Merge pull request #17 from cofide/ci-integration-tests

Add single trust zone and federation integration tests, run in CI

.github/workflows/integration.yml

@@ -0,0 +1,71 @@
+---
+name: integration
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  single-trust-zone:
+    name: single trust zone
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install just
+        uses: taiki-e/install-action@just
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Build and run tests
+        run: just build
+
+      - name: Install kind
+        run: just install-kind
+
+      - name: Install ko
+        uses: ko-build/[email protected]
+        env:
+          KO_DOCKER_REPO: kind.local
+
+      - name: Create a kind cluster
+        run: just create-kind-cluster
+
+      - name: Test
+        run: just integration-test single-trust-zone
+
+  federation:
+    name: federation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install just
+        uses: taiki-e/install-action@just
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Build and run tests
+        run: just build
+
+      - name: Install kind
+        run: just install-kind
+
+      - name: Install ko
+        uses: ko-build/[email protected]
+        env:
+          KO_DOCKER_REPO: kind.local
+
+      - name: Create kind clusters
+        run: just create-kind-clusters 2
+
+      - name: Test
+        run: just integration-test federation

Justfile

@@ -6,3 +6,15 @@ test:
 
 lint *args:
     golangci-lint run --show-stats {{args}}
+
+install-kind:
+    tests/integration/install-kind.sh
+
+create-kind-cluster:
+    tests/integration/create-kind-cluster.sh
+
+create-kind-clusters num_clusters:
+    tests/integration/create-kind-clusters.sh {{num_clusters}}
+
+integration-test test:
+    tests/integration/{{test}}/test.sh

demos/Justfile

@@ -1,12 +1,11 @@
 set export
 set shell := ["bash", "-euo", "pipefail", "-c"]
 
-# assume a local kind cluster for the demos unless otherwise configured
-export KO_DOCKER_REPO := env_var_or_default("KO_DOCKER_REPO", "kind.local")
-export KIND_CLUSTER_NAME := env_var_or_default("KIND_CLUSTER_NAME", "kind")
-
 namespace := "demo"
 
+# Set prompt_namespace=no to avoid prompting before namespace creation.
+prompt_namespace := 'yes'
+
 # Check for demo script dependencies
 check-deps:
     for cmd in ko kubectl; do \
@@ -21,28 +20,23 @@ ensure-namespace context:
     if [[ ! -z "{{context}}" ]]; then \
         if ! kubectl --context {{context}} get namespace "{{namespace}}" &> /dev/null; then \
             echo "Namespace {{namespace}} does not exist"; \
-            read -p "Create namespace? (y/n) " -r; \
-            if [[ $REPLY =~ ^[Yy]$ ]]; then \
-                kubectl --context {{context}} create namespace "{{namespace}}"; \
-            else \
-                echo "Aborting..."; \
-                exit 1; \
-            fi \
+            if [[ "{{prompt_namespace}}" != "no" ]]; then \
+                read -p "Create namespace? (y/n) " -r; \
+                if [[ ! $REPLY =~ ^[Yy]$ ]]; then \
+                    echo "Aborting..."; \
+                    exit 1; \
+                fi \
+            fi; \
+            kubectl --context {{context}} create namespace "{{namespace}}"; \
         fi \
     fi
 
+# Clone the cofide-demos git repo.
 clone-demos-repo:
     if [ ! -d cofide-demos ]; then \
         git clone https://github.com/cofide/cofide-demos; \
     fi
 
-# Build all demo ping-pong applications
-build-demos: build-ping-pong
-
-# Build the ping-pong application
-build-ping-pong: clone-demos-repo
-    just -f cofide-demos/Justfile build-ping-pong
-
 # Deploy ping-pong server and client
-deploy-ping-pong client_context server_context="": build-ping-pong (ensure-namespace client_context) (ensure-namespace server_context)
+deploy-ping-pong client_context server_context="": clone-demos-repo (ensure-namespace client_context) (ensure-namespace server_context)
     ping-pong/deploy.sh {{namespace}} {{client_context}} {{server_context}}

demos/ping-pong/deploy.sh

@@ -6,6 +6,9 @@ NAMESPACE="$1"
 CLIENT_CTX="$2"
 SERVER_CTX="${3:-$CLIENT_CTX}"
 
+# assume a local kind cluster for the demos unless otherwise configured
+export KO_DOCKER_REPO=${KO_DOCKER_REPO:-kind.local}
+
 pushd cofide-demos
 
 echo "Deploying pong server to: $SERVER_CTX"
@@ -17,12 +20,12 @@ if ! ko resolve -f workloads/ping-pong/server/deploy.yaml | kubectl apply -n "$N
     exit 1
 fi
 echo "Server deployment complete"
-if [ "$CLIENT_CTX" == kind-* ]; then
+if [[ "$CLIENT_CTX" == kind-* ]]; then
     export KIND_CLUSTER_NAME="${CLIENT_CTX#kind-}"
 fi
 
 echo "Deploying ping client to: $CLIENT_CTX"
-if [ "$SERVER_CTX" != "$CLIENT_CTX" ]; then
+if [[ "$SERVER_CTX" != "$CLIENT_CTX" ]]; then
     echo "Discovering server IP..."
     export PING_PONG_SERVER_SERVICE_HOST=$(kubectl --context "$SERVER_CTX" wait --for=jsonpath="{.status.loadBalancer.ingress[0].ip}" service/ping-pong-server -n $NAMESPACE --timeout=60s > /dev/null 2>&1 \
         && kubectl --context "$SERVER_CTX" get service ping-pong-server -n $NAMESPACE -o "jsonpath={.status.loadBalancer.ingress[0].ip}")

tests/integration/create-kind-cluster.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# This script deploys a local Kubernetes cluster using Kind (https://kind.sigs.k8s.io).
+
+set -euxo pipefail
+
+DELETE_EXISTING_KIND_CLUSTER=${DELETE_EXISTING_KIND_CLUSTER:-true}
+
+K8S_CLUSTER_NAME=${K8S_CLUSTER_NAME:-local1}
+
+CLOUD_PROVIDER_KIND_CONTAINER_NAME=cloud-provider-kind-cloud-provider-1
+
+function delete_kind_cluster() {
+  cluster=$(kind get clusters | egrep "\b${K8S_CLUSTER_NAME}\b" || true)
+  if [[ -n $cluster ]]; then
+    kind delete cluster -n $K8S_CLUSTER_NAME
+  fi
+}
+
+function create_kind_cluster() {
+  kind create cluster -n $K8S_CLUSTER_NAME
+}
+
+function restart_cloud_provider_kind() {
+  # Workaround: cloud-provider-kind often stops working when a kind cluster is created. Restart it.
+  if [[ $(docker ps -q --filter "name=$CLOUD_PROVIDER_KIND_CONTAINER_NAME") != "" ]]; then
+    docker restart $CLOUD_PROVIDER_KIND_CONTAINER_NAME
+  fi
+}
+
+function main() {
+  if $DELETE_EXISTING_KIND_CLUSTER; then
+    delete_kind_cluster
+  fi
+  create_kind_cluster
+  restart_cloud_provider_kind
+  echo "Success!"
+}
+
+main

tests/integration/create-kind-clusters.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# This script deploys multiple local Kubernetes clusters using Kind (https://kind.sigs.k8s.io).
+
+set -euxo pipefail
+
+NUM_K8S_CLUSTERS=${1:?Number of kind clusters to create}
+K8S_CLUSTER_NAME_PREFIX=${K8S_CLUSTER_NAME_PREFIX:-local}
+
+function create_kind_cluster() {
+  suffix=$1
+  parent_dir=$(dirname $BASH_SOURCE)
+  export K8S_CLUSTER_NAME=${K8S_CLUSTER_NAME_PREFIX}${suffix}
+  $parent_dir/create-kind-cluster.sh
+}
+
+function main() {
+  for i in $(seq $NUM_K8S_CLUSTERS); do
+    create_kind_cluster $i
+  done
+}
+
+main

tests/integration/federation/test.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+# This script deploys two federated trust zones, runs some basic tests against them, then tears them down.
+# The trust zones have one attestation policy matching workloads in namespace ns1, and another matching pods with a label foo=bar.
+
+set -euxo pipefail
+
+K8S_CLUSTER_1_NAME=${K8S_CLUSTER_1_NAME:-local1}
+K8S_CLUSTER_1_CONTEXT=${K8S_CLUSTER_1_CONTEXT:-kind-$K8S_CLUSTER_1_NAME}
+
+K8S_CLUSTER_2_NAME=${K8S_CLUSTER_2_NAME:-local2}
+K8S_CLUSTER_2_CONTEXT=${K8S_CLUSTER_2_CONTEXT:-kind-$K8S_CLUSTER_2_NAME}
+
+TRUST_ZONE_1=${TRUST_ZONE_1:-tz1}
+TRUST_DOMAIN_1=${TRUST_DOMAIN_1:-td1}
+
+TRUST_ZONE_2=${TRUST_ZONE_2:-tz2}
+TRUST_DOMAIN_2=${TRUST_DOMAIN_2:-td2}
+
+NAMESPACE_POLICY_NAMESPACE=${NAMESPACE_POLICY_NAMESPACE:-demo}
+POD_POLICY_POD_LABEL=${POD_POLICY_POD_LABEL:-"foo=bar"}
+
+function configure() {
+  rm -f cofide.yaml
+  ./cofidectl init
+  ./cofidectl trust-zone add $TRUST_ZONE_1 --trust-domain $TRUST_DOMAIN_1 --kubernetes-context $K8S_CLUSTER_1_CONTEXT --kubernetes-cluster $K8S_CLUSTER_1_NAME --profile kubernetes
+  ./cofidectl trust-zone add $TRUST_ZONE_2 --trust-domain $TRUST_DOMAIN_2 --kubernetes-context $K8S_CLUSTER_2_CONTEXT --kubernetes-cluster $K8S_CLUSTER_2_NAME --profile kubernetes
+  ./cofidectl federation add --from $TRUST_ZONE_1 --to $TRUST_ZONE_2
+  ./cofidectl federation add --from $TRUST_ZONE_2 --to $TRUST_ZONE_1
+  ./cofidectl attestation-policy add kubernetes --name namespace --namespace $NAMESPACE_POLICY_NAMESPACE
+  ./cofidectl attestation-policy add kubernetes --name pod-label --pod-label $POD_POLICY_POD_LABEL
+  ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE_1 --attestation-policy namespace --federates-with $TRUST_ZONE_2
+  ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE_1 --attestation-policy pod-label --federates-with $TRUST_ZONE_2
+  ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE_2 --attestation-policy namespace --federates-with $TRUST_ZONE_1
+  ./cofidectl attestation-policy-binding add --trust-zone $TRUST_ZONE_2 --attestation-policy pod-label --federates-with $TRUST_ZONE_1
+}
+
+function up() {
+  ./cofidectl up
+}
+
+function list_resources() {
+  ./cofidectl trust-zone list
+  ./cofidectl attestation-policy list
+  ./cofidectl attestation-policy-binding list
+}
+
+function show_config() {
+  cat cofide.yaml
+}
+
+function show_status() {
+  ./cofidectl workload discover
+  ./cofidectl workload list
+  ./cofidectl trust-zone status $TRUST_ZONE_1
+  ./cofidectl trust-zone status $TRUST_ZONE_2
+}
+
+function run_tests() {
+  just -f demos/Justfile prompt_namespace=no deploy-ping-pong $K8S_CLUSTER_1_CONTEXT $K8S_CLUSTER_2_CONTEXT
+  if ! wait_for_pong; then
+    echo "Timed out waiting for pong from server"
+    echo "Client logs:"
+    kubectl --context $K8S_CLUSTER_1_CONTEXT logs -n demo deployments/ping-pong-client
+    echo "Server logs:"
+    kubectl --context $K8S_CLUSTER_2_CONTEXT logs -n demo deployments/ping-pong-server
+    exit 1
+  fi
+}
+
+function wait_for_pong() {
+  kubectl --context $K8S_CLUSTER_1_CONTEXT wait -n demo --for=condition=Available --timeout 60s deployments/ping-pong-client
+  for i in $(seq 30); do
+    if kubectl --context $K8S_CLUSTER_1_CONTEXT logs -n demo deployments/ping-pong-client | grep pong; then
+      return
+    fi
+    sleep 2
+  done
+}
+
+function down() {
+  ./cofidectl down
+}
+
+function main() {
+  configure
+  up
+  list_resources
+  show_config
+  show_status
+  run_tests
+  down
+  echo "Success!"
+}
+
+main