Set up minikube for k8s testing

Install and set up minikube so that we can
create a k8s cluster for testing.

Signed-off-by: Urvashi Mohnani <[email protected]>

.cirrus.yml

@@ -33,7 +33,7 @@ env:
     UBUNTU_NAME: "ubuntu-2204"
 
     # Image identifiers
-    IMAGE_SUFFIX: "c5823947156488192"
+    IMAGE_SUFFIX: "c4678746211876864"
     # EC2 images
     FEDORA_AMI: "fedora-aws-${IMAGE_SUFFIX}"
     FEDORA_AARCH64_AMI: "fedora-podman-aws-arm64-${IMAGE_SUFFIX}"
@@ -770,6 +770,23 @@ rootless_system_test_task:
     main_script: *main
     always: *logs_artifacts
 
+minikube_test_task:
+    name: *std_name_fmt
+    alias: minikube_test
+    # Docs: ./contrib/cirrus/CIModes.md
+    only_if: *not_tag_build_docs_multiarch
+    depends_on:
+        - build
+        - rootless_system_test
+    gce_instance: *standardvm
+    env:
+        <<: *stdenvars
+        TEST_FLAVOR: minikube
+        PRIV_NAME: rootless
+    clone_script: *get_gosrc
+    setup_script: *setup
+    main_script: *main
+    always: *logs_artifacts
 
 buildah_bud_test_task:
     name: *std_name_fmt
@@ -963,6 +980,7 @@ success_task:
         - remote_system_test_aarch64
         - rootless_system_test
         - rootless_remote_system_test
+        - minikube_test
         - buildah_bud_test
         - rootless_gitlab_test
         - upgrade_test

contrib/cirrus/runner.sh

@@ -118,6 +118,12 @@ function _run_endpoint() {
     make endpoint
 }
 
+function _run_minikube() {
+    _bail_if_test_can_be_skipped test/minikube
+    msg "Testing  minikube."
+    bats test/minikube |& logformatter
+}
+
 exec_container() {
     local var_val
     local cmd

contrib/cirrus/setup_environment.sh

@@ -297,6 +297,14 @@ case "$TEST_FLAVOR" in
             die "Invalid value for \$TEST_ENVIRON=$TEST_ENVIRON"
         fi
 
+        install_test_configs
+        ;;
+    minikube)
+        dnf install -y $PACKAGE_DOWNLOAD_DIR/minikube-latest*
+        remove_packaged_podman_files
+        make install.tools
+        make install PREFIX=/usr ETCDIR=/etc
+        minikube config set driver podman
         install_test_configs
         ;;
     machine)

test/minikube/001-kube.bats

@@ -0,0 +1,61 @@
+#!/usr/bin/env bats
+#
+# Tests of podman kube commands with minikube
+#
+
+load helpers.bash
+
+###############################################################################
+# BEGIN tests
+
+@test "minikube - check cluster is up" {
+    run minikube kubectl get nodes
+    assert "$status" -eq 0 "get status of nodes"
+    assert "$output" =~ "Ready"
+    run minikube kubectl get pods
+    assert "$status" -eq 0 "get pods in the default namespace"
+    assert "$output" == "No resources found in default namespace."
+    wait_for_default_sa
+}
+
+@test "minikube - deploy generated container yaml to minikube" {
+    cname="test-ctr"
+    fname="/tmp/minikube_deploy_$(random_string 6).yaml"
+    run_podman container create --name $cname $IMAGE top
+    run_podman kube generate -f $fname $cname
+
+    # deploy to the minikube cluster
+    project="ctr-ns"
+    run minikube kubectl create namespace $project
+    assert "$status" -eq 0 "create new namespace $project"
+    run minikube kubectl -- apply -f $fname
+    echo $output >&2
+    assert "$status" -eq 0 "deploy $fname to the cluster"
+    assert "$output" == "pod/$cname-pod created"
+    wait_for_pods_to_start
+    run minikube kubectl delete namespace $project
+    assert $status -eq 0 "delete namespace $project"
+}
+
+@test "minikube - deploy generated pod yaml to minikube" {
+    pname="test-pod"
+    cname1="test-ctr1"
+    cname2="test-ctr2"
+    fname="/tmp/minikube_deploy_$(random_string 6).yaml"
+
+    run_podman pod create --name $pname --publish 9999:8888
+    run_podman container create --name $cname1 --pod $pname $IMAGE sleep 1000
+    run_podman container create --name $cname2 --pod $pname $IMAGE sleep 2000
+    run_podman kube generate -f $fname $pname
+
+    # deploy to the minikube cluster
+    project="pod-ns"
+    run minikube kubectl create namespace $project
+    assert "$status" -eq 0 "create new namespace $project"
+    run minikube kubectl -- apply -f $fname
+    assert "$status" -eq 0 "deploy $fname to the cluster"
+    assert "$output" == "pod/$pname created"
+    wait_for_pods_to_start
+    run minikube kubectl delete namespace $project
+    assert $status -eq 0 "delete namespace $project"
+}

test/minikube/helpers.bash

@@ -0,0 +1,60 @@
+# -*- bash -*-
+
+load ../system/helpers.bash
+
+function setup(){
+    # only set up the minikube cluster before the first test
+    if [[ "$BATS_TEST_NUMBER" -eq 1 ]]; then
+        minikube start
+    fi
+    basic_setup
+}
+
+function teardown(){
+    # only delete the minikube cluster if we are done with the last test
+    # the $DEBUG_MINIKUBE env can be set to preserve the cluster to debug if needed
+    if [[ "$BATS_TEST_NUMBER" -eq ${#BATS_TEST_NAMES[@]} ]] && [[ "$DEBUG_MINIKUBE" == "" ]]; then
+        minikube delete
+    fi
+    basic_teardown
+}
+
+function wait_for_default_sa(){
+    count=0
+    sa_ready=false
+    # timeout after 30 seconds
+    # if the default service account hasn't been created yet, there is something else wrong
+    while [[ $count -lt 30 ]] && [[ $sa_ready == false ]]
+    do
+        run minikube kubectl get sa
+        assert "$status" -eq 0
+        if [[ "$output" != "No resources found in default namespace." ]]; then
+            sa_ready=true
+        fi
+        count=$((count + 1))
+        sleep 1
+    done
+    if [[ $sa_ready == false ]]; then
+        die "Timed out waiting for default service account to be created"
+    fi
+}
+
+function wait_for_pods_to_start(){
+    count=0
+    running=false
+    # timeout after 30 seconds
+    # if the pod hasn't started running after 30 seconds, there is something else wrong
+    while [[ $count -lt 30 ]] && [[ $running == false ]]
+    do
+        run minikube kubectl get pods
+        assert "$status" -eq 0
+        if [[ "$output" =~ "Running" ]]; then
+            running=true
+        fi
+        count=$((count + 1))
+        sleep 1
+    done
+    if [[ $running == false ]]; then
+        die "Timed out waiting for pod to move to 'Running' state"
+    fi
+}