unit-tests: Add utility to run container commands for tests

Add utility to run container commands during tests when a runtime is available.

Add failover test for redis loadbalancer using keydb.

Signed-off-by: Heathcliff <[email protected]>

pkg/lock-manager/storage/redis/loadbalancer_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/alicebob/miniredis/v2"
+	"github.com/heathcliff26/fleetlock/tests/utils"
 	"github.com/redis/go-redis/v9"
 	"github.com/stretchr/testify/assert"
 )
@@ -29,4 +30,49 @@ func TestLoadbalancer(t *testing.T) {
 			t.FailNow()
 		}
 	})
+	t.Run("Failover", func(t *testing.T) {
+		if !utils.HasContainerRuntimer() {
+			t.Skip("Missing Container Runtime")
+		}
+
+		err := utils.ExecCRI("run", "--name", "fleetlock-redis-loadbalancer-failover-1", "-d", "--rm", "--net", "host", "docker.io/eqalpha/keydb:latest", "--port", "6379", "--active-replica", "yes", "--replicaof", "localhost", "6380")
+		if err != nil {
+			t.Fatalf("Failed to start test db: %v\n", err)
+		}
+		t.Cleanup(func() {
+			_ = utils.ExecCRI("stop", "fleetlock-redis-loadbalancer-failover-1")
+		})
+		err = utils.ExecCRI("run", "--name", "fleetlock-redis-loadbalancer-failover-2", "-d", "--rm", "--net", "host", "docker.io/eqalpha/keydb:latest", "--port", "6380", "--active-replica", "yes", "--replicaof", "localhost", "6379")
+		if err != nil {
+			t.Fatalf("Failed to start test db: %v\n", err)
+		}
+		t.Cleanup(func() {
+			_ = utils.ExecCRI("stop", "fleetlock-redis-loadbalancer-failover-2")
+		})
+
+		addrs := []string{"localhost:6379", "localhost:6380"}
+		client, lb := NewRedisClientWithLoadbalancer(addrs, &redis.Options{})
+		t.Cleanup(func() {
+			lb.Close()
+			client.Close()
+		})
+
+		assert := assert.New(t)
+
+		assert.Equal(0, lb.selected, "Should have currently the first client selected")
+
+		err = utils.ExecCRI("stop", "fleetlock-redis-loadbalancer-failover-1")
+		if err != nil {
+			t.Fatalf("Failed to stop keydb instance: %v\n", err)
+		}
+		lb.HealthCheck()
+
+		if !assert.Equal(1, lb.selected, "Should have failed over") {
+			t.FailNow()
+		}
+		res, err := client.Ping(context.Background()).Result()
+		if !assert.Nil(err, "Should have failed over") || !assert.Equal("PONG", res, "Should have failed over") {
+			t.FailNow()
+		}
+	})
 }

tests/utils/container.go

@@ -0,0 +1,43 @@
+package utils
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+var (
+	containerRuntime string
+	initialized      = false
+)
+
+func findContainerRuntime() string {
+	for _, cmd := range []string{"docker", "podman"} {
+		path, err := exec.LookPath(cmd)
+		if err != nil {
+			continue
+		}
+		err = exec.Command(path, "ps").Run()
+		if err == nil {
+			fmt.Printf("Found container runtime %s, path=%s\n", cmd, path)
+			return path
+		}
+	}
+	fmt.Println("Did not find any container runtimes")
+	return ""
+}
+
+func HasContainerRuntimer() bool {
+	if !initialized {
+		containerRuntime = findContainerRuntime()
+		initialized = true
+	}
+	return containerRuntime != ""
+}
+
+func ExecCRI(args ...string) error {
+	if !initialized {
+		containerRuntime = findContainerRuntime()
+		initialized = true
+	}
+	return exec.Command(containerRuntime, args...).Run()
+}