feat(e2e): add podchaos api and test cases

test/e2e/chaos/podChaos.go

@@ -0,0 +1,90 @@
+package chaos
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework"
+	"github.com/radondb/radondb-mysql-kubernetes/utils"
+)
+
+var _ = Describe("podchaos", Ordered, func() {
+	f := framework.NewFramework("e2e-test")
+	clusterKey := &types.NamespacedName{Name: framework.TestContext.ClusterReleaseName, Namespace: f.Namespace.Name}
+
+	BeforeAll(func() {
+		f.BeforeEach()
+		// Make sure operator is available
+		By("checking webhook")
+		f.WaitUntilServiceAvailable(framework.WebhookServiceName)
+		By("checking manager")
+		Expect(f.CreateOperatorHealthCheckService()).Should(Succeed())
+		Expect(f.CheckServiceEndpoint(framework.HealthCheckServiceName, 8081, "healthz")).Should(Succeed())
+
+		By("checking mysql cluster")
+		f.WaitClusterReadiness(clusterKey)
+	})
+
+	AfterEach(func() {
+		f.CleanUpChaos()
+	})
+
+	It("kill leader", func() {
+		killLeader(f)
+		time.Sleep(10 * time.Second)
+		f.WaitClusterReadiness(clusterKey)
+	})
+
+	It("kill leader mysql", func() {
+		killLeaderMySQL(f)
+		time.Sleep(10 * time.Second)
+		f.WaitClusterReadiness(clusterKey)
+	})
+
+	It("kill leader xenon", func() {
+		killLeaderXenon(f)
+		time.Sleep(10 * time.Second)
+		f.WaitClusterReadiness(clusterKey)
+	})
+
+	It("leader failure", func() {
+		LeaderFailure(f, "30s")
+		time.Sleep(10 * time.Second)
+		f.WaitClusterReadiness(clusterKey)
+	})
+})
+
+func killLeader(f *framework.Framework) {
+	By("killing leader")
+	f.KillPod(&framework.PodChaosOptions{
+		Labels: map[string]string{"role": string(utils.Leader)},
+	})
+}
+
+func killLeaderMySQL(f *framework.Framework) {
+	By("killing leader mysql")
+	f.KillContainers(&framework.PodChaosOptions{
+		Labels:     map[string]string{"role": string(utils.Leader)},
+		Containers: []string{"mysql"},
+	})
+}
+
+func killLeaderXenon(f *framework.Framework) {
+	By("killing leader xenon")
+	f.KillContainers(&framework.PodChaosOptions{
+		Labels:     map[string]string{"role": string(utils.Leader)},
+		Containers: []string{"xenon"},
+	})
+}
+
+func LeaderFailure(f *framework.Framework, duration string) {
+	By(fmt.Sprintf("leader failure: %v", duration))
+	f.PodFailure(&framework.PodChaosOptions{
+		Labels:   map[string]string{"role": string(utils.Leader)},
+		Duration: duration,
+	})
+}

test/e2e/e2e_test.go

@@ -31,6 +31,7 @@ import (
 
 	// Test case source.
 	// Comment out the package that you don't want to run.
+	_ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/chaos"
 	_ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/install"
 	_ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/simplecase"
 	_ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/user"

test/e2e/framework/chaos.go

@@ -0,0 +1,109 @@
+package framework
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"time"
+
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	chaosapis "github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+)
+
+type PodChaosOptions struct {
+	Name       string
+	Duration   string
+	Mode       chaosapis.SelectorMode
+	Containers []string
+	Labels     map[string]string
+}
+
+func newPodChaos(name, ns string) *chaosapis.PodChaos {
+	return &chaosapis.PodChaos{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+	}
+}
+
+func (f *Framework) KillPod(option *PodChaosOptions) {
+	if option.Name == "" {
+		option.Name = fmt.Sprintf("kill-pod-%d", rand.Intn(1000))
+	}
+	if option.Mode == "" {
+		option.Mode = chaosapis.OneMode
+	}
+	Expect(option.Labels).ShouldNot(BeEmpty())
+
+	podChaos := newPodChaos(option.Name, f.kubectlOptions.Namespace)
+	podChaos.Spec.Action = chaosapis.PodKillAction
+	podChaos.Spec.Mode = option.Mode
+	podChaos.Spec.Selector.LabelSelectors = option.Labels
+
+	Expect(f.Client.Create(context.TODO(), podChaos, &client.CreateOptions{})).Should(Succeed())
+	f.waitChaosInjected(podChaos.Name, 30)
+}
+
+func (f *Framework) KillContainers(option *PodChaosOptions) {
+	if option.Name == "" {
+		option.Name = fmt.Sprintf("kill-containers-%d", rand.Intn(1000))
+	}
+	if option.Mode == "" {
+		option.Mode = chaosapis.OneMode
+	}
+	Expect(option.Labels).ShouldNot(BeEmpty())
+
+	podChaos := newPodChaos(option.Name, f.kubectlOptions.Namespace)
+	podChaos.Spec.Action = chaosapis.ContainerKillAction
+	podChaos.Spec.Mode = option.Mode
+	podChaos.Spec.Selector.LabelSelectors = option.Labels
+	podChaos.Spec.ContainerNames = option.Containers
+
+	Expect(f.Client.Create(context.TODO(), podChaos, &client.CreateOptions{})).Should(Succeed())
+	f.waitChaosInjected(podChaos.Name, 30)
+}
+
+func (f *Framework) PodFailure(option *PodChaosOptions) {
+	if option.Name == "" {
+		option.Name = fmt.Sprintf("pod-failure-%d", rand.Intn(1000))
+	}
+	if option.Mode == "" {
+		option.Mode = chaosapis.OneMode
+	}
+	if option.Duration == "" {
+		option.Duration = "9999s"
+	}
+	Expect(option.Labels).ShouldNot(BeEmpty())
+
+	podChaos := newPodChaos(option.Name, f.kubectlOptions.Namespace)
+	podChaos.Spec.Action = chaosapis.PodFailureAction
+	podChaos.Spec.Mode = option.Mode
+	podChaos.Spec.Selector.LabelSelectors = option.Labels
+	podChaos.Spec.Duration = &option.Duration
+
+	Expect(f.Client.Create(context.TODO(), podChaos, &client.CreateOptions{})).Should(Succeed())
+	f.waitChaosInjected(podChaos.Name, 30)
+}
+
+func (f *Framework) waitChaosInjected(chaosName string, timeout time.Duration) {
+	err := wait.PollImmediate(2*time.Second, timeout, func() (bool, error) {
+		f.Log.Logf(f.t, "%s injecting", chaosName)
+
+		chaos := chaosapis.PodChaos{}
+		f.Client.Get(context.TODO(), client.ObjectKey{Name: chaosName, Namespace: f.kubectlOptions.Namespace}, &chaos)
+		for _, cond := range chaos.Status.Conditions {
+			if cond.Type == chaosapis.ConditionAllInjected && cond.Status == corev1.ConditionTrue {
+				f.Log.Logf(f.t, "%s injected", chaosName)
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+	Expect(err).Should(BeNil(), "failed to inject chaos")
+}

test/e2e/framework/cleanup.go

@@ -76,6 +76,10 @@ func (f *Framework) CleanUpNS() {
 	IgnoreNotFound(k8s.DeleteNamespaceE(f.t, f.kubectlOptions, TestContext.E2ETestNamespace))
 }
 
+func (f *Framework) CleanUpChaos() {
+	k8s.RunKubectl(f.t, f.kubectlOptions, "delete", "podchaos", "--all", "-n", f.kubectlOptions.Namespace)
+}
+
 func IgnoreNotFound(err error) {
 	if err != nil && strings.Contains(err.Error(), "not found") {
 		// Do nothing

test/e2e/framework/framework.go

@@ -23,8 +23,8 @@ import (
 	"time"
 
 	chaosapis "github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
-	"github.com/go-logr/logr"
 	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/testing"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -49,7 +49,7 @@ type Framework struct {
 
 	Timeout time.Duration
 
-	Log logr.Logger
+	Log logger.Logger
 }
 
 func NewFramework(baseName string) *Framework {
@@ -67,7 +67,7 @@ func NewFramework(baseName string) *Framework {
 				Name: TestContext.E2ETestNamespace,
 			},
 		},
-		Log: Log,
+		Log: *logger.Default,
 	}
 
 	return f

test/e2e/framework/mysqlcluster.go

@@ -93,6 +93,7 @@ func (f *Framework) ClusterEventuallyReplicas(cluster *apiv1alpha1.MysqlCluster,
 	Eventually(func() int {
 		cl := &apiv1alpha1.MysqlCluster{}
 		f.Client.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cl)
+		f.Log.Logf(f.t, "ready nodes: %d/%d", cl.Status.ReadyNodes, *cl.Spec.Replicas)
 		return cl.Status.ReadyNodes
 	}, timeout, POLLING).Should(Equal(int(*cluster.Spec.Replicas)), "Not ready replicas of cluster '%s'", cluster.Name)
 }
@@ -101,6 +102,7 @@ func (f *Framework) ClusterEventuallyRaftStatus(cluster *apiv1alpha1.MysqlCluste
 	Eventually(func() bool {
 		cl := &apiv1alpha1.MysqlCluster{}
 		f.Client.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cl)
+		f.Log.Logf(f.t, "checking xenon")
 		return isXenonReadiness(cl)
 	}, TIMEOUT, POLLING).Should(BeTrue(), "Not ready xenon of cluster '%s'", cluster.Name)
 }