From adeeb4a32962ff90194245f1539099057bcbdfa0 Mon Sep 17 00:00:00 2001 From: chengbaitai Date: Tue, 6 Sep 2022 14:25:03 +0800 Subject: [PATCH] feat(e2e): add podchaos api and test cases --- test/e2e/chaos/podChaos.go | 90 ++++++++++++++++++++++++ test/e2e/e2e_test.go | 1 + test/e2e/framework/chaos.go | 109 +++++++++++++++++++++++++++++ test/e2e/framework/cleanup.go | 4 ++ test/e2e/framework/framework.go | 6 +- test/e2e/framework/mysqlcluster.go | 2 + 6 files changed, 209 insertions(+), 3 deletions(-) create mode 100644 test/e2e/chaos/podChaos.go create mode 100644 test/e2e/framework/chaos.go diff --git a/test/e2e/chaos/podChaos.go b/test/e2e/chaos/podChaos.go new file mode 100644 index 00000000..56613763 --- /dev/null +++ b/test/e2e/chaos/podChaos.go @@ -0,0 +1,90 @@ +package chaos + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + + "github.com/radondb/radondb-mysql-kubernetes/test/e2e/framework" + "github.com/radondb/radondb-mysql-kubernetes/utils" +) + +var _ = Describe("podchaos", Ordered, func() { + f := framework.NewFramework("e2e-test") + clusterKey := &types.NamespacedName{Name: framework.TestContext.ClusterReleaseName, Namespace: f.Namespace.Name} + + BeforeAll(func() { + f.BeforeEach() + // Make sure operator is available + By("checking webhook") + f.WaitUntilServiceAvailable(framework.WebhookServiceName) + By("checking manager") + Expect(f.CreateOperatorHealthCheckService()).Should(Succeed()) + Expect(f.CheckServiceEndpoint(framework.HealthCheckServiceName, 8081, "healthz")).Should(Succeed()) + + By("checking mysql cluster") + f.WaitClusterReadiness(clusterKey) + }) + + AfterEach(func() { + f.CleanUpChaos() + }) + + It("kill leader", func() { + killLeader(f) + time.Sleep(10 * time.Second) + f.WaitClusterReadiness(clusterKey) + }) + + It("kill leader mysql", func() { + killLeaderMySQL(f) + time.Sleep(10 * time.Second) + f.WaitClusterReadiness(clusterKey) + }) + + It("kill leader xenon", func() { + killLeaderXenon(f) + time.Sleep(10 * time.Second) + f.WaitClusterReadiness(clusterKey) + }) + + It("leader failure", func() { + LeaderFailure(f, "30s") + time.Sleep(10 * time.Second) + f.WaitClusterReadiness(clusterKey) + }) +}) + +func killLeader(f *framework.Framework) { + By("killing leader") + f.KillPod(&framework.PodChaosOptions{ + Labels: map[string]string{"role": string(utils.Leader)}, + }) +} + +func killLeaderMySQL(f *framework.Framework) { + By("killing leader mysql") + f.KillContainers(&framework.PodChaosOptions{ + Labels: map[string]string{"role": string(utils.Leader)}, + Containers: []string{"mysql"}, + }) +} + +func killLeaderXenon(f *framework.Framework) { + By("killing leader xenon") + f.KillContainers(&framework.PodChaosOptions{ + Labels: map[string]string{"role": string(utils.Leader)}, + Containers: []string{"xenon"}, + }) +} + +func LeaderFailure(f *framework.Framework, duration string) { + By(fmt.Sprintf("leader failure: %v", duration)) + f.PodFailure(&framework.PodChaosOptions{ + Labels: map[string]string{"role": string(utils.Leader)}, + Duration: duration, + }) +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 36b6572f..04eb673e 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -31,6 +31,7 @@ import ( // Test case source. // Comment out the package that you don't want to run. + _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/chaos" _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/install" _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/simplecase" _ "github.com/radondb/radondb-mysql-kubernetes/test/e2e/user" diff --git a/test/e2e/framework/chaos.go b/test/e2e/framework/chaos.go new file mode 100644 index 00000000..7495be96 --- /dev/null +++ b/test/e2e/framework/chaos.go @@ -0,0 +1,109 @@ +package framework + +import ( + "context" + "fmt" + "math/rand" + "time" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + chaosapis "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" +) + +type PodChaosOptions struct { + Name string + Duration string + Mode chaosapis.SelectorMode + Containers []string + Labels map[string]string +} + +func newPodChaos(name, ns string) *chaosapis.PodChaos { + return &chaosapis.PodChaos{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + } +} + +func (f *Framework) KillPod(option *PodChaosOptions) { + if option.Name == "" { + option.Name = fmt.Sprintf("kill-pod-%d", rand.Intn(1000)) + } + if option.Mode == "" { + option.Mode = chaosapis.OneMode + } + Expect(option.Labels).ShouldNot(BeEmpty()) + + podChaos := newPodChaos(option.Name, f.kubectlOptions.Namespace) + podChaos.Spec.Action = chaosapis.PodKillAction + podChaos.Spec.Mode = option.Mode + podChaos.Spec.Selector.LabelSelectors = option.Labels + + Expect(f.Client.Create(context.TODO(), podChaos, &client.CreateOptions{})).Should(Succeed()) + f.waitChaosInjected(podChaos.Name, 30) +} + +func (f *Framework) KillContainers(option *PodChaosOptions) { + if option.Name == "" { + option.Name = fmt.Sprintf("kill-containers-%d", rand.Intn(1000)) + } + if option.Mode == "" { + option.Mode = chaosapis.OneMode + } + Expect(option.Labels).ShouldNot(BeEmpty()) + + podChaos := newPodChaos(option.Name, f.kubectlOptions.Namespace) + podChaos.Spec.Action = chaosapis.ContainerKillAction + podChaos.Spec.Mode = option.Mode + podChaos.Spec.Selector.LabelSelectors = option.Labels + podChaos.Spec.ContainerNames = option.Containers + + Expect(f.Client.Create(context.TODO(), podChaos, &client.CreateOptions{})).Should(Succeed()) + f.waitChaosInjected(podChaos.Name, 30) +} + +func (f *Framework) PodFailure(option *PodChaosOptions) { + if option.Name == "" { + option.Name = fmt.Sprintf("pod-failure-%d", rand.Intn(1000)) + } + if option.Mode == "" { + option.Mode = chaosapis.OneMode + } + if option.Duration == "" { + option.Duration = "9999s" + } + Expect(option.Labels).ShouldNot(BeEmpty()) + + podChaos := newPodChaos(option.Name, f.kubectlOptions.Namespace) + podChaos.Spec.Action = chaosapis.PodFailureAction + podChaos.Spec.Mode = option.Mode + podChaos.Spec.Selector.LabelSelectors = option.Labels + podChaos.Spec.Duration = &option.Duration + + Expect(f.Client.Create(context.TODO(), podChaos, &client.CreateOptions{})).Should(Succeed()) + f.waitChaosInjected(podChaos.Name, 30) +} + +func (f *Framework) waitChaosInjected(chaosName string, timeout time.Duration) { + err := wait.PollImmediate(2*time.Second, timeout, func() (bool, error) { + f.Log.Logf(f.t, "%s injecting", chaosName) + + chaos := chaosapis.PodChaos{} + f.Client.Get(context.TODO(), client.ObjectKey{Name: chaosName, Namespace: f.kubectlOptions.Namespace}, &chaos) + for _, cond := range chaos.Status.Conditions { + if cond.Type == chaosapis.ConditionAllInjected && cond.Status == corev1.ConditionTrue { + f.Log.Logf(f.t, "%s injected", chaosName) + return true, nil + } + } + return false, nil + }) + Expect(err).Should(BeNil(), "failed to inject chaos") +} diff --git a/test/e2e/framework/cleanup.go b/test/e2e/framework/cleanup.go index b59c3696..b23f884d 100644 --- a/test/e2e/framework/cleanup.go +++ b/test/e2e/framework/cleanup.go @@ -76,6 +76,10 @@ func (f *Framework) CleanUpNS() { IgnoreNotFound(k8s.DeleteNamespaceE(f.t, f.kubectlOptions, TestContext.E2ETestNamespace)) } +func (f *Framework) CleanUpChaos() { + k8s.RunKubectl(f.t, f.kubectlOptions, "delete", "podchaos", "--all", "-n", f.kubectlOptions.Namespace) +} + func IgnoreNotFound(err error) { if err != nil && strings.Contains(err.Error(), "not found") { // Do nothing diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 6862959a..389ca601 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -23,8 +23,8 @@ import ( "time" chaosapis "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" - "github.com/go-logr/logr" "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -49,7 +49,7 @@ type Framework struct { Timeout time.Duration - Log logr.Logger + Log logger.Logger } func NewFramework(baseName string) *Framework { @@ -67,7 +67,7 @@ func NewFramework(baseName string) *Framework { Name: TestContext.E2ETestNamespace, }, }, - Log: Log, + Log: *logger.Default, } return f diff --git a/test/e2e/framework/mysqlcluster.go b/test/e2e/framework/mysqlcluster.go index 46392a73..3a3624c9 100644 --- a/test/e2e/framework/mysqlcluster.go +++ b/test/e2e/framework/mysqlcluster.go @@ -93,6 +93,7 @@ func (f *Framework) ClusterEventuallyReplicas(cluster *apiv1alpha1.MysqlCluster, Eventually(func() int { cl := &apiv1alpha1.MysqlCluster{} f.Client.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cl) + f.Log.Logf(f.t, "ready nodes: %d/%d", cl.Status.ReadyNodes, *cl.Spec.Replicas) return cl.Status.ReadyNodes }, timeout, POLLING).Should(Equal(int(*cluster.Spec.Replicas)), "Not ready replicas of cluster '%s'", cluster.Name) } @@ -101,6 +102,7 @@ func (f *Framework) ClusterEventuallyRaftStatus(cluster *apiv1alpha1.MysqlCluste Eventually(func() bool { cl := &apiv1alpha1.MysqlCluster{} f.Client.Get(context.TODO(), types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cl) + f.Log.Logf(f.t, "checking xenon") return isXenonReadiness(cl) }, TIMEOUT, POLLING).Should(BeTrue(), "Not ready xenon of cluster '%s'", cluster.Name) }