From 9b6da31da7e4aef79f889e9b8182432c69cc6550 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 15:15:13 +0100 Subject: [PATCH 1/6] feat: terminate workers Allow to terminate one or all workers --- go-chaos/cmd/terminate.go | 38 +++++++++++++++++++++++++++++++++++++ go-chaos/internal/labels.go | 7 +++++++ go-chaos/internal/pods.go | 15 +++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index 1ef77523a..32b14d14e 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -22,6 +22,10 @@ import ( "github.com/zeebe-io/zeebe-chaos/go-chaos/internal" ) +var ( + all bool +) + func init() { rootCmd.AddCommand(terminateCmd) @@ -31,6 +35,9 @@ func init() { terminateCmd.MarkFlagsMutuallyExclusive("partitionId", "nodeId") terminateCmd.AddCommand(terminateGatewayCmd) + + terminateCmd.AddCommand(terminateWorkerCmd) + terminateWorkerCmd.Flags().BoolVar(&all, "all", false, "Specify whether all workers should be terminated") } var terminateCmd = &cobra.Command{ @@ -91,3 +98,34 @@ var terminateGatewayCmd = &cobra.Command{ fmt.Printf("Terminated %s\n", gatewayPod) }, } + +var terminateWorkerCmd = &cobra.Command{ + Use: "worker", + Short: "Terminates a Zeebe worker", + Long: `Terminates a Zeebe worker.`, + Run: func(cmd *cobra.Command, args []string) { + k8Client, err := internal.CreateK8Client() + ensureNoError(err) + + workerPods, err := k8Client.GetWorkerPods() + ensureNoError(err) + + if workerPods == nil || len(workerPods.Items) <= 0 { + panic(errors.New(fmt.Sprintf("Expected to find workers in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) + } + + if all { + for _, worker := range workerPods.Items { + err = k8Client.TerminatePod(worker.Name) + ensureNoError(err) + fmt.Printf("Terminated %s\n", worker.Name) + } + } else { + workerPod := workerPods.Items[0] + err = k8Client.TerminatePod(workerPod.Name) + ensureNoError(err) + + fmt.Printf("Terminated %s\n", workerPod.Name) + } + }, +} diff --git a/go-chaos/internal/labels.go b/go-chaos/internal/labels.go index 8f830f018..561afc53b 100644 --- a/go-chaos/internal/labels.go +++ b/go-chaos/internal/labels.go @@ -74,3 +74,10 @@ func (c K8Client) getGatewayLabels() string { return getSelfManagedGatewayLabels() } } + +func (c K8Client) getWorkerLabels() string { + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "worker"}, + } + return labels.Set(labelSelector.MatchLabels).String() +} diff --git a/go-chaos/internal/pods.go b/go-chaos/internal/pods.go index 4e9dc0536..09f9616a3 100644 --- a/go-chaos/internal/pods.go +++ b/go-chaos/internal/pods.go @@ -94,6 +94,21 @@ func (c K8Client) GetGatewayPodNames() ([]string, error) { return c.extractPodNames(list) } +func (c K8Client) GetWorkerPods() (*v1.PodList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: c.getWorkerLabels(), + // we check for running workers, since terminated workers can be lying around + FieldSelector: "status.phase=Running", + } + + list, err := c.Clientset.CoreV1().Pods(c.GetCurrentNamespace()).List(context.TODO(), listOptions) + if err != nil { + return nil, err + } + + return list, err +} + func (c K8Client) TerminatePod(podName string) error { gracePeriodSec := int64(0) options := metav1.DeleteOptions{GracePeriodSeconds: &gracePeriodSec} From d04398bd87a8a3096895f38fd015fe67b03e820b Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 15:31:07 +0100 Subject: [PATCH 2/6] feat: restart worker --- go-chaos/cmd/restart.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/go-chaos/cmd/restart.go b/go-chaos/cmd/restart.go index 33a66f2bf..293060709 100644 --- a/go-chaos/cmd/restart.go +++ b/go-chaos/cmd/restart.go @@ -15,6 +15,7 @@ package cmd import ( + "errors" "fmt" "github.com/spf13/cobra" @@ -34,6 +35,9 @@ func init() { if err := restartCmd.MarkFlagRequired("partitionId"); err != nil { panic(err) } + + restartCmd.AddCommand(restartWorkerCmd) + restartWorkerCmd.Flags().BoolVar(&all, "all", false, "Specify whether all workers should be restarted") } var restartCmd = &cobra.Command{ @@ -69,3 +73,34 @@ var restartCmd = &cobra.Command{ fmt.Println() }, } + +var restartWorkerCmd = &cobra.Command{ + Use: "worker", + Short: "Restart a Zeebe worker", + Long: `Restart a Zeebe worker.`, + Run: func(cmd *cobra.Command, args []string) { + k8Client, err := internal.CreateK8Client() + ensureNoError(err) + + workerPods, err := k8Client.GetWorkerPods() + ensureNoError(err) + + if workerPods == nil || len(workerPods.Items) <= 0 { + panic(errors.New(fmt.Sprintf("Expected to find workers in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) + } + + if all { + for _, worker := range workerPods.Items { + err = k8Client.RestartPod(worker.Name) + ensureNoError(err) + fmt.Printf("Restart %s\n", worker.Name) + } + } else { + workerPod := workerPods.Items[0] + err = k8Client.RestartPod(workerPod.Name) + ensureNoError(err) + + fmt.Printf("Restart %s\n", workerPod.Name) + } + }, +} From 56ff7a9f58586b7e1429c6bcce0b5833df0bace2 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 15:36:31 +0100 Subject: [PATCH 3/6] feat: add restart gateway --- go-chaos/cmd/restart.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/go-chaos/cmd/restart.go b/go-chaos/cmd/restart.go index 293060709..3cec98bed 100644 --- a/go-chaos/cmd/restart.go +++ b/go-chaos/cmd/restart.go @@ -36,6 +36,7 @@ func init() { panic(err) } + restartCmd.AddCommand(restartGatewayCmd) restartCmd.AddCommand(restartWorkerCmd) restartWorkerCmd.Flags().BoolVar(&all, "all", false, "Specify whether all workers should be restarted") } @@ -74,6 +75,35 @@ var restartCmd = &cobra.Command{ }, } +var restartGatewayCmd = &cobra.Command{ + Use: "gateway", + Short: "Restarts a Zeebe gateway", + Long: `Restarts a Zeebe gateway.`, + Run: func(cmd *cobra.Command, args []string) { + k8Client, err := internal.CreateK8Client() + if err != nil { + panic(err) + } + + gatewayPodNames, err := k8Client.GetGatewayPodNames() + if err != nil { + panic(err) + } + + if len(gatewayPodNames) <= 0 { + panic(errors.New(fmt.Sprintf("Expected to find Zeebe gateway in namespace %s, but none found.", k8Client.GetCurrentNamespace()))) + } + + gatewayPod := gatewayPodNames[0] + err = k8Client.RestartPod(gatewayPod) + if err != nil { + panic(err) + } + + fmt.Printf("Restarted %s\n", gatewayPod) + }, +} + var restartWorkerCmd = &cobra.Command{ Use: "worker", Short: "Restart a Zeebe worker", From 86fda4417673cd6f978d5cc25da78ff34dfacfca Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 15:39:42 +0100 Subject: [PATCH 4/6] refactor: introduce sub-command for broker restart --- go-chaos/cmd/restart.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/go-chaos/cmd/restart.go b/go-chaos/cmd/restart.go index 3cec98bed..8f225b906 100644 --- a/go-chaos/cmd/restart.go +++ b/go-chaos/cmd/restart.go @@ -24,17 +24,9 @@ import ( func init() { rootCmd.AddCommand(restartCmd) - - restartCmd.Flags().StringVar(&role, "role", "LEADER", "Specify the partition role [LEADER, FOLLOWER, INACTIVE]") - restartCmd.Flags().IntVar(&partitionId, "partitionId", 1, "Specify the id of the partition") - - if err := restartCmd.MarkFlagRequired("role"); err != nil { - panic(err) - } - - if err := restartCmd.MarkFlagRequired("partitionId"); err != nil { - panic(err) - } + restartCmd.AddCommand(restartBrokerCmd) + restartBrokerCmd.Flags().StringVar(&role, "role", "LEADER", "Specify the partition role [LEADER, FOLLOWER, INACTIVE]") + restartBrokerCmd.Flags().IntVar(&partitionId, "partitionId", 1, "Specify the id of the partition") restartCmd.AddCommand(restartGatewayCmd) restartCmd.AddCommand(restartWorkerCmd) @@ -43,6 +35,12 @@ func init() { var restartCmd = &cobra.Command{ Use: "restart", + Short: "Restarts a Zeebe node", + Long: `Restarts a Zeebe node, it can be chosen between: broker, gateway or a worker.`, +} + +var restartBrokerCmd = &cobra.Command{ + Use: "broker", Short: "Restarts a Zeebe broker", Long: `Restarts a Zeebe broker with a certain role and given partition.`, Run: func(cmd *cobra.Command, args []string) { @@ -70,7 +68,7 @@ var restartCmd = &cobra.Command{ panic(err) } - fmt.Printf("\nDeleted %s", broker) + fmt.Printf("Deleted %s", broker) fmt.Println() }, } From 52c58ed84d8787bb9a60a484fdf2882dc8ec518d Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 15:43:27 +0100 Subject: [PATCH 5/6] refactor: introduce sub-command broker for termination --- go-chaos/cmd/terminate.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/go-chaos/cmd/terminate.go b/go-chaos/cmd/terminate.go index 32b14d14e..4c107309a 100644 --- a/go-chaos/cmd/terminate.go +++ b/go-chaos/cmd/terminate.go @@ -29,10 +29,11 @@ var ( func init() { rootCmd.AddCommand(terminateCmd) - terminateCmd.Flags().StringVar(&role, "role", "LEADER", "Specify the partition role [LEADER, FOLLOWER]") - terminateCmd.Flags().IntVar(&partitionId, "partitionId", 1, "Specify the id of the partition") - terminateCmd.Flags().IntVar(&nodeId, "nodeId", -1, "Specify the nodeId of the Broker") - terminateCmd.MarkFlagsMutuallyExclusive("partitionId", "nodeId") + terminateCmd.AddCommand(terminateBrokerCmd) + terminateBrokerCmd.Flags().StringVar(&role, "role", "LEADER", "Specify the partition role [LEADER, FOLLOWER]") + terminateBrokerCmd.Flags().IntVar(&partitionId, "partitionId", 1, "Specify the id of the partition") + terminateBrokerCmd.Flags().IntVar(&nodeId, "nodeId", -1, "Specify the nodeId of the Broker") + terminateBrokerCmd.MarkFlagsMutuallyExclusive("partitionId", "nodeId") terminateCmd.AddCommand(terminateGatewayCmd) @@ -42,6 +43,12 @@ func init() { var terminateCmd = &cobra.Command{ Use: "terminate", + Short: "Terminates a Zeebe node", + Long: `Terminates a Zeebe node, it can be chosen between: broker, gateway or a worker.`, +} + +var terminateBrokerCmd = &cobra.Command{ + Use: "broker", Short: "Terminates a Zeebe broker", Long: `Terminates a Zeebe broker with a certain role and given partition.`, Run: func(cmd *cobra.Command, args []string) { From e6c7f27500e2e0f4471d24303367116827e498c2 Mon Sep 17 00:00:00 2001 From: Christopher Zell Date: Tue, 22 Nov 2022 16:31:34 +0100 Subject: [PATCH 6/6] refactor: update println MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ole Schönburg --- go-chaos/cmd/restart.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-chaos/cmd/restart.go b/go-chaos/cmd/restart.go index 8f225b906..7da0fd89f 100644 --- a/go-chaos/cmd/restart.go +++ b/go-chaos/cmd/restart.go @@ -68,7 +68,7 @@ var restartBrokerCmd = &cobra.Command{ panic(err) } - fmt.Printf("Deleted %s", broker) + fmt.Printf("Restarted %s", broker) fmt.Println() }, }