Migrate follower restart experiment

go-chaos/cmd/deploy.go

@@ -24,13 +24,12 @@ func init() {
 	rootCmd.AddCommand(deployCmd)
 
 	deployCmd.AddCommand(deployProcessModelCmd)
-
 	deployProcessModelCmd.Flags().StringVar(&processModelPath, "processModelPath", "",
-		"Specify the path to a BPMN process model, which should be deployed.")
-	deployProcessModelCmd.Flags().IntVar(&multipleVersions, "multipleVersions", 10,
-		"Specify how many different versions of a default BPMN and DMN model should be deployed. Useful for testing deployment distribution.")
+		"Specify the path to a BPMN process model, which should be deployed. Defaults to a benchmark process model with one task (included in zbchaos).")
 
-	deployProcessModelCmd.MarkFlagsMutuallyExclusive("processModelPath", "multipleVersions")
+	deployCmd.AddCommand(deployMultiVersionProcessModelCmd)
+	deployMultiVersionProcessModelCmd.Flags().IntVar(&versionCount, "versionCount", 10,
+		"Specify how many different versions of a default BPMN and DMN model should be deployed. Useful for testing deployment distribution.")
 
 	deployCmd.AddCommand(deployWorkerCmd)
 	deployCmd.AddCommand(deployChaosModels)
@@ -60,17 +59,33 @@ Defaults to the later, which is useful for experimenting with deployment distrib
 		ensureNoError(err)
 		defer zbClient.Close()
 
-		if len(processModelPath) == 0 {
-			// deploy multi version
-			err := internal.DeployDifferentVersions(zbClient, int32(multipleVersions))
-			ensureNoError(err)
-			internal.LogInfo("Deployed different process models of different types and versions to zeebe!")
-		} else {
-			processDefinitionKey, err := internal.DeployModel(zbClient, processModelPath)
-			ensureNoError(err)
-
-			internal.LogInfo("Deployed given process model %s, under key %d!", processModelPath, processDefinitionKey)
-		}
+		processDefinitionKey, err := internal.DeployModel(zbClient, processModelPath)
+		ensureNoError(err)
+
+		internal.LogInfo("Deployed given process model %s, under key %d!", processModelPath, processDefinitionKey)
+	},
+}
+
+var deployMultiVersionProcessModelCmd = &cobra.Command{
+	Use:   "multi-version",
+	Short: "Deploy multiple versions to Zeebe",
+	Long: `Deploy multiple versions of process and dmn models to Zeebe.
+Useful for experimenting with deployment distribution.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		k8Client, err := internal.CreateK8Client()
+		ensureNoError(err)
+
+		port := 26500
+		closeFn := k8Client.MustGatewayPortForward(port, port)
+		defer closeFn()
+
+		zbClient, err := internal.CreateZeebeClient(port)
+		ensureNoError(err)
+		defer zbClient.Close()
+
+		err = internal.DeployDifferentVersions(zbClient, int32(versionCount))
+		ensureNoError(err)
+		internal.LogInfo("Deployed different process models of different types and versions to zeebe!")
 	},
 }
 

go-chaos/cmd/root.go

@@ -30,7 +30,7 @@ var (
 	role               string
 	nodeId             int
 	processModelPath   string
-	multipleVersions   int
+	versionCount       int
 	variables          string
 	msgName            string
 	awaitResult        bool

go-chaos/internal/chaos-experiments/camunda-cloud/production-s/deployment-distribution/experiment.json

@@ -39,7 +39,7 @@
             "provider": {
                 "type": "process",
                 "path": "zbchaos",
-                "arguments": ["deploy", "process", "--multipleVersions", "10"],
+                "arguments": ["deploy", "multi-version", "--versionCount", "10"],
                 "timeout": 900
             }
         },

go-chaos/internal/chaos-experiments/camunda-cloud/production-s/follower-restart/experiment.json

@@ -15,7 +15,19 @@
                 "tolerance": 0,
                 "provider": {
                     "type": "process",
-                    "path": "verify-readiness.sh",
+                    "path": "zbchaos",
+                    "arguments": ["verify", "readiness"],
+                    "timeout": 900
+                }
+            },
+            {
+                "name": "Can deploy process model",
+                "type": "probe",
+                "tolerance": 0,
+                "provider": {
+                    "type": "process",
+                    "path": "zbchaos",
+                    "arguments": ["deploy", "process"],
                     "timeout": 900
                 }
             },
@@ -25,8 +37,8 @@
                 "tolerance": 0,
                 "provider": {
                     "type": "process",
-                    "path": "verify-steady-state.sh",
-                    "arguments": "1",
+                    "path": "zbchaos",
+                    "arguments": ["verify", "instance-creation", "--partitionId", "1"],
                     "timeout": 900
                 }
             }
@@ -38,8 +50,8 @@
             "name": "Restart follower of partition 1 gracefully",
             "provider": {
                 "type": "process",
-                "path": "shutdown-gracefully-partition.sh",
-                "arguments": ["Follower", "1"]
+                "path": "zbchaos",
+                "arguments": ["restart", "broker", "--role", "FOLLOWER", "--partitionId", "1"]
             }
         }
     ],

go-chaos/internal/zeebe.go

@@ -284,10 +284,9 @@ type ProcessInstanceCreationOptions struct {
 
 func CreateProcessInstanceCreator(zbClient zbc.Client, options ProcessInstanceCreationOptions) (ProcessInstanceCreator, error) {
 	var processInstanceCreator ProcessInstanceCreator
-	LogVerbose("Create process instance with BPMN process ID %s and version %d [variables: '%s', awaitResult: %t]",
-		options.BpmnProcessId, options.Version, options.Variables, options.AwaitResult)
-
 	processInstanceCreator = func() (int64, error) {
+		LogVerbose("Send create process instance command, with BPMN process ID '%s' and version '%d' (-1 means latest) [variables: '%s', awaitResult: %t]",
+			options.BpmnProcessId, options.Version, options.Variables, options.AwaitResult)
 		commandStep3 := zbClient.NewCreateInstanceCommand().BPMNProcessId(options.BpmnProcessId).Version(options.Version)
 		if len(options.Variables) != 0 {
 			_, err := commandStep3.VariablesFromString(options.Variables)

go-chaos/worker/chaos_worker.go

@@ -76,11 +76,15 @@ func HandleZbChaosJob(client worker.JobClient, job entities.Job, commandRunner C
 
 	err = commandRunner(commandArgs, commandCtx)
 	if err != nil {
+		internal.LogInfo("Error on running command. [key: %d, args: %s]. Error: %s", job.Key, commandArgs, err.Error())
 		_, _ = client.NewFailJobCommand().JobKey(job.Key).Retries(job.Retries - 1).Send(ctx)
 		return
 	}
 
-	_, _ = client.NewCompleteJobCommand().JobKey(job.Key).Send(ctx)
+	_, err = client.NewCompleteJobCommand().JobKey(job.Key).Send(ctx)
+	if err != nil {
+		internal.LogInfo("Error on completing the job [key: %d]. Error: %s", job.Key, err.Error())
+	}
 }
 
 func HandleReadExperiments(client worker.JobClient, job entities.Job) {