Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate worker restart experiment #281

Merged
merged 2 commits into from
Dec 8, 2022
Merged

Migrate worker restart experiment #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5609edd
fix: handle worker deployment in self-managed env
ChrisKujawa Dec 8, 2022
e4887b9
feat: migrate worker restart
ChrisKujawa Dec 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 56 additions & 7 deletions ...haos/internal/chaos-experiments/camunda-cloud/production-s/worker-restart/experiment.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,85 @@
"tolerance": 0,
"provider": {
"type": "process",
"path": "verify-readiness.sh",
"path": "zbchaos",
"arguments": ["verify", "readiness"],
"timeout": 900
}
},
{
"name": "Should be able to create a process and await the result",
"name": "Can deploy process model",
"type": "probe",
"tolerance": 0,
"provider": {
"type": "process",
"path": "await-processes-with-result.sh",
"arguments": "1",
"path": "zbchaos",
"arguments": ["deploy", "process"],
"timeout": 900
}
},
{
"name": "Should be able to create process instances on partition 1",
"type": "probe",
"tolerance": 0,
"provider": {
"type": "process",
"path": "zbchaos",
"arguments": ["verify", "instance-creation", "--partitionId", "1"],
"timeout": 900
}
}
]
},
"method": [
{
"type": "action",
"name": "Restart worker pod",
"name": "Deploy Workers",
"type": "probe",
"tolerance": 0,
"provider": {
"type": "process",
"path": "terminate-workers.sh",
"path": "zbchaos",
"arguments": ["deploy", "worker"],
"timeout": 900
},
"pauses": {
"after": 5
}
},
{
"name": "Should be able to create a process and await the result",
"type": "probe",
"tolerance": 0,
"provider": {
"type": "process",
"path": "zbchaos",
"arguments": ["verify", "instance-creation", "--partitionId", "1", "--awaitResult"],
"timeout": 900
}
},
{
"name": "Restart Workers",
"type": "probe",
"tolerance": 0,
"provider": {
"type": "process",
"path": "zbchaos",
"arguments": ["restart", "worker"],
"timeout": 900
},
"pauses": {
"after": 5
}
},
{
"name": "Should be able to create a process and await the result",
"type": "probe",
"tolerance": 0,
"provider": {
"type": "process",
"path": "zbchaos",
"arguments": ["verify", "instance-creation", "--partitionId", "1", "--awaitResult"],
"timeout": 900
}
}
],
"rollbacks": []
Expand Down
23 changes: 23 additions & 0 deletions go-chaos/internal/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"embed"
"errors"
"fmt"
"strings"

v12 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -64,6 +65,28 @@ func (c K8Client) CreateWorkerDeployment() error {
return err
}

if !c.SaaSEnv {
// We are in self-managed environment
// We have to update the service url such that our workers can connect
// We expect that the used helm release name is == to the namespace name

// JAVA_OPTIONS
envVar := deployment.Spec.Template.Spec.Containers[0].Env[0]
envVar.Value = strings.Replace(envVar.Value, "zeebe-service:26500", fmt.Sprintf("%s-zeebe-gateway:26500", c.GetCurrentNamespace()), 1)
deployment.Spec.Template.Spec.Containers[0].Env[0] = envVar
}

_, err = c.Clientset.AppsV1().Deployments(c.GetCurrentNamespace()).Create(context.TODO(), deployment, metav1.CreateOptions{})

if err != nil {
if err.Error() == "deployments.apps \"worker\" already exists" {
LogInfo("Workers have already deployed, update deployment.")
_, err = c.Clientset.AppsV1().Deployments(c.GetCurrentNamespace()).Update(context.TODO(), deployment, metav1.UpdateOptions{})
if err != nil {
return err
}
return nil
}
}
return err
}
36 changes: 36 additions & 0 deletions go-chaos/internal/deployment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,4 +112,40 @@ func Test_ShouldDeployWorkerDeployment(t *testing.T) {

assert.Equal(t, 1, len(deploymentList.Items))
assert.Equal(t, "worker", deploymentList.Items[0].Name)
assert.Contains(t, deploymentList.Items[0].Spec.Template.Spec.Containers[0].Env[0].Value, "-Dapp.brokerUrl=testNamespace-zeebe-gateway:26500")
}

func Test_ShouldNotReturnErrorWhenWorkersAlreadyDeployed(t *testing.T) {
// given
k8Client := CreateFakeClient()
_ = k8Client.CreateWorkerDeployment()

// when
err := k8Client.CreateWorkerDeployment()

// then
require.NoError(t, err)
deploymentList, err := k8Client.Clientset.AppsV1().Deployments(k8Client.GetCurrentNamespace()).List(context.TODO(), metav1.ListOptions{})
require.NoError(t, err)

assert.Equal(t, 1, len(deploymentList.Items))
assert.Equal(t, "worker", deploymentList.Items[0].Name)
}

func Test_ShouldDeployWorkerInSaas(t *testing.T) {
// given
k8Client := CreateFakeClient()
k8Client.createSaaSCRD(t)

// when
err := k8Client.CreateWorkerDeployment()

// then
require.NoError(t, err)
deploymentList, err := k8Client.Clientset.AppsV1().Deployments(k8Client.GetCurrentNamespace()).List(context.TODO(), metav1.ListOptions{})
require.NoError(t, err)

assert.Equal(t, 1, len(deploymentList.Items))
assert.Equal(t, "worker", deploymentList.Items[0].Name)
assert.Contains(t, deploymentList.Items[0].Spec.Template.Spec.Containers[0].Env[0].Value, "-Dapp.brokerUrl=zeebe-service:26500")
}