Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set recommended kubernetes labels on services and workflows #482

Merged
merged 1 commit into from
Jul 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -354,4 +354,4 @@ create-cluster: install-kind

.PHONY: delete-cluster
delete-cluster: install-kind
kind delete cluster && docker rm -f kind-registry
kind delete cluster && docker rm -f kind-registry
Original file line number Diff line number Diff line change
Expand Up @@ -733,6 +733,7 @@ spec:
serviceAccountName: sonataflow-operator-controller-manager
deployments:
- label:
app.kubernetes.io/name: sonataflow-operator
control-plane: sonataflow-operator
name: sonataflow-operator-controller-manager
spec:
Expand Down
1 change: 1 addition & 0 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ metadata:
namespace: system
labels:
control-plane: sonataflow-operator
app.kubernetes.io/name: sonataflow-operator
spec:
selector:
matchLabels:
Expand Down
7 changes: 5 additions & 2 deletions controllers/platform/k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,11 @@ func createOrUpdateService(ctx context.Context, client client.Client, platform *

func getLabels(platform *operatorapi.SonataFlowPlatform, psh services.PlatformServiceHandler) (map[string]string, map[string]string) {
lbl := map[string]string{
workflowproj.LabelApp: platform.Name,
workflowproj.LabelService: psh.GetServiceName(),
workflowproj.LabelService: psh.GetServiceName(),
workflowproj.LabelK8SName: psh.GetContainerName(),
workflowproj.LabelK8SComponent: psh.GetServiceName(),
workflowproj.LabelK8SPartOF: platform.Name,
workflowproj.LabelK8SManagedBy: "sonataflow-operator",
}
selectorLbl := map[string]string{
workflowproj.LabelService: psh.GetServiceName(),
Expand Down
28 changes: 27 additions & 1 deletion controllers/profiles/common/mutate_visitors.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,17 @@ package common

import (
"context"
"maps"
"reflect"
"slices"

"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/discovery"
"github.com/apache/incubator-kie-kogito-serverless-operator/controllers/profiles/common/properties"
"github.com/imdario/mergo"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
servingv1 "knative.dev/serving/pkg/apis/serving/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
Expand Down Expand Up @@ -86,7 +91,28 @@ func DeploymentMutateVisitor(workflow *operatorapi.SonataFlow, plf *operatorapi.
if err != nil {
return err
}
return EnsureDeployment(original.(*appsv1.Deployment), object.(*appsv1.Deployment))
src := original.(*appsv1.Deployment)
dst := object.(*appsv1.Deployment)
// merge new and old labels, but prevent overriding to keep exiting immutable selector working.
mergo.Merge(&dst.ObjectMeta.Labels, src.ObjectMeta.Labels, mergo.WithAppendSlice)
// to prevent furhter merge conflcts set the same lables on both src and dst
src.ObjectMeta.Labels = dst.ObjectMeta.Labels
if !maps.Equal(dst.Spec.Selector.MatchLabels, src.Spec.Selector.MatchLabels) {
// mutating selector labels is not supported so to prevent merge conflicts we set src and dst
// values to be identical
src.Spec.Selector.MatchLabels = dst.Spec.Selector.MatchLabels
}
if !slices.EqualFunc(
dst.Spec.Selector.MatchExpressions,
src.Spec.Selector.MatchExpressions,
func(lsr1, lsr2 metav1.LabelSelectorRequirement) bool {
return reflect.DeepEqual(lsr1, lsr2)
}) {
// mutating selector matchExpressions is not supported so to prevent merge conflicts we set src and dst
// values to be identical
src.Spec.Selector.MatchExpressions = dst.Spec.Selector.MatchExpressions
}
return EnsureDeployment(src, dst)
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion controllers/profiles/common/object_creators.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func DeploymentCreator(workflow *operatorapi.SonataFlow, plf *operatorapi.Sonata
Spec: appsv1.DeploymentSpec{
Replicas: getReplicasOrDefault(workflow),
Selector: &metav1.LabelSelector{
MatchLabels: lbl,
MatchLabels: workflowproj.GetSelectorLabels(workflow),
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Expand Down
16 changes: 14 additions & 2 deletions controllers/profiles/common/object_creators_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ import (
"github.com/apache/incubator-kie-kogito-serverless-operator/workflowproj"
)

const platformName = "test-platform"

func Test_ensureWorkflowPropertiesConfigMapMutator(t *testing.T) {
workflow := test.GetBaseSonataFlowWithDevProfile(t.Name())
platform := test.GetBasePlatform()
Expand Down Expand Up @@ -192,7 +194,12 @@ func Test_ensureWorkflowSinkBindingIsCreated(t *testing.T) {
assert.NotEmpty(t, reflectSinkBinding.Spec.Sink)
assert.Equal(t, reflectSinkBinding.Spec.Sink.Ref.Kind, "Broker")
assert.NotNil(t, reflectSinkBinding.GetLabels())
assert.Equal(t, reflectSinkBinding.ObjectMeta.Labels, map[string]string{"app": "vet", "sonataflow.org/workflow-app": "vet"})
assert.Equal(t, reflectSinkBinding.ObjectMeta.Labels, map[string]string{
"sonataflow.org/workflow-app": "vet",
"app.kubernetes.io/name": "vet",
"app.kubernetes.io/component": "serverless-workflow",
"app.kubernetes.io/managed-by": "sonataflow-operator",
})
}

func Test_ensureWorkflowTriggersAreCreated(t *testing.T) {
Expand All @@ -206,7 +213,12 @@ func Test_ensureWorkflowTriggersAreCreated(t *testing.T) {
for _, trigger := range triggers {
assert.Contains(t, []string{"vet-vetappointmentrequestreceived-trigger", "vet-vetappointmentinfo-trigger"}, trigger.GetName())
assert.NotNil(t, trigger.GetLabels())
assert.Equal(t, trigger.GetLabels(), map[string]string{"app": "vet", "sonataflow.org/workflow-app": "vet"})
assert.Equal(t, trigger.GetLabels(), map[string]string{
"sonataflow.org/workflow-app": "vet",
"app.kubernetes.io/name": "vet",
"app.kubernetes.io/component": "serverless-workflow",
"app.kubernetes.io/managed-by": "sonataflow-operator",
})
}
}

Expand Down
8 changes: 7 additions & 1 deletion controllers/profiles/dev/object_creators_dev_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,5 +43,11 @@ func Test_ensureWorkflowDevServiceIsExposed(t *testing.T) {
assert.Equal(t, reflectService.Spec.Type, v1.ServiceTypeNodePort)
assert.NotNil(t, reflectService.ObjectMeta)
assert.NotNil(t, reflectService.ObjectMeta.Labels)
assert.Equal(t, reflectService.ObjectMeta.Labels, map[string]string{"test": "test", "app": "greeting", "sonataflow.org/workflow-app": "greeting"})
assert.Equal(t, reflectService.ObjectMeta.Labels, map[string]string{
"test": "test",
"sonataflow.org/workflow-app": "greeting",
"app.kubernetes.io/name": "greeting",
"app.kubernetes.io/component": "serverless-workflow",
"app.kubernetes.io/managed-by": "sonataflow-operator",
})
}
2 changes: 1 addition & 1 deletion controllers/profiles/dev/status_enricher_dev.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func statusEnricher(ctx context.Context, c client.Client, workflow *operatorapi.
podList := &v1.PodList{}
opts := []client.ListOption{
client.InNamespace(workflow.Namespace),
client.MatchingLabels{workflowproj.LabelApp: labels[workflowproj.LabelApp]},
client.MatchingLabels{workflowproj.LabelK8SName: labels[workflowproj.LabelK8SName]},
}
err := c.List(ctx, podList, opts...)
if err != nil {
Expand Down
9 changes: 8 additions & 1 deletion controllers/profiles/gitops/profile_gitops_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,5 +64,12 @@ func Test_Reconciler_ProdOps(t *testing.T) {

assert.NotNil(t, deployment.ObjectMeta)
assert.NotNil(t, deployment.ObjectMeta.Labels)
assert.Equal(t, deployment.ObjectMeta.Labels, map[string]string{"test": "test", "app": "simple", "sonataflow.org/workflow-app": "simple"})
assert.Equal(t, deployment.ObjectMeta.Labels, map[string]string{
"test": "test",
"sonataflow.org/workflow-app": "simple",
"app.kubernetes.io/name": "simple",
"app.kubernetes.io/component": "serverless-workflow",
"app.kubernetes.io/managed-by": "sonataflow-operator",
"app.kubernetes.io/part-of": "sonataflow-platform",
})
}
8 changes: 7 additions & 1 deletion controllers/profiles/preview/profile_preview_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,13 @@ func Test_Reconciler_ProdCustomPod(t *testing.T) {
assert.Len(t, deployment.Spec.Template.Spec.Containers[0].VolumeMounts, 1)
assert.NotNil(t, deployment.ObjectMeta)
assert.NotNil(t, deployment.ObjectMeta.Labels)
assert.Equal(t, deployment.ObjectMeta.Labels, map[string]string{"test": "test", "app": "greeting", "sonataflow.org/workflow-app": "greeting"})
assert.Equal(t, deployment.ObjectMeta.Labels, map[string]string{
"test": "test",
"sonataflow.org/workflow-app": "greeting",
"app.kubernetes.io/name": "greeting",
"app.kubernetes.io/component": "serverless-workflow",
"app.kubernetes.io/managed-by": "sonataflow-operator",
})
}

func Test_reconcilerProdBuildConditions(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27076,6 +27076,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app.kubernetes.io/name: sonataflow-operator
control-plane: sonataflow-operator
name: sonataflow-operator-controller-manager
namespace: sonataflow-operator-system
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/platform_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,12 @@ var _ = Describe("Validate the persistence", Ordered, func() {
By("Wait for SonataFlowPlatform CR to complete deployment")
// wait for service deployments to be ready
EventuallyWithOffset(1, func() error {
cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app=sonataflow-platform", "--for", "condition=Ready", "--timeout=5s")
cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "--for", "condition=Ready", "--timeout=5s")
_, err = utils.Run(cmd)
return err
}, 20*time.Minute, 5).Should(Succeed())
By("Evaluate status of service's health endpoint")
cmd = exec.Command("kubectl", "get", "pod", "-l", "app=sonataflow-platform", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}")
cmd = exec.Command("kubectl", "get", "pod", "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}")
output, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
// remove the last CR that is added by default as the last character of the string.
Expand Down Expand Up @@ -140,12 +140,12 @@ var _ = Describe("Validate the persistence", Ordered, func() {
By("Wait for SonatatFlowPlatform CR to complete deployment")
// wait for service deployments to be ready
EventuallyWithOffset(1, func() error {
cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app=sonataflow-platform", "--for", "condition=Ready", "--timeout=5s")
cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "--for", "condition=Ready", "--timeout=5s")
_, err = utils.Run(cmd)
return err
}, 10*time.Minute, 5).Should(Succeed())
By("Evaluate status of all service's health endpoint")
cmd = exec.Command("kubectl", "get", "pod", "-l", "app=sonataflow-platform", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}")
cmd = exec.Command("kubectl", "get", "pod", "-l", "app.kubernetes.io/name in (jobs-service,data-index-service)", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}")
output, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
for _, pn := range strings.Split(string(output), " ") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ metadata:
sonataflow.org/version: 0.0.1
labels:
test: test
app: not-simple
spec:
podTemplate:
container:
Expand Down
26 changes: 20 additions & 6 deletions workflowproj/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,14 @@ const (
// ApplicationPropertiesFileName is the default application properties file name holding user properties
ApplicationPropertiesFileName = "application.properties"
workflowManagedConfigMapNameSuffix = "-managed-props"
// LabelApp key to use among object selectors, "app" is used among k8s applications to group objects in some UI consoles
LabelApp = "app"
// LabelService key to use among object selectors
LabelService = metadata.Domain + "/service"
// LabelWorkflow specialized label managed by the controller
LabelWorkflow = metadata.Domain + "/workflow-app"
LabelWorkflow = metadata.Domain + "/workflow-app"
LabelK8SName = "app.kubernetes.io/name"
LabelK8SComponent = "app.kubernetes.io/component"
LabelK8SPartOF = "app.kubernetes.io/part-of"
LabelK8SManagedBy = "app.kubernetes.io/managed-by"
)

// SetTypeToObject sets the Kind and ApiVersion to a given object since the default constructor won't do it.
Expand Down Expand Up @@ -84,10 +86,22 @@ func GetManagedPropertiesFileName(workflow *operatorapi.SonataFlow) string {

// GetDefaultLabels gets the default labels based on the given workflow.
func GetDefaultLabels(workflow *operatorapi.SonataFlow) map[string]string {
return map[string]string{
LabelApp: workflow.Name,
LabelWorkflow: workflow.Name,
labels := map[string]string{
LabelWorkflow: workflow.Name,
LabelK8SName: workflow.Name,
LabelK8SComponent: "serverless-workflow",
LabelK8SManagedBy: "sonataflow-operator",
}
if workflow.Status.Platform != nil {
labels[LabelK8SPartOF] = workflow.Status.Platform.Name
}
return labels

}
func GetSelectorLabels(workflow *operatorapi.SonataFlow) map[string]string {
labels := GetDefaultLabels(workflow)
delete(labels, LabelK8SPartOF)
return labels
}

// SetMergedLabels adds the merged labels to the given object.
Expand Down
7 changes: 6 additions & 1 deletion workflowproj/workflowproj_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,12 @@ func Test_Handler_WorkflowMinimalAndPropsAndSpec(t *testing.T) {
assert.NoError(t, err)
assert.NotNil(t, proj.Workflow)
assert.NotNil(t, proj.Workflow.ObjectMeta)
assert.Equal(t, proj.Workflow.ObjectMeta.Labels, map[string]string{"app": "hello", "sonataflow.org/workflow-app": "hello"})
assert.Equal(t, proj.Workflow.ObjectMeta.Labels, map[string]string{
"sonataflow.org/workflow-app": "hello",
"app.kubernetes.io/name": "hello",
"app.kubernetes.io/component": "serverless-workflow",
"app.kubernetes.io/managed-by": "sonataflow-operator",
})
assert.NotNil(t, proj.Properties)
assert.NotEmpty(t, proj.Resources)
assert.Equal(t, "hello", proj.Workflow.Name)
Expand Down
Loading