sidecarset support inject&upgrade pod annotations (#992)

Signed-off-by: liheng.zms <[email protected]>

apis/apps/defaults/v1alpha1.go

@@ -44,6 +44,14 @@ func SetDefaultsSidecarSet(obj *v1alpha1.SidecarSet) {
 	//default setting history revision limitation
 	SetDefaultRevisionHistoryLimit(&obj.Spec.RevisionHistoryLimit)
 
+	// default patchPolicy is 'Retain'
+	for i := range obj.Spec.PatchPodMetadata {
+		patch := &obj.Spec.PatchPodMetadata[i]
+		if patch.PatchPolicy == "" {
+			patch.PatchPolicy = v1alpha1.SidecarSetRetainPatchPolicy
+		}
+	}
+
 	//default setting injectRevisionStrategy
 	SetDefaultInjectRevision(&obj.Spec.InjectionStrategy)
 }

apis/apps/v1alpha1/sidecarset_types.go

@@ -66,8 +66,42 @@ type SidecarSetSpec struct {
 	// RevisionHistoryLimit indicates the maximum quantity of stored revisions about the SidecarSet.
 	// default value is 10
 	RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"`
+
+	// SidecarSet support to inject & in-place update metadata in pod.
+	PatchPodMetadata []SidecarSetPatchPodMetadata `json:"patchPodMetadata,omitempty"`
+}
+
+type SidecarSetPatchPodMetadata struct {
+	// annotations
+	Annotations map[string]string `json:"annotations,omitempty"`
+
+	// labels map[string]string `json:"labels,omitempty"`
+	// patch pod metadata policy, Default is "Retain"
+	PatchPolicy SidecarSetPatchPolicyType `json:"patchPolicy,omitempty"`
 }
 
+type SidecarSetPatchPolicyType string
+
+var (
+	// SidecarSetRetainPatchPolicy indicates if PatchPodFields conflicts with Pod,
+	// will ignore PatchPodFields, and retain the corresponding fields of pods.
+	// SidecarSet webhook cannot allow the conflict of PatchPodFields between SidecarSets under this policy type.
+	// Note: Retain is only supported for injection, and the Metadata will not be updated when upgrading the Sidecar Container in-place.
+	SidecarSetRetainPatchPolicy SidecarSetPatchPolicyType = "Retain"
+
+	// SidecarSetOverwritePatchPolicy indicates if PatchPodFields conflicts with Pod,
+	// SidecarSet will apply PatchPodFields to overwrite the corresponding fields of pods.
+	// SidecarSet webhook cannot allow the conflict of PatchPodFields between SidecarSets under this policy type.
+	// Overwrite support to inject and in-place metadata.
+	SidecarSetOverwritePatchPolicy SidecarSetPatchPolicyType = "Overwrite"
+
+	// SidecarSetMergePatchJsonPatchPolicy indicate that sidecarSet use application/merge-patch+json to patch annotation value,
+	// for example, A patch annotation[oom-score] = '{"log-agent": 1}' and B patch annotation[oom-score] = '{"envoy": 2}'
+	// result pod annotation[oom-score] = '{"log-agent": 1, "envoy": 2}'
+	// MergePatchJson support to inject and in-place metadata.
+	SidecarSetMergePatchJsonPatchPolicy SidecarSetPatchPolicyType = "MergePatchJson"
+)
+
 // SidecarContainer defines the container of Sidecar
 type SidecarContainer struct {
 	// +kubebuilder:pruning:PreserveUnknownFields

config/crd/bases/apps.kruise.io_sidecarsets.yaml

@@ -255,6 +255,22 @@ spec:
                 description: Namespace sidecarSet will only match the pods in the
                   namespace otherwise, match pods in all namespaces(in cluster)
                 type: string
+              patchPodMetadata:
+                description: SidecarSet support to inject & in-place update metadata
+                  in pod.
+                items:
+                  properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      description: annotations
+                      type: object
+                    patchPolicy:
+                      description: labels map[string]string `json:"labels,omitempty"`
+                        patch pod metadata policy, Default is "Retain"
+                      type: string
+                  type: object
+                type: array
               revisionHistoryLimit:
                 description: RevisionHistoryLimit indicates the maximum quantity of
                   stored revisions about the SidecarSet. default value is 10

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/containerd/containerd v1.5.10
 	github.com/docker/distribution v2.8.0+incompatible
 	github.com/docker/docker v20.10.2+incompatible
+	github.com/evanphx/json-patch v4.11.0+incompatible
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/gorilla/mux v1.8.0
@@ -66,7 +67,6 @@ require (
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
-	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
 	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/go-openapi/analysis v0.21.2 // indirect
 	github.com/go-openapi/errors v0.20.2 // indirect

main.go

@@ -114,6 +114,7 @@ func main() {
 		})
 	}
 
+	ctx := ctrl.SetupSignalHandler()
 	cfg := ctrl.GetConfigOrDie()
 	setRestConfig(cfg)
 	cfg.UserAgent = "kruise-manager"
@@ -169,8 +170,6 @@ func main() {
 	}
 
 	// +kubebuilder:scaffold:builder
-
-	ctx := ctrl.SetupSignalHandler()
 	setupLog.Info("initialize webhook")
 	if err := webhook.Initialize(ctx, cfg); err != nil {
 		setupLog.Error(err, "unable to initialize webhook")

pkg/control/sidecarcontrol/hash.go

@@ -21,9 +21,8 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"k8s.io/apimachinery/pkg/util/rand"
-
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
+	"k8s.io/apimachinery/pkg/util/rand"
 )
 
 // SidecarSetHash returns a hash of the SidecarSet.

pkg/control/sidecarcontrol/util.go

@@ -22,19 +22,22 @@ import (
 	"regexp"
 	"strings"
 
+	jsonpatch "github.com/evanphx/json-patch"
 	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
+	"github.com/openkruise/kruise/pkg/features"
 	"github.com/openkruise/kruise/pkg/util"
-
+	"github.com/openkruise/kruise/pkg/util/configuration"
+	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/validation"
-
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/klog/v2"
 	kubecontroller "k8s.io/kubernetes/pkg/controller"
 	"k8s.io/kubernetes/pkg/fieldpath"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const (
@@ -389,3 +392,116 @@ func ConvertDownwardAPIFieldLabel(version, label, value string) (string, string,
 	}
 	return "", "", fmt.Errorf("field label not supported: %s", label)
 }
+
+func PatchPodMetadata(originMetadata *metav1.ObjectMeta, patches []appsv1alpha1.SidecarSetPatchPodMetadata) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("%v", r)
+		}
+	}()
+
+	if originMetadata.Annotations == nil {
+		originMetadata.Annotations = map[string]string{}
+	}
+	for _, patch := range patches {
+		switch patch.PatchPolicy {
+		case appsv1alpha1.SidecarSetRetainPatchPolicy, "":
+			retainPatchPodMetadata(originMetadata, patch)
+		case appsv1alpha1.SidecarSetOverwritePatchPolicy:
+			overwritePatchPodMetadata(originMetadata, patch)
+		case appsv1alpha1.SidecarSetMergePatchJsonPatchPolicy:
+			if err = mergePatchJsonPodMetadata(originMetadata, patch); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func retainPatchPodMetadata(originMetadata *metav1.ObjectMeta, patchPodField appsv1alpha1.SidecarSetPatchPodMetadata) {
+	for k, v := range patchPodField.Annotations {
+		if _, ok := originMetadata.Annotations[k]; !ok {
+			originMetadata.Annotations[k] = v
+		}
+	}
+}
+
+func overwritePatchPodMetadata(originMetadata *metav1.ObjectMeta, patchPodField appsv1alpha1.SidecarSetPatchPodMetadata) {
+	for k, v := range patchPodField.Annotations {
+		originMetadata.Annotations[k] = v
+	}
+}
+
+func mergePatchJsonPodMetadata(originMetadata *metav1.ObjectMeta, patchPodField appsv1alpha1.SidecarSetPatchPodMetadata) error {
+	for key, patchJSON := range patchPodField.Annotations {
+		if origin, ok := originMetadata.Annotations[key]; ok && origin != "" {
+			modified, err := jsonpatch.MergePatch([]byte(origin), []byte(patchJSON))
+			if err != nil {
+				return err
+			}
+			originMetadata.Annotations[key] = string(modified)
+		} else {
+			originMetadata.Annotations[key] = patchJSON
+		}
+	}
+	return nil
+}
+
+func ValidateSidecarSetPatchMetadataWhitelist(c client.Client, sidecarSet *appsv1alpha1.SidecarSet) error {
+	if len(sidecarSet.Spec.PatchPodMetadata) == 0 {
+		return nil
+	}
+
+	regAnnotations := make([]*regexp.Regexp, 0)
+	whitelist, err := configuration.GetSidecarSetPatchMetadataWhiteList(c)
+	if err != nil {
+		return err
+	} else if whitelist == nil {
+		if utilfeature.DefaultFeatureGate.Enabled(features.SidecarSetPatchPodMetadataDefaultsAllowed) {
+			return nil
+		}
+		return fmt.Errorf("SidecarSet patch metadata whitelist not found")
+	}
+
+	for _, rule := range whitelist.Rules {
+		if rule.Selector != nil {
+			selector, err := util.GetFastLabelSelector(rule.Selector)
+			if err != nil {
+				return err
+			}
+			if !selector.Matches(labels.Set(sidecarSet.Labels)) {
+				continue
+			}
+		}
+		for _, key := range rule.AllowedAnnotationKeyExprs {
+			reg, err := regexp.Compile(key)
+			if err != nil {
+				return err
+			}
+			regAnnotations = append(regAnnotations, reg)
+		}
+	}
+	if len(regAnnotations) == 0 {
+		if utilfeature.DefaultFeatureGate.Enabled(features.SidecarSetPatchPodMetadataDefaultsAllowed) {
+			return nil
+		}
+		return fmt.Errorf("sidecarSet patch metadata annotation is not allowed")
+	}
+	for _, patch := range sidecarSet.Spec.PatchPodMetadata {
+		for key := range patch.Annotations {
+			if !matchRegKey(key, regAnnotations) {
+				return fmt.Errorf("sidecarSet patch metadata annotation(%s) is not allowed", key)
+			}
+		}
+	}
+	return nil
+}
+
+func matchRegKey(key string, regs []*regexp.Regexp) bool {
+	for _, reg := range regs {
+		if reg.MatchString(key) {
+			return true
+		}
+	}
+	return false
+}