sidecarset support k8s 1.28 sidecarContainers

.gitignore

@@ -28,3 +28,5 @@ test/e2e/generated/bindata.go
 .vscode
 
 .DS_Store
+
+vendor/

pkg/control/sidecarcontrol/hash.go

@@ -43,6 +43,9 @@ func SidecarSetHashWithoutImage(sidecarSet *appsv1alpha1.SidecarSet) (string, er
 	for i := range ss.Spec.Containers {
 		ss.Spec.Containers[i].Image = ""
 	}
+	for i := range ss.Spec.InitContainers {
+		ss.Spec.InitContainers[i].Image = ""
+	}
 	encoded, err := encodeSidecarSet(ss)
 	if err != nil {
 		return "", err
@@ -53,7 +56,17 @@ func SidecarSetHashWithoutImage(sidecarSet *appsv1alpha1.SidecarSet) (string, er
 func encodeSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (string, error) {
 	// json.Marshal sorts the keys in a stable order in the encoding
 	m := map[string]interface{}{"containers": sidecarSet.Spec.Containers}
-	//m["initContainers"] = sidecarSet.Spec.InitContainers
+	// when k8s 1.28, if initContainer restartPolicy = Always, indicates it is sidecar container, so the hash needs to contain it.
+	initContainer := make([]appsv1alpha1.SidecarContainer, 0)
+	for i := range sidecarSet.Spec.InitContainers {
+		container := &sidecarSet.Spec.InitContainers[i]
+		if IsSidecarContainer(container.Container) {
+			initContainer = append(initContainer, *container)
+		}
+	}
+	if len(initContainer) > 0 {
+		m["initContainers"] = sidecarSet.Spec.InitContainers
+	}
 	data, err := json.Marshal(m)
 	if err != nil {
 		return "", err

pkg/control/sidecarcontrol/util.go

@@ -564,3 +564,11 @@ func matchRegKey(key string, regs []*regexp.Regexp) bool {
 	}
 	return false
 }
+
+// IsSidecarContainer check whether initContainer is sidecar container in k8s 1.28.
+func IsSidecarContainer(container corev1.Container) bool {
+	if container.RestartPolicy != nil && *container.RestartPolicy == corev1.ContainerRestartPolicyAlways {
+		return true
+	}
+	return false
+}

pkg/webhook/pod/mutating/sidecarset.go

@@ -368,11 +368,13 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
 		}
 
 		isInjecting := false
+		sidecarList := sets.NewString()
 		//process initContainers
 		//only when created pod, inject initContainer and pullSecrets
 		if !isUpdated {
 			for i := range sidecarSet.Spec.InitContainers {
 				initContainer := &sidecarSet.Spec.InitContainers[i]
+				sidecarList.Insert(initContainer.Name)
 				// volumeMounts that injected into sidecar container
 				// when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs)
 				injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(control, initContainer, pod)
@@ -393,13 +395,22 @@ func buildSidecars(isUpdated bool, pod *corev1.Pod, oldPod *corev1.Pod, matchedS
 				// merged Env from sidecar.Env and transfer envs
 				initContainer.Env = util.MergeEnvVar(initContainer.Env, transferEnvs)
 				isInjecting = true
-				sidecarInitContainers = append(sidecarInitContainers, initContainer)
+
+				// when sidecar container UpgradeStrategy is HotUpgrade
+				if sidecarcontrol.IsSidecarContainer(initContainer.Container) && sidecarcontrol.IsHotUpgradeContainer(initContainer) {
+					hotContainers, annotations := injectHotUpgradeContainers(hotUpgradeWorkInfo, initContainer)
+					sidecarInitContainers = append(sidecarInitContainers, hotContainers...)
+					for k, v := range annotations {
+						injectedAnnotations[k] = v
+					}
+				} else {
+					sidecarInitContainers = append(sidecarInitContainers, initContainer)
+				}
 			}
 			//process imagePullSecrets
 			sidecarSecrets = append(sidecarSecrets, sidecarSet.Spec.ImagePullSecrets...)
 		}
 
-		sidecarList := sets.NewString()
 		//process containers
 		for i := range sidecarSet.Spec.Containers {
 			sidecarContainer := &sidecarSet.Spec.Containers[i]

pkg/webhook/pod/mutating/sidecarset_test.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 
 	"github.com/openkruise/kruise/apis"
@@ -60,6 +61,8 @@ func TestMain(m *testing.M) {
 }
 
 var (
+	always = corev1.ContainerRestartPolicyAlways
+
 	sidecarSet1 = &appsv1alpha1.SidecarSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "sidecarset1",
@@ -113,6 +116,43 @@ var (
 		},
 	}
 
+	sidecarSetWithInitContainer = &appsv1alpha1.SidecarSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Annotations: map[string]string{
+				sidecarcontrol.SidecarSetHashAnnotation: "c4k2dbb95d",
+			},
+			Name:   "sidecarset",
+			Labels: map[string]string{},
+		},
+		Spec: appsv1alpha1.SidecarSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": "suxing-test",
+				},
+			},
+			InitContainers: []appsv1alpha1.SidecarContainer{
+				{
+					Container: corev1.Container{
+						Name:  "dns-e",
+						Image: "dns-e-image:1.0",
+						VolumeMounts: []corev1.VolumeMount{
+							{Name: "volume-1"},
+						},
+					},
+					TransferEnv: []appsv1alpha1.TransferEnvVar{
+						{
+							SourceContainerName: "nginx",
+							EnvName:             "hello2",
+						},
+					},
+				},
+			},
+			Volumes: []corev1.Volume{
+				{Name: "volume-1"},
+			},
+		},
+	}
+
 	sidecarsetWithTransferEnv = &appsv1alpha1.SidecarSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Annotations: map[string]string{
@@ -1108,6 +1148,113 @@ func TestMergeSidecarContainers(t *testing.T) {
 	}
 }
 
+func TestInjectInitContainer(t *testing.T) {
+	cases := []struct {
+		name                     string
+		getSidecarSets           func() []*appsv1alpha1.SidecarSet
+		getPod                   func() *corev1.Pod
+		expectedPod              func() *corev1.Pod
+		expectedInitContainerLen int
+	}{
+		{
+			name: "inject initContainers-1",
+			getSidecarSets: func() []*appsv1alpha1.SidecarSet {
+				obj1 := sidecarSetWithInitContainer.DeepCopy()
+				obj1.Name = "sidecarset-1"
+				obj1.Annotations[sidecarcontrol.SidecarSetHashAnnotation] = "sidecarset-1-hash"
+				obj1.Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = "sidecarset-1-without-image-hash"
+				obj1.Spec.InitContainers[0].Name = "init-1"
+				obj1.Spec.InitContainers[0].RestartPolicy = &always
+				obj2 := sidecarSetWithInitContainer.DeepCopy()
+				obj2.Name = "sidecarset-2"
+				obj2.Annotations[sidecarcontrol.SidecarSetHashAnnotation] = "sidecarset-2-hash"
+				obj2.Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = "sidecarset-2-without-image-hash"
+				obj2.Spec.InitContainers[0].Name = "hot-init"
+				obj2.Spec.InitContainers[0].RestartPolicy = &always
+				obj2.Spec.InitContainers[0].UpgradeStrategy.UpgradeType = appsv1alpha1.SidecarContainerHotUpgrade
+				obj2.Spec.InitContainers[0].UpgradeStrategy.HotUpgradeEmptyImage = "empty:v1"
+				return []*appsv1alpha1.SidecarSet{obj1, obj2}
+			},
+			getPod: func() *corev1.Pod {
+				obj := pod1.DeepCopy()
+				return obj
+			},
+			expectedPod: func() *corev1.Pod {
+				obj := pod1.DeepCopy()
+				obj.Annotations = map[string]string{}
+				sidecarSetHash := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec)
+				sidecarSetHashWithoutImage := make(map[string]sidecarcontrol.SidecarSetUpgradeSpec)
+				sidecarSetHash["sidecarset-1"] = sidecarcontrol.SidecarSetUpgradeSpec{
+					UpdateTimestamp: metav1.Now(),
+					SidecarSetHash:  "sidecarset-1-hash",
+					SidecarSetName:  "sidecarset-1",
+					SidecarList:     []string{"init-1"},
+				}
+				sidecarSetHash["sidecarset-2"] = sidecarcontrol.SidecarSetUpgradeSpec{
+					UpdateTimestamp: metav1.Now(),
+					SidecarSetHash:  "sidecarset-2-hash",
+					SidecarSetName:  "sidecarset-2",
+					SidecarList:     []string{"hot-init"},
+				}
+				sidecarSetHashWithoutImage["sidecarset-1"] = sidecarcontrol.SidecarSetUpgradeSpec{
+					UpdateTimestamp: metav1.Now(),
+					SidecarSetHash:  "sidecarset-1-without-image-hash",
+					SidecarSetName:  "sidecarset-1",
+					SidecarList:     []string{"init-1"},
+				}
+				sidecarSetHashWithoutImage["sidecarset-2"] = sidecarcontrol.SidecarSetUpgradeSpec{
+					UpdateTimestamp: metav1.Now(),
+					SidecarSetHash:  "sidecarset-2-without-image-hash",
+					SidecarSetName:  "sidecarset-2",
+					SidecarList:     []string{"hot-init"},
+				}
+				by, _ := json.Marshal(sidecarSetHash)
+				obj.Annotations[sidecarcontrol.SidecarSetHashAnnotation] = string(by)
+				by, _ = json.Marshal(sidecarSetHashWithoutImage)
+				obj.Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = string(by)
+				// store matched sidecarset list in pod annotations
+				obj.Annotations[sidecarcontrol.SidecarSetListAnnotation] = "sidecarset-1,sidecarset-2"
+				hotUpgradeWorkInfo := map[string]string{
+					"hot-init": "hot-init-1",
+				}
+				by, _ = json.Marshal(hotUpgradeWorkInfo)
+				obj.Annotations[sidecarcontrol.SidecarSetWorkingHotUpgradeContainer] = string(by)
+				obj.Annotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation("hot-init-1")] = "1"
+				obj.Annotations[sidecarcontrol.GetPodSidecarSetVersionAltAnnotation("hot-init-1")] = "0"
+				// "0" indicates sidecar container is hot upgrade empty container
+				obj.Annotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation("hot-init-2")] = "0"
+				obj.Annotations[sidecarcontrol.GetPodSidecarSetVersionAltAnnotation("hot-init-2")] = "1"
+				obj.Spec.Volumes = append(obj.Spec.Volumes, corev1.Volume{Name: "volume-1"})
+				return obj
+			},
+			expectedInitContainerLen: 4,
+		},
+	}
+
+	for _, cs := range cases {
+		t.Run(cs.name, func(t *testing.T) {
+			decoder := admission.NewDecoder(scheme.Scheme)
+			ss := cs.getSidecarSets()
+			client := fake.NewClientBuilder().WithObjects(ss[0], ss[1]).WithIndex(
+				&appsv1alpha1.SidecarSet{}, fieldindex.IndexNameForSidecarSetNamespace, fieldindex.IndexSidecarSet,
+			).Build()
+			pod := cs.getPod()
+			podHandler := &PodCreateHandler{Decoder: decoder, Client: client}
+			req := newAdmission(admissionv1.Create, runtime.RawExtension{}, runtime.RawExtension{}, "")
+			_, err := podHandler.sidecarsetMutatingPod(context.Background(), req, pod)
+			if err != nil {
+				t.Fatalf("inject sidecar into pod failed, err: %v", err)
+			}
+			if !reflect.DeepEqual(pod.Annotations, cs.expectedPod().Annotations) {
+				t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(cs.expectedPod()), util.DumpJSON(pod))
+			}
+			if len(pod.Spec.InitContainers) != cs.expectedInitContainerLen {
+				t.Fatalf("expect(%d), but get(%d)", cs.expectedInitContainerLen, len(pod.Spec.InitContainers))
+			}
+		})
+	}
+}
+
 func newAdmission(op admissionv1.Operation, object, oldObject runtime.RawExtension, subResource string) admission.Request {
 	return admission.Request{
 		AdmissionRequest: newAdmissionRequest(op, object, oldObject, subResource),

pkg/webhook/sidecarset/validating/sidecarset_create_update_handler.go

@@ -112,6 +112,12 @@ func validateSidecarSetName(name string, prefix bool) (allErrs []string) {
 func (h *SidecarSetCreateUpdateHandler) validateSidecarSetSpec(obj *appsv1alpha1.SidecarSet, fldPath *field.Path) field.ErrorList {
 	spec := &obj.Spec
 	allErrs := field.ErrorList{}
+	// currently when initContainer restartPolicy = Always, kruise don't support in-place update
+	for _, c := range obj.Spec.InitContainers {
+		if sidecarcontrol.IsSidecarContainer(c.Container) && obj.Spec.UpdateStrategy.Type == appsv1alpha1.RollingUpdateSidecarSetStrategyType {
+			allErrs = append(allErrs, field.Required(fldPath.Child("updateStrategy"), "The initContainer in-place upgrade is not currently supported."))
+		}
+	}
 
 	//validate spec selector
 	if spec.Selector == nil {

pkg/webhook/sidecarset/validating/sidecarset_validating_test.go

@@ -47,6 +47,7 @@ var (
 var (
 	testScheme *runtime.Scheme
 	handler    = &SidecarSetCreateUpdateHandler{}
+	always     = corev1.ContainerRestartPolicyAlways
 )
 
 func init() {
@@ -499,6 +500,39 @@ func TestValidateSidecarSet(t *testing.T) {
 			},
 			expectErrs: 1,
 		},
+		{
+			caseName: "The initContainer in-place upgrade is not currently supported.",
+			sidecarSet: appsv1alpha1.SidecarSet{
+				ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"},
+				Spec: appsv1alpha1.SidecarSetSpec{
+					Selector: &metav1.LabelSelector{
+						MatchLabels: map[string]string{"a": "b"},
+					},
+					UpdateStrategy: appsv1alpha1.SidecarSetUpdateStrategy{
+						Type: appsv1alpha1.RollingUpdateSidecarSetStrategyType,
+					},
+					InitContainers: []appsv1alpha1.SidecarContainer{
+						{
+							PodInjectPolicy: appsv1alpha1.BeforeAppContainerType,
+							ShareVolumePolicy: appsv1alpha1.ShareVolumePolicy{
+								Type: appsv1alpha1.ShareVolumePolicyDisabled,
+							},
+							UpgradeStrategy: appsv1alpha1.SidecarContainerUpgradeStrategy{
+								UpgradeType: appsv1alpha1.SidecarContainerColdUpgrade,
+							},
+							Container: corev1.Container{
+								Name:                     "test-sidecar",
+								Image:                    "test-image",
+								ImagePullPolicy:          corev1.PullIfNotPresent,
+								TerminationMessagePolicy: corev1.TerminationMessageReadFile,
+								RestartPolicy:            &always,
+							},
+						},
+					},
+				},
+			},
+			expectErrs: 1,
+		},
 	}
 
 	SidecarSetRevisions := []client.Object{