From a701385a832d6a120f6fd5028d4cb33fe8171f33 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Mon, 14 Oct 2024 20:03:10 +0800 Subject: [PATCH 1/6] remove-lifecycle-before-update --- .../collaset/collaset_controller_test.go | 112 +++- .../collaset/synccontrol/sync_control.go | 2 +- .../collaset/synccontrol/update.go | 3 +- .../operationjob/operationjob_manager.go | 2 +- .../operationjob/utils/lifecycle.go | 13 +- pkg/controllers/podopslifecycle/utils.go | 17 +- .../utils/podopslifecycle/utils.go | 23 + test/e2e/apps/collaset.go | 485 +++--------------- 8 files changed, 204 insertions(+), 453 deletions(-) diff --git a/pkg/controllers/collaset/collaset_controller_test.go b/pkg/controllers/collaset/collaset_controller_test.go index 44b492a4..b31174b3 100644 --- a/pkg/controllers/collaset/collaset_controller_test.go +++ b/pkg/controllers/collaset/collaset_controller_test.go @@ -614,6 +614,9 @@ var _ = Describe("collaset controller", func() { }, 5*time.Second, 1*time.Second).Should(BeTrue()) Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + for i := range podList.Items { + Expect(cleanPodLifecycleLabels(c, podList.Items[i].Namespace, podList.Items[i].Name)).Should(BeNil()) + } for _, number := range []int32{1, 2, 3, 4} { pod := podList.Items[number-1] // label pod to trigger update @@ -625,10 +628,24 @@ var _ = Describe("collaset controller", func() { return true })).Should(BeNil()) + Eventually(func() bool { + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, &pod)).Should(BeNil()) + if !podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &pod) { + return false + } + // allow Pod to do update + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) + return true + })).Should(BeNil()) + return true + }, 10*time.Second, 1*time.Second).Should(BeTrue()) + Eventually(func() error { // check updated pod replicas by CollaSet status return expectedStatusReplicas(c, cs, 0, 0, 0, 4, number, 1, 0, 0) - }, 5*time.Second, 1*time.Second).Should(BeNil()) + }, 10*time.Second, 1*time.Second).Should(BeNil()) Expect(updatePodStatusWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { pod.Status.ContainerStatuses = []corev1.ContainerStatus{ @@ -957,6 +974,16 @@ var _ = Describe("collaset controller", func() { return true })).Should(BeNil()) + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + for k := range podList.Items[0].Labels { + if strings.HasPrefix(k, appsv1alpha1.PodOperatingLabelPrefix) { + return true + } + } + return false + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + // allow Pod to update Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) @@ -1069,15 +1096,27 @@ var _ = Describe("collaset controller", func() { return true })) - for i := range podList.Items { - pod := &podList.Items[i] - // allow Pod to update - Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { - labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) - pod.Labels[labelOperate] = "true" - return true - })).Should(BeNil()) - } + Eventually(func() int { + count := 0 + for i := range podList.Items { + pod := &podList.Items[i] + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, pod)).Should(BeNil()) + for k := range pod.Labels { + if strings.HasPrefix(k, appsv1alpha1.PodOperatingLabelPrefix) { + count++ + // allow Pod to update + Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = "true" + return true + })).Should(BeNil()) + } + } + + } + return count + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(1)) + Eventually(func() error { // check updated pod replicas by CollaSet status return expectedStatusReplicas(c, cs, 0, 0, 0, 2, 1, 1, 0, 0) @@ -2082,8 +2121,7 @@ var _ = Describe("collaset controller", func() { Eventually(func() bool { pod := &corev1.Pod{} - error := c.Get(context.TODO(), types.NamespacedName{Namespace: originPod1.Namespace, Name: originPod1.Name}, pod) - Expect(error).Should(BeNil()) + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: originPod1.Namespace, Name: originPod1.Name}, pod)).Should(BeNil()) Expect(pod.Labels).ShouldNot(BeNil()) _, replaceIndicate := pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] _, replaceByUpdate := pod.Labels[appsv1alpha1.PodReplaceByReplaceUpdateLabelKey] @@ -2101,14 +2139,21 @@ var _ = Describe("collaset controller", func() { })).Should(BeNil()) // allow originPod2 to do inPlace update + Eventually(func() bool { + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: originPod2.Namespace, Name: originPod2.Name}, &originPod2)).Should(BeNil()) + for k := range originPod2.Labels { + if strings.HasPrefix(k, appsv1alpha1.PodOperatingLabelPrefix) { + return true + } + } + return false + }, time.Second*10, time.Second).Should(BeTrue()) Expect(updatePodWithRetry(c, originPod2.Namespace, originPod2.Name, func(pod *corev1.Pod) bool { labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return true })).Should(BeNil()) - time.Sleep(3 * time.Second) - Eventually(func() error { // check updated pod replicas by CollaSet status return expectedStatusReplicas(c, cs, 0, 0, 0, 3, 2, 2, 0, 0) @@ -3912,9 +3957,20 @@ var _ = Describe("collaset controller", func() { }, } + Expect(c.Create(context.TODO(), podDecoration)).Should(BeNil()) + // allow Pod to do update for i := range podList.Items { pod := podList.Items[i] + Eventually(func() bool { + Expect(c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, &pod)).Should(BeNil()) + for k := range pod.Labels { + if strings.HasPrefix(k, appsv1alpha1.PodOperatingLabelPrefix) { + return true + } + } + return false + }, time.Second*10, time.Second).Should(BeTrue()) Expect(updatePodWithRetry(c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) @@ -3922,7 +3978,6 @@ var _ = Describe("collaset controller", func() { })).Should(BeNil()) } - Expect(c.Create(context.TODO(), podDecoration)).Should(BeNil()) Eventually(func() int32 { err := c.Get(context.TODO(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration) if !errors.IsNotFound(err) { @@ -4165,6 +4220,33 @@ func updatePodStatusWithRetry(c client.Client, namespace, name string, updateFn }) } +func cleanPodLifecycleLabels(c client.Client, namespace, name string) error { + pod := &corev1.Pod{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, pod); err != nil { + return err + } + newLabels := make(map[string]string) + for k, v := range pod.Labels { + has := false + for _, prefix := range appsv1alpha1.WellKnownLabelPrefixesWithID { + if strings.HasPrefix(k, prefix) { + has = true + break + } + } + if !has { + newLabels[k] = v + } + } + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: name}, pod); err != nil { + return err + } + pod.Labels = newLabels + return c.Update(context.TODO(), pod) + }) +} + func testReconcile(inner reconcile.Reconciler) (reconcile.Reconciler, chan reconcile.Request) { requests := make(chan reconcile.Request, 5) fn := reconcile.Func(func(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { diff --git a/pkg/controllers/collaset/synccontrol/sync_control.go b/pkg/controllers/collaset/synccontrol/sync_control.go index a9d906e2..084773c2 100644 --- a/pkg/controllers/collaset/synccontrol/sync_control.go +++ b/pkg/controllers/collaset/synccontrol/sync_control.go @@ -762,7 +762,7 @@ func (r *RealSyncControl) Update( "UpdatePodCanceled", "pod %s/%s with revision %s update is canceled due to not started and not included by partition", podInfo.Namespace, podInfo.Name, podInfo.CurrentRevision.Name) - return ojutils.CancelOpsLifecycle(ctx, r.client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod) + return ojutils.CancelOpsLifecycle(r.client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod) } // not allowedOps, skip GetPodUpdateFinishStatus return nil diff --git a/pkg/controllers/collaset/synccontrol/update.go b/pkg/controllers/collaset/synccontrol/update.go index 80c48c77..154f922a 100644 --- a/pkg/controllers/collaset/synccontrol/update.go +++ b/pkg/controllers/collaset/synccontrol/update.go @@ -378,7 +378,8 @@ func (u *GenericPodUpdater) BeginUpdatePod(_ context.Context, resources *collase succCount, err := controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(int, error) error { podInfo := <-podCh u.Recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "PodUpdateLifecycle", "try to begin PodOpsLifecycle for updating Pod of CollaSet") - if updated, err := podopslifecycle.Begin(u.Client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod, func(obj client.Object) (bool, error) { + + if updated, err := podopslifecycle.BeginWithCleaningOld(u.Client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod, func(obj client.Object) (bool, error) { if !podInfo.OnlyMetadataChanged && !podInfo.InPlaceUpdateSupport { return podopslifecycle.WhenBeginDelete(obj) } diff --git a/pkg/controllers/operationjob/operationjob_manager.go b/pkg/controllers/operationjob/operationjob_manager.go index 87376948..721e5b62 100644 --- a/pkg/controllers/operationjob/operationjob_manager.go +++ b/pkg/controllers/operationjob/operationjob_manager.go @@ -323,7 +323,7 @@ func (r *ReconcileOperationJob) cleanCandidateOpsLifecycle(ctx context.Context, lifecycleAdapter := NewLifecycleAdapter(operationJob.Name, operationJob.Spec.Action) if forced { - err := ojutils.CancelOpsLifecycle(ctx, r.Client, lifecycleAdapter, candidate.Pod) + err := ojutils.CancelOpsLifecycle(r.Client, lifecycleAdapter, candidate.Pod) if err != nil { return err } diff --git a/pkg/controllers/operationjob/utils/lifecycle.go b/pkg/controllers/operationjob/utils/lifecycle.go index 03154dbc..09adf748 100644 --- a/pkg/controllers/operationjob/utils/lifecycle.go +++ b/pkg/controllers/operationjob/utils/lifecycle.go @@ -17,11 +17,9 @@ limitations under the License. package utils import ( - "context" "fmt" corev1 "k8s.io/api/core/v1" - appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" podopslifecycleutil "kusionstack.io/kuperator/pkg/controllers/podopslifecycle" @@ -66,22 +64,17 @@ func FinishOperateLifecycle(client client.Client, adapter podopslifecycle.Lifecy return nil } -func CancelOpsLifecycle(ctx context.Context, client client.Client, adapter podopslifecycle.LifecycleAdapter, pod *corev1.Pod) error { +func CancelOpsLifecycle(client client.Client, adapter podopslifecycle.LifecycleAdapter, pod *corev1.Pod) error { if pod == nil { return nil } // only cancel when lifecycle exist on pod if exist, err := podopslifecycleutil.IsLifecycleOnPod(adapter.GetID(), pod); err != nil { - return err + return fmt.Errorf("fail to check %s PodOpsLifecycle on Pod %s/%s: %s", adapter.GetID(), pod.Namespace, pod.Name, err) } else if !exist { return nil } - labelUndo := fmt.Sprintf("%s/%s", appsv1alpha1.PodUndoOperationTypeLabelPrefix, adapter.GetID()) - if pod.Labels == nil { - pod.Labels = make(map[string]string) - } - pod.Labels[labelUndo] = string(adapter.GetType()) - return client.Update(ctx, pod) + return podopslifecycle.Undo(client, adapter, pod) } diff --git a/pkg/controllers/podopslifecycle/utils.go b/pkg/controllers/podopslifecycle/utils.go index 635c2cee..bc063c0e 100644 --- a/pkg/controllers/podopslifecycle/utils.go +++ b/pkg/controllers/podopslifecycle/utils.go @@ -22,6 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kube-api/apps/v1alpha1" ) @@ -33,12 +34,15 @@ func IDToLabelsMap(pod *corev1.Pod) (map[string]map[string]string, map[string]in ids := sets.String{} for k := range pod.Labels { - if strings.HasPrefix(k, v1alpha1.PodOperatingLabelPrefix) || strings.HasPrefix(k, v1alpha1.PodOperateLabelPrefix) || strings.HasPrefix(k, v1alpha1.PodOperatedLabelPrefix) { - s := strings.Split(k, "/") - if len(s) < 2 { - return nil, nil, fmt.Errorf("invalid label %s", k) + for _, prefix := range v1alpha1.WellKnownLabelPrefixesWithID { + if strings.HasPrefix(k, prefix) { + s := strings.Split(k, "/") + if len(s) < 2 { + return nil, nil, fmt.Errorf("invalid label %s", k) + } + ids.Insert(s[1]) + break } - ids.Insert(s[1]) } } @@ -76,7 +80,8 @@ func IDToLabelsMap(pod *corev1.Pod) (map[string]map[string]string, map[string]in } // IsLifecycleOnPod returns true if the lifecycle with lifecycleId exist on pod, otherwise returns false -func IsLifecycleOnPod(lifecycleId string, pod *corev1.Pod) (bool, error) { +func IsLifecycleOnPod(lifecycleId string, obj client.Object) (bool, error) { + pod := obj.(*corev1.Pod) if pod == nil { return false, nil } diff --git a/pkg/controllers/utils/podopslifecycle/utils.go b/pkg/controllers/utils/podopslifecycle/utils.go index 440cc0c2..961b9d70 100644 --- a/pkg/controllers/utils/podopslifecycle/utils.go +++ b/pkg/controllers/utils/podopslifecycle/utils.go @@ -27,6 +27,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "kusionstack.io/kube-api/apps/v1alpha1" + + podopslifecycleutil "kusionstack.io/kuperator/pkg/controllers/podopslifecycle" ) // IsDuringOps decides whether the Pod is during ops or not @@ -85,6 +87,16 @@ func Begin(c client.Client, adapter LifecycleAdapter, obj client.Object, updateF return false, nil } +// BeginWithCleaningOld is used for an CRD Operator to begin a lifecycle with cleaning the old lifecycle +func BeginWithCleaningOld(c client.Client, adapter LifecycleAdapter, obj client.Object, updateFunc ...UpdateFunc) (updated bool, err error) { + if podInUpdateLifecycle, err := podopslifecycleutil.IsLifecycleOnPod(adapter.GetID(), obj); err != nil { + return false, fmt.Errorf("fail to check %s PodOpsLifecycle on Pod %s/%s: %s", adapter.GetID(), obj.GetNamespace(), obj.GetName(), err) + } else if podInUpdateLifecycle { + return false, Undo(c, adapter, obj) + } + return Begin(c, adapter, obj, updateFunc...) +} + // AllowOps is used to check whether the PodOpsLifecycle phase is in UPGRADE to do following operations. func AllowOps(adapter LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter *time.Duration, allow bool) { if !IsDuringOps(adapter, obj) { @@ -139,6 +151,12 @@ func Finish(c client.Client, adapter LifecycleAdapter, obj client.Object, update return false, err } +// Undo is used for an CRD Operator to undo a lifecycle +func Undo(c client.Client, adapter LifecycleAdapter, obj client.Object) error { + setUndo(adapter, obj) + return c.Update(context.Background(), obj) +} + func checkOperatingID(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { labelID := fmt.Sprintf("%s/%s", v1alpha1.PodOperatingLabelPrefix, adapter.GetID()) _, ok = obj.GetLabels()[labelID] @@ -177,6 +195,11 @@ func setOperate(adapter LifecycleAdapter, obj client.Object) (val string, ok boo return } +func setUndo(adapter LifecycleAdapter, obj client.Object) { + labelUndo := fmt.Sprintf("%s/%s", v1alpha1.PodUndoOperationTypeLabelPrefix, adapter.GetID()) + obj.GetLabels()[labelUndo] = string(adapter.GetType()) +} + func deleteOperatingID(adapter LifecycleAdapter, obj client.Object) (val string, ok bool) { labelID := fmt.Sprintf("%s/%s", v1alpha1.PodOperatingLabelPrefix, adapter.GetID()) delete(obj.GetLabels(), labelID) diff --git a/test/e2e/apps/collaset.go b/test/e2e/apps/collaset.go index 691afe97..d62f77f1 100644 --- a/test/e2e/apps/collaset.go +++ b/test/e2e/apps/collaset.go @@ -19,7 +19,6 @@ package apps import ( "context" "fmt" - "strconv" "strings" "time" @@ -30,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" @@ -40,6 +38,7 @@ import ( collasetutils "kusionstack.io/kuperator/pkg/controllers/collaset/utils" "kusionstack.io/kuperator/pkg/controllers/utils/podopslifecycle" + "kusionstack.io/kuperator/pkg/utils" "kusionstack.io/kuperator/test/e2e/framework" ) @@ -1260,7 +1259,7 @@ var _ = SIGDescribe("CollaSet", func() { } } return duringOpsCount - }) + }, 10*time.Second, 3*time.Second).Should(Equal(1)) By("Remove finalizers from pods") pods, err = tester.ListPodsForCollaSet(cls) @@ -1274,462 +1273,110 @@ var _ = SIGDescribe("CollaSet", func() { By("Wait for update finish") Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 2, 2, 1, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) }) - }) - framework.KusionstackDescribe("CollaSet Replacing", func() { - - framework.ConformanceIt("replace pod by label", func() { - cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{}) - cls.Spec.ScaleStrategy.PersistentVolumeClaimRetentionPolicy = &appsv1alpha1.PersistentVolumeClaimRetentionPolicy{ - WhenScaled: appsv1alpha1.RetainPersistentVolumeClaimRetentionPolicyType, - } - cls.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-test", - }, - Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - "storage": resource.MustParse("100m"), - }, - }, - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, - }, - }, - } - cls.Spec.Template.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{ - { - MountPath: "/path/to/mount", - Name: "pvc-test", - }, - } + framework.ConformanceIt("update with cleaning old lifecycle", func() { + cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{PodUpdatePolicy: appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType}) Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) By("Wait for status replicas satisfied") Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 1, 1, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Replace pod by label") pods, err := tester.ListPodsForCollaSet(cls) Expect(err).NotTo(HaveOccurred()) - podToReplace := pods[0] - Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { - podToReplace.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" - })).NotTo(HaveOccurred()) - By("Wait for replace finished") - Eventually(func() bool { - pods, err = tester.ListPodsForCollaSet(cls) - if err != nil { - return false - } - for i := range pods { - if pods[i].Labels[appsv1alpha1.PodReplaceIndicationLabelKey] != "" || - pods[i].Labels[appsv1alpha1.PodReplacePairOriginName] != "" { - return false - } - } - return len(pods) == 1 - }, 30*time.Second, 3*time.Second).Should(BeTrue()) - - By("Check replace new pod") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 1, 1, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - podNewCreate := pods[0] - Expect(podNewCreate.Name).ShouldNot(Equal(podToReplace.Name)) - - By("Check pvc and resourceContext") - newPvcs, err := tester.ListPVCForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - currResourceContexts, err := tester.ListResourceContextsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - Expect(len(newPvcs)).To(Equal(1)) - Expect(newPvcs[0].Labels[appsv1alpha1.PodInstanceIDLabelKey]).To(Equal(podNewCreate.Labels[appsv1alpha1.PodInstanceIDLabelKey])) - Expect(len(currResourceContexts[0].Spec.Contexts)).To(Equal(1)) - Expect(strconv.Itoa(currResourceContexts[0].Spec.Contexts[0].ID)).To(Equal(podNewCreate.Labels[appsv1alpha1.PodInstanceIDLabelKey])) - }) - - framework.ConformanceIt("cancel replace pod by label", func() { - cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{}) - cls.Spec.ScaleStrategy.PersistentVolumeClaimRetentionPolicy = &appsv1alpha1.PersistentVolumeClaimRetentionPolicy{ - WhenScaled: appsv1alpha1.RetainPersistentVolumeClaimRetentionPolicyType, - } - cls.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "pvc-test", - }, - Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - "storage": resource.MustParse("100m"), - }, - }, - AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + By("Mock traffic finalizer on pods") + Expect(tester.UpdatePod(pods[0], func(pod *v1.Pod) { + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + finalizer := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperationProtectionFinalizerPrefix, "test") + pod.Finalizers = []string{finalizer} + pod.Annotations[appsv1alpha1.PodAvailableConditionsAnnotation] = utils.DumpJSON(&appsv1alpha1.PodAvailableConditions{ + ExpectedFinalizers: map[string]string{ + "finalizer/test": finalizer, }, - }, - } - cls.Spec.Template.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{ - { - MountPath: "/path/to/mount", - Name: "pvc-test", - }, - } - // use bad image to mock new replace pod unavailable - cls.Spec.Template.Spec.Containers[0].Image = "nginx:non-exist" - Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) - - By("Wait for status replicas satisfied") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - resourceContexts, err := tester.ListResourceContextsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) + }) + })).NotTo(HaveOccurred()) - By("Replace pod by label") - pods, err := tester.ListPodsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - podToReplace := pods[0] - Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { - podToReplace.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" + By("Update CollaSet from v1 to v2") + Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { + cls.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.Redis) })).NotTo(HaveOccurred()) - By("Wait for replace new pod created") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + By("Wait for CollaSet reconciled") Eventually(func() bool { - pvcs, err := tester.ListPVCForCollaSet(cls) - if err != nil { - return false - } - if len(pvcs) != 2 { + if err = tester.GetCollaSet(cls); err != nil { return false } - return true - }, 30*time.Second, 3*time.Second).Should(Equal(true)) + return cls.Generation == cls.Status.ObservedGeneration + }, 10*time.Second, 3*time.Second).Should(Equal(true)) + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 0, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - By("Cancel replace pod by delete to-replace label") - Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { - delete(pod.Labels, appsv1alpha1.PodReplaceIndicationLabelKey) + By("Remove finalizers from pods") + Expect(tester.UpdatePod(pods[0], func(pod *v1.Pod) { + pod.Finalizers = []string{} })).NotTo(HaveOccurred()) - By("Wait for replace cancel finished") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Check replace origin pod") - pods, err = tester.ListPodsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - Expect(pods[0].Name).To(Equal(podToReplace.Name)) - - By("Check pvc and resourceContext") + By("Wait for pod operated") Eventually(func() bool { - pvcs, err := tester.ListPVCForCollaSet(cls) - if err != nil { - return false - } - if len(pvcs) != 1 { - return false - } - return pvcs[0].Labels[appsv1alpha1.PodInstanceIDLabelKey] == podToReplace.Labels[appsv1alpha1.PodInstanceIDLabelKey] - }, 30*time.Second, 3*time.Second).Should(Equal(true)) - Expect(err).NotTo(HaveOccurred()) - var currResourceContexts []*appsv1alpha1.ResourceContext - Eventually(func() bool { - currResourceContexts, err = tester.ListResourceContextsForCollaSet(cls) - return len(currResourceContexts[0].Spec.Contexts) == 1 - }, 30*time.Second, 3*time.Second).Should(BeTrue()) - Expect(len(currResourceContexts[0].Spec.Contexts)).To(Equal(len(resourceContexts[0].Spec.Contexts))) - Expect(currResourceContexts[0].Spec.Contexts[0].ID).To(Equal(resourceContexts[0].Spec.Contexts[0].ID)) - for k := range resourceContexts[0].Spec.Contexts[0].Data { - Expect(currResourceContexts[0].Spec.Contexts[0].Data[k]).To(Equal(resourceContexts[0].Spec.Contexts[0].Data[k])) - } - for k := range currResourceContexts[0].Spec.Contexts[0].Data { - Expect(currResourceContexts[0].Spec.Contexts[0].Data[k]).To(Equal(resourceContexts[0].Spec.Contexts[0].Data[k])) - } - }) - - framework.ConformanceIt("replace pod with update", func() { - for _, updateStrategy := range []appsv1alpha1.PodUpdateStrategyType{appsv1alpha1.CollaSetRecreatePodUpdateStrategyType, appsv1alpha1.CollaSetReplacePodUpdateStrategyType, appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType} { - By(fmt.Sprintf("Test replace pod with %s update", updateStrategy)) - cls := tester.NewCollaSet(fmt.Sprintf("collaset-%s-%s", randStr, strings.ToLower(string(updateStrategy))), 1, appsv1alpha1.UpdateStrategy{PodUpdatePolicy: appsv1alpha1.CollaSetRecreatePodUpdateStrategyType}) - // use bad image to mock new replace pod unavailable - cls.Spec.Template.Spec.Containers[0].Image = "nginx:non-exist" - Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) - - By("Wait for status replicas satisfied") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Replace pod by label") - pods, err := tester.ListPodsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - podToReplace := pods[0] - Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { - podToReplace.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" - })).NotTo(HaveOccurred()) - - By("Wait for replace new pod created") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) pods, err = tester.ListPodsForCollaSet(cls) Expect(err).NotTo(HaveOccurred()) - var replaceNewPod *v1.Pod - for _, pod := range pods { - if pod.Name != podToReplace.Name { - replaceNewPod = pod - } - } - Expect(replaceNewPod).ShouldNot(BeNil()) - - By("Update collaset image") - Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { - cls.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) - })).NotTo(HaveOccurred()) - - By("Wait for replace and update finished") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 1, 1, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Check replace new pod is deleted") - Eventually(func() bool { - pods, err = tester.ListPodsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - for _, pod := range pods { - if pod.Name == replaceNewPod.Name { - return false - } - } - return len(pods) == 1 && pods[0].Spec.Containers[0].Image == imageutils.GetE2EImage(imageutils.NginxNew) - }, 30*time.Second, 3*time.Second) - - By("Check resource contexts") - Eventually(func() bool { - resourceContexts, err := tester.ListResourceContextsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - if len(resourceContexts[0].Spec.Contexts) != 1 { - return false - } - pods, err = tester.ListPodsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - return strconv.Itoa(resourceContexts[0].Spec.Contexts[0].ID) == pods[0].Labels[appsv1alpha1.PodInstanceIDLabelKey] - }, 30*time.Second, 3*time.Second).Should(BeTrue()) - } - }) - - framework.ConformanceIt("separate pd and collaset update", func() { - for _, updateStrategy := range []appsv1alpha1.PodUpdateStrategyType{appsv1alpha1.CollaSetRecreatePodUpdateStrategyType, appsv1alpha1.CollaSetReplacePodUpdateStrategyType, appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType} { - By(fmt.Sprintf("Test separate pd and collaset update %s strategy", updateStrategy)) - cls := tester.NewCollaSet(fmt.Sprintf("collaset-%s-%s", randStr, strings.ToLower(string(updateStrategy))), 3, appsv1alpha1.UpdateStrategy{PodUpdatePolicy: appsv1alpha1.CollaSetRecreatePodUpdateStrategyType}) - - By("Create CollaSet") - Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) - - By("Wait for CollaSet status replicas satisfied") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 3, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Create PodDecoration") - podDecoration := &appsv1alpha1.PodDecoration{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: cls.Namespace, - Name: fmt.Sprintf("pd-%s-%s", randStr, strings.ToLower(string(updateStrategy))), - }, - Spec: appsv1alpha1.PodDecorationSpec{ - HistoryLimit: 5, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "owner": cls.Name, - }, - }, - Weight: int32Pointer(1), - UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ - RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ - Partition: int32Pointer(0), - }, - }, - Template: appsv1alpha1.PodDecorationPodTemplate{ - Containers: []*appsv1alpha1.ContainerPatch{ - { - InjectPolicy: appsv1alpha1.AfterPrimaryContainer, - Container: v1.Container{ - Name: "sidecar", - Image: imageutils.GetE2EImage(imageutils.Nginx), - Command: []string{ - "sleep", - "2h", - }, - }, - }, - }, - }, - }, - } - Expect(client.Create(context.Background(), podDecoration)).Should(BeNil()) - - By("Wait for PodDecoration status replicas satisfied") - Eventually(func() int32 { - Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) - return podDecoration.Status.UpdatedPods - }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(3)) - - By("Update CollaSet 1 pod") - Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { - cls.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) - cls.Spec.UpdateStrategy.RollingUpdate = &appsv1alpha1.RollingUpdateCollaSetStrategy{ - ByPartition: &appsv1alpha1.ByPartition{ - Partition: int32Pointer(2), - }, - } - })).NotTo(HaveOccurred()) - - By("Check CollaSet and PodDecoration upgrade progress") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 1, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - Eventually(func() int32 { - Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) - return podDecoration.Status.UpdatedPods - }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(3)) - - By("Update PodDecoration 2 pod") - Eventually(func() error { - podDecoration.Spec.Template.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) - podDecoration.Spec.UpdateStrategy.RollingUpdate.Partition = int32Pointer(1) - return client.Update(context.Background(), podDecoration) - }, 30*time.Second, 3*time.Second).Should(BeNil()) - - By("Check CollaSet and PodDecoration upgrade progress") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 1, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - Eventually(func() int32 { - Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) - return podDecoration.Status.UpdatedPods - }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(2)) - - By("Update CollaSet all pods") - Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { - cls.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) - cls.Spec.UpdateStrategy.RollingUpdate = &appsv1alpha1.RollingUpdateCollaSetStrategy{ - ByPartition: &appsv1alpha1.ByPartition{ - Partition: int32Pointer(0), - }, - } - })).NotTo(HaveOccurred()) - - By("Check CollaSet and PodDecoration upgrade progress") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 3, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - Eventually(func() int32 { - Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) - return podDecoration.Status.UpdatedPods - }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(2)) - - By("Update PodDecoration all pods") - Eventually(func() error { - podDecoration.Spec.Template.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) - podDecoration.Spec.UpdateStrategy.RollingUpdate.Partition = int32Pointer(0) - return client.Update(context.Background(), podDecoration) - }, 30*time.Second, 3*time.Second).Should(BeNil()) - - By("Check CollaSet and PodDecoration upgrade progress") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 3, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - Eventually(func() int32 { - Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) - return podDecoration.Status.UpdatedPods - }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(3)) - } - }) - - framework.ConformanceIt("scaleIn origin pod before new pod service available", func() { - cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{}) - // use bad image to mock new replace pod unavailable - cls.Spec.Template.Spec.Containers[0].Image = "nginx:non-exist" - Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) - - By("Wait for status replicas satisfied") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Replace pod by label") - pods, err := tester.ListPodsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - podToReplace := pods[0] - Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { - pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" - })).NotTo(HaveOccurred()) - - By("Wait for replace new pod created") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Selective scaleIn origin pod") - Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { - cls.Spec.Replicas = int32Pointer(0) - cls.Spec.ScaleStrategy = appsv1alpha1.ScaleStrategy{ - PodToDelete: []string{podToReplace.Name}, - } - })).NotTo(HaveOccurred()) - - By("Wait for pods deleted") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 0, 0, 0, 0, 0) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Check resourceContext") - var currResourceContexts []*appsv1alpha1.ResourceContext - Eventually(func() bool { - currResourceContexts, err = tester.ListResourceContextsForCollaSet(cls) - Expect(err).Should(BeNil()) - return len(currResourceContexts) == 0 - }, 30*time.Second, 3*time.Second).Should(BeTrue()) - }) - - framework.ConformanceIt("scaleIn new pod", func() { - cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{}) - // use bad image to mock new replace pod unavailable - cls.Spec.Template.Spec.Containers[0].Image = "nginx:non-exist" - Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) - - By("Wait for status replicas satisfied") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - - By("Replace pod by label") - pods, err := tester.ListPodsForCollaSet(cls) - Expect(err).NotTo(HaveOccurred()) - podToReplace := pods[0] - Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { - pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" - })).NotTo(HaveOccurred()) - - By("Wait for replace new pod created") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + _, exist := pods[0].Labels[fmt.Sprintf("%s/%s", appsv1alpha1.PodOperatedLabelPrefix, "collaset")] + return exist + }, 10*time.Second, 3*time.Second).Should(Equal(true)) - By("Selective scaleIn new pod") + By("Record lifecycle label values") pods, err = tester.ListPodsForCollaSet(cls) Expect(err).NotTo(HaveOccurred()) - var newPod *v1.Pod - for _, pod := range pods { - if pod.Name != podToReplace.Name { - newPod = pod + lifecycleLabelValues := make(map[string]string) + for k := range pods[0].Labels { + if strings.HasPrefix(k, appsv1alpha1.PodOperatedLabelPrefix) { + lifecycleLabelValues[k] = pods[0].Labels[k] } } + + By("Update CollaSet from v2 to v3") Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { - cls.Spec.Replicas = int32Pointer(0) - cls.Spec.ScaleStrategy = appsv1alpha1.ScaleStrategy{ - PodToDelete: []string{newPod.Name}, - } + cls.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) })).NotTo(HaveOccurred()) By("Wait for CollaSet reconciled") Eventually(func() bool { - if err := tester.GetCollaSet(cls); err != nil { + if err = tester.GetCollaSet(cls); err != nil { return false } return cls.Generation == cls.Status.ObservedGeneration }, 10*time.Second, 3*time.Second).Should(Equal(true)) - By("New pod will not be deleted") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + By("Wait for pod operated") + Eventually(func() bool { + pods, err = tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + _, exist := pods[0].Labels[fmt.Sprintf("%s/%s", appsv1alpha1.PodOperatedLabelPrefix, "collaset")] + return exist + }, 10*time.Second, 3*time.Second).Should(Equal(true)) + + By("Check old lifecycle are cleaned and new lifecycle created") + pods, err = tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + for k := range pods[0].Labels { + if strings.HasPrefix(k, appsv1alpha1.PodOperatingLabelPrefix) { + Expect(lifecycleLabelValues[k] == pods[0].Labels[k]).ShouldNot(BeTrue()) + } + } - By("Mock new pod service available") - Expect(tester.UpdatePod(newPod, func(pod *v1.Pod) { - newPod.Labels[appsv1alpha1.PodServiceAvailableLabel] = "true" + By("Add finalizers to pods") + Expect(tester.UpdatePod(pods[0], func(pod *v1.Pod) { + finalizer := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperationProtectionFinalizerPrefix, "test") + pod.Finalizers = []string{finalizer} })).NotTo(HaveOccurred()) - By("Wait for pods are deleted") - Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 0, 0, 0, 0, 0) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + By("Wait for update finish") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 1, 1, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) - By("Check resourceContext") - var currResourceContexts []*appsv1alpha1.ResourceContext - Eventually(func() bool { - currResourceContexts, err = tester.ListResourceContextsForCollaSet(cls) - Expect(err).Should(BeNil()) - return len(currResourceContexts) == 0 - }, 30*time.Second, 3*time.Second).Should(BeTrue()) + By("Remove finalizers from pods") + Expect(tester.UpdatePod(pods[0], func(pod *v1.Pod) { + pod.Finalizers = []string{} + })).NotTo(HaveOccurred()) }) }) }) From 3cb23b26afa9d128279a762f3ef521291c21c4b0 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Mon, 11 Nov 2024 14:24:50 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20e2e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/e2e/apps/collaset.go | 459 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) diff --git a/test/e2e/apps/collaset.go b/test/e2e/apps/collaset.go index d62f77f1..820bc899 100644 --- a/test/e2e/apps/collaset.go +++ b/test/e2e/apps/collaset.go @@ -19,6 +19,7 @@ package apps import ( "context" "fmt" + "strconv" "strings" "time" @@ -29,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" clientset "k8s.io/client-go/kubernetes" @@ -1379,6 +1381,463 @@ var _ = SIGDescribe("CollaSet", func() { })).NotTo(HaveOccurred()) }) }) + + framework.KusionstackDescribe("CollaSet Replacing", func() { + + framework.ConformanceIt("replace pod by label", func() { + cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{}) + cls.Spec.ScaleStrategy.PersistentVolumeClaimRetentionPolicy = &appsv1alpha1.PersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1alpha1.RetainPersistentVolumeClaimRetentionPolicyType, + } + cls.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-test", + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "storage": resource.MustParse("100m"), + }, + }, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + }, + }, + } + cls.Spec.Template.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{ + { + MountPath: "/path/to/mount", + Name: "pvc-test", + }, + } + Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) + + By("Wait for status replicas satisfied") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 1, 1, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Replace pod by label") + pods, err := tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + podToReplace := pods[0] + Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { + podToReplace.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" + })).NotTo(HaveOccurred()) + + By("Wait for replace finished") + Eventually(func() bool { + pods, err = tester.ListPodsForCollaSet(cls) + if err != nil { + return false + } + for i := range pods { + if pods[i].Labels[appsv1alpha1.PodReplaceIndicationLabelKey] != "" || + pods[i].Labels[appsv1alpha1.PodReplacePairOriginName] != "" { + return false + } + } + return len(pods) == 1 + }, 30*time.Second, 3*time.Second).Should(BeTrue()) + + By("Check replace new pod") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 1, 1, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + podNewCreate := pods[0] + Expect(podNewCreate.Name).ShouldNot(Equal(podToReplace.Name)) + + By("Check pvc and resourceContext") + newPvcs, err := tester.ListPVCForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + currResourceContexts, err := tester.ListResourceContextsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + Expect(len(newPvcs)).To(Equal(1)) + Expect(newPvcs[0].Labels[appsv1alpha1.PodInstanceIDLabelKey]).To(Equal(podNewCreate.Labels[appsv1alpha1.PodInstanceIDLabelKey])) + Expect(len(currResourceContexts[0].Spec.Contexts)).To(Equal(1)) + Expect(strconv.Itoa(currResourceContexts[0].Spec.Contexts[0].ID)).To(Equal(podNewCreate.Labels[appsv1alpha1.PodInstanceIDLabelKey])) + }) + + framework.ConformanceIt("cancel replace pod by label", func() { + cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{}) + cls.Spec.ScaleStrategy.PersistentVolumeClaimRetentionPolicy = &appsv1alpha1.PersistentVolumeClaimRetentionPolicy{ + WhenScaled: appsv1alpha1.RetainPersistentVolumeClaimRetentionPolicyType, + } + cls.Spec.VolumeClaimTemplates = []v1.PersistentVolumeClaim{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-test", + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "storage": resource.MustParse("100m"), + }, + }, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + }, + }, + } + cls.Spec.Template.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{ + { + MountPath: "/path/to/mount", + Name: "pvc-test", + }, + } + // use bad image to mock new replace pod unavailable + cls.Spec.Template.Spec.Containers[0].Image = "nginx:non-exist" + Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) + + By("Wait for status replicas satisfied") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + resourceContexts, err := tester.ListResourceContextsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + + By("Replace pod by label") + pods, err := tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + podToReplace := pods[0] + Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { + podToReplace.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" + })).NotTo(HaveOccurred()) + + By("Wait for replace new pod created") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + Eventually(func() bool { + pvcs, err := tester.ListPVCForCollaSet(cls) + if err != nil { + return false + } + if len(pvcs) != 2 { + return false + } + return true + }, 30*time.Second, 3*time.Second).Should(Equal(true)) + + By("Cancel replace pod by delete to-replace label") + Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { + delete(pod.Labels, appsv1alpha1.PodReplaceIndicationLabelKey) + })).NotTo(HaveOccurred()) + + By("Wait for replace cancel finished") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Check replace origin pod") + pods, err = tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + Expect(pods[0].Name).To(Equal(podToReplace.Name)) + + By("Check pvc and resourceContext") + Eventually(func() bool { + pvcs, err := tester.ListPVCForCollaSet(cls) + if err != nil { + return false + } + if len(pvcs) != 1 { + return false + } + return pvcs[0].Labels[appsv1alpha1.PodInstanceIDLabelKey] == podToReplace.Labels[appsv1alpha1.PodInstanceIDLabelKey] + }, 30*time.Second, 3*time.Second).Should(Equal(true)) + Expect(err).NotTo(HaveOccurred()) + var currResourceContexts []*appsv1alpha1.ResourceContext + Eventually(func() bool { + currResourceContexts, err = tester.ListResourceContextsForCollaSet(cls) + return len(currResourceContexts[0].Spec.Contexts) == 1 + }, 30*time.Second, 3*time.Second).Should(BeTrue()) + Expect(len(currResourceContexts[0].Spec.Contexts)).To(Equal(len(resourceContexts[0].Spec.Contexts))) + Expect(currResourceContexts[0].Spec.Contexts[0].ID).To(Equal(resourceContexts[0].Spec.Contexts[0].ID)) + for k := range resourceContexts[0].Spec.Contexts[0].Data { + Expect(currResourceContexts[0].Spec.Contexts[0].Data[k]).To(Equal(resourceContexts[0].Spec.Contexts[0].Data[k])) + } + for k := range currResourceContexts[0].Spec.Contexts[0].Data { + Expect(currResourceContexts[0].Spec.Contexts[0].Data[k]).To(Equal(resourceContexts[0].Spec.Contexts[0].Data[k])) + } + }) + + framework.ConformanceIt("replace pod with update", func() { + for _, updateStrategy := range []appsv1alpha1.PodUpdateStrategyType{appsv1alpha1.CollaSetRecreatePodUpdateStrategyType, appsv1alpha1.CollaSetReplacePodUpdateStrategyType, appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType} { + By(fmt.Sprintf("Test replace pod with %s update", updateStrategy)) + cls := tester.NewCollaSet(fmt.Sprintf("collaset-%s-%s", randStr, strings.ToLower(string(updateStrategy))), 1, appsv1alpha1.UpdateStrategy{PodUpdatePolicy: appsv1alpha1.CollaSetRecreatePodUpdateStrategyType}) + // use bad image to mock new replace pod unavailable + cls.Spec.Template.Spec.Containers[0].Image = "nginx:non-exist" + Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) + + By("Wait for status replicas satisfied") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Replace pod by label") + pods, err := tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + podToReplace := pods[0] + Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { + podToReplace.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" + })).NotTo(HaveOccurred()) + + By("Wait for replace new pod created") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + pods, err = tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + var replaceNewPod *v1.Pod + for _, pod := range pods { + if pod.Name != podToReplace.Name { + replaceNewPod = pod + } + } + Expect(replaceNewPod).ShouldNot(BeNil()) + + By("Update collaset image") + Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { + cls.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) + })).NotTo(HaveOccurred()) + + By("Wait for replace and update finished") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 1, 1, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Check replace new pod is deleted") + Eventually(func() bool { + pods, err = tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + for _, pod := range pods { + if pod.Name == replaceNewPod.Name { + return false + } + } + return len(pods) == 1 && pods[0].Spec.Containers[0].Image == imageutils.GetE2EImage(imageutils.NginxNew) + }, 30*time.Second, 3*time.Second) + + By("Check resource contexts") + Eventually(func() bool { + resourceContexts, err := tester.ListResourceContextsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + if len(resourceContexts[0].Spec.Contexts) != 1 { + return false + } + pods, err = tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + return strconv.Itoa(resourceContexts[0].Spec.Contexts[0].ID) == pods[0].Labels[appsv1alpha1.PodInstanceIDLabelKey] + }, 30*time.Second, 3*time.Second).Should(BeTrue()) + } + }) + + framework.ConformanceIt("separate pd and collaset update", func() { + for _, updateStrategy := range []appsv1alpha1.PodUpdateStrategyType{appsv1alpha1.CollaSetRecreatePodUpdateStrategyType, appsv1alpha1.CollaSetReplacePodUpdateStrategyType, appsv1alpha1.CollaSetInPlaceIfPossiblePodUpdateStrategyType} { + By(fmt.Sprintf("Test separate pd and collaset update %s strategy", updateStrategy)) + cls := tester.NewCollaSet(fmt.Sprintf("collaset-%s-%s", randStr, strings.ToLower(string(updateStrategy))), 3, appsv1alpha1.UpdateStrategy{PodUpdatePolicy: appsv1alpha1.CollaSetRecreatePodUpdateStrategyType}) + + By("Create CollaSet") + Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) + + By("Wait for CollaSet status replicas satisfied") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 3, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Create PodDecoration") + podDecoration := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cls.Namespace, + Name: fmt.Sprintf("pd-%s-%s", randStr, strings.ToLower(string(updateStrategy))), + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "owner": cls.Name, + }, + }, + Weight: int32Pointer(1), + UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ + RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ + Partition: int32Pointer(0), + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: v1.Container{ + Name: "sidecar", + Image: imageutils.GetE2EImage(imageutils.Nginx), + Command: []string{ + "sleep", + "2h", + }, + }, + }, + }, + }, + }, + } + Expect(client.Create(context.Background(), podDecoration)).Should(BeNil()) + + By("Wait for PodDecoration status replicas satisfied") + Eventually(func() int32 { + Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) + return podDecoration.Status.UpdatedPods + }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(3)) + + By("Update CollaSet 1 pod") + Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { + cls.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) + cls.Spec.UpdateStrategy.RollingUpdate = &appsv1alpha1.RollingUpdateCollaSetStrategy{ + ByPartition: &appsv1alpha1.ByPartition{ + Partition: int32Pointer(2), + }, + } + })).NotTo(HaveOccurred()) + + By("Check CollaSet and PodDecoration upgrade progress") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 1, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + Eventually(func() int32 { + Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) + return podDecoration.Status.UpdatedPods + }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(3)) + + By("Update PodDecoration 2 pod") + Eventually(func() error { + podDecoration.Spec.Template.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) + podDecoration.Spec.UpdateStrategy.RollingUpdate.Partition = int32Pointer(1) + return client.Update(context.Background(), podDecoration) + }, 30*time.Second, 3*time.Second).Should(BeNil()) + + By("Check CollaSet and PodDecoration upgrade progress") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 1, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + Eventually(func() int32 { + Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) + return podDecoration.Status.UpdatedPods + }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(2)) + + By("Update CollaSet all pods") + Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { + cls.Spec.Template.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) + cls.Spec.UpdateStrategy.RollingUpdate = &appsv1alpha1.RollingUpdateCollaSetStrategy{ + ByPartition: &appsv1alpha1.ByPartition{ + Partition: int32Pointer(0), + }, + } + })).NotTo(HaveOccurred()) + + By("Check CollaSet and PodDecoration upgrade progress") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 3, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + Eventually(func() int32 { + Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) + return podDecoration.Status.UpdatedPods + }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(2)) + + By("Update PodDecoration all pods") + Eventually(func() error { + podDecoration.Spec.Template.Containers[0].Image = imageutils.GetE2EImage(imageutils.NginxNew) + podDecoration.Spec.UpdateStrategy.RollingUpdate.Partition = int32Pointer(0) + return client.Update(context.Background(), podDecoration) + }, 30*time.Second, 3*time.Second).Should(BeNil()) + + By("Check CollaSet and PodDecoration upgrade progress") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 3, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + Eventually(func() int32 { + Expect(client.Get(context.Background(), types.NamespacedName{Namespace: podDecoration.Namespace, Name: podDecoration.Name}, podDecoration)).Should(BeNil()) + return podDecoration.Status.UpdatedPods + }, 30*time.Second, 3*time.Second).Should(BeEquivalentTo(3)) + } + }) + + framework.ConformanceIt("scaleIn origin pod before new pod service available", func() { + cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{}) + // use bad image to mock new replace pod unavailable + cls.Spec.Template.Spec.Containers[0].Image = "nginx:non-exist" + Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) + + By("Wait for status replicas satisfied") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Replace pod by label") + pods, err := tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + podToReplace := pods[0] + Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { + pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" + })).NotTo(HaveOccurred()) + + By("Wait for replace new pod created") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Selective scaleIn origin pod") + Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { + cls.Spec.Replicas = int32Pointer(0) + cls.Spec.ScaleStrategy = appsv1alpha1.ScaleStrategy{ + PodToDelete: []string{podToReplace.Name}, + } + })).NotTo(HaveOccurred()) + + By("Wait for pods deleted") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 0, 0, 0, 0, 0) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Check resourceContext") + var currResourceContexts []*appsv1alpha1.ResourceContext + Eventually(func() bool { + currResourceContexts, err = tester.ListResourceContextsForCollaSet(cls) + Expect(err).Should(BeNil()) + return len(currResourceContexts) == 0 + }, 30*time.Second, 3*time.Second).Should(BeTrue()) + }) + + framework.ConformanceIt("scaleIn new pod", func() { + cls := tester.NewCollaSet("collaset-"+randStr, 1, appsv1alpha1.UpdateStrategy{}) + // use bad image to mock new replace pod unavailable + cls.Spec.Template.Spec.Containers[0].Image = "nginx:non-exist" + Expect(tester.CreateCollaSet(cls)).NotTo(HaveOccurred()) + + By("Wait for status replicas satisfied") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 1, 0, 0, 1, 1) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Replace pod by label") + pods, err := tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + podToReplace := pods[0] + Expect(tester.UpdatePod(podToReplace, func(pod *v1.Pod) { + pod.Labels[appsv1alpha1.PodReplaceIndicationLabelKey] = "true" + })).NotTo(HaveOccurred()) + + By("Wait for replace new pod created") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Selective scaleIn new pod") + pods, err = tester.ListPodsForCollaSet(cls) + Expect(err).NotTo(HaveOccurred()) + var newPod *v1.Pod + for _, pod := range pods { + if pod.Name != podToReplace.Name { + newPod = pod + } + } + Expect(tester.UpdateCollaSet(cls, func(cls *appsv1alpha1.CollaSet) { + cls.Spec.Replicas = int32Pointer(0) + cls.Spec.ScaleStrategy = appsv1alpha1.ScaleStrategy{ + PodToDelete: []string{newPod.Name}, + } + })).NotTo(HaveOccurred()) + + By("Wait for CollaSet reconciled") + Eventually(func() bool { + if err := tester.GetCollaSet(cls); err != nil { + return false + } + return cls.Generation == cls.Status.ObservedGeneration + }, 10*time.Second, 3*time.Second).Should(Equal(true)) + + By("New pod will not be deleted") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 2, 0, 0, 2, 2) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Mock new pod service available") + Expect(tester.UpdatePod(newPod, func(pod *v1.Pod) { + newPod.Labels[appsv1alpha1.PodServiceAvailableLabel] = "true" + })).NotTo(HaveOccurred()) + + By("Wait for pods are deleted") + Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 0, 0, 0, 0, 0) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) + + By("Check resourceContext") + var currResourceContexts []*appsv1alpha1.ResourceContext + Eventually(func() bool { + currResourceContexts, err = tester.ListResourceContextsForCollaSet(cls) + Expect(err).Should(BeNil()) + return len(currResourceContexts) == 0 + }, 30*time.Second, 3*time.Second).Should(BeTrue()) + }) + }) }) func int32Pointer(val int32) *int32 { From 408fe0bc8932ca20681a2611e0c00953b1388ace Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Mon, 11 Nov 2024 15:33:01 +0800 Subject: [PATCH 3/6] revert IDToLabelsMap logic --- pkg/controllers/podopslifecycle/utils.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/controllers/podopslifecycle/utils.go b/pkg/controllers/podopslifecycle/utils.go index bc063c0e..f1ea5cf3 100644 --- a/pkg/controllers/podopslifecycle/utils.go +++ b/pkg/controllers/podopslifecycle/utils.go @@ -34,15 +34,12 @@ func IDToLabelsMap(pod *corev1.Pod) (map[string]map[string]string, map[string]in ids := sets.String{} for k := range pod.Labels { - for _, prefix := range v1alpha1.WellKnownLabelPrefixesWithID { - if strings.HasPrefix(k, prefix) { - s := strings.Split(k, "/") - if len(s) < 2 { - return nil, nil, fmt.Errorf("invalid label %s", k) - } - ids.Insert(s[1]) - break + if strings.HasPrefix(k, v1alpha1.PodOperatingLabelPrefix) || strings.HasPrefix(k, v1alpha1.PodOperateLabelPrefix) || strings.HasPrefix(k, v1alpha1.PodOperatedLabelPrefix) { + s := strings.Split(k, "/") + if len(s) < 2 { + return nil, nil, fmt.Errorf("invalid label %s", k) } + ids.Insert(s[1]) } } From df74087b76343004c76aab6395b238b26adf75f6 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Mon, 11 Nov 2024 16:51:47 +0800 Subject: [PATCH 4/6] fix include ut --- .../collaset/collaset_controller_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/collaset/collaset_controller_test.go b/pkg/controllers/collaset/collaset_controller_test.go index b31174b3..82c13970 100644 --- a/pkg/controllers/collaset/collaset_controller_test.go +++ b/pkg/controllers/collaset/collaset_controller_test.go @@ -3731,7 +3731,7 @@ var _ = Describe("collaset controller", func() { return true })).Should(BeNil()) - // included pod should not have ownerReference + // included pod should have ownerReference Eventually(func() bool { Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) podIncluded := false @@ -3748,7 +3748,7 @@ var _ = Describe("collaset controller", func() { return podIncluded }, 10*time.Second, 1*time.Second).Should(BeTrue()) - // included pvc should not have ownerReference + // included pvc should have ownerReference Eventually(func() bool { pvcIncluded := false Expect(c.List(context.TODO(), pvcList, client.InNamespace(cs.Namespace))).Should(BeNil()) @@ -3783,6 +3783,16 @@ var _ = Describe("collaset controller", func() { return true })).Should(BeNil()) + Eventually(func() bool { + Expect(c.List(context.TODO(), podList, client.InNamespace(cs.Namespace))).Should(BeNil()) + for k := range podList.Items[0].Labels { + if strings.HasPrefix(k, appsv1alpha1.PodOperatingLabelPrefix) { + return true + } + } + return false + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + // allow pod to update Expect(c.List(context.TODO(), pvcList, client.InNamespace(cs.Namespace))).Should(BeNil()) for i := range podList.Items { From 654d1e8d5e49f9a54c73b207a1fbf507da41e594 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Mon, 11 Nov 2024 18:06:32 +0800 Subject: [PATCH 5/6] fix e2e: check ods --- test/e2e/apps/collaset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/apps/collaset.go b/test/e2e/apps/collaset.go index 820bc899..643452e9 100644 --- a/test/e2e/apps/collaset.go +++ b/test/e2e/apps/collaset.go @@ -1058,7 +1058,7 @@ var _ = SIGDescribe("CollaSet", func() { } return false }, 12*time.Second, time.Second).Should(BeTrue()) - Expect(duration > time.Duration(*cls.Spec.UpdateStrategy.OperationDelaySeconds)*time.Second).Should(BeTrue()) + Expect(duration > time.Duration(*cls.Spec.UpdateStrategy.OperationDelaySeconds)*time.Second-4).Should(BeTrue()) By("Wait for update finished") Eventually(func() error { return tester.ExpectedStatusReplicas(cls, 3, 3, 3, 1, 3) }, 30*time.Second, 3*time.Second).ShouldNot(HaveOccurred()) From 947ca3c8b1c14af6f76c5052a132f59718c472a2 Mon Sep 17 00:00:00 2001 From: ColdsteelRail <574252631@qq.com> Date: Tue, 12 Nov 2024 13:06:03 +0800 Subject: [PATCH 6/6] fix comment --- pkg/controllers/podopslifecycle/utils.go | 5 +---- pkg/controllers/utils/podopslifecycle/utils.go | 7 +++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/podopslifecycle/utils.go b/pkg/controllers/podopslifecycle/utils.go index f1ea5cf3..78b4603f 100644 --- a/pkg/controllers/podopslifecycle/utils.go +++ b/pkg/controllers/podopslifecycle/utils.go @@ -22,8 +22,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" - "kusionstack.io/kube-api/apps/v1alpha1" ) @@ -77,8 +75,7 @@ func IDToLabelsMap(pod *corev1.Pod) (map[string]map[string]string, map[string]in } // IsLifecycleOnPod returns true if the lifecycle with lifecycleId exist on pod, otherwise returns false -func IsLifecycleOnPod(lifecycleId string, obj client.Object) (bool, error) { - pod := obj.(*corev1.Pod) +func IsLifecycleOnPod(lifecycleId string, pod *corev1.Pod) (bool, error) { if pod == nil { return false, nil } diff --git a/pkg/controllers/utils/podopslifecycle/utils.go b/pkg/controllers/utils/podopslifecycle/utils.go index 961b9d70..41be069b 100644 --- a/pkg/controllers/utils/podopslifecycle/utils.go +++ b/pkg/controllers/utils/podopslifecycle/utils.go @@ -23,6 +23,7 @@ import ( "strings" "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" @@ -89,10 +90,12 @@ func Begin(c client.Client, adapter LifecycleAdapter, obj client.Object, updateF // BeginWithCleaningOld is used for an CRD Operator to begin a lifecycle with cleaning the old lifecycle func BeginWithCleaningOld(c client.Client, adapter LifecycleAdapter, obj client.Object, updateFunc ...UpdateFunc) (updated bool, err error) { - if podInUpdateLifecycle, err := podopslifecycleutil.IsLifecycleOnPod(adapter.GetID(), obj); err != nil { + if podInUpdateLifecycle, err := podopslifecycleutil.IsLifecycleOnPod(adapter.GetID(), obj.(*corev1.Pod)); err != nil { return false, fmt.Errorf("fail to check %s PodOpsLifecycle on Pod %s/%s: %s", adapter.GetID(), obj.GetNamespace(), obj.GetName(), err) } else if podInUpdateLifecycle { - return false, Undo(c, adapter, obj) + if err := Undo(c, adapter, obj); err != nil { + return false, err + } } return Begin(c, adapter, obj, updateFunc...) }