From 065601e90f7a7ab12037d4ce46fc491abe9b02cc Mon Sep 17 00:00:00 2001 From: "mingzhou.swx" Date: Thu, 8 Sep 2022 15:57:59 +0800 Subject: [PATCH] add postCreatedHook for lifecycle Signed-off-by: mingzhou.swx --- apis/apps/pub/lifecycle.go | 3 + apis/apps/pub/zz_generated.deepcopy.go | 5 ++ .../crd/bases/apps.kruise.io_clonesets.yaml | 20 ++++++ .../crd/bases/apps.kruise.io_daemonsets.yaml | 20 ++++++ .../bases/apps.kruise.io_statefulsets.yaml | 20 ++++++ .../apps.kruise.io_uniteddeployments.yaml | 40 +++++++++++ .../cloneset/sync/cloneset_scale.go | 2 +- .../cloneset/sync/cloneset_update.go | 7 ++ .../podreadiness/pod_readiness_controller.go | 71 ++++++++++++++++++- pkg/util/lifecycle/lifecycle_utils.go | 5 ++ pkg/util/podreadiness/pod_readiness_utils.go | 2 +- 11 files changed, 192 insertions(+), 3 deletions(-) diff --git a/apis/apps/pub/lifecycle.go b/apis/apps/pub/lifecycle.go index 39c8ae33ce..c5e0d85bc0 100644 --- a/apis/apps/pub/lifecycle.go +++ b/apis/apps/pub/lifecycle.go @@ -20,6 +20,7 @@ const ( LifecycleStateKey = "lifecycle.apps.kruise.io/state" LifecycleTimestampKey = "lifecycle.apps.kruise.io/timestamp" + LifecycleStateCreating LifecycleStateType = "Creating" LifecycleStateNormal LifecycleStateType = "Normal" LifecycleStatePreparingUpdate LifecycleStateType = "PreparingUpdate" LifecycleStateUpdating LifecycleStateType = "Updating" @@ -35,6 +36,8 @@ type Lifecycle struct { PreDelete *LifecycleHook `json:"preDelete,omitempty"` // InPlaceUpdate is the hook before Pod to update and after Pod has been updated. InPlaceUpdate *LifecycleHook `json:"inPlaceUpdate,omitempty"` + // PostCreated is the hook after Pod to be created and ready to be Normal. + PostCreated *LifecycleHook `json:"postCreated,omitempty"` } type LifecycleHook struct { diff --git a/apis/apps/pub/zz_generated.deepcopy.go b/apis/apps/pub/zz_generated.deepcopy.go index afa9250c4e..69349041f1 100644 --- a/apis/apps/pub/zz_generated.deepcopy.go +++ b/apis/apps/pub/zz_generated.deepcopy.go @@ -158,6 +158,11 @@ func (in *Lifecycle) DeepCopyInto(out *Lifecycle) { *out = new(LifecycleHook) (*in).DeepCopyInto(*out) } + if in.PostCreated != nil { + in, out := &in.PostCreated, &out.PostCreated + *out = new(LifecycleHook) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Lifecycle. diff --git a/config/crd/bases/apps.kruise.io_clonesets.yaml b/config/crd/bases/apps.kruise.io_clonesets.yaml index da2fb66cdc..1a18d08056 100644 --- a/config/crd/bases/apps.kruise.io_clonesets.yaml +++ b/config/crd/bases/apps.kruise.io_clonesets.yaml @@ -105,6 +105,26 @@ spec: to false.' type: boolean type: object + postCreated: + description: PostCreated is the hook after Pod to be created and + ready to be Normal. + properties: + finalizersHandler: + items: + type: string + type: array + labelsHandler: + additionalProperties: + type: string + type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean + type: object preDelete: description: PreDelete is the hook before Pod to be deleted. properties: diff --git a/config/crd/bases/apps.kruise.io_daemonsets.yaml b/config/crd/bases/apps.kruise.io_daemonsets.yaml index a03c84523a..fc0ccf66b7 100644 --- a/config/crd/bases/apps.kruise.io_daemonsets.yaml +++ b/config/crd/bases/apps.kruise.io_daemonsets.yaml @@ -109,6 +109,26 @@ spec: to false.' type: boolean type: object + postCreated: + description: PostCreated is the hook after Pod to be created and + ready to be Normal. + properties: + finalizersHandler: + items: + type: string + type: array + labelsHandler: + additionalProperties: + type: string + type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean + type: object preDelete: description: PreDelete is the hook before Pod to be deleted. properties: diff --git a/config/crd/bases/apps.kruise.io_statefulsets.yaml b/config/crd/bases/apps.kruise.io_statefulsets.yaml index 2250fdf00b..9dacc20c5d 100644 --- a/config/crd/bases/apps.kruise.io_statefulsets.yaml +++ b/config/crd/bases/apps.kruise.io_statefulsets.yaml @@ -522,6 +522,26 @@ spec: to false.' type: boolean type: object + postCreated: + description: PostCreated is the hook after Pod to be created and + ready to be Normal. + properties: + finalizersHandler: + items: + type: string + type: array + labelsHandler: + additionalProperties: + type: string + type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - Pod will be + set to ''NotReady'' at preparingDelete/preparingUpdate state. + - Pod will be restored to ''Ready'' at Updated state if + it was set to ''NotReady'' at preparingUpdate state. Default + to false.' + type: boolean + type: object preDelete: description: PreDelete is the hook before Pod to be deleted. properties: diff --git a/config/crd/bases/apps.kruise.io_uniteddeployments.yaml b/config/crd/bases/apps.kruise.io_uniteddeployments.yaml index a274937b18..5df69dbcae 100644 --- a/config/crd/bases/apps.kruise.io_uniteddeployments.yaml +++ b/config/crd/bases/apps.kruise.io_uniteddeployments.yaml @@ -153,6 +153,26 @@ spec: at preparingUpdate state. Default to false.' type: boolean type: object + postCreated: + description: PostCreated is the hook after Pod to + be created and ready to be Normal. + properties: + finalizersHandler: + items: + type: string + type: array + labelsHandler: + additionalProperties: + type: string + type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - + Pod will be set to ''NotReady'' at preparingDelete/preparingUpdate + state. - Pod will be restored to ''Ready'' at + Updated state if it was set to ''NotReady'' + at preparingUpdate state. Default to false.' + type: boolean + type: object preDelete: description: PreDelete is the hook before Pod to be deleted. @@ -568,6 +588,26 @@ spec: at preparingUpdate state. Default to false.' type: boolean type: object + postCreated: + description: PostCreated is the hook after Pod to + be created and ready to be Normal. + properties: + finalizersHandler: + items: + type: string + type: array + labelsHandler: + additionalProperties: + type: string + type: object + markPodNotReady: + description: 'MarkPodNotReady = true means: - + Pod will be set to ''NotReady'' at preparingDelete/preparingUpdate + state. - Pod will be restored to ''Ready'' at + Updated state if it was set to ''NotReady'' + at preparingUpdate state. Default to false.' + type: boolean + type: object preDelete: description: PreDelete is the hook before Pod to be deleted. diff --git a/pkg/controller/cloneset/sync/cloneset_scale.go b/pkg/controller/cloneset/sync/cloneset_scale.go index 69ca12c685..7388eea209 100644 --- a/pkg/controller/cloneset/sync/cloneset_scale.go +++ b/pkg/controller/cloneset/sync/cloneset_scale.go @@ -214,7 +214,7 @@ func (r *realControl) createPods( if clonesetutils.EqualToRevisionHash("", pod, currentRevision) { cs = currentCS } - lifecycle.SetPodLifecycle(appspub.LifecycleStateNormal)(pod) + lifecycle.SetPodLifecycle(appspub.LifecycleStateCreating)(pod) var createErr error if createErr = r.createOnePod(cs, pod, existingPVCNames); createErr != nil { diff --git a/pkg/controller/cloneset/sync/cloneset_update.go b/pkg/controller/cloneset/sync/cloneset_update.go index 828135efe2..a45c7baeee 100644 --- a/pkg/controller/cloneset/sync/cloneset_update.go +++ b/pkg/controller/cloneset/sync/cloneset_update.go @@ -168,6 +168,13 @@ func (c *realControl) refreshPodState(cs *appsv1alpha1.CloneSet, coreControl clo var state appspub.LifecycleStateType switch lifecycle.GetPodLifecycleState(pod) { + case appspub.LifecycleStateCreating: + if cs.Spec.Lifecycle == nil || + cs.Spec.Lifecycle.PostCreated == nil || + lifecycle.IsPodAllHooked(cs.Spec.Lifecycle.PostCreated, pod) { + state = appspub.LifecycleStateNormal + } + case appspub.LifecycleStateUpdating: if opts.CheckPodUpdateCompleted(pod) == nil { if cs.Spec.Lifecycle != nil && !lifecycle.IsPodAllHooked(cs.Spec.Lifecycle.InPlaceUpdate, pod) { diff --git a/pkg/controller/podreadiness/pod_readiness_controller.go b/pkg/controller/podreadiness/pod_readiness_controller.go index fd155497bb..cca6049e75 100644 --- a/pkg/controller/podreadiness/pod_readiness_controller.go +++ b/pkg/controller/podreadiness/pod_readiness_controller.go @@ -18,6 +18,11 @@ package podreadiness import ( "context" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + appsv1beta1 "github.com/openkruise/kruise/apis/apps/v1beta1" + lifecycleutil "github.com/openkruise/kruise/pkg/util/lifecycle" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "time" appspub "github.com/openkruise/kruise/apis/apps/pub" @@ -38,6 +43,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) +const ( + CloneSetKind = "CloneSet" + AdvancedStatefulSetKind = "StatefulSet" + AdvancedDaemonSetKind = "DaemonSet" +) + var ( concurrentReconciles = 3 ) @@ -48,8 +59,10 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) *ReconcilePodReadiness { + cli := utilclient.NewClientFromManager(mgr, "pod-readiness-controller") return &ReconcilePodReadiness{ - Client: utilclient.NewClientFromManager(mgr, "pod-readiness-controller"), + Client: cli, + lifecycleControl: lifecycleutil.New(cli), } } @@ -89,6 +102,7 @@ var _ reconcile.Reconciler = &ReconcilePodReadiness{} // ReconcilePodReadiness reconciles a Pod object type ReconcilePodReadiness struct { client.Client + lifecycleControl lifecycleutil.Interface } // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete @@ -127,6 +141,19 @@ func (r *ReconcilePodReadiness) Reconcile(_ context.Context, request reconcile.R return nil } + lifecycle, err := r.getLifecycleHook(pod) + if err != nil { + return err + } + + if lifecycle != nil && + lifecycle.PostCreated != nil && + lifecycle.PostCreated.MarkPodNotReady && + !lifecycleutil.IsPodAllHooked(lifecycle.PostCreated, pod) { + _, _, err = r.lifecycleControl.UpdatePodLifecycle(pod, appspub.LifecycleStateCreating, true) + return err + } + pod.Status.Conditions = append(pod.Status.Conditions, v1.PodCondition{ Type: appspub.KruisePodReadyConditionType, Status: v1.ConditionTrue, @@ -136,3 +163,45 @@ func (r *ReconcilePodReadiness) Reconcile(_ context.Context, request reconcile.R }) return reconcile.Result{}, err } + +func (r *ReconcilePodReadiness) getLifecycleHook(pod *v1.Pod) (*appspub.Lifecycle, error) { + if pod.Labels[appspub.LifecycleStateKey] != string(appspub.LifecycleStateCreating) { + return nil, nil + } + + owner := metav1.GetControllerOfNoCopy(pod) + if owner == nil { + return nil, nil + } + + ownerGVK := schema.FromAPIVersionAndKind(owner.APIVersion, owner.Kind) + if ownerGVK.Group != appsv1alpha1.GroupVersion.Group { + return nil, nil + } + ownerKey := types.NamespacedName{Namespace: pod.Namespace, Name: owner.Name} + + switch ownerGVK.Kind { + case CloneSetKind: + clone := &appsv1alpha1.CloneSet{} + if err := r.Get(context.TODO(), ownerKey, clone); err != nil { + return nil, err + } + return clone.Spec.Lifecycle, nil + + case AdvancedStatefulSetKind: + asts := &appsv1beta1.StatefulSet{} + if err := r.Get(context.TODO(), ownerKey, asts); err != nil { + return nil, err + } + return asts.Spec.Lifecycle, nil + + case AdvancedDaemonSetKind: + ads := &appsv1alpha1.DaemonSet{} + if err := r.Get(context.TODO(), ownerKey, ads); err != nil { + return nil, err + } + return ads.Spec.Lifecycle, nil + } + + return nil, nil +} diff --git a/pkg/util/lifecycle/lifecycle_utils.go b/pkg/util/lifecycle/lifecycle_utils.go index 8a16536ea4..fab436b052 100644 --- a/pkg/util/lifecycle/lifecycle_utils.go +++ b/pkg/util/lifecycle/lifecycle_utils.go @@ -37,6 +37,7 @@ const ( // these keys for MarkPodNotReady Policy of pod lifecycle preparingDeleteHookKey = "preDeleteHook" preparingUpdateHookKey = "preUpdateHook" + postCreateHookKey = "postCreateHook" ) func newPodReadinessControl(adp podadapter.Adapter) podreadiness.Interface { @@ -111,6 +112,10 @@ func SetPodLifecycle(state appspub.LifecycleStateType) func(*v1.Pod) { func (c *realControl) executePodNotReadyPolicy(pod *v1.Pod, state appspub.LifecycleStateType) (err error) { switch state { + case appspub.LifecycleStateCreating: + err = c.podReadinessControl.AddNotReadyKey(pod, getReadinessMessage(postCreateHookKey)) + case appspub.LifecycleStateNormal: + err = c.podReadinessControl.RemoveNotReadyKey(pod, getReadinessMessage(postCreateHookKey)) case appspub.LifecycleStatePreparingDelete: err = c.podReadinessControl.AddNotReadyKey(pod, getReadinessMessage(preparingDeleteHookKey)) case appspub.LifecycleStatePreparingUpdate: diff --git a/pkg/util/podreadiness/pod_readiness_utils.go b/pkg/util/podreadiness/pod_readiness_utils.go index 012cf8a854..f863fe009d 100644 --- a/pkg/util/podreadiness/pod_readiness_utils.go +++ b/pkg/util/podreadiness/pod_readiness_utils.go @@ -46,7 +46,7 @@ func addNotReadyKey(adp podadapter.Adapter, pod *v1.Pod, msg Message, condType v if condition == nil { _, messages := addMessage("", msg) newPod.Status.Conditions = append(newPod.Status.Conditions, v1.PodCondition{ - Type: appspub.KruisePodReadyConditionType, + Type: condType, Message: messages.dump(), LastTransitionTime: metav1.Now(), })