TAS API (kubernetes-sigs#3235)

apis/kueue/v1alpha1/tas_types.go

@@ -16,7 +16,33 @@ limitations under the License.
 
 package v1alpha1
 
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
 const (
+	// PodSetRequiredTopologyAnnotation indicates that a PodSet requires
+	// Topology Aware Scheduling, and requires scheduling all pods on nodes
+	// within the same topology domain corresponding to the topology level
+	// indicated by the annotation value (e.g. within a rack or within a block).
+	PodSetRequiredTopologyAnnotation = "kueue.x-k8s.io/podset-required-topology"
+
+	// PodSetPreferredTopologyAnnotation indicates that a PodSet requires
+	// Topology Aware Scheduling, but scheduling all pods within pods on nodes
+	// within the same topology domain is a preference rather than requirement.
+	//
+	// The levels are evaluated one-by-one going up from the level indicated by
+	// the annotation. If the PodSet cannot fit within a given topology domain
+	// then the next topology level up is considered. If the PodSet cannot fit
+	// at the highest topology level, then it gets admitted as distributed
+	// among multiple topology domains.
+	PodSetPreferredTopologyAnnotation = "kueue.x-k8s.io/podset-preferred-topology"
+
+	// TopologySchedulingGate is used to delay scheduling of a Pod until the
+	// nodeSelectors corresponding to the assigned topology domain are injected
+	// into the Pod.
+	TopologySchedulingGate = "kueue.x-k8s.io/topology"
+
 	// WorkloadAnnotation is an annotation set on the Job's PodTemplate to
 	// indicate the name of the admitted Workload corresponding to the Job. The
 	// annotation is set when starting the Job, and removed on stopping the Job.
@@ -27,3 +53,64 @@ const (
 	// The label is set when starting the Job, and removed on stopping the Job.
 	PodSetLabel = "kueue.x-k8s.io/podset"
 )
+
+// TopologySpec defines the desired state of Topology
+type TopologySpec struct {
+	// levels define the levels of topology.
+	//
+	// +required
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=8
+	Levels []TopologyLevel `json:"levels,omitempty"`
+}
+
+// TopologyLevel defines the desired state of TopologyLevel
+type TopologyLevel struct {
+	// nodeLabel indicates the name of the node label for a specific topology
+	// level.
+	//
+	// Examples:
+	// - cloud.provider.com/topology-block
+	// - cloud.provider.com/topology-rack
+	//
+	// +required
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=316
+	// +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$`
+	NodeLabel string `json:"nodeLabel"`
+}
+
+// TopologyStatus defines the observed state of Topology
+type TopologyStatus struct {
+}
+
+// +genclient
+// +genclient:nonNamespaced
+// +kubebuilder:object:root=true
+// +kubebuilder:storageversion
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster
+
+// Topology is the Schema for the topology API
+type Topology struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   TopologySpec   `json:"spec,omitempty"`
+	Status TopologyStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// TopologyList contains a list of Topology
+type TopologyList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Topology `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Topology{}, &TopologyList{})
+}

apis/kueue/v1beta1/resourceflavor_types.go

@@ -85,6 +85,13 @@ type ResourceFlavorSpec struct {
 	// +kubebuilder:validation:XValidation:rule="self.all(x, has(x.operator) && x.operator == 'Exists' ? !has(x.value) : true)", message="a value must be empty when 'operator' is 'Exists'"
 	// +kubebuilder:validation:XValidation:rule="self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', 'NoExecute'])", message="supported taint effect values: 'NoSchedule', 'PreferNoSchedule', 'NoExecute'"
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+
+	// topologyName indicates topology for the TAS ResourceFlavor.
+	// When specified, it enables scraping of the topology information from the
+	// nodes matching to the Resource Flavor node labels.
+	//
+	// +optional
+	TopologyName *string `json:"topologyName,omitempty"`
 }
 
 // +kubebuilder:object:root=true

apis/kueue/v1beta1/workload_types.go

@@ -77,6 +77,23 @@ type WorkloadSpec struct {
 	Active *bool `json:"active,omitempty"`
 }
 
+// PodSetTopologyRequest defines the topology request for a PodSet.
+type PodSetTopologyRequest struct {
+	// required indicates the topology level required by the PodSet, as
+	// indicated by the `kueue.x-k8s.io/podset-required-topology` PodSet
+	// annotation.
+	//
+	// +optional
+	Required *string `json:"required,omitempty"`
+
+	// preferred indicates the topology level preferred by the PodSet, as
+	// indicated by the `kueue.x-k8s.io/podset-preferred-topology` PodSet
+	// annotation.
+	//
+	// +optional
+	Preferred *string `json:"preferred,omitempty"`
+}
+
 type Admission struct {
 	// clusterQueue is the name of the ClusterQueue that admitted this workload.
 	ClusterQueue ClusterQueueReference `json:"clusterQueue"`
@@ -113,6 +130,74 @@ type PodSetAssignment struct {
 	// +optional
 	// +kubebuilder:validation:Minimum=0
 	Count *int32 `json:"count,omitempty"`
+
+	// topologyAssignment indicates the topology assignment divided into
+	// topology domains corresponding to the lowest level of the topology.
+	// The assignment specifies the number of Pods to be scheduled per topology
+	// domain and specifies the node selectors for each topology domain, in the
+	// following way: the node selector keys are specified by the levels field
+	// (same for all domains), and the corresponding node selector value is
+	// specified by the domains.values subfield.
+	//
+	// Example:
+	//
+	// topologyAssignment:
+	//   levels:
+	//   - cloud.provider.com/topology-block
+	//   - cloud.provider.com/topology-rack
+	//   domains:
+	//   - values: [block-1, rack-1]
+	//     count: 4
+	//   - values: [block-1, rack-2]
+	//     count: 2
+	//
+	// Here:
+	// - 4 Pods are to be scheduled on nodes matching the node selector:
+	//   cloud.provider.com/topology-block: block-1
+	//   cloud.provider.com/topology-rack: rack-1
+	// - 2 Pods are to be scheduled on nodes matching the node selector:
+	//   cloud.provider.com/topology-block: block-1
+	//   cloud.provider.com/topology-rack: rack-2
+	//
+	// +optional
+	TopologyAssignment *TopologyAssignment `json:"topologyAssignment,omitempty"`
+}
+
+type TopologyAssignment struct {
+	// levels is an ordered list of keys denoting the levels of the assigned
+	// topology (i.e. node label keys), from the highest to the lowest level of
+	// the topology.
+	//
+	// +required
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=8
+	Levels []string `json:"levels"`
+
+	// domains is a list of topology assignments split by topology domains at
+	// the lowest level of the topology.
+	//
+	// +required
+	Domains []TopologyDomainAssignment `json:"domains"`
+}
+
+type TopologyDomainAssignment struct {
+	// values is an ordered list of node selector values describing a topology
+	// domain. The values correspond to the consecutive topology levels, from
+	// the highest to the lowest.
+	//
+	// +required
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=8
+	Values []string `json:"values"`
+
+	// count indicates the number of Pods to be scheduled in the topology
+	// domain indicated by the values field.
+	//
+	// +required
+	// +kubebuilder:validation:Minimum=1
+	Count int32 `json:"count"`
 }
 
 // +kubebuilder:validation:XValidation:rule="has(self.minCount) ? self.minCount <= self.count : true", message="minCount should be positive and less or equal to count"
@@ -156,6 +241,11 @@ type PodSet struct {
 	// +optional
 	// +kubebuilder:validation:Minimum=1
 	MinCount *int32 `json:"minCount,omitempty"`
+
+	// topologyRequest defines the topology request for the PodSet.
+	//
+	// +optional
+	TopologyRequest *PodSetTopologyRequest `json:"topologyRequest,omitempty"`
 }
 
 // WorkloadStatus defines the observed state of Workload