TAS API

apis/kueue/v1alpha1/tas_types.go

@@ -16,7 +16,33 @@ limitations under the License.
 
 package v1alpha1
 
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
 const (
+	// PodSetTopologyRequiredAnnotation indicates that a PodSet requires
+	// Topology Aware Scheduling, and requires scheduling all pods on nodes
+	// within the same topology domain corresponding to the topology level
+	// indicated by the annotation value (e.g. within a rack or within a block).
+	PodSetTopologyRequiredAnnotation = "kueue.x-k8s.io/podset-required-topology"
+
+	// PodSetPreferredTopologyAnnotation indicates that a PodSet requires
+	// Topology Aware Scheduling, but scheduling all pods within pods on nodes
+	// within the same topology domain is a preference rather than requirement.
+	//
+	// The levels are evaluated one-by-one going up from the level indicated by
+	// the annotation. If the PodSet cannot fit within a given topology domain
+	// then the next topology level up is considered. If the PodSet cannot fit
+	// at the highest topology level, then it gets admitted as distributed
+	// among multiple topology domains.
+	PodSetPreferredTopologyAnnotation = "kueue.x-k8s.io/podset-preferred-topology"
+
+	// TopologySchedulingGate is used to delay scheduling of a Pod until the
+	// nodeSelectors corresponding to the assigned topology domain are injected
+	// into the Pod.
+	TopologySchedulingGate = "kueue.x-k8s.io/topology"
+
 	// WorkloadAnnotation is an annotation set on the Job's PodTemplate to
 	// indicate the name of the admitted Workload corresponding to the Job. The
 	// annotation is set when starting the Job, and removed on stopping the Job.
@@ -27,3 +53,59 @@ const (
 	// The label is set when starting the Job, and removed on stopping the Job.
 	PodSetLabel = "kueue.x-k8s.io/podset"
 )
+
+// TopologySpec defines the desired state of Topology
+type TopologySpec struct {
+	// levels define the levels of topology.
+	//
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=5
+	Levels []TopologyLevel `json:"levels,omitempty"`
+}
+
+// TopologyLevel defines the desired state of TopologyLevel
+type TopologyLevel struct {
+	// nodeLabel indicates the name of the node label for a specific topology
+	// level.
+	//
+	// +required
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=316
+	// +kubebuilder:validation:Pattern=`^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$`
+	NodeLabel string `json:"nodeLabel"`
+}
+
+// TopologyStatus defines the observed state of Topology
+type TopologyStatus struct {
+}
+
+// +genclient
+// +genclient:nonNamespaced
+// +kubebuilder:object:root=true
+// +kubebuilder:storageversion
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster
+
+// Topology is the Schema for the topology API
+type Topology struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   TopologySpec   `json:"spec,omitempty"`
+	Status TopologyStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// TopologyList contains a list of Topology
+type TopologyList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Topology `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Topology{}, &TopologyList{})
+}

apis/kueue/v1beta1/resourceflavor_types.go

@@ -85,6 +85,13 @@ type ResourceFlavorSpec struct {
 	// +kubebuilder:validation:XValidation:rule="self.all(x, has(x.operator) && x.operator == 'Exists' ? !has(x.value) : true)", message="a value must be empty when 'operator' is 'Exists'"
 	// +kubebuilder:validation:XValidation:rule="self.all(x, !has(x.effect) || x.effect in ['NoSchedule', 'PreferNoSchedule', 'NoExecute'])", message="supported taint effect values: 'NoSchedule', 'PreferNoSchedule', 'NoExecute'"
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+
+	// topologyName indicates topology for the TAS ResourceFlavor.
+	// When specified, it enables scraping of the topology information from the
+	// nodes matching to the Resource Flavor node labels.
+	//
+	// +optional
+	TopologyName *string `json:"topologyName,omitempty"`
 }
 
 // +kubebuilder:object:root=true

apis/kueue/v1beta1/workload_types.go

@@ -77,6 +77,18 @@ type WorkloadSpec struct {
 	Active *bool `json:"active,omitempty"`
 }
 
+type PodSetTopologyRequest struct {
+	// required is the level label indicated by the `kueue.x-k8s.io/podset-required-topology` annotation
+	//
+	// +optional
+	Required *string `json:"required,omitempty"`
+
+	// preferred is the level label indicated by the `kueue.x-k8s.io/podset-preferred-topology` annotation
+	//
+	// +optional
+	Preferred *string `json:"preferred,omitempty"`
+}
+
 type Admission struct {
 	// clusterQueue is the name of the ClusterQueue that admitted this workload.
 	ClusterQueue ClusterQueueReference `json:"clusterQueue"`
@@ -113,6 +125,27 @@ type PodSetAssignment struct {
 	// +optional
 	// +kubebuilder:validation:Minimum=0
 	Count *int32 `json:"count,omitempty"`
+
+	// topologyAssignment indicates the resources assigned per topology level
+	// +optional
+	TopologyAssignment *TopologyAssignment `json:"topologyAssignment,omitempty"`
+}
+
+type TopologyAssignment struct {
+	// domains contains the list of assignments split into lowest-level groups
+	// +optional
+	Domains []TopologyDomainAssignment `json:"domains,omitempty"`
+
+	// levels specifies the ordered list of keys for the assigned nodeSelectors.
+	Levels []string `json:"levels,omitempty"`
+}
+
+type TopologyDomainAssignment struct {
+	// Values specifies the values of nodeSelectors for the corresponding topology levels.
+	Values []string `json:"values,omitempty"`
+
+	// Count indicates the number of pods in a given TopologyAssignmentSlice
+	Count int `json:"count,omitempty"`
 }
 
 // +kubebuilder:validation:XValidation:rule="has(self.minCount) ? self.minCount <= self.count : true", message="minCount should be positive and less or equal to count"
@@ -156,6 +189,11 @@ type PodSet struct {
 	// +optional
 	// +kubebuilder:validation:Minimum=1
 	MinCount *int32 `json:"minCount,omitempty"`
+
+	// TopologyRequest defines the topology requested for the corresponding Job.
+	//
+	// +optional
+	TopologyRequest *PodSetTopologyRequest `json:"topologyRequest,omitempty"`
 }
 
 // WorkloadStatus defines the observed state of Workload