Release soperator 1.16.0

.github/workflows/github_release.yml

@@ -117,7 +117,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create GitHub Release with changelog
-        uses: softprops/action-gh-release@01570a1f39cb168c169c802c3bceb9e93fb10974 # v2.1.0
+        uses: softprops/action-gh-release@7b4da11513bf3f43f9999e90eabced41ab8bb048 # v2.2.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.github/workflows/gpubench_only.yml

@@ -51,7 +51,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install GO
-        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
             go-version-file: 'go.mod'
 

.github/workflows/one_job.yml

@@ -11,6 +11,9 @@ on:
       - 'README.md'
       - 'SECURITY.md'
       - 'images/jail/gpubench/**'
+  pull_request:
+    branches:
+      - main
 
 permissions:
   contents: read
@@ -59,7 +62,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install GO
-        uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
+        uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
             go-version-file: 'go.mod'
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22@sha256:4594271250150c1a322ed749abfd218e1a8c6eb1ade90872e325a664412e2037 AS operator_builder
+FROM golang:1.23@sha256:70031844b8c225351d0bb63e2c383f80db85d92ba894e3da7e13bcf80efa9a37 AS operator_builder
 
 ARG GO_LDFLAGS=""
 ARG BUILD_TIME
@@ -16,7 +16,7 @@ RUN GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED GO_LDFLAGS=$GO_LDFLAGS \
     go build -o slurm_operator ./cmd/
 
 #######################################################################################################################
-FROM alpine:latest@sha256:1e42bbe2508154c9126d48c2b8a75420c3544343bf86fd041fb7527e017a4b4a AS slurm-operator
+FROM alpine:latest@sha256:21dc6063fd678b478f57c0e13f47560d0ea4eeba26dfc947b2a4f81f686b9f45 AS slurm-operator
 
 COPY --from=operator_builder /operator/slurm_operator /usr/bin/
 

Makefile

@@ -105,13 +105,12 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 	$(GOLANGCI_LINT) run --fix
 
 .PHONY: helm
-helm: kustomize helmify yq ## Update soperator Helm chart
-	rm -rf $(CHART_OPERATOR_PATH)
-	$(KUSTOMIZE) build config/default | $(HELMIFY) --crd-dir $(CHART_OPERATOR_PATH)
-	rm -f $(CHART_PATH)/operatorAppVersion
-	cp -r $(CHART_OPERATOR_PATH)/crds/* $(CHART_OPERATOR_CRDS_PATH)/templates/
-	@$(YQ) -i ".name = \"helm-soperator\"" "$(CHART_OPERATOR_PATH)/Chart.yaml"
-	@$(SED_COMMAND) '/^#/d' "$(CHART_OPERATOR_PATH)/Chart.yaml"
+helm: generate manifests ## Update soperator Helm chart
+	$(KUSTOMIZE) build config/crd > $(CHART_OPERATOR_PATH)/crds/slurmcluster-crd.yaml 
+	$(KUSTOMIZE) build config/crd > $(CHART_OPERATOR_CRDS_PATH)/templates/slurmcluster-crd.yaml
+	mv $(CHART_OPERATOR_PATH)/values.yaml $(CHART_OPERATOR_PATH)/values.yaml.bak
+	$(KUSTOMIZE)  build --load-restrictor LoadRestrictionsNone config/rbac/soperator-helm  | $(HELMIFY) $(CHART_OPERATOR_PATH)
+	mv $(CHART_OPERATOR_PATH)/values.yaml.bak $(CHART_OPERATOR_PATH)/values.yaml
 
 .PHONY: get-version
 get-version:
@@ -297,11 +296,11 @@ YQ             ?= $(LOCALBIN)/yq
 
 ## Tool Versions
 KUSTOMIZE_VERSION        ?= v5.5.0
-CONTROLLER_TOOLS_VERSION ?= v0.14.0
+CONTROLLER_TOOLS_VERSION ?= v0.16.4
 ENVTEST_VERSION          ?= release-0.17
 GOLANGCI_LINT_VERSION    ?= v1.57.2
 HELMIFY_VERSION          ?= 0.4.13
-YQ_VERSION               ?= 4.44.1
+YQ_VERSION               ?= 4.44.3
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.

PROJECT

@@ -17,4 +17,12 @@ resources:
   kind: SlurmCluster
   path: nebius.ai/slurm-operator/api/v1
   version: v1
+- core: true
+  group: core
+  kind: Secret
+  path: k8s.io/api/core/v1
+  version: v1
+  webhooks:
+    validation: true
+    webhookVersion: v1
 version: "3"

README.md

@@ -101,6 +101,7 @@ Slurm's accounting system records detailed job information such as:
 - User and group identities
 - Job start/end times
 - Resource requests and allocations
+- If `protectedSecret` is set to `true`, the user secret for MariaDB will not be deleted after the MariaDB CR is deleted
 
 This helps cluster administrators and users monitor resource utilization, enforce quotas, and generate usage reports for performance optimization or billing purposes.
 
@@ -114,11 +115,10 @@ This helps cluster administrators and users monitor resource utilization, enforc
       [22.04](https://releases.ubuntu.com/jammy/).
     - Slurm: versions `23.11.6` and `24.05.3`.
     - CUDA: version [12.2.2](https://developer.nvidia.com/cuda-12-2-2-download-archive).
-    - Kubernetes: >= [1.28](https://kubernetes.io/blog/2023/08/15/kubernetes-v1-28-release/).
+    - Kubernetes: >= [1.29](https://kubernetes.io/blog/2023/08/15/kubernetes-v1-28-release/).
     - Versions of some preinstalled software packages can't be changed.
 
 
-
 ## 🚀 Installation
 The steps required to deploy Soperator to your Kubernetes cluster depend on whether you are using Kubernetes
 on premises or in a cloud.

VERSION

@@ -1 +1 @@
-1.15.5
+1.16.0

api/v1/slurmcluster_types.go

@@ -6,7 +6,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	mariadv1alpha1 "github.com/mariadb-operator/mariadb-operator/api/v1alpha1"
+	mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/api/v1alpha1"
 	prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 )
 
@@ -73,6 +73,40 @@ type SlurmClusterSpec struct {
 	// https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION
 	// +kubebuilder:validation:Optional
 	PartitionConfiguration PartitionConfiguration `json:"partitionConfiguration,omitempty"`
+
+	// SlurmConfig represents the Slurm configuration in slurm.conf. Not all options are supported.
+	//
+	// +kubebuilder:validation:Optional
+	SlurmConfig SlurmConfig `json:"slurmConfig,omitempty"`
+}
+
+// SlurmConfig represents the Slurm configuration in slurm.conf
+type SlurmConfig struct {
+	// Default real memory size available per allocated node in mebibytes.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=1228800
+	DefMemPerNode int32 `json:"defMemPerNode,omitempty"`
+	// Default count of CPUs allocated per allocated GPU
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=16
+	DefCpuPerGPU int32 `json:"defCpuPerGPU,omitempty"`
+	// The time to wait, in seconds, when any job is in the COMPLETING state before any additional jobs are scheduled.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=5
+	CompleteWait int32 `json:"completeWait,omitempty"`
+	// Defines specific subsystems which should provide more detailed event logging.
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default="Cgroup,CPU_Bind,Gres,JobComp,Priority,Script,SelectType,Steps,TraceJobs"
+	// +kubebuilder:validation:Pattern="^((Accrue|Agent|AuditRPCs|Backfill|BackfillMap|BurstBuffer|Cgroup|ConMgr|CPU_Bind|CpuFrequency|Data|DBD_Agent|Dependency|Elasticsearch|Energy|Federation|FrontEnd|Gres|Hetjob|Gang|GLOB_SILENCE|JobAccountGather|JobComp|JobContainer|License|Network|NetworkRaw|NodeFeatures|NO_CONF_HASH|Power|Priority|Profile|Protocol|Reservation|Route|Script|SelectType|Steps|Switch|TLS|TraceJobs|Triggers)(,)?)+$"
+	DebugFlags string `json:"debugFlags,omitempty"`
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default="Verbose"
+	// +kubebuilder:validation:Pattern="^((None|Cores|Sockets|Threads|SlurmdOffSpec|OOMKillStep|Verbose|Autobind)(,)?)+$"
+	TaskPluginParam string `json:"taskPluginParam,omitempty"`
 }
 
 type PartitionConfiguration struct {
@@ -277,7 +311,6 @@ type K8sNodeFilter struct {
 	Name string `json:"name"`
 
 	// Affinity defines the desired affinity for the node
-	//
 	// NOTE: Affinity could not be set if NodeSelector is specified
 	//
 	// +kubebuilder:validation:Optional
@@ -442,13 +475,26 @@ type MariaDbOperator struct {
 	// +kubebuilder:validation:Optional
 	Enabled bool `json:"enabled"`
 
+	// If enabled, secret cannot be deleted until custom resource slurmcluster is deleted
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=false
+	// +kubebuilder:validation:Immutable
+	ProtectedSecret bool `json:"protectedSecret"`
+
 	NodeContainer      `json:",inline"`
-	PodSecurityContext *corev1.PodSecurityContext     `json:"podSecurityContext,omitempty"`
-	SecurityContext    *corev1.SecurityContext        `json:"securityContext,omitempty"`
-	Replicas           int32                          `json:"replicas,omitempty"`
-	Metrics            *mariadv1alpha1.MariadbMetrics `json:"metrics,omitempty"`
-	Replication        *mariadv1alpha1.Replication    `json:"replication,omitempty"`
-	Storage            mariadv1alpha1.Storage         `json:"storage,omitempty"`
+	PodSecurityContext *mariadbv1alpha1.PodSecurityContext `json:"podSecurityContext,omitempty"`
+	SecurityContext    *mariadbv1alpha1.SecurityContext    `json:"securityContext,omitempty"`
+	Replicas           int32                               `json:"replicas,omitempty"`
+	Metrics            MariadbMetrics                      `json:"metrics,omitempty"`
+	Replication        *mariadbv1alpha1.Replication        `json:"replication,omitempty"`
+	Storage            mariadbv1alpha1.Storage             `json:"storage,omitempty"`
+}
+
+type MariadbMetrics struct {
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=true
+	Enabled bool `json:"enabled,omitempty"`
 }
 
 type SlurmdbdConfig struct {
@@ -577,6 +623,11 @@ type SlurmNodeWorker struct {
 	// +kubebuilder:validation:Required
 	Munge NodeContainer `json:"munge"`
 
+	// SupervisordConfigMapRefName is the name of the supervisord config, which runs in slurmd container
+	//
+	// +kubebuilder:validation:Optional
+	SupervisordConfigMapRefName string `json:"supervisordConfigMapRefName,omitempty"`
+
 	// Volumes represents the volume configurations for the worker node
 	//
 	// +kubebuilder:validation:Required
@@ -587,6 +638,13 @@ type SlurmNodeWorker struct {
 	// +kubebuilder:default="v2"
 	// +kubebuilder:validation:Enum="v1";"v2"
 	CgroupVersion string `json:"cgroupVersion,omitempty"`
+
+	// EnableGDRCopy driver propagation into containers (this feature must also be enabled in NVIDIA GPU operator)
+	// https://developer.nvidia.com/gdrcopy
+	//
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=false
+	EnableGDRCopy bool `json:"enableGDRCopy,omitempty"`
 }
 
 // SlurmNodeWorkerVolumes defines the volumes for the Slurm worker node

cmd/main.go

@@ -37,14 +37,15 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
-	mariadv1alpha1 "github.com/mariadb-operator/mariadb-operator/api/v1alpha1"
+	mariadbv1alpha1 "github.com/mariadb-operator/mariadb-operator/api/v1alpha1"
 	otelv1beta1 "github.com/open-telemetry/opentelemetry-operator/apis/v1beta1"
 	prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 
 	slurmv1 "nebius.ai/slurm-operator/api/v1"
 	"nebius.ai/slurm-operator/internal/check"
 	"nebius.ai/slurm-operator/internal/consts"
 	"nebius.ai/slurm-operator/internal/controller/clustercontroller"
+	webhookcorev1 "nebius.ai/slurm-operator/internal/webhook/v1"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -62,7 +63,7 @@ func init() {
 		utilruntime.Must(prometheusv1.AddToScheme(scheme))
 	}
 	if check.IsMariaDbCRDInstalled() {
-		utilruntime.Must(mariadv1alpha1.AddToScheme(scheme))
+		utilruntime.Must(mariadbv1alpha1.AddToScheme(scheme))
 	}
 
 	utilruntime.Must(slurmv1.AddToScheme(scheme))
@@ -193,6 +194,13 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", reflect.TypeOf(slurmv1.SlurmCluster{}).Name())
 		os.Exit(1)
 	}
+	// nolint:goconst
+	if os.Getenv("ENABLE_WEBHOOKS") != "false" {
+		if err = webhookcorev1.SetupSecretWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to create webhook", "webhook", "Secret")
+			os.Exit(1)
+		}
+	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {

config/certmanager/certificate.yaml

@@ -0,0 +1,35 @@
+# The following manifests contain a self-signed issuer CR and a certificate CR.
+# More document can be found at https://docs.cert-manager.io
+# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  labels:
+    app.kubernetes.io/name: slurm-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: selfsigned-issuer
+  namespace: system
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    app.kubernetes.io/name: certificate
+    app.kubernetes.io/instance: serving-cert
+    app.kubernetes.io/component: certificate
+    app.kubernetes.io/created-by: slurm-operator
+    app.kubernetes.io/part-of: slurm-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+  dnsNames:
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc
+  - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize

config/certmanager/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+- certificate.yaml
+
+configurations:
+- kustomizeconfig.yaml