From c7c167da22657559ae2c330e3aa8ef7150300214 Mon Sep 17 00:00:00 2001 From: ameade <847570+ameade@users.noreply.github.com> Date: Fri, 14 Jan 2022 11:35:17 -0500 Subject: [PATCH] AdAdd Trident Controller plugin pod scheduling options to trident operator and helm chart This commit adds options on the TridentOrchestrator CRD to specify nodeSelectors and tolerations for the Trident controller service pod. --- cli/k8s_client/types.go | 40 +++--- cli/k8s_client/yaml_factory.go | 6 + cli/k8s_client/yaml_factory_test.go | 115 ++++++++++++++++++ .../templates/tridentorchestrator.yaml | 14 +++ helm/trident-operator/values.yaml | 6 + .../orchestrator/apis/netapp/v1/types.go | 44 +++---- .../orchestrator/installer/installer.go | 22 +++- 7 files changed, 205 insertions(+), 42 deletions(-) diff --git a/cli/k8s_client/types.go b/cli/k8s_client/types.go index 440071d37..134b9d325 100644 --- a/cli/k8s_client/types.go +++ b/cli/k8s_client/types.go @@ -140,25 +140,27 @@ type KubernetesClient interface { } type DeploymentYAMLArguments struct { - DeploymentName string `json:"deploymentName"` - TridentImage string `json:"tridentImage"` - AutosupportImage string `json:"autosupportImage"` - AutosupportProxy string `json:"autosupportProxy"` - AutosupportCustomURL string `json:"autosupportCustomURL"` - AutosupportSerialNumber string `json:"autosupportSerialNumber"` - AutosupportHostname string `json:"autosupportHostname"` - ImageRegistry string `json:"imageRegistry"` - LogFormat string `json:"logFormat"` - SnapshotCRDVersion string `json:"snapshotCRDVersion"` - ImagePullSecrets []string `json:"imagePullSecrets"` - Labels map[string]string `json:"labels"` - ControllingCRDetails map[string]string `json:"controllingCRDetails"` - Debug bool `json:"debug"` - UseIPv6 bool `json:"useIPv6"` - SilenceAutosupport bool `json:"silenceAutosupport"` - Version *utils.Version `json:"version"` - TopologyEnabled bool `json:"topologyEnabled"` - HTTPRequestTimeout string `json:"httpRequestTimeout"` + DeploymentName string `json:"deploymentName"` + TridentImage string `json:"tridentImage"` + AutosupportImage string `json:"autosupportImage"` + AutosupportProxy string `json:"autosupportProxy"` + AutosupportCustomURL string `json:"autosupportCustomURL"` + AutosupportSerialNumber string `json:"autosupportSerialNumber"` + AutosupportHostname string `json:"autosupportHostname"` + ImageRegistry string `json:"imageRegistry"` + LogFormat string `json:"logFormat"` + SnapshotCRDVersion string `json:"snapshotCRDVersion"` + ImagePullSecrets []string `json:"imagePullSecrets"` + Labels map[string]string `json:"labels"` + ControllingCRDetails map[string]string `json:"controllingCRDetails"` + Debug bool `json:"debug"` + UseIPv6 bool `json:"useIPv6"` + SilenceAutosupport bool `json:"silenceAutosupport"` + Version *utils.Version `json:"version"` + TopologyEnabled bool `json:"topologyEnabled"` + HTTPRequestTimeout string `json:"httpRequestTimeout"` + NodeSelector map[string]string `json:"nodeSelector"` + Tolerations []map[string]string `json:"tolerations"` } type DaemonsetYAMLArguments struct { diff --git a/cli/k8s_client/yaml_factory.go b/cli/k8s_client/yaml_factory.go index 97e38c664..813558993 100644 --- a/cli/k8s_client/yaml_factory.go +++ b/cli/k8s_client/yaml_factory.go @@ -380,6 +380,8 @@ func GetCSIDeploymentYAML(args *DeploymentYAMLArguments) string { deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "LABELS", constructLabels(args.Labels)) deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails)) deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "IMAGE_PULL_SECRETS", constructImagePullSecrets(args.ImagePullSecrets)) + deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector)) + deploymentYAML = replaceMultilineYAMLTag(deploymentYAML, "NODE_TOLERATIONS", constructTolerations(args.Tolerations)) return deploymentYAML } @@ -527,6 +529,8 @@ spec: nodeSelector: kubernetes.io/os: linux kubernetes.io/arch: amd64 + {NODE_SELECTOR} + {NODE_TOLERATIONS} volumes: - name: socket-dir emptyDir: @@ -682,6 +686,8 @@ spec: nodeSelector: kubernetes.io/os: linux kubernetes.io/arch: amd64 + {NODE_SELECTOR} + {NODE_TOLERATIONS} volumes: - name: socket-dir emptyDir: diff --git a/cli/k8s_client/yaml_factory_test.go b/cli/k8s_client/yaml_factory_test.go index 9afbb59b8..0dd1db62b 100644 --- a/cli/k8s_client/yaml_factory_test.go +++ b/cli/k8s_client/yaml_factory_test.go @@ -212,6 +212,121 @@ func TestValidateGetCSIDeploymentYAMLFail(t *testing.T) { } } +func TestGetCSIDeploymentYAML(t *testing.T) { + versions := []string{"1.17.0", "1.18.0", "1.19.1", "1.21.0"} + + for _, versionString := range versions { + version := utils.MustParseSemantic(versionString) + deploymentArgs := &DeploymentYAMLArguments{Version: version} + + yamlData := GetCSIDeploymentYAML(deploymentArgs) + _, err := yaml.YAMLToJSON([]byte(yamlData)) + if err != nil { + t.Fatalf("expected valid YAML for version %s", versionString) + } + } +} + +func TestGetCSIDeploymentYAML_NodeSelectors(t *testing.T) { + deploymentArgs := &DeploymentYAMLArguments{ + NodeSelector: map[string]string{"foo": "bar"}, + } + expectedNodeSelectorString := ` + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/arch: amd64 + foo: bar +` + + yamlData := GetCSIDeploymentYAML(deploymentArgs) + _, err := yaml.YAMLToJSON([]byte(yamlData)) + if err != nil { + t.Fatalf("expected valid YAML, got %s", yamlData) + } + assert.Contains(t, yamlData, expectedNodeSelectorString, fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData)) + + // Defaults + deploymentArgs = &DeploymentYAMLArguments{} + expectedNodeSelectorString = ` + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/arch: amd64 +` + + yamlData = GetCSIDeploymentYAML(deploymentArgs) + _, err = yaml.YAMLToJSON([]byte(yamlData)) + if err != nil { + t.Fatalf("expected valid YAML, got %s", yamlData) + } + assert.Contains(t, yamlData, expectedNodeSelectorString, fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData)) + + // Defaults used when empty list specified + deploymentArgs = &DeploymentYAMLArguments{ + NodeSelector: map[string]string{}, + } + expectedNodeSelectorString = ` + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/arch: amd64 +` + + yamlData = GetCSIDeploymentYAML(deploymentArgs) + _, err = yaml.YAMLToJSON([]byte(yamlData)) + if err != nil { + t.Fatalf("expected valid YAML, got %s", yamlData) + } + assert.Contains(t, yamlData, expectedNodeSelectorString, fmt.Sprintf("expected nodeSelector in final YAML: %s", yamlData)) +} + +func TestGetCSIDeploymentYAMLTolerations(t *testing.T) { + deploymentArgs := &DeploymentYAMLArguments{ + Tolerations: []map[string]string{ + {"key": "foo", "value": "bar", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "foo2", "value": "bar2", "operator": "Equals", "effect": "NoExecute", "tolerationSeconds": "20"}, + }, + } + expectedTolerationString := ` + tolerations: + - key: "foo" + value: "bar" + effect: "NoSchedule" + operator: "Exists" + - key: "foo2" + value: "bar2" + effect: "NoExecute" + operator: "Equals" + tolerationSeconds: 20 +` + + yamlData := GetCSIDeploymentYAML(deploymentArgs) + _, err := yaml.YAMLToJSON([]byte(yamlData)) + if err != nil { + t.Fatalf("expected valid YAML, got %s", yamlData) + } + assert.Contains(t, yamlData, expectedTolerationString, fmt.Sprintf("expected toleration in final YAML: %s", yamlData)) + + // Test empty tolerations specified + deploymentArgs = &DeploymentYAMLArguments{Tolerations: []map[string]string{}} + expectedTolerationString = ` + tolerations: [] +` + defaultTolerationString := ` + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" +` + + yamlData = GetCSIDeploymentYAML(deploymentArgs) + _, err = yaml.YAMLToJSON([]byte(yamlData)) + if err != nil { + t.Fatalf("expected valid YAML, got %s", yamlData) + } + assert.Contains(t, yamlData, expectedTolerationString, fmt.Sprintf("expected toleration in final YAML: %s", yamlData)) + assert.NotContains(t, yamlData, defaultTolerationString, fmt.Sprintf("expected default tolerations to not appear in final YAML: %s", yamlData)) +} + func TestGetCSIDaemonSetYAML(t *testing.T) { versions := []string{"1.17.0", "1.18.0", "1.21.0"} diff --git a/helm/trident-operator/templates/tridentorchestrator.yaml b/helm/trident-operator/templates/tridentorchestrator.yaml index a5f04b5f7..2b4e495bd 100644 --- a/helm/trident-operator/templates/tridentorchestrator.yaml +++ b/helm/trident-operator/templates/tridentorchestrator.yaml @@ -23,6 +23,20 @@ spec: {{- toYaml . | nindent 2 }} {{- end }} enableNodePrep: {{ include "trident.enableNodePrep" $ }} + {{- if .Values.tridentControllerPluginNodeSelector }} + controllerPluginNodeSelector: + {{- range $key, $value := .Values.tridentControllerPluginNodeSelector }} + {{ $key }}: "{{ $value }}" + {{- end}} + {{- end }} + {{- if .Values.tridentControllerPluginTolerations }} + controllerPluginTolerations: + {{- range $value := .Values.tridentControllerPluginTolerations }} + - {{- range $k, $v := $value }} + {{ $k }}: "{{ $v }}" + {{- end}} + {{- end}} + {{- end }} {{- if .Values.tridentNodePluginNodeSelector }} nodePluginNodeSelector: {{- range $key, $value := .Values.tridentNodePluginNodeSelector }} diff --git a/helm/trident-operator/values.yaml b/helm/trident-operator/values.yaml index 2ac30817f..698e1f520 100644 --- a/helm/trident-operator/values.yaml +++ b/helm/trident-operator/values.yaml @@ -19,6 +19,12 @@ tolerations: [] ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity affinity: {} +# tridentControllerPluginNodeSelector additional nodeSelectors for the Pod running the Trident Controller CSI Plugin +# tridentControllerPluginNodeSelector : {} + +# tridentControllerPluginTolerations overrides tolerations for the Pod running the Trident Controler CSI Plugin +# tridentControllerPluginTolerations: [] + # tridentNodePluginNodeSelector additional nodeSelectors for Pods running the Trident Node CSI Plugin # tridentNodePluginNodeSelector : {} diff --git a/operator/controllers/orchestrator/apis/netapp/v1/types.go b/operator/controllers/orchestrator/apis/netapp/v1/types.go index 5a4ae0a43..9f16d2a07 100644 --- a/operator/controllers/orchestrator/apis/netapp/v1/types.go +++ b/operator/controllers/orchestrator/apis/netapp/v1/types.go @@ -33,27 +33,29 @@ type TridentOrchestratorList struct { // TridentOrchestratorSpec defines the desired state of TridentOrchestrator type TridentOrchestratorSpec struct { - Debug bool `json:"debug"` - Namespace string `json:"namespace"` - IPv6 bool `json:"IPv6,omitempty"` - K8sTimeout int `json:"k8sTimeout,omitempty"` - HTTPRequestTimeout string `json:"httpRequestTimeout,omitempty"` - SilenceAutosupport bool `json:"silenceAutosupport,omitempty"` - AutosupportImage string `json:"autosupportImage,omitempty"` - AutosupportProxy string `json:"autosupportProxy,omitempty"` - AutosupportSerialNumber string `json:"autosupportSerialNumber,omitempty"` - AutosupportHostname string `json:"autosupportHostname,omitempty"` - Uninstall bool `json:"uninstall,omitempty"` - LogFormat string `json:"logFormat,omitempty"` - ProbePort *int64 `json:"probePort,omitempty"` - TridentImage string `json:"tridentImage,omitempty"` - ImageRegistry string `json:"imageRegistry,omitempty"` - KubeletDir string `json:"kubeletDir,omitempty"` - Wipeout []string `json:"wipeout,omitempty"` - ImagePullSecrets []string `json:"imagePullSecrets,omitempty"` - EnableNodePrep bool `json:"enableNodePrep,omitempty"` - NodePluginNodeSelector map[string]string `json:"nodePluginNodeSelector,omitempty"` - NodePluginTolerations []Toleration `json:"nodePluginTolerations,omitempty"` + Debug bool `json:"debug"` + Namespace string `json:"namespace"` + IPv6 bool `json:"IPv6,omitempty"` + K8sTimeout int `json:"k8sTimeout,omitempty"` + HTTPRequestTimeout string `json:"httpRequestTimeout,omitempty"` + SilenceAutosupport bool `json:"silenceAutosupport,omitempty"` + AutosupportImage string `json:"autosupportImage,omitempty"` + AutosupportProxy string `json:"autosupportProxy,omitempty"` + AutosupportSerialNumber string `json:"autosupportSerialNumber,omitempty"` + AutosupportHostname string `json:"autosupportHostname,omitempty"` + Uninstall bool `json:"uninstall,omitempty"` + LogFormat string `json:"logFormat,omitempty"` + ProbePort *int64 `json:"probePort,omitempty"` + TridentImage string `json:"tridentImage,omitempty"` + ImageRegistry string `json:"imageRegistry,omitempty"` + KubeletDir string `json:"kubeletDir,omitempty"` + Wipeout []string `json:"wipeout,omitempty"` + ImagePullSecrets []string `json:"imagePullSecrets,omitempty"` + EnableNodePrep bool `json:"enableNodePrep,omitempty"` + ControllerPluginNodeSelector map[string]string `json:"controllerPluginNodeSelector,omitempty"` + ControllerPluginTolerations []Toleration `json:"controllerPluginTolerations,omitempty"` + NodePluginNodeSelector map[string]string `json:"nodePluginNodeSelector,omitempty"` + NodePluginTolerations []Toleration `json:"nodePluginTolerations,omitempty"` } // Toleration diff --git a/operator/controllers/orchestrator/installer/installer.go b/operator/controllers/orchestrator/installer/installer.go index a37169e17..49de94c58 100644 --- a/operator/controllers/orchestrator/installer/installer.go +++ b/operator/controllers/orchestrator/installer/installer.go @@ -74,8 +74,10 @@ var ( appLabelKey string appLabelValue string - nodePluginNodeSelector map[string]string - nodePluginTolerations []netappv1.Toleration + controllerPluginNodeSelector map[string]string + controllerPluginTolerations []netappv1.Toleration + nodePluginNodeSelector map[string]string + nodePluginTolerations []netappv1.Toleration CRDnames = []string{ BackendCRDName, @@ -342,6 +344,12 @@ func (i *Installer) setInstallationParams( appLabelKey = TridentCSILabelKey appLabelValue = TridentCSILabelValue + if cr.Spec.ControllerPluginNodeSelector != nil { + controllerPluginNodeSelector = cr.Spec.ControllerPluginNodeSelector + } + if cr.Spec.ControllerPluginTolerations != nil { + controllerPluginTolerations = cr.Spec.ControllerPluginTolerations + } if cr.Spec.NodePluginNodeSelector != nil { nodePluginNodeSelector = cr.Spec.NodePluginNodeSelector } @@ -1047,6 +1055,14 @@ func (i *Installer) createOrPatchTridentDeployment( createDeployment = true } + var tolerations []map[string]string + if controllerPluginTolerations != nil { + tolerations = make([]map[string]string, 0) + for _, t := range controllerPluginTolerations { + tolerations = append(tolerations, t.GetMap()) + } + } + if err = i.client.RemoveMultipleDeployments(unwantedDeployments); err != nil { return fmt.Errorf("failed to remove unwanted Trident deployments; %v", err) } @@ -1073,6 +1089,8 @@ func (i *Installer) createOrPatchTridentDeployment( Version: i.client.ServerVersion(), TopologyEnabled: topologyEnabled, HTTPRequestTimeout: httpTimeout, + NodeSelector: controllerPluginNodeSelector, + Tolerations: tolerations, } newDeploymentYAML := k8sclient.GetCSIDeploymentYAML(deploymentArgs)