Merge pull request #3252 from terascope/add-k8s-priority-class-support

Add k8s priority class support
terascope · Apr 8, 2022 · b31a03b · b31a03b
2 parents 5467eca + fa4fdc0
commit b31a03b
Show file tree

Hide file tree

Showing 20 changed files with 152 additions and 13 deletions.
diff --git a/docs/configuration/clustering-k8s.md b/docs/configuration/clustering-k8s.md
@@ -103,6 +103,7 @@ support k8s based Teraslice deployments.
 |  kubernetes_image_pull_secret  |                                                    Secret used to pull docker images from private repo                                                     | String  | optional |
 |   kubernetes_config_map_name   | Name of the configmap used by worker and execution_controller containers for config.  If this is not provided, the default will be `<CLUSTER_NAME>-worker` | String  | optional |
 |      kubernetes_namespace      |                                       Kubernetes Namespace that Teraslice will run in, default namespace: 'default'                                        | String  | optional |
+| kubernetes_priority_class_name |                       Priority class that the Teraslice master, execution controller, and stateful workers should run with                                 | String  | optional |
 | kubernetes_worker_antiaffinity |                                   If `true`, pod antiaffinity will be enabled for Teraslice workers, `false` by default                                    | Boolean | optional |
 |  memory_execution_controller   |                                         Memory resources to use for Execution Controller request and limit values                                          | Number  | optional |
 
@@ -112,6 +113,12 @@ Targets specified in the `execution_controller_targets` setting will result in
 required NodeAffinities and tolerations being added to the execution controller
 Jobs so that they can be targeted to specific parts of your k8s infrastructure.
 
+In order for the setting `kubernetes_priority_class_name` to be useful, you
+must create a Kubernetes `PriorityClass` with an appropriate priority for your
+Kubernetes cluster.  See the
+[Kubernetes `PriorityClass` documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/)
+for details.
+
 ## Teraslice Job Properties
 
 Support for Kubernetes based clustering adds additional properties to a
@@ -244,6 +251,25 @@ settings in the Master configuration.  The behavior of these two settings is
 the same as the Worker settings with the exception of the default being applied
 in the Execution Controller case.
 
+### Stateful Workers
+
+Teraslice jobs which use processors that maintain internal state might need
+special handling in Kubernetes.  To support this we have the job property
+`stateful`.  Setting it `stateful: true` in your Teraslice job will result the
+following things:
+
+* Teraslice workers for this job will have `priorityClassName` set equal to the
+`kubernetes_priority_class_name` setting.  This is meant to prevent preemption
+of the worker pods which could otherwise happen.
+* All of the Teraslice worker pods will have the Kubernetes label
+`job-property.teraslice.terascope.io/stateful: true`.
+
+This property is a boolean, so is simply set like:
+
+```json
+"stateful": true
+```
+
 ### Node Affinity and Tolerance Using Teraslice Job Targets
 
 If you need the workers (and execution controller) of your job to execute on

diff --git a/docs/configuration/overview.md b/docs/configuration/overview.md
@@ -70,6 +70,7 @@ teraslice:
 |              **kubernetes_image**               |              `String`              |  `"terascope/teraslice"`   |                             Specify a custom image name for kubernetes, this only applies to kubernetes systems                             |
 |        **kubernetes_image_pull_secret**         |              `String`              |             -              |                                Name of Kubernetes secret used to pull docker images from private repository                                 |
 |            **kubernetes_namespace**             |              `String`              |        `"default"`         |                               Specify a custom kubernetes namespace, this only applies to kubernetes systems                                |
+|       **kubernetes_priority_class_name**        |              `String`              |        -                   |                               Priority class that the Teraslice master, execution controller, and stateful workers should run with systems                                |
 |                   **master**                    |             `Boolean`              |          `false`           |                                     boolean for determining if cluster_master should live on this node                                      |
 |               **master_hostname**               |              `String`              |       `"localhost"`        |                         hostname where the cluster_master resides, used to notify all node_masters where to connect                         |
 |                   **memory**                    |              `Number`              |             -              |                                       memory, in bytes, to reserve per teraslice worker in kubernetes                                       |

diff --git a/docs/jobs/configuration.md b/docs/jobs/configuration.md
@@ -20,6 +20,7 @@ The first operation in the [operations](#operations) list, reads from a particul
 | `max_retries`         | Number of times a given slice of data will attempt to process before continuing on                                                                                                                                  | `Number`                | optional                                                                                                                                                                                                |
 | `slicers`             | Number of slicer functions that will chunk and prep the data for worker                                                                                                                                             | `Number`                | optional, defaults to 1                                                                                                                                                                                 |
 | `workers`             | Number of worker instances that will process data, depending on the nature of the operations you may choose to over subscribe the number of workers compared to the number of cpu's                                 | `Number`                | optional, defaults to 5, if the number of workers for the job is set above workers specified in system configuration, a warning is passed and the workers set in the system configuration will be used. |
+| `stateful`            | Indicates that the Teraslice worker maintains internal state, and must be handled differently.                                                                                                                                          | `Boolean`               | optional, defaults to `false`                                                                                                                                                                           |
 | `assets`              | An array of strings that are the id's for the corresponding assets zip files.                                                                                                                                       | `Array`                 | optional                                                                                                                                                                                                |
 | `operations`          | An array containing all the [operations](#operations) as well as their configurations. Typically the first is the reader/slicer.                                                                                    | `Array`                 | required                                                                                                                                                                                                |
 | `apis`                | An array containing all the [apis](#apis) as well as their configurations.                                                                                                                                          | `Array`                 | required                                                                                                                                                                                                |

diff --git a/examples/k8s/Makefile b/examples/k8s/Makefile
@@ -134,6 +134,15 @@ showauth: ## Show roles and roleBindings
 	kubectl get --namespace $(NAMESPACE) roles teraslice-all-$(NAMESPACE) -o yaml
 	kubectl get --namespace $(NAMESPACE) roleBindings teraslice-all-$(NAMESPACE) -o yaml
 
+priority: ## Setup Kubernetes Priority Class
+	kubectl apply -f priorityClass.yaml
+
+delete-priority: ## Remove Kubernetes Priority Class
+	kubectl delete priorityclasses high-priority
+
+showpriority:  ## Show Kubernetes Priority Class
+	kubectl get priorityclasses high-priority -o yaml
+
 configs: ## create the configmaps
 ifeq ($(TERASLICE_MODE),minikube)
 	yq eval ".teraslice.kubernetes_image = \"$(TERASLICE_K8S_IMAGE)\"" teraslice-worker.yaml.tpl | yq eval ".teraslice.kubernetes_namespace = \"$(NAMESPACE)\"" - > teraslice.yaml
@@ -160,7 +169,7 @@ build: ## builds docker images
 push: ## push final docker image
 	docker push $(TERASLICE_K8S_IMAGE)
 
-setup-all: namespace elasticsearch auth setup ## setup EVERYTHING
+setup-all: namespace elasticsearch auth priority setup ## setup EVERYTHING
 	earl aliases remove ${TERASLICE_ALIAS} 2> /dev/null || true
 	earl aliases add ${TERASLICE_ALIAS} ${TERASLICE_MASTER_URL}
 
@@ -193,6 +202,7 @@ register: ## creates asset and registers job
 	earl tjm register ${TERASLICE_ALIAS} example-job-labels.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-resource.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-separate-resources.json
+	earl tjm register ${TERASLICE_ALIAS} example-job-stateful.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-targets.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-volume.json
 
@@ -201,6 +211,7 @@ deregister: ## resets jobs
 	earl tjm reset example-job-labels.json || echo '* it is okay'
 	earl tjm reset example-job-resource.json || echo '* it is okay'
 	earl tjm reset example-job-separate-resources.json || echo '* it is okay'
+	earl tjm reset example-job-stateful.json || echo '* it is okay'
 	earl tjm reset example-job-targets.json || echo '* it is okay'
 	earl tjm reset example-job-volume.json || echo '* it is okay'
 

diff --git a/examples/k8s/example-job-stateful.json b/examples/k8s/example-job-stateful.json
@@ -0,0 +1,31 @@
+{
+    "name": "example-data-generator-job-stateful",
+    "lifecycle": "once",
+    "workers": 2,
+    "stateful": true,
+    "assets": [
+        "example",
+        "elasticsearch",
+        "standard"
+    ],
+    "ephemeral_storage": true,
+    "operations": [
+        {
+            "_op": "data_generator",
+            "size": 5000000
+        },
+        {
+            "_op": "example-op"
+        },
+        {
+            "_op": "delay",
+            "ms": 30000
+        },
+        {
+            "_op": "elasticsearch_bulk",
+            "index": "terak8s-example-data",
+            "type": "events",
+            "size": 5000
+        }
+    ]
+}
diff --git a/examples/k8s/priorityClass.yaml b/examples/k8s/priorityClass.yaml
@@ -0,0 +1,7 @@
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: high-priority
+value: 1000000
+globalDefault: false
+description: "This priority class is for Teraslice pods."
diff --git a/examples/k8s/teraslice-master.yaml.tpl b/examples/k8s/teraslice-master.yaml.tpl
@@ -20,6 +20,7 @@ teraslice:
     kubernetes_image_pull_secrets:
         - "docker-tera1-secret"
     kubernetes_namespace: "ts-dev1"
+    kubernetes_priority_class_name: 'high-priority'
     name: "ts-dev1"
     cpu: 1
     memory: 536870912
diff --git a/examples/k8s/teraslice-worker.yaml.tpl b/examples/k8s/teraslice-worker.yaml.tpl
@@ -18,6 +18,7 @@ teraslice:
     master_hostname: "teraslice-master"
     kubernetes_image: "teraslice-k8sdev"
     kubernetes_namespace: "ts-dev1"
+    kubernetes_priority_class_name: 'high-priority'
     name: "ts-dev1"
     cpu: 1
     memory: 536870912
diff --git a/packages/job-components/package.json b/packages/job-components/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/job-components",
     "displayName": "Job Components",
-    "version": "0.56.2",
+    "version": "0.56.3",
     "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
     "bugs": {

diff --git a/packages/job-components/src/interfaces/context.ts b/packages/job-components/src/interfaces/context.ts
@@ -37,6 +37,7 @@ export interface TerasliceConfig {
     kubernetes_image_pull_secret?: string|'';
     kubernetes_image?: string|'terascope/teraslice';
     kubernetes_namespace?: string|'default';
+    kubernetes_priority_class_name?: string|'';
     kubernetes_worker_antiaffinity?: boolean|false;
     master_hostname: string|'localhost';
     master: boolean|false;

diff --git a/packages/job-components/src/interfaces/jobs.ts b/packages/job-components/src/interfaces/jobs.ts
@@ -95,6 +95,7 @@ export interface ValidatedJobConfig {
     env_vars: { [key: string]: string };
     slicers: number;
     workers: number;
+    stateful?: boolean;
     /** This will only be available in the context of k8s */
     labels?: { [key: string]: string };
     /** This will only be available in the context of k8s */

diff --git a/packages/job-components/src/job-schemas.ts b/packages/job-components/src/job-schemas.ts
@@ -164,6 +164,11 @@ export function jobSchema(context: Context): convict.Schema<any> {
             doc: 'the number of workers dedicated for the job',
             format: 'positive_int'
         },
+        stateful: {
+            default: false,
+            doc: 'Indicates that the Teraslice worker maintains internal state, and must be handled differently',
+            format: Boolean,
+        },
         labels: {
             default: null,
             doc: 'An array of arrays containing key value pairs used to label kubernetes resources.',

diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-cli",
     "displayName": "Teraslice CLI",
-    "version": "0.48.2",
+    "version": "0.48.3",
     "description": "Command line manager for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "teraslice"
@@ -51,7 +51,7 @@
         "pretty-bytes": "^5.6.0",
         "prompts": "^2.4.2",
         "signale": "^1.4.0",
-        "teraslice-client-js": "^0.44.2",
+        "teraslice-client-js": "^0.44.3",
         "tmp": "^0.2.0",
         "tty-table": "^4.1.5",
         "yargs": "^17.3.1",

diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-client-js",
     "displayName": "Teraslice Client (JavaScript)",
-    "version": "0.44.2",
+    "version": "0.44.3",
     "description": "A Node.js client for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "elasticsearch",
@@ -31,7 +31,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/job-components": "^0.56.2",
+        "@terascope/job-components": "^0.56.3",
         "auto-bind": "^4.0.0",
         "got": "^11.8.3"
     },

diff --git a/packages/teraslice-op-test-harness/package.json b/packages/teraslice-op-test-harness/package.json
@@ -21,10 +21,10 @@
         "bluebird": "^3.7.2"
     },
     "devDependencies": {
-        "@terascope/job-components": "^0.56.2"
+        "@terascope/job-components": "^0.56.3"
     },
     "peerDependencies": {
-        "@terascope/job-components": ">=0.56.2"
+        "@terascope/job-components": ">=0.56.3"
     },
     "engines": {
         "node": "^12.22.0 || >=14.17.0",

diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json
@@ -36,10 +36,10 @@
         "fs-extra": "^10.0.0"
     },
     "devDependencies": {
-        "@terascope/job-components": "^0.56.2"
+        "@terascope/job-components": "^0.56.3"
     },
     "peerDependencies": {
-        "@terascope/job-components": ">=0.56.2"
+        "@terascope/job-components": ">=0.56.3"
     },
     "engines": {
         "node": "^12.22.0 || >=14.17.0",

diff --git a/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js b/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js
@@ -25,6 +25,7 @@ class K8sResource {
     constructor(resourceType, resourceName, terasliceConfig, execution) {
         this.execution = execution;
         this.jobLabelPrefix = 'job.teraslice.terascope.io';
+        this.jobPropertyLabelPrefix = 'job-property.teraslice.terascope.io';
         this.nodeType = resourceName;
         this.terasliceConfig = terasliceConfig;
 
@@ -54,6 +55,7 @@ class K8sResource {
         this._setImagePullSecret();
         this._setEphemeralStorage();
         this._setExternalPorts();
+        this._setPriorityClassName();
 
         if (resourceName === 'worker') {
             this._setWorkerAntiAffinity();
@@ -219,6 +221,25 @@ class K8sResource {
         }
     }
 
+    _setPriorityClassName() {
+        if (this.terasliceConfig.kubernetes_priority_class_name) {
+            if (this.nodeType === 'execution_controller') {
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
+                if (this.execution.stateful) {
+                    // eslint-disable-next-line max-len
+                    this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
+                }
+            }
+            if (this.nodeType === 'worker' && this.execution.stateful) {
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.spec.priorityClassName = this.terasliceConfig.kubernetes_priority_class_name;
+                // eslint-disable-next-line max-len
+                this.resource.spec.template.metadata.labels[`${this.jobPropertyLabelPrefix}/stateful`] = 'true';
+            }
+        }
+    }
+
     _setAssetsVolume() {
         if (this.terasliceConfig.assets_directory && this.terasliceConfig.assets_volume) {
             this.resource.spec.template.spec.volumes.push({

diff --git a/packages/teraslice/lib/config/schemas/system.js b/packages/teraslice/lib/config/schemas/system.js
@@ -295,6 +295,11 @@ const schema = {
         default: 'default',
         format: 'optional_String'
     },
+    kubernetes_priority_class_name: {
+        doc: 'Priority class that the Teraslice master, execution controller, and stateful workers should run with',
+        default: undefined,
+        format: 'optional_String'
+    },
     kubernetes_config_map_name: {
         doc: 'Specify the name of the Kubernetes ConfigMap used to configure worker pods',
         default: 'teraslice-worker',

diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
@@ -38,7 +38,7 @@
     },
     "dependencies": {
         "@terascope/elasticsearch-api": "^3.0.2",
-        "@terascope/job-components": "^0.56.2",
+        "@terascope/job-components": "^0.56.3",
         "@terascope/teraslice-messaging": "^0.27.1",
         "@terascope/utils": "^0.44.1",
         "async-mutex": "^0.3.2",