Merge pull request #3157 from terascope/k8s-improvements-summer-2021-2

[k8s] add support for separate req/lim
terascope · Apr 1, 2022 · 5467eca · 5467eca
2 parents a992562 + 4b7095b
commit 5467eca
Show file tree

Hide file tree

Showing 17 changed files with 274 additions and 152 deletions.
diff --git a/docs/configuration/clustering-k8s.md b/docs/configuration/clustering-k8s.md
@@ -6,6 +6,8 @@ sidebar_label: Kubernetes Clustering
 Teraslice supports the use of Kubernetes as a cluster manager. The following
 versions of Kubernetes have been used:
 
+* `1.22.*`
+* `1.20.*`
 * `1.19.*`
 * `1.18.*`
 * `1.17.*`
@@ -175,8 +177,37 @@ labels that Teraslice uses.
 
 It is possible to set CPU and memory resource constraints for your Teraslice
 Workers that translate to Kubernetes resource constraints.  Resources for
-Execution Controllers are handled separately and described below.  Currently you
-can specify optional integer values on your job or in the Teraslice master
+Execution Controllers are handled separately and described below.
+
+#### New Method for Setting Resources
+
+The new method for setting CPU and memory resources on Teraslice workers allows
+you to explicitly set the CPU and memory requests and limits separately on the
+Teraslice Job, which will result in the Kubernetes deployment for the Teraslice
+workers having the corresponding resource set.  You may set any combination of
+the following resources on your job or omit them entirely.
+
+```json
+"resources_requests_cpu": 0.25,
+"resources_limits_cpu": 1,
+"resources_requests_memory": 128000000,
+"resources_limits_memory": 536870912,
+```
+
+The cpu settings are in [Kubernetes CPU Units](https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units)
+and the memory settings are in bytes.
+
+Note: The old method of setting just `cpu` or `memory` on the job or in the
+Teraslice master config file will be ignored if any of the new `resource_*` job
+properties are configured.
+
+#### Old Method for Setting Resources
+
+**DEPRECATED** This older method of specifying CPU and memory resources should
+be avoided and will be removed in future versions.  They are only present
+briefly to aid in the migration of jobs.
+
+You can specify optional integer values on your job or in the Teraslice master
 configuration as shown below. The `cpu` setting is in vcores and the `memory`
 setting is in bytes.  Teraslice `cpu` and `memory` settings on your Teraslice
 Job override any settings in the master configuration.  Both are optional,
@@ -417,9 +448,7 @@ cd examples/k8s
 export NAMESPACE=ts-dev1
 export TERASLICE_K8S_IMAGE=teraslice-k8sdev:1
 export TERASLICE_MODE=minikube
-# Set the version of Kubernetes you want to run
-export KUBERNETES_VERSION=v1.19.7
-minikube start --memory 4096 --cpus 4 --kubernetes-version=${KUBERNETES_VERSION}
+minikube start --memory 4096 --cpus 4
 eval $(minikube docker-env)
 make build
 make setup-all

diff --git a/examples/k8s/Makefile b/examples/k8s/Makefile
@@ -192,13 +192,15 @@ register: ## creates asset and registers job
 	earl tjm register ${TERASLICE_ALIAS} example-job.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-labels.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-resource.json
+	earl tjm register ${TERASLICE_ALIAS} example-job-separate-resources.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-targets.json
 	earl tjm register ${TERASLICE_ALIAS} example-job-volume.json
 
 deregister: ## resets jobs
 	earl tjm reset example-job.json || echo '* it is okay'
 	earl tjm reset example-job-labels.json || echo '* it is okay'
 	earl tjm reset example-job-resource.json || echo '* it is okay'
+	earl tjm reset example-job-separate-resources.json || echo '* it is okay'
 	earl tjm reset example-job-targets.json || echo '* it is okay'
 	earl tjm reset example-job-volume.json || echo '* it is okay'
 

diff --git a/examples/k8s/example-job-separate-resources.json b/examples/k8s/example-job-separate-resources.json
@@ -0,0 +1,29 @@
+{
+    "name": "example-data-generator-job-separate-resources",
+    "lifecycle": "once",
+    "workers": 2,
+    "assets": [
+        "example",
+        "elasticsearch",
+        "standard"
+    ],
+    "resources_requests_cpu": 0.25,
+    "resources_limits_cpu": 1,
+    "resources_requests_memory": 128000000,
+    "resources_limits_memory": 536870912,
+    "operations": [
+        {
+            "_op": "data_generator",
+            "size": 5000000
+        },
+        {
+            "_op": "example-op"
+        },
+        {
+            "_op": "elasticsearch_bulk",
+            "index": "terak8s-example-data",
+            "type": "events",
+            "size": 5000
+        }
+    ]
+}
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.78.0",
+    "version": "0.79.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

diff --git a/packages/data-mate/src/function-configs/date/setTimezone.ts b/packages/data-mate/src/function-configs/date/setTimezone.ts
@@ -3,6 +3,7 @@ import {
     isNumber,
     isString,
     setTimezoneFP,
+    timezoneToOffset,
     toISO8601
 } from '@terascope/utils';
 import {
@@ -62,8 +63,7 @@ export const setTimezoneConfig: FieldTransformConfig<SetTimezoneArgs> = {
             },
             field: 'testField',
             input: '2020-02-14T20:45:30.091Z',
-            // FIXME this test breaks during daylight savings
-            output: [new Date('2020-02-14T20:45:30.091Z').getTime(), 1 * 60],
+            output: [new Date('2020-02-14T20:45:30.091Z').getTime(), timezoneToOffset('Europe/Paris')],
             serialize_output: toISO8601
         }
     ],

diff --git a/packages/job-components/package.json b/packages/job-components/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/job-components",
     "displayName": "Job Components",
-    "version": "0.56.1",
+    "version": "0.56.2",
     "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme",
     "bugs": {

diff --git a/packages/job-components/src/config-validators.ts b/packages/job-components/src/config-validators.ts
@@ -64,5 +64,15 @@ export function validateJobConfig<T>(
         throw new Error(`Validation failed for job config: ${inputConfig.name} - ${err.message}`);
     }
 
-    return config.getProperties();
+    const jobProperties = config.getProperties();
+
+    if ((jobProperties.cpu && jobProperties.resources_limits_cpu)
+        || (jobProperties.cpu && jobProperties.resources_requests_cpu)
+        || (jobProperties.memory && jobProperties.resources_limits_memory)
+        || (jobProperties.memory && jobProperties.resources_requests_memory)
+    ) {
+        throw new Error(`Validation failed for job config: ${inputConfig.name} - cpu/memory can't be mixed with resource settings of the same type.`);
+    }
+
+    return jobProperties;
 }
diff --git a/packages/job-components/src/interfaces/jobs.ts b/packages/job-components/src/interfaces/jobs.ts
@@ -112,6 +112,14 @@ export interface ValidatedJobConfig {
     /** This will only be available in the context of k8s */
     memory_execution_controller?: number;
     /** This will only be available in the context of k8s */
+    resources_requests_cpu?: number;
+    /** This will only be available in the context of k8s */
+    resources_requests_memory?: number;
+    /** This will only be available in the context of k8s */
+    resources_limits_cpu?: number;
+    /** This will only be available in the context of k8s */
+    resources_limits_memory?: number;
+    /** This will only be available in the context of k8s */
     volumes?: Volume[];
     /** This will only be available in the context of k8s */
     kubernetes_image?: string;

diff --git a/packages/job-components/src/job-schemas.ts b/packages/job-components/src/job-schemas.ts
@@ -303,6 +303,30 @@ export function jobSchema(context: Context): convict.Schema<any> {
             format: 'Number',
         };
 
+        schemas.resources_requests_cpu = {
+            doc: 'kubernetes CPU request, in cores, to set on Teraslice workers',
+            default: undefined,
+            format: 'Number'
+        };
+
+        schemas.resources_requests_memory = {
+            doc: 'kubernetes memory request, in bytes, to set on Teraslice workers',
+            default: undefined,
+            format: 'Number'
+        };
+
+        schemas.resources_limits_cpu = {
+            doc: 'kubernetes CPU limit, in cores, to set on Teraslice workers',
+            default: undefined,
+            format: 'Number'
+        };
+
+        schemas.resources_limits_memory = {
+            doc: 'kubernetes memory limit, in bytes, to set on Teraslice workers',
+            default: undefined,
+            format: 'Number'
+        };
+
         schemas.volumes = {
             default: [],
             doc: 'array of volumes to be mounted by job workers',

diff --git a/packages/job-components/test/config-validators-spec.ts b/packages/job-components/test/config-validators-spec.ts
@@ -598,7 +598,7 @@ describe('when validating k8s clustering', () => {
     const context = new TestContext('teraslice-operations');
     context.sysconfig.teraslice.cluster_manager_type = 'kubernetes';
 
-    describe('when passed a jobConfig with resources', () => {
+    describe('when passed a jobConfig with cpu and memory', () => {
         it('should return a completed and valid jobConfig', () => {
             const schema = jobSchema(context);
             const job = {
@@ -620,6 +620,58 @@ describe('when validating k8s clustering', () => {
         });
     });
 
+    describe('when passed a jobConfig with cpu and memory resources', () => {
+        it('should return a completed and valid jobConfig', () => {
+            const schema = jobSchema(context);
+            const job = {
+                resources_requests_cpu: 1,
+                resources_requests_memory: 805306368,
+                resources_limits_cpu: 1.5,
+                resources_limits_memory: 905306368,
+                operations: [
+                    {
+                        _op: 'noop',
+                    },
+                    {
+                        _op: 'noop',
+                    },
+                ],
+            };
+
+            const jobConfig = validateJobConfig(schema, job);
+            expect(jobConfig.resources_requests_cpu).toEqual(job.resources_requests_cpu);
+            expect(jobConfig.resources_requests_memory).toEqual(job.resources_requests_memory);
+            expect(jobConfig.resources_limits_cpu).toEqual(job.resources_limits_cpu);
+            expect(jobConfig.resources_limits_memory).toEqual(job.resources_limits_memory);
+        });
+    });
+
+    describe('when passed a jobConfig with old and new cpu and memory resources', () => {
+        it('should throw an exception', () => {
+            const schema = jobSchema(context);
+            const job = {
+                cpu: 1,
+                memory: 805306368,
+                resources_requests_cpu: 1,
+                resources_limits_cpu: 1.5,
+                resources_requests_memory: 805306368,
+                resources_limits_memory: 905306368,
+                operations: [
+                    {
+                        _op: 'noop',
+                    },
+                    {
+                        _op: 'noop',
+                    },
+                ],
+            };
+
+            expect(() => {
+                validateJobConfig(schema, job);
+            }).toThrowError('Validation failed for job config: undefined - cpu/memory can\'t be mixed with resource settings of the same type.');
+        });
+    });
+
     describe('when passed a jobConfig with targets', () => {
         it('should return a completed and valid jobConfig', () => {
             const schema = jobSchema(context);

diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-cli",
     "displayName": "Teraslice CLI",
-    "version": "0.48.1",
+    "version": "0.48.2",
     "description": "Command line manager for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "teraslice"
@@ -51,7 +51,7 @@
         "pretty-bytes": "^5.6.0",
         "prompts": "^2.4.2",
         "signale": "^1.4.0",
-        "teraslice-client-js": "^0.44.1",
+        "teraslice-client-js": "^0.44.2",
         "tmp": "^0.2.0",
         "tty-table": "^4.1.5",
         "yargs": "^17.3.1",

diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-client-js",
     "displayName": "Teraslice Client (JavaScript)",
-    "version": "0.44.1",
+    "version": "0.44.2",
     "description": "A Node.js client for teraslice jobs, assets, and cluster references.",
     "keywords": [
         "elasticsearch",
@@ -31,7 +31,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/job-components": "^0.56.1",
+        "@terascope/job-components": "^0.56.2",
         "auto-bind": "^4.0.0",
         "got": "^11.8.3"
     },

diff --git a/packages/teraslice-op-test-harness/package.json b/packages/teraslice-op-test-harness/package.json
@@ -21,10 +21,10 @@
         "bluebird": "^3.7.2"
     },
     "devDependencies": {
-        "@terascope/job-components": "^0.56.1"
+        "@terascope/job-components": "^0.56.2"
     },
     "peerDependencies": {
-        "@terascope/job-components": ">=0.56.1"
+        "@terascope/job-components": ">=0.56.2"
     },
     "engines": {
         "node": "^12.22.0 || >=14.17.0",

diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json
@@ -36,10 +36,10 @@
         "fs-extra": "^10.0.0"
     },
     "devDependencies": {
-        "@terascope/job-components": "^0.56.1"
+        "@terascope/job-components": "^0.56.2"
     },
     "peerDependencies": {
-        "@terascope/job-components": ">=0.56.1"
+        "@terascope/job-components": ">=0.56.2"
     },
     "engines": {
         "node": "^12.22.0 || >=14.17.0",