Merge pull request #2651 from terascope/improve-external-ports

add external port object type

docs/configuration/clustering-k8s.md

@@ -4,7 +4,7 @@ sidebar_label: Kubernetes Clustering
 ---
 
 Teraslice supports the use of Kubernetes as a cluster manager. The following
-versions of Kuberenetes have been used:
+versions of Kubernetes have been used:
 
 * `1.19.*`
 * `1.18.*`
@@ -78,7 +78,7 @@ The Teraslice master node needs to be deployed in k8s by the user.  It should
 be deployed in a namespace mentioned above.  It should have a k8s service that
 exposes port `5678` for user interaction.  The cluster master will only show
 up in cluster state if it is deployed with the label `clusterName` set to the
-clustername modified as follows:
+cluster name modified as follows:
 
 ```js
 clusterName.replace(/[^a-zA-Z0-9_\-.]/g, '_').substring(0, 63)
@@ -108,7 +108,7 @@ Note that the `assets_volume` should also be mounted to your Teraslice master po
 
 Targets specified in the `execution_controller_targets` setting will result in
 required NodeAffinities and tolerations being added to the execution controller
-Jobs so that they can be targetted to specific parts of your k8s infrastructure.
+Jobs so that they can be targeted to specific parts of your k8s infrastructure.
 
 ## Teraslice Job Properties
 
@@ -118,7 +118,7 @@ Teraslice job definition.  These are outlined below.
 ### Ephemeral Storage
 
 If your Teraslice job uses a processor that needs temporary local storage that
-persists in the Kubernets Pod across container restarts, you can set the
+persists in the Kubernetes Pod across container restarts, you can set the
 Teraslice Job Property `ephemeral_storage` to `true` on your job as shown
 below.  This will create an [`emptyDir`](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir)
 style Ephemeral Volume accessible in your pod at the path `/ephemeral0`.
@@ -127,6 +127,35 @@ style Ephemeral Volume accessible in your pod at the path `/ephemeral0`.
     "ephemeral_storage": true
 ```
 
+### External Ports
+
+If for some reason you need to expose ports on your Teraslice workers in
+Kubernetes, you can do so with the `external_ports` job property.  You can
+provide either a simple number (that matches the port) or you can specify an
+object with `name` (a string) and `port` (a number) properties.
+
+```json
+    "external_ports": [
+        9090,
+        {"name": "metrics", "port": 3333}
+    ]
+```
+
+A job containing the `external_ports` shown above would result in the following
+Kubernetes snippet:
+
+```yaml
+    ports:
+    - containerPort: 9090
+      protocol: TCP
+    - containerPort: 3333
+      name: metrics
+      protocol: TCP
+```
+
+The reason this was added was to expose a Prometheus exporter on Teraslice
+worker pods.
+
 ### Labels
 
 Key value pairs added into a job's `labels` array, as shown below, will result
@@ -180,7 +209,7 @@ following resource requests and limits:
 
 These defaults can be overridden either in your Teraslice Job or in the
 Teraslice Master configuration.  Settings on your Teraslice Job override the
-settings in the Master configuration.  The behaviour of these two settings is
+settings in the Master configuration.  The behavior of these two settings is
 the same as the Worker settings with the exception of the default being applied
 in the Execution Controller case.
 
@@ -267,7 +296,7 @@ translated into the Kubernetes manifest.  Those details are described below.
 
 #### Details of the `required` constraint
 
-For each entry in `targets` without a `constraint` or if `contraint` is set to
+For each entry in `targets` without a `constraint` or if `constraint` is set to
 `required`, the pod spec will include an entry in the `matchExpressions` list
 under the `requiredDuringSchedulingIgnoredDuringExecution` affinity property.
 The example below is what you would get if you provided two `required` targets:

package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.75.0",
+    "version": "0.75.1",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

packages/job-components/src/interfaces/jobs.ts

@@ -106,7 +106,7 @@ export interface ValidatedJobConfig {
     /** This will only be available in the context of k8s */
     ephemeral_storage?: boolean;
     /** This will only be available in the context of k8s */
-    exposed_ports?: number[];
+    external_ports?: (number|ExternalPort)[];
     /** This will only be available in the context of k8s */
     memory?: number;
     /** This will only be available in the context of k8s */
@@ -122,6 +122,11 @@ export interface Targets {
     value: string;
 }
 
+export interface ExternalPort {
+    name: string;
+    containerPort: number
+}
+
 export interface Volume {
     name: string;
     path: string;

packages/job-components/src/job-schemas.ts

@@ -1,14 +1,16 @@
 import os from 'os';
 import convict from 'convict';
 import {
-    getField,
+    DataEncoding,
+    dataEncodings,
     flatten,
+    getField,
     getTypeOf,
+    hasOwn,
+    isNotNil,
+    isNumber,
     isPlainObject,
-    dataEncodings,
     isString,
-    DataEncoding,
-    isNotNil,
 } from '@terascope/utils';
 import { Context } from './interfaces';
 
@@ -110,7 +112,7 @@ export function jobSchema(context: Context): convict.Schema<any> {
             default: [],
             doc: `An array of apis to load and any configurations they require.
             Validated similar to operations, with the exception of no apis are required.
-            The _name property is required, and it is required to be unqiue
+            The _name property is required, and it is required to be unique
             but can be suffixed with a identifier by using the format "example:0",
             anything after the ":" is stripped out when searching for the file or folder.`,
             format(arr: any[]) {
@@ -164,7 +166,7 @@ export function jobSchema(context: Context): convict.Schema<any> {
         },
         labels: {
             default: null,
-            doc: 'An array of arrays containing key value pairs used to label kubetnetes resources.',
+            doc: 'An array of arrays containing key value pairs used to label kubernetes resources.',
             // TODO: Refactor this out as format, copied from env_vars
             format(obj: any[]) {
                 if (obj != null) {
@@ -208,7 +210,7 @@ export function jobSchema(context: Context): convict.Schema<any> {
     if (clusteringType === 'kubernetes') {
         schemas.targets = {
             default: [],
-            doc: 'array of key/value labels used for targetting teraslice jobs to nodes',
+            doc: 'array of key/value labels used for targeting teraslice jobs to nodes',
             format(arr: any[]) {
                 if (!Array.isArray(arr)) {
                     throw new Error('must be array');
@@ -254,17 +256,43 @@ export function jobSchema(context: Context): convict.Schema<any> {
                 // processor requires port X this code should throw an error.
                 if (arr != null) {
                     if (!Array.isArray(arr)) {
-                        throw new Error('external_ports is required to be an array');
+                        throw new Error('external_ports is required to be an array of numbers or objects like {name: \'myJob\', port: 9000}.');
                     }
-                    if (arr.includes(45680)) {
-                        throw new Error('Port 45680 cannot be included in external_ports, it is reserved by Teraslice.');
+                    for (const portValue of arr) {
+                        if (isNumber(portValue)) {
+                            if (portValue === 45680) {
+                                throw new Error('Port 45680 cannot be included in external_ports, it is reserved by Teraslice.');
+                            }
+                        } else if (isPlainObject(portValue)) {
+                            Object.entries(portValue).forEach(([key, val]) => {
+                                if (key == null || key === '') {
+                                    throw new Error('key must be not empty');
+                                }
+
+                                if (val == null || val === '') {
+                                    throw new Error(`value for key "${key}" must be not empty`);
+                                }
+                                if (hasOwn(portValue, 'name') && hasOwn(portValue, 'port')) {
+                                    if (!isNumber(portValue.port)) {
+                                        throw new Error('The port set on an external_ports object must be a number.');
+                                    }
+                                    if (portValue.name === '' || !isString(portValue.name)) {
+                                        throw new Error('The name set on an external ports object must be a non empty string.');
+                                    }
+                                } else {
+                                    throw new Error('An external_ports entry must be an object like {name: \'myJob\', port: 9000} or a number.');
+                                }
+                            });
+                        } else {
+                            throw new Error('An external_ports entry must be a number or an object like {name: \'myJob\', port: 9000}.');
+                        }
                     }
                 }
             }
         };
 
         schemas.memory = {
-            doc: 'memory, in bytes, to reserve per teraslice worker in kubernetes',
+            doc: 'memory, in bytes, to reserve per teraslice worker in Kubernetes',
             default: undefined,
             format: 'Number',
         };
@@ -341,7 +369,7 @@ export const opSchema: convict.Schema<any> = {
 export const apiSchema: convict.Schema<any> = {
     _name: {
         default: '',
-        doc: `The _name property is required, and it is required to be unqiue
+        doc: `The _name property is required, and it is required to be unique
         but can be suffixed with a identifier by using the format "example:0",
         anything after the ":" is stripped out when searching for the file or folder.`,
         format: 'required_String',

packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8sResource.js

@@ -6,6 +6,8 @@ const path = require('path');
 const barbe = require('barbe');
 const _ = require('lodash');
 
+const { isNumber } = require('@terascope/utils');
+
 const { safeEncode } = require('../../../../../utils/encoding_utils');
 const { setMaxOldSpaceViaEnv } = require('./utils');
 
@@ -183,9 +185,19 @@ class K8sResource {
 
     _setExternalPorts() {
         if (this.execution.external_ports) {
-            _.forEach(this.execution.external_ports, (port) => {
-                this.resource.spec.template.spec.containers[0].ports
-                    .push({ containerPort: port });
+            _.forEach(this.execution.external_ports, (portValue) => {
+                if (isNumber(portValue)) {
+                    this.resource.spec.template.spec.containers[0].ports
+                        .push({ containerPort: portValue });
+                } else {
+                    this.resource.spec.template.spec.containers[0].ports
+                        .push(
+                            {
+                                name: portValue.name,
+                                containerPort: portValue.port
+                            }
+                        );
+                }
             });
         }
     }

packages/teraslice/package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.75.0",
+    "version": "0.75.1",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {

packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8sResource-spec.js

@@ -691,7 +691,10 @@ describe('k8sResource', () => {
 
     describe('teraslice job with two valid external_ports set', () => {
         it('generates k8s worker deployment with containerPort on container', () => {
-            execution.external_ports = [9090, 9091];
+            execution.external_ports = [
+                9090,
+                { name: 'metrics', port: 9091 }
+            ];
             const kr = new K8sResource(
                 'deployments', 'worker', terasliceConfig, execution
             );
@@ -702,7 +705,7 @@ describe('k8sResource', () => {
                 .toEqual([
                     { containerPort: 45680 },
                     { containerPort: 9090 },
-                    { containerPort: 9091 }
+                    { name: 'metrics', containerPort: 9091 }
                 ]);
         });