Merge pull request #3584 from terascope/s3-caCertificate-option

Add caCertificate option for s3 connector

package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "1.1.3",
+    "version": "1.2.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

packages/terafoundation/package.json

@@ -1,7 +1,7 @@
 {
     "name": "terafoundation",
     "displayName": "Terafoundation",
-    "version": "0.59.1",
+    "version": "0.60.0",
     "description": "A Clustering and Foundation tool for Terascope Tools",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme",
     "bugs": {
@@ -27,7 +27,7 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/file-asset-apis": "^0.12.2",
+        "@terascope/file-asset-apis": "^0.13.0",
         "@terascope/types": "^0.15.0",
         "@terascope/utils": "^0.57.0",
         "aws-sdk": "^2.1401.0",

packages/terafoundation/src/connector-utils.ts

@@ -1,5 +1,6 @@
 import path from 'path';
 import { TSError, parseError, Logger } from '@terascope/utils';
+import { Terafoundation } from '@terascope/types';
 
 type ErrorResult = {
     filePath: string;
@@ -25,6 +26,14 @@ function requireConnector(filePath: string, errors: ErrorResult[]) {
         valid = false;
     }
 
+    if (mod && mod.validate_config && typeof mod.validate_config !== 'function') {
+        errors.push({
+            filePath,
+            message: `Connector ${filePath} validate_config must be a function`,
+        });
+        valid = false;
+    }
+
     if (mod && typeof mod.create !== 'function') {
         errors.push({
             filePath,
@@ -97,15 +106,17 @@ export function getConnectorModule(name: string, reason: string): any {
     return null;
 }
 
-export function getConnectorSchema(name: string): Record<string, any> {
+export function getConnectorSchemaAndValFn<S>(
+    name: string
+): Terafoundation.Initializers<S> {
     const reason = `Could not retrieve schema code for: ${name}\n`;
 
     const mod = getConnectorModule(name, reason);
     if (!mod) {
         console.warn(`[WARNING] ${reason}`);
-        return {};
+        return { schema: {} };
     }
-    return mod.config_schema();
+    return { schema: mod.config_schema(), validatorFn: mod.validate_config };
 }
 
 export function createConnection(

packages/terafoundation/src/connectors/s3.ts

@@ -1,11 +1,12 @@
 import { Logger } from '@terascope/utils';
-import { createS3Client } from '@terascope/file-asset-apis';
+import { createS3Client, S3ClientConfig } from '@terascope/file-asset-apis';
+import { Terafoundation } from '@terascope/types';
 
 export default {
     create() {
         throw new Error('s3 does not support the deprecated "create" method, please use file-assets >= v2.4.0');
     },
-    async createClient(customConfig: Record<string, any>, logger: Logger) {
+    async createClient(customConfig: S3ClientConfig, logger: Logger) {
         const client = await createS3Client(customConfig, logger);
         return { client, logger };
     },
@@ -35,18 +36,18 @@ export default {
                 default: 3,
                 format: Number
             },
-            maxRedirects: {
-                doc: '',
-                default: 10,
-                format: Number
-            },
             sslEnabled: {
                 doc: '',
                 default: true,
                 format: Boolean
             },
             certLocation: {
-                doc: 'Location of ssl cert. Must be provided if `sslEnabled` is true',
+                doc: 'DEPRECATED - use caCertificate. Location of ssl cert.',
+                default: '',
+                format: String
+            },
+            caCertificate: {
+                doc: 'A string containing a single or multiple ca certificates',
                 default: '',
                 format: String
             },
@@ -61,5 +62,22 @@ export default {
                 format: Boolean
             }
         };
+    },
+    validate_config<S>(
+        subconfig: S3ClientConfig,
+        _sysconfig: Terafoundation.SysConfig<S>
+    ): void {
+        const caCertExists: boolean = (subconfig.caCertificate?.length !== 0);
+        const certLocationExists: boolean = (subconfig.certLocation?.length !== 0);
+        if (caCertExists && certLocationExists) {
+            throw new Error('"caCertificate" and "certLocation" contradict.\n'
+            + '  Use only one or the other.');
+        } else if (
+            (caCertExists && !subconfig.sslEnabled)
+            || (certLocationExists && !subconfig.sslEnabled)
+        ) {
+            throw new Error('A certificate is provided but sslEnabled is set to "false".\n'
+                + '  Set sslEnabled to "true" or don\'t provide a certificate.');
+        }
     }
 };

packages/terafoundation/src/schema.ts

@@ -5,8 +5,8 @@ import { Terafoundation } from '@terascope/types';
 const workerCount = cpus().length;
 const DEFAULT_ASSET_STORAGE_CONNECTION_TYPE = 'elasticsearch-next';
 
-export function foundationSchema(sysconfig: Terafoundation.SysConfig<any>): convict.Schema<any> {
-    const schema: convict.Schema<any> = {
+export function foundationSchema() {
+    const schema: convict.Schema<Record<string, any>> = {
         environment: {
             doc: 'If set to `production`, console logging will be disabled and logs will be sent to a file',
             default: 'development',
@@ -74,59 +74,58 @@ export function foundationSchema(sysconfig: Terafoundation.SysConfig<any>): conv
         asset_storage_connection_type: {
             doc: 'Name of the connection type used to store assets',
             default: DEFAULT_ASSET_STORAGE_CONNECTION_TYPE,
-            format(connectionTypeName: any): void {
-                const validConnectionTypes = ['elasticsearch-next', 's3'];
-                const connectionTypesPresent = Object.keys(sysconfig.terafoundation.connectors);
-                if (!connectionTypesPresent.includes(connectionTypeName)) {
-                    throw new Error('asset_storage_connection_type not found in terafoundation.connectors');
-                }
-                if (!validConnectionTypes.includes(connectionTypeName)) {
-                    throw new Error(`Invalid asset_storage_connection_type. Valid types: ${validConnectionTypes.toString()}`);
-                }
-            }
+            format: String
         },
         asset_storage_connection: {
             doc: 'Name of the connection used to store assets.',
             default: 'default',
-            format(connectionName: any): void {
-                let connectionType: string;
-                if (sysconfig.terafoundation.asset_storage_connection_type) {
-                    connectionType = sysconfig.terafoundation.asset_storage_connection_type;
-                } else {
-                    connectionType = DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
-                }
-
-                const connectionsPresent = Object.keys(sysconfig.terafoundation.connectors[`${connectionType}`]);
-                /// Check to make sure the asset_storage_connection exists inside the connector
-                /// Exclude elasticsearch as this connection type does not utilize this value
-                if (
-                    !connectionsPresent.includes(connectionName)
-                    && !connectionType.includes('elasticsearch-next')
-                ) {
-                    throw new Error(`${connectionName} not found in terafoundation.connectors.${connectionType}`);
-                }
-            }
+            format: String
         },
         asset_storage_bucket: {
             doc: 'Name of S3 bucket used to store assets. Can only be used if "asset_storage_connection_type" is "s3".',
             default: undefined,
-            format(bucketName: any): void {
-                let connectionType;
-                if (sysconfig.terafoundation.asset_storage_connection_type) {
-                    connectionType = sysconfig.terafoundation.asset_storage_connection_type;
-                } else {
-                    connectionType = DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
-                }
-                if (typeof bucketName !== 'string') {
-                    throw new Error('asset_storage_bucket must be a string');
-                }
-                if (connectionType !== 's3') {
-                    throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"');
-                }
-                // TODO: add regex to check if valid bucket name
-            }
+            format: String
         }
     };
 
     return schema;
 }
+
+export function foundationValidatorFn<S>(
+    subconfig: Record<string, any>,
+    _sysconfig: Terafoundation.SysConfig<S>
+): void {
+    const typedSubconfig = subconfig as Terafoundation.Foundation;
+    const connectionType = typedSubconfig.asset_storage_connection_type
+        || DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
+    const connectionTypesPresent = Object.keys(typedSubconfig.connectors);
+    const validConnectionTypes = ['elasticsearch-next', 's3'];
+
+    // checks on asset_storage_connection_type
+    if (!validConnectionTypes
+        .includes(connectionType)) {
+        throw new Error(`Invalid asset_storage_connection_type. Valid types: ${validConnectionTypes.toString()}`);
+    }
+    if (!connectionTypesPresent
+        .includes(connectionType)) {
+        throw new Error('asset_storage_connection_type not found in terafoundation.connectors');
+    }
+
+    // checks on asset_storage_connection
+    const connectionsPresent = Object.keys(typedSubconfig.connectors[`${connectionType}`]);
+    /// Check to make sure the asset_storage_connection exists inside the connector
+    /// Exclude elasticsearch as this connection type does not utilize this value
+    if (
+        connectionType !== 'elasticsearch-next'
+        && typedSubconfig.asset_storage_connection
+        && !connectionsPresent.includes(typedSubconfig.asset_storage_connection)
+    ) {
+        throw new Error(`${typedSubconfig.asset_storage_connection} not found in terafoundation.connectors.${connectionType}`);
+    }
+
+    // checks on asset_storage_bucket
+    if (typedSubconfig.asset_storage_bucket && connectionType !== 's3') {
+        throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"');
+    }
+    // TODO: add regex to check if valid bucket name
+}

packages/terafoundation/src/validate-configs.ts

@@ -2,14 +2,15 @@ import os from 'os';
 import convict, { addFormats } from 'convict';
 import {
     TSError, isFunction, isPlainObject,
-    isEmpty, concat, PartialDeep
+    isEmpty, concat, PartialDeep, cloneDeep
 } from '@terascope/utils';
 // @ts-expect-error no types
 import convict_format_with_validator from 'convict-format-with-validator';
 // @ts-expect-error no types
 import convict_format_with_moment from 'convict-format-with-moment';
-import { getConnectorSchema } from './connector-utils';
-import { foundationSchema } from './schema';
+import { Terafoundation } from '@terascope/types';
+import { getConnectorSchemaAndValFn } from './connector-utils';
+import { foundationSchema, foundationValidatorFn } from './schema';
 import * as i from './interfaces';
 
 addFormats(convict_format_with_validator);
@@ -43,9 +44,13 @@ function validateConfig(
 function extractSchema<S>(
     fn: any,
     sysconfig: PartialDeep<i.FoundationSysConfig<S>>
-): Record<string, any> {
+): Terafoundation.Schema<Record<string, any>> {
     if (isFunction(fn)) {
-        return fn(sysconfig);
+        const result = fn(sysconfig);
+        if (result.schema) {
+            return result.schema;
+        }
+        return result;
     }
     if (isPlainObject(fn)) {
         return fn;
@@ -54,6 +59,23 @@ function extractSchema<S>(
     return {};
 }
 
+function extractValidatorFn<S>(
+    fn: any,
+    sysconfig: PartialDeep<i.FoundationSysConfig<S>>
+): Terafoundation.ValidatorFn<S> | undefined {
+    if (isFunction(fn)) {
+        const result = fn(sysconfig);
+        if (result.validatorFn) {
+            return result.validatorFn;
+        }
+    }
+    if (isPlainObject(fn)) {
+        return fn.validatorFn;
+    }
+
+    return undefined;
+}
+
 /**
  * @param cluster the nodejs cluster metadata
  * @param config the config object passed to the library terafoundation
@@ -76,8 +98,10 @@ export default function validateConfigs<
         throw new Error('Terafoundation requires a valid system configuration');
     }
 
+    const listOfValidations: Record<string, Terafoundation.ValidationObj<S>> = {};
     const schema = extractSchema(config.config_schema, sysconfig);
-    schema.terafoundation = foundationSchema(sysconfig);
+    schema.terafoundation = foundationSchema();
+
     const result: any = {};
 
     if (config.schema_formats) {
@@ -90,25 +114,46 @@ export default function validateConfigs<
     for (const schemaKey of schemaKeys) {
         const subSchema = schema[schemaKey] || {};
         const subConfig: Record<string, any> = sysconfig[schemaKey] || {};
-
-        result[schemaKey] = validateConfig(cluster, subSchema, subConfig);
+        const validatedConfig = validateConfig(cluster, subSchema, subConfig);
+        result[schemaKey] = validatedConfig;
 
         if (schemaKey === 'terafoundation') {
             result[schemaKey].connectors = {};
 
             const connectors: Record<string, any> = subConfig.connectors || {};
             for (const [connector, connectorConfig] of Object.entries(connectors)) {
-                const connectorSchema = getConnectorSchema(connector);
-                result[schemaKey].connectors[connector] = {};
+                const {
+                    schema: connectorSchema,
+                    validatorFn: connValidatorFn
+                } = getConnectorSchemaAndValFn<S>(connector);
 
+                result[schemaKey].connectors[connector] = {};
                 for (const [connection, connectionConfig] of Object.entries(connectorConfig)) {
-                    result[schemaKey].connectors[connector][connection] = validateConfig(
+                    const validatedConnConfig = validateConfig(
                         cluster,
                         connectorSchema,
                         connectionConfig as any
                     );
+
+                    result[schemaKey].connectors[connector][connection] = validatedConnConfig;
+
+                    listOfValidations[`${connector}:${connection}`] = {
+                        validatorFn: connValidatorFn,
+                        subconfig: validatedConnConfig,
+                        connector: true
+                    };
                 }
             }
+
+            listOfValidations[schemaKey] = {
+                validatorFn: foundationValidatorFn<S>,
+                subconfig: validatedConfig
+            };
+        } else {
+            listOfValidations[schemaKey] = {
+                validatorFn: extractValidatorFn<S>(config.config_schema, sysconfig),
+                subconfig: validatedConfig
+            };
         }
     }
 
@@ -122,5 +167,18 @@ export default function validateConfigs<
         result._nodeName = hostname;
     }
 
+    // Cross-field validation
+    for (const entry of Object.entries(listOfValidations)) {
+        const [name, validatorObj] = entry;
+
+        if (validatorObj.validatorFn) {
+            try {
+                validatorObj.validatorFn(cloneDeep(validatorObj.subconfig), cloneDeep(result));
+            } catch (err) {
+                throw new TSError(`Cross-field validation failed for ${validatorObj.connector ? 'connector ' : ''}'${name}': ${err}`);
+            }
+        }
+    }
+
     return result;
 }

packages/terafoundation/test/process-context-spec.ts

@@ -9,6 +9,9 @@ describe('Terafoundation (ProcessContext)', () => {
             configfile: {
                 terafoundation: {
                     environment: process.env.NODE_ENV,
+                    connectors: {
+                        'elasticsearch-next': {}
+                    }
                 }
             }
         } as any);