pass subconfig into validator fn

packages/terafoundation/src/connector-utils.ts

@@ -1,6 +1,6 @@
 import path from 'path';
 import { TSError, parseError, Logger } from '@terascope/utils';
-import { Initializers } from 'packages/types/dist/src/terafoundation'; // FIXME
+import { Terafoundation } from '@terascope/types';
 
 type ErrorResult = {
     filePath: string;
@@ -106,7 +106,7 @@ export function getConnectorModule(name: string, reason: string): any {
     return null;
 }
 
-export function getConnectorInitializers(name: string): Initializers {
+export function getConnectorInitializers(name: string): Terafoundation.Initializers {
     const reason = `Could not retrieve schema code for: ${name}\n`;
 
     const mod = getConnectorModule(name, reason);

packages/terafoundation/src/connectors/s3.ts

@@ -1,6 +1,5 @@
 import { Logger } from '@terascope/utils';
 import { createS3Client, S3ClientConfig } from '@terascope/file-asset-apis';
-import { Schema } from 'packages/types/dist/src/terafoundation'; // FIXME
 import * as i from '../interfaces';
 
 export default {
@@ -66,22 +65,22 @@ export default {
     },
     validate_config(
         sysconfig: i.FoundationSysConfig<any>,
-        schema: Schema<S3ClientConfig>,
-        connectorName: string,
-        connectionName: string
+        subconfig: S3ClientConfig,
+        name: string
     ): void {
-        const connection = sysconfig.terafoundation.connectors[connectorName][connectionName];
-        const caCertExists: boolean = (connection.caCertificate.length !== 0);
-        const certLocationExists: boolean = (connection.certLocation.length !== 0);
+        const connectorName = name.split(':')[0];
+        const connectionName = name.split(':')[1];
+        const caCertExists: boolean = (subconfig.caCertificate?.length !== 0);
+        const certLocationExists: boolean = (subconfig.certLocation?.length !== 0);
         if (caCertExists && certLocationExists) {
-            throw new Error('"caCertificate" and "certLocation" contradict inside of the s3 connection config.\n'
+            throw new Error(`"caCertificate" and "certLocation" contradict inside of the ${connectorName}.${connectionName} connector.\n`
             + '  Use only one or the other.');
         } else if (
-            (caCertExists && !connection.sslEnabled)
-            || (certLocationExists && !connection.sslEnabled)
+            (caCertExists && !subconfig.sslEnabled)
+            || (certLocationExists && !subconfig.sslEnabled)
         ) {
             throw new Error('A certificate is provided but sslEnabled is set to "false".\n'
-                + '  Set sslEnabled to "true" or don\'t provide a certificate inside of the s3 connection config.');
+                + `  Set sslEnabled to "true" or don't provide a certificate inside of the ${connectorName}.${connectionName} connector.`);
         }
     }
 };

packages/terafoundation/src/schema.ts

@@ -1,12 +1,11 @@
 import convict from 'convict';
 import { cpus } from 'os';
-import { Terafoundation } from '@terascope/types';
-import { Initializers } from 'packages/types/dist/src/terafoundation'; // FIXME add to @terascope/types
+import { PartialDeep, Terafoundation } from '@terascope/types';
 
 const workerCount = cpus().length;
 const DEFAULT_ASSET_STORAGE_CONNECTION_TYPE = 'elasticsearch-next';
 
-export function getFoundationInitializers(): Initializers {
+export function getFoundationInitializers(): Terafoundation.Initializers {
     const schema: convict.Schema<any> = {
         environment: {
             doc: 'If set to `production`, console logging will be disabled and logs will be sent to a file',
@@ -89,10 +88,13 @@ export function getFoundationInitializers(): Initializers {
         }
     };
 
-    function validate_config(sysconfig: Terafoundation.SysConfig<any>): void {
-        const connectionType = sysconfig.terafoundation.asset_storage_connection_type
+    function validate_config(sysconfig: Terafoundation.SysConfig<any>,
+        subconfig: PartialDeep<Terafoundation.SysConfig<any>>
+    ): void {
+        const typedSubconfig = subconfig as unknown as Terafoundation.Foundation;
+        const connectionType = typedSubconfig.asset_storage_connection_type
                                 || DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
-        const connectionTypesPresent = Object.keys(sysconfig.terafoundation.connectors);
+        const connectionTypesPresent = Object.keys(typedSubconfig.connectors);
         const validConnectionTypes = ['elasticsearch-next', 's3'];
 
         // checks on asset_storage_connection_type
@@ -106,18 +108,19 @@ export function getFoundationInitializers(): Initializers {
         }
 
         // checks on asset_storage_connection
-        const connectionsPresent = Object.keys(sysconfig.terafoundation.connectors[`${connectionType}`]);
+        const connectionsPresent = Object.keys(typedSubconfig.connectors[`${connectionType}`]);
         /// Check to make sure the asset_storage_connection exists inside the connector
         /// Exclude elasticsearch as this connection type does not utilize this value
         if (
-            !connectionsPresent.includes(sysconfig.terafoundation.asset_storage_connection)
-            && !connectionType.includes('elasticsearch-next')
+            connectionType !== 'elasticsearch-next'
+            && typedSubconfig.asset_storage_connection
+            && !connectionsPresent.includes(typedSubconfig.asset_storage_connection)
         ) {
-            throw new Error(`${sysconfig.terafoundation.asset_storage_connection} not found in terafoundation.connectors.${connectionType}`);
+            throw new Error(`${typedSubconfig.asset_storage_connection} not found in terafoundation.connectors.${connectionType}`);
         }
 
         // checks on asset_storage_bucket
-        if (sysconfig.terafoundation.asset_storage_bucket && connectionType !== 's3') {
+        if (typedSubconfig.asset_storage_bucket && connectionType !== 's3') {
             throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"');
         }
         // TODO: add regex to check if valid bucket name

packages/terafoundation/src/validate-configs.ts

@@ -8,7 +8,7 @@ import {
 import convict_format_with_validator from 'convict-format-with-validator';
 // @ts-expect-error no types
 import convict_format_with_moment from 'convict-format-with-moment';
-import { Initializers } from 'packages/types/dist/src/terafoundation'; // FIXME
+import { Terafoundation } from '@terascope/types';
 import { getConnectorInitializers } from './connector-utils';
 import { getFoundationInitializers } from './schema';
 import * as i from './interfaces';
@@ -44,7 +44,7 @@ function validateConfig(
 function extractInitializers<S>(
     fn: any,
     sysconfig: PartialDeep<i.FoundationSysConfig<S>>
-): Initializers {
+): Terafoundation.Initializers {
     if (isFunction(fn)) {
         return fn(sysconfig);
     }
@@ -77,19 +77,17 @@ export default function validateConfigs<
         throw new Error('Terafoundation requires a valid system configuration');
     }
 
-    const listOfValidations: Initializers[] = [];
+    const listOfValidations: Record<string, Terafoundation.ValidationObj> = {};
     const {
         schema: sysconfigSchema,
-        validatorFn: sysconfigValidatorFn
+        validatorFn: terasliceValidatorFn
     } = extractInitializers(config.config_schema, sysconfig);
-
-    listOfValidations.push({ schema: sysconfigSchema, validatorFn: sysconfigValidatorFn });
-
+    listOfValidations.teraslice = { validatorFn: terasliceValidatorFn, subconfig: {} };
     const {
         schema: foundationSchema,
         validatorFn: foundationValidatorFn
     } = getFoundationInitializers();
-    listOfValidations.push({ schema: foundationSchema, validatorFn: foundationValidatorFn });
+    listOfValidations.terafoundation = { validatorFn: foundationValidatorFn, subconfig: {} };
     sysconfigSchema.terafoundation = foundationSchema;
 
     const result: any = {};
@@ -104,7 +102,17 @@ export default function validateConfigs<
     for (const schemaKey of schemaKeys) {
         const subSchema = sysconfigSchema[schemaKey] || {};
         const subConfig: Record<string, any> = sysconfig[schemaKey] || {};
-        result[schemaKey] = validateConfig(cluster, subSchema, subConfig);
+        const validatedConfig = validateConfig(cluster, subSchema, subConfig);
+        result[schemaKey] = validatedConfig;
+
+        if (listOfValidations[schemaKey]) {
+            listOfValidations[schemaKey].subconfig = validatedConfig;
+        } else {
+            listOfValidations[schemaKey] = {
+                validatorFn: undefined,
+                subconfig: validatedConfig
+            };
+        }
 
         if (schemaKey === 'terafoundation') {
             result[schemaKey].connectors = {};
@@ -118,18 +126,18 @@ export default function validateConfigs<
 
                 result[schemaKey].connectors[connector] = {};
                 for (const [connection, connectionConfig] of Object.entries(connectorConfig)) {
-                    result[schemaKey].connectors[connector][connection] = validateConfig(
+                    const validatedConnConfig = validateConfig(
                         cluster,
                         connSchema,
                         connectionConfig as any
                     );
 
-                    listOfValidations.push({
-                        schema: connSchema,
+                    result[schemaKey].connectors[connector][connection] = validatedConnConfig;
+
+                    listOfValidations[`${connector}:${connection}`] = {
                         validatorFn: connValidatorFn,
-                        connector,
-                        connection
-                    });
+                        subconfig: validatedConnConfig
+                    };
                 }
             }
         }
@@ -145,12 +153,18 @@ export default function validateConfigs<
         result._nodeName = hostname;
     }
 
-    const resultCopy = cloneDeep(result);
-    for (const {
-        schema, validatorFn, connector, connection
-    } of listOfValidations) {
-        if (validatorFn) {
-            validatorFn(resultCopy, schema, connector, connection);
+    // Cross-field validation
+    for (const key in listOfValidations) {
+        if (Object.prototype.hasOwnProperty.call(listOfValidations, key)) {
+            const obj = listOfValidations[key];
+
+            if (obj.validatorFn) {
+                try {
+                    obj.validatorFn(cloneDeep(result), cloneDeep(obj.subconfig), key);
+                } catch (err) {
+                    throw new TSError(`Cross-field validation failed: ${err}`);
+                }
+            }
         }
     }
 

packages/terafoundation/test/validate-configs-spec.ts

@@ -1,5 +1,6 @@
 import 'jest-extended';
 import os from 'os';
+import { PartialDeep, Terafoundation } from 'packages/types/dist/src';
 import { Cluster } from '../src';
 import validateConfigs from '../src/validate-configs';
 import { getFoundationInitializers } from '../src/schema';
@@ -366,7 +367,7 @@ describe('Validate Configs', () => {
         });
     });
 
-    describe('when given a config with an elasticsearch with no default connection', () => {
+    describe('when given a config without an asset_storage_connection_type and with an elasticsearch-next with no default connection', () => {
         const configFile = {
             terafoundation: {
                 connectors: {
@@ -411,16 +412,55 @@ describe('Validate Configs', () => {
             });
         });
     });
+
+    describe('when given a config_schema with a validator fn that fails', () => {
+        const configFile = {
+            teraslice: {
+                workers: 4
+            },
+            terafoundation: {
+                asset_storage_bucket: 'testBucket',
+                connectors: {
+                    'elasticsearch-next': {
+                        default: {}
+                    }
+                },
+                workers: 3
+            }
+        };
+        const cluster = {
+            isMaster: true,
+        };
+
+        const testFn = (
+            sysconfig: Terafoundation.SysConfig<any>,
+            subconfig: PartialDeep<Terafoundation.SysConfig<any>>,
+            name: string) => {
+            const typedSubconfig = subconfig as unknown as Terafoundation.Foundation;
+            if (sysconfig.terafoundation.workers !== typedSubconfig.workers) {
+                throw new Error(`${name} validatorFn test failed`);
+            }
+        };
+        const config = {
+            config_schema() {
+                return { schema: {}, validatorFn: testFn };
+            }
+        };
+
+        it('should throw an error', () => {
+            expect(() => validateConfigs(cluster as any, config as any, configFile as any)).toThrow('teraslice validatorFn test failed');
+        });
+    });
 });
 
 describe('getFoundationInitializers', () => {
-    it('should return an initializer with schema and validatorFn keys', () => {
+    it('should return an initializer with schema key', () => {
         expect(getFoundationInitializers()).toContainKey('schema');
     });
 });
 
 describe('getConnectorInitializers', () => {
-    it('should return an initializer with schema and validatorFn keys', () => {
+    it('should return an initializer with schema key', () => {
         const connector = 'elasticsearch-next';
         expect(getConnectorInitializers(connector)).toContainKey('schema');
     });

packages/teraslice/src/lib/config/schemas/system.ts

@@ -323,10 +323,13 @@ export const schema = {
     }
 };
 
+// This function is used to make cross-field validations if needed
+export const validatorFn = () => {};
+
 export function configSchema(): Initializers {
     return {
         schema: { teraslice: schema },
-        validatorFn: undefined
+        validatorFn
     };
 }
 

packages/types/src/terafoundation.ts

@@ -3,7 +3,7 @@ import {
     Cluster as NodeJSCluster,
     Worker as NodeJSWorker
 } from 'node:cluster';
-import type { Overwrite } from './utility';
+import type { Overwrite, PartialDeep } from './utility';
 import type { Logger } from './logger';
 
 interface Format {
@@ -29,11 +29,18 @@ export type Schema<T> = {
 
 export type Initializers = {
     schema: Schema<any>,
-    validatorFn?: (
-        sysconfig: SysConfig<any>, schema: Schema<any>, connector?: string, connection?: string
-    ) => void,
-    connector?: string,
-    connection?: string
+    validatorFn?: ValidatorFn
+}
+
+export type ValidatorFn = (
+    sysconfig: SysConfig<any>,
+    subConfig: PartialDeep<SysConfig<any>>,
+    name: string
+) => void
+
+export type ValidationObj= {
+    validatorFn: ValidatorFn | undefined,
+    subconfig: PartialDeep<SysConfig<any>>
 }
 
 export type Config<
@@ -125,19 +132,21 @@ export type Cluster = Overwrite<NodeJSCluster, {
 
 export type SysConfig<S> = {
     _nodeName: string;
-    terafoundation: {
-        workers: number;
-        environment: 'production'|'development'|'test'|string;
-        connectors: Record<string, any>;
-        log_path: string;
-        log_level: LogLevelConfig;
-        logging: LogType[];
-        asset_storage_connection_type?: string;
-        asset_storage_connection?: string;
-        asset_storage_bucket?: string;
-    };
+    terafoundation: Foundation
 } & S;
 
+export type Foundation = {
+    workers: number;
+    environment: 'production'|'development'|'test'|string;
+    connectors: Record<string, any>;
+    log_path: string;
+    log_level: LogLevelConfig;
+    logging: LogType[];
+    asset_storage_connection_type?: string;
+    asset_storage_connection?: string;
+    asset_storage_bucket?: string;
+};
+
 export type Context<
     S = Record<string, any>,
     A = Record<string, any>,