allow validator fns on a terafoundation schemas

packages/terafoundation/src/connector-utils.ts

@@ -1,5 +1,6 @@
 import path from 'path';
 import { TSError, parseError, Logger } from '@terascope/utils';
+import { Initializers } from 'packages/types/dist/src/terafoundation'; // FIXME
 
 type ErrorResult = {
     filePath: string;
@@ -105,15 +106,15 @@ export function getConnectorModule(name: string, reason: string): any {
     return null;
 }
 
-export function getConnectorInitializers(name: string): Record<string, any> {
+export function getConnectorInitializers(name: string): Initializers {
     const reason = `Could not retrieve schema code for: ${name}\n`;
 
     const mod = getConnectorModule(name, reason);
     if (!mod) {
         console.warn(`[WARNING] ${reason}`);
-        return {};
+        return { schema: {} };
     }
-    return { connectorSchema: mod.config_schema(), validatorFn: mod.validate_config };
+    return { schema: mod.config_schema(), validatorFn: mod.validate_config };
 }
 
 export function createConnection(

packages/terafoundation/src/connectors/s3.ts

@@ -1,11 +1,13 @@
 import { Logger } from '@terascope/utils';
-import { createS3Client } from '@terascope/file-asset-apis';
+import { createS3Client, S3ClientConfig } from '@terascope/file-asset-apis';
+import { Schema } from 'packages/types/dist/src/terafoundation'; // FIXME
+import * as i from '../interfaces';
 
 export default {
     create() {
         throw new Error('s3 does not support the deprecated "create" method, please use file-assets >= v2.4.0');
     },
-    async createClient(customConfig: Record<string, any>, logger: Logger) {
+    async createClient(customConfig: S3ClientConfig, logger: Logger) {
         const client = await createS3Client(customConfig, logger);
         return { client, logger };
     },
@@ -62,16 +64,21 @@ export default {
             }
         };
     },
-    validate_config(config: Record<string, any>): void {
-        /// Cross validate s3 configuration
-        const caCertExists: boolean = (config.caCertificate.length !== 0);
-        const certLocationExists: boolean = (config.certLocation.length !== 0);
+    validate_config(
+        sysconfig: i.FoundationSysConfig<any>,
+        schema: Schema<S3ClientConfig>,
+        connectorName: string,
+        connectionName: string
+    ): void {
+        const connection = sysconfig.terafoundation.connectors[connectorName][connectionName];
+        const caCertExists: boolean = (connection.caCertificate.length !== 0);
+        const certLocationExists: boolean = (connection.certLocation.length !== 0);
         if (caCertExists && certLocationExists) {
             throw new Error('"caCertificate" and "certLocation" contradict inside of the s3 connection config.\n'
             + '  Use only one or the other.');
         } else if (
-            (caCertExists && !config.sslEnabled)
-            || (certLocationExists && !config.sslEnabled)
+            (caCertExists && !connection.sslEnabled)
+            || (certLocationExists && !connection.sslEnabled)
         ) {
             throw new Error('A certificate is provided but sslEnabled is set to "false".\n'
                 + '  Set sslEnabled to "true" or don\'t provide a certificate inside of the s3 connection config.');

packages/terafoundation/src/schema.ts

@@ -1,11 +1,12 @@
 import convict from 'convict';
 import { cpus } from 'os';
 import { Terafoundation } from '@terascope/types';
+import { Initializers } from 'packages/types/dist/src/terafoundation'; // FIXME add to @terascope/types
 
 const workerCount = cpus().length;
 const DEFAULT_ASSET_STORAGE_CONNECTION_TYPE = 'elasticsearch-next';
 
-export function foundationSchema(sysconfig: Terafoundation.SysConfig<any>): convict.Schema<any> {
+export function getFoundationInitializers(): Initializers {
     const schema: convict.Schema<any> = {
         environment: {
             doc: 'If set to `production`, console logging will be disabled and logs will be sent to a file',
@@ -74,59 +75,53 @@ export function foundationSchema(sysconfig: Terafoundation.SysConfig<any>): conv
         asset_storage_connection_type: {
             doc: 'Name of the connection type used to store assets',
             default: DEFAULT_ASSET_STORAGE_CONNECTION_TYPE,
-            format(connectionTypeName: any): void {
-                const validConnectionTypes = ['elasticsearch-next', 's3'];
-                const connectionTypesPresent = Object.keys(sysconfig.terafoundation.connectors);
-                if (!connectionTypesPresent.includes(connectionTypeName)) {
-                    throw new Error('asset_storage_connection_type not found in terafoundation.connectors');
-                }
-                if (!validConnectionTypes.includes(connectionTypeName)) {
-                    throw new Error(`Invalid asset_storage_connection_type. Valid types: ${validConnectionTypes.toString()}`);
-                }
-            }
+            format: String
         },
         asset_storage_connection: {
             doc: 'Name of the connection used to store assets.',
             default: 'default',
-            format(connectionName: any): void {
-                let connectionType: string;
-                if (sysconfig.terafoundation.asset_storage_connection_type) {
-                    connectionType = sysconfig.terafoundation.asset_storage_connection_type;
-                } else {
-                    connectionType = DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
-                }
-
-                const connectionsPresent = Object.keys(sysconfig.terafoundation.connectors[`${connectionType}`]);
-                /// Check to make sure the asset_storage_connection exists inside the connector
-                /// Exclude elasticsearch as this connection type does not utilize this value
-                if (
-                    !connectionsPresent.includes(connectionName)
-                    && !connectionType.includes('elasticsearch-next')
-                ) {
-                    throw new Error(`${connectionName} not found in terafoundation.connectors.${connectionType}`);
-                }
-            }
+            format: String
         },
         asset_storage_bucket: {
             doc: 'Name of S3 bucket used to store assets. Can only be used if "asset_storage_connection_type" is "s3".',
             default: undefined,
-            format(bucketName: any): void {
-                let connectionType;
-                if (sysconfig.terafoundation.asset_storage_connection_type) {
-                    connectionType = sysconfig.terafoundation.asset_storage_connection_type;
-                } else {
-                    connectionType = DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
-                }
-                if (typeof bucketName !== 'string') {
-                    throw new Error('asset_storage_bucket must be a string');
-                }
-                if (connectionType !== 's3') {
-                    throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"');
-                }
-                // TODO: add regex to check if valid bucket name
-            }
+            format: String
         }
     };
 
-    return schema;
+    function validate_config(sysconfig: Terafoundation.SysConfig<any>): void {
+        const connectionType = sysconfig.terafoundation.asset_storage_connection_type
+                                || DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
+        const connectionTypesPresent = Object.keys(sysconfig.terafoundation.connectors);
+        const validConnectionTypes = ['elasticsearch-next', 's3'];
+
+        // checks on asset_storage_connection_type
+        if (!validConnectionTypes
+            .includes(connectionType)) {
+            throw new Error(`Invalid asset_storage_connection_type. Valid types: ${validConnectionTypes.toString()}`);
+        }
+        if (!connectionTypesPresent
+            .includes(connectionType)) {
+            throw new Error('asset_storage_connection_type not found in terafoundation.connectors');
+        }
+
+        // checks on asset_storage_connection
+        const connectionsPresent = Object.keys(sysconfig.terafoundation.connectors[`${connectionType}`]);
+        /// Check to make sure the asset_storage_connection exists inside the connector
+        /// Exclude elasticsearch as this connection type does not utilize this value
+        if (
+            !connectionsPresent.includes(sysconfig.terafoundation.asset_storage_connection)
+            && !connectionType.includes('elasticsearch-next')
+        ) {
+            throw new Error(`${sysconfig.terafoundation.asset_storage_connection} not found in terafoundation.connectors.${connectionType}`);
+        }
+
+        // checks on asset_storage_bucket
+        if (sysconfig.terafoundation.asset_storage_bucket && connectionType !== 's3') {
+            throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"');
+        }
+        // TODO: add regex to check if valid bucket name
+    }
+
+    return { schema, validatorFn: validate_config };
 }

packages/terafoundation/src/validate-configs.ts

@@ -2,14 +2,15 @@ import os from 'os';
 import convict, { addFormats } from 'convict';
 import {
     TSError, isFunction, isPlainObject,
-    isEmpty, concat, PartialDeep
+    isEmpty, concat, PartialDeep, cloneDeep
 } from '@terascope/utils';
 // @ts-expect-error no types
 import convict_format_with_validator from 'convict-format-with-validator';
 // @ts-expect-error no types
 import convict_format_with_moment from 'convict-format-with-moment';
+import { Initializers } from 'packages/types/dist/src/terafoundation'; // FIXME
 import { getConnectorInitializers } from './connector-utils';
-import { foundationSchema } from './schema';
+import { getFoundationInitializers } from './schema';
 import * as i from './interfaces';
 
 addFormats(convict_format_with_validator);
@@ -18,8 +19,7 @@ addFormats(convict_format_with_moment);
 function validateConfig(
     cluster: { isMaster: boolean },
     schema: convict.Schema<any>,
-    namespaceConfig: any,
-    crossFieldValidation?: ((config: Record<string, any>) => void) | undefined
+    namespaceConfig: any
 ) {
     try {
         const config = convict(schema || {});
@@ -34,28 +34,25 @@ function validateConfig(
                 allowed: true,
             } as any);
         }
-        if (crossFieldValidation) {
-            crossFieldValidation(config.getProperties());
-        }
 
         return config.getProperties();
     } catch (err) {
         throw new TSError(err, { reason: 'Error validating configuration' });
     }
 }
 
-function extractSchema<S>(
+function extractInitializers<S>(
     fn: any,
     sysconfig: PartialDeep<i.FoundationSysConfig<S>>
-): Record<string, any> {
+): Initializers {
     if (isFunction(fn)) {
         return fn(sysconfig);
     }
     if (isPlainObject(fn)) {
         return fn;
     }
 
-    return {};
+    return { schema: {} };
 }
 
 /**
@@ -80,8 +77,23 @@ export default function validateConfigs<
         throw new Error('Terafoundation requires a valid system configuration');
     }
 
-    const schema = extractSchema(config.config_schema, sysconfig);
-    schema.terafoundation = foundationSchema(sysconfig);
+    const listOfValidations: Initializers[] = [];
+    const {
+        schema: sysconfigSchema,
+        validatorFn: sysconfigValidatorFn
+    } = extractInitializers(config.config_schema, sysconfig); // FixMe test this
+    console.log('@@@@ sysconfigSchema: ', sysconfigSchema);
+
+    listOfValidations.push({ schema: sysconfigSchema, validatorFn: sysconfigValidatorFn });
+
+    const {
+        schema: foundationSchema,
+        validatorFn: foundationValidatorFn
+    } = getFoundationInitializers(); // FixMe test this
+    listOfValidations.push({ schema: foundationSchema, validatorFn: foundationValidatorFn });
+    sysconfigSchema.terafoundation = foundationSchema;
+    console.log('@@@@ sysconfigSchema: ', sysconfigSchema);
+
     const result: any = {};
 
     if (config.schema_formats) {
@@ -90,9 +102,9 @@ export default function validateConfigs<
         });
     }
 
-    const schemaKeys = concat(Object.keys(schema), Object.keys(sysconfig));
+    const schemaKeys = concat(Object.keys(sysconfigSchema), Object.keys(sysconfig));
     for (const schemaKey of schemaKeys) {
-        const subSchema = schema[schemaKey] || {};
+        const subSchema = sysconfigSchema[schemaKey] || {};
         const subConfig: Record<string, any> = sysconfig[schemaKey] || {};
         result[schemaKey] = validateConfig(cluster, subSchema, subConfig);
 
@@ -101,16 +113,25 @@ export default function validateConfigs<
 
             const connectors: Record<string, any> = subConfig.connectors || {};
             for (const [connector, connectorConfig] of Object.entries(connectors)) {
-                const { connectorSchema, validatorFn } = getConnectorInitializers(connector);
+                const {
+                    schema: connSchema,
+                    validatorFn: connValidatorFn
+                } = getConnectorInitializers(connector);
 
                 result[schemaKey].connectors[connector] = {};
                 for (const [connection, connectionConfig] of Object.entries(connectorConfig)) {
                     result[schemaKey].connectors[connector][connection] = validateConfig(
                         cluster,
-                        connectorSchema,
-                        connectionConfig as any,
-                        validatorFn
+                        connSchema,
+                        connectionConfig as any
                     );
+
+                    listOfValidations.push({
+                        schema: connSchema,
+                        validatorFn: connValidatorFn,
+                        connector,
+                        connection
+                    });
                 }
             }
         }
@@ -126,5 +147,15 @@ export default function validateConfigs<
         result._nodeName = hostname;
     }
 
+    console.log('@@@@ listofValidations: ', listOfValidations);
+    const resultCopy = cloneDeep(result);
+    for (const {
+        schema, validatorFn, connector, connection
+    } of listOfValidations) {
+        if (validatorFn) {
+            validatorFn(resultCopy, schema, connector, connection);
+        }
+    }
+
     return result;
 }

packages/terafoundation/test/validate-configs-spec.ts

@@ -137,7 +137,7 @@ describe('Validate Configs', () => {
         });
     });
 
-    describe("when using using a connector that doesn't exist", () => {
+    describe("when using a connector that doesn't exist", () => {
         const configFile = {
             terafoundation: {
                 connectors: {

packages/teraslice/src/lib/config/schemas/system.ts

@@ -5,6 +5,7 @@ import {
     isInteger
 } from '@terascope/utils';
 import { cpus } from 'node:os';
+import { Initializers } from 'packages/types/dist/src/terafoundation'; // FIXME
 
 const workerCount = cpus().length;
 
@@ -322,8 +323,11 @@ export const schema = {
     }
 };
 
-export function configSchema() {
-    return { teraslice: schema };
+export function configSchema(): Initializers {
+    return {
+        schema: { teraslice: schema },
+        validatorFn: undefined
+    };
 }
 
 // TODO: fix this

packages/teraslice/test/config/schemas/system_schema-spec.ts

@@ -4,7 +4,7 @@ import { config_schema } from '../../../src/lib/config/schemas/system.js';
 import('@terascope/job-components');
 
 describe('system_schema', () => {
-    const schema = config_schema().teraslice;
+    const schema = config_schema().schema.teraslice;
 
     function checkValidation(config: Record<string, any>) {
         try {

packages/types/src/terafoundation.ts

@@ -23,6 +23,19 @@ interface SchemaObj<T = any> {
     [key: string]: any;
 }
 
+export type Schema<T> = {
+    [P in keyof T]: Schema<T[P]> | SchemaObj<T[P]>;
+};
+
+export type Initializers = {
+    schema: Schema<any>,
+    validatorFn?: (
+        sysconfig: SysConfig<any>, schema: Schema<any>, connector?: string, connection?: string
+    ) => void,
+    connector?: string,
+    connection?: string
+}
+
 export type Config<
     S = Record<string, any>,
     A = Record<string, any>,