From 7654b139e4e453122f749c9646a36a79673c23a2 Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 11 Apr 2024 14:05:13 -0700 Subject: [PATCH] remove 'initializer' verbage. refactor --- .../terafoundation/src/connector-utils.ts | 2 +- packages/terafoundation/src/connectors/s3.ts | 9 +- packages/terafoundation/src/schema.ts | 70 +++++++------- .../terafoundation/src/validate-configs.ts | 93 ++++++++++--------- .../test/validate-configs-spec.ts | 19 ++-- packages/types/src/terafoundation.ts | 6 +- 6 files changed, 98 insertions(+), 101 deletions(-) diff --git a/packages/terafoundation/src/connector-utils.ts b/packages/terafoundation/src/connector-utils.ts index 65cad7c7c48..2ac7805ef0c 100644 --- a/packages/terafoundation/src/connector-utils.ts +++ b/packages/terafoundation/src/connector-utils.ts @@ -106,7 +106,7 @@ export function getConnectorModule(name: string, reason: string): any { return null; } -export function getConnectorInitializers( +export function getConnectorSchemaAndValFn( name: string ): Terafoundation.Initializers { const reason = `Could not retrieve schema code for: ${name}\n`; diff --git a/packages/terafoundation/src/connectors/s3.ts b/packages/terafoundation/src/connectors/s3.ts index 9ab0c590ff6..d6e7e43c86a 100644 --- a/packages/terafoundation/src/connectors/s3.ts +++ b/packages/terafoundation/src/connectors/s3.ts @@ -65,22 +65,19 @@ export default { }, validate_config( _sysconfig: Terafoundation.SysConfig, - subconfig: S3ClientConfig, - name: string + subconfig: S3ClientConfig ): void { - const connectorName = name.split(':')[0]; - const connectionName = name.split(':')[1]; const caCertExists: boolean = (subconfig.caCertificate?.length !== 0); const certLocationExists: boolean = (subconfig.certLocation?.length !== 0); if (caCertExists && certLocationExists) { - throw new Error(`"caCertificate" and "certLocation" contradict inside of the ${connectorName}.${connectionName} connector.\n` + throw new Error('"caCertificate" and "certLocation" contradict.\n' + ' Use only one or the other.'); } else if ( (caCertExists && !subconfig.sslEnabled) || (certLocationExists && !subconfig.sslEnabled) ) { throw new Error('A certificate is provided but sslEnabled is set to "false".\n' - + ` Set sslEnabled to "true" or don't provide a certificate inside of the ${connectorName}.${connectionName} connector.`); + + ' Set sslEnabled to "true" or don\'t provide a certificate.'); } } }; diff --git a/packages/terafoundation/src/schema.ts b/packages/terafoundation/src/schema.ts index 3c1238d167a..04202a59d4e 100644 --- a/packages/terafoundation/src/schema.ts +++ b/packages/terafoundation/src/schema.ts @@ -5,7 +5,7 @@ import { Terafoundation } from '@terascope/types'; const workerCount = cpus().length; const DEFAULT_ASSET_STORAGE_CONNECTION_TYPE = 'elasticsearch-next'; -export function getFoundationInitializers(): Terafoundation.Initializers { +export function foundationSchema() { const schema: convict.Schema> = { environment: { doc: 'If set to `production`, console logging will be disabled and logs will be sent to a file', @@ -88,43 +88,43 @@ export function getFoundationInitializers(): Terafoundation.Initializers { } }; - function validate_config(_sysconfig: Terafoundation.SysConfig, - subconfig: Record - ): void { - const typedSubconfig = subconfig as Terafoundation.Foundation; - const connectionType = typedSubconfig.asset_storage_connection_type - || DEFAULT_ASSET_STORAGE_CONNECTION_TYPE; - const connectionTypesPresent = Object.keys(typedSubconfig.connectors); - const validConnectionTypes = ['elasticsearch-next', 's3']; + return schema; +} - // checks on asset_storage_connection_type - if (!validConnectionTypes - .includes(connectionType)) { - throw new Error(`Invalid asset_storage_connection_type. Valid types: ${validConnectionTypes.toString()}`); - } - if (!connectionTypesPresent - .includes(connectionType)) { - throw new Error('asset_storage_connection_type not found in terafoundation.connectors'); - } +export function foundationValidatorFn(_sysconfig: Terafoundation.SysConfig, + subconfig: Record +): void { + const typedSubconfig = subconfig as Terafoundation.Foundation; + const connectionType = typedSubconfig.asset_storage_connection_type + || DEFAULT_ASSET_STORAGE_CONNECTION_TYPE; + const connectionTypesPresent = Object.keys(typedSubconfig.connectors); + const validConnectionTypes = ['elasticsearch-next', 's3']; - // checks on asset_storage_connection - const connectionsPresent = Object.keys(typedSubconfig.connectors[`${connectionType}`]); - /// Check to make sure the asset_storage_connection exists inside the connector - /// Exclude elasticsearch as this connection type does not utilize this value - if ( - connectionType !== 'elasticsearch-next' - && typedSubconfig.asset_storage_connection - && !connectionsPresent.includes(typedSubconfig.asset_storage_connection) - ) { - throw new Error(`${typedSubconfig.asset_storage_connection} not found in terafoundation.connectors.${connectionType}`); - } + // checks on asset_storage_connection_type + if (!validConnectionTypes + .includes(connectionType)) { + throw new Error(`Invalid asset_storage_connection_type. Valid types: ${validConnectionTypes.toString()}`); + } + if (!connectionTypesPresent + .includes(connectionType)) { + throw new Error('asset_storage_connection_type not found in terafoundation.connectors'); + } - // checks on asset_storage_bucket - if (typedSubconfig.asset_storage_bucket && connectionType !== 's3') { - throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"'); - } - // TODO: add regex to check if valid bucket name + // checks on asset_storage_connection + const connectionsPresent = Object.keys(typedSubconfig.connectors[`${connectionType}`]); + /// Check to make sure the asset_storage_connection exists inside the connector + /// Exclude elasticsearch as this connection type does not utilize this value + if ( + connectionType !== 'elasticsearch-next' + && typedSubconfig.asset_storage_connection + && !connectionsPresent.includes(typedSubconfig.asset_storage_connection) + ) { + throw new Error(`${typedSubconfig.asset_storage_connection} not found in terafoundation.connectors.${connectionType}`); } - return { schema, validatorFn: validate_config }; + // checks on asset_storage_bucket + if (typedSubconfig.asset_storage_bucket && connectionType !== 's3') { + throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"'); + } + // TODO: add regex to check if valid bucket name } diff --git a/packages/terafoundation/src/validate-configs.ts b/packages/terafoundation/src/validate-configs.ts index 4c1b4388aca..5c98d882603 100644 --- a/packages/terafoundation/src/validate-configs.ts +++ b/packages/terafoundation/src/validate-configs.ts @@ -9,8 +9,8 @@ import convict_format_with_validator from 'convict-format-with-validator'; // @ts-expect-error no types import convict_format_with_moment from 'convict-format-with-moment'; import { Terafoundation } from '@terascope/types'; -import { getConnectorInitializers } from './connector-utils'; -import { getFoundationInitializers } from './schema'; +import { getConnectorSchemaAndValFn } from './connector-utils'; +import { foundationSchema, foundationValidatorFn } from './schema'; import * as i from './interfaces'; addFormats(convict_format_with_validator); @@ -41,22 +41,39 @@ function validateConfig( } } -function extractInitializers( +function extractSchema( fn: any, sysconfig: PartialDeep> -): Terafoundation.Initializers { +): Terafoundation.Schema> { if (isFunction(fn)) { const result = fn(sysconfig); if (result.schema) { - return result; + return result.schema; } - return { schema: fn(sysconfig) }; + return result; } if (isPlainObject(fn)) { - return { schema: fn }; + return fn; } - return { schema: {} }; + return {}; +} + +function extractValidatorFn( + fn: any, + sysconfig: PartialDeep> +): Terafoundation.ValidatorFn | undefined { + if (isFunction(fn)) { + const result = fn(sysconfig); + if (result.validatorFn) { + return result.validatorFn; + } + } + if (isPlainObject(fn)) { + return fn.validatorFn; + } + + return undefined; } /** @@ -82,18 +99,8 @@ export default function validateConfigs< } const listOfValidations: Record> = {}; - const { - schema: sysconfigSchema, - validatorFn: mainSvcValidatorFn - } = extractInitializers(config.config_schema, sysconfig); - const mainSvcName = Object.keys(sysconfigSchema)[0]; - listOfValidations[mainSvcName] = { validatorFn: mainSvcValidatorFn, subconfig: {} }; - const { - schema: foundationSchema, - validatorFn: foundationValidatorFn - } = getFoundationInitializers(); - listOfValidations.terafoundation = { validatorFn: foundationValidatorFn, subconfig: {} }; - sysconfigSchema.terafoundation = foundationSchema; + const schema = extractSchema(config.config_schema, sysconfig); + schema.terafoundation = foundationSchema(); const result: any = {}; @@ -103,22 +110,13 @@ export default function validateConfigs< }); } - const schemaKeys = concat(Object.keys(sysconfigSchema), Object.keys(sysconfig)); + const schemaKeys = concat(Object.keys(schema), Object.keys(sysconfig)); for (const schemaKey of schemaKeys) { - const subSchema = sysconfigSchema[schemaKey] || {}; + const subSchema = schema[schemaKey] || {}; const subConfig: Record = sysconfig[schemaKey] || {}; const validatedConfig = validateConfig(cluster, subSchema, subConfig); result[schemaKey] = validatedConfig; - if (listOfValidations[schemaKey]) { - listOfValidations[schemaKey].subconfig = validatedConfig; - } else { - listOfValidations[schemaKey] = { - validatorFn: undefined, - subconfig: validatedConfig - }; - } - if (schemaKey === 'terafoundation') { result[schemaKey].connectors = {}; @@ -127,7 +125,7 @@ export default function validateConfigs< const { schema: connectorSchema, validatorFn: connValidatorFn - } = getConnectorInitializers(connector); + } = getConnectorSchemaAndValFn(connector); result[schemaKey].connectors[connector] = {}; for (const [connection, connectionConfig] of Object.entries(connectorConfig)) { @@ -141,10 +139,21 @@ export default function validateConfigs< listOfValidations[`${connector}:${connection}`] = { validatorFn: connValidatorFn, - subconfig: validatedConnConfig + subconfig: validatedConnConfig, + connector: true }; } } + + listOfValidations[schemaKey] = { + validatorFn: foundationValidatorFn, + subconfig: validatedConfig + }; + } else { + listOfValidations[schemaKey] = { + validatorFn: extractValidatorFn(config.config_schema, sysconfig), + subconfig: validatedConfig + }; } } @@ -159,16 +168,14 @@ export default function validateConfigs< } // Cross-field validation - for (const key in listOfValidations) { - if (Object.prototype.hasOwnProperty.call(listOfValidations, key)) { - const obj = listOfValidations[key]; - - if (obj.validatorFn) { - try { - obj.validatorFn(cloneDeep(result), cloneDeep(obj.subconfig), key); - } catch (err) { - throw new TSError(`Cross-field validation failed: ${err}`); - } + for (const entry of Object.entries(listOfValidations)) { + const [name, validatorObj] = entry; + + if (validatorObj.validatorFn) { + try { + validatorObj.validatorFn(cloneDeep(result), cloneDeep(validatorObj.subconfig)); + } catch (err) { + throw new TSError(`Cross-field validation failed for ${validatorObj.connector ? 'connector ' : ''}'${name}': ${err}`); } } } diff --git a/packages/terafoundation/test/validate-configs-spec.ts b/packages/terafoundation/test/validate-configs-spec.ts index f923d8b3a44..ceeac36c613 100644 --- a/packages/terafoundation/test/validate-configs-spec.ts +++ b/packages/terafoundation/test/validate-configs-spec.ts @@ -3,8 +3,7 @@ import os from 'os'; import { PartialDeep, Terafoundation } from 'packages/types/dist/src'; import { Cluster } from '../src'; import validateConfigs from '../src/validate-configs'; -import { getFoundationInitializers } from '../src/schema'; -import { getConnectorInitializers } from '../src/connector-utils'; +import { getConnectorSchemaAndValFn } from '../src/connector-utils'; describe('Validate Configs', () => { describe('when using mainly defaults', () => { @@ -434,11 +433,10 @@ describe('Validate Configs', () => { const testFn = ( sysconfig: Terafoundation.SysConfig, - subconfig: PartialDeep>, - name: string) => { + subconfig: PartialDeep>) => { const typedSubconfig = subconfig as unknown as Terafoundation.Foundation; if (sysconfig.terafoundation.workers !== typedSubconfig.workers) { - throw new Error(`${name} validatorFn test failed`); + throw new Error('validatorFn test failed'); } }; const config = { @@ -448,20 +446,15 @@ describe('Validate Configs', () => { }; it('should throw an error', () => { - expect(() => validateConfigs(cluster as any, config as any, configFile as any)).toThrow('teraslice validatorFn test failed'); + expect(() => validateConfigs(cluster as any, config as any, configFile as any)) + .toThrow('Cross-field validation failed for \'teraslice\': Error: validatorFn test failed'); }); }); }); -describe('getFoundationInitializers', () => { - it('should return an initializer with schema key', () => { - expect(getFoundationInitializers()).toContainKey('schema'); - }); -}); - describe('getConnectorInitializers', () => { it('should return an initializer with schema key', () => { const connector = 'elasticsearch-next'; - expect(getConnectorInitializers(connector)).toContainKey('schema'); + expect(getConnectorSchemaAndValFn(connector)).toContainKey('schema'); }); }); diff --git a/packages/types/src/terafoundation.ts b/packages/types/src/terafoundation.ts index 1e1bb57f401..2cfaa599788 100644 --- a/packages/types/src/terafoundation.ts +++ b/packages/types/src/terafoundation.ts @@ -34,13 +34,13 @@ export type Initializers = { export type ValidationObj= { subconfig: Record, - validatorFn?: ValidatorFn + validatorFn?: ValidatorFn, + connector?: boolean } export type ValidatorFn = ( sysconfig: SysConfig, - subconfig: Record, - name: string + subconfig: Record ) => void export type Config<