Skip to content

Commit

Permalink
remove 'initializer' verbage. refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@busma13
busma13 committed Apr 11, 2024
1 parent e3b648e commit 7654b13
Show file tree
Hide file tree
Showing 6 changed files with 98 additions and 101 deletions.
2 changes: 1 addition & 1 deletion packages/terafoundation/src/connector-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ export function getConnectorModule(name: string, reason: string): any {
return null;
}

export function getConnectorInitializers<S>(
export function getConnectorSchemaAndValFn<S>(
name: string
): Terafoundation.Initializers<S> {
const reason = `Could not retrieve schema code for: ${name}\n`;
Expand Down
9 changes: 3 additions & 6 deletions packages/terafoundation/src/connectors/s3.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,22 +65,19 @@ export default {
},
validate_config<S>(
_sysconfig: Terafoundation.SysConfig<S>,
subconfig: S3ClientConfig,
name: string
subconfig: S3ClientConfig
): void {
const connectorName = name.split(':')[0];
const connectionName = name.split(':')[1];
const caCertExists: boolean = (subconfig.caCertificate?.length !== 0);
const certLocationExists: boolean = (subconfig.certLocation?.length !== 0);
if (caCertExists && certLocationExists) {
throw new Error(`"caCertificate" and "certLocation" contradict inside of the ${connectorName}.${connectionName} connector.\n`
throw new Error('"caCertificate" and "certLocation" contradict.\n'
+ ' Use only one or the other.');
} else if (
(caCertExists && !subconfig.sslEnabled)
|| (certLocationExists && !subconfig.sslEnabled)
) {
throw new Error('A certificate is provided but sslEnabled is set to "false".\n'
+ ` Set sslEnabled to "true" or don't provide a certificate inside of the ${connectorName}.${connectionName} connector.`);
+ ' Set sslEnabled to "true" or don\'t provide a certificate.');
}
}
};
70 changes: 35 additions & 35 deletions packages/terafoundation/src/schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { Terafoundation } from '@terascope/types';
const workerCount = cpus().length;
const DEFAULT_ASSET_STORAGE_CONNECTION_TYPE = 'elasticsearch-next';

export function getFoundationInitializers<S>(): Terafoundation.Initializers<S> {
export function foundationSchema() {
const schema: convict.Schema<Record<string, any>> = {
environment: {
doc: 'If set to `production`, console logging will be disabled and logs will be sent to a file',
Expand Down Expand Up @@ -88,43 +88,43 @@ export function getFoundationInitializers<S>(): Terafoundation.Initializers<S> {
}
};

function validate_config(_sysconfig: Terafoundation.SysConfig<S>,
subconfig: Record<string, any>
): void {
const typedSubconfig = subconfig as Terafoundation.Foundation;
const connectionType = typedSubconfig.asset_storage_connection_type
|| DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
const connectionTypesPresent = Object.keys(typedSubconfig.connectors);
const validConnectionTypes = ['elasticsearch-next', 's3'];
return schema;
}

// checks on asset_storage_connection_type
if (!validConnectionTypes
.includes(connectionType)) {
throw new Error(`Invalid asset_storage_connection_type. Valid types: ${validConnectionTypes.toString()}`);
}
if (!connectionTypesPresent
.includes(connectionType)) {
throw new Error('asset_storage_connection_type not found in terafoundation.connectors');
}
export function foundationValidatorFn<S>(_sysconfig: Terafoundation.SysConfig<S>,
subconfig: Record<string, any>
): void {
const typedSubconfig = subconfig as Terafoundation.Foundation;
const connectionType = typedSubconfig.asset_storage_connection_type
|| DEFAULT_ASSET_STORAGE_CONNECTION_TYPE;
const connectionTypesPresent = Object.keys(typedSubconfig.connectors);
const validConnectionTypes = ['elasticsearch-next', 's3'];

// checks on asset_storage_connection
const connectionsPresent = Object.keys(typedSubconfig.connectors[`${connectionType}`]);
/// Check to make sure the asset_storage_connection exists inside the connector
/// Exclude elasticsearch as this connection type does not utilize this value
if (
connectionType !== 'elasticsearch-next'
&& typedSubconfig.asset_storage_connection
&& !connectionsPresent.includes(typedSubconfig.asset_storage_connection)
) {
throw new Error(`${typedSubconfig.asset_storage_connection} not found in terafoundation.connectors.${connectionType}`);
}
// checks on asset_storage_connection_type
if (!validConnectionTypes
.includes(connectionType)) {
throw new Error(`Invalid asset_storage_connection_type. Valid types: ${validConnectionTypes.toString()}`);
}
if (!connectionTypesPresent
.includes(connectionType)) {
throw new Error('asset_storage_connection_type not found in terafoundation.connectors');
}

// checks on asset_storage_bucket
if (typedSubconfig.asset_storage_bucket && connectionType !== 's3') {
throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"');
}
// TODO: add regex to check if valid bucket name
// checks on asset_storage_connection
const connectionsPresent = Object.keys(typedSubconfig.connectors[`${connectionType}`]);
/// Check to make sure the asset_storage_connection exists inside the connector
/// Exclude elasticsearch as this connection type does not utilize this value
if (
connectionType !== 'elasticsearch-next'
&& typedSubconfig.asset_storage_connection
&& !connectionsPresent.includes(typedSubconfig.asset_storage_connection)
) {
throw new Error(`${typedSubconfig.asset_storage_connection} not found in terafoundation.connectors.${connectionType}`);
}

return { schema, validatorFn: validate_config };
// checks on asset_storage_bucket
if (typedSubconfig.asset_storage_bucket && connectionType !== 's3') {
throw new Error('asset_storage_bucket can only be used if asset_storage_connection_type is set to "s3"');
}
// TODO: add regex to check if valid bucket name
}
93 changes: 50 additions & 43 deletions packages/terafoundation/src/validate-configs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import convict_format_with_validator from 'convict-format-with-validator';
// @ts-expect-error no types
import convict_format_with_moment from 'convict-format-with-moment';
import { Terafoundation } from '@terascope/types';
import { getConnectorInitializers } from './connector-utils';
import { getFoundationInitializers } from './schema';
import { getConnectorSchemaAndValFn } from './connector-utils';
import { foundationSchema, foundationValidatorFn } from './schema';
import * as i from './interfaces';

addFormats(convict_format_with_validator);
Expand Down Expand Up @@ -41,22 +41,39 @@ function validateConfig(
}
}

function extractInitializers<S>(
function extractSchema<S>(
fn: any,
sysconfig: PartialDeep<i.FoundationSysConfig<S>>
): Terafoundation.Initializers<S> {
): Terafoundation.Schema<Record<string, any>> {
if (isFunction(fn)) {
const result = fn(sysconfig);
if (result.schema) {
return result;
return result.schema;
}
return { schema: fn(sysconfig) };
return result;
}
if (isPlainObject(fn)) {
return { schema: fn };
return fn;
}

return { schema: {} };
return {};
}

function extractValidatorFn<S>(
fn: any,
sysconfig: PartialDeep<i.FoundationSysConfig<S>>
): Terafoundation.ValidatorFn<S> | undefined {
if (isFunction(fn)) {
const result = fn(sysconfig);
if (result.validatorFn) {
return result.validatorFn;
}
}
if (isPlainObject(fn)) {
return fn.validatorFn;
}

return undefined;
}

/**
Expand All @@ -82,18 +99,8 @@ export default function validateConfigs<
}

const listOfValidations: Record<string, Terafoundation.ValidationObj<S>> = {};
const {
schema: sysconfigSchema,
validatorFn: mainSvcValidatorFn
} = extractInitializers<S>(config.config_schema, sysconfig);
const mainSvcName = Object.keys(sysconfigSchema)[0];
listOfValidations[mainSvcName] = { validatorFn: mainSvcValidatorFn, subconfig: {} };
const {
schema: foundationSchema,
validatorFn: foundationValidatorFn
} = getFoundationInitializers<S>();
listOfValidations.terafoundation = { validatorFn: foundationValidatorFn, subconfig: {} };
sysconfigSchema.terafoundation = foundationSchema;
const schema = extractSchema(config.config_schema, sysconfig);
schema.terafoundation = foundationSchema();

const result: any = {};

Expand All @@ -103,22 +110,13 @@ export default function validateConfigs<
});
}

const schemaKeys = concat(Object.keys(sysconfigSchema), Object.keys(sysconfig));
const schemaKeys = concat(Object.keys(schema), Object.keys(sysconfig));
for (const schemaKey of schemaKeys) {
const subSchema = sysconfigSchema[schemaKey] || {};
const subSchema = schema[schemaKey] || {};
const subConfig: Record<string, any> = sysconfig[schemaKey] || {};
const validatedConfig = validateConfig(cluster, subSchema, subConfig);
result[schemaKey] = validatedConfig;

if (listOfValidations[schemaKey]) {
listOfValidations[schemaKey].subconfig = validatedConfig;
} else {
listOfValidations[schemaKey] = {
validatorFn: undefined,
subconfig: validatedConfig
};
}

if (schemaKey === 'terafoundation') {
result[schemaKey].connectors = {};

Expand All @@ -127,7 +125,7 @@ export default function validateConfigs<
const {
schema: connectorSchema,
validatorFn: connValidatorFn
} = getConnectorInitializers<S>(connector);
} = getConnectorSchemaAndValFn<S>(connector);

result[schemaKey].connectors[connector] = {};
for (const [connection, connectionConfig] of Object.entries(connectorConfig)) {
Expand All @@ -141,10 +139,21 @@ export default function validateConfigs<

listOfValidations[`${connector}:${connection}`] = {
validatorFn: connValidatorFn,
subconfig: validatedConnConfig
subconfig: validatedConnConfig,
connector: true
};
}
}

listOfValidations[schemaKey] = {
validatorFn: foundationValidatorFn<S>,
subconfig: validatedConfig
};
} else {
listOfValidations[schemaKey] = {
validatorFn: extractValidatorFn<S>(config.config_schema, sysconfig),
subconfig: validatedConfig
};
}
}

Expand All @@ -159,16 +168,14 @@ export default function validateConfigs<
}

// Cross-field validation
for (const key in listOfValidations) {
if (Object.prototype.hasOwnProperty.call(listOfValidations, key)) {
const obj = listOfValidations[key];

if (obj.validatorFn) {
try {
obj.validatorFn(cloneDeep(result), cloneDeep(obj.subconfig), key);
} catch (err) {
throw new TSError(`Cross-field validation failed: ${err}`);
}
for (const entry of Object.entries(listOfValidations)) {
const [name, validatorObj] = entry;

if (validatorObj.validatorFn) {
try {
validatorObj.validatorFn(cloneDeep(result), cloneDeep(validatorObj.subconfig));
} catch (err) {
throw new TSError(`Cross-field validation failed for ${validatorObj.connector ? 'connector ' : ''}'${name}': ${err}`);
}
}
}
Expand Down
19 changes: 6 additions & 13 deletions packages/terafoundation/test/validate-configs-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ import os from 'os';
import { PartialDeep, Terafoundation } from 'packages/types/dist/src';
import { Cluster } from '../src';
import validateConfigs from '../src/validate-configs';
import { getFoundationInitializers } from '../src/schema';
import { getConnectorInitializers } from '../src/connector-utils';
import { getConnectorSchemaAndValFn } from '../src/connector-utils';

describe('Validate Configs', () => {
describe('when using mainly defaults', () => {
Expand Down Expand Up @@ -434,11 +433,10 @@ describe('Validate Configs', () => {

const testFn = (
sysconfig: Terafoundation.SysConfig<any>,
subconfig: PartialDeep<Terafoundation.SysConfig<any>>,
name: string) => {
subconfig: PartialDeep<Terafoundation.SysConfig<any>>) => {
const typedSubconfig = subconfig as unknown as Terafoundation.Foundation;
if (sysconfig.terafoundation.workers !== typedSubconfig.workers) {
throw new Error(`${name} validatorFn test failed`);
throw new Error('validatorFn test failed');
}
};
const config = {
Expand All @@ -448,20 +446,15 @@ describe('Validate Configs', () => {
};

it('should throw an error', () => {
expect(() => validateConfigs(cluster as any, config as any, configFile as any)).toThrow('teraslice validatorFn test failed');
expect(() => validateConfigs(cluster as any, config as any, configFile as any))
.toThrow('Cross-field validation failed for \'teraslice\': Error: validatorFn test failed');
});
});
});

describe('getFoundationInitializers', () => {
it('should return an initializer with schema key', () => {
expect(getFoundationInitializers()).toContainKey('schema');
});
});

describe('getConnectorInitializers', () => {
it('should return an initializer with schema key', () => {
const connector = 'elasticsearch-next';
expect(getConnectorInitializers(connector)).toContainKey('schema');
expect(getConnectorSchemaAndValFn(connector)).toContainKey('schema');
});
});
6 changes: 3 additions & 3 deletions packages/types/src/terafoundation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ export type Initializers<S> = {

export type ValidationObj<S>= {
subconfig: Record<string, any>,
validatorFn?: ValidatorFn<S>
validatorFn?: ValidatorFn<S>,
connector?: boolean
}

export type ValidatorFn<S> = (
sysconfig: SysConfig<S>,
subconfig: Record<string, any>,
name: string
subconfig: Record<string, any>
) => void

export type Config<
Expand Down

0 comments on commit 7654b13

Please sign in to comment.