Skip to content

Commit

Permalink
feat(tools): improve JSON schema inference
Browse files Browse the repository at this point in the history 
Signed-off-by: Matt Roberts <[email protected]>
  • Loading branch information
@mttrbrts
mttrbrts committed Nov 17, 2022
1 parent 31ee08c commit 1eb78ca
Show file tree
Hide file tree
Showing 9 changed files with 155 additions and 74 deletions.
22 changes: 17 additions & 5 deletions packages/concerto-cli/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

const Logger = require('@accordproject/concerto-util').Logger;
const { glob } = require('glob');
const fs = require('fs');
const Commands = require('./lib/commands');

require('yargs')
Expand Down Expand Up @@ -307,30 +308,41 @@ require('yargs')
});
})
.command('infer', 'generate a concerto model from a source schema', (yargs) => {
yargs.demandOption(['input', 'format', 'namespace', 'output']);
yargs.demandOption(['input', 'namespace']);
yargs.option('input', {
describe: 'path to the input file',
type: 'string',
});
yargs.option('output', {
describe: 'path to the output file',
type: 'string'
type: 'string',
});
yargs.option('format', {
describe: 'either `openapi` or `jsonSchema',
describe: 'either `openapi` or `jsonSchema`',
default: 'jsonSchema',
type: 'string'
});
yargs.option('namespace', {
describe: 'The namepspace for the output model',
describe: 'The namespace for the output model',
type: 'string',
});
yargs.option('typeName', {
describe: 'The name of the root type',
type: 'string',
default: 'Root'
});
}, (argv) => {
if (argv.verbose) {
Logger.info(`Infer Concerto model from ${argv.input} in the ${argv.format} format`);
}

try {
return Commands.inferConcertoSchema(argv.input, argv.output, argv.format, argv.namespace);
const cto = Commands.inferConcertoSchema(argv.input, argv.namespace, argv.typeName, argv.format, argv.output);
if (argv.output){
fs.writeFileSync(argv.output, cto);
} else {
console.log(cto);
}
} catch (err){
Logger.error(err);
return;
Expand Down
20 changes: 11 additions & 9 deletions packages/concerto-cli/lib/commands.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const fs = require('fs');
const path = require('path');
const semver = require('semver');
const toJsonSchema = require('@openapi-contrib/openapi-schema-to-json-schema');
const migrate = require('json-schema-migrate');

const Logger = require('@accordproject/concerto-util').Logger;
const FileWriter = require('@accordproject/concerto-util').FileWriter;
Expand Down Expand Up @@ -563,21 +564,22 @@ class Commands {
/**
* Generate a Concerto model from another schema format
* @param {string} input The source file.
* @param {string} output The target file.
* @param {string} format The source format
* @param {string} namespace The namepspace for the output model
* @param {string} [typeName] The name of the root concept
* @param {string} [format] The source format
* @param {string} [output] The target file.
*
* @returns {string} a CTO string
*/
static inferConcertoSchema(input, output, format, namespace) {
static inferConcertoSchema(input, namespace, typeName = 'Root', format = 'jsonSchema', output) {
let schema = JSON.parse(fs.readFileSync(input, 'utf8'));

if (format.toLowerCase() === 'openapi'){
schema = toJsonSchema(schema);
fs.writeFileSync(`${output}.jsonschema.json`, JSON.stringify(schema, null, 2));
const jsonSchema = toJsonSchema(schema);
migrate.draft2020(jsonSchema);
return CodeGen.InferFromJsonSchema(namespace, typeName, jsonSchema);
}
const outputPath = output || `${input}.cto`;
const cto = CodeGen.InferFromJsonSchema(namespace, 'Model', schema);
Logger.info('Creating file: ' + outputPath);
fs.writeFileSync(outputPath, cto);
return CodeGen.InferFromJsonSchema(namespace, typeName, schema);
}
}

Expand Down
74 changes: 51 additions & 23 deletions packages/concerto-tools/lib/codegen/fromJsonSchema/cto/inferModel.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,20 +73,25 @@ function parseIdUri(id) {

/**
* Infer a type name for a definition. Examines $id, title and parent declaration
* @param {*} definition the input object
* @param {*} context the processing context
* @param {object} definition - the input object
* @param {*} context - the processing context
* @param {boolean} skipDictionary - if true, this function will not use the dictionary help inference
* @returns {string} A name for the definition
* @private
*/
function inferTypeName(definition, context) {
function inferTypeName(definition, context, skipDictionary) {
if (definition.$ref) {
return normalizeType(definition.$ref);
}
const name = context.parents.peek();
const { type } = parseIdUri(definition.$id) ||
{ type: definition.title || name };

// TODO Resolve $ref definitions
// if (definition.$ref) {}

return normalizeType(type);
if (skipDictionary || context.dictionary.has(normalizeType(type))){
return normalizeType(type);
}
// We fallback to a stringified object representation. This is "untyped".
return 'String';
}

/**
Expand All @@ -107,12 +112,12 @@ function inferType(definition, context) {
return parent;
}

return inferTypeName(definition, context);
return inferTypeName(definition, context, false);
}

// TODO Also add local sub-schema definition
if (definition.enum) {
return inferTypeName(definition, context);
return inferTypeName(definition, context, false);
}

if (definition.type) {
Expand All @@ -121,10 +126,9 @@ function inferType(definition, context) {
if (definition.format) {
if (definition.format === 'date-time' || definition.format === 'date') {
return 'DateTime';
} else if (definition.format === 'decimal') {
return 'Double';
} else {
throw new Error(`Format '${definition.format}' in '${name}' is not supported`);
console.warn(`Format '${definition.format}' in '${name}' is not supported. It has been ignored.`);
return 'String';
}
}
return 'String';
Expand All @@ -137,17 +141,23 @@ function inferType(definition, context) {
case 'array':
return inferType(definition.items, context) + '[]';
case 'object':
return inferTypeName(definition, context);
return inferTypeName(definition, context, false);
default:
throw new Error(`Type keyword '${definition.type}' in '${name}' is not supported`);
}
}

// Hack until we support union types.
// https://github.com/accordproject/concerto/issues/292
if (definition.anyOf){
const alternative = definition.anyOf || definition.oneOf;
if (alternative){
const keyword = definition.anyOf ? 'anyOf' : 'oneOf';
console.warn(
`Keyword '${keyword}' in definition '${name}' is not fully supported. Defaulting to first alternative.`
);

// Just choose the first item
return inferType(definition.anyOf[0], context);
return inferType(alternative[0], context);
}

throw new Error(`Unsupported definition: ${JSON.stringify(definition)}`);
Expand All @@ -162,7 +172,7 @@ function inferType(definition, context) {
function inferEnum(definition, context) {
const { writer } = context;

writer.writeLine(0, `enum ${inferTypeName(definition, context)} {`);
writer.writeLine(0, `enum ${inferTypeName(definition, context, false)} {`);
definition.enum.forEach((value) => {
writer.writeLine(
1,
Expand Down Expand Up @@ -251,9 +261,13 @@ function inferDeclaration(definition, context) {
} else if (definition.type) {
if (definition.type === 'object') {
inferConcept(definition, context);
} else if (definition.type === 'array') {
console.warn(
`Type keyword 'array' in definition '${name}' is not supported. It has been ignored.`
);
} else {
throw new Error(
`Type keyword '${definition.type}' in definition '${name}' not supported.`
`Type keyword '${definition.type}' in definition '${name}' is not supported.`
);
}
} else {
Expand All @@ -263,7 +277,7 @@ function inferDeclaration(definition, context) {
!key.startsWith('x-') // Ignore custom extensions
);
console.warn(
`Keyword(s) '${badKeys.join('\', \'')}' in definition '${name}' not supported.`
`Keyword(s) '${badKeys.join('\', \'')}' in definition '${name}' are not supported.`
);
}
}
Expand Down Expand Up @@ -298,18 +312,32 @@ function inferModelFile(defaultNamespace, defaultType, schema) {
const context = {
parents: new TypedStack(),
writer: new Writer(),
dictionary: new Set(),
};

context.writer.writeLine(0, `namespace ${namespace}`);
context.writer.writeLine(0, '');

// Add imports
// TODO we need some heuristic or metadata to identify Concerto dependencies rather than making assumptions
context.writer.writeLine(0, 'import org.accordproject.time.* from https://models.accordproject.org/[email protected]');
context.writer.writeLine(0, '');

// Create definitions
const defs = schema.definitions || schema.$defs || schema?.components?.schemas ||[];

// Build a dictionary
context.dictionary.add(defaultType);
if (schema.$id) {
context.dictionary.add(normalizeType(parseIdUri(schema.$id).type));
}
Object.keys(defs).forEach((key) => {
context.parents.push(key);
const definition = defs[key];
const typeName = inferTypeName(definition, context, true);
if (context.dictionary.has(typeName)){
throw new Error(`Duplicate definition found for type '${typeName}'.`);
}
context.dictionary.add(typeName);
context.parents.pop();
});

// Visit each declaration
Object.keys(defs).forEach((key) => {
context.parents.push(key);
const definition = defs[key];
Expand Down
4 changes: 2 additions & 2 deletions packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/full.cto
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
namespace org.acme

import org.accordproject.time.* from https://models.accordproject.org/[email protected]

concept Children {
o String name
o Integer age
Expand All @@ -16,6 +14,8 @@ concept Children {
o String[] emptyArray
o Pet[] favoritePets
o Stuff[] stuff
o String json optional
o Double alternation optional
}

enum Color {
Expand Down
10 changes: 10 additions & 0 deletions packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,16 @@
},
"required": ["$class", "sku", "price", "product"]
}
},
"json": {
"type": "object"
},
"alternation": {
"oneOf": [
{ "type": "number" },
{ "type": "string" }
]

}
},
"required": [
Expand Down
Loading

0 comments on commit 1eb78ca

Please sign in to comment.