Skip to content

Commit

Permalink
feat(tools): improve JSON schema inference
Browse files Browse the repository at this point in the history 
Signed-off-by: Matt Roberts <[email protected]>
  • Loading branch information
@mttrbrts
mttrbrts committed Nov 17, 2022
1 parent 346c5b0 commit 408c69a
Show file tree
Hide file tree
Showing 7 changed files with 115 additions and 52 deletions.
8 changes: 7 additions & 1 deletion packages/concerto-cli/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
const Logger = require('@accordproject/concerto-util').Logger;
const fs = require('fs');
const { glob } = require('glob');
const fs = require('fs');
const Commands = require('./lib/commands');

require('yargs')
Expand Down Expand Up @@ -323,8 +324,13 @@ require('yargs')
type: 'string'
});
yargs.option('namespace', {
describe: 'The namepspace for the output model',
describe: 'The namespace for the output model',
type: 'string',
});
yargs.option('typeName', {
describe: 'The name of the root type',
type: 'string',
default: 'Root'
});
yargs.option('typeName', {
describe: 'The name of the root type',
Expand Down
63 changes: 45 additions & 18 deletions packages/concerto-tools/lib/codegen/fromJsonSchema/cto/inferModel.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,9 @@ function parseIdUri(id) {

/**
* Infer a type name for a definition. Examines $id, title and parent declaration
* @param {*} definition the input object
* @param {*} context the processing context
* @param {object} definition - the input object
* @param {*} context - the processing context
* @param {boolean} skipDictionary - if true, this function will not use the dictionary help inference
* @returns {string} A name for the definition
* @private
*/
Expand All @@ -86,7 +87,12 @@ function inferTypeName(definition, context) {
const name = context.parents.peek();
const { type } = parseIdUri(definition.$id) ||
{ type: definition.title || name };
return normalizeType(type);

if (skipDictionary || context.dictionary.has(normalizeType(type))){
return normalizeType(type);
}
// We fallback to a stringified object representation. This is "untyped".
return 'String';
}

/**
Expand All @@ -107,12 +113,12 @@ function inferType(definition, context) {
return parent;
}

return inferTypeName(definition, context);
return inferTypeName(definition, context, false);
}

// TODO Also add local sub-schema definition
if (definition.enum) {
return inferTypeName(definition, context);
return inferTypeName(definition, context, false);
}

if (definition.type) {
Expand All @@ -122,7 +128,8 @@ function inferType(definition, context) {
if (definition.format === 'date-time' || definition.format === 'date') {
return 'DateTime';
} else {
throw new Error(`Format '${definition.format}' in '${name}' is not supported`);
console.warn(`Format '${definition.format}' in '${name}' is not supported. It has been ignored.`);
return 'String';
}
}
return 'String';
Expand All @@ -135,21 +142,23 @@ function inferType(definition, context) {
case 'array':
return inferType(definition.items, context) + '[]';
case 'object':
return inferTypeName(definition, context);
return inferTypeName(definition, context, false);
default:
throw new Error(`Type keyword '${definition.type}' in '${name}' is not supported`);
}
}

// Hack until we support union types.
// https://github.com/accordproject/concerto/issues/292
if (definition.anyOf){
const alternative = definition.anyOf || definition.oneOf;
if (alternative){
const keyword = definition.anyOf ? 'anyOf' : 'oneOf';
console.warn(
`Keyword 'anyOf' in definition '${name}' is not fully supported. Defaulting to first alternative.`
`Keyword '${keyword}' in definition '${name}' is not fully supported. Defaulting to first alternative.`
);

// Just choose the first item
return inferType(definition.anyOf[0], context);
return inferType(alternative[0], context);
}

throw new Error(`Unsupported definition: ${JSON.stringify(definition)}`);
Expand All @@ -164,7 +173,7 @@ function inferType(definition, context) {
function inferEnum(definition, context) {
const { writer } = context;

writer.writeLine(0, `enum ${inferTypeName(definition, context)} {`);
writer.writeLine(0, `enum ${inferTypeName(definition, context, false)} {`);
definition.enum.forEach((value) => {
writer.writeLine(
1,
Expand Down Expand Up @@ -253,9 +262,13 @@ function inferDeclaration(definition, context) {
} else if (definition.type) {
if (definition.type === 'object') {
inferConcept(definition, context);
} else if (definition.type === 'array') {
console.warn(
`Type keyword 'array' in definition '${name}' is not supported. It has been ignored.`
);
} else {
throw new Error(
`Type keyword '${definition.type}' in definition '${name}' not supported.`
`Type keyword '${definition.type}' in definition '${name}' is not supported.`
);
}
} else {
Expand All @@ -265,7 +278,7 @@ function inferDeclaration(definition, context) {
!key.startsWith('x-') // Ignore custom extensions
);
console.warn(
`Keyword(s) '${badKeys.join('\', \'')}' in definition '${name}' is not supported.`
`Keyword(s) '${badKeys.join('\', \'')}' in definition '${name}' are not supported.`
);
}
}
Expand Down Expand Up @@ -304,18 +317,32 @@ function inferModelFile(defaultNamespace, defaultType, schema) {
const context = {
parents: new TypedStack(),
writer: new Writer(),
dictionary: new Set(),
};

context.writer.writeLine(0, `namespace ${namespace}`);
context.writer.writeLine(0, '');

// Add imports
// TODO we need some heuristic or metadata to identify Concerto dependencies rather than making assumptions
context.writer.writeLine(0, 'import org.accordproject.time.* from https://models.accordproject.org/[email protected]');
context.writer.writeLine(0, '');

// Create definitions
const defs = schema.definitions || schema.$defs || schema?.components?.schemas ||[];

// Build a dictionary
context.dictionary.add(defaultType);
if (schema.$id) {
context.dictionary.add(normalizeType(parseIdUri(schema.$id).type));
}
Object.keys(defs).forEach((key) => {
context.parents.push(key);
const definition = defs[key];
const typeName = inferTypeName(definition, context, true);
if (context.dictionary.has(typeName)){
throw new Error(`Duplicate definition found for type '${typeName}'.`);
}
context.dictionary.add(typeName);
context.parents.pop();
});

// Visit each declaration
Object.keys(defs).forEach((key) => {
context.parents.push(key);
const definition = defs[key];
Expand Down
4 changes: 2 additions & 2 deletions packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/full.cto
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
namespace org.acme

import org.accordproject.time.* from https://models.accordproject.org/[email protected]

concept Children {
o String name
o Integer age
Expand All @@ -16,6 +14,8 @@ concept Children {
o String[] emptyArray
o Pet[] favoritePets
o Stuff[] stuff
o String json optional
o Double alternation optional
}

enum Color {
Expand Down
10 changes: 10 additions & 0 deletions packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,16 @@
},
"required": ["$class", "sku", "price", "product"]
}
},
"json": {
"type": "object"
},
"alternation": {
"oneOf": [
{ "type": "number" },
{ "type": "string" }
]

}
},
"required": [
Expand Down
78 changes: 49 additions & 29 deletions packages/concerto-tools/test/codegen/fromJsonSchema/cto/inferModel.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,6 @@ describe('inferModel', function () {
});
cto.should.equal(`namespace org.acme
import org.accordproject.time.* from https://models.accordproject.org/[email protected]
enum Root {
o one
o two
Expand All @@ -86,13 +84,11 @@ enum Root {
}
}
});
// TODO This is not a valid CTO model, because we don't generate definitions for inline sub-schemas.
// TODO Generate definitions for inline sub-schemas.
cto.should.equal(`namespace org.acme
import org.accordproject.time.* from https://models.accordproject.org/[email protected]
concept Root {
o Xs[] xs optional
o String[] xs optional
}
`);
Expand All @@ -112,8 +108,6 @@ concept Root {
);
cto.should.equal(`namespace org.acme
import org.accordproject.time.* from https://models.accordproject.org/[email protected]
concept Root {
o String name optional
o Root[] children optional
Expand All @@ -127,8 +121,6 @@ concept Root {
const cto = inferModel('org.acme', 'Root', schema);
cto.should.equal(`namespace com.example
import org.accordproject.time.* from https://models.accordproject.org/[email protected]
concept Veggie {
o String veggieName
o Boolean veggieLike
Expand All @@ -147,8 +139,6 @@ concept Arrays {
const cto = inferModel('org.acme', 'Root', schema);
cto.should.equal(`namespace com.example
import org.accordproject.time.* from https://models.accordproject.org/[email protected]
concept Geographical_location {
o String name default="home" regex=/[\\w\\s]+/ optional
o Double latitude
Expand Down Expand Up @@ -190,23 +180,28 @@ concept Geographical_location {
}).should.throw('\'additionalProperties\' are not supported in Concerto');
});

it('should not generate when unsupported formats are used', async () => {
(function () {
inferModel('org.acme', 'Root', {
$schema: 'http://json-schema.org/draft-07/schema#',
definitions: {
Foo: {
type: 'object',
properties: {
email: {
type: 'string',
format: 'email'
}
it('should quietly accept unsupported formats', async () => {
const cto = inferModel('org.acme', 'Root', {
$schema: 'http://json-schema.org/draft-07/schema#',
definitions: {
Foo: {
type: 'object',
properties: {
email: {
type: 'string',
format: 'email'
}
}
}
});
}).should.throw('Format \'email\' in \'email\' is not supported');
}
});
cto.should.equal(`namespace org.acme
concept Foo {
o String email optional
}
`);
});

it('should not generate when unsupported type keywords are used', async () => {
Expand All @@ -219,7 +214,7 @@ concept Geographical_location {
}
}
});
}).should.throw('Type keyword \'null\' in definition \'Foo\' not supported.');
}).should.throw('Type keyword \'null\' in definition \'Foo\' is not supported.');
});

it('should not generate when unsupported type keywords are used in an object', async () => {
Expand All @@ -240,11 +235,36 @@ concept Geographical_location {
}).should.throw('Type keyword \'null\' in \'email\' is not supported');
});

it('should not fail for unsupported keywords', async () => {
inferModel('org.acme', 'Root', {
it('should not generate when duplicate definitions are found', async () => {
(function () {
inferModel('org.acme', 'Foo', {
$schema: 'http://json-schema.org/draft-07/schema#',
definitions: {
'Foo': {
'type': 'object',
},
}
});
}).should.throw('Duplicate definition found for type \'Foo\'');
});

it('should quietly accept array definitions', async () => {
const cto = inferModel('org.acme', 'Root', {
type: 'array'
});
cto.should.equal(`namespace org.acme
`);
});

it('should quietly accept unsupported definitions', async () => {
const cto = inferModel('org.acme', 'Root', {
'allOf': [
{ 'type': 'string' }
]
});
cto.should.equal(`namespace org.acme
`);
});
});
1 change: 1 addition & 0 deletions packages/concerto-tools/types/index.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,6 @@ export var CodeGen: {
markdown: typeof import("./lib/codegen/fromcto/markdown/markdownvisitor");
protobuf: typeof import("./lib/codegen/fromcto/protobuf/protobufvisitor");
};
InferFromJsonSchema: typeof import("./lib/codegen/fromJsonSchema/cto/inferModel");
};
export var version: any;
3 changes: 1 addition & 2 deletions packages/concerto-tools/types/lib/codegen/codegen.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import CSharpVisitor = require("./fromcto/csharp/csharpvisitor");
import ODataVisitor = require("./fromcto/odata/odatavisitor");
import MermaidVisitor = require("./fromcto/mermaid/mermaidvisitor");
import MarkdownVisitor = require("./fromcto/markdown/markdownvisitor");
import ProtobufVisitor = require("./fromcto/protobuf/protobufvisitor");
import InferFromJsonSchema = require("./fromJsonSchema/cto/inferModel");
export declare namespace formats {
export { GoLangVisitor as golang };
Expand All @@ -26,4 +25,4 @@ export declare namespace formats {
export { MarkdownVisitor as markdown };
export { ProtobufVisitor as protobuf };
}
export { AbstractPlugin, GoLangVisitor, JSONSchemaVisitor, XmlSchemaVisitor, PlantUMLVisitor, TypescriptVisitor, JavaVisitor, GraphQLVisitor, CSharpVisitor, ODataVisitor, MermaidVisitor, MarkdownVisitor, ProtobufVisitor, InferFromJsonSchema };
export { AbstractPlugin, GoLangVisitor, JSONSchemaVisitor, XmlSchemaVisitor, PlantUMLVisitor, TypescriptVisitor, JavaVisitor, GraphQLVisitor, CSharpVisitor, ODataVisitor, MermaidVisitor, MarkdownVisitor, InferFromJsonSchema };

0 comments on commit 408c69a

Please sign in to comment.