From 408c69ae0996487208008a41f72d07fe382f65f6 Mon Sep 17 00:00:00 2001 From: Matt Roberts Date: Thu, 17 Nov 2022 16:47:17 +0000 Subject: [PATCH] feat(tools): improve JSON schema inference Signed-off-by: Matt Roberts --- packages/concerto-cli/index.js | 8 +- .../codegen/fromJsonSchema/cto/inferModel.js | 63 ++++++++++----- .../codegen/fromJsonSchema/cto/data/full.cto | 4 +- .../fromJsonSchema/cto/data/schema.json | 10 +++ .../codegen/fromJsonSchema/cto/inferModel.js | 78 ++++++++++++------- packages/concerto-tools/types/index.d.ts | 1 + .../types/lib/codegen/codegen.d.ts | 3 +- 7 files changed, 115 insertions(+), 52 deletions(-) diff --git a/packages/concerto-cli/index.js b/packages/concerto-cli/index.js index 319cc8cec9..7a548769f1 100755 --- a/packages/concerto-cli/index.js +++ b/packages/concerto-cli/index.js @@ -18,6 +18,7 @@ const Logger = require('@accordproject/concerto-util').Logger; const fs = require('fs'); const { glob } = require('glob'); +const fs = require('fs'); const Commands = require('./lib/commands'); require('yargs') @@ -323,8 +324,13 @@ require('yargs') type: 'string' }); yargs.option('namespace', { - describe: 'The namepspace for the output model', + describe: 'The namespace for the output model', + type: 'string', + }); + yargs.option('typeName', { + describe: 'The name of the root type', type: 'string', + default: 'Root' }); yargs.option('typeName', { describe: 'The name of the root type', diff --git a/packages/concerto-tools/lib/codegen/fromJsonSchema/cto/inferModel.js b/packages/concerto-tools/lib/codegen/fromJsonSchema/cto/inferModel.js index 5cf912c619..edc08792a6 100644 --- a/packages/concerto-tools/lib/codegen/fromJsonSchema/cto/inferModel.js +++ b/packages/concerto-tools/lib/codegen/fromJsonSchema/cto/inferModel.js @@ -73,8 +73,9 @@ function parseIdUri(id) { /** * Infer a type name for a definition. Examines $id, title and parent declaration - * @param {*} definition the input object - * @param {*} context the processing context + * @param {object} definition - the input object + * @param {*} context - the processing context + * @param {boolean} skipDictionary - if true, this function will not use the dictionary help inference * @returns {string} A name for the definition * @private */ @@ -86,7 +87,12 @@ function inferTypeName(definition, context) { const name = context.parents.peek(); const { type } = parseIdUri(definition.$id) || { type: definition.title || name }; - return normalizeType(type); + + if (skipDictionary || context.dictionary.has(normalizeType(type))){ + return normalizeType(type); + } + // We fallback to a stringified object representation. This is "untyped". + return 'String'; } /** @@ -107,12 +113,12 @@ function inferType(definition, context) { return parent; } - return inferTypeName(definition, context); + return inferTypeName(definition, context, false); } // TODO Also add local sub-schema definition if (definition.enum) { - return inferTypeName(definition, context); + return inferTypeName(definition, context, false); } if (definition.type) { @@ -122,7 +128,8 @@ function inferType(definition, context) { if (definition.format === 'date-time' || definition.format === 'date') { return 'DateTime'; } else { - throw new Error(`Format '${definition.format}' in '${name}' is not supported`); + console.warn(`Format '${definition.format}' in '${name}' is not supported. It has been ignored.`); + return 'String'; } } return 'String'; @@ -135,7 +142,7 @@ function inferType(definition, context) { case 'array': return inferType(definition.items, context) + '[]'; case 'object': - return inferTypeName(definition, context); + return inferTypeName(definition, context, false); default: throw new Error(`Type keyword '${definition.type}' in '${name}' is not supported`); } @@ -143,13 +150,15 @@ function inferType(definition, context) { // Hack until we support union types. // https://github.com/accordproject/concerto/issues/292 - if (definition.anyOf){ + const alternative = definition.anyOf || definition.oneOf; + if (alternative){ + const keyword = definition.anyOf ? 'anyOf' : 'oneOf'; console.warn( - `Keyword 'anyOf' in definition '${name}' is not fully supported. Defaulting to first alternative.` + `Keyword '${keyword}' in definition '${name}' is not fully supported. Defaulting to first alternative.` ); // Just choose the first item - return inferType(definition.anyOf[0], context); + return inferType(alternative[0], context); } throw new Error(`Unsupported definition: ${JSON.stringify(definition)}`); @@ -164,7 +173,7 @@ function inferType(definition, context) { function inferEnum(definition, context) { const { writer } = context; - writer.writeLine(0, `enum ${inferTypeName(definition, context)} {`); + writer.writeLine(0, `enum ${inferTypeName(definition, context, false)} {`); definition.enum.forEach((value) => { writer.writeLine( 1, @@ -253,9 +262,13 @@ function inferDeclaration(definition, context) { } else if (definition.type) { if (definition.type === 'object') { inferConcept(definition, context); + } else if (definition.type === 'array') { + console.warn( + `Type keyword 'array' in definition '${name}' is not supported. It has been ignored.` + ); } else { throw new Error( - `Type keyword '${definition.type}' in definition '${name}' not supported.` + `Type keyword '${definition.type}' in definition '${name}' is not supported.` ); } } else { @@ -265,7 +278,7 @@ function inferDeclaration(definition, context) { !key.startsWith('x-') // Ignore custom extensions ); console.warn( - `Keyword(s) '${badKeys.join('\', \'')}' in definition '${name}' is not supported.` + `Keyword(s) '${badKeys.join('\', \'')}' in definition '${name}' are not supported.` ); } } @@ -304,18 +317,32 @@ function inferModelFile(defaultNamespace, defaultType, schema) { const context = { parents: new TypedStack(), writer: new Writer(), + dictionary: new Set(), }; context.writer.writeLine(0, `namespace ${namespace}`); context.writer.writeLine(0, ''); - // Add imports - // TODO we need some heuristic or metadata to identify Concerto dependencies rather than making assumptions - context.writer.writeLine(0, 'import org.accordproject.time.* from https://models.accordproject.org/time@0.2.0.cto'); - context.writer.writeLine(0, ''); - // Create definitions const defs = schema.definitions || schema.$defs || schema?.components?.schemas ||[]; + + // Build a dictionary + context.dictionary.add(defaultType); + if (schema.$id) { + context.dictionary.add(normalizeType(parseIdUri(schema.$id).type)); + } + Object.keys(defs).forEach((key) => { + context.parents.push(key); + const definition = defs[key]; + const typeName = inferTypeName(definition, context, true); + if (context.dictionary.has(typeName)){ + throw new Error(`Duplicate definition found for type '${typeName}'.`); + } + context.dictionary.add(typeName); + context.parents.pop(); + }); + + // Visit each declaration Object.keys(defs).forEach((key) => { context.parents.push(key); const definition = defs[key]; diff --git a/packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/full.cto b/packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/full.cto index de831413bd..3915974ffe 100644 --- a/packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/full.cto +++ b/packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/full.cto @@ -1,7 +1,5 @@ namespace org.acme -import org.accordproject.time.* from https://models.accordproject.org/time@0.2.0.cto - concept Children { o String name o Integer age @@ -16,6 +14,8 @@ concept Children { o String[] emptyArray o Pet[] favoritePets o Stuff[] stuff + o String json optional + o Double alternation optional } enum Color { diff --git a/packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/schema.json b/packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/schema.json index ac616390b2..96bde41110 100644 --- a/packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/schema.json +++ b/packages/concerto-tools/test/codegen/fromJsonSchema/cto/data/schema.json @@ -142,6 +142,16 @@ }, "required": ["$class", "sku", "price", "product"] } + }, + "json": { + "type": "object" + }, + "alternation": { + "oneOf": [ + { "type": "number" }, + { "type": "string" } + ] + } }, "required": [ diff --git a/packages/concerto-tools/test/codegen/fromJsonSchema/cto/inferModel.js b/packages/concerto-tools/test/codegen/fromJsonSchema/cto/inferModel.js index a02bdcee1b..ca8b00a2e8 100644 --- a/packages/concerto-tools/test/codegen/fromJsonSchema/cto/inferModel.js +++ b/packages/concerto-tools/test/codegen/fromJsonSchema/cto/inferModel.js @@ -63,8 +63,6 @@ describe('inferModel', function () { }); cto.should.equal(`namespace org.acme -import org.accordproject.time.* from https://models.accordproject.org/time@0.2.0.cto - enum Root { o one o two @@ -86,13 +84,11 @@ enum Root { } } }); - // TODO This is not a valid CTO model, because we don't generate definitions for inline sub-schemas. + // TODO Generate definitions for inline sub-schemas. cto.should.equal(`namespace org.acme -import org.accordproject.time.* from https://models.accordproject.org/time@0.2.0.cto - concept Root { - o Xs[] xs optional + o String[] xs optional } `); @@ -112,8 +108,6 @@ concept Root { ); cto.should.equal(`namespace org.acme -import org.accordproject.time.* from https://models.accordproject.org/time@0.2.0.cto - concept Root { o String name optional o Root[] children optional @@ -127,8 +121,6 @@ concept Root { const cto = inferModel('org.acme', 'Root', schema); cto.should.equal(`namespace com.example -import org.accordproject.time.* from https://models.accordproject.org/time@0.2.0.cto - concept Veggie { o String veggieName o Boolean veggieLike @@ -147,8 +139,6 @@ concept Arrays { const cto = inferModel('org.acme', 'Root', schema); cto.should.equal(`namespace com.example -import org.accordproject.time.* from https://models.accordproject.org/time@0.2.0.cto - concept Geographical_location { o String name default="home" regex=/[\\w\\s]+/ optional o Double latitude @@ -190,23 +180,28 @@ concept Geographical_location { }).should.throw('\'additionalProperties\' are not supported in Concerto'); }); - it('should not generate when unsupported formats are used', async () => { - (function () { - inferModel('org.acme', 'Root', { - $schema: 'http://json-schema.org/draft-07/schema#', - definitions: { - Foo: { - type: 'object', - properties: { - email: { - type: 'string', - format: 'email' - } + it('should quietly accept unsupported formats', async () => { + const cto = inferModel('org.acme', 'Root', { + $schema: 'http://json-schema.org/draft-07/schema#', + definitions: { + Foo: { + type: 'object', + properties: { + email: { + type: 'string', + format: 'email' } } } - }); - }).should.throw('Format \'email\' in \'email\' is not supported'); + } + }); + cto.should.equal(`namespace org.acme + +concept Foo { + o String email optional +} + +`); }); it('should not generate when unsupported type keywords are used', async () => { @@ -219,7 +214,7 @@ concept Geographical_location { } } }); - }).should.throw('Type keyword \'null\' in definition \'Foo\' not supported.'); + }).should.throw('Type keyword \'null\' in definition \'Foo\' is not supported.'); }); it('should not generate when unsupported type keywords are used in an object', async () => { @@ -240,11 +235,36 @@ concept Geographical_location { }).should.throw('Type keyword \'null\' in \'email\' is not supported'); }); - it('should not fail for unsupported keywords', async () => { - inferModel('org.acme', 'Root', { + it('should not generate when duplicate definitions are found', async () => { + (function () { + inferModel('org.acme', 'Foo', { + $schema: 'http://json-schema.org/draft-07/schema#', + definitions: { + 'Foo': { + 'type': 'object', + }, + } + }); + }).should.throw('Duplicate definition found for type \'Foo\''); + }); + + it('should quietly accept array definitions', async () => { + const cto = inferModel('org.acme', 'Root', { + type: 'array' + }); + cto.should.equal(`namespace org.acme + +`); + }); + + it('should quietly accept unsupported definitions', async () => { + const cto = inferModel('org.acme', 'Root', { 'allOf': [ { 'type': 'string' } ] }); + cto.should.equal(`namespace org.acme + +`); }); }); diff --git a/packages/concerto-tools/types/index.d.ts b/packages/concerto-tools/types/index.d.ts index 785ac75017..f0d84649f1 100644 --- a/packages/concerto-tools/types/index.d.ts +++ b/packages/concerto-tools/types/index.d.ts @@ -27,5 +27,6 @@ export var CodeGen: { markdown: typeof import("./lib/codegen/fromcto/markdown/markdownvisitor"); protobuf: typeof import("./lib/codegen/fromcto/protobuf/protobufvisitor"); }; + InferFromJsonSchema: typeof import("./lib/codegen/fromJsonSchema/cto/inferModel"); }; export var version: any; diff --git a/packages/concerto-tools/types/lib/codegen/codegen.d.ts b/packages/concerto-tools/types/lib/codegen/codegen.d.ts index 23bb69b136..9e4db3f8e2 100644 --- a/packages/concerto-tools/types/lib/codegen/codegen.d.ts +++ b/packages/concerto-tools/types/lib/codegen/codegen.d.ts @@ -10,7 +10,6 @@ import CSharpVisitor = require("./fromcto/csharp/csharpvisitor"); import ODataVisitor = require("./fromcto/odata/odatavisitor"); import MermaidVisitor = require("./fromcto/mermaid/mermaidvisitor"); import MarkdownVisitor = require("./fromcto/markdown/markdownvisitor"); -import ProtobufVisitor = require("./fromcto/protobuf/protobufvisitor"); import InferFromJsonSchema = require("./fromJsonSchema/cto/inferModel"); export declare namespace formats { export { GoLangVisitor as golang }; @@ -26,4 +25,4 @@ export declare namespace formats { export { MarkdownVisitor as markdown }; export { ProtobufVisitor as protobuf }; } -export { AbstractPlugin, GoLangVisitor, JSONSchemaVisitor, XmlSchemaVisitor, PlantUMLVisitor, TypescriptVisitor, JavaVisitor, GraphQLVisitor, CSharpVisitor, ODataVisitor, MermaidVisitor, MarkdownVisitor, ProtobufVisitor, InferFromJsonSchema }; +export { AbstractPlugin, GoLangVisitor, JSONSchemaVisitor, XmlSchemaVisitor, PlantUMLVisitor, TypescriptVisitor, JavaVisitor, GraphQLVisitor, CSharpVisitor, ODataVisitor, MermaidVisitor, MarkdownVisitor, InferFromJsonSchema };