From 4b0e1c754184f48417bfca2328415aa0428aaf3b Mon Sep 17 00:00:00 2001 From: Matt Roberts Date: Tue, 16 Aug 2022 11:52:10 +0100 Subject: [PATCH 1/3] feat(stringvalidator): client provided regex Signed-off-by: Matt Roberts --- package-lock.json | 15 ---- .../concerto-core/lib/basemodelmanager.js | 2 + .../lib/introspect/stringvalidator.js | 7 +- packages/concerto-core/lib/modelmanager.js | 1 + packages/concerto-core/package-lock.json | 74 +++++++++++++++---- packages/concerto-core/package.json | 2 +- packages/concerto-core/re2.js | 22 ------ .../test/introspect/stringvalidator.js | 20 ++++- packages/concerto-core/test/modelmanager.js | 20 +++++ packages/concerto-core/webpack.config.js | 3 - 10 files changed, 104 insertions(+), 62 deletions(-) delete mode 100644 packages/concerto-core/re2.js diff --git a/package-lock.json b/package-lock.json index 58c370fe4d..f271705d24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,7 +42,6 @@ "peggy": "2.0.1", "process": "0.11.10", "randexp": "0.5.3", - "re2-wasm": "1.0.2", "rimraf": "3.0.2", "sinon": "12.0.0", "sinon-chai": "3.7.0", @@ -15704,15 +15703,6 @@ "safe-buffer": "^5.1.0" } }, - "node_modules/re2-wasm": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/re2-wasm/-/re2-wasm-1.0.2.tgz", - "integrity": "sha1-eMCdxlG4liqoFLVa5/5eRy7BW7s=", - "license": "Apache-2.0", - "engines": { - "node": ">=10" - } - }, "node_modules/react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -30403,11 +30393,6 @@ "safe-buffer": "^5.1.0" } }, - "re2-wasm": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/re2-wasm/-/re2-wasm-1.0.2.tgz", - "integrity": "sha1-eMCdxlG4liqoFLVa5/5eRy7BW7s=" - }, "react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", diff --git a/packages/concerto-core/lib/basemodelmanager.js b/packages/concerto-core/lib/basemodelmanager.js index 2d4a5c83c2..f74294feb1 100644 --- a/packages/concerto-core/lib/basemodelmanager.js +++ b/packages/concerto-core/lib/basemodelmanager.js @@ -75,6 +75,7 @@ class BaseModelManager { * @constructor * @param {object} [options] - ModelManager options, also passed to Serializer * @param {boolean} [options.versionedNamespacesStrict] - require versioned namespaces and imports + * @param {Object} [options.regExp] - An alternative regular expression engine. * @param {*} [processFile] - how to obtain a concerto AST from an input to the model manager */ constructor(options, processFile) { @@ -84,6 +85,7 @@ class BaseModelManager { this.serializer = new Serializer(this.factory, this, options); this.decoratorFactories = []; this.versionedNamespacesStrict = !!options?.versionedNamespacesStrict; + this.options = options; this.addRootModel(); } diff --git a/packages/concerto-core/lib/introspect/stringvalidator.js b/packages/concerto-core/lib/introspect/stringvalidator.js index ab1bd9b154..83965aa025 100644 --- a/packages/concerto-core/lib/introspect/stringvalidator.js +++ b/packages/concerto-core/lib/introspect/stringvalidator.js @@ -14,7 +14,6 @@ 'use strict'; -const { RE2 } = require('re2-wasm'); const Validator = require('./validator'); // Types needed for TypeScript generation. @@ -37,16 +36,14 @@ class StringValidator extends Validator{ * Create a StringValidator. * @param {Field} field - the field this validator is attached to * @param {Object} validator - The validation string. This must be a regex - * expression. * * @throws {IllegalModelException} */ constructor(field, validator) { super(field, validator); try { - // RE2 must always be run in unicode mode, it's safe to add 'u' even if it already exists - const flags = validator.flags ? validator.flags + 'u' : 'u'; - this.regex = new RE2(validator.pattern, flags); + const CustomRegExp = field?.parent?.getModelFile()?.getModelManager()?.options?.regExp || RegExp; + this.regex = new CustomRegExp(validator.pattern, validator.flags); } catch(exception) { this.reportError(field.getName(), exception.message); diff --git a/packages/concerto-core/lib/modelmanager.js b/packages/concerto-core/lib/modelmanager.js index 511c4091d0..347b283e48 100644 --- a/packages/concerto-core/lib/modelmanager.js +++ b/packages/concerto-core/lib/modelmanager.js @@ -58,6 +58,7 @@ class ModelManager extends BaseModelManager { * @constructor * @param {object} [options] - ModelManager options, also passed to Serializer * @param {boolean} [options.versionedNamespacesStrict] - require versioned namespaces and imports + * @param {Object} [options.regExp] - An alternative regular expression engine. */ constructor(options) { super(options, ctoProcessFile(options)); diff --git a/packages/concerto-core/package-lock.json b/packages/concerto-core/package-lock.json index d5ba8144c7..7e88362154 100644 --- a/packages/concerto-core/package-lock.json +++ b/packages/concerto-core/package-lock.json @@ -16,7 +16,6 @@ "debug": "4.3.1", "lorem-ipsum": "2.0.3", "randexp": "0.5.3", - "re2-wasm": "1.0.2", "semver": "7.3.5", "slash": "3.0.0", "urijs": "1.19.11", @@ -49,6 +48,7 @@ "typescript": "4.6.3", "webpack": "5.64.2", "webpack-cli": "4.9.1", + "xregexp": "5.1.1", "yargs": "17.3.1" }, "engines": { @@ -1868,6 +1868,19 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/runtime-corejs3": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.18.9.tgz", + "integrity": "sha512-qZEWeccZCrHA2Au4/X05QW5CMdm4VjUDCrGq5gf1ZDcM4hRqreKrtwAn7yci9zfgAS9apvnsFXiGBHBAxZdK9A==", + "dev": true, + "dependencies": { + "core-js-pure": "^3.20.2", + "regenerator-runtime": "^0.13.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.18.10", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.10.tgz", @@ -3557,6 +3570,17 @@ "semver": "bin/semver.js" } }, + "node_modules/core-js-pure": { + "version": "3.24.1", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.24.1.tgz", + "integrity": "sha512-r1nJk41QLLPyozHUUPmILCEMtMw24NG4oWK6RbsDdjzQgg9ZvrUsPBj1MnG0wXXp1DCDU6j+wUvEmBSrtRbLXg==", + "dev": true, + "hasInstallScript": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, "node_modules/create-ecdh": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", @@ -7546,15 +7570,6 @@ "safe-buffer": "^5.1.0" } }, - "node_modules/re2-wasm": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/re2-wasm/-/re2-wasm-1.0.2.tgz", - "integrity": "sha1-eMCdxlG4liqoFLVa5/5eRy7BW7s=", - "license": "Apache-2.0", - "engines": { - "node": ">=10" - } - }, "node_modules/readable-stream": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.1.0.tgz", @@ -9130,6 +9145,15 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/xregexp": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-5.1.1.tgz", + "integrity": "sha512-fKXeVorD+CzWvFs7VBuKTYIW63YD1e1osxwQ8caZ6o1jg6pDAbABDG54LCIq0j5cy7PjRvGIq6sef9DYPXpncg==", + "dev": true, + "dependencies": { + "@babel/runtime-corejs3": "^7.16.5" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -10403,6 +10427,16 @@ "regenerator-runtime": "^0.13.4" } }, + "@babel/runtime-corejs3": { + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.18.9.tgz", + "integrity": "sha512-qZEWeccZCrHA2Au4/X05QW5CMdm4VjUDCrGq5gf1ZDcM4hRqreKrtwAn7yci9zfgAS9apvnsFXiGBHBAxZdK9A==", + "dev": true, + "requires": { + "core-js-pure": "^3.20.2", + "regenerator-runtime": "^0.13.4" + } + }, "@babel/template": { "version": "7.18.10", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.10.tgz", @@ -11662,6 +11696,12 @@ } } }, + "core-js-pure": { + "version": "3.24.1", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.24.1.tgz", + "integrity": "sha512-r1nJk41QLLPyozHUUPmILCEMtMw24NG4oWK6RbsDdjzQgg9ZvrUsPBj1MnG0wXXp1DCDU6j+wUvEmBSrtRbLXg==", + "dev": true + }, "create-ecdh": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", @@ -14429,11 +14469,6 @@ "safe-buffer": "^5.1.0" } }, - "re2-wasm": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/re2-wasm/-/re2-wasm-1.0.2.tgz", - "integrity": "sha1-eMCdxlG4liqoFLVa5/5eRy7BW7s=" - }, "readable-stream": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.1.0.tgz", @@ -15535,6 +15570,15 @@ "integrity": "sha1-DFqw+ZzdAqgQZfqc2Piuh2JIib4=", "dev": true }, + "xregexp": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-5.1.1.tgz", + "integrity": "sha512-fKXeVorD+CzWvFs7VBuKTYIW63YD1e1osxwQ8caZ6o1jg6pDAbABDG54LCIq0j5cy7PjRvGIq6sef9DYPXpncg==", + "dev": true, + "requires": { + "@babel/runtime-corejs3": "^7.16.5" + } + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/packages/concerto-core/package.json b/packages/concerto-core/package.json index 9f92b77b64..493ded7646 100644 --- a/packages/concerto-core/package.json +++ b/packages/concerto-core/package.json @@ -64,6 +64,7 @@ "typescript": "4.6.3", "webpack": "5.64.2", "webpack-cli": "4.9.1", + "xregexp": "5.1.1", "yargs": "17.3.1" }, "dependencies": { @@ -74,7 +75,6 @@ "debug": "4.3.1", "lorem-ipsum": "2.0.3", "randexp": "0.5.3", - "re2-wasm": "1.0.2", "semver": "7.3.5", "slash": "3.0.0", "urijs": "1.19.11", diff --git a/packages/concerto-core/re2.js b/packages/concerto-core/re2.js deleted file mode 100644 index 7a7b113e7b..0000000000 --- a/packages/concerto-core/re2.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -'use strict'; - -// This file is used to replace the re2-wasm module in the browser -// https://github.com/google/re2-wasm/issues/15 - -module.exports = { - RE2: RegExp -}; diff --git a/packages/concerto-core/test/introspect/stringvalidator.js b/packages/concerto-core/test/introspect/stringvalidator.js index 62e74f04ab..2bd5471694 100644 --- a/packages/concerto-core/test/introspect/stringvalidator.js +++ b/packages/concerto-core/test/introspect/stringvalidator.js @@ -21,6 +21,8 @@ require('chai').should(); const sinon = require('sinon'); const NumberValidator = require('../../lib/introspect/numbervalidator'); +const XRegExp = require('xregexp'); + describe('StringValidator', () => { let mockField; @@ -44,7 +46,7 @@ describe('StringValidator', () => { it('should ignore a null string', () => { let v = new StringValidator(mockField, { pattern: '^[A-z][A-z][0-9]{7}' }); - v.getRegex().toString().should.equal('/^[A-z][A-z][0-9]{7}/u'); + v.getRegex().toString().should.equal('/^[A-z][A-z][0-9]{7}/'); v.validate('id', null); }); @@ -87,6 +89,22 @@ describe('StringValidator', () => { }); + describe('#validate with custom RegEx engine', () => { + const options = { + regExp: XRegExp + }; + it('should ignore a null string', () => { + let v = new StringValidator(mockField, { pattern: '^[A-z][A-z][0-9]{7}' }, options); + v.getRegex().toString().should.equal('/^[A-z][A-z][0-9]{7}/'); + v.validate('id', null); + }); + + it('should validate a string', () => { + let v = new StringValidator(mockField, { pattern: '^[\\p{Letter}\\p{Number}]{7}' }, options); + v.validate('id', 'AB1234567'); + }); + }); + describe('#compatibleWith', () => { it('should return false for a number validator', () => { const other = new NumberValidator(mockField, { lower: -1, upper: 1 }); diff --git a/packages/concerto-core/test/modelmanager.js b/packages/concerto-core/test/modelmanager.js index c59e67ef92..62bbe29669 100644 --- a/packages/concerto-core/test/modelmanager.js +++ b/packages/concerto-core/test/modelmanager.js @@ -15,6 +15,7 @@ 'use strict'; const fs = require('fs'); +const XRegExp = require('xregexp'); const FileDownloader = require('@accordproject/concerto-util').FileDownloader; const AssetDeclaration = require('../lib/introspect/assetdeclaration'); @@ -27,6 +28,7 @@ const ModelFile = require('../lib/introspect/modelfile'); const ModelManager = require('../lib/modelmanager'); const ParticipantDeclaration = require('../lib/introspect/participantdeclaration'); const Serializer = require('../lib/serializer'); +const Concerto = require('../lib/concerto'); const TypeNotFoundException = require('../lib/typenotfoundexception'); const Util = require('./composer/composermodelutility'); const COMPOSER_MODEL = require('./composer/composermodel'); @@ -186,6 +188,24 @@ describe('ModelManager', () => { JSON.stringify(ast).should.not.contain('location'); }); + it('should add a model file with alternative RegExp option', () => { + const regExp = XRegExp; + const modelManagerWithOptions = new ModelManager({ regExp }); + Util.addComposerModel(modelManagerWithOptions); + + modelManagerWithOptions.addCTOModel(`namespace org.acme + concept Bar { + o String foo regex=/\\p{S}/ + }`, 'internal.cto', true); + + const bar = { + $class : 'org.acme.Bar', + foo : '😊' + }; + const concerto = new Concerto(modelManagerWithOptions); + concerto.validate(bar); + }); + it('should return error for duplicate namespaces for a string', () => { modelManager.addCTOModel(modelBase); let mf1 = sinon.createStubInstance(ModelFile); diff --git a/packages/concerto-core/webpack.config.js b/packages/concerto-core/webpack.config.js index a05b64c6eb..273451895b 100644 --- a/packages/concerto-core/webpack.config.js +++ b/packages/concerto-core/webpack.config.js @@ -69,9 +69,6 @@ module.exports = { ] }, resolve: { - alias: { - 're2-wasm': path.resolve(__dirname, 're2.js'), - }, fallback: { // Webpack 5 no longer polyfills Node.js core modules automatically. // see https://webpack.js.org/configuration/resolve/#resolvefallback From 1f58582bfcb80c142d1b97dde22734a98bba9f39 Mon Sep 17 00:00:00 2001 From: Matt Roberts Date: Tue, 16 Aug 2022 12:55:17 +0100 Subject: [PATCH 2/3] chore(core): update API Signed-off-by: Matt Roberts --- packages/concerto-core/api.txt | 4 ++-- packages/concerto-core/changelog.txt | 4 ++++ packages/concerto-core/types/lib/basemodelmanager.d.ts | 6 ++++++ .../concerto-core/types/lib/introspect/stringvalidator.d.ts | 3 +-- packages/concerto-core/types/lib/modelmanager.d.ts | 2 ++ 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/concerto-core/api.txt b/packages/concerto-core/api.txt index cececfbc18..82e91c2a85 100644 --- a/packages/concerto-core/api.txt +++ b/packages/concerto-core/api.txt @@ -2,7 +2,7 @@ class AstModelManager extends BaseModelManager { + void constructor(object?) } class BaseModelManager { - + void constructor(object?,boolean?,processFile?) + + void constructor(object?,boolean?,Object?,processFile?) + boolean isModelManager() + boolean isVersionedNamespacesStrict() + Object accept(Object,Object) @@ -244,7 +244,7 @@ class ModelLoader { + ModelManager[] loadModelManagerFromModelFiles(object[],string[],object,boolean?,boolean?,number?) } class ModelManager extends BaseModelManager { - + void constructor(object?,boolean?) + + void constructor(object?,boolean?,Object?) + ModelFile addCTOModel(string,string?,boolean?) throws IllegalModelException } + object getRootModel() diff --git a/packages/concerto-core/changelog.txt b/packages/concerto-core/changelog.txt index ec1ece3b6f..43d63741aa 100644 --- a/packages/concerto-core/changelog.txt +++ b/packages/concerto-core/changelog.txt @@ -23,6 +23,10 @@ # # Note that the latest public API is documented using JSDocs and is available in api.txt. # + +Version 3.0.0 {b0c9b0951c58e3deeed8fbccf2dfd052} 2022-08-16 +- Allow client-provided RegExp engine to ModelManager + Version 2.3.0 {774980a857090905fe276b6e94f1dbb1} 2022-07-21 - Versioned namespaces diff --git a/packages/concerto-core/types/lib/basemodelmanager.d.ts b/packages/concerto-core/types/lib/basemodelmanager.d.ts index d807895068..eeeadfd8f5 100644 --- a/packages/concerto-core/types/lib/basemodelmanager.d.ts +++ b/packages/concerto-core/types/lib/basemodelmanager.d.ts @@ -20,10 +20,12 @@ declare class BaseModelManager { * @constructor * @param {object} [options] - ModelManager options, also passed to Serializer * @param {boolean} [options.versionedNamespacesStrict] - require versioned namespaces and imports + * @param {Object} [options.regExp] - An alternative regular expression engine. * @param {*} [processFile] - how to obtain a concerto AST from an input to the model manager */ constructor(options?: { versionedNamespacesStrict?: boolean; + regExp?: any; }, processFile?: any); processFile: any; modelFiles: {}; @@ -31,6 +33,10 @@ declare class BaseModelManager { serializer: any; decoratorFactories: any[]; versionedNamespacesStrict: boolean; + options: { + versionedNamespacesStrict?: boolean; + regExp?: any; + }; /** * Returns true * @returns {boolean} true diff --git a/packages/concerto-core/types/lib/introspect/stringvalidator.d.ts b/packages/concerto-core/types/lib/introspect/stringvalidator.d.ts index b1fb7bbb7c..e724f5f0f2 100644 --- a/packages/concerto-core/types/lib/introspect/stringvalidator.d.ts +++ b/packages/concerto-core/types/lib/introspect/stringvalidator.d.ts @@ -6,7 +6,7 @@ export = StringValidator; * @memberof module:concerto-core */ declare class StringValidator extends Validator { - regex: RE2; + regex: any; /** * Returns the RegExp object associated with the string validator * @returns {RegExp} the RegExp object @@ -14,4 +14,3 @@ declare class StringValidator extends Validator { getRegex(): RegExp; } import Validator = require("./validator"); -import { RE2 } from "re2-wasm"; diff --git a/packages/concerto-core/types/lib/modelmanager.d.ts b/packages/concerto-core/types/lib/modelmanager.d.ts index 1b668b5358..6455d85802 100644 --- a/packages/concerto-core/types/lib/modelmanager.d.ts +++ b/packages/concerto-core/types/lib/modelmanager.d.ts @@ -18,9 +18,11 @@ declare class ModelManager extends BaseModelManager { * @constructor * @param {object} [options] - ModelManager options, also passed to Serializer * @param {boolean} [options.versionedNamespacesStrict] - require versioned namespaces and imports + * @param {Object} [options.regExp] - An alternative regular expression engine. */ constructor(options?: { versionedNamespacesStrict?: boolean; + regExp?: any; }); /** * Adds a model in CTO format to the ModelManager. From c7984b4b96e7bb0948d57f75c089d2d16f10dd0a Mon Sep 17 00:00:00 2001 From: Matt Roberts Date: Tue, 16 Aug 2022 13:08:45 +0100 Subject: [PATCH 3/3] test(stringvalidator): fix tests Signed-off-by: Matt Roberts --- packages/concerto-core/test/introspect/stringvalidator.js | 2 +- packages/concerto-core/test/modelmanager.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/concerto-core/test/introspect/stringvalidator.js b/packages/concerto-core/test/introspect/stringvalidator.js index 2bd5471694..2b182069b3 100644 --- a/packages/concerto-core/test/introspect/stringvalidator.js +++ b/packages/concerto-core/test/introspect/stringvalidator.js @@ -100,7 +100,7 @@ describe('StringValidator', () => { }); it('should validate a string', () => { - let v = new StringValidator(mockField, { pattern: '^[\\p{Letter}\\p{Number}]{7}' }, options); + let v = new StringValidator(mockField, { pattern: '^[\\p{Letter}\\p{Number}]{7}', flags: 'u' }, options); v.validate('id', 'AB1234567'); }); }); diff --git a/packages/concerto-core/test/modelmanager.js b/packages/concerto-core/test/modelmanager.js index 62bbe29669..95b3d54c3d 100644 --- a/packages/concerto-core/test/modelmanager.js +++ b/packages/concerto-core/test/modelmanager.js @@ -189,6 +189,7 @@ describe('ModelManager', () => { }); it('should add a model file with alternative RegExp option', () => { + XRegExp.install('astral'); const regExp = XRegExp; const modelManagerWithOptions = new ModelManager({ regExp }); Util.addComposerModel(modelManagerWithOptions);