From e2a0da59e6e9f3cf4cbc408d943babe7f380f6ec Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sat, 20 Feb 2021 11:11:31 +0000 Subject: [PATCH 01/19] refactor isOwnProperty --- .../applicator/additionalProperties.ts | 9 ++------ lib/vocabularies/applicator/dependencies.ts | 4 ++-- lib/vocabularies/applicator/properties.ts | 2 +- lib/vocabularies/code.ts | 22 ++++++++++++------- lib/vocabularies/jtd/properties.ts | 11 +++------- lib/vocabularies/validation/required.ts | 4 ++-- 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/lib/vocabularies/applicator/additionalProperties.ts b/lib/vocabularies/applicator/additionalProperties.ts index 381495a5a..b8bdb76a4 100644 --- a/lib/vocabularies/applicator/additionalProperties.ts +++ b/lib/vocabularies/applicator/additionalProperties.ts @@ -5,7 +5,7 @@ import type { KeywordErrorDefinition, AnySchema, } from "../../types" -import {allSchemaProperties, usePattern} from "../code" +import {allSchemaProperties, usePattern, isOwnProperty} from "../code" import {_, nil, or, not, Code, Name} from "../../compile/codegen" import N from "../../compile/names" import {SubschemaArgs, Type} from "../../compile/subschema" @@ -52,13 +52,8 @@ const def: CodeKeywordDefinition & AddedKeywordDefinition = { let definedProp: Code if (props.length > 8) { // TODO maybe an option instead of hard-coded 8? - const hasProp = gen.scopeValue("func", { - // eslint-disable-next-line @typescript-eslint/unbound-method - ref: Object.prototype.hasOwnProperty, - code: _`Object.prototype.hasOwnProperty`, - }) const propsSchema = schemaRefOrVal(it, parentSchema.properties, "properties") - definedProp = _`${hasProp}.call(${propsSchema}, ${key})` + definedProp = isOwnProperty(gen, propsSchema as Code, key) } else if (props.length) { definedProp = or(...props.map((p) => _`${key} === ${p}`)) } else { diff --git a/lib/vocabularies/applicator/dependencies.ts b/lib/vocabularies/applicator/dependencies.ts index d3084f479..6d6436edd 100644 --- a/lib/vocabularies/applicator/dependencies.ts +++ b/lib/vocabularies/applicator/dependencies.ts @@ -72,7 +72,7 @@ export function validatePropertyDeps( for (const prop in propertyDeps) { const deps = propertyDeps[prop] as string[] if (deps.length === 0) continue - const hasProperty = propertyInData(data, prop, it.opts.ownProperties) + const hasProperty = propertyInData(gen, data, prop, it.opts.ownProperties) cxt.setParams({ property: prop, depsCount: deps.length, @@ -98,7 +98,7 @@ export function validateSchemaDeps(cxt: KeywordCxt, schemaDeps: SchemaMap = cxt. for (const prop in schemaDeps) { if (alwaysValidSchema(it, schemaDeps[prop] as AnySchema)) continue gen.if( - propertyInData(data, prop, it.opts.ownProperties), + propertyInData(gen, data, prop, it.opts.ownProperties), () => { const schCxt = cxt.subschema({keyword, schemaProp: prop}, valid) cxt.mergeValidEvaluated(schCxt, valid) diff --git a/lib/vocabularies/applicator/properties.ts b/lib/vocabularies/applicator/properties.ts index 5ab96acd0..b469346d8 100644 --- a/lib/vocabularies/applicator/properties.ts +++ b/lib/vocabularies/applicator/properties.ts @@ -25,7 +25,7 @@ const def: CodeKeywordDefinition = { if (hasDefault(prop)) { applyPropertySchema(prop) } else { - gen.if(propertyInData(data, prop, it.opts.ownProperties)) + gen.if(propertyInData(gen, data, prop, it.opts.ownProperties)) applyPropertySchema(prop) if (!it.allErrors) gen.else().var(valid, true) gen.endIf() diff --git a/lib/vocabularies/code.ts b/lib/vocabularies/code.ts index b02cd3e7b..47c2c57d2 100644 --- a/lib/vocabularies/code.ts +++ b/lib/vocabularies/code.ts @@ -8,20 +8,20 @@ import N from "../compile/names" export function checkReportMissingProp(cxt: KeywordCxt, prop: string): void { const {gen, data, it} = cxt - gen.if(noPropertyInData(data, prop, it.opts.ownProperties), () => { + gen.if(noPropertyInData(gen, data, prop, it.opts.ownProperties), () => { cxt.setParams({missingProperty: _`${prop}`}, true) cxt.error() }) } export function checkMissingProp( - {data, it: {opts}}: KeywordCxt, + {gen, data, it: {opts}}: KeywordCxt, properties: string[], missing: Name ): Code { return or( ...properties.map( - (prop) => _`${noPropertyInData(data, prop, opts.ownProperties)} && (${missing} = ${prop})` + (prop) => _`${noPropertyInData(gen, data, prop, opts.ownProperties)} && (${missing} = ${prop})` ) ) } @@ -31,22 +31,28 @@ export function reportMissingProp(cxt: KeywordCxt, missing: Name): void { cxt.error() } -function isOwnProperty(data: Name, property: Name | string): Code { - return _`Object.prototype.hasOwnProperty.call(${data}, ${property})` +export function isOwnProperty(gen: CodeGen, data: Name, property: Name | string): Code { + const hasProp = gen.scopeValue("func", { + // eslint-disable-next-line @typescript-eslint/unbound-method + ref: Object.prototype.hasOwnProperty, + code: _`Object.prototype.hasOwnProperty`, + }) + return _`${hasProp}.call(${data}, ${property})` } -export function propertyInData(data: Name, property: Name | string, ownProperties?: boolean): Code { +export function propertyInData(gen: CodeGen, data: Name, property: Name | string, ownProperties?: boolean): Code { const cond = _`${data}${getProperty(property)} !== undefined` - return ownProperties ? _`${cond} && ${isOwnProperty(data, property)}` : cond + return ownProperties ? _`${cond} && ${isOwnProperty(gen, data, property)}` : cond } export function noPropertyInData( + gen: CodeGen, data: Name, property: Name | string, ownProperties?: boolean ): Code { const cond = _`${data}${getProperty(property)} === undefined` - return ownProperties ? _`${cond} || !${isOwnProperty(data, property)}` : cond + return ownProperties ? _`${cond} || !${isOwnProperty(gen, data, property)}` : cond } export function allSchemaProperties(schemaMap?: SchemaMap): string[] { diff --git a/lib/vocabularies/jtd/properties.ts b/lib/vocabularies/jtd/properties.ts index 5d881d6ac..1ef172b5d 100644 --- a/lib/vocabularies/jtd/properties.ts +++ b/lib/vocabularies/jtd/properties.ts @@ -1,6 +1,6 @@ import type {CodeKeywordDefinition} from "../../types" import type KeywordCxt from "../../compile/context" -import {propertyInData, allSchemaProperties} from "../code" +import {propertyInData, allSchemaProperties, isOwnProperty} from "../code" import {alwaysValidSchema, schemaRefOrVal} from "../../compile/util" import {_, and, Code, Name} from "../../compile/codegen" import {checkMetadata} from "./metadata" @@ -63,7 +63,7 @@ export function validateProperties(cxt: KeywordCxt): void { const _valid = gen.var("valid") for (const prop of props) { gen.if( - propertyInData(data, prop, it.opts.ownProperties), + propertyInData(gen, data, prop, it.opts.ownProperties), () => applyPropertySchema(prop, keyword, _valid), missingProperty ) @@ -116,12 +116,7 @@ export function validateProperties(cxt: KeywordCxt): void { if (props.length > 8) { // TODO maybe an option instead of hard-coded 8? const propsSchema = schemaRefOrVal(it, parentSchema[keyword], keyword) - const hasProp = gen.scopeValue("func", { - // eslint-disable-next-line @typescript-eslint/unbound-method - ref: Object.prototype.hasOwnProperty, - code: _`Object.prototype.hasOwnProperty`, - }) - additional = _`!${hasProp}.call(${propsSchema}, ${key})` + additional = isOwnProperty(gen, propsSchema as Code, key) } else if (props.length) { additional = and(...props.map((p) => _`${key} !== ${p}`)) } else { diff --git a/lib/vocabularies/validation/required.ts b/lib/vocabularies/validation/required.ts index d803b0578..1bfb2a475 100644 --- a/lib/vocabularies/validation/required.ts +++ b/lib/vocabularies/validation/required.ts @@ -60,7 +60,7 @@ const def: CodeKeywordDefinition = { function loopAllRequired(): void { gen.forOf("prop", schemaCode as Code, (prop) => { cxt.setParams({missingProperty: prop}) - gen.if(noPropertyInData(data, prop, opts.ownProperties), () => cxt.error()) + gen.if(noPropertyInData(gen, data, prop, opts.ownProperties), () => cxt.error()) }) } @@ -70,7 +70,7 @@ const def: CodeKeywordDefinition = { missing, schemaCode as Code, () => { - gen.assign(valid, propertyInData(data, missing, opts.ownProperties)) + gen.assign(valid, propertyInData(gen, data, missing, opts.ownProperties)) gen.if(not(valid), () => { cxt.error() gen.break() From 802b53a53841f8fc1688b428067044b38d77701f Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sat, 20 Feb 2021 14:41:08 +0000 Subject: [PATCH 02/19] compile serializers based on JTD schema (WIP) --- lib/compile/codegen/index.ts | 22 +++- lib/compile/serialize.ts | 189 +++++++++++++++++++++++++++++++++++ lib/jtd.ts | 8 +- 3 files changed, 213 insertions(+), 6 deletions(-) create mode 100644 lib/compile/serialize.ts diff --git a/lib/compile/codegen/index.ts b/lib/compile/codegen/index.ts index 0957ca9a7..e33de346b 100644 --- a/lib/compile/codegen/index.ts +++ b/lib/compile/codegen/index.ts @@ -21,6 +21,7 @@ export const operators = { NOT: new _Code("!"), OR: new _Code("||"), AND: new _Code("&&"), + ADD: new _Code("+"), } abstract class Node { @@ -62,11 +63,7 @@ class Def extends Node { } class Assign extends Node { - constructor( - private readonly lhs: Code, - private rhs: SafeExpr, - private readonly sideEffects?: boolean - ) { + constructor(readonly lhs: Code, public rhs: SafeExpr, private readonly sideEffects?: boolean) { super() } @@ -86,6 +83,16 @@ class Assign extends Node { } } +class AssignOp extends Assign { + constructor(lhs: Code, private readonly op: Code, rhs: SafeExpr, sideEffects?: boolean) { + super(lhs, rhs, sideEffects) + } + + render({_n}: CGOptions): string { + return `${this.lhs} ${this.op}= ${this.rhs};` + _n + } +} + class Label extends Node { readonly names: UsedNames = {} constructor(readonly label: Name) { @@ -508,6 +515,11 @@ export class CodeGen { return this._leafNode(new Assign(lhs, rhs, sideEffects)) } + // `+=` code + add(lhs: Code, rhs: SafeExpr): CodeGen { + return this._leafNode(new AssignOp(lhs, operators.ADD, rhs)) + } + // appends passed SafeExpr to code or executes Block code(c: Block | SafeExpr): CodeGen { if (typeof c == "function") c() diff --git a/lib/compile/serialize.ts b/lib/compile/serialize.ts new file mode 100644 index 000000000..06cfd9771 --- /dev/null +++ b/lib/compile/serialize.ts @@ -0,0 +1,189 @@ +import type Ajv from "../core" +import type {SchemaObject} from "../types" +import {_, str, and, getProperty, CodeGen, Code, Name} from "./codegen" +import N from "./names" +import {isOwnProperty} from "../vocabularies/code" + +type SchemaObjectMap = {[Ref in string]?: SchemaObject} + +const jtdForms = [ + "elements", + "values", + "discriminator", + "properties", + "optionalProperties", + "enum", + "type", + "ref", +] as const + +type JTDForm = typeof jtdForms[number] + +const genSerialize: {[S in JTDForm]: (cxt: SerializeCxt) => void} = { + elements: serializeElements, + values: serializeValues, + discriminator: serializeDiscriminator, + properties: serializeProperties, + optionalProperties: serializeProperties, + enum: serializeString, + type: serializeType, + ref: serializeRef, +} + +export type SerializeFunction = (data: T) => string + +interface SerializeCxt { + gen: CodeGen + schema: SchemaObject + definitions: SchemaObjectMap + data: Code + result: Name +} + +export function compileSerializer( + this: Ajv, + schema: SchemaObject, + definitions: SchemaObjectMap +): SerializeFunction { + const {es5, lines} = this.opts.code + const {ownProperties} = this.opts + const gen = new CodeGen(this.scope, {es5, lines, ownProperties}) + const serializeName = gen.scopeName("serialize") + gen.func(serializeName, N.data, false, () => { + const result = gen.let("result", str``) + const cxt = {gen, schema, definitions, result, data: N.data} + genSerializeCode(cxt) + }) + gen.optimize(this.opts.code.optimize) + const serializeCode = gen.toString() + const sourceCode = `${gen.scopeRefs(N.scope)}return ${serializeCode}` + const makeSerialize = new Function(`${N.scope}`, sourceCode) + const serialize: SerializeFunction = makeSerialize(this.scope.get()) + this.scope.value(serializeName, {ref: serialize}) + return serialize +} + +function genSerializeCode(cxt: SerializeCxt): void { + let form: JTDForm | undefined + for (const key of jtdForms) { + if (key in cxt.schema) { + form = key + break + } + } + serializeNullable(cxt, form ? genSerialize[form] : serializeEmpty) +} + +function serializeNullable(cxt: SerializeCxt, serializeForm: (_cxt: SerializeCxt) => void): void { + const {gen, schema, result, data} = cxt + if (!schema.nullable) return serializeForm(cxt) + gen.if( + _`${data} === undefined || ${data} === null`, + () => gen.add(result, _`null`), + () => serializeForm(cxt) + ) +} + +function serializeElements(cxt: SerializeCxt): void { + const {gen, schema, result, data} = cxt + gen.add(result, str`[`) + let first = true + gen.forOf("element", data, (el) => { + if (!first) gen.add(result, str`,`) + first = false + genSerializeCode({...cxt, schema: schema.elements, data: el}) + }) + gen.add(result, str`]`) +} + +function serializeValues(cxt: SerializeCxt): void { + const {gen, schema, result, data} = cxt + gen.add(result, str`{`) + let first = true + gen.forIn("key", data, (key) => { + if (!first) gen.add(result, str`,`) + first = false + serializeString({...cxt, data: key}) + gen.add(result, str`:`) + const value = gen.const("value", _`${data}${getProperty(key)}`) + genSerializeCode({...cxt, schema: schema.values, data: value}) + }) + gen.add(result, str`}`) +} + +function serializeDiscriminator(cxt: SerializeCxt): void { + const {gen, schema, result, data} = cxt + const {discriminator} = schema + gen.add(result, str`{${JSON.stringify(discriminator)}:`) + const tag = gen.const("tag", _`${data}${getProperty(discriminator)}`) + serializeString({...cxt, data: tag}) + gen.if(false) + for (const tagValue in schema.mapping) { + gen.elseIf(_`${tag} === ${tagValue}`) + serializeSchemaProperties({...cxt, schema: schema.mapping[tagValue]}, false) + } + gen.endIf() + gen.add(result, str`}`) +} + +function serializeProperties(cxt: SerializeCxt): void { + const {gen, result} = cxt + gen.add(result, str`{`) + serializeSchemaProperties(cxt, true) + gen.add(result, str`}`) +} + +function serializeSchemaProperties(cxt: SerializeCxt, first: boolean): void { + const {gen, schema, result, data} = cxt + const {properties, optionalProperties} = schema + for (const key in properties || {}) { + serializeProperty(key, keyValue(key)) + } + for (const key in optionalProperties || {}) { + const value = keyValue(key) + gen.if(and(_`${value} !== undefined`, isOwnProperty(gen, data, key)), () => + serializeProperty(key, value) + ) + } + + function keyValue(key: string): Name { + return gen.const("value", _`${data}${getProperty(key)}`) + } + + function serializeProperty(key: string, value: Name): void { + if (!first) gen.add(result, str`,`) + first = false + gen.add(result, str`${JSON.stringify(key)}:`) + genSerializeCode({...cxt, data: value}) + } +} + +function serializeType(cxt: SerializeCxt): void { + const {gen, schema, result, data} = cxt + switch (schema.type) { + case "boolean": + gen.add(result, _`${data} ? "true" : "false`) + break + case "string": + serializeString(cxt) + break + case "timestamp": + gen.if(_`${data} instanceof Date`, + () => gen.add(result, _`${data}.toISOString()`), + () => serializeString(cxt) + ) + break + default: + serializeNumber(cxt) + } +} + +function serializeString(_cxt: SerializeCxt): void {} + +function serializeNumber({gen, result, data}: SerializeCxt): void { + gen.add(result, _`"" + ${data}`) +} + +function serializeRef(_cxt: SerializeCxt): void {} + +function serializeEmpty(_cxt: SerializeCxt): void {} diff --git a/lib/jtd.ts b/lib/jtd.ts index 356b0bd8a..e604ba36f 100644 --- a/lib/jtd.ts +++ b/lib/jtd.ts @@ -27,10 +27,12 @@ export {KeywordCxt} export {_, str, stringify, nil, Name, Code, CodeGen, CodeGenOptions} from "./compile/codegen" import type {AnySchemaObject} from "./types" -export {JTDSchemaType} from "./types/jtd-schema" +import type {JTDSchemaType} from "./types/jtd-schema" +export {JTDSchemaType} import AjvCore, {CurrentOptions} from "./core" import jtdVocabulary from "./vocabularies/jtd" import jtdMetaSchema from "./refs/jtd-schema" +// import {SerializeFunction, compileSerializer} from "./compile/serialize" // const META_SUPPORT_DATA = ["/properties"] @@ -88,4 +90,8 @@ export default class Ajv extends AjvCore { return (this.opts.defaultMeta = super.defaultMeta() || (this.getSchema(META_SCHEMA_ID) ? META_SCHEMA_ID : undefined)) } + + // compileSerializer(schema: JTDSchemaType): SerializeFunction { + // return compileSerializer.call(this, schema, schema.definitions || {}) as SerializeFunction + // } } From f23825867ac3d1086c2635cfd8bf6d2923e08834 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sat, 20 Feb 2021 22:23:52 +0000 Subject: [PATCH 03/19] JTD: compile serializers (some tests skipped) --- lib/compile/index.ts | 4 +- lib/compile/serialize.ts | 262 +++++++++++++++++++++++++----------- lib/core.ts | 3 +- lib/jtd.ts | 19 ++- lib/vocabularies/code.ts | 10 +- lib/vocabularies/jtd/ref.ts | 16 +-- spec/jtd-schema.spec.ts | 20 ++- 7 files changed, 241 insertions(+), 93 deletions(-) diff --git a/lib/compile/index.ts b/lib/compile/index.ts index 9102cc570..ba928640f 100644 --- a/lib/compile/index.ts +++ b/lib/compile/index.ts @@ -79,6 +79,8 @@ export class SchemaEnv implements SchemaEnvArgs { readonly dynamicAnchors: {[Ref in string]?: true} = {} validate?: AnyValidateFunction validateName?: ValueScopeName + serialize?: (data: unknown) => string + serializeName?: ValueScopeName constructor(env: SchemaEnvArgs) { let schema: AnySchemaObject | undefined @@ -216,7 +218,7 @@ function inlineOrCompile(this: Ajv, sch: SchemaEnv): AnySchema | SchemaEnv { } // Index of schema compilation in the currently compiled list -function getCompilingSchema(this: Ajv, schEnv: SchemaEnv): SchemaEnv | void { +export function getCompilingSchema(this: Ajv, schEnv: SchemaEnv): SchemaEnv | void { for (const sch of this._compilations) { if (sameSchemaEnv(sch, schEnv)) return sch } diff --git a/lib/compile/serialize.ts b/lib/compile/serialize.ts index 06cfd9771..a8ae6a50a 100644 --- a/lib/compile/serialize.ts +++ b/lib/compile/serialize.ts @@ -1,8 +1,12 @@ import type Ajv from "../core" import type {SchemaObject} from "../types" -import {_, str, and, getProperty, CodeGen, Code, Name} from "./codegen" +import {SchemaEnv, getCompilingSchema} from "." +import {_, str, and, getProperty, stringify, CodeGen, Code, Name} from "./codegen" +import {_Code} from "./codegen/code" +import {MissingRefError} from "./error_classes" import N from "./names" import {isOwnProperty} from "../vocabularies/code" +import {hasRef} from "../vocabularies/jtd/ref" type SchemaObjectMap = {[Ref in string]?: SchemaObject} @@ -30,40 +34,66 @@ const genSerialize: {[S in JTDForm]: (cxt: SerializeCxt) => void} = { ref: serializeRef, } -export type SerializeFunction = (data: T) => string - interface SerializeCxt { - gen: CodeGen + readonly gen: CodeGen + readonly self: Ajv // current Ajv instance + readonly schemaEnv: SchemaEnv + readonly definitions: SchemaObjectMap + readonly jsonStr: Name schema: SchemaObject - definitions: SchemaObjectMap data: Code - result: Name } export function compileSerializer( this: Ajv, - schema: SchemaObject, + sch: SchemaEnv, definitions: SchemaObjectMap -): SerializeFunction { +): SchemaEnv { + const _sch = getCompilingSchema.call(this, sch) + if (_sch) return _sch const {es5, lines} = this.opts.code const {ownProperties} = this.opts const gen = new CodeGen(this.scope, {es5, lines, ownProperties}) const serializeName = gen.scopeName("serialize") - gen.func(serializeName, N.data, false, () => { - const result = gen.let("result", str``) - const cxt = {gen, schema, definitions, result, data: N.data} - genSerializeCode(cxt) - }) - gen.optimize(this.opts.code.optimize) - const serializeCode = gen.toString() - const sourceCode = `${gen.scopeRefs(N.scope)}return ${serializeCode}` - const makeSerialize = new Function(`${N.scope}`, sourceCode) - const serialize: SerializeFunction = makeSerialize(this.scope.get()) - this.scope.value(serializeName, {ref: serialize}) - return serialize + const jsonStr = gen.name("json") + const cxt = { + self: this, + gen, + schema: sch.schema as SchemaObject, + schemaEnv: sch, + definitions, + jsonStr, + data: N.data, + } + + let sourceCode: string | undefined + try { + this._compilations.add(sch) + sch.serializeName = serializeName + gen.func(serializeName, N.data, false, () => { + gen.let(jsonStr, str``) + serializeCode(cxt) + gen.return(jsonStr) + }) + gen.optimize(this.opts.code.optimize) + const serializeFuncCode = gen.toString() + sourceCode = `${gen.scopeRefs(N.scope)}return ${serializeFuncCode}` + const makeSerialize = new Function(`${N.scope}`, sourceCode) + const serialize: (data: unknown) => string = makeSerialize(this.scope.get()) + this.scope.value(serializeName, {ref: serialize}) + sch.serialize = serialize + } catch (e) { + if (sourceCode) this.logger.error("Error compiling serializer, function code:", sourceCode) + delete sch.serialize + delete sch.serializeName + throw e + } finally { + this._compilations.delete(sch) + } + return sch } -function genSerializeCode(cxt: SerializeCxt): void { +function serializeCode(cxt: SerializeCxt): void { let form: JTDForm | undefined for (const key of jtdForms) { if (key in cxt.schema) { @@ -75,66 +105,66 @@ function genSerializeCode(cxt: SerializeCxt): void { } function serializeNullable(cxt: SerializeCxt, serializeForm: (_cxt: SerializeCxt) => void): void { - const {gen, schema, result, data} = cxt + const {gen, schema, jsonStr, data} = cxt if (!schema.nullable) return serializeForm(cxt) gen.if( _`${data} === undefined || ${data} === null`, - () => gen.add(result, _`null`), + () => gen.add(jsonStr, _`"null"`), () => serializeForm(cxt) ) } function serializeElements(cxt: SerializeCxt): void { - const {gen, schema, result, data} = cxt - gen.add(result, str`[`) - let first = true - gen.forOf("element", data, (el) => { - if (!first) gen.add(result, str`,`) - first = false - genSerializeCode({...cxt, schema: schema.elements, data: el}) + const {gen, schema, jsonStr, data} = cxt + gen.add(jsonStr, str`[`) + const first = gen.let("first", true) + gen.forOf("el", data, (el) => { + addComma(cxt, first) + serializeCode({...cxt, schema: schema.elements, data: el}) }) - gen.add(result, str`]`) + gen.add(jsonStr, str`]`) } function serializeValues(cxt: SerializeCxt): void { - const {gen, schema, result, data} = cxt - gen.add(result, str`{`) - let first = true + const {gen, schema, jsonStr, data} = cxt + gen.add(jsonStr, str`{`) + const first = gen.let("first", true) gen.forIn("key", data, (key) => { - if (!first) gen.add(result, str`,`) - first = false + addComma(cxt, first) serializeString({...cxt, data: key}) - gen.add(result, str`:`) + gen.add(jsonStr, str`:`) const value = gen.const("value", _`${data}${getProperty(key)}`) - genSerializeCode({...cxt, schema: schema.values, data: value}) + serializeCode({...cxt, schema: schema.values, data: value}) }) - gen.add(result, str`}`) + gen.add(jsonStr, str`}`) } function serializeDiscriminator(cxt: SerializeCxt): void { - const {gen, schema, result, data} = cxt + const {gen, schema, jsonStr, data} = cxt const {discriminator} = schema - gen.add(result, str`{${JSON.stringify(discriminator)}:`) + gen.add(jsonStr, str`{${JSON.stringify(discriminator)}:`) const tag = gen.const("tag", _`${data}${getProperty(discriminator)}`) serializeString({...cxt, data: tag}) + const first = gen.let("first", false) gen.if(false) for (const tagValue in schema.mapping) { gen.elseIf(_`${tag} === ${tagValue}`) - serializeSchemaProperties({...cxt, schema: schema.mapping[tagValue]}, false) + serializeSchemaProperties({...cxt, schema: schema.mapping[tagValue]}, first) } gen.endIf() - gen.add(result, str`}`) + gen.add(jsonStr, str`}`) } function serializeProperties(cxt: SerializeCxt): void { - const {gen, result} = cxt - gen.add(result, str`{`) - serializeSchemaProperties(cxt, true) - gen.add(result, str`}`) + const {gen, jsonStr} = cxt + gen.add(jsonStr, str`{`) + const first = gen.let("first", true) + serializeSchemaProperties(cxt, first) + gen.add(jsonStr, str`}`) } -function serializeSchemaProperties(cxt: SerializeCxt, first: boolean): void { - const {gen, schema, result, data} = cxt +function serializeSchemaProperties(cxt: SerializeCxt, first: Name): void { + const {gen, schema, jsonStr, data} = cxt const {properties, optionalProperties} = schema for (const key in properties || {}) { serializeProperty(key, keyValue(key)) @@ -151,39 +181,121 @@ function serializeSchemaProperties(cxt: SerializeCxt, first: boolean): void { } function serializeProperty(key: string, value: Name): void { - if (!first) gen.add(result, str`,`) - first = false - gen.add(result, str`${JSON.stringify(key)}:`) - genSerializeCode({...cxt, data: value}) + addComma(cxt, first) + gen.add(jsonStr, str`${JSON.stringify(key)}:`) + serializeCode({...cxt, data: value}) } } function serializeType(cxt: SerializeCxt): void { - const {gen, schema, result, data} = cxt - switch (schema.type) { - case "boolean": - gen.add(result, _`${data} ? "true" : "false`) - break - case "string": - serializeString(cxt) - break - case "timestamp": - gen.if(_`${data} instanceof Date`, - () => gen.add(result, _`${data}.toISOString()`), - () => serializeString(cxt) - ) - break - default: - serializeNumber(cxt) - } + const {gen, schema, jsonStr, data} = cxt + switch (schema.type) { + case "boolean": + gen.add(jsonStr, _`${data} ? "true" : "false"`) + break + case "string": + serializeString(cxt) + break + case "timestamp": + gen.if( + _`${data} instanceof Date`, + () => gen.add(jsonStr, _`${data}.toISOString()`), + () => serializeString(cxt) + ) + break + default: + serializeNumber(cxt) + } +} + +function serializeString({gen, jsonStr, data}: SerializeCxt): void { + gen.add(jsonStr, _`${quoteFunc(gen)}(${data})`) } -function serializeString(_cxt: SerializeCxt): void {} +function serializeNumber({gen, jsonStr, data}: SerializeCxt): void { + gen.add(jsonStr, _`"" + ${data}`) +} + +function serializeRef(cxt: SerializeCxt): void { + const {gen, self, jsonStr, data, definitions, schema, schemaEnv} = cxt + const {ref} = schema + const refSchema = definitions[ref] + if (!refSchema) throw new MissingRefError("", ref, `No definition ${ref}`) + if (!hasRef(refSchema)) return serializeCode({...cxt, schema: refSchema}) + const {root} = schemaEnv + const sch = compileSerializer.call(self, new SchemaEnv({schema: refSchema, root}), definitions) + gen.add(jsonStr, _`${getSerialize(gen, sch)}(${data})`) +} -function serializeNumber({gen, result, data}: SerializeCxt): void { - gen.add(result, _`"" + ${data}`) +function getSerialize(gen: CodeGen, sch: SchemaEnv): Code { + return sch.serialize + ? gen.scopeValue("serialize", {ref: sch.serialize}) + : _`${gen.scopeValue("wrapper", {ref: sch})}.serialize` } -function serializeRef(_cxt: SerializeCxt): void {} +function serializeEmpty({gen, jsonStr, data}: SerializeCxt): void { + gen.add(jsonStr, _`JSON.stringify(${data})`) +} -function serializeEmpty(_cxt: SerializeCxt): void {} +function addComma({gen, jsonStr}: SerializeCxt, first: Name): void { + gen.if( + first, + () => gen.assign(first, false), + () => gen.add(jsonStr, str`,`) + ) +} + +// eslint-disable-next-line no-control-regex, no-misleading-character-class +const rxEscapable = /[\\"\u0000-\u001f\u007f-\u009f\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g + +const rxEscapableRx = /rxEscapable/g + +const escaped: {[K in string]?: string} = { + "\b": "\\b", + "\t": "\\t", + "\n": "\\n", + "\f": "\\f", + "\r": "\\r", + '"': '\\"', + "\\": "\\\\", +} + +const escapedRx = /escapedRx/g + +function quote(s: string): string { + rxEscapable.lastIndex = 0 + return ( + '"' + + (rxEscapable.test(s) + ? s.replace(rxEscapable, (a) => { + const c = escaped[a] + return typeof c === "string" + ? c + : "\\u" + ("0000" + a.charCodeAt(0).toString(16)).slice(-4) + }) + : s) + + '"' + ) +} + +function quoteFunc(gen: CodeGen): Name { + // const quoteName = gen.getScopeValue("func", quote) + // if (quoteName) return quoteName + const escapedName = gen.scopeValue("obj", { + ref: escaped, + code: stringify(escaped), + }) + const rxEscapableName = gen.scopeValue("obj", { + ref: rxEscapable, + code: new _Code(rxEscapable.toString()), + }) + return gen.scopeValue("func", { + ref: quote, + code: new _Code( + quote + .toString() + .replace(rxEscapableRx, rxEscapableName.toString()) + .replace(escapedRx, escapedName.toString()) + ), + }) +} diff --git a/lib/core.ts b/lib/core.ts index f469d5d82..649f24875 100644 --- a/lib/core.ts +++ b/lib/core.ts @@ -63,6 +63,7 @@ import $dataRefSchema = require("./refs/data.json") const META_IGNORE_OPTIONS: (keyof Options)[] = ["removeAdditional", "useDefaults", "coerceTypes"] const EXT_SCOPE_NAMES = new Set([ "validate", + "serialize", "wrapper", "root", "schema", @@ -622,7 +623,7 @@ export default class Ajv { } } - private _addSchema( + _addSchema( schema: AnySchema, meta?: boolean, validateSchema = this.opts.validateSchema, diff --git a/lib/jtd.ts b/lib/jtd.ts index e604ba36f..ba9e4d454 100644 --- a/lib/jtd.ts +++ b/lib/jtd.ts @@ -26,13 +26,14 @@ export {KeywordCxt} // export {DefinedError} from "./vocabularies/errors" export {_, str, stringify, nil, Name, Code, CodeGen, CodeGenOptions} from "./compile/codegen" -import type {AnySchemaObject} from "./types" +import type {AnySchemaObject, SchemaObject} from "./types" import type {JTDSchemaType} from "./types/jtd-schema" export {JTDSchemaType} import AjvCore, {CurrentOptions} from "./core" import jtdVocabulary from "./vocabularies/jtd" import jtdMetaSchema from "./refs/jtd-schema" -// import {SerializeFunction, compileSerializer} from "./compile/serialize" +import {compileSerializer} from "./compile/serialize" +import {SchemaEnv} from "./compile" // const META_SUPPORT_DATA = ["/properties"] @@ -91,7 +92,15 @@ export default class Ajv extends AjvCore { super.defaultMeta() || (this.getSchema(META_SCHEMA_ID) ? META_SCHEMA_ID : undefined)) } - // compileSerializer(schema: JTDSchemaType): SerializeFunction { - // return compileSerializer.call(this, schema, schema.definitions || {}) as SerializeFunction - // } + compileSerializer(schema: SchemaObject | JTDSchemaType): (data: T) => string { + const sch = this._addSchema(schema) + return sch.serialize || this._compileSerializer(sch) + } + + private _compileSerializer(sch: SchemaEnv): (data: T) => string { + compileSerializer.call(this, sch, (sch.schema as AnySchemaObject).definitions || {}) + /* istanbul ignore if */ + if (!sch.serialize) throw new Error("ajv implementation error") + return sch.serialize + } } diff --git a/lib/vocabularies/code.ts b/lib/vocabularies/code.ts index 47c2c57d2..618960a6d 100644 --- a/lib/vocabularies/code.ts +++ b/lib/vocabularies/code.ts @@ -21,7 +21,8 @@ export function checkMissingProp( ): Code { return or( ...properties.map( - (prop) => _`${noPropertyInData(gen, data, prop, opts.ownProperties)} && (${missing} = ${prop})` + (prop) => + _`${noPropertyInData(gen, data, prop, opts.ownProperties)} && (${missing} = ${prop})` ) ) } @@ -40,7 +41,12 @@ export function isOwnProperty(gen: CodeGen, data: Name, property: Name | string) return _`${hasProp}.call(${data}, ${property})` } -export function propertyInData(gen: CodeGen, data: Name, property: Name | string, ownProperties?: boolean): Code { +export function propertyInData( + gen: CodeGen, + data: Name, + property: Name | string, + ownProperties?: boolean +): Code { const cond = _`${data}${getProperty(property)} !== undefined` return ownProperties ? _`${cond} && ${isOwnProperty(gen, data, property)}` : cond } diff --git a/lib/vocabularies/jtd/ref.ts b/lib/vocabularies/jtd/ref.ts index 5bec32d18..4aed9a9a9 100644 --- a/lib/vocabularies/jtd/ref.ts +++ b/lib/vocabularies/jtd/ref.ts @@ -57,15 +57,15 @@ const def: CodeKeywordDefinition = { valid ) } - - function hasRef(schema: AnySchemaObject): boolean { - for (const key in schema) { - let sch: AnySchemaObject - if (key === "ref" || (typeof (sch = schema[key]) == "object" && hasRef(sch))) return true - } - return false - } }, } +export function hasRef(schema: AnySchemaObject): boolean { + for (const key in schema) { + let sch: AnySchemaObject + if (key === "ref" || (typeof (sch = schema[key]) == "object" && hasRef(sch))) return true + } + return false +} + export default def diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index 3bc386f18..7fbf53b3f 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -48,7 +48,7 @@ describe("JSON Type Definition", () => { allErrors: true, inlineRefs: false, code: {es5: true, lines: true, optimize: false}, - }) + }) as AjvJTD[] ajvs.forEach((ajv) => (ajv.opts.code.source = true)) }) @@ -93,6 +93,24 @@ describe("JSON Type Definition", () => { ) } }) + + describe.skip("serialize", () => { + const ajv = new _AjvJTD() + + for (const testName in jtdValidationTests) { + const {schema, instance, errors} = jtdValidationTests[testName] as TestCase + const valid = errors.length === 0 + if (!valid) continue + describeOnly(testName, () => + it(`should serialize data`, () => { + const serialize = ajv.compileSerializer(schema) + console.log(serialize.toString()) + assert.deepStrictEqual(JSON.parse(serialize(instance)), instance) + // const opts = ajv instanceof AjvPack ? ajv.ajv.opts : ajv.opts + }) + ) + } + }) }) function describeOnly(name: string, func: () => void) { From af534f48e1fbc3727c15f8433b1d54d0bfe6e84d Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun, 21 Feb 2021 08:46:11 +0000 Subject: [PATCH 04/19] JTD: serialize (all tests pass) --- lib/compile/serialize.ts | 122 +++++++++++++++++---------------------- lib/runtime/quote.ts | 28 +++++++++ spec/jtd-schema.spec.ts | 4 +- 3 files changed, 82 insertions(+), 72 deletions(-) create mode 100644 lib/runtime/quote.ts diff --git a/lib/compile/serialize.ts b/lib/compile/serialize.ts index a8ae6a50a..63feab849 100644 --- a/lib/compile/serialize.ts +++ b/lib/compile/serialize.ts @@ -1,12 +1,12 @@ import type Ajv from "../core" import type {SchemaObject} from "../types" import {SchemaEnv, getCompilingSchema} from "." -import {_, str, and, getProperty, stringify, CodeGen, Code, Name} from "./codegen" -import {_Code} from "./codegen/code" +import {_, str, and, getProperty, CodeGen, Code, Name} from "./codegen" import {MissingRefError} from "./error_classes" import N from "./names" import {isOwnProperty} from "../vocabularies/code" import {hasRef} from "../vocabularies/jtd/ref" +import quote from "../runtime/quote" type SchemaObjectMap = {[Ref in string]?: SchemaObject} @@ -129,27 +129,30 @@ function serializeValues(cxt: SerializeCxt): void { const {gen, schema, jsonStr, data} = cxt gen.add(jsonStr, str`{`) const first = gen.let("first", true) - gen.forIn("key", data, (key) => { - addComma(cxt, first) - serializeString({...cxt, data: key}) - gen.add(jsonStr, str`:`) - const value = gen.const("value", _`${data}${getProperty(key)}`) - serializeCode({...cxt, schema: schema.values, data: value}) - }) + gen.forIn("key", data, (key) => serializeKeyValue(cxt, key, schema.values, first)) gen.add(jsonStr, str`}`) } +function serializeKeyValue(cxt: SerializeCxt, key: Name, schema: SchemaObject, first: Name): void { + const {gen, jsonStr, data} = cxt + addComma(cxt, first) + serializeString({...cxt, data: key}) + gen.add(jsonStr, str`:`) + const value = gen.const("value", _`${data}${getProperty(key)}`) + serializeCode({...cxt, schema, data: value}) +} + function serializeDiscriminator(cxt: SerializeCxt): void { const {gen, schema, jsonStr, data} = cxt const {discriminator} = schema gen.add(jsonStr, str`{${JSON.stringify(discriminator)}:`) const tag = gen.const("tag", _`${data}${getProperty(discriminator)}`) serializeString({...cxt, data: tag}) - const first = gen.let("first", false) gen.if(false) for (const tagValue in schema.mapping) { gen.elseIf(_`${tag} === ${tagValue}`) - serializeSchemaProperties({...cxt, schema: schema.mapping[tagValue]}, first) + const sch = schema.mapping[tagValue] + serializeSchemaProperties({...cxt, schema: sch}, discriminator) } gen.endIf() gen.add(jsonStr, str`}`) @@ -158,32 +161,59 @@ function serializeDiscriminator(cxt: SerializeCxt): void { function serializeProperties(cxt: SerializeCxt): void { const {gen, jsonStr} = cxt gen.add(jsonStr, str`{`) - const first = gen.let("first", true) - serializeSchemaProperties(cxt, first) + serializeSchemaProperties(cxt) gen.add(jsonStr, str`}`) } -function serializeSchemaProperties(cxt: SerializeCxt, first: Name): void { +function serializeSchemaProperties(cxt: SerializeCxt, discriminator?: string): void { const {gen, schema, jsonStr, data} = cxt const {properties, optionalProperties} = schema - for (const key in properties || {}) { - serializeProperty(key, keyValue(key)) + const props = keys(properties) + const optProps = keys(optionalProperties) + const allProps = allProperties(props.concat(optProps)) + let first = !discriminator + for (const key of props) { + serializeProperty(key, properties[key], keyValue(key)) } - for (const key in optionalProperties || {}) { + for (const key of optProps) { const value = keyValue(key) gen.if(and(_`${value} !== undefined`, isOwnProperty(gen, data, key)), () => - serializeProperty(key, value) + serializeProperty(key, optionalProperties[key], value) ) } + if (schema.additionalProperties) { + gen.forIn("key", data, (key) => + gen.if(isAdditional(key, allProps), () => + serializeKeyValue(cxt, key, {}, gen.let("first", first)) + ) + ) + } + + function keys(ps?: SchemaObjectMap): string[] { + return ps ? Object.keys(ps) : [] + } + + function allProperties(ps: string[]): string[] { + if (discriminator) ps.push(discriminator) + if (new Set(ps).size !== ps.length) { + throw new Error("JTD: properties/optionalProperties/disciminator overlap") + } + return ps + } function keyValue(key: string): Name { return gen.const("value", _`${data}${getProperty(key)}`) } - function serializeProperty(key: string, value: Name): void { - addComma(cxt, first) + function serializeProperty(key: string, propSchema: SchemaObject, value: Name): void { + if (first) first = false + else gen.add(jsonStr, str`,`) gen.add(jsonStr, str`${JSON.stringify(key)}:`) - serializeCode({...cxt, data: value}) + serializeCode({...cxt, schema: propSchema, data: value}) + } + + function isAdditional(key: Name, ps: string[]): Code | true { + return ps.length ? and(...ps.map((p) => _`${key} !== ${p}`)) : true } } @@ -245,57 +275,9 @@ function addComma({gen, jsonStr}: SerializeCxt, first: Name): void { ) } -// eslint-disable-next-line no-control-regex, no-misleading-character-class -const rxEscapable = /[\\"\u0000-\u001f\u007f-\u009f\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g - -const rxEscapableRx = /rxEscapable/g - -const escaped: {[K in string]?: string} = { - "\b": "\\b", - "\t": "\\t", - "\n": "\\n", - "\f": "\\f", - "\r": "\\r", - '"': '\\"', - "\\": "\\\\", -} - -const escapedRx = /escapedRx/g - -function quote(s: string): string { - rxEscapable.lastIndex = 0 - return ( - '"' + - (rxEscapable.test(s) - ? s.replace(rxEscapable, (a) => { - const c = escaped[a] - return typeof c === "string" - ? c - : "\\u" + ("0000" + a.charCodeAt(0).toString(16)).slice(-4) - }) - : s) + - '"' - ) -} - function quoteFunc(gen: CodeGen): Name { - // const quoteName = gen.getScopeValue("func", quote) - // if (quoteName) return quoteName - const escapedName = gen.scopeValue("obj", { - ref: escaped, - code: stringify(escaped), - }) - const rxEscapableName = gen.scopeValue("obj", { - ref: rxEscapable, - code: new _Code(rxEscapable.toString()), - }) return gen.scopeValue("func", { ref: quote, - code: new _Code( - quote - .toString() - .replace(rxEscapableRx, rxEscapableName.toString()) - .replace(escapedRx, escapedName.toString()) - ), + code: _`require("ajv/dist/runtime/quote").default`, }) } diff --git a/lib/runtime/quote.ts b/lib/runtime/quote.ts new file mode 100644 index 000000000..c3309a0c4 --- /dev/null +++ b/lib/runtime/quote.ts @@ -0,0 +1,28 @@ +// eslint-disable-next-line no-control-regex, no-misleading-character-class +const rxEscapable = /[\\"\u0000-\u001f\u007f-\u009f\u00ad\u0600-\u0604\u070f\u17b4\u17b5\u200c-\u200f\u2028-\u202f\u2060-\u206f\ufeff\ufff0-\uffff]/g + +const escaped: {[K in string]?: string} = { + "\b": "\\b", + "\t": "\\t", + "\n": "\\n", + "\f": "\\f", + "\r": "\\r", + '"': '\\"', + "\\": "\\\\", +} + +export default function quote(s: string): string { + rxEscapable.lastIndex = 0 + return ( + '"' + + (rxEscapable.test(s) + ? s.replace(rxEscapable, (a) => { + const c = escaped[a] + return typeof c === "string" + ? c + : "\\u" + ("0000" + a.charCodeAt(0).toString(16)).slice(-4) + }) + : s) + + '"' + ) +} diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index 7fbf53b3f..881df665b 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -94,7 +94,7 @@ describe("JSON Type Definition", () => { } }) - describe.skip("serialize", () => { + describe("serialize", () => { const ajv = new _AjvJTD() for (const testName in jtdValidationTests) { @@ -104,7 +104,7 @@ describe("JSON Type Definition", () => { describeOnly(testName, () => it(`should serialize data`, () => { const serialize = ajv.compileSerializer(schema) - console.log(serialize.toString()) + // console.log(serialize.toString()) assert.deepStrictEqual(JSON.parse(serialize(instance)), instance) // const opts = ajv instanceof AjvPack ? ajv.ajv.opts : ajv.opts }) From d5d1b7c13d8b8d554b058e0365df19507dc0996a Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun, 21 Feb 2021 09:34:07 +0000 Subject: [PATCH 05/19] refactor: fixed name for JSON string in JTD serialize --- lib/compile/names.ts | 2 ++ lib/compile/serialize.ts | 75 ++++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/lib/compile/names.ts b/lib/compile/names.ts index 325c80a2d..4f1231d0d 100644 --- a/lib/compile/names.ts +++ b/lib/compile/names.ts @@ -17,6 +17,8 @@ const names = { // "globals" self: new Name("self"), scope: new Name("scope"), + // JTD serialize/parse name for JSON string + json: new Name("json") } export default names diff --git a/lib/compile/serialize.ts b/lib/compile/serialize.ts index 63feab849..22237a18c 100644 --- a/lib/compile/serialize.ts +++ b/lib/compile/serialize.ts @@ -39,7 +39,6 @@ interface SerializeCxt { readonly self: Ajv // current Ajv instance readonly schemaEnv: SchemaEnv readonly definitions: SchemaObjectMap - readonly jsonStr: Name schema: SchemaObject data: Code } @@ -55,14 +54,12 @@ export function compileSerializer( const {ownProperties} = this.opts const gen = new CodeGen(this.scope, {es5, lines, ownProperties}) const serializeName = gen.scopeName("serialize") - const jsonStr = gen.name("json") const cxt = { self: this, gen, schema: sch.schema as SchemaObject, schemaEnv: sch, definitions, - jsonStr, data: N.data, } @@ -71,9 +68,9 @@ export function compileSerializer( this._compilations.add(sch) sch.serializeName = serializeName gen.func(serializeName, N.data, false, () => { - gen.let(jsonStr, str``) + gen.let(N.json, str``) serializeCode(cxt) - gen.return(jsonStr) + gen.return(N.json) }) gen.optimize(this.opts.code.optimize) const serializeFuncCode = gen.toString() @@ -105,47 +102,49 @@ function serializeCode(cxt: SerializeCxt): void { } function serializeNullable(cxt: SerializeCxt, serializeForm: (_cxt: SerializeCxt) => void): void { - const {gen, schema, jsonStr, data} = cxt + const {gen, schema, data} = cxt if (!schema.nullable) return serializeForm(cxt) gen.if( _`${data} === undefined || ${data} === null`, - () => gen.add(jsonStr, _`"null"`), + () => gen.add(N.json, _`"null"`), () => serializeForm(cxt) ) } function serializeElements(cxt: SerializeCxt): void { - const {gen, schema, jsonStr, data} = cxt - gen.add(jsonStr, str`[`) + const {gen, schema, data} = cxt + gen.add(N.json, str`[`) const first = gen.let("first", true) gen.forOf("el", data, (el) => { addComma(cxt, first) serializeCode({...cxt, schema: schema.elements, data: el}) }) - gen.add(jsonStr, str`]`) + gen.add(N.json, str`]`) } function serializeValues(cxt: SerializeCxt): void { - const {gen, schema, jsonStr, data} = cxt - gen.add(jsonStr, str`{`) + const {gen, schema, data} = cxt + gen.add(N.json, str`{`) const first = gen.let("first", true) - gen.forIn("key", data, (key) => serializeKeyValue(cxt, key, schema.values, first)) - gen.add(jsonStr, str`}`) + gen.forIn("key", data, (key) => + serializeKeyValue(cxt, key, schema.values, first) + ) + gen.add(N.json, str`}`) } function serializeKeyValue(cxt: SerializeCxt, key: Name, schema: SchemaObject, first: Name): void { - const {gen, jsonStr, data} = cxt + const {gen, data} = cxt addComma(cxt, first) serializeString({...cxt, data: key}) - gen.add(jsonStr, str`:`) + gen.add(N.json, str`:`) const value = gen.const("value", _`${data}${getProperty(key)}`) serializeCode({...cxt, schema, data: value}) } function serializeDiscriminator(cxt: SerializeCxt): void { - const {gen, schema, jsonStr, data} = cxt + const {gen, schema, data} = cxt const {discriminator} = schema - gen.add(jsonStr, str`{${JSON.stringify(discriminator)}:`) + gen.add(N.json, str`{${JSON.stringify(discriminator)}:`) const tag = gen.const("tag", _`${data}${getProperty(discriminator)}`) serializeString({...cxt, data: tag}) gen.if(false) @@ -155,18 +154,18 @@ function serializeDiscriminator(cxt: SerializeCxt): void { serializeSchemaProperties({...cxt, schema: sch}, discriminator) } gen.endIf() - gen.add(jsonStr, str`}`) + gen.add(N.json, str`}`) } function serializeProperties(cxt: SerializeCxt): void { - const {gen, jsonStr} = cxt - gen.add(jsonStr, str`{`) + const {gen} = cxt + gen.add(N.json, str`{`) serializeSchemaProperties(cxt) - gen.add(jsonStr, str`}`) + gen.add(N.json, str`}`) } function serializeSchemaProperties(cxt: SerializeCxt, discriminator?: string): void { - const {gen, schema, jsonStr, data} = cxt + const {gen, schema, data} = cxt const {properties, optionalProperties} = schema const props = keys(properties) const optProps = keys(optionalProperties) @@ -207,8 +206,8 @@ function serializeSchemaProperties(cxt: SerializeCxt, discriminator?: string): v function serializeProperty(key: string, propSchema: SchemaObject, value: Name): void { if (first) first = false - else gen.add(jsonStr, str`,`) - gen.add(jsonStr, str`${JSON.stringify(key)}:`) + else gen.add(N.json, str`,`) + gen.add(N.json, str`${JSON.stringify(key)}:`) serializeCode({...cxt, schema: propSchema, data: value}) } @@ -218,10 +217,10 @@ function serializeSchemaProperties(cxt: SerializeCxt, discriminator?: string): v } function serializeType(cxt: SerializeCxt): void { - const {gen, schema, jsonStr, data} = cxt + const {gen, schema, data} = cxt switch (schema.type) { case "boolean": - gen.add(jsonStr, _`${data} ? "true" : "false"`) + gen.add(N.json, _`${data} ? "true" : "false"`) break case "string": serializeString(cxt) @@ -229,7 +228,7 @@ function serializeType(cxt: SerializeCxt): void { case "timestamp": gen.if( _`${data} instanceof Date`, - () => gen.add(jsonStr, _`${data}.toISOString()`), + () => gen.add(N.json, _`${data}.toISOString()`), () => serializeString(cxt) ) break @@ -238,23 +237,23 @@ function serializeType(cxt: SerializeCxt): void { } } -function serializeString({gen, jsonStr, data}: SerializeCxt): void { - gen.add(jsonStr, _`${quoteFunc(gen)}(${data})`) +function serializeString({gen, data}: SerializeCxt): void { + gen.add(N.json, _`${quoteFunc(gen)}(${data})`) } -function serializeNumber({gen, jsonStr, data}: SerializeCxt): void { - gen.add(jsonStr, _`"" + ${data}`) +function serializeNumber({gen, data}: SerializeCxt): void { + gen.add(N.json, _`"" + ${data}`) } function serializeRef(cxt: SerializeCxt): void { - const {gen, self, jsonStr, data, definitions, schema, schemaEnv} = cxt + const {gen, self, data, definitions, schema, schemaEnv} = cxt const {ref} = schema const refSchema = definitions[ref] if (!refSchema) throw new MissingRefError("", ref, `No definition ${ref}`) if (!hasRef(refSchema)) return serializeCode({...cxt, schema: refSchema}) const {root} = schemaEnv const sch = compileSerializer.call(self, new SchemaEnv({schema: refSchema, root}), definitions) - gen.add(jsonStr, _`${getSerialize(gen, sch)}(${data})`) + gen.add(N.json, _`${getSerialize(gen, sch)}(${data})`) } function getSerialize(gen: CodeGen, sch: SchemaEnv): Code { @@ -263,15 +262,15 @@ function getSerialize(gen: CodeGen, sch: SchemaEnv): Code { : _`${gen.scopeValue("wrapper", {ref: sch})}.serialize` } -function serializeEmpty({gen, jsonStr, data}: SerializeCxt): void { - gen.add(jsonStr, _`JSON.stringify(${data})`) +function serializeEmpty({gen, data}: SerializeCxt): void { + gen.add(N.json, _`JSON.stringify(${data})`) } -function addComma({gen, jsonStr}: SerializeCxt, first: Name): void { +function addComma({gen}: SerializeCxt, first: Name): void { gen.if( first, () => gen.assign(first, false), - () => gen.add(jsonStr, str`,`) + () => gen.add(N.json, str`,`) ) } From c59fa7f98d912e43bf097f88314bc1eb71cabf4c Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun, 21 Feb 2021 10:23:12 +0000 Subject: [PATCH 06/19] add benchmarks for JTD serializers to compare with JSON.stringify --- benchmark/package.json | 6 ++++ benchmark/serialize.js | 73 ++++++++++++++++++++++++++++++++++++++++++ package.json | 3 +- 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 benchmark/package.json create mode 100644 benchmark/serialize.js diff --git a/benchmark/package.json b/benchmark/package.json new file mode 100644 index 000000000..2fd10970a --- /dev/null +++ b/benchmark/package.json @@ -0,0 +1,6 @@ +{ + "private": true, + "devDependencies": { + "benchmark": "^2.1.4" + } +} diff --git a/benchmark/serialize.js b/benchmark/serialize.js new file mode 100644 index 000000000..23815c1a2 --- /dev/null +++ b/benchmark/serialize.js @@ -0,0 +1,73 @@ +/* eslint-disable no-console */ +const Ajv = require("ajv/dist/jtd").default +const Benchmark = require('benchmark') +const jtdValidationTests = require("../spec/json-typedef-spec/tests/validation.json") + +const ajv = new Ajv +const suite = new Benchmark.Suite +const tests = [] + +for (const testName in jtdValidationTests) { + const {schema, instance, errors} = jtdValidationTests[testName] + const valid = errors.length === 0 + if (!valid) continue + tests.push({ + serialize: ajv.compileSerializer(schema), + data: instance + }) +} + +suite.add("JTD test suite: compiled JTD serializers", () => { + for (const test of tests) { + test.serialize(test.data) + } +}) + +suite.add("JTD test suite: JSON.stringify", () => { + for (const test of tests) { + JSON.stringify(test.data) + } +}) + +const testSchema = { + definitions: { + obj: { + properties: { + foo: {type: "string"}, + bar: {type: "int8"} + } + } + }, + properties: { + a: {ref: "obj"} + }, + optionalProperties: { + b: {ref: "obj"} + } +} + +const testData = { + a: { + foo: "foo1", + bar: 1 + }, + b: { + foo: "foo2", + bar: 2 + } +} + +const serializer = ajv.compileSerializer(testSchema) + +suite.add("test data: compiled JTD serializer", () => serializer(testData)) +suite.add("test data: JSON.stringify", () => JSON.stringify(testData)) + +console.log() + +suite + .on("cycle", (event) => console.log(String(event.target))) + .on("complete", function () { + // eslint-disable-next-line no-invalid-this + console.log('The fastest is "' + this.filter('fastest').map('name') + '"'); + }) + .run({async: true}) diff --git a/package.json b/package.json index e73f18b7e..668b1bffd 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "test-all": "npm run test-cov && if-node-version 12 npm run test-browser", "test": "npm link && npm link ajv && npm run json-tests && npm run eslint && npm run test-cov", "test-ci": "AJV_FULL_TEST=true npm test", - "prepublish": "npm run build" + "prepublish": "npm run build", + "benchmark": "npm i && npm run build && npm link && cd ./benchmark && npm link ajv && npm i && node ./serialize" }, "nyc": { "exclude": [ From 1e10ab38b887a6f507153afe7f36e71f1e625f57 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun, 21 Feb 2021 10:59:33 +0000 Subject: [PATCH 07/19] move serialize.ts to jtd folder --- lib/compile/{ => jtd}/serialize.ts | 18 +++++++++--------- lib/jtd.ts | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) rename lib/compile/{ => jtd}/serialize.ts (94%) diff --git a/lib/compile/serialize.ts b/lib/compile/jtd/serialize.ts similarity index 94% rename from lib/compile/serialize.ts rename to lib/compile/jtd/serialize.ts index 22237a18c..4b1e9dfc0 100644 --- a/lib/compile/serialize.ts +++ b/lib/compile/jtd/serialize.ts @@ -1,12 +1,12 @@ -import type Ajv from "../core" -import type {SchemaObject} from "../types" -import {SchemaEnv, getCompilingSchema} from "." -import {_, str, and, getProperty, CodeGen, Code, Name} from "./codegen" -import {MissingRefError} from "./error_classes" -import N from "./names" -import {isOwnProperty} from "../vocabularies/code" -import {hasRef} from "../vocabularies/jtd/ref" -import quote from "../runtime/quote" +import type Ajv from "../../core" +import type {SchemaObject} from "../../types" +import {SchemaEnv, getCompilingSchema} from ".." +import {_, str, and, getProperty, CodeGen, Code, Name} from "../codegen" +import {MissingRefError} from "../error_classes" +import N from "../names" +import {isOwnProperty} from "../../vocabularies/code" +import {hasRef} from "../../vocabularies/jtd/ref" +import quote from "../../runtime/quote" type SchemaObjectMap = {[Ref in string]?: SchemaObject} diff --git a/lib/jtd.ts b/lib/jtd.ts index ba9e4d454..e6225367e 100644 --- a/lib/jtd.ts +++ b/lib/jtd.ts @@ -32,7 +32,7 @@ export {JTDSchemaType} import AjvCore, {CurrentOptions} from "./core" import jtdVocabulary from "./vocabularies/jtd" import jtdMetaSchema from "./refs/jtd-schema" -import {compileSerializer} from "./compile/serialize" +import {compileSerializer} from "./compile/jtd/serialize" import {SchemaEnv} from "./compile" // const META_SUPPORT_DATA = ["/properties"] From 07f0a1903c4c4c17612022f8ac8bec75a54d6497 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun, 21 Feb 2021 14:09:46 +0000 Subject: [PATCH 08/19] JTD: generate JSON parsers (WIP) --- benchmark/serialize.js | 28 ++-- lib/compile/index.ts | 2 + lib/compile/jtd/parse.ts | 314 +++++++++++++++++++++++++++++++++++ lib/compile/jtd/serialize.ts | 26 +-- lib/compile/jtd/types.ts | 16 ++ lib/compile/names.ts | 5 +- lib/core.ts | 1 + lib/jtd.ts | 15 +- lib/runtime/jsonParse.ts | 15 ++ spec/jtd-schema.spec.ts | 25 ++- 10 files changed, 408 insertions(+), 39 deletions(-) create mode 100644 lib/compile/jtd/parse.ts create mode 100644 lib/compile/jtd/types.ts create mode 100644 lib/runtime/jsonParse.ts diff --git a/benchmark/serialize.js b/benchmark/serialize.js index 23815c1a2..692da1888 100644 --- a/benchmark/serialize.js +++ b/benchmark/serialize.js @@ -1,10 +1,10 @@ /* eslint-disable no-console */ const Ajv = require("ajv/dist/jtd").default -const Benchmark = require('benchmark') +const Benchmark = require("benchmark") const jtdValidationTests = require("../spec/json-typedef-spec/tests/validation.json") -const ajv = new Ajv -const suite = new Benchmark.Suite +const ajv = new Ajv() +const suite = new Benchmark.Suite() const tests = [] for (const testName in jtdValidationTests) { @@ -13,7 +13,7 @@ for (const testName in jtdValidationTests) { if (!valid) continue tests.push({ serialize: ajv.compileSerializer(schema), - data: instance + data: instance, }) } @@ -34,27 +34,27 @@ const testSchema = { obj: { properties: { foo: {type: "string"}, - bar: {type: "int8"} - } - } + bar: {type: "int8"}, + }, + }, }, properties: { - a: {ref: "obj"} + a: {ref: "obj"}, }, optionalProperties: { - b: {ref: "obj"} - } + b: {ref: "obj"}, + }, } const testData = { a: { foo: "foo1", - bar: 1 + bar: 1, }, b: { foo: "foo2", - bar: 2 - } + bar: 2, + }, } const serializer = ajv.compileSerializer(testSchema) @@ -68,6 +68,6 @@ suite .on("cycle", (event) => console.log(String(event.target))) .on("complete", function () { // eslint-disable-next-line no-invalid-this - console.log('The fastest is "' + this.filter('fastest').map('name') + '"'); + console.log('The fastest is "' + this.filter("fastest").map("name") + '"') }) .run({async: true}) diff --git a/lib/compile/index.ts b/lib/compile/index.ts index ba928640f..04eb4e5b4 100644 --- a/lib/compile/index.ts +++ b/lib/compile/index.ts @@ -81,6 +81,8 @@ export class SchemaEnv implements SchemaEnvArgs { validateName?: ValueScopeName serialize?: (data: unknown) => string serializeName?: ValueScopeName + parse?: (data: string) => unknown + parseName?: ValueScopeName constructor(env: SchemaEnvArgs) { let schema: AnySchemaObject | undefined diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts new file mode 100644 index 000000000..161e6f940 --- /dev/null +++ b/lib/compile/jtd/parse.ts @@ -0,0 +1,314 @@ +import type Ajv from "../../core" +import type {SchemaObject} from "../../types" +import {jtdForms, JTDForm, SchemaObjectMap} from "./types" +import {SchemaEnv, getCompilingSchema} from ".." +import {_, str, and, getProperty, CodeGen, Code, Name} from "../codegen" +import {_Code} from "../codegen/code" +import {MissingRefError} from "../error_classes" +import N from "../names" +import {isOwnProperty} from "../../vocabularies/code" +import {hasRef} from "../../vocabularies/jtd/ref" +import jsonParse from "../../runtime/jsonParse" + +type GenParse = (cxt: ParseCxt) => void + +const genParse: {[F in JTDForm]: GenParse} = { + elements: parseElements, + values: parseValues, + discriminator: parseDiscriminator, + properties: parseProperties, + optionalProperties: parseProperties, + enum: parseString, + type: parseType, + ref: parseRef, +} + +interface ParseCxt { + readonly gen: CodeGen + readonly self: Ajv // current Ajv instance + readonly schemaEnv: SchemaEnv + readonly definitions: SchemaObjectMap + schema: SchemaObject + data: Code + jsonPos: Name | number +} + +export default function compileParser( + this: Ajv, + sch: SchemaEnv, + definitions: SchemaObjectMap +): SchemaEnv { + const _sch = getCompilingSchema.call(this, sch) + if (_sch) return _sch + const {es5, lines} = this.opts.code + const {ownProperties} = this.opts + const gen = new CodeGen(this.scope, {es5, lines, ownProperties}) + const parseName = gen.scopeName("parse") + const cxt: ParseCxt = { + self: this, + gen, + schema: sch.schema as SchemaObject, + schemaEnv: sch, + definitions, + data: N.data, + jsonPos: 0, + } + + let sourceCode: string | undefined + try { + this._compilations.add(sch) + sch.parseName = parseName + gen.func(parseName, N.json, false, () => { + gen.let(N.data) + gen.let(N.jsonPos) + parseCode(cxt) + gen.if( + _`${N.jsonPos} === ${N.json}.length`, + () => gen.return(N.data), + () => throwSyntaxError(gen) + // gen.throw( + // _`new SyntaxError("Unexpected token " + ${N.json}[${N.jsonPos}] + " in JSON at position " + ${N.jsonPos})` + // ) + ) + }) + gen.optimize(this.opts.code.optimize) + const parseFuncCode = gen.toString() + sourceCode = `${gen.scopeRefs(N.scope)}return ${parseFuncCode}` + const makeParse = new Function(`${N.scope}`, sourceCode) + const parse: (json: string) => unknown = makeParse(this.scope.get()) + this.scope.value(parseName, {ref: parse}) + sch.parse = parse + } catch (e) { + if (sourceCode) this.logger.error("Error compiling parser, function code:", sourceCode) + delete sch.parse + delete sch.parseName + throw e + } finally { + this._compilations.delete(sch) + } + return sch +} + +function parseCode(cxt: ParseCxt): void { + let form: JTDForm | undefined + for (const key of jtdForms) { + if (key in cxt.schema) { + form = key + break + } + } + parseNullable(cxt, form ? genParse[form] : parseEmpty) +} + +function parseNullable(cxt: ParseCxt, parseForm: GenParse): void { + const {gen, schema, data} = cxt + if (!schema.nullable) return parseForm(cxt) + tryParse(cxt, "null", () => gen.assign(data, null), parseForm) +} + +function tryParse(cxt: ParseCxt, s: string, success: GenParse, fail: GenParse): void { + const {gen, jsonPos} = cxt + const slice = + s.length === 1 ? _`${N.json}[${jsonPos}]` : _`${N.json}.slice(${jsonPos}, ${s.length})` + gen.if( + _`${slice} === ${s}`, + () => { + addJsonPos() + success(cxt) + }, + () => fail(cxt) + ) + + function addJsonPos(): void { + if (jsonPos instanceof Name) { + gen.add(jsonPos, s.length) + } else { + gen.assign(N.jsonPos, jsonPos + s.length) + cxt.jsonPos = N.jsonPos + } + } +} + +function parseElements(cxt: ParseCxt): void { + const {gen, schema, data} = cxt + gen.add(N.json, str`[`) + const first = gen.let("first", true) + gen.forOf("el", data, (el) => { + addComma(cxt, first) + parseCode({...cxt, schema: schema.elements, data: el}) + }) + gen.add(N.json, str`]`) +} + +function parseValues(cxt: ParseCxt): void { + const {gen, schema, data} = cxt + gen.add(N.json, str`{`) + const first = gen.let("first", true) + gen.forIn("key", data, (key) => parseKeyValue(cxt, key, schema.values, first)) + gen.add(N.json, str`}`) +} + +function parseKeyValue(cxt: ParseCxt, key: Name, schema: SchemaObject, first: Name): void { + const {gen, data} = cxt + addComma(cxt, first) + parseString({...cxt, data: key}) + gen.add(N.json, str`:`) + const value = gen.const("value", _`${data}${getProperty(key)}`) + parseCode({...cxt, schema, data: value}) +} + +function parseDiscriminator(cxt: ParseCxt): void { + const {gen, schema, data} = cxt + const {discriminator} = schema + gen.add(N.json, str`{${JSON.stringify(discriminator)}:`) + const tag = gen.const("tag", _`${data}${getProperty(discriminator)}`) + parseString({...cxt, data: tag}) + gen.if(false) + for (const tagValue in schema.mapping) { + gen.elseIf(_`${tag} === ${tagValue}`) + const sch = schema.mapping[tagValue] + parseSchemaProperties({...cxt, schema: sch}, discriminator) + } + gen.endIf() + gen.add(N.json, str`}`) +} + +function parseProperties(cxt: ParseCxt): void { + const {gen} = cxt + gen.add(N.json, str`{`) + parseSchemaProperties(cxt) + gen.add(N.json, str`}`) +} + +function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { + const {gen, schema, data} = cxt + const {properties, optionalProperties} = schema + const props = keys(properties) + const optProps = keys(optionalProperties) + const allProps = allProperties(props.concat(optProps)) + let first = !discriminator + for (const key of props) { + parseProperty(key, properties[key], keyValue(key)) + } + for (const key of optProps) { + const value = keyValue(key) + gen.if(and(_`${value} !== undefined`, isOwnProperty(gen, data, key)), () => + parseProperty(key, optionalProperties[key], value) + ) + } + if (schema.additionalProperties) { + gen.forIn("key", data, (key) => + gen.if(isAdditional(key, allProps), () => + parseKeyValue(cxt, key, {}, gen.let("first", first)) + ) + ) + } + + function keys(ps?: SchemaObjectMap): string[] { + return ps ? Object.keys(ps) : [] + } + + function allProperties(ps: string[]): string[] { + if (discriminator) ps.push(discriminator) + if (new Set(ps).size !== ps.length) { + throw new Error("JTD: properties/optionalProperties/disciminator overlap") + } + return ps + } + + function keyValue(key: string): Name { + return gen.const("value", _`${data}${getProperty(key)}`) + } + + function parseProperty(key: string, propSchema: SchemaObject, value: Name): void { + if (first) first = false + else gen.add(N.json, str`,`) + gen.add(N.json, str`${JSON.stringify(key)}:`) + parseCode({...cxt, schema: propSchema, data: value}) + } + + function isAdditional(key: Name, ps: string[]): Code | true { + return ps.length ? and(...ps.map((p) => _`${key} !== ${p}`)) : true + } +} + +function parseType(cxt: ParseCxt): void { + const {gen, schema, data} = cxt + switch (schema.type) { + case "boolean": + gen.add(N.json, _`${data} ? "true" : "false"`) + break + case "string": + parseString(cxt) + break + case "timestamp": + gen.if( + _`${data} instanceof Date`, + () => gen.add(N.json, _`${data}.toISOString()`), + () => parseString(cxt) + ) + break + default: + parseNumber(cxt) + } +} + +function parseString({gen, data}: ParseCxt): void { + gen.add(N.json, _`${jsonParseFunc(gen)}(${data})`) +} + +function parseNumber({gen, data}: ParseCxt): void { + gen.add(N.json, _`"" + ${data}`) +} + +function parseRef(cxt: ParseCxt): void { + const {gen, self, data, definitions, schema, schemaEnv} = cxt + const {ref} = schema + const refSchema = definitions[ref] + if (!refSchema) throw new MissingRefError("", ref, `No definition ${ref}`) + if (!hasRef(refSchema)) return parseCode({...cxt, schema: refSchema}) + const {root} = schemaEnv + const sch = compileParser.call(self, new SchemaEnv({schema: refSchema, root}), definitions) + gen.add(N.json, _`${getParse(gen, sch)}(${data})`) +} + +function getParse(gen: CodeGen, sch: SchemaEnv): Code { + return sch.parse + ? gen.scopeValue("parse", {ref: sch.parse}) + : _`${gen.scopeValue("wrapper", {ref: sch})}.parse` +} + +function parseEmpty(cxt: ParseCxt): void { + const {gen, data, jsonPos} = cxt + gen.assign(_`[${data}, ${N.jsonPos}]`, _`${jsonParseFunc(gen)}(${N.json}, ${jsonPos})`) + cxt.jsonPos = N.jsonPos +} + +function addComma({gen}: ParseCxt, first: Name): void { + gen.if( + first, + () => gen.assign(first, false), + () => gen.add(N.json, str`,`) + ) +} + +function jsonParseFunc(gen: CodeGen): Name { + return gen.scopeValue("func", { + ref: jsonParse, + code: _`require("ajv/dist/runtime/jsonParse").default`, + }) +} + +function syntaxError(json: string, jsonPos: number): void { + throw new SyntaxError(`Unexpected token ${json[jsonPos]} in JSON at position ${jsonPos}`) +} + +const syntaxErrorCode = new _Code(syntaxError.toString()) + +function throwSyntaxError(gen: CodeGen): void { + const throwError = gen.scopeValue("func", { + ref: syntaxError, + code: syntaxErrorCode, + }) + gen.code(_`${throwError}(${N.json}, ${N.jsonPos})`) +} diff --git a/lib/compile/jtd/serialize.ts b/lib/compile/jtd/serialize.ts index 4b1e9dfc0..63b253dd4 100644 --- a/lib/compile/jtd/serialize.ts +++ b/lib/compile/jtd/serialize.ts @@ -1,5 +1,6 @@ import type Ajv from "../../core" import type {SchemaObject} from "../../types" +import {jtdForms, JTDForm, SchemaObjectMap} from "./types" import {SchemaEnv, getCompilingSchema} from ".." import {_, str, and, getProperty, CodeGen, Code, Name} from "../codegen" import {MissingRefError} from "../error_classes" @@ -8,22 +9,7 @@ import {isOwnProperty} from "../../vocabularies/code" import {hasRef} from "../../vocabularies/jtd/ref" import quote from "../../runtime/quote" -type SchemaObjectMap = {[Ref in string]?: SchemaObject} - -const jtdForms = [ - "elements", - "values", - "discriminator", - "properties", - "optionalProperties", - "enum", - "type", - "ref", -] as const - -type JTDForm = typeof jtdForms[number] - -const genSerialize: {[S in JTDForm]: (cxt: SerializeCxt) => void} = { +const genSerialize: {[F in JTDForm]: (cxt: SerializeCxt) => void} = { elements: serializeElements, values: serializeValues, discriminator: serializeDiscriminator, @@ -43,7 +29,7 @@ interface SerializeCxt { data: Code } -export function compileSerializer( +export default function compileSerializer( this: Ajv, sch: SchemaEnv, definitions: SchemaObjectMap @@ -54,7 +40,7 @@ export function compileSerializer( const {ownProperties} = this.opts const gen = new CodeGen(this.scope, {es5, lines, ownProperties}) const serializeName = gen.scopeName("serialize") - const cxt = { + const cxt: SerializeCxt = { self: this, gen, schema: sch.schema as SchemaObject, @@ -126,9 +112,7 @@ function serializeValues(cxt: SerializeCxt): void { const {gen, schema, data} = cxt gen.add(N.json, str`{`) const first = gen.let("first", true) - gen.forIn("key", data, (key) => - serializeKeyValue(cxt, key, schema.values, first) - ) + gen.forIn("key", data, (key) => serializeKeyValue(cxt, key, schema.values, first)) gen.add(N.json, str`}`) } diff --git a/lib/compile/jtd/types.ts b/lib/compile/jtd/types.ts new file mode 100644 index 000000000..7f3619576 --- /dev/null +++ b/lib/compile/jtd/types.ts @@ -0,0 +1,16 @@ +import type {SchemaObject} from "../../types" + +export type SchemaObjectMap = {[Ref in string]?: SchemaObject} + +export const jtdForms = [ + "elements", + "values", + "discriminator", + "properties", + "optionalProperties", + "enum", + "type", + "ref", +] as const + +export type JTDForm = typeof jtdForms[number] diff --git a/lib/compile/names.ts b/lib/compile/names.ts index 4f1231d0d..4b91756c1 100644 --- a/lib/compile/names.ts +++ b/lib/compile/names.ts @@ -17,8 +17,9 @@ const names = { // "globals" self: new Name("self"), scope: new Name("scope"), - // JTD serialize/parse name for JSON string - json: new Name("json") + // JTD serialize/parse name for JSON string and position + json: new Name("json"), + jsonPos: new Name("jsonPos"), } export default names diff --git a/lib/core.ts b/lib/core.ts index 649f24875..56431c490 100644 --- a/lib/core.ts +++ b/lib/core.ts @@ -64,6 +64,7 @@ const META_IGNORE_OPTIONS: (keyof Options)[] = ["removeAdditional", "useDefaults const EXT_SCOPE_NAMES = new Set([ "validate", "serialize", + "parse", "wrapper", "root", "schema", diff --git a/lib/jtd.ts b/lib/jtd.ts index e6225367e..d73452dc4 100644 --- a/lib/jtd.ts +++ b/lib/jtd.ts @@ -32,7 +32,8 @@ export {JTDSchemaType} import AjvCore, {CurrentOptions} from "./core" import jtdVocabulary from "./vocabularies/jtd" import jtdMetaSchema from "./refs/jtd-schema" -import {compileSerializer} from "./compile/jtd/serialize" +import compileSerializer from "./compile/jtd/serialize" +import compileParser from "./compile/jtd/parse" import {SchemaEnv} from "./compile" // const META_SUPPORT_DATA = ["/properties"] @@ -97,10 +98,22 @@ export default class Ajv extends AjvCore { return sch.serialize || this._compileSerializer(sch) } + compileParser(schema: SchemaObject | JTDSchemaType): (json: string) => T { + const sch = this._addSchema(schema) + return (sch.parse || this._compileParser(sch)) as (json: string) => T + } + private _compileSerializer(sch: SchemaEnv): (data: T) => string { compileSerializer.call(this, sch, (sch.schema as AnySchemaObject).definitions || {}) /* istanbul ignore if */ if (!sch.serialize) throw new Error("ajv implementation error") return sch.serialize } + + private _compileParser(sch: SchemaEnv): (json: string) => unknown { + compileParser.call(this, sch, (sch.schema as AnySchemaObject).definitions || {}) + /* istanbul ignore if */ + if (!sch.parse) throw new Error("ajv implementation error") + return sch.parse + } } diff --git a/lib/runtime/jsonParse.ts b/lib/runtime/jsonParse.ts new file mode 100644 index 000000000..fd0807537 --- /dev/null +++ b/lib/runtime/jsonParse.ts @@ -0,0 +1,15 @@ +const rxJsonParse = /position\s(\d+)$/ + +export default function jsonParse(s: string, pos: number): [unknown, number] { + let endPos: number | undefined + if (pos) s = s.slice(pos) + try { + return [JSON.parse(s), pos + s.length] + } catch (e) { + const matches = rxJsonParse.exec(e.message) + if (!matches) throw e + endPos = +matches[1] + s = s.slice(0, endPos) + return [JSON.parse(s), pos + endPos] + } +} diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index 881df665b..e3dfd4c2b 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -106,11 +106,34 @@ describe("JSON Type Definition", () => { const serialize = ajv.compileSerializer(schema) // console.log(serialize.toString()) assert.deepStrictEqual(JSON.parse(serialize(instance)), instance) - // const opts = ajv instanceof AjvPack ? ajv.ajv.opts : ajv.opts }) ) } }) + + describe.skip("parse", () => { + const ajv = new _AjvJTD() + + for (const testName in jtdValidationTests) { + const {schema, instance, errors} = jtdValidationTests[testName] as TestCase + const valid = errors.length === 0 + describeOnly(testName, () => { + if (valid) { + it(`should parse valid JSON string`, () => { + const parse = ajv.compileParser(schema) + console.log(parse.toString()) + assert.deepStrictEqual(parse(JSON.stringify(instance)), instance) + }) + } else { + it(`should throw exception on invalid JSON string`, () => { + const parse = ajv.compileParser(schema) + console.log(parse.toString()) + assert.throws(() => parse(JSON.stringify(instance))) + }) + } + }) + } + }) }) function describeOnly(name: string, func: () => void) { From ce4a476de17486a3d3bfb96d95f66b47125ee191 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun, 21 Feb 2021 21:24:32 +0000 Subject: [PATCH 09/19] JTD: compile parsers (WIP - 12 tests fail) --- lib/compile/jtd/parse.ts | 210 ++++++++++++++++++++--------------- lib/compile/names.ts | 1 + lib/compile/timestamp.ts | 4 + lib/compile/util.ts | 7 ++ lib/runtime/jsonParse.ts | 15 --- lib/runtime/parseJson.ts | 152 +++++++++++++++++++++++++ lib/vocabularies/jtd/type.ts | 10 +- spec/jtd-schema.spec.ts | 2 +- 8 files changed, 292 insertions(+), 109 deletions(-) delete mode 100644 lib/runtime/jsonParse.ts create mode 100644 lib/runtime/parseJson.ts diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index 161e6f940..e68e4696e 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -3,12 +3,19 @@ import type {SchemaObject} from "../../types" import {jtdForms, JTDForm, SchemaObjectMap} from "./types" import {SchemaEnv, getCompilingSchema} from ".." import {_, str, and, getProperty, CodeGen, Code, Name} from "../codegen" -import {_Code} from "../codegen/code" import {MissingRefError} from "../error_classes" import N from "../names" import {isOwnProperty} from "../../vocabularies/code" import {hasRef} from "../../vocabularies/jtd/ref" -import jsonParse from "../../runtime/jsonParse" +import {intRange, IntType} from "../../vocabularies/jtd/type" +import { + parseJson, + parseJsonNumber, + parseJsonInteger, + parseJsonString, +} from "../../runtime/parseJson" +import {func} from "../util" +import validTimestamp from "../timestamp" type GenParse = (cxt: ParseCxt) => void @@ -18,7 +25,7 @@ const genParse: {[F in JTDForm]: GenParse} = { discriminator: parseDiscriminator, properties: parseProperties, optionalProperties: parseProperties, - enum: parseString, + enum: parseEnum, type: parseType, ref: parseRef, } @@ -30,7 +37,6 @@ interface ParseCxt { readonly definitions: SchemaObjectMap schema: SchemaObject data: Code - jsonPos: Name | number } export default function compileParser( @@ -51,7 +57,6 @@ export default function compileParser( schemaEnv: sch, definitions, data: N.data, - jsonPos: 0, } let sourceCode: string | undefined @@ -60,15 +65,13 @@ export default function compileParser( sch.parseName = parseName gen.func(parseName, N.json, false, () => { gen.let(N.data) - gen.let(N.jsonPos) + gen.let(N.jsonPos, 0) + gen.const(N.jsonLen, _`${N.json}.length`) parseCode(cxt) gen.if( - _`${N.jsonPos} === ${N.json}.length`, + _`${N.jsonPos} === ${N.jsonLen}`, () => gen.return(N.data), - () => throwSyntaxError(gen) - // gen.throw( - // _`new SyntaxError("Unexpected token " + ${N.json}[${N.jsonPos}] + " in JSON at position " + ${N.jsonPos})` - // ) + () => jsonSyntaxError(cxt) ) }) gen.optimize(this.opts.code.optimize) @@ -103,58 +106,43 @@ function parseCode(cxt: ParseCxt): void { function parseNullable(cxt: ParseCxt, parseForm: GenParse): void { const {gen, schema, data} = cxt if (!schema.nullable) return parseForm(cxt) - tryParse(cxt, "null", () => gen.assign(data, null), parseForm) -} - -function tryParse(cxt: ParseCxt, s: string, success: GenParse, fail: GenParse): void { - const {gen, jsonPos} = cxt - const slice = - s.length === 1 ? _`${N.json}[${jsonPos}]` : _`${N.json}.slice(${jsonPos}, ${s.length})` - gen.if( - _`${slice} === ${s}`, - () => { - addJsonPos() - success(cxt) - }, - () => fail(cxt) - ) - - function addJsonPos(): void { - if (jsonPos instanceof Name) { - gen.add(jsonPos, s.length) - } else { - gen.assign(N.jsonPos, jsonPos + s.length) - cxt.jsonPos = N.jsonPos - } - } + tryParseToken(cxt, "null", parseForm, () => gen.assign(data, null)) } function parseElements(cxt: ParseCxt): void { const {gen, schema, data} = cxt - gen.add(N.json, str`[`) - const first = gen.let("first", true) - gen.forOf("el", data, (el) => { - addComma(cxt, first) + parseToken(cxt, "[") + const ix = gen.let("i", 0) + gen.assign(data, _`[]`) + parseItems(cxt, "]", () => { + const el = gen.let("el") parseCode({...cxt, schema: schema.elements, data: el}) + gen.assign(_`${data}[${ix}++]`, el) }) - gen.add(N.json, str`]`) } function parseValues(cxt: ParseCxt): void { const {gen, schema, data} = cxt - gen.add(N.json, str`{`) - const first = gen.let("first", true) - gen.forIn("key", data, (key) => parseKeyValue(cxt, key, schema.values, first)) - gen.add(N.json, str`}`) + parseToken(cxt, "{") + gen.assign(data, _`{}`) + parseItems(cxt, "}", () => parseKeyValue(cxt, schema.values)) } -function parseKeyValue(cxt: ParseCxt, key: Name, schema: SchemaObject, first: Name): void { +function parseItems(cxt: ParseCxt, endToken: string, block: () => void): void { + const {gen} = cxt + gen.for(_`;${N.jsonPos}<${N.jsonLen} && ${jsonSlice(1)}!==${endToken};`, () => { + block() + tryParseToken(cxt, ",", () => gen.break()) + }) + parseToken(cxt, endToken) +} + +function parseKeyValue(cxt: ParseCxt, schema: SchemaObject): void { const {gen, data} = cxt - addComma(cxt, first) + const key = gen.let("key") parseString({...cxt, data: key}) - gen.add(N.json, str`:`) - const value = gen.const("value", _`${data}${getProperty(key)}`) - parseCode({...cxt, schema, data: value}) + parseToken(cxt, ":") + parseCode({...cxt, schema, data: _`${data}[${key}]`}) } function parseDiscriminator(cxt: ParseCxt): void { @@ -198,9 +186,7 @@ function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { } if (schema.additionalProperties) { gen.forIn("key", data, (key) => - gen.if(isAdditional(key, allProps), () => - parseKeyValue(cxt, key, {}, gen.let("first", first)) - ) + gen.if(isAdditional(key, allProps), () => parseKeyValue(cxt, {})) ) } @@ -236,29 +222,73 @@ function parseType(cxt: ParseCxt): void { const {gen, schema, data} = cxt switch (schema.type) { case "boolean": - gen.add(N.json, _`${data} ? "true" : "false"`) + parseBoolean(true, parseBoolean(false, jsonSyntaxError))(cxt) break case "string": parseString(cxt) break - case "timestamp": - gen.if( - _`${data} instanceof Date`, - () => gen.add(N.json, _`${data}.toISOString()`), - () => parseString(cxt) - ) + case "timestamp": { + // TODO parse timestamp? + parseString(cxt) + const vts = func(gen, validTimestamp) + gen.if(_`!${vts}(${data})`, () => gen.throw(_`new SyntaxError("JSON: invalid timestamp")`)) break - default: + } + case "float32": + case "float64": parseNumber(cxt) + break + default: { + parseNumber(cxt, true) + const [min, max] = intRange[schema.type as IntType] + gen.if(_`${data} < ${min} || ${data} > ${max}`, () => + gen.throw(_`new SyntaxError("JSON: integer out of range")`) + ) + } } } -function parseString({gen, data}: ParseCxt): void { - gen.add(N.json, _`${jsonParseFunc(gen)}(${data})`) +function parseString(cxt: ParseCxt): void { + parseToken(cxt, '"') + parseWith(cxt, parseJsonString) } -function parseNumber({gen, data}: ParseCxt): void { - gen.add(N.json, _`"" + ${data}`) +function parseEnum(cxt: ParseCxt): void { + const {gen, data, schema} = cxt + const enumSch = schema.enum + parseToken(cxt, '"') + // TODO loopEnum + gen.if(false) + for (const value of enumSch) { + const valueStr = JSON.stringify(value).slice(1) // remove starting quote + gen.elseIf(_`${jsonSlice(valueStr.length)} === ${valueStr}`) + gen.assign(data, str`${value}`) + gen.add(N.jsonPos, valueStr.length) + } + gen.else() + jsonSyntaxError(cxt) + gen.endIf() +} + +function parseNumber(cxt: ParseCxt, int?: boolean): void { + const {gen} = cxt + gen.if( + _`"-0123456789".indexOf(${jsonSlice(1)}) < 0`, + () => jsonSyntaxError(cxt), + () => parseWith(cxt, int ? parseJsonInteger : parseJsonNumber) + ) +} + +function parseBoolean(bool: boolean, fail: GenParse): GenParse { + return (cxt) => { + const {gen, data} = cxt + tryParseToken( + cxt, + `${bool}`, + () => fail(cxt), + () => gen.assign(data, bool) + ) + } } function parseRef(cxt: ParseCxt): void { @@ -279,36 +309,42 @@ function getParse(gen: CodeGen, sch: SchemaEnv): Code { } function parseEmpty(cxt: ParseCxt): void { - const {gen, data, jsonPos} = cxt - gen.assign(_`[${data}, ${N.jsonPos}]`, _`${jsonParseFunc(gen)}(${N.json}, ${jsonPos})`) - cxt.jsonPos = N.jsonPos + parseWith(cxt, parseJson) } -function addComma({gen}: ParseCxt, first: Name): void { - gen.if( - first, - () => gen.assign(first, false), - () => gen.add(N.json, str`,`) - ) +function parseWith({gen, data}: ParseCxt, parseFunc: {code: Code}): void { + const func = gen.scopeValue("func", { + ref: parseFunc, + code: parseFunc.code, + }) + gen.assign(_`[${data}, ${N.jsonPos}]`, _`${func}(${N.json}, ${N.jsonPos})`) } -function jsonParseFunc(gen: CodeGen): Name { - return gen.scopeValue("func", { - ref: jsonParse, - code: _`require("ajv/dist/runtime/jsonParse").default`, - }) +function parseToken(cxt: ParseCxt, tok: string): void { + tryParseToken(cxt, tok, jsonSyntaxError) } -function syntaxError(json: string, jsonPos: number): void { - throw new SyntaxError(`Unexpected token ${json[jsonPos]} in JSON at position ${jsonPos}`) +function tryParseToken(cxt: ParseCxt, tok: string, fail: GenParse, success?: GenParse): void { + const {gen} = cxt + const n = tok.length + gen.if( + _`${jsonSlice(n)} === ${tok}`, + () => { + gen.add(N.jsonPos, n) + success?.(cxt) + }, + () => fail(cxt) + ) } -const syntaxErrorCode = new _Code(syntaxError.toString()) +function jsonSlice(len: number | Name): Code { + return len === 1 + ? _`${N.json}[${N.jsonPos}]` + : _`${N.json}.slice(${N.jsonPos}, ${N.jsonPos}+${len})` +} -function throwSyntaxError(gen: CodeGen): void { - const throwError = gen.scopeValue("func", { - ref: syntaxError, - code: syntaxErrorCode, - }) - gen.code(_`${throwError}(${N.json}, ${N.jsonPos})`) +function jsonSyntaxError(cxt: ParseCxt): void { + cxt.gen.throw( + _`new SyntaxError("Unexpected token "+${N.json}[${N.jsonPos}]+" in JSON at position "+${N.jsonPos})` + ) } diff --git a/lib/compile/names.ts b/lib/compile/names.ts index 4b91756c1..1db2615a8 100644 --- a/lib/compile/names.ts +++ b/lib/compile/names.ts @@ -20,6 +20,7 @@ const names = { // JTD serialize/parse name for JSON string and position json: new Name("json"), jsonPos: new Name("jsonPos"), + jsonLen: new Name("jsonLen"), } export default names diff --git a/lib/compile/timestamp.ts b/lib/compile/timestamp.ts index 6543ba56e..0a16954a4 100644 --- a/lib/compile/timestamp.ts +++ b/lib/compile/timestamp.ts @@ -1,3 +1,5 @@ +import {_} from "./codegen" + const DATE_TIME = /^(\d\d\d\d)-(\d\d)-(\d\d)(?:t|\s)(\d\d):(\d\d):(\d\d)(?:\.\d+)?(?:z|([+-]\d\d)(?::?(\d\d))?)$/i const DAYS = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -25,3 +27,5 @@ export default function validTimestamp(str: string): boolean { (hr - tzH === 23 && min - tzM === 59 && sec === 60)) ) } + +validTimestamp.code = _`require("ajv/dist/compile/timestamp").default` diff --git a/lib/compile/util.ts b/lib/compile/util.ts index 329842093..e6a40468a 100644 --- a/lib/compile/util.ts +++ b/lib/compile/util.ts @@ -167,3 +167,10 @@ export function evaluatedPropsToName(gen: CodeGen, ps?: EvaluatedProperties): Na export function setEvaluated(gen: CodeGen, props: Name, ps: {[K in string]?: true}): void { Object.keys(ps).forEach((p) => gen.assign(_`${props}${getProperty(p)}`, true)) } + +export function func(gen: CodeGen, f: {code: Code}): Name { + return gen.scopeValue("func", { + ref: f, + code: f.code, + }) +} diff --git a/lib/runtime/jsonParse.ts b/lib/runtime/jsonParse.ts deleted file mode 100644 index fd0807537..000000000 --- a/lib/runtime/jsonParse.ts +++ /dev/null @@ -1,15 +0,0 @@ -const rxJsonParse = /position\s(\d+)$/ - -export default function jsonParse(s: string, pos: number): [unknown, number] { - let endPos: number | undefined - if (pos) s = s.slice(pos) - try { - return [JSON.parse(s), pos + s.length] - } catch (e) { - const matches = rxJsonParse.exec(e.message) - if (!matches) throw e - endPos = +matches[1] - s = s.slice(0, endPos) - return [JSON.parse(s), pos + endPos] - } -} diff --git a/lib/runtime/parseJson.ts b/lib/runtime/parseJson.ts new file mode 100644 index 000000000..0acab88bc --- /dev/null +++ b/lib/runtime/parseJson.ts @@ -0,0 +1,152 @@ +import {_} from "../compile/codegen" + +const rxParseJson = /position\s(\d+)$/ + +export function parseJson(s: string, pos: number): [unknown, number] { + let endPos: number | undefined + if (pos) s = s.slice(pos) + try { + return [JSON.parse(s), pos + s.length] + } catch (e) { + const matches = rxParseJson.exec(e.message) + if (!matches) throw e + endPos = +matches[1] + s = s.slice(0, endPos) + return [JSON.parse(s), pos + endPos] + } +} + +parseJson.code = _`require("ajv/dist/runtime/parseJson").parseJson` + +export function parseJsonNumber(s: string, pos: number): [number, number] { + let numStr + [numStr, pos] = parseIntStr(s, pos) + let c: string + let digits: string + if (s[pos] === ".") { + numStr += "." + pos++ + [digits, pos] = parseDigits(s, pos) + numStr += digits + } + if (((c = s[pos]), c === "e" || c === "E")) { + numStr += "e" + pos++ + if (((c = s[pos]), c === "+" || c === "-")) { + numStr += c + pos++ + } + [digits, pos] = parseDigits(s, pos) + numStr += digits + } + return [+numStr, pos] +} + +parseJsonNumber.code = _`require("ajv/dist/runtime/parseJson").parseJsonNumber` + +export function parseJsonInteger(s: string, pos: number): [number, number] { + const res: [string | number, number] = parseIntStr(s, pos) + res[0] = +res[0] + return res as [number, number] +} + +function parseIntStr(s: string, pos: number): [string, number] { + let numStr = "" + if (s[pos] === "-") { + numStr += "-" + pos++ + } + if (s[pos] === "0") { + numStr += "0" + pos++ + } else { + let digits: string + [digits, pos] = parseDigits(s, pos) + numStr += digits + } + return [numStr, pos] +} + +function parseDigits(s: string, pos: number): [string, number] { + let numStr = "" + let c: string + let digit: boolean | undefined + while (((c = s[pos]), c >= "0" && c <= "9")) { + digit = true + numStr += c + pos++ + } + if (!digit) { + if (pos < s.length) unexpectedToken(s[pos], pos) + else unexpectedEnd() + } + return [numStr, pos] +} + +parseJsonInteger.code = _`require("ajv/dist/runtime/parseJson").parseJsonInteger` + +const escapedChars: {[X in string]?: string} = { + b: "\b", + f: "\f", + n: "\n", + r: "\r", + t: "\t", + '"': '"', + "/": "/", + "\\": "\\", +} + +const A_CODE: number = "a".charCodeAt(0) + +export function parseJsonString(s: string, pos: number): [string, number] { + let str = "" + let c: string | undefined + // eslint-disable-next-line no-constant-condition, @typescript-eslint/no-unnecessary-condition + while (true) { + c = s[pos] + pos++ + if (c === '"') break + if (c === "\\") { + c = s[pos] + if (c in escapedChars) str += escapedChars[c] + else if (c === "u") getCharCode() + else unexpectedToken(c, pos) + pos++ + } else if (c === undefined) { + throw unexpectedEnd() + } else { + str += c + } + } + return [str, pos] + + function getCharCode(): void { + let count = 4 + let code = 0 + while (count--) { + code <<= 4 + c = s[pos].toLowerCase() + if (c >= "a" && c <= "f") { + c += c.charCodeAt(0) - A_CODE + 10 + } else if (c >= "0" && c <= "9") { + code += +c + } else if (c === undefined) { + unexpectedEnd() + } else { + unexpectedToken(c, pos) + } + pos++ + } + str += String.fromCharCode(code) + } +} + +parseJsonString.code = _`require("ajv/dist/runtime/parseJson").parseJsonString` + +function unexpectedEnd(): never { + throw new SyntaxError("Unexpected end of JSON input") +} + +function unexpectedToken(c: string, pos: number): never { + throw new SyntaxError(`Unexpected token ${c} in JSON at position ${pos}`) +} diff --git a/lib/vocabularies/jtd/type.ts b/lib/vocabularies/jtd/type.ts index 86cecffd0..80a4846ce 100644 --- a/lib/vocabularies/jtd/type.ts +++ b/lib/vocabularies/jtd/type.ts @@ -2,11 +2,12 @@ import type {CodeKeywordDefinition} from "../../types" import type KeywordCxt from "../../compile/context" import {_, or, Code} from "../../compile/codegen" import validTimestamp from "../../compile/timestamp" +import {func} from "../../compile/util" import {checkMetadata} from "./metadata" -type IntType = "int8" | "uint8" | "int16" | "uint16" | "int32" | "uint32" +export type IntType = "int8" | "uint8" | "int16" | "uint16" | "int32" | "uint32" -const intRange: {[T in IntType]: [number, number]} = { +export const intRange: {[T in IntType]: [number, number]} = { int8: [-128, 127], uint8: [0, 255], int16: [-32768, 32767], @@ -28,10 +29,7 @@ const def: CodeKeywordDefinition = { cond = _`typeof ${data} == ${schema}` break case "timestamp": { - const vts = gen.scopeValue("func", { - ref: validTimestamp, - code: _`require("ajv/dist/compile/timestamp").default`, - }) + const vts = func(gen, validTimestamp) cond = _`${data} instanceof Date || (typeof ${data} == "string" && ${vts}(${data}))` break } diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index e3dfd4c2b..881f00f34 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -121,7 +121,7 @@ describe("JSON Type Definition", () => { if (valid) { it(`should parse valid JSON string`, () => { const parse = ajv.compileParser(schema) - console.log(parse.toString()) + console.log(schema, instance, `"${JSON.stringify(instance)}"`, parse.toString()) assert.deepStrictEqual(parse(JSON.stringify(instance)), instance) }) } else { From fa36fe78b202cdc202392bb3d69812f64e825538 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun, 21 Feb 2021 22:28:12 +0000 Subject: [PATCH 10/19] add JTD parsing benchmarks --- benchmark/{serialize.js => jtd.js} | 28 ++++++++++++++++++++++++++++ lib/runtime/parseJson.ts | 9 +++++---- package.json | 2 +- 3 files changed, 34 insertions(+), 5 deletions(-) rename benchmark/{serialize.js => jtd.js} (68%) diff --git a/benchmark/serialize.js b/benchmark/jtd.js similarity index 68% rename from benchmark/serialize.js rename to benchmark/jtd.js index 692da1888..70c11bb6d 100644 --- a/benchmark/serialize.js +++ b/benchmark/jtd.js @@ -1,3 +1,4 @@ +/* eslint-disable no-empty */ /* eslint-disable no-console */ const Ajv = require("ajv/dist/jtd").default const Benchmark = require("benchmark") @@ -62,6 +63,33 @@ const serializer = ajv.compileSerializer(testSchema) suite.add("test data: compiled JTD serializer", () => serializer(testData)) suite.add("test data: JSON.stringify", () => JSON.stringify(testData)) +const nestedElementsSchema = { + elements: { + elements: { + type: "int32", + }, + }, +} + +const validNestedElements = JSON.stringify([[], [1, 2], [3, 4, 5]]) +const invalidNestedElements = JSON.stringify([[], [1, 2], {}, [3, 4, 5]]) + +const parse = ajv.compileParser(nestedElementsSchema) + +suite.add("valid test data: compiled JTD parser", () => parse(validNestedElements)) +suite.add("valid test data: JSON.parse", () => JSON.parse(validNestedElements)) +suite.add("invalid test data: compiled JTD parser", () => { + try { + parse(invalidNestedElements) + } catch (e) {} +}) +suite.add("invalid test data: JSON.parse", () => { + try { + JSON.parse(invalidNestedElements) + throw new Error() + } catch(e) {} +}) + console.log() suite diff --git a/lib/runtime/parseJson.ts b/lib/runtime/parseJson.ts index 0acab88bc..f094fdeb3 100644 --- a/lib/runtime/parseJson.ts +++ b/lib/runtime/parseJson.ts @@ -18,15 +18,16 @@ export function parseJson(s: string, pos: number): [unknown, number] { parseJson.code = _`require("ajv/dist/runtime/parseJson").parseJson` +// TODO combile parsing integers and numbers with extra parameters for digits size export function parseJsonNumber(s: string, pos: number): [number, number] { let numStr - [numStr, pos] = parseIntStr(s, pos) + ;[numStr, pos] = parseIntStr(s, pos) let c: string let digits: string if (s[pos] === ".") { numStr += "." pos++ - [digits, pos] = parseDigits(s, pos) + ;[digits, pos] = parseDigits(s, pos) numStr += digits } if (((c = s[pos]), c === "e" || c === "E")) { @@ -36,7 +37,7 @@ export function parseJsonNumber(s: string, pos: number): [number, number] { numStr += c pos++ } - [digits, pos] = parseDigits(s, pos) + ;[digits, pos] = parseDigits(s, pos) numStr += digits } return [+numStr, pos] @@ -61,7 +62,7 @@ function parseIntStr(s: string, pos: number): [string, number] { pos++ } else { let digits: string - [digits, pos] = parseDigits(s, pos) + ;[digits, pos] = parseDigits(s, pos) numStr += digits } return [numStr, pos] diff --git a/package.json b/package.json index 668b1bffd..342c4f5ca 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "test": "npm link && npm link ajv && npm run json-tests && npm run eslint && npm run test-cov", "test-ci": "AJV_FULL_TEST=true npm test", "prepublish": "npm run build", - "benchmark": "npm i && npm run build && npm link && cd ./benchmark && npm link ajv && npm i && node ./serialize" + "benchmark": "npm i && npm run build && npm link && cd ./benchmark && npm link ajv && npm i && node ./jtd" }, "nyc": { "exclude": [ From e5f21389ba32f87fe76ae97642a9509d65816494 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun, 21 Feb 2021 22:58:45 +0000 Subject: [PATCH 11/19] refactor JTD number parsing --- benchmark/jtd.js | 2 +- lib/compile/jtd/parse.ts | 26 ++++++------ lib/runtime/parseJson.ts | 76 +++++++++++++----------------------- lib/vocabularies/jtd/type.ts | 14 +++---- spec/jtd-schema.spec.ts | 26 ++++++------ 5 files changed, 60 insertions(+), 84 deletions(-) diff --git a/benchmark/jtd.js b/benchmark/jtd.js index 70c11bb6d..9a73a4968 100644 --- a/benchmark/jtd.js +++ b/benchmark/jtd.js @@ -87,7 +87,7 @@ suite.add("invalid test data: JSON.parse", () => { try { JSON.parse(invalidNestedElements) throw new Error() - } catch(e) {} + } catch (e) {} }) console.log() diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index e68e4696e..fc44d9e8a 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -2,18 +2,13 @@ import type Ajv from "../../core" import type {SchemaObject} from "../../types" import {jtdForms, JTDForm, SchemaObjectMap} from "./types" import {SchemaEnv, getCompilingSchema} from ".." -import {_, str, and, getProperty, CodeGen, Code, Name} from "../codegen" +import {_, str, and, nil, getProperty, CodeGen, Code, Name, SafeExpr} from "../codegen" import {MissingRefError} from "../error_classes" import N from "../names" import {isOwnProperty} from "../../vocabularies/code" import {hasRef} from "../../vocabularies/jtd/ref" import {intRange, IntType} from "../../vocabularies/jtd/type" -import { - parseJson, - parseJsonNumber, - parseJsonInteger, - parseJsonString, -} from "../../runtime/parseJson" +import {parseJson, parseJsonNumber, parseJsonString} from "../../runtime/parseJson" import {func} from "../util" import validTimestamp from "../timestamp" @@ -239,8 +234,8 @@ function parseType(cxt: ParseCxt): void { parseNumber(cxt) break default: { - parseNumber(cxt, true) - const [min, max] = intRange[schema.type as IntType] + const [min, max, maxDigits] = intRange[schema.type as IntType] + parseNumber(cxt, maxDigits) gen.if(_`${data} < ${min} || ${data} > ${max}`, () => gen.throw(_`new SyntaxError("JSON: integer out of range")`) ) @@ -270,12 +265,12 @@ function parseEnum(cxt: ParseCxt): void { gen.endIf() } -function parseNumber(cxt: ParseCxt, int?: boolean): void { +function parseNumber(cxt: ParseCxt, maxDigits?: number): void { const {gen} = cxt gen.if( _`"-0123456789".indexOf(${jsonSlice(1)}) < 0`, () => jsonSyntaxError(cxt), - () => parseWith(cxt, int ? parseJsonInteger : parseJsonNumber) + () => parseWith(cxt, parseJsonNumber, maxDigits) ) } @@ -312,12 +307,15 @@ function parseEmpty(cxt: ParseCxt): void { parseWith(cxt, parseJson) } -function parseWith({gen, data}: ParseCxt, parseFunc: {code: Code}): void { - const func = gen.scopeValue("func", { +function parseWith({gen, data}: ParseCxt, parseFunc: {code: Code}, args?: SafeExpr): void { + const f = gen.scopeValue("func", { ref: parseFunc, code: parseFunc.code, }) - gen.assign(_`[${data}, ${N.jsonPos}]`, _`${func}(${N.json}, ${N.jsonPos})`) + gen.assign( + _`[${data}, ${N.jsonPos}]`, + _`${f}(${N.json}, ${N.jsonPos}${args ? _`, ${args}` : nil})` + ) } function parseToken(cxt: ParseCxt, tok: string): void { diff --git a/lib/runtime/parseJson.ts b/lib/runtime/parseJson.ts index f094fdeb3..0623a322a 100644 --- a/lib/runtime/parseJson.ts +++ b/lib/runtime/parseJson.ts @@ -18,17 +18,24 @@ export function parseJson(s: string, pos: number): [unknown, number] { parseJson.code = _`require("ajv/dist/runtime/parseJson").parseJson` -// TODO combile parsing integers and numbers with extra parameters for digits size -export function parseJsonNumber(s: string, pos: number): [number, number] { - let numStr - ;[numStr, pos] = parseIntStr(s, pos) +export function parseJsonNumber(s: string, pos: number, maxDigits?: number): [number, number] { + let numStr = "" let c: string - let digits: string + if (s[pos] === "-") { + numStr += "-" + pos++ + } + if (s[pos] === "0") { + numStr += "0" + pos++ + } else { + parseDigits(maxDigits) + } + if (maxDigits) return [+numStr, pos] if (s[pos] === ".") { numStr += "." pos++ - ;[digits, pos] = parseDigits(s, pos) - numStr += digits + parseDigits() } if (((c = s[pos]), c === "e" || c === "E")) { numStr += "e" @@ -37,54 +44,25 @@ export function parseJsonNumber(s: string, pos: number): [number, number] { numStr += c pos++ } - ;[digits, pos] = parseDigits(s, pos) - numStr += digits + parseDigits() } return [+numStr, pos] -} - -parseJsonNumber.code = _`require("ajv/dist/runtime/parseJson").parseJsonNumber` - -export function parseJsonInteger(s: string, pos: number): [number, number] { - const res: [string | number, number] = parseIntStr(s, pos) - res[0] = +res[0] - return res as [number, number] -} - -function parseIntStr(s: string, pos: number): [string, number] { - let numStr = "" - if (s[pos] === "-") { - numStr += "-" - pos++ - } - if (s[pos] === "0") { - numStr += "0" - pos++ - } else { - let digits: string - ;[digits, pos] = parseDigits(s, pos) - numStr += digits - } - return [numStr, pos] -} -function parseDigits(s: string, pos: number): [string, number] { - let numStr = "" - let c: string - let digit: boolean | undefined - while (((c = s[pos]), c >= "0" && c <= "9")) { - digit = true - numStr += c - pos++ - } - if (!digit) { - if (pos < s.length) unexpectedToken(s[pos], pos) - else unexpectedEnd() + function parseDigits(maxLen?: number): void { + let digit: boolean | undefined + while (((c = s[pos]), c >= "0" && c <= "9" && (maxLen === undefined || maxLen-- > 0))) { + digit = true + numStr += c + pos++ + } + if (!digit) { + if (pos < s.length) unexpectedToken(s[pos], pos) + else unexpectedEnd() + } } - return [numStr, pos] } -parseJsonInteger.code = _`require("ajv/dist/runtime/parseJson").parseJsonInteger` +parseJsonNumber.code = _`require("ajv/dist/runtime/parseJson").parseJsonNumber` const escapedChars: {[X in string]?: string} = { b: "\b", diff --git a/lib/vocabularies/jtd/type.ts b/lib/vocabularies/jtd/type.ts index 80a4846ce..3e7429c86 100644 --- a/lib/vocabularies/jtd/type.ts +++ b/lib/vocabularies/jtd/type.ts @@ -7,13 +7,13 @@ import {checkMetadata} from "./metadata" export type IntType = "int8" | "uint8" | "int16" | "uint16" | "int32" | "uint32" -export const intRange: {[T in IntType]: [number, number]} = { - int8: [-128, 127], - uint8: [0, 255], - int16: [-32768, 32767], - uint16: [0, 65535], - int32: [-2147483648, 2147483647], - uint32: [0, 4294967295], +export const intRange: {[T in IntType]: [number, number, number]} = { + int8: [-128, 127, 3], + uint8: [0, 255, 3], + int16: [-32768, 32767, 5], + uint16: [0, 65535, 5], + int32: [-2147483648, 2147483647, 10], + uint32: [0, 4294967295, 10], } const def: CodeKeywordDefinition = { diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index 881f00f34..f223367bd 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -24,19 +24,19 @@ interface TestCaseError { // schemaPath: string // } -// const ONLY: RegExp[] = [ -// "empty", +const ONLY: RegExp[] = [ + "empty", // "ref", -// "type", -// "enum", -// "elements", + "type", + "enum", + "elements", // "properties", // "optionalProperties", // "discriminator", -// "values", -// ].map((s) => new RegExp(`(^|.*\\s)${s}\\s.*-`)) + "values", +].map((s) => new RegExp(`(^|.*\\s)${s}\\s.*-`)) -const ONLY: RegExp[] = [] +// const ONLY: RegExp[] = [] describe("JSON Type Definition", () => { describe("validation", function () { @@ -55,7 +55,7 @@ describe("JSON Type Definition", () => { for (const testName in jtdValidationTests) { const {schema, instance, errors} = jtdValidationTests[testName] as TestCase const valid = errors.length === 0 - describeOnly(testName, () => + describe(testName, () => it(`should be ${valid ? "valid" : "invalid"}`, () => withStandalone(ajvs).forEach((ajv) => { // console.log(ajv.compile(schema).toString()) @@ -101,7 +101,7 @@ describe("JSON Type Definition", () => { const {schema, instance, errors} = jtdValidationTests[testName] as TestCase const valid = errors.length === 0 if (!valid) continue - describeOnly(testName, () => + describe(testName, () => it(`should serialize data`, () => { const serialize = ajv.compileSerializer(schema) // console.log(serialize.toString()) @@ -111,7 +111,7 @@ describe("JSON Type Definition", () => { } }) - describe.skip("parse", () => { + describe("parse", () => { const ajv = new _AjvJTD() for (const testName in jtdValidationTests) { @@ -121,13 +121,13 @@ describe("JSON Type Definition", () => { if (valid) { it(`should parse valid JSON string`, () => { const parse = ajv.compileParser(schema) - console.log(schema, instance, `"${JSON.stringify(instance)}"`, parse.toString()) + // console.log(schema, instance, `"${JSON.stringify(instance)}"`, parse.toString()) assert.deepStrictEqual(parse(JSON.stringify(instance)), instance) }) } else { it(`should throw exception on invalid JSON string`, () => { const parse = ajv.compileParser(schema) - console.log(parse.toString()) + // console.log(parse.toString()) assert.throws(() => parse(JSON.stringify(instance))) }) } From 6eab2a840629e03122a8b2567b2bc3d97f68f3c7 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Mon, 22 Feb 2021 08:51:15 +0000 Subject: [PATCH 12/19] JTD: parsing properties (TODO check for missing properties) --- lib/compile/jtd/parse.ts | 47 +++++++++++++++++++++++++++++++++++----- spec/jtd-schema.spec.ts | 8 +++---- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index fc44d9e8a..4d5dfed34 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -133,16 +133,18 @@ function parseItems(cxt: ParseCxt, endToken: string, block: () => void): void { } function parseKeyValue(cxt: ParseCxt, schema: SchemaObject): void { - const {gen, data} = cxt + const {gen} = cxt const key = gen.let("key") parseString({...cxt, data: key}) + checkDuplicateProperty(cxt, key) parseToken(cxt, ":") - parseCode({...cxt, schema, data: _`${data}[${key}]`}) + parsePropertyValue(cxt, key, schema) } function parseDiscriminator(cxt: ParseCxt): void { const {gen, schema, data} = cxt const {discriminator} = schema + gen.add(N.json, str`{${JSON.stringify(discriminator)}:`) const tag = gen.const("tag", _`${data}${getProperty(discriminator)}`) parseString({...cxt, data: tag}) @@ -157,10 +159,45 @@ function parseDiscriminator(cxt: ParseCxt): void { } function parseProperties(cxt: ParseCxt): void { + // TODO check missing properties + const {gen, schema, data} = cxt + const {properties, optionalProperties, additionalProperties} = schema + parseToken(cxt, "{") + gen.assign(data, _`{}`) + parseItems(cxt, "}", () => { + const key = gen.let("key") + parseString({...cxt, data: key}) + checkDuplicateProperty(cxt, key) + parseToken(cxt, ":") + gen.if(false) + parseDefinedProperty(cxt, key, properties) + parseDefinedProperty(cxt, key, optionalProperties) + gen.else() + if (additionalProperties) { + parseEmpty({...cxt, data: _`${data}[${key}]`}) + } else { + gen.throw(_`new Error("JSON: property "+${key}+" not allowed")`) + } + gen.endIf() + }) +} + +function parseDefinedProperty(cxt: ParseCxt, key: Name, schemas: SchemaObjectMap = {}): void { const {gen} = cxt - gen.add(N.json, str`{`) - parseSchemaProperties(cxt) - gen.add(N.json, str`}`) + for (const prop in schemas) { + gen.elseIf(_`${key} === ${prop}`) + parsePropertyValue(cxt, key, schemas[prop] as SchemaObject) + } +} + +function checkDuplicateProperty({gen, data}: ParseCxt, key: Name): void { + gen.if(isOwnProperty(gen, data, key), () => + gen.throw(_`new Error("JSON: duplicate property " + ${key})`) + ) +} + +function parsePropertyValue(cxt: ParseCxt, key: Name, schema: SchemaObject): void { + parseCode({...cxt, schema, data: _`${cxt.data}[${key}]`}) } function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index f223367bd..e8b046f91 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -26,13 +26,13 @@ interface TestCaseError { const ONLY: RegExp[] = [ "empty", -// "ref", + // "ref", "type", "enum", "elements", -// "properties", -// "optionalProperties", -// "discriminator", + // "properties", + // "optionalProperties", + // "discriminator", "values", ].map((s) => new RegExp(`(^|.*\\s)${s}\\s.*-`)) From 934a18f4b9e8d6b44757ddded6ac2f43b6f1150e Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Mon, 22 Feb 2021 21:35:24 +0000 Subject: [PATCH 13/19] JTD parsing: discriminator schema --- lib/compile/jtd/parse.ts | 127 ++++++++++++++++++--------------------- lib/vocabularies/code.ts | 9 ++- spec/jtd-schema.spec.ts | 6 +- 3 files changed, 68 insertions(+), 74 deletions(-) diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index 4d5dfed34..ca9bfbddd 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -2,10 +2,10 @@ import type Ajv from "../../core" import type {SchemaObject} from "../../types" import {jtdForms, JTDForm, SchemaObjectMap} from "./types" import {SchemaEnv, getCompilingSchema} from ".." -import {_, str, and, nil, getProperty, CodeGen, Code, Name, SafeExpr} from "../codegen" +import {_, str, and, nil, not, CodeGen, Code, Name, SafeExpr} from "../codegen" import {MissingRefError} from "../error_classes" import N from "../names" -import {isOwnProperty} from "../../vocabularies/code" +import {isOwnProperty, hasPropFunc} from "../../vocabularies/code" import {hasRef} from "../../vocabularies/jtd/ref" import {intRange, IntType} from "../../vocabularies/jtd/type" import {parseJson, parseJsonNumber, parseJsonString} from "../../runtime/parseJson" @@ -124,12 +124,16 @@ function parseValues(cxt: ParseCxt): void { } function parseItems(cxt: ParseCxt, endToken: string, block: () => void): void { + tryParseItems(cxt, endToken, block) + parseToken(cxt, endToken) +} + +function tryParseItems(cxt: ParseCxt, endToken: string, block: () => void): void { const {gen} = cxt gen.for(_`;${N.jsonPos}<${N.jsonLen} && ${jsonSlice(1)}!==${endToken};`, () => { block() tryParseToken(cxt, ",", () => gen.break()) }) - parseToken(cxt, endToken) } function parseKeyValue(cxt: ParseCxt, schema: SchemaObject): void { @@ -142,36 +146,66 @@ function parseKeyValue(cxt: ParseCxt, schema: SchemaObject): void { } function parseDiscriminator(cxt: ParseCxt): void { - const {gen, schema, data} = cxt - const {discriminator} = schema - - gen.add(N.json, str`{${JSON.stringify(discriminator)}:`) - const tag = gen.const("tag", _`${data}${getProperty(discriminator)}`) - parseString({...cxt, data: tag}) - gen.if(false) - for (const tagValue in schema.mapping) { + const {gen, data, schema} = cxt + const {discriminator, mapping} = schema + parseToken(cxt, "{") + gen.assign(data, _`{}`) + const startPos = gen.const("pos", N.jsonPos) + const value = gen.let("value") + const tag = gen.let("tag") + tryParseItems(cxt, "}", () => { + const key = gen.let("key") + parseString({...cxt, data: key}) + parseToken(cxt, ":") + gen.if( + _`${key} === ${discriminator}`, + () => { + parseString({...cxt, data: tag}) + gen.assign(_`${data}[${key}]`, tag) + gen.break() + }, + () => parseEmpty({...cxt, data: value}) // can be discarded/skipped + ) + }) + gen.assign(N.jsonPos, startPos) + gen.if(_`${tag} === undefined`) + gen.throw(_`new Error("JSON: discriminator tag not found")`) + for (const tagValue in mapping) { gen.elseIf(_`${tag} === ${tagValue}`) - const sch = schema.mapping[tagValue] - parseSchemaProperties({...cxt, schema: sch}, discriminator) + parseSchemaProperties({...cxt, schema: mapping[tagValue]}, discriminator) } + gen.else() + gen.throw(_`new Error("JSON: discriminator value not in schema")`) gen.endIf() - gen.add(N.json, str`}`) } function parseProperties(cxt: ParseCxt): void { - // TODO check missing properties - const {gen, schema, data} = cxt - const {properties, optionalProperties, additionalProperties} = schema + const {gen, data} = cxt parseToken(cxt, "{") gen.assign(data, _`{}`) + parseSchemaProperties(cxt) +} + +function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { + const {gen, schema, data} = cxt + const {properties, optionalProperties, additionalProperties} = schema parseItems(cxt, "}", () => { const key = gen.let("key") parseString({...cxt, data: key}) - checkDuplicateProperty(cxt, key) + if (discriminator) { + gen.if(_`${key} !== ${discriminator}`, () => checkDuplicateProperty(cxt, key)) + } else { + checkDuplicateProperty(cxt, key) + } parseToken(cxt, ":") gen.if(false) parseDefinedProperty(cxt, key, properties) parseDefinedProperty(cxt, key, optionalProperties) + if (discriminator) { + gen.elseIf(_`${key} === ${discriminator}`) + const tag = gen.let("tag") + parseString({...cxt, data: tag}) // can be discarded, it is already assigned + } gen.else() if (additionalProperties) { parseEmpty({...cxt, data: _`${data}[${key}]`}) @@ -180,6 +214,13 @@ function parseProperties(cxt: ParseCxt): void { } gen.endIf() }) + if (properties) { + const hasProp = hasPropFunc(gen) + const allProps: Code = and( + ...Object.keys(properties).map((p): Code => _`${hasProp}.call(${data}, ${p})`) + ) + gen.if(not(allProps), () => gen.throw(_`new Error("JSON: missing required properties")`)) + } } function parseDefinedProperty(cxt: ParseCxt, key: Name, schemas: SchemaObjectMap = {}): void { @@ -200,56 +241,6 @@ function parsePropertyValue(cxt: ParseCxt, key: Name, schema: SchemaObject): voi parseCode({...cxt, schema, data: _`${cxt.data}[${key}]`}) } -function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { - const {gen, schema, data} = cxt - const {properties, optionalProperties} = schema - const props = keys(properties) - const optProps = keys(optionalProperties) - const allProps = allProperties(props.concat(optProps)) - let first = !discriminator - for (const key of props) { - parseProperty(key, properties[key], keyValue(key)) - } - for (const key of optProps) { - const value = keyValue(key) - gen.if(and(_`${value} !== undefined`, isOwnProperty(gen, data, key)), () => - parseProperty(key, optionalProperties[key], value) - ) - } - if (schema.additionalProperties) { - gen.forIn("key", data, (key) => - gen.if(isAdditional(key, allProps), () => parseKeyValue(cxt, {})) - ) - } - - function keys(ps?: SchemaObjectMap): string[] { - return ps ? Object.keys(ps) : [] - } - - function allProperties(ps: string[]): string[] { - if (discriminator) ps.push(discriminator) - if (new Set(ps).size !== ps.length) { - throw new Error("JTD: properties/optionalProperties/disciminator overlap") - } - return ps - } - - function keyValue(key: string): Name { - return gen.const("value", _`${data}${getProperty(key)}`) - } - - function parseProperty(key: string, propSchema: SchemaObject, value: Name): void { - if (first) first = false - else gen.add(N.json, str`,`) - gen.add(N.json, str`${JSON.stringify(key)}:`) - parseCode({...cxt, schema: propSchema, data: value}) - } - - function isAdditional(key: Name, ps: string[]): Code | true { - return ps.length ? and(...ps.map((p) => _`${key} !== ${p}`)) : true - } -} - function parseType(cxt: ParseCxt): void { const {gen, schema, data} = cxt switch (schema.type) { diff --git a/lib/vocabularies/code.ts b/lib/vocabularies/code.ts index 618960a6d..f3be79b43 100644 --- a/lib/vocabularies/code.ts +++ b/lib/vocabularies/code.ts @@ -32,13 +32,16 @@ export function reportMissingProp(cxt: KeywordCxt, missing: Name): void { cxt.error() } -export function isOwnProperty(gen: CodeGen, data: Name, property: Name | string): Code { - const hasProp = gen.scopeValue("func", { +export function hasPropFunc(gen: CodeGen): Name { + return gen.scopeValue("func", { // eslint-disable-next-line @typescript-eslint/unbound-method ref: Object.prototype.hasOwnProperty, code: _`Object.prototype.hasOwnProperty`, }) - return _`${hasProp}.call(${data}, ${property})` +} + +export function isOwnProperty(gen: CodeGen, data: Name, property: Name | string): Code { + return _`${hasPropFunc(gen)}.call(${data}, ${property})` } export function propertyInData( diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index e8b046f91..2f930a176 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -30,9 +30,9 @@ const ONLY: RegExp[] = [ "type", "enum", "elements", - // "properties", - // "optionalProperties", - // "discriminator", + "properties", + "optionalProperties", + "discriminator", "values", ].map((s) => new RegExp(`(^|.*\\s)${s}\\s.*-`)) From 749401a0ef56acdaaaebbee3e482d95d0e859a14 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Tue, 23 Feb 2021 20:50:25 +0000 Subject: [PATCH 14/19] JTD parse: parse nested and recursive refs --- lib/compile/jtd/parse.ts | 41 +++++++++++++++++++++++----------------- lib/compile/names.ts | 1 + spec/jtd-schema.spec.ts | 26 ++++++++++++------------- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index ca9bfbddd..d42c8ea1d 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -58,17 +58,7 @@ export default function compileParser( try { this._compilations.add(sch) sch.parseName = parseName - gen.func(parseName, N.json, false, () => { - gen.let(N.data) - gen.let(N.jsonPos, 0) - gen.const(N.jsonLen, _`${N.json}.length`) - parseCode(cxt) - gen.if( - _`${N.jsonPos} === ${N.jsonLen}`, - () => gen.return(N.data), - () => jsonSyntaxError(cxt) - ) - }) + parserFunction(cxt, parseName) gen.optimize(this.opts.code.optimize) const parseFuncCode = gen.toString() sourceCode = `${gen.scopeRefs(N.scope)}return ${parseFuncCode}` @@ -87,6 +77,19 @@ export default function compileParser( return sch } +function parserFunction(cxt: ParseCxt, parseName: Name): void { + const {gen} = cxt + gen.func(parseName, _`${N.json}, ${N.jsonPos}, ${N.jsonPart}`, false, () => { + gen.let(N.data) + gen.assign(N.jsonPos, _`${N.jsonPos} || 0`) + gen.const(N.jsonLen, _`${N.json}.length`) + parseCode(cxt) + gen.if(N.jsonPart, () => gen.return(_`[${N.data}, ${N.jsonPos}]`)) + gen.if(_`${N.jsonPos} === ${N.jsonLen}`, () => gen.return(N.data)) + jsonSyntaxError(cxt) + }) +} + function parseCode(cxt: ParseCxt): void { let form: JTDForm | undefined for (const key of jtdForms) { @@ -315,17 +318,17 @@ function parseBoolean(bool: boolean, fail: GenParse): GenParse { } function parseRef(cxt: ParseCxt): void { - const {gen, self, data, definitions, schema, schemaEnv} = cxt + const {gen, self, definitions, schema, schemaEnv} = cxt const {ref} = schema const refSchema = definitions[ref] if (!refSchema) throw new MissingRefError("", ref, `No definition ${ref}`) if (!hasRef(refSchema)) return parseCode({...cxt, schema: refSchema}) const {root} = schemaEnv const sch = compileParser.call(self, new SchemaEnv({schema: refSchema, root}), definitions) - gen.add(N.json, _`${getParse(gen, sch)}(${data})`) + partialParse(cxt, getParser(gen, sch), true) } -function getParse(gen: CodeGen, sch: SchemaEnv): Code { +function getParser(gen: CodeGen, sch: SchemaEnv): Code { return sch.parse ? gen.scopeValue("parse", {ref: sch.parse}) : _`${gen.scopeValue("wrapper", {ref: sch})}.parse` @@ -335,14 +338,18 @@ function parseEmpty(cxt: ParseCxt): void { parseWith(cxt, parseJson) } -function parseWith({gen, data}: ParseCxt, parseFunc: {code: Code}, args?: SafeExpr): void { - const f = gen.scopeValue("func", { +function parseWith(cxt: ParseCxt, parseFunc: {code: Code}, args?: SafeExpr): void { + const f = cxt.gen.scopeValue("func", { ref: parseFunc, code: parseFunc.code, }) + partialParse(cxt, f, args) +} + +function partialParse({gen, data}: ParseCxt, parseFunc: Name, args?: SafeExpr): void { gen.assign( _`[${data}, ${N.jsonPos}]`, - _`${f}(${N.json}, ${N.jsonPos}${args ? _`, ${args}` : nil})` + _`${parseFunc}(${N.json}, ${N.jsonPos}${args ? _`, ${args}` : nil})` ) } diff --git a/lib/compile/names.ts b/lib/compile/names.ts index 1db2615a8..b7f18ca45 100644 --- a/lib/compile/names.ts +++ b/lib/compile/names.ts @@ -21,6 +21,7 @@ const names = { json: new Name("json"), jsonPos: new Name("jsonPos"), jsonLen: new Name("jsonLen"), + jsonPart: new Name("jsonPart"), } export default names diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index 2f930a176..208c760aa 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -24,19 +24,19 @@ interface TestCaseError { // schemaPath: string // } -const ONLY: RegExp[] = [ - "empty", - // "ref", - "type", - "enum", - "elements", - "properties", - "optionalProperties", - "discriminator", - "values", -].map((s) => new RegExp(`(^|.*\\s)${s}\\s.*-`)) - -// const ONLY: RegExp[] = [] +// const ONLY: RegExp[] = [ +// "empty", +// "ref", +// "type", +// "enum", +// "elements", +// "properties", +// "optionalProperties", +// "discriminator", +// "values", +// ].map((s) => new RegExp(`(^|.*\\s)${s}\\s.*-`)) + +const ONLY: RegExp[] = [] describe("JSON Type Definition", () => { describe("validation", function () { From 7783b7108c96edcf1ca08405b4b1efa3dd829dee Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Tue, 23 Feb 2021 21:32:52 +0000 Subject: [PATCH 15/19] JTD parsing: sckip whitespace --- lib/compile/jtd/parse.ts | 9 ++++++++- lib/runtime/parseJson.ts | 8 ++++++++ spec/jtd-schema.spec.ts | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index d42c8ea1d..dee24033a 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -8,7 +8,7 @@ import N from "../names" import {isOwnProperty, hasPropFunc} from "../../vocabularies/code" import {hasRef} from "../../vocabularies/jtd/ref" import {intRange, IntType} from "../../vocabularies/jtd/type" -import {parseJson, parseJsonNumber, parseJsonString} from "../../runtime/parseJson" +import {parseJson, parseJsonNumber, parseJsonString, skipWhitespace} from "../../runtime/parseJson" import {func} from "../util" import validTimestamp from "../timestamp" @@ -84,6 +84,7 @@ function parserFunction(cxt: ParseCxt, parseName: Name): void { gen.assign(N.jsonPos, _`${N.jsonPos} || 0`) gen.const(N.jsonLen, _`${N.json}.length`) parseCode(cxt) + whitespace(gen) gen.if(N.jsonPart, () => gen.return(_`[${N.data}, ${N.jsonPos}]`)) gen.if(_`${N.jsonPos} === ${N.jsonLen}`, () => gen.return(N.data)) jsonSyntaxError(cxt) @@ -298,6 +299,7 @@ function parseEnum(cxt: ParseCxt): void { function parseNumber(cxt: ParseCxt, maxDigits?: number): void { const {gen} = cxt + gen.assign(N.jsonPos, _`${func(gen, skipWhitespace)}(${N.json}, ${N.jsonPos})`) gen.if( _`"-0123456789".indexOf(${jsonSlice(1)}) < 0`, () => jsonSyntaxError(cxt), @@ -360,6 +362,7 @@ function parseToken(cxt: ParseCxt, tok: string): void { function tryParseToken(cxt: ParseCxt, tok: string, fail: GenParse, success?: GenParse): void { const {gen} = cxt const n = tok.length + whitespace(gen) gen.if( _`${jsonSlice(n)} === ${tok}`, () => { @@ -370,6 +373,10 @@ function tryParseToken(cxt: ParseCxt, tok: string, fail: GenParse, success?: Gen ) } +function whitespace(gen: CodeGen): void { + gen.assign(N.jsonPos, _`${func(gen, skipWhitespace)}(${N.json}, ${N.jsonPos})`) +} + function jsonSlice(len: number | Name): Code { return len === 1 ? _`${N.json}[${N.jsonPos}]` diff --git a/lib/runtime/parseJson.ts b/lib/runtime/parseJson.ts index 0623a322a..3ef8d28f2 100644 --- a/lib/runtime/parseJson.ts +++ b/lib/runtime/parseJson.ts @@ -122,6 +122,14 @@ export function parseJsonString(s: string, pos: number): [string, number] { parseJsonString.code = _`require("ajv/dist/runtime/parseJson").parseJsonString` +export function skipWhitespace(s: string, pos: number): number { + let c: string + while (((c = s[pos]), c === " " || c === "\n" || c === "\r" || c === "\t")) pos++ + return pos +} + +skipWhitespace.code = _`require("ajv/dist/runtime/parseJson").skipWhitespace` + function unexpectedEnd(): never { throw new SyntaxError("Unexpected end of JSON input") } diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index 208c760aa..9f6415e11 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -123,12 +123,14 @@ describe("JSON Type Definition", () => { const parse = ajv.compileParser(schema) // console.log(schema, instance, `"${JSON.stringify(instance)}"`, parse.toString()) assert.deepStrictEqual(parse(JSON.stringify(instance)), instance) + assert.deepStrictEqual(parse(` ${JSON.stringify(instance, null, 2)} `), instance) }) } else { it(`should throw exception on invalid JSON string`, () => { const parse = ajv.compileParser(schema) // console.log(parse.toString()) assert.throws(() => parse(JSON.stringify(instance))) + assert.throws(() => parse(` ${JSON.stringify(instance, null, 2)} `)) }) } }) From 8c3356cc45962abefeaf8d08158bfa998ec05922 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Tue, 23 Feb 2021 22:42:14 +0000 Subject: [PATCH 16/19] JTD parse: do not throw parsing errors, return undefined --- benchmark/jtd.js | 51 +++++++++++++++++++++++----------------- lib/compile/jtd/parse.ts | 40 +++++++++++++++++++++---------- lib/jtd.ts | 11 +++++---- lib/types/index.ts | 16 +++++++++++++ spec/jtd-schema.spec.ts | 20 ++++++++++++---- 5 files changed, 94 insertions(+), 44 deletions(-) diff --git a/benchmark/jtd.js b/benchmark/jtd.js index 9a73a4968..775f5fc7c 100644 --- a/benchmark/jtd.js +++ b/benchmark/jtd.js @@ -14,7 +14,9 @@ for (const testName in jtdValidationTests) { if (!valid) continue tests.push({ serialize: ajv.compileSerializer(schema), + parse: ajv.compileParser(schema), data: instance, + json: JSON.stringify(instance), }) } @@ -63,33 +65,38 @@ const serializer = ajv.compileSerializer(testSchema) suite.add("test data: compiled JTD serializer", () => serializer(testData)) suite.add("test data: JSON.stringify", () => JSON.stringify(testData)) -const nestedElementsSchema = { - elements: { - elements: { - type: "int32", - }, - }, -} +suite.add("JTD test suite: compiled JTD parsers", () => { + for (const test of tests) { + test.parse(test.json) + } +}) -const validNestedElements = JSON.stringify([[], [1, 2], [3, 4, 5]]) -const invalidNestedElements = JSON.stringify([[], [1, 2], {}, [3, 4, 5]]) +suite.add("JTD test suite: JSON.parse", () => { + for (const test of tests) { + JSON.parse(test.json) + } +}) -const parse = ajv.compileParser(nestedElementsSchema) +const validTestData = JSON.stringify(testData) -suite.add("valid test data: compiled JTD parser", () => parse(validNestedElements)) -suite.add("valid test data: JSON.parse", () => JSON.parse(validNestedElements)) -suite.add("invalid test data: compiled JTD parser", () => { - try { - parse(invalidNestedElements) - } catch (e) {} -}) -suite.add("invalid test data: JSON.parse", () => { - try { - JSON.parse(invalidNestedElements) - throw new Error() - } catch (e) {} +const invalidTestData = JSON.stringify({ + a: { + foo: "foo1", + bar: "1", + }, + b: { + foo: "foo2", + bar: 2, + }, }) +const parse = ajv.compileParser(testSchema) + +suite.add("valid test data: compiled JTD parser", () => parse(validTestData)) +suite.add("valid test data: JSON.parse", () => JSON.parse(validTestData)) +suite.add("invalid test data: compiled JTD parser", () => parse(invalidTestData)) +suite.add("invalid test data: JSON.parse", () => JSON.parse(invalidTestData)) + console.log() suite diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index dee24033a..5eb23339b 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -32,6 +32,7 @@ interface ParseCxt { readonly definitions: SchemaObjectMap schema: SchemaObject data: Code + parseName: Name } export default function compileParser( @@ -52,13 +53,14 @@ export default function compileParser( schemaEnv: sch, definitions, data: N.data, + parseName, } let sourceCode: string | undefined try { this._compilations.add(sch) sch.parseName = parseName - parserFunction(cxt, parseName) + parserFunction(cxt) gen.optimize(this.opts.code.optimize) const parseFuncCode = gen.toString() sourceCode = `${gen.scopeRefs(N.scope)}return ${parseFuncCode}` @@ -77,10 +79,13 @@ export default function compileParser( return sch } -function parserFunction(cxt: ParseCxt, parseName: Name): void { - const {gen} = cxt +const undef = _`undefined` + +function parserFunction(cxt: ParseCxt): void { + const {gen, parseName} = cxt gen.func(parseName, _`${N.json}, ${N.jsonPos}, ${N.jsonPart}`, false, () => { gen.let(N.data) + gen.assign(_`${parseName}.error`, undef) gen.assign(N.jsonPos, _`${N.jsonPos} || 0`) gen.const(N.jsonLen, _`${N.json}.length`) parseCode(cxt) @@ -173,13 +178,13 @@ function parseDiscriminator(cxt: ParseCxt): void { }) gen.assign(N.jsonPos, startPos) gen.if(_`${tag} === undefined`) - gen.throw(_`new Error("JSON: discriminator tag not found")`) + parsingErrorMsg(cxt, "discriminator tag not found") for (const tagValue in mapping) { gen.elseIf(_`${tag} === ${tagValue}`) parseSchemaProperties({...cxt, schema: mapping[tagValue]}, discriminator) } gen.else() - gen.throw(_`new Error("JSON: discriminator value not in schema")`) + parsingErrorMsg(cxt, "discriminator value not in schema") gen.endIf() } @@ -214,7 +219,7 @@ function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { if (additionalProperties) { parseEmpty({...cxt, data: _`${data}[${key}]`}) } else { - gen.throw(_`new Error("JSON: property "+${key}+" not allowed")`) + parsingErrorMsg(cxt, str`property ${key} not allowed`) } gen.endIf() }) @@ -223,7 +228,7 @@ function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { const allProps: Code = and( ...Object.keys(properties).map((p): Code => _`${hasProp}.call(${data}, ${p})`) ) - gen.if(not(allProps), () => gen.throw(_`new Error("JSON: missing required properties")`)) + gen.if(not(allProps), () => parsingErrorMsg(cxt, "missing required properties")) } } @@ -258,7 +263,7 @@ function parseType(cxt: ParseCxt): void { // TODO parse timestamp? parseString(cxt) const vts = func(gen, validTimestamp) - gen.if(_`!${vts}(${data})`, () => gen.throw(_`new SyntaxError("JSON: invalid timestamp")`)) + gen.if(_`!${vts}(${data})`, () => parsingErrorMsg(cxt, "invalid timestamp")) break } case "float32": @@ -269,7 +274,7 @@ function parseType(cxt: ParseCxt): void { const [min, max, maxDigits] = intRange[schema.type as IntType] parseNumber(cxt, maxDigits) gen.if(_`${data} < ${min} || ${data} > ${max}`, () => - gen.throw(_`new SyntaxError("JSON: integer out of range")`) + parsingErrorMsg(cxt, "integer out of range") ) } } @@ -348,11 +353,13 @@ function parseWith(cxt: ParseCxt, parseFunc: {code: Code}, args?: SafeExpr): voi partialParse(cxt, f, args) } -function partialParse({gen, data}: ParseCxt, parseFunc: Name, args?: SafeExpr): void { +function partialParse(cxt: ParseCxt, parseFunc: Name, args?: SafeExpr): void { + const {gen, data} = cxt gen.assign( _`[${data}, ${N.jsonPos}]`, _`${parseFunc}(${N.json}, ${N.jsonPos}${args ? _`, ${args}` : nil})` ) + gen.if(_`${data} === undefined`, () => parsingError(cxt, _`${parseFunc}.error`)) } function parseToken(cxt: ParseCxt, tok: string): void { @@ -384,7 +391,14 @@ function jsonSlice(len: number | Name): Code { } function jsonSyntaxError(cxt: ParseCxt): void { - cxt.gen.throw( - _`new SyntaxError("Unexpected token "+${N.json}[${N.jsonPos}]+" in JSON at position "+${N.jsonPos})` - ) + parsingErrorMsg(cxt, _`"unexpected token " + ${N.json}[${N.jsonPos}]`) +} + +function parsingErrorMsg(cxt: ParseCxt, msg: Code | string): void { + parsingError(cxt, cxt.gen.object(["message", msg], ["position", N.jsonPos])) +} + +function parsingError({gen, parseName}: ParseCxt, err: Code): void { + gen.assign(_`${parseName}.error`, err) + gen.return(_`${N.jsonPart} ? [undefined, ${N.jsonPos}] : undefined`) } diff --git a/lib/jtd.ts b/lib/jtd.ts index d73452dc4..04cc4a4ee 100644 --- a/lib/jtd.ts +++ b/lib/jtd.ts @@ -17,6 +17,9 @@ export { AsyncValidateFunction, ErrorObject, ErrorNoParams, + JTDParser, + JTDParserError, + JTDErrorObject, } from "./types" export {Plugin, Options, CodeOptions, InstanceOptions, Logger, ErrorsTextOptions} from "./core" @@ -26,7 +29,7 @@ export {KeywordCxt} // export {DefinedError} from "./vocabularies/errors" export {_, str, stringify, nil, Name, Code, CodeGen, CodeGenOptions} from "./compile/codegen" -import type {AnySchemaObject, SchemaObject} from "./types" +import type {AnySchemaObject, SchemaObject, JTDParser} from "./types" import type {JTDSchemaType} from "./types/jtd-schema" export {JTDSchemaType} import AjvCore, {CurrentOptions} from "./core" @@ -98,9 +101,9 @@ export default class Ajv extends AjvCore { return sch.serialize || this._compileSerializer(sch) } - compileParser(schema: SchemaObject | JTDSchemaType): (json: string) => T { + compileParser(schema: SchemaObject | JTDSchemaType): JTDParser { const sch = this._addSchema(schema) - return (sch.parse || this._compileParser(sch)) as (json: string) => T + return (sch.parse || this._compileParser(sch)) as JTDParser } private _compileSerializer(sch: SchemaEnv): (data: T) => string { @@ -110,7 +113,7 @@ export default class Ajv extends AjvCore { return sch.serialize } - private _compileParser(sch: SchemaEnv): (json: string) => unknown { + private _compileParser(sch: SchemaEnv): JTDParser { compileParser.call(this, sch, (sch.schema as AnySchemaObject).definitions || {}) /* istanbul ignore if */ if (!sch.parse) throw new Error("ajv implementation error") diff --git a/lib/types/index.ts b/lib/types/index.ts index 5fc9b6d2a..c478a695d 100644 --- a/lib/types/index.ts +++ b/lib/types/index.ts @@ -53,6 +53,22 @@ export interface ValidateFunction { source?: SourceCode } +export interface JTDParser { + (json: string): T | undefined + error?: JTDParserError +} + +export interface JTDParserError { + message: string + position: number + jtd?: JTDErrorObject +} + +export interface JTDErrorObject { + schemaPath: string + instancePath: string +} + export type EvaluatedProperties = {[K in string]?: true} | true export type EvaluatedItems = number | true diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index 9f6415e11..b163978f9 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -1,5 +1,5 @@ import type AjvJTD from "../dist/jtd" -import type {SchemaObject} from "../dist/jtd" +import type {SchemaObject, JTDParser} from "../dist/jtd" import _AjvJTD from "./ajv_jtd" import getAjvInstances from "./ajv_instances" import {withStandalone} from "./ajv_standalone" @@ -122,19 +122,29 @@ describe("JSON Type Definition", () => { it(`should parse valid JSON string`, () => { const parse = ajv.compileParser(schema) // console.log(schema, instance, `"${JSON.stringify(instance)}"`, parse.toString()) - assert.deepStrictEqual(parse(JSON.stringify(instance)), instance) - assert.deepStrictEqual(parse(` ${JSON.stringify(instance, null, 2)} `), instance) + shouldParse(parse, JSON.stringify(instance), instance) + shouldParse(parse, ` ${JSON.stringify(instance, null, 2)} `, instance) }) } else { it(`should throw exception on invalid JSON string`, () => { const parse = ajv.compileParser(schema) // console.log(parse.toString()) - assert.throws(() => parse(JSON.stringify(instance))) - assert.throws(() => parse(` ${JSON.stringify(instance, null, 2)} `)) + shouldFail(parse, JSON.stringify(instance)) + shouldFail(parse, ` ${JSON.stringify(instance, null, 2)} `) }) } }) } + + function shouldParse(parse: JTDParser, str: string, res: unknown): void { + assert.deepStrictEqual(parse(str), res) + assert.strictEqual(parse.error, undefined) + } + + function shouldFail(parse: JTDParser, str: string): void { + assert.strictEqual(parse(str), undefined) + assert.strictEqual(typeof parse.error, "object") + } }) }) From 4c7ba37cd08e9b0bf1ef09bba9b7bcea4604e537 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Wed, 24 Feb 2021 08:19:39 +0000 Subject: [PATCH 17/19] JTD parse: inline skipping whitespace --- benchmark/jtd.js | 28 ++++++++++++------------ lib/compile/jtd/parse.ts | 47 +++++++++++++++++++++------------------- lib/jtd.ts | 2 -- lib/runtime/parseJson.ts | 8 ------- lib/types/index.ts | 14 ++---------- spec/jtd-schema.spec.ts | 8 ++++--- 6 files changed, 46 insertions(+), 61 deletions(-) diff --git a/benchmark/jtd.js b/benchmark/jtd.js index 775f5fc7c..88c2a5ae2 100644 --- a/benchmark/jtd.js +++ b/benchmark/jtd.js @@ -20,17 +20,17 @@ for (const testName in jtdValidationTests) { }) } -suite.add("JTD test suite: compiled JTD serializers", () => { - for (const test of tests) { - test.serialize(test.data) - } -}) - -suite.add("JTD test suite: JSON.stringify", () => { - for (const test of tests) { - JSON.stringify(test.data) - } -}) +// suite.add("JTD test suite: compiled JTD serializers", () => { +// for (const test of tests) { +// test.serialize(test.data) +// } +// }) + +// suite.add("JTD test suite: JSON.stringify", () => { +// for (const test of tests) { +// JSON.stringify(test.data) +// } +// }) const testSchema = { definitions: { @@ -60,10 +60,10 @@ const testData = { }, } -const serializer = ajv.compileSerializer(testSchema) +// const serializer = ajv.compileSerializer(testSchema) -suite.add("test data: compiled JTD serializer", () => serializer(testData)) -suite.add("test data: JSON.stringify", () => JSON.stringify(testData)) +// suite.add("test data: compiled JTD serializer", () => serializer(testData)) +// suite.add("test data: JSON.stringify", () => JSON.stringify(testData)) suite.add("JTD test suite: compiled JTD parsers", () => { for (const test of tests) { diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index 5eb23339b..90fbf497a 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -8,7 +8,7 @@ import N from "../names" import {isOwnProperty, hasPropFunc} from "../../vocabularies/code" import {hasRef} from "../../vocabularies/jtd/ref" import {intRange, IntType} from "../../vocabularies/jtd/type" -import {parseJson, parseJsonNumber, parseJsonString, skipWhitespace} from "../../runtime/parseJson" +import {parseJson, parseJsonNumber, parseJsonString} from "../../runtime/parseJson" import {func} from "../util" import validTimestamp from "../timestamp" @@ -33,6 +33,7 @@ interface ParseCxt { schema: SchemaObject data: Code parseName: Name + char: Name } export default function compileParser( @@ -54,6 +55,7 @@ export default function compileParser( definitions, data: N.data, parseName, + char: gen.name("c"), } let sourceCode: string | undefined @@ -82,14 +84,16 @@ export default function compileParser( const undef = _`undefined` function parserFunction(cxt: ParseCxt): void { - const {gen, parseName} = cxt + const {gen, parseName, char} = cxt gen.func(parseName, _`${N.json}, ${N.jsonPos}, ${N.jsonPart}`, false, () => { gen.let(N.data) - gen.assign(_`${parseName}.error`, undef) + gen.let(char) + gen.assign(_`${parseName}.message`, undef) + gen.assign(_`${parseName}.position`, undef) gen.assign(N.jsonPos, _`${N.jsonPos} || 0`) gen.const(N.jsonLen, _`${N.json}.length`) parseCode(cxt) - whitespace(gen) + _skipWhitespace(cxt) gen.if(N.jsonPart, () => gen.return(_`[${N.data}, ${N.jsonPos}]`)) gen.if(_`${N.jsonPos} === ${N.jsonLen}`, () => gen.return(N.data)) jsonSyntaxError(cxt) @@ -178,13 +182,13 @@ function parseDiscriminator(cxt: ParseCxt): void { }) gen.assign(N.jsonPos, startPos) gen.if(_`${tag} === undefined`) - parsingErrorMsg(cxt, "discriminator tag not found") + parsingError(cxt, str`discriminator tag not found`) for (const tagValue in mapping) { gen.elseIf(_`${tag} === ${tagValue}`) parseSchemaProperties({...cxt, schema: mapping[tagValue]}, discriminator) } gen.else() - parsingErrorMsg(cxt, "discriminator value not in schema") + parsingError(cxt, str`discriminator value not in schema`) gen.endIf() } @@ -219,7 +223,7 @@ function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { if (additionalProperties) { parseEmpty({...cxt, data: _`${data}[${key}]`}) } else { - parsingErrorMsg(cxt, str`property ${key} not allowed`) + parsingError(cxt, str`property ${key} not allowed`) } gen.endIf() }) @@ -228,7 +232,7 @@ function parseSchemaProperties(cxt: ParseCxt, discriminator?: string): void { const allProps: Code = and( ...Object.keys(properties).map((p): Code => _`${hasProp}.call(${data}, ${p})`) ) - gen.if(not(allProps), () => parsingErrorMsg(cxt, "missing required properties")) + gen.if(not(allProps), () => parsingError(cxt, str`missing required properties`)) } } @@ -263,7 +267,7 @@ function parseType(cxt: ParseCxt): void { // TODO parse timestamp? parseString(cxt) const vts = func(gen, validTimestamp) - gen.if(_`!${vts}(${data})`, () => parsingErrorMsg(cxt, "invalid timestamp")) + gen.if(_`!${vts}(${data})`, () => parsingError(cxt, str`invalid timestamp`)) break } case "float32": @@ -274,7 +278,7 @@ function parseType(cxt: ParseCxt): void { const [min, max, maxDigits] = intRange[schema.type as IntType] parseNumber(cxt, maxDigits) gen.if(_`${data} < ${min} || ${data} > ${max}`, () => - parsingErrorMsg(cxt, "integer out of range") + parsingError(cxt, str`integer out of range`) ) } } @@ -304,7 +308,7 @@ function parseEnum(cxt: ParseCxt): void { function parseNumber(cxt: ParseCxt, maxDigits?: number): void { const {gen} = cxt - gen.assign(N.jsonPos, _`${func(gen, skipWhitespace)}(${N.json}, ${N.jsonPos})`) + _skipWhitespace(cxt) gen.if( _`"-0123456789".indexOf(${jsonSlice(1)}) < 0`, () => jsonSyntaxError(cxt), @@ -359,7 +363,7 @@ function partialParse(cxt: ParseCxt, parseFunc: Name, args?: SafeExpr): void { _`[${data}, ${N.jsonPos}]`, _`${parseFunc}(${N.json}, ${N.jsonPos}${args ? _`, ${args}` : nil})` ) - gen.if(_`${data} === undefined`, () => parsingError(cxt, _`${parseFunc}.error`)) + gen.if(_`${data} === undefined`, () => parsingError(cxt, _`${parseFunc}.message`)) } function parseToken(cxt: ParseCxt, tok: string): void { @@ -369,7 +373,7 @@ function parseToken(cxt: ParseCxt, tok: string): void { function tryParseToken(cxt: ParseCxt, tok: string, fail: GenParse, success?: GenParse): void { const {gen} = cxt const n = tok.length - whitespace(gen) + _skipWhitespace(cxt) gen.if( _`${jsonSlice(n)} === ${tok}`, () => { @@ -380,8 +384,10 @@ function tryParseToken(cxt: ParseCxt, tok: string, fail: GenParse, success?: Gen ) } -function whitespace(gen: CodeGen): void { - gen.assign(N.jsonPos, _`${func(gen, skipWhitespace)}(${N.json}, ${N.jsonPos})`) +function _skipWhitespace({gen, char: c}: ParseCxt): void { + gen.code( + _`while((${c}=${N.json}[${N.jsonPos}],${c}===" "||${c}==="\\n"||${c}==="\\r"||${c}==="\\t"))${N.jsonPos}++;` + ) } function jsonSlice(len: number | Name): Code { @@ -391,14 +397,11 @@ function jsonSlice(len: number | Name): Code { } function jsonSyntaxError(cxt: ParseCxt): void { - parsingErrorMsg(cxt, _`"unexpected token " + ${N.json}[${N.jsonPos}]`) -} - -function parsingErrorMsg(cxt: ParseCxt, msg: Code | string): void { - parsingError(cxt, cxt.gen.object(["message", msg], ["position", N.jsonPos])) + parsingError(cxt, _`"unexpected token " + ${N.json}[${N.jsonPos}]`) } -function parsingError({gen, parseName}: ParseCxt, err: Code): void { - gen.assign(_`${parseName}.error`, err) +function parsingError({gen, parseName}: ParseCxt, msg: Code): void { + gen.assign(_`${parseName}.message`, msg) + gen.assign(_`${parseName}.position`, N.jsonPos) gen.return(_`${N.jsonPart} ? [undefined, ${N.jsonPos}] : undefined`) } diff --git a/lib/jtd.ts b/lib/jtd.ts index 04cc4a4ee..6a7439d46 100644 --- a/lib/jtd.ts +++ b/lib/jtd.ts @@ -18,8 +18,6 @@ export { ErrorObject, ErrorNoParams, JTDParser, - JTDParserError, - JTDErrorObject, } from "./types" export {Plugin, Options, CodeOptions, InstanceOptions, Logger, ErrorsTextOptions} from "./core" diff --git a/lib/runtime/parseJson.ts b/lib/runtime/parseJson.ts index 3ef8d28f2..0623a322a 100644 --- a/lib/runtime/parseJson.ts +++ b/lib/runtime/parseJson.ts @@ -122,14 +122,6 @@ export function parseJsonString(s: string, pos: number): [string, number] { parseJsonString.code = _`require("ajv/dist/runtime/parseJson").parseJsonString` -export function skipWhitespace(s: string, pos: number): number { - let c: string - while (((c = s[pos]), c === " " || c === "\n" || c === "\r" || c === "\t")) pos++ - return pos -} - -skipWhitespace.code = _`require("ajv/dist/runtime/parseJson").skipWhitespace` - function unexpectedEnd(): never { throw new SyntaxError("Unexpected end of JSON input") } diff --git a/lib/types/index.ts b/lib/types/index.ts index c478a695d..79ffa12df 100644 --- a/lib/types/index.ts +++ b/lib/types/index.ts @@ -55,18 +55,8 @@ export interface ValidateFunction { export interface JTDParser { (json: string): T | undefined - error?: JTDParserError -} - -export interface JTDParserError { - message: string - position: number - jtd?: JTDErrorObject -} - -export interface JTDErrorObject { - schemaPath: string - instancePath: string + message?: string + position?: number } export type EvaluatedProperties = {[K in string]?: true} | true diff --git a/spec/jtd-schema.spec.ts b/spec/jtd-schema.spec.ts index b163978f9..cdd768296 100644 --- a/spec/jtd-schema.spec.ts +++ b/spec/jtd-schema.spec.ts @@ -126,7 +126,7 @@ describe("JSON Type Definition", () => { shouldParse(parse, ` ${JSON.stringify(instance, null, 2)} `, instance) }) } else { - it(`should throw exception on invalid JSON string`, () => { + it(`should return undefined on invalid JSON string`, () => { const parse = ajv.compileParser(schema) // console.log(parse.toString()) shouldFail(parse, JSON.stringify(instance)) @@ -138,12 +138,14 @@ describe("JSON Type Definition", () => { function shouldParse(parse: JTDParser, str: string, res: unknown): void { assert.deepStrictEqual(parse(str), res) - assert.strictEqual(parse.error, undefined) + assert.strictEqual(parse.message, undefined) + assert.strictEqual(parse.position, undefined) } function shouldFail(parse: JTDParser, str: string): void { assert.strictEqual(parse(str), undefined) - assert.strictEqual(typeof parse.error, "object") + assert.strictEqual(typeof parse.message, "string") + assert.strictEqual(typeof parse.position, "number") } }) }) From 6b4a910cc65a0cf3e2e2c56f5050b17a78568882 Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Fri, 26 Feb 2021 21:48:48 +0000 Subject: [PATCH 18/19] JTD parse: return position and error via function properties --- lib/compile/jtd/parse.ts | 52 +++++++++++---- lib/runtime/parseJson.ts | 134 +++++++++++++++++++++++++-------------- 2 files changed, 123 insertions(+), 63 deletions(-) diff --git a/lib/compile/jtd/parse.ts b/lib/compile/jtd/parse.ts index 90fbf497a..f2f2fa0b9 100644 --- a/lib/compile/jtd/parse.ts +++ b/lib/compile/jtd/parse.ts @@ -93,8 +93,11 @@ function parserFunction(cxt: ParseCxt): void { gen.assign(N.jsonPos, _`${N.jsonPos} || 0`) gen.const(N.jsonLen, _`${N.json}.length`) parseCode(cxt) - _skipWhitespace(cxt) - gen.if(N.jsonPart, () => gen.return(_`[${N.data}, ${N.jsonPos}]`)) + skipWhitespace(cxt) + gen.if(N.jsonPart, () => { + gen.assign(_`${parseName}.position`, N.jsonPos) + gen.return(N.data) + }) gen.if(_`${N.jsonPos} === ${N.jsonLen}`, () => gen.return(N.data)) jsonSyntaxError(cxt) }) @@ -108,8 +111,31 @@ function parseCode(cxt: ParseCxt): void { break } } - parseNullable(cxt, form ? genParse[form] : parseEmpty) -} + if (form) parseNullable(cxt, genParse[form]) + else parseEmpty(cxt) +} + +const parseBoolean = parseBooleanToken(true, parseBooleanToken(false, jsonSyntaxError)) + +// function parseEmptyCode(cxt: ParseCxt): void { +// const {gen, data, char: c} = cxt +// skipWhitespace(cxt) +// gen.assign(c, _`${N.json}[${N.jsonPos}]`) +// gen.if(_`${c} === "t" || ${c} === "f"`) +// parseBoolean(cxt) +// gen.elseIf(_`${c} === "n"`) +// tryParseToken(cxt, "null", jsonSyntaxError, () => gen.assign(data, null)) +// gen.elseIf(_`${c} === '"'`) +// parseString(cxt) +// gen.elseIf(_`${c} === "["`) +// parseElements({...cxt, schema: {elements: {}}}) +// gen.elseIf(_`${c} === "{"`) +// parseValues({...cxt, schema: {values: {}}}) +// gen.else() +// parseNumber(cxt) +// gen.endIf() +// skipWhitespace(cxt) +// } function parseNullable(cxt: ParseCxt, parseForm: GenParse): void { const {gen, schema, data} = cxt @@ -258,7 +284,7 @@ function parseType(cxt: ParseCxt): void { const {gen, schema, data} = cxt switch (schema.type) { case "boolean": - parseBoolean(true, parseBoolean(false, jsonSyntaxError))(cxt) + parseBoolean(cxt) break case "string": parseString(cxt) @@ -308,7 +334,7 @@ function parseEnum(cxt: ParseCxt): void { function parseNumber(cxt: ParseCxt, maxDigits?: number): void { const {gen} = cxt - _skipWhitespace(cxt) + skipWhitespace(cxt) gen.if( _`"-0123456789".indexOf(${jsonSlice(1)}) < 0`, () => jsonSyntaxError(cxt), @@ -316,7 +342,7 @@ function parseNumber(cxt: ParseCxt, maxDigits?: number): void { ) } -function parseBoolean(bool: boolean, fail: GenParse): GenParse { +function parseBooleanToken(bool: boolean, fail: GenParse): GenParse { return (cxt) => { const {gen, data} = cxt tryParseToken( @@ -359,10 +385,8 @@ function parseWith(cxt: ParseCxt, parseFunc: {code: Code}, args?: SafeExpr): voi function partialParse(cxt: ParseCxt, parseFunc: Name, args?: SafeExpr): void { const {gen, data} = cxt - gen.assign( - _`[${data}, ${N.jsonPos}]`, - _`${parseFunc}(${N.json}, ${N.jsonPos}${args ? _`, ${args}` : nil})` - ) + gen.assign(data, _`${parseFunc}(${N.json}, ${N.jsonPos}${args ? _`, ${args}` : nil})`) + gen.assign(N.jsonPos, _`${parseFunc}.position`) gen.if(_`${data} === undefined`, () => parsingError(cxt, _`${parseFunc}.message`)) } @@ -373,7 +397,7 @@ function parseToken(cxt: ParseCxt, tok: string): void { function tryParseToken(cxt: ParseCxt, tok: string, fail: GenParse, success?: GenParse): void { const {gen} = cxt const n = tok.length - _skipWhitespace(cxt) + skipWhitespace(cxt) gen.if( _`${jsonSlice(n)} === ${tok}`, () => { @@ -384,7 +408,7 @@ function tryParseToken(cxt: ParseCxt, tok: string, fail: GenParse, success?: Gen ) } -function _skipWhitespace({gen, char: c}: ParseCxt): void { +function skipWhitespace({gen, char: c}: ParseCxt): void { gen.code( _`while((${c}=${N.json}[${N.jsonPos}],${c}===" "||${c}==="\\n"||${c}==="\\r"||${c}==="\\t"))${N.jsonPos}++;` ) @@ -403,5 +427,5 @@ function jsonSyntaxError(cxt: ParseCxt): void { function parsingError({gen, parseName}: ParseCxt, msg: Code): void { gen.assign(_`${parseName}.message`, msg) gen.assign(_`${parseName}.position`, N.jsonPos) - gen.return(_`${N.jsonPart} ? [undefined, ${N.jsonPos}] : undefined`) + gen.return(undef) } diff --git a/lib/runtime/parseJson.ts b/lib/runtime/parseJson.ts index 0623a322a..2539bbc73 100644 --- a/lib/runtime/parseJson.ts +++ b/lib/runtime/parseJson.ts @@ -2,25 +2,40 @@ import {_} from "../compile/codegen" const rxParseJson = /position\s(\d+)$/ -export function parseJson(s: string, pos: number): [unknown, number] { +export function parseJson(s: string, pos: number): unknown { let endPos: number | undefined + parseJson.message = undefined + let matches: RegExpExecArray | null if (pos) s = s.slice(pos) try { - return [JSON.parse(s), pos + s.length] + parseJson.position = pos + s.length + return JSON.parse(s) } catch (e) { - const matches = rxParseJson.exec(e.message) - if (!matches) throw e + matches = rxParseJson.exec(e.message) + if (!matches) { + parseJson.message = "unexpected end" + return undefined + } endPos = +matches[1] s = s.slice(0, endPos) - return [JSON.parse(s), pos + endPos] + parseJson.position = pos + endPos + try { + return JSON.parse(s) + } catch (e1) { + parseJson.message = `unexpected token ${s[endPos]}` + return undefined + } } } +parseJson.message = undefined as string | undefined +parseJson.position = 0 as number parseJson.code = _`require("ajv/dist/runtime/parseJson").parseJson` -export function parseJsonNumber(s: string, pos: number, maxDigits?: number): [number, number] { +export function parseJsonNumber(s: string, pos: number, maxDigits?: number): number | undefined { let numStr = "" let c: string + parseJsonNumber.message = undefined if (s[pos] === "-") { numStr += "-" pos++ @@ -29,13 +44,22 @@ export function parseJsonNumber(s: string, pos: number, maxDigits?: number): [nu numStr += "0" pos++ } else { - parseDigits(maxDigits) + if (!parseDigits(maxDigits)) { + errorMessage() + return undefined + } + } + if (maxDigits) { + parseJsonNumber.position = pos + return +numStr } - if (maxDigits) return [+numStr, pos] if (s[pos] === ".") { numStr += "." pos++ - parseDigits() + if (!parseDigits()) { + errorMessage() + return undefined + } } if (((c = s[pos]), c === "e" || c === "E")) { numStr += "e" @@ -44,24 +68,31 @@ export function parseJsonNumber(s: string, pos: number, maxDigits?: number): [nu numStr += c pos++ } - parseDigits() + if (!parseDigits()) { + errorMessage() + return undefined + } } - return [+numStr, pos] + parseJsonNumber.position = pos + return +numStr - function parseDigits(maxLen?: number): void { - let digit: boolean | undefined + function parseDigits(maxLen?: number): boolean { + let digit = false while (((c = s[pos]), c >= "0" && c <= "9" && (maxLen === undefined || maxLen-- > 0))) { digit = true numStr += c pos++ } - if (!digit) { - if (pos < s.length) unexpectedToken(s[pos], pos) - else unexpectedEnd() - } + return digit + } + + function errorMessage(): void { + parseJson.message = pos < s.length ? `unexpected token ${s[pos]}` : "unexpected end" } } +parseJsonNumber.message = undefined as string | undefined +parseJsonNumber.position = 0 as number parseJsonNumber.code = _`require("ajv/dist/runtime/parseJson").parseJsonNumber` const escapedChars: {[X in string]?: string} = { @@ -77,9 +108,10 @@ const escapedChars: {[X in string]?: string} = { const A_CODE: number = "a".charCodeAt(0) -export function parseJsonString(s: string, pos: number): [string, number] { +export function parseJsonString(s: string, pos: number): string | undefined { let str = "" let c: string | undefined + parseJsonString.message = undefined // eslint-disable-next-line no-constant-condition, @typescript-eslint/no-unnecessary-condition while (true) { c = s[pos] @@ -87,45 +119,49 @@ export function parseJsonString(s: string, pos: number): [string, number] { if (c === '"') break if (c === "\\") { c = s[pos] - if (c in escapedChars) str += escapedChars[c] - else if (c === "u") getCharCode() - else unexpectedToken(c, pos) + if (c in escapedChars) { + str += escapedChars[c] + } else if (c === "u") { + let count = 4 + let code = 0 + while (count--) { + code <<= 4 + c = s[pos].toLowerCase() + if (c >= "a" && c <= "f") { + c += c.charCodeAt(0) - A_CODE + 10 + } else if (c >= "0" && c <= "9") { + code += +c + } else if (c === undefined) { + errorMessage("unexpected end") + return undefined + } else { + errorMessage(`unexpected token ${s[pos]}`) + return undefined + } + pos++ + } + str += String.fromCharCode(code) + } else { + errorMessage(`unexpected token ${s[pos]}`) + return undefined + } pos++ } else if (c === undefined) { - throw unexpectedEnd() + errorMessage("unexpected end") + return undefined } else { str += c } } - return [str, pos] + parseJsonString.position = pos + return str - function getCharCode(): void { - let count = 4 - let code = 0 - while (count--) { - code <<= 4 - c = s[pos].toLowerCase() - if (c >= "a" && c <= "f") { - c += c.charCodeAt(0) - A_CODE + 10 - } else if (c >= "0" && c <= "9") { - code += +c - } else if (c === undefined) { - unexpectedEnd() - } else { - unexpectedToken(c, pos) - } - pos++ - } - str += String.fromCharCode(code) + function errorMessage(msg: string): void { + parseJsonString.position = pos + parseJsonString.message = msg } } +parseJsonString.message = undefined as string | undefined +parseJsonString.position = 0 as number parseJsonString.code = _`require("ajv/dist/runtime/parseJson").parseJsonString` - -function unexpectedEnd(): never { - throw new SyntaxError("Unexpected end of JSON input") -} - -function unexpectedToken(c: string, pos: number): never { - throw new SyntaxError(`Unexpected token ${c} in JSON at position ${pos}`) -} From 1123cee793333ea799e9a8ac429404681ba2f72a Mon Sep 17 00:00:00 2001 From: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Tue, 2 Mar 2021 08:05:37 +0000 Subject: [PATCH 19/19] JTD parsers/serializers docs --- README.md | 2 +- docs/api.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ec11b9be7..dcf29af22 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,8 @@ Supports JSON Schema draft-06/07/2019-09 (draft-04 is supported in [version 6](h Ajv version 7 has these new features: +- NEW: support of JSON Type Definition [RFC8927](https://datatracker.ietf.org/doc/rfc8927/) (from [v7.1.0](https://github.com/ajv-validator/ajv-keywords/releases/tag/v7.1.0)), including generation of [serializers](./docs/api.md#jtd-serialize) and [parsers](./docs/api.md#jtd-parse) from JTD schemas that are more efficient than native JSON serialization/parsing, combining JSON string parsing and validation in one function. - support of JSON Schema draft-2019-09 features: [`unevaluatedProperties`](./docs/json-schema.md#unevaluatedproperties) and [`unevaluatedItems`](./docs/json-schema.md#unevaluateditems), [dynamic recursive references](./docs/validation.md#extending-recursive-schemas) and other [additional keywords](./docs/json-schema.md#json-schema-draft-2019-09). -- NEW: support of JSON Type Definition [RFC8927](https://datatracker.ietf.org/doc/rfc8927/) (from [v7.1.0](https://github.com/ajv-validator/ajv-keywords/releases/tag/v7.1.0)) - to reduce the mistakes in JSON schemas and unexpected validation results, [strict mode](./docs/strict-mode.md) is added - it prohibits ignored or ambiguous JSON Schema elements. - to make code injection from untrusted schemas impossible, [code generation](./docs/codegen.md) is fully re-written to be safe and to allow code optimization (compiled schema code size is reduced by more than 10%). - to simplify Ajv extensions, the new keyword API that is used by pre-defined keywords is available to user-defined keywords - it is much easier to define any keywords now, especially with subschemas. [ajv-keywords](https://github.com/ajv-validator/ajv-keywords) package was updated to use the new API (in [v4.0.0](https://github.com/ajv-validator/ajv-keywords/releases/tag/v4.0.0)) diff --git a/docs/api.md b/docs/api.md index c6692972c..862941ddc 100644 --- a/docs/api.md +++ b/docs/api.md @@ -50,6 +50,69 @@ if (validate(data)) { See more advanced example in [the test](../spec/types/json-schema.spec.ts). +#### ajv.compileSerializer(schema: object): (data: any) =\> string (NEW) + +Generate serializing function based on the [JTD schema](./json-type-definition.md) (caches the schema) - only in JTD instance of Ajv (see example below). + +Serializers compiled from JTD schemas can be more than 10 times faster than using `JSON.stringify`, because they do not traverse all the data, only the properties that are defined in the schema. + +Properties not defined in the schema will not be included in serialized JSON, unless the schema has `additionalProperties: true` flag. It can also be beneficial from the application security point of view, as it prevents leaking accidentally/temporarily added additional properties to the API responses. + +If you use JTD with typescript, the type for the schema can be derived from the data type, and generated serializer would only accept correct data type in this case: + +```typescript +import Ajv, {JTDSchemaType} from "ajv/dist/jtd" +const ajv = new Ajv() + +interface MyData = { + foo: number + bar?: string +} + +const mySchema: JTDSchemaType = { + properties: { + foo: {type: "int32"} // any JTD number type would be accepted here + }, + optionalProperties: { + bar: {type: "string"} + } +} + +const serializeMyData = ajv.compileSerializer(mySchema) + +// serializeMyData has type (x: MyData) => string +// it prevents you from accidentally passing the wrong type +``` + +**Please note**: Compiled serializers do NOT validate passed data, it is assumed that the data is valid according to the schema. In the future there may be an option added that would make serializers also validate the data. + +#### ajv.compileParser(schema: object): (json: string) =\> any (NEW) + +Generate parsing function based on the [JTD schema](./json-type-definition.md) (caches the schema) - only in JTD instance of Ajv (see example below). + +Parsers compiled from JTD schemas have comparable performance to `JSON.parse`* in case JSON string is valid according to the schema (and they do not just parse JSON - they ensure that parsed JSON is valid according to the schema as they parse), but they can be many times faster in case the string is invalid - for example, if schema expects an object, and JSON string is array the parser would fail on the first character. + +Parsing will fail if there are properties not defined in the schema, unless the schema has `additionalProperties: true` flag. + +If you use JTD with typescript, the type for the schema can be derived from the data type, and generated parser will return correct data type (see definitions example in the [serialize](#jtd-serialize) section): + +```typescript +const parseMyData = ajv.compileParser(mySchema) + +// parseMyData has type (s: string) => MyData | undefined +// it returns correct data type in case parsing is successful and undefined if not + +const validData = parseMyData('{"foo":1}') // {foo: 1} - success + +const invalidData = parseMyData('{"x":1}') // undefined - failure +console.log(parseMyData.position) // 4 +console.log(parseMyData.message) // property x not allowed +``` + +**Please note**: generated parsers is a NEW Ajv functionality (as of March 2021), there can be some edge cases that are not handled correctly - please report any issues/submit fixes. + +* As long as empty schema `{}` is not used - there is a possibility to improve performance in this case. Also, the performance of parsing `discriminator` schemas depends on the position of discriminator tag in the schema - the best parsing performance will be achieved if the tag is the first property - this is how compiled JTD serializers generate JSON in case of discriminator schemas. + #### ajv.compileAsync(schema: object, meta?: boolean): Promise\ Asynchronous version of `compile` method that loads missing remote schemas using asynchronous function in `options.loadSchema`. This function returns a Promise that resolves to a validation function. An optional callback passed to `compileAsync` will be called with 2 parameters: error (or null) and validating function. The returned promise will reject (and the callback will be called with an error) when: