Skip to content

Commit

Permalink
Use js-yaml (instead of yaml.js) for YAML parsing
Browse files Browse the repository at this point in the history
Resolves #165

* Fixes #63
* Fixes kaitai-io/kaitai_struct#456
* Fixes #150
* Fixes #27
* Fixes #62
* Fixes kaitai-io/kaitai_struct#693
  • Loading branch information
generalmimon committed Feb 4, 2024
1 parent 80c94b1 commit a47b86e
Show file tree
Hide file tree
Showing 8 changed files with 248 additions and 12 deletions.
32 changes: 32 additions & 0 deletions LICENSE-3RD-PARTY.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1230,6 +1230,38 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================================================

================================================================================
js-yaml

License name: MIT
License URL: https://github.com/nodeca/js-yaml/blob/master/LICENSE
License applies to files under the folder lib/_npm/js-yaml/

Source: https://github.com/nodeca/js-yaml
================================================================================
(The MIT License)

Copyright (C) 2011-2015 by Vitaly Puzrin

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
================================================================================

================================================================================
jsTree

Expand Down
5 changes: 5 additions & 0 deletions docs/wiki/3rd-party-libraries.md
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ Source: https://github.com/jquery/jquery

License: MIT (https://raw.githubusercontent.com/jquery/jquery/master/LICENSE.txt)

## js-yaml
Source: https://github.com/nodeca/js-yaml

License: MIT (https://github.com/nodeca/js-yaml/blob/master/LICENSE)

## jsTree
Website: https://www.jstree.com/

Expand Down
2 changes: 1 addition & 1 deletion index.html
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
<script src="lib/_npm/requirejs/require.js"></script>
<script src="lib/_npm/jquery/jquery.min.js"></script>
<script src="lib/_npm/bootstrap/js/bootstrap.min.js"></script>
<script src="lib/_npm/yamljs/yaml.js"></script>
<script src="lib/_npm/ace/ace.js"></script>

<script src="js/autorefresh.js"></script>
Expand All @@ -37,6 +36,7 @@
["bowser", "jstree", "localforage", "goldenlayout", "vue", "kaitai-struct-compiler", "dateformat"].forEach(
name => paths[name] = `../../lib/_npm/${name}/${name}`);
paths["big-integer"] = "../../lib/_npm/BigInteger/BigInteger";
paths["js-yaml"] = "../../lib/_npm/js-yaml/js-yaml.min";

requirejs.config({ baseUrl: "js/v1/", paths: paths });
require(["app.unsupportedBrowser"]);
Expand Down
165 changes: 165 additions & 0 deletions lib/ts-types/js-yaml.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
// Type definitions for js-yaml v4.0.0
// Project: https://github.com/nodeca/js-yaml

// Adapted from https://github.com/DefinitelyTyped/DefinitelyTyped/blob/65afafadd93759348909de1cdad5df5789c94326/types/js-yaml/index.d.ts
// 1. fixed compatibility with TypeScript 2 (which we're still
// using at the time of writing) by replacing `unknown` with `any`
// 2. wrap everything in `declare namespace jsyaml { ... }` and export it
// as "js-yaml" module so that it works in our setup

declare namespace jsyaml {
export function load(str: string, opts?: LoadOptions): any;

export class Type {
constructor(tag: string, opts?: TypeConstructorOptions);

kind: "sequence" | "scalar" | "mapping" | null;

resolve(data: any): boolean;

construct(data: any, type?: string): any;

instanceOf: object | null;
predicate: ((data: object) => boolean) | null;
represent: ((data: object) => any) | { [x: string]: (data: object) => any } | null;
representName: ((data: object) => any) | null;
defaultStyle: string | null;
multi: boolean;
styleAliases: { [x: string]: any };
}

export class Schema {
constructor(definition: SchemaDefinition | Type[] | Type);

extend(types: SchemaDefinition | Type[] | Type): Schema;
}

export function loadAll(str: string, iterator?: null, opts?: LoadOptions): any[];
export function loadAll(str: string, iterator: (doc: any) => void, opts?: LoadOptions): void;

export function dump(obj: any, opts?: DumpOptions): string;

export interface LoadOptions {
/** string to be used as a file path in error/warning messages. */
filename?: string | undefined;

/** function to call on warning messages. */
onWarning?(this: null, e: YAMLException): void;

/** specifies a schema to use. */
schema?: Schema | undefined;
/** compatibility with JSON.parse behaviour. */
json?: boolean | undefined;

/** listener for parse events */
listener?(this: State, eventType: EventType, state: State): void;
}

export type EventType = "open" | "close";

export interface State {
input: string;
filename: string | null;
schema: Schema;
onWarning: (this: null, e: YAMLException) => void;
json: boolean;
length: number;
position: number;
line: number;
lineStart: number;
lineIndent: number;
version: null | number;
checkLineBreaks: boolean;
kind: string;
result: any;
implicitTypes: Type[];
}

export interface DumpOptions {
/** indentation width to use (in spaces). */
indent?: number | undefined;
/** when true, will not add an indentation level to array elements */
noArrayIndent?: boolean | undefined;
/** do not throw on invalid types (like function in the safe schema) and skip pairs and single values with such types. */
skipInvalid?: boolean | undefined;
/** specifies level of nesting, when to switch from block to flow style for collections. -1 means block style everwhere */
flowLevel?: number | undefined;
/** Each tag may have own set of styles. - "tag" => "style" map. */
styles?: { [x: string]: any } | undefined;
/** specifies a schema to use. */
schema?: Schema | undefined;
/** if true, sort keys when dumping YAML. If a function, use the function to sort the keys. (default: false) */
sortKeys?: boolean | ((a: any, b: any) => number) | undefined;
/** set max line width. (default: 80) */
lineWidth?: number | undefined;
/** if true, don't convert duplicate objects into references (default: false) */
noRefs?: boolean | undefined;
/** if true don't try to be compatible with older yaml versions. Currently: don't quote "yes", "no" and so on, as required for YAML 1.1 (default: false) */
noCompatMode?: boolean | undefined;
/**
* if true flow sequences will be condensed, omitting the space between `key: value` or `a, b`. Eg. `'[a,b]'` or `{a:{b:c}}`.
* Can be useful when using yaml for pretty URL query params as spaces are %-encoded. (default: false).
*/
condenseFlow?: boolean | undefined;
/** strings will be quoted using this quoting style. If you specify single quotes, double quotes will still be used for non-printable characters. (default: `'`) */
quotingType?: "'" | "\"" | undefined;
/** if true, all non-key strings will be quoted even if they normally don't need to. (default: false) */
forceQuotes?: boolean | undefined;
/** callback `function (key, value)` called recursively on each key/value in source object (see `replacer` docs for `JSON.stringify`). */
replacer?: ((key: string, value: any) => any) | undefined;
}

export interface TypeConstructorOptions {
kind?: "sequence" | "scalar" | "mapping" | undefined;
resolve?: ((data: any) => boolean) | undefined;
construct?: ((data: any, type?: string) => any) | undefined;
instanceOf?: object | undefined;
predicate?: ((data: object) => boolean) | undefined;
represent?: ((data: object) => any) | { [x: string]: (data: object) => any } | undefined;
representName?: ((data: object) => any) | undefined;
defaultStyle?: string | undefined;
multi?: boolean | undefined;
styleAliases?: { [x: string]: any } | undefined;
}

export interface SchemaDefinition {
implicit?: Type[] | undefined;
explicit?: Type[] | undefined;
}

/** only strings, arrays and plain objects: http://www.yaml.org/spec/1.2/spec.html#id2802346 */
export let FAILSAFE_SCHEMA: Schema;
/** only strings, arrays and plain objects: http://www.yaml.org/spec/1.2/spec.html#id2802346 */
export let JSON_SCHEMA: Schema;
/** same as JSON_SCHEMA: http://www.yaml.org/spec/1.2/spec.html#id2804923 */
export let CORE_SCHEMA: Schema;
/** all supported YAML types */
export let DEFAULT_SCHEMA: Schema;

export interface Mark {
buffer: string;
column: number;
line: number;
name: string;
position: number;
snippet: string;
}

export class YAMLException extends Error {
constructor(reason?: string, mark?: Mark);

toString(compact?: boolean): string;

name: string;

reason: string;

message: string;

mark: Mark;
}
}

declare module "js-yaml" {
export = jsyaml;
}
29 changes: 23 additions & 6 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"font-awesome": "^4.7.0",
"golden-layout": "^1.5.9",
"jquery": "^3.5.0",
"js-yaml": "^4.1.0",
"jstree": "^3.3.4",
"kaitai-struct": "next",
"kaitai-struct-compiler": "next",
Expand Down
19 changes: 14 additions & 5 deletions src/v1/KaitaiServices.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { fss, IFsItem } from "./app.files";
import { performanceHelper } from "./utils/PerformanceHelper";
import KaitaiStructCompiler = require("kaitai-struct-compiler");
import * as jsyaml from "js-yaml";

class SchemaUtils {
static ksyNameToJsName(ksyName: string, isProp: boolean) {
Expand Down Expand Up @@ -62,7 +63,7 @@ class JsImporter implements IYamlImporter {
const sourceAppendix = mode === 'abs' ? 'kaitai.io' : 'local storage';
let ksyContent;
try {
ksyContent = await fss[importedFsType].get(`${loadFn}.ksy`);
ksyContent = await fss[importedFsType].get(fn);
} catch (e) {
const error = new Error(`failed to import spec ${fn} from ${sourceAppendix}${e.message ? ': ' + e.message : ''}`);

Expand All @@ -78,11 +79,11 @@ class JsImporter implements IYamlImporter {
};
throw error;
}
const ksyModel = <KsySchema.IKsyFile>YAML.parse(<string>ksyContent);
const ksyModel = <KsySchema.IKsyFile>parseYaml(<string>ksyContent, fn);
Object.assign(this.ksyTypes, SchemaUtils.collectKsyTypes(ksyModel));

// we have to modify the schema (add typesByJsName for example) before sending into the compiler, so we need a copy
const compilerSchema = <KsySchema.IKsyFile>YAML.parse(<string>ksyContent);
const compilerSchema = <KsySchema.IKsyFile>parseYaml(<string>ksyContent, fn);
return compilerSchema;
}
}
Expand All @@ -100,11 +101,11 @@ export class CompilerService {
var perfYamlParse = performanceHelper.measureAction("YAML parsing");

try {
this.ksySchema = <KsySchema.IKsyFile>YAML.parse(srcYaml);
this.ksySchema = <KsySchema.IKsyFile>parseYaml(srcYaml, srcYamlFsItem.fn);
this.ksyTypes = SchemaUtils.collectKsyTypes(this.ksySchema);

// we have to modify the schema (add typesByJsName for example) before sending into the compiler, so we need a copy
var compilerSchema = <KsySchema.IKsyFile>YAML.parse(srcYaml);
var compilerSchema = <KsySchema.IKsyFile>parseYaml(srcYaml, srcYamlFsItem.fn);
} catch (parseErr) {
return Promise.reject(new CompilationError("yaml", parseErr));
}
Expand Down Expand Up @@ -132,3 +133,11 @@ export class CompilerService {
}
}
}

function parseYaml(yamlContents: string, filename: string) {
const options = {
schema: jsyaml.CORE_SCHEMA,
filename: filename,
};
return jsyaml.load(yamlContents, options);
}
7 changes: 7 additions & 0 deletions vendor.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,13 @@ libs:
npmDir: kaitai-struct
files: [LICENSE, KaitaiStream.js]

js-yaml:
source: https://github.com/nodeca/js-yaml
licenseName: MIT
licenseUrl: https://github.com/nodeca/js-yaml/blob/master/LICENSE
npmDir: js-yaml
files: [LICENSE, dist/js-yaml.min.js]

yaml.js:
source: https://github.com/jeremyfa/yaml.js
licenseName: MIT
Expand Down

0 comments on commit a47b86e

Please sign in to comment.