From 1531e88f7083fc7773ea139bcbf943d90c42734f Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 29 Aug 2023 15:37:31 +1000 Subject: [PATCH 1/8] feat(developer): eliminate source .model_info and .keyboard_info files Removes all references to source .keyboard_info and .model_info files in the kmc compiler and types. Updates the model info compiler to support building purely from model sources (isRTL, license fields are TODO). --- .../types/src/kpj/keyman-developer-project.ts | 46 +++++---- common/web/types/src/kpj/kpj-file-reader.ts | 1 - common/web/types/src/util/file-types.ts | 18 +--- .../types/test/kpj/test-kpj-file-reader.ts | 4 +- developer/src/README.md | 39 +++++--- developer/src/inst/kmdev.wxs | 4 - developer/src/inst/node/kmlmi.cmd | 4 - developer/src/kmc-keyboard-info/src/index.ts | 6 +- .../test/test-keyboard-info-compiler.ts | 2 +- .../kmc-model-info/src/model-info-compiler.ts | 94 +++++++------------ .../src/kmc-model-info/src/model-info-file.ts | 7 -- .../sil.cmo.bw/build/sil.cmo.bw.model_info | 3 +- .../test/fixtures/sil.cmo.bw/sil.cmo.bw.kpj | 7 -- .../fixtures/sil.cmo.bw/sil.cmo.bw.model_info | 7 -- .../sil.cmo.bw/source/sil.cmo.bw.model.kps | 1 + .../test/test-model-info-compiler.ts | 9 +- developer/src/kmc/README.md | 7 +- developer/src/kmc/build-bundler.js | 1 - developer/src/kmc/package.json | 6 +- developer/src/kmc/src/commands/build.ts | 4 - .../buildClasses/BuildKeyboardInfo.ts | 33 +++---- .../commands/buildClasses/BuildModelInfo.ts | 28 +++--- .../src/commands/buildClasses/BuildProject.ts | 18 +++- .../commands/buildClasses/buildActivities.ts | 7 +- developer/src/kmc/src/kmlmi.ts | 91 ------------------ developer/src/tike/Makefile | 1 - developer/src/tike/kmlmi.cmd | 20 ---- .../source/dmg.dv.test/dmg.dv.test.kpj | 7 -- .../source/dmg.dv.test/dmg.dv.test.model_info | 7 -- 29 files changed, 145 insertions(+), 337 deletions(-) delete mode 100644 developer/src/inst/node/kmlmi.cmd delete mode 100644 developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/sil.cmo.bw.model_info delete mode 100644 developer/src/kmc/src/kmlmi.ts delete mode 100644 developer/src/tike/kmlmi.cmd delete mode 100644 web/src/test/manual/web/issue2924/source/dmg.dv.test/dmg.dv.test.model_info diff --git a/common/web/types/src/kpj/keyman-developer-project.ts b/common/web/types/src/kpj/keyman-developer-project.ts index 1eee3a18cce..a6814a5302f 100644 --- a/common/web/types/src/kpj/keyman-developer-project.ts +++ b/common/web/types/src/kpj/keyman-developer-project.ts @@ -1,5 +1,5 @@ // -// Version 1.0 of Keyman Developer Project .kpj file +// Version 1.0 and 2.0 of Keyman Developer Project .kpj file // import { KeymanFileTypes } from '../main.js'; @@ -9,11 +9,17 @@ export class KeymanDeveloperProject { options: KeymanDeveloperProjectOptions; files: KeymanDeveloperProjectFile[]; projectPath: string = ''; + readonly projectFile: KeymanDeveloperProjectFile; - constructor(private projectFilename: string, version: KeymanDeveloperProjectVersion, private callbacks: CompilerCallbacks) { - this.projectPath = this.callbacks.path.dirname(this.projectFilename); + get projectFilename() { + return this._projectFilename; + } + + constructor(private _projectFilename: string, version: KeymanDeveloperProjectVersion, private callbacks: CompilerCallbacks) { + this.projectPath = this.callbacks.path.dirname(this._projectFilename); this.options = new KeymanDeveloperProjectOptions(version); this.files = []; + this.projectFile = new KeymanDeveloperProjectFile20(_projectFilename, callbacks); } /** * Adds .kmn, .xml, .kps to project based on options.sourcePath @@ -38,8 +44,6 @@ export class KeymanDeveloperProject { this.files.push(file); } } - - this.addMetadataFile(); } public isKeyboardProject() { @@ -50,26 +54,32 @@ export class KeymanDeveloperProject { return !!this.files.find(file => file.fileType == KeymanFileTypes.Source.Model); } - public addMetadataFile() { - const ext = this.isLexicalModelProject() ? KeymanFileTypes.Source.ModelInfo : KeymanFileTypes.Source.KeyboardInfo; + private resolveProjectPath(p: string): string { + // Replace placeholders in the target path + return p.replace('$PROJECTPATH', this.projectPath); + } + + getOutputFilePath(type: KeymanFileTypes.Binary) { + // Roughly corresponds to Delphi TProject.GetTargetFileName + let p = this.options.version == '1.0' ? + this.options.buildPath || '$SOURCEPATH' : + this.options.buildPath; - if(this.files.find(file => KeymanFileTypes.filenameIs(file.filename, ext))) { - return; + // Replace placeholders in the target path + if(this.options.version == '1.0') { + // TODO: do we need to support $VERSION? + // $SOURCEPATH only supported in 1.0 projects + p = p.replace('$SOURCEPATH', this.callbacks.path.dirname(this._projectFilename)); } - const infoFile = - this.callbacks.path.join(this.projectPath, - this.callbacks.path.basename(this.projectFilename, KeymanFileTypes.Source.Project) + ext); - this.files.push(new KeymanDeveloperProjectFile20(infoFile, this.callbacks)); - } + p = this.resolveProjectPath(p); - private resolveProjectPath(p: string): string { - // Replace placeholders in the target path - return p.replace('$PROJECTPATH', this.projectPath); + const f = this.callbacks.path.basename(this._projectFilename, KeymanFileTypes.Source.Project) + type; + return this.callbacks.path.normalize(this.callbacks.path.join(p, f)); } resolveInputFilePath(file: KeymanDeveloperProjectFile): string { - return this.callbacks.resolveFilename(this.projectFilename, file.filePath); + return this.callbacks.resolveFilename(this._projectFilename, file.filePath); } resolveOutputFilePath(file: KeymanDeveloperProjectFile, sourceExt: string, targetExt: string): string { diff --git a/common/web/types/src/kpj/kpj-file-reader.ts b/common/web/types/src/kpj/kpj-file-reader.ts index 5693d3a0b69..5472d5c8d24 100644 --- a/common/web/types/src/kpj/kpj-file-reader.ts +++ b/common/web/types/src/kpj/kpj-file-reader.ts @@ -77,7 +77,6 @@ export class KPJFileReader { if(result.options.version == '1.0') { this.transformFilesVersion10(project, result); - result.addMetadataFile(); } else { result.populateFiles(); } diff --git a/common/web/types/src/util/file-types.ts b/common/web/types/src/util/file-types.ts index 52aa7495457..e3d3bd54523 100644 --- a/common/web/types/src/util/file-types.ts +++ b/common/web/types/src/util/file-types.ts @@ -10,9 +10,7 @@ export const enum Source { LdmlKeyboard = '.xml', // Warning, also other possible uses Package = '.kps', VisualKeyboard = '.kvks', - TouchLayout = '.keyman-touch-layout', - KeyboardInfo = '.keyboard_info', - ModelInfo = '.model_info', + TouchLayout = '.keyman-touch-layout' }; /** @@ -26,9 +24,7 @@ export const ALL_SOURCE: ReadonlyArray = [ Source.LdmlKeyboard, Source.Package, Source.VisualKeyboard, - Source.TouchLayout, - Source.KeyboardInfo, - Source.ModelInfo, + Source.TouchLayout ] as const; /** @@ -146,16 +142,6 @@ export function filenameIs(filename: string, fileType: Source | Binary) { return filename.toLowerCase().endsWith(fileType); } -/** - * Returns true if the file is either a .keyboard_info file or a .model_info - * file - * @param filename - * @returns - */ -export function filenameIsMetadata(filename: string) { - return filenameIs(filename, Source.KeyboardInfo) || filenameIs(filename, Source.ModelInfo); -} - /** * Replaces a filename extension with the new extension. Returns `null` if the * filename does not end with oldExtension. diff --git a/common/web/types/test/kpj/test-kpj-file-reader.ts b/common/web/types/test/kpj/test-kpj-file-reader.ts index 0523bac078b..12c1e7a134e 100644 --- a/common/web/types/test/kpj/test-kpj-file-reader.ts +++ b/common/web/types/test/kpj/test-kpj-file-reader.ts @@ -23,7 +23,7 @@ describe('kpj-file-reader', function () { assert.equal(kpj.KeymanDeveloperProject.Options.CompilerWarningsAsErrors, 'True'); assert.equal(kpj.KeymanDeveloperProject.Options.ProjectType, 'keyboard'); assert.equal(kpj.KeymanDeveloperProject.Options.WarnDeprecatedCode, 'True'); - assert.equal(kpj.KeymanDeveloperProject.Options.SkipMetadataFiles, 'True'); // because this is a 1.0 version file + assert.isUndefined(kpj.KeymanDeveloperProject.Options.SkipMetadataFiles); // because this is a 1.0 version file assert.isUndefined(kpj.KeymanDeveloperProject.Options.Version); assert.lengthOf(kpj.KeymanDeveloperProject.Files.File, 21); @@ -77,7 +77,7 @@ describe('kpj-file-reader', function () { assert.isTrue(project.options.skipMetadataFiles); assert.equal(project.options.version, '1.0'); - assert.lengthOf(project.files, 3); + assert.lengthOf(project.files, 2); let f: KeymanDeveloperProjectFile10 = project.files[0]; assert.equal(f.id, 'id_f347675c33d2e6b1c705c787fad4941a'); diff --git a/developer/src/README.md b/developer/src/README.md index 95b33c36f44..7622abaca5d 100644 --- a/developer/src/README.md +++ b/developer/src/README.md @@ -68,31 +68,40 @@ details. ## src/kmc -node-based next generation compiler, hosts kmc, kmlmi, kmlmc, kmlmp +node-based next generation compiler, hosts kmc, (and legacy kmlmc, kmlmp) + +### src/kmc-analyze - Analysis tools + +File analysis tools for Keyman files. + +### src/kmc-keyboard-info - Keyboard Info Compiler + +Builds .keyboard_info files for use on the Keyman Cloud keyboard repository +at https://github.com/keymanapp/keyboards. Command line access through kmc. + +### src/kmc-kmn - Keyboard Compiler + +Builds .kmx files from .kmn. Command line access through kmc. ### src/kmc-ldml - LDML Keyboard Compiler -Next Generation keyboard compiler package - LDML keyboards only at present. -Command line access through kmc. +Next Generation keyboard compiler - LDML keyboards. Command line access through +kmc. ### src/kmc-model - Lexical Model Compiler -The Lexical Model Compiler, kmlmc, runs on nodeJS on all supported desktop -platforms. Command line access through kmc/kmlmc. +The Lexical Model Compiler, runs on nodeJS on all supported desktop platforms. +Command line access through kmc. -### src/kmc-package - Package Compiler +### src/kmc-model-info - Model Info Compiler -The package compiler is broadly compatible with the kmcomp .kps package -compiler. However at this stage it is only tested with lexical models, and use -with keyboards (either .js or .kmx) is not tested or supported. It is likely in -the future that the kmcomp .kps compiler will be deprecated in favour of this -one. Command line access through kmc/kmlmp. +Builds .model_info files for use on the Keyman Cloud lexical model repository at +https://github.com/keymanapp/lexical-models. Command line access through kmc. -### src/kmc-model-info - Model Info Compiler +### src/kmc-package - Package Compiler -Merges .model_info files for use on the Keyman Cloud lexical model repository at -https://github.com/keymanapp/lexical-models. Command line access through -kmc/kmlmi. +Compiles .kps packages into .kmp files. Works with both lexical model packages +and keyboard packages. Command line access through kmc. ## src/samples diff --git a/developer/src/inst/kmdev.wxs b/developer/src/inst/kmdev.wxs index 436d532f4f5..f44cd0b883f 100644 --- a/developer/src/inst/kmdev.wxs +++ b/developer/src/inst/kmdev.wxs @@ -253,10 +253,6 @@ - - - - diff --git a/developer/src/inst/node/kmlmi.cmd b/developer/src/inst/node/kmlmi.cmd deleted file mode 100644 index e674fe4d2c0..00000000000 --- a/developer/src/inst/node/kmlmi.cmd +++ /dev/null @@ -1,4 +0,0 @@ -@rem This script avoids path dependencies for node for distribution -@rem with Keyman Developer. When used on platforms other than Windows, -@rem node can be used directly with the compiler (`npm link` will setup). -@"%~dp0\node.js\node.exe" "%~dp0\kmc\kmlmi.mjs" %* diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts index ef86efed05d..0c0876cf247 100644 --- a/developer/src/kmc-keyboard-info/src/index.ts +++ b/developer/src/kmc-keyboard-info/src/index.ts @@ -62,7 +62,7 @@ export class KeyboardInfoCompiler { } /** - * Merges source .keyboard_info file with metadata from the keyboard and package source file. + * Builds a .keyboard_info file with metadata from the keyboard and package source file. * This function is intended for use within the keyboards repository. While many of the * parameters could be deduced from each other, they are specified here to reduce the * number of places the filenames are constructed. @@ -71,7 +71,7 @@ export class KeyboardInfoCompiler { * * @param sources Details on files from which to extract metadata */ - public writeMergedKeyboardInfoFile( + public writeKeyboardInfoFile( sources: KeyboardInfoSources ): Uint8Array { @@ -158,7 +158,7 @@ export class KeyboardInfoCompiler { // description if(kmpJsonData.info.description?.description) { - keyboard_info.description = kmpJsonData.info.description?.description.trim(); + keyboard_info.description = kmpJsonData.info.description.description.trim(); } // extract the language identifiers from the language metadata arrays for diff --git a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts index 27c2a383bea..939b76407ab 100644 --- a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts +++ b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts @@ -19,7 +19,7 @@ describe('keyboard-info-compiler', function () { const buildKeyboardInfoFilename = makePathToFixture('khmer_angkor', 'build', 'khmer_angkor.keyboard_info'); const compiler = new KeyboardInfoCompiler(callbacks); - const data = compiler.writeMergedKeyboardInfoFile({ + const data = compiler.writeKeyboardInfoFile({ kmpFilename, sourcePath: 'release/k/khmer_angkor', kpsFilename, diff --git a/developer/src/kmc-model-info/src/model-info-compiler.ts b/developer/src/kmc-model-info/src/model-info-compiler.ts index 32730dd034f..04ebf6cef8d 100644 --- a/developer/src/kmc-model-info/src/model-info-compiler.ts +++ b/developer/src/kmc-model-info/src/model-info-compiler.ts @@ -1,5 +1,5 @@ /** - * Merges a source .model_info file with metadata extracted from .kps file and + * Builds a source .model_info file with metadata extracted from .kps file and * compiled files to produce a comprehensive .model_info file. */ @@ -8,6 +8,8 @@ import { ModelInfoFile } from "./model-info-file.js"; import { CompilerCallbacks, KmpJsonFile } from "@keymanapp/common-types"; import { ModelInfoCompilerMessages } from "./messages.js"; +const HelpRoot = 'https://help.keyman.com/model/'; + /* c8 ignore start */ export class ModelInfoOptions { /** The identifier for the model */ @@ -28,16 +30,15 @@ export class ModelInfoOptions { /* c8 ignore stop */ /** - * Merges source .model_info file with metadata from the model and package source file. + * Builds .model_info file with metadata from the model and package source file. * This function is intended for use within the lexical-models repository. While many of the * parameters could be deduced from each other, they are specified here to reduce the * number of places the filenames are constructed. * - * @param sourceModelInfoFileName Path for the source .model_info file + * @param callbacks * @param options Details on files from which to extract additional metadata */ -export function writeMergedModelMetadataFile( - sourceModelInfoFileName: string, +export function writeModelMetadataFile( callbacks: CompilerCallbacks, options: ModelInfoOptions ): Uint8Array { @@ -58,66 +59,44 @@ export function writeMergedModelMetadataFile( * For full documentation, see: * https://help.keyman.com/developer/cloud/model_info/1.0/ */ - const dataInput = callbacks.loadFile(sourceModelInfoFileName); - if(!dataInput) { - callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileDoesNotExist({filename: sourceModelInfoFileName})); - return null; - } - let model_info: ModelInfoFile = null; - try { - const jsonInput = new TextDecoder('utf-8', {fatal: true}).decode(dataInput); - model_info = JSON.parse(jsonInput); - } catch(e) { - callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileIsNotValid({filename: sourceModelInfoFileName, e})); - return null; - } + let model_info: ModelInfoFile = { + languages: [], + license: 'mit' + }; // - // Build merged .model_info file + // Build .model_info file -- some fields have "special" behaviours -- see below // https://api.keyman.com/schemas/model_info.source.json and // https://api.keyman.com/schemas/model_info.distribution.json // https://help.keyman.com/developer/cloud/model_info/1.0 // - function setModelMetadata(field: keyof ModelInfoFile, expected: unknown, warn: boolean = true) { - /* c8 ignore next 4 */ - if (model_info[field] && model_info[field] !== expected) { - if (warn ?? true) { - callbacks.reportMessage(ModelInfoCompilerMessages.Warn_MetadataFieldInconsistent({ - field, value:model_info[field], expected - })); - } - - } - // TypeScript gets upset with this assignment, because it cannot deduce - // the exact type of model_info[field] -- there are many possibilities! - // So we assert that it's unknown so that TypeScript can chill. - ( model_info[field]) = model_info[field] || expected; - } - - // - // Merge model info file -- some fields have "special" behaviours -- see below - // - - setModelMetadata('id', options.model_id); + // TODO: isrtl + // TODO: license - setModelMetadata('name', options.kmpJsonData.info.name.description); + model_info.id = options.model_id; + model_info.name = options.kmpJsonData.info.name.description; - let author = options.kmpJsonData.info.author; - setModelMetadata('authorName', author?.description); + const author = options.kmpJsonData.info.author; + model_info.authorName = author?.description ?? ''; if (author?.url) { // we strip the mailto: from the .kps file for the .model_info - let match = author.url.match(/^(mailto\:)?(.+)$/); + const match = author.url.match(/^(mailto\:)?(.+)$/); /* c8 ignore next 3 */ if (match === null) { callbacks.reportMessage(ModelInfoCompilerMessages.Error_InvalidAuthorEmail({email:author.url})); return null; } - let email = match[2]; - setModelMetadata('authorEmail', email, false); + model_info.authorEmail = match[2]; + } + + // description + + if(options.kmpJsonData.info.description?.description) { + model_info.description = options.kmpJsonData.info.description.description.trim(); } // extract the language identifiers from the language metadata @@ -125,37 +104,32 @@ export function writeMergedModelMetadataFile( // and merge into a single array of identifiers in the // .model_info file. - model_info.languages = model_info.languages || options.kmpJsonData.lexicalModels.reduce((a, e) => [].concat(a, e.languages.map((f) => f.id)), []); + model_info.languages = options.kmpJsonData.lexicalModels.reduce((a, e) => [].concat(a, e.languages.map((f) => f.id)), []); - setModelMetadata('lastModifiedDate', (new Date).toISOString()); - setModelMetadata('packageFilename', callbacks.path.basename(options.kmpFileName)); + // TODO: use git date + model_info.lastModifiedDate = (new Date).toISOString(); - // Always overwrite with actual file size + model_info.packageFilename = callbacks.path.basename(options.kmpFileName); model_info.packageFileSize = callbacks.fileSize(options.kmpFileName); if(model_info.packageFileSize === undefined) { callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileDoesNotExist({filename:options.kmpFileName})); return null; } - setModelMetadata('jsFilename', callbacks.path.basename(options.modelFileName)); - - // Always overwrite with actual file size + model_info.jsFilename = callbacks.path.basename(options.modelFileName); model_info.jsFileSize = callbacks.fileSize(options.modelFileName); if(model_info.jsFileSize === undefined) { callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileDoesNotExist({filename:options.modelFileName})); return null; } - // Always overwrite source data model_info.packageIncludes = options.kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : []; - - setModelMetadata('version', options.kmpJsonData.info.version.description); - - // The minimum Keyman version detected in the package file may be manually set higher by the developer - setModelMetadata('minKeymanVersion', minKeymanVersion, false); + model_info.version = options.kmpJsonData.info.version.description; + model_info.minKeymanVersion = minKeymanVersion; + model_info.helpLink = HelpRoot + model_info.id; if(options.sourcePath) { - setModelMetadata('sourcePath', options.sourcePath); + model_info.sourcePath = options.sourcePath; } const jsonOutput = JSON.stringify(model_info, null, 2); diff --git a/developer/src/kmc-model-info/src/model-info-file.ts b/developer/src/kmc-model-info/src/model-info-file.ts index a70cfba20a0..c993242e349 100644 --- a/developer/src/kmc-model-info/src/model-info-file.ts +++ b/developer/src/kmc-model-info/src/model-info-file.ts @@ -7,7 +7,6 @@ export interface ModelInfoFile { license: "mit"; languages: Array; lastModifiedDate?: string; - links: ModelInfoFileLink[]; packageFilename?: string; packageFileSize?: number; jsFilename?: string; @@ -21,13 +20,7 @@ export interface ModelInfoFile { related?: {[id:string]:ModelInfoFileRelated}; } -export interface ModelInfoFileLink { - name: string; - url: string; -} - export interface ModelInfoFileRelated { deprecates?: string; deprecatedBy?: string; - note?: string; } \ No newline at end of file diff --git a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/build/sil.cmo.bw.model_info b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/build/sil.cmo.bw.model_info index a9f7bbe1548..1f8d5c03548 100644 --- a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/build/sil.cmo.bw.model_info +++ b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/build/sil.cmo.bw.model_info @@ -3,7 +3,8 @@ "languages": [ "cmo-Khmr" ], - "description": "Bunong Wordlist is based on the Bible wordlist of 1,157 unique entries with their frequencies.", + "description": "

Bunong Wordlist is based on the Bible wordlist of 1,157 unique entries with their frequencies.

", + "helpLink": "https://help.keyman.com/model/sil.cmo.bw", "id": "sil.cmo.bw", "name": "Bunong Wordlist", "authorName": "", diff --git a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/sil.cmo.bw.kpj b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/sil.cmo.bw.kpj index 12aca7bbf28..11d0f84b757 100644 --- a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/sil.cmo.bw.kpj +++ b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/sil.cmo.bw.kpj @@ -48,13 +48,6 @@ .md - - id_8146e45ce008c5c83d5a824f2121c46c - sil.cmo.bw.model_info - sil.cmo.bw.model_info - - .model_info - id_46ebc8fccf1bc10c92484653167449df sil.cmo.bw.model.js diff --git a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/sil.cmo.bw.model_info b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/sil.cmo.bw.model_info deleted file mode 100644 index c2d2b6070c5..00000000000 --- a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/sil.cmo.bw.model_info +++ /dev/null @@ -1,7 +0,0 @@ -{ - "license": "mit", - "languages": [ - "cmo-Khmr" - ], - "description": "Bunong Wordlist is based on the Bible wordlist of 1,157 unique entries with their frequencies." -} diff --git a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/source/sil.cmo.bw.model.kps b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/source/sil.cmo.bw.model.kps index 4fd452897cb..f53475848e7 100644 --- a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/source/sil.cmo.bw.model.kps +++ b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/source/sil.cmo.bw.model.kps @@ -19,6 +19,7 @@ © 2021 - 2022 SIL International 1.1 + Bunong Wordlist is based on the Bible wordlist of 1,157 unique entries with their frequencies. diff --git a/developer/src/kmc-model-info/test/test-model-info-compiler.ts b/developer/src/kmc-model-info/test/test-model-info-compiler.ts index 418b62955dc..ebfb82f4f89 100644 --- a/developer/src/kmc-model-info/test/test-model-info-compiler.ts +++ b/developer/src/kmc-model-info/test/test-model-info-compiler.ts @@ -3,7 +3,7 @@ import { assert } from 'chai'; import 'mocha'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; -import { writeMergedModelMetadataFile } from '../src/model-info-compiler.js'; +import { writeModelMetadataFile } from '../src/model-info-compiler.js'; import { KmpCompiler } from '@keymanapp/kmc-package'; const callbacks = new TestCompilerCallbacks(); @@ -14,7 +14,6 @@ beforeEach(function() { describe('model-info-compiler', function () { it('compile a .model_info file correctly', function() { - const path = makePathToFixture('sil.cmo.bw', 'sil.cmo.bw.model_info'); const kpsFileName = makePathToFixture('sil.cmo.bw', 'source', 'sil.cmo.bw.model.kps'); const kmpFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.kmp'); const buildModelInfoFilename = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model_info'); @@ -23,7 +22,7 @@ describe('model-info-compiler', function () { const kmpJsonData = kmpCompiler.transformKpsToKmpObject(kpsFileName); const modelFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.js'); - const data = writeMergedModelMetadataFile(path, callbacks, { + const data = writeModelMetadataFile(callbacks, { kmpFileName, kmpJsonData, model_id: 'sil.cmo.bw', @@ -41,8 +40,4 @@ describe('model-info-compiler', function () { assert.deepEqual(actual, expected); }); - - it('compile a .model_info file correctly when no source .model_info exists', function() { - this.skip(); // TODO: support model_info when no source file exists (determine license from LICENSE.md) - }); }); diff --git a/developer/src/kmc/README.md b/developer/src/kmc/README.md index f0c816bbb54..448f42e4b43 100644 --- a/developer/src/kmc/README.md +++ b/developer/src/kmc/README.md @@ -9,13 +9,11 @@ This package provides the following Keyman **command line tools**: file. - `kmlmp` — uses a `.model.kmp` file to generate a redistributable **lexical model package**. - - `kmlmi` — merges Keyman lexical model `.model_info` files. `kmlmc` is intended to be used standalone, or as part of a build system. `kmlmp` -is used only by command line tools. `kmlmi` is used exclusively in the -[lexical-models repository][lexical models]. +is used only by command line tools. -Note: `kmc` will in the future replace `kmlmc`, `kmlmp`, and `kmlmi`. +Note: `kmc` will in the future replace `kmlmc` and `kmlmp`. In order to build [lexical models][], these tools must be built and compiled. @@ -51,7 +49,6 @@ To see more command line options by using the `--help` option: kmlmc --help kmlmp --help - kmlmi --help How to build from source ------------------------ diff --git a/developer/src/kmc/build-bundler.js b/developer/src/kmc/build-bundler.js index e24ca39da0f..9b5f9cdd9bd 100644 --- a/developer/src/kmc/build-bundler.js +++ b/developer/src/kmc/build-bundler.js @@ -8,7 +8,6 @@ await esbuild.build({ entryPoints: [ 'build/src/kmc.js', 'build/src/kmlmc.js', - 'build/src/kmlmi.js', 'build/src/kmlmp.js', ], bundle: true, diff --git a/developer/src/kmc/package.json b/developer/src/kmc/package.json index b3ae21c078e..892acd93d73 100644 --- a/developer/src/kmc/package.json +++ b/developer/src/kmc/package.json @@ -11,10 +11,9 @@ ], "scripts": { "build": "tsc -b", - "bundle": "npm run bundle-kmc && npm run bundle-kmlmc && npm run bundle-kmlmi && npm run bundle-kmlmp", + "bundle": "npm run bundle-kmc && npm run bundle-kmlmc && npm run bundle-kmlmp", "bundle-kmc": "esbuild build/src/kmc.js --bundle --platform=node --target=es2022 > build/cjs-src/kmc.cjs", "bundle-kmlmc": "esbuild build/src/kmlmc.js --bundle --platform=node --target=es2022 > build/cjs-src/kmlmc.cjs", - "bundle-kmlmi": "esbuild build/src/kmlmi.js --bundle --platform=node --target=es2022 > build/cjs-src/kmlmi.cjs", "bundle-kmlmp": "esbuild build/src/kmlmp.js --bundle --platform=node --target=es2022 > build/cjs-src/kmlmp.cjs", "test": "eslint . && cd test && tsc -b && cd .. && mocha", "prepublishOnly": "npm run build" @@ -33,8 +32,7 @@ "bin": { "kmc": "build/src/kmc.js", "kmlmc": "build/src/kmlmc.js", - "kmlmp": "build/src/kmlmp.js", - "kmlmi": "build/src/kmlmi.js" + "kmlmp": "build/src/kmlmp.js" }, "dependencies": { "@keymanapp/common-types": "*", diff --git a/developer/src/kmc/src/commands/build.ts b/developer/src/kmc/src/commands/build.ts index 1047dd8208c..9568da70209 100644 --- a/developer/src/kmc/src/commands/build.ts +++ b/developer/src/kmc/src/commands/build.ts @@ -41,10 +41,6 @@ Supported file types: * .model.ts: Keyman lexical model * .kps: Keyman keyboard or lexical model package -The following two metadata file types are also supported: - * .model_info: lexical model metadata file - * .keyboard_info: keyboard metadata file - File lists can be referenced with @filelist.txt. If no input file is supplied, kmc will build the current folder.`) diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index e392f1c62c4..0af8343b2da 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -9,45 +9,34 @@ import { getLastGitCommitDate } from '../../util/getLastGitCommitDate.js'; export class BuildKeyboardInfo extends BuildActivity { public get name(): string { return 'Keyboard metadata'; } - public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.KeyboardInfo; } + public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.KeyboardInfo; } public get description(): string { return 'Build a keyboard metadata file'; } public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - if(KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.KeyboardInfo)) { - // We are given a .keyboard_info but need to use the project file in the - // same folder, so that we can find the related files. This also supports - // version 2.0 projects (where the .kpj file is optional). - infile = KeymanFileTypes.replaceExtension(infile, KeymanFileTypes.Source.KeyboardInfo, KeymanFileTypes.Source.Project); - } - - if(!callbacks.fs.existsSync(infile)) { - // We cannot build a .keyboard_info if we don't have a repository-style project - return false; + if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { + // Even if the project file does not exist, we use its name as our reference + // in order to avoid ambiguity + throw new Error(`BuildKeyboardInfo called with unexpected file type ${infile}`); } const project = loadProject(infile, callbacks); if(!project) { - return false; - } - - const metadata = findProjectFile(callbacks, project, KeymanFileTypes.Source.KeyboardInfo); - if(!metadata) { - // Project loader should always have added a metadata file + // Error messages written by loadProject return false; } const keyboard = findProjectFile(callbacks, project, KeymanFileTypes.Source.KeymanKeyboard); const kps = findProjectFile(callbacks, project, KeymanFileTypes.Source.Package); if(!keyboard || !kps) { + // Error messages written by findProjectFile return false; } - const jsFilename = project.resolveOutputFilePath(keyboard, KeymanFileTypes.Source.KeymanKeyboard, KeymanFileTypes.Binary.WebKeyboard); - const lastCommitDate = getLastGitCommitDate(callbacks.path.dirname(project.resolveInputFilePath(metadata))); + const lastCommitDate = getLastGitCommitDate(project.projectPath); const compiler = new KeyboardInfoCompiler(callbacks); - const data = compiler.writeMergedKeyboardInfoFile({ + const data = compiler.writeKeyboardInfoFile({ kmpFilename: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package), kpsFilename: project.resolveInputFilePath(kps), jsFilename: fs.existsSync(jsFilename) ? jsFilename : undefined, @@ -60,8 +49,10 @@ export class BuildKeyboardInfo extends BuildActivity { return false; } + const outputFilename = project.getOutputFilePath(KeymanFileTypes.Binary.KeyboardInfo); + fs.writeFileSync( - project.resolveOutputFilePath(metadata, KeymanFileTypes.Source.KeyboardInfo, KeymanFileTypes.Binary.KeyboardInfo), + outputFilename, data ); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 9dd4d476e7c..92a6a08c72a 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; -import { writeMergedModelMetadataFile } from '@keymanapp/kmc-model-info'; +import { writeModelMetadataFile } from '@keymanapp/kmc-model-info'; import { KmpCompiler } from '@keymanapp/kmc-package'; import { loadProject } from '../../util/projectLoader.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; @@ -9,7 +9,7 @@ import { calculateSourcePath } from '../../util/calculateSourcePath.js'; export class BuildModelInfo extends BuildActivity { public get name(): string { return 'Lexical model metadata'; } - public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.ModelInfo; } + public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.ModelInfo; } public get description(): string { return 'Build a lexical model metadata file'; } @@ -24,23 +24,18 @@ export class BuildModelInfo extends BuildActivity { * @returns */ public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { - if(KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.ModelInfo)) { - // We are given a .model_info but need to use the project file in the - // same folder, so that we can find the related files. - infile = KeymanFileTypes.replaceExtension(infile, KeymanFileTypes.Source.ModelInfo, KeymanFileTypes.Source.Project); + if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { + // Even if the project file does not exist, we use its name as our reference + // in order to avoid ambiguity + throw new Error(`BuildModelInfo called with unexpected file type ${infile}`); } + const project = loadProject(infile, callbacks); if(!project) { // Error messages will be reported by loadProject return false; } - const metadata = project.files.find(file => file.fileType == KeymanFileTypes.Source.ModelInfo); - if(!metadata) { - callbacks.reportMessage(InfrastructureMessages.Error_FileTypeNotFound({ext: KeymanFileTypes.Source.ModelInfo})); - return false; - } - const model = project.files.find(file => file.fileType == KeymanFileTypes.Source.Model); if(!model) { callbacks.reportMessage(InfrastructureMessages.Error_FileTypeNotFound({ext: KeymanFileTypes.Source.Model})); @@ -60,11 +55,10 @@ export class BuildModelInfo extends BuildActivity { return false; } - const data = writeMergedModelMetadataFile( - project.resolveInputFilePath(metadata), + const data = writeModelMetadataFile( callbacks, { - model_id: callbacks.path.basename(metadata.filename, KeymanFileTypes.Source.ModelInfo), + model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project), kmpJsonData, sourcePath: calculateSourcePath(infile), modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model), @@ -73,12 +67,12 @@ export class BuildModelInfo extends BuildActivity { ); if(data == null) { - // Error messages have already been emitted by writeMergedModelMetadataFile + // Error messages have already been emitted by writeModelMetadataFile return false; } fs.writeFileSync( - project.resolveOutputFilePath(metadata, KeymanFileTypes.Source.ModelInfo, KeymanFileTypes.Binary.ModelInfo), + project.getOutputFilePath(KeymanFileTypes.Binary.ModelInfo), data ); diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index fdced30e021..380be1b7245 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs'; import { CompilerCallbacks, CompilerFileCallbacks, CompilerOptions, KeymanDeveloperProject, KeymanDeveloperProjectFile, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; -import { buildActivities } from './buildActivities.js'; +import { buildActivities, buildKeyboardInfoActivity, buildModelInfoActivity } from './buildActivities.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; import { loadProject } from '../../util/projectLoader.js'; @@ -47,11 +47,17 @@ class ProjectBuilder { continue; } - if(KeymanFileTypes.filenameIsMetadata(builder.sourceExtension) && this.project.options.skipMetadataFiles) { - continue; + if(!await this.buildProjectTargets(builder)) { + return false; } + } - if(!await this.buildProjectTargets(builder)) { + // Build project metadata + if(!this.project.options.skipMetadataFiles) { + if(!await (this.buildProjectTargets( + this.project.isKeyboardProject() + ? buildKeyboardInfoActivity + : buildModelInfoActivity))) { return false; } } @@ -60,6 +66,10 @@ class ProjectBuilder { } async buildProjectTargets(activity: BuildActivity): Promise { + if(activity.sourceExtension == KeymanFileTypes.Source.Project) { + return await this.buildTarget(this.project.projectFile, activity); + } + let result = true; for(let file of this.project.files) { if(file.fileType.toLowerCase() == activity.sourceExtension) { diff --git a/developer/src/kmc/src/commands/buildClasses/buildActivities.ts b/developer/src/kmc/src/commands/buildClasses/buildActivities.ts index aae271fbacb..d8d1a9b5d55 100644 --- a/developer/src/kmc/src/commands/buildClasses/buildActivities.ts +++ b/developer/src/kmc/src/commands/buildClasses/buildActivities.ts @@ -13,10 +13,13 @@ export const buildActivities: BuildActivity[] = [ new BuildLdmlKeyboard(), new BuildModel(), new BuildPackage(), - new BuildKeyboardInfo(), - new BuildModelInfo(), ]; +// These are built from the .kpj reference after all others +export const buildKeyboardInfoActivity = new BuildKeyboardInfo(); +export const buildModelInfoActivity = new BuildModelInfo(); + + // Note: BuildProject is not listed here to avoid circular references, // because it depends on the other activities here. This means that // BuildProject must be separately checked. diff --git a/developer/src/kmc/src/kmlmi.ts b/developer/src/kmc/src/kmlmi.ts deleted file mode 100644 index 219032d0446..00000000000 --- a/developer/src/kmc/src/kmlmi.ts +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env node -/** - * kmlmi - Keyman Lexical Model model_info Compiler - */ - -import * as fs from 'fs'; -import * as path from 'path'; -import { Command } from 'commander'; -import { KmpCompiler, PackageValidation } from '@keymanapp/kmc-package'; -import { ModelInfoOptions, writeMergedModelMetadataFile } from '@keymanapp/kmc-model-info'; -import { SysExits } from './util/sysexits.js'; -import KEYMAN_VERSION from "@keymanapp/keyman-version"; -import { NodeCompilerCallbacks } from './util/NodeCompilerCallbacks.js'; - -let inputFilename: string; -const program = new Command(); - -/* Arguments */ -program - .description('Merges Keyman lexical model model_info files. Intended for use within the keymanapp/lexical-models repository.') - .version(KEYMAN_VERSION.VERSION_WITH_TAG) - .arguments('') - .action(infile => inputFilename = infile) - .option('-o, --outFile ', 'where to save the resultant file') - .option('-m, --model ', 'model id, defaults to basename of input file sans .model_info extension') - .option('-s, --source ', 'path to source of model, relative to lexical-models repo root') - .option('--kpsFilename ', 'path to .model.kps file, defaults to source/.model.kps') - .option('--kmpFilename ', 'path to .model.kmp file, defaults to build/.model.kmp') - .option('--jsFilename ', 'path to .model.js file, defaults to build/.model.js'); - -program.parse(process.argv); - -// Deal with input arguments: - -if (!inputFilename) { - exitDueToUsageError('Must provide a lexical model .model_info source file.'); -} - -let model_id: string = program.opts().model ? program.opts().model : path.basename(inputFilename).replace(/\.model_info$/, ""); -let outputFilename: string = program.opts().outFile ? program.opts().outFile : path.join(path.dirname(inputFilename), 'build', path.basename(inputFilename)); -let kpsFilename = program.opts().kpsFilename ? program.opts().kpsFilename : path.join(path.dirname(inputFilename), 'source', path.basename(inputFilename).replace(/\.model_info$/, '.model.kps')); -let kmpFilename = program.opts().kmpFilename ? program.opts().kmpFilename : path.join(path.dirname(inputFilename), 'build', path.basename(inputFilename).replace(/\.model_info$/, '.model.kmp')); -let jsFilename = program.opts().jsFilename ? program.opts().jsFilename : path.join(path.dirname(inputFilename), 'build', path.basename(inputFilename).replace(/\.model_info$/, '.model.js')); - -// -// Load .kps source data -// - -const callbacks = new NodeCompilerCallbacks({logLevel: 'info'}); -let kmpCompiler = new KmpCompiler(callbacks); -let kmpJsonData = kmpCompiler.transformKpsToKmpObject(kpsFilename); -if(!kmpJsonData) { - process.exit(SysExits.EX_DATAERR); -} - -// -// Validate the package file -// - -const validation = new PackageValidation(callbacks, {}); -if(!validation.validate(kpsFilename, kmpJsonData)) { - process.exit(SysExits.EX_DATAERR); -} - -// -// Write out the merged .model_info file -// - -let modelInfoOptions: ModelInfoOptions = { - model_id: model_id, - kmpJsonData: kmpJsonData, - sourcePath: program.opts().source, - modelFileName: jsFilename, - kmpFileName: kmpFilename -}; - -const data = writeMergedModelMetadataFile( - inputFilename, - callbacks, - modelInfoOptions); -if(!data) { - process.exit(SysExits.EX_DATAERR); -} -fs.writeFileSync(outputFilename, data); - -function exitDueToUsageError(message: string): never { - console.error(`${program.name()}: ${message}`); - console.error(); - program.outputHelp(); - return process.exit(SysExits.EX_USAGE); -} diff --git a/developer/src/tike/Makefile b/developer/src/tike/Makefile index ce20a3673c6..c6a8cb16044 100644 --- a/developer/src/tike/Makefile +++ b/developer/src/tike/Makefile @@ -11,7 +11,6 @@ build: version.res manifest.res icons dirs xml xsd pull-core $(TDS2DBG) $(WIN32_TARGET_PATH)\tike.exe $(COPY) $(WIN32_TARGET_PATH)\tike.exe $(DEVELOPER_PROGRAM) $(COPY) kmlmc.cmd $(DEVELOPER_PROGRAM) - $(COPY) kmlmi.cmd $(DEVELOPER_PROGRAM) $(COPY) kmlmp.cmd $(DEVELOPER_PROGRAM) $(COPY) kmc.cmd $(DEVELOPER_PROGRAM) $(COPY) $(KEYMAN_ROOT)\core\build\x86\$(TARGET_PATH)\src\kmnkbp0-0.dll $(DEVELOPER_PROGRAM) diff --git a/developer/src/tike/kmlmi.cmd b/developer/src/tike/kmlmi.cmd deleted file mode 100644 index 32f6a256a95..00000000000 --- a/developer/src/tike/kmlmi.cmd +++ /dev/null @@ -1,20 +0,0 @@ -@echo off -setlocal -rem This script is based on /developer/src/node/inst/kmlmi.cmd. It is stored here -rem in order to allow TIKE to call out to the compiler while debugging. -if exist "%~dp0..\inst\node\dist\node.exe" ( - rem If running in developer/src/tike/: - set nodeexe="%~dp0..\inst\node\dist\node.exe" - set nodecli="%~dp0..\kmc\build\src\kmlmi.js" -) else if exist "%~dp0..\src\inst\node\dist\node.exe" ( - rem If running in developer/bin/: - set nodeexe="%~dp0..\src\inst\node\dist\node.exe" - set nodecli="%~dp0..\src\kmc\build\src\kmlmi.js" -) else ( - rem Cannot find node or kmlmi.js relative to execution path - echo Error: node.exe or kmlmi.js not found. - exit /b 1 -) - -%nodeexe% --enable-source-maps %nodecli% %* -exit /b %errorlevel% diff --git a/web/src/test/manual/web/issue2924/source/dmg.dv.test/dmg.dv.test.kpj b/web/src/test/manual/web/issue2924/source/dmg.dv.test/dmg.dv.test.kpj index e4d5c9c567a..855646cf269 100644 --- a/web/src/test/manual/web/issue2924/source/dmg.dv.test/dmg.dv.test.kpj +++ b/web/src/test/manual/web/issue2924/source/dmg.dv.test/dmg.dv.test.kpj @@ -48,13 +48,6 @@ .md - - id_4b12eec2a4d0bfc7b5441a4a59691eab - dmg.dv.test.model_info - dmg.dv.test.model_info - - .model_info - id_6ee5fa96871fb96ecebf2cbe9ec7425b dmg.dv.test.model.js diff --git a/web/src/test/manual/web/issue2924/source/dmg.dv.test/dmg.dv.test.model_info b/web/src/test/manual/web/issue2924/source/dmg.dv.test/dmg.dv.test.model_info deleted file mode 100644 index 03399a9f8e5..00000000000 --- a/web/src/test/manual/web/issue2924/source/dmg.dv.test/dmg.dv.test.model_info +++ /dev/null @@ -1,7 +0,0 @@ -{ - "license": "mit", - "languages": [ - "dv" - ], - "description": "test generated from template" -} From 232c918c314a859823b957938995efa8a7ec92b1 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 29 Aug 2023 16:37:37 +1000 Subject: [PATCH 2/8] feat(developer): support isRTL and license fields in .model_info Adds support for calculating isRTL and license fields in .model_info compiler. Also: * establishes @keymanapp/developer-utils shared module * moves license validation into @keymanapp/developer-utils * refactors kmc-model-info to a class and general cleanup --- developer/build.sh | 1 + developer/src/common/web/utils/.gitignore | 1 + developer/src/common/web/utils/build.sh | 36 +++ developer/src/common/web/utils/index.ts | 1 + developer/src/common/web/utils/package.json | 32 +++ .../web/utils}/src/validate-mit-license.ts | 0 .../license/LICENSE-changed-clause.md | 0 .../license/LICENSE-missing-copyright.md | 0 .../fixtures/license/LICENSE-missing-title.md | 0 .../test/fixtures/license/LICENSE-too-long.md | 0 .../fixtures/license/LICENSE-too-short.md | 0 .../utils}/test/fixtures/license/LICENSE.md | 0 .../common/web/utils/test/helpers/index.ts | 18 ++ .../web/utils}/test/test-license.ts | 0 .../src/common/web/utils/test/tsconfig.json | 24 ++ developer/src/common/web/utils/tsconfig.json | 18 ++ developer/src/kmc-keyboard-info/package.json | 3 +- developer/src/kmc-keyboard-info/src/index.ts | 2 +- developer/src/kmc-keyboard-info/tsconfig.json | 2 + developer/src/kmc-model-info/package.json | 3 +- developer/src/kmc-model-info/src/messages.ts | 15 ++ .../kmc-model-info/src/model-info-compiler.ts | 254 +++++++++++------- .../src/kmc-model-info/src/model-info-file.ts | 2 +- .../test/fixtures/sil.cmo.bw/LICENSE.md | 2 +- .../sil.cmo.bw/source/sil.cmo.bw.model.kps | 7 + .../test/test-model-info-compiler.ts | 14 +- developer/src/kmc-model-info/tsconfig.json | 2 + .../commands/buildClasses/BuildModelInfo.ts | 5 + package.json | 2 + tsconfig.esm.json | 2 + 30 files changed, 345 insertions(+), 101 deletions(-) create mode 100644 developer/src/common/web/utils/.gitignore create mode 100755 developer/src/common/web/utils/build.sh create mode 100644 developer/src/common/web/utils/index.ts create mode 100644 developer/src/common/web/utils/package.json rename developer/src/{kmc-keyboard-info => common/web/utils}/src/validate-mit-license.ts (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/test/fixtures/license/LICENSE-changed-clause.md (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/test/fixtures/license/LICENSE-missing-copyright.md (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/test/fixtures/license/LICENSE-missing-title.md (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/test/fixtures/license/LICENSE-too-long.md (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/test/fixtures/license/LICENSE-too-short.md (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/test/fixtures/license/LICENSE.md (100%) create mode 100644 developer/src/common/web/utils/test/helpers/index.ts rename developer/src/{kmc-keyboard-info => common/web/utils}/test/test-license.ts (100%) create mode 100644 developer/src/common/web/utils/test/tsconfig.json create mode 100644 developer/src/common/web/utils/tsconfig.json diff --git a/developer/build.sh b/developer/build.sh index 2315cd3b34c..47cc05f533b 100755 --- a/developer/build.sh +++ b/developer/build.sh @@ -27,6 +27,7 @@ builder_describe \ configure \ build \ test \ + ":utils=src/common/web/utils Developer utils" \ ":kmcmplib=src/kmcmplib Compiler - .kmn compiler" \ ":kmc-analyze=src/kmc-analyze Compiler - Analysis Tools" \ ":kmc-keyboard-info=src/kmc-keyboard-info Compiler - .keyboard_info Module" \ diff --git a/developer/src/common/web/utils/.gitignore b/developer/src/common/web/utils/.gitignore new file mode 100644 index 00000000000..d16386367f7 --- /dev/null +++ b/developer/src/common/web/utils/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/developer/src/common/web/utils/build.sh b/developer/src/common/web/utils/build.sh new file mode 100755 index 00000000000..d0da84b56fe --- /dev/null +++ b/developer/src/common/web/utils/build.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../../../resources/build/build-utils.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +cd "$THIS_SCRIPT_PATH" + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +builder_describe "Build Keyman Developer web utility module" \ + "@/common/web/types" \ + "clean" \ + "configure" \ + "build" \ + "test" + +builder_describe_outputs \ + configure /node_modules \ + build /developer/src/common/web/utils/build/index.js + +builder_parse "$@" + +#------------------------------------------------------------------------------------------------------------------- + +builder_run_action clean rm -rf ./build/ +builder_run_action configure verify_npm_setup +builder_run_action build tsc --build + +if builder_start_action test; then + eslint . + tsc --build test + c8 --reporter=lcov --reporter=text --exclude-after-remap mocha + builder_finish_action success test +fi diff --git a/developer/src/common/web/utils/index.ts b/developer/src/common/web/utils/index.ts new file mode 100644 index 00000000000..f18f2e90de5 --- /dev/null +++ b/developer/src/common/web/utils/index.ts @@ -0,0 +1 @@ +export { validateMITLicense } from './src/validate-mit-license.js'; \ No newline at end of file diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json new file mode 100644 index 00000000000..277066b0464 --- /dev/null +++ b/developer/src/common/web/utils/package.json @@ -0,0 +1,32 @@ +{ + "name": "@keymanapp/developer-utils", + "description": "Keyman Developer utilities", + "type": "module", + "exports": { + ".": "./build/index.js" + }, + "files": [ + "/build/" + ], + "devDependencies": { + "@types/node": "^20.4.1", + "ts-node": "^9.1.1", + "typescript": "^4.9.5", + "c8": "^7.12.0", + "chai": "^4.3.4", + "mocha": "^8.4.0" + }, + "scripts": { + "build": "tsc -b" + }, + "license": "MIT", + "dependencies": { + "@keymanapp/common-types": "*" + }, + "mocha": { + "spec": "build/test/**/test-*.js", + "require": [ + "source-map-support/register" + ] + } +} diff --git a/developer/src/kmc-keyboard-info/src/validate-mit-license.ts b/developer/src/common/web/utils/src/validate-mit-license.ts similarity index 100% rename from developer/src/kmc-keyboard-info/src/validate-mit-license.ts rename to developer/src/common/web/utils/src/validate-mit-license.ts diff --git a/developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-changed-clause.md b/developer/src/common/web/utils/test/fixtures/license/LICENSE-changed-clause.md similarity index 100% rename from developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-changed-clause.md rename to developer/src/common/web/utils/test/fixtures/license/LICENSE-changed-clause.md diff --git a/developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-missing-copyright.md b/developer/src/common/web/utils/test/fixtures/license/LICENSE-missing-copyright.md similarity index 100% rename from developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-missing-copyright.md rename to developer/src/common/web/utils/test/fixtures/license/LICENSE-missing-copyright.md diff --git a/developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-missing-title.md b/developer/src/common/web/utils/test/fixtures/license/LICENSE-missing-title.md similarity index 100% rename from developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-missing-title.md rename to developer/src/common/web/utils/test/fixtures/license/LICENSE-missing-title.md diff --git a/developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-too-long.md b/developer/src/common/web/utils/test/fixtures/license/LICENSE-too-long.md similarity index 100% rename from developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-too-long.md rename to developer/src/common/web/utils/test/fixtures/license/LICENSE-too-long.md diff --git a/developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-too-short.md b/developer/src/common/web/utils/test/fixtures/license/LICENSE-too-short.md similarity index 100% rename from developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE-too-short.md rename to developer/src/common/web/utils/test/fixtures/license/LICENSE-too-short.md diff --git a/developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE.md b/developer/src/common/web/utils/test/fixtures/license/LICENSE.md similarity index 100% rename from developer/src/kmc-keyboard-info/test/fixtures/license/LICENSE.md rename to developer/src/common/web/utils/test/fixtures/license/LICENSE.md diff --git a/developer/src/common/web/utils/test/helpers/index.ts b/developer/src/common/web/utils/test/helpers/index.ts new file mode 100644 index 00000000000..9968fbfea6d --- /dev/null +++ b/developer/src/common/web/utils/test/helpers/index.ts @@ -0,0 +1,18 @@ +/** + * Helpers and utilities for the Mocha tests. + */ +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +/** + * Builds a path to the fixture with the given path components. + * + * e.g., makePathToFixture('example.qaa.trivial') + * e.g., makePathToFixture('example.qaa.trivial', 'model.ts') + * + * @param components One or more path components. + */ + export function makePathToFixture(...components: string[]): string { + return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url)); +} + diff --git a/developer/src/kmc-keyboard-info/test/test-license.ts b/developer/src/common/web/utils/test/test-license.ts similarity index 100% rename from developer/src/kmc-keyboard-info/test/test-license.ts rename to developer/src/common/web/utils/test/test-license.ts diff --git a/developer/src/common/web/utils/test/tsconfig.json b/developer/src/common/web/utils/test/tsconfig.json new file mode 100644 index 00000000000..a28eeb03ab2 --- /dev/null +++ b/developer/src/common/web/utils/test/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../../kmc/tsconfig.kmc-base.json", + + "compilerOptions": { + "rootDir": ".", + "rootDirs": ["./", "../src/"], + "outDir": "../build/test", + "esModuleInterop": true, + "moduleResolution": "node16", + "allowSyntheticDefaultImports": true, + "baseUrl": ".", + "paths": { + "@keymanapp/developer-test-helpers": ["../../test-helpers/index"], + }, + }, + "include": [ + "**/test-*.ts", + "helpers/*.ts", + ], + "references": [ + { "path": "../" }, + { "path": "../../test-helpers/" }, + ] +} \ No newline at end of file diff --git a/developer/src/common/web/utils/tsconfig.json b/developer/src/common/web/utils/tsconfig.json new file mode 100644 index 00000000000..dbb068a7a22 --- /dev/null +++ b/developer/src/common/web/utils/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../../../tsconfig.esm-base.json", + + "compilerOptions": { + "rootDir": ".", + "outDir": "./build/", + "baseUrl": ".", + "paths": { + "@keymanapp/common-types": ["../../../../../common/web/types/src/main"], + }, + }, + "include": [ + "./*.ts", "src/validate-mit-license.ts", + ], + "references": [ + { "path": "../../../../../common/web/types/" }, + ] +} \ No newline at end of file diff --git a/developer/src/kmc-keyboard-info/package.json b/developer/src/kmc-keyboard-info/package.json index ad3572ec1e8..7b028ec1294 100644 --- a/developer/src/kmc-keyboard-info/package.json +++ b/developer/src/kmc-keyboard-info/package.json @@ -25,7 +25,8 @@ }, "dependencies": { "@keymanapp/common-types": "*", - "@keymanapp/kmc-package": "*" + "@keymanapp/kmc-package": "*", + "@keymanapp/developer-utils": "*" }, "devDependencies": { "@types/chai": "^4.3.5", diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts index 0c0876cf247..a677f5f5a9f 100644 --- a/developer/src/kmc-keyboard-info/src/index.ts +++ b/developer/src/kmc-keyboard-info/src/index.ts @@ -8,7 +8,7 @@ import { KeyboardInfoFile, KeyboardInfoFileIncludes, KeyboardInfoFilePlatform } import { KeymanFileTypes, CompilerCallbacks, KmpJsonFile, KmxFileReader, KMX, KeymanTargets } from "@keymanapp/common-types"; import { KeyboardInfoCompilerMessages } from "./messages.js"; import langtags from "./imports/langtags.js"; -import { validateMITLicense } from "./validate-mit-license.js"; +import { validateMITLicense } from "@keymanapp/developer-utils"; import { KmpCompiler } from "@keymanapp/kmc-package"; const regionNames = new Intl.DisplayNames(['en'], { type: "region" }); diff --git a/developer/src/kmc-keyboard-info/tsconfig.json b/developer/src/kmc-keyboard-info/tsconfig.json index c10b82f8637..bf1ad29bd95 100644 --- a/developer/src/kmc-keyboard-info/tsconfig.json +++ b/developer/src/kmc-keyboard-info/tsconfig.json @@ -13,6 +13,7 @@ "paths": { "@keymanapp/common-types": ["../../../common/web/types/src/main"], "@keymanapp/kmc-package": ["../kmc-package/source"], + "@keymanapp/developer-utils": ["../common/web/utils/index"], } }, "include": [ @@ -22,5 +23,6 @@ "references": [ { "path": "../../../common/web/types" }, { "path": "../kmc-package/" }, + { "path": "../common/web/utils/" }, ] } diff --git a/developer/src/kmc-model-info/package.json b/developer/src/kmc-model-info/package.json index 624be5ff807..e0499ad0e60 100644 --- a/developer/src/kmc-model-info/package.json +++ b/developer/src/kmc-model-info/package.json @@ -32,7 +32,8 @@ }, "dependencies": { "@keymanapp/common-types": "*", - "@keymanapp/models-types": "*" + "@keymanapp/models-types": "*", + "@keymanapp/developer-utils": "*" }, "devDependencies": { "@types/chai": "^4.1.7", diff --git a/developer/src/kmc-model-info/src/messages.ts b/developer/src/kmc-model-info/src/messages.ts index a4d61e8b232..03dce78613a 100644 --- a/developer/src/kmc-model-info/src/messages.ts +++ b/developer/src/kmc-model-info/src/messages.ts @@ -26,5 +26,20 @@ export class ModelInfoCompilerMessages { `Invalid author email: ${o.email}`); static ERROR_InvalidAuthorEmail = SevError | 0x0005; + static Error_LicenseFileDoesNotExist = (o:{filename:string}) => m(this.ERROR_LicenseFileIsMissing, + `License file ${o.filename} does not exist.`); + static ERROR_LicenseFileIsMissing = SevError | 0x0006; + + static Error_LicenseFileIsDamaged = (o:{filename:string}) => m(this.ERROR_LicenseFileIsDamaged, + `License file ${o.filename} could not be loaded or decoded.`); + static ERROR_LicenseFileIsDamaged = SevError | 0x0007; + + static Error_LicenseIsNotValid = (o:{filename:string,message:string}) => m(this.ERROR_LicenseIsNotValid, + `An error was encountered parsing license file ${o.filename}: ${o.message}.`); + static ERROR_LicenseIsNotValid = SevError | 0x0008; + + static Error_NoLicenseFound = () => m(this.ERROR_NoLicenseFound, + `No license for the model was found. MIT license is required for publication to Keyman lexical-models repository.`); + static ERROR_NoLicenseFound = SevError | 0x0009; } diff --git a/developer/src/kmc-model-info/src/model-info-compiler.ts b/developer/src/kmc-model-info/src/model-info-compiler.ts index 04ebf6cef8d..4563963aa13 100644 --- a/developer/src/kmc-model-info/src/model-info-compiler.ts +++ b/developer/src/kmc-model-info/src/model-info-compiler.ts @@ -7,11 +7,12 @@ import { minKeymanVersion } from "./min-keyman-version.js"; import { ModelInfoFile } from "./model-info-file.js"; import { CompilerCallbacks, KmpJsonFile } from "@keymanapp/common-types"; import { ModelInfoCompilerMessages } from "./messages.js"; +import { validateMITLicense } from "@keymanapp/developer-utils"; const HelpRoot = 'https://help.keyman.com/model/'; /* c8 ignore start */ -export class ModelInfoOptions { +export class ModelInfoSources { /** The identifier for the model */ model_id: string; @@ -19,119 +20,190 @@ export class ModelInfoOptions { kmpJsonData: KmpJsonFile.KmpJsonFile; /** The path in the keymanapp/lexical-models repo where this model may be found (optional) */ - sourcePath?: string; + sourcePath: string; /** The compiled model filename and relative path (.js) */ modelFileName: string; /** The compiled package filename and relative path (.kmp) */ kmpFileName: string; + + /** The source package filename and relative path (.kps) */ + kpsFilename: string; + + /** Last modification date for files in the project folder 'YYYY-MM-DDThh:mm:ssZ' */ + lastCommitDate?: string; }; /* c8 ignore stop */ -/** - * Builds .model_info file with metadata from the model and package source file. - * This function is intended for use within the lexical-models repository. While many of the - * parameters could be deduced from each other, they are specified here to reduce the - * number of places the filenames are constructed. - * - * @param callbacks - * @param options Details on files from which to extract additional metadata - */ -export function writeModelMetadataFile( - callbacks: CompilerCallbacks, - options: ModelInfoOptions +export class ModelInfoCompiler { + constructor(private callbacks: CompilerCallbacks) { + } + + /** + * Builds .model_info file with metadata from the model and package source file. + * This function is intended for use within the lexical-models repository. While many of the + * parameters could be deduced from each other, they are specified here to reduce the + * number of places the filenames are constructed. + * + * @param sources Details on files from which to extract additional metadata + */ + writeModelMetadataFile( + sources: ModelInfoSources ): Uint8Array { - /* - * Model info looks like this: - * - * { - * "name": "Example Template Model" - * "license": "mit", - * "version": "1.0.0", - * "languages": ["en"], - * "authorName": "Example Author", - * "authorEmail": "nobody@example.com", - * "description": "Example wordlist model" - * } - * - * For full documentation, see: - * https://help.keyman.com/developer/cloud/model_info/1.0/ - */ - - let model_info: ModelInfoFile = { - languages: [], - license: 'mit' - }; - - // - // Build .model_info file -- some fields have "special" behaviours -- see below - // https://api.keyman.com/schemas/model_info.source.json and - // https://api.keyman.com/schemas/model_info.distribution.json - // https://help.keyman.com/developer/cloud/model_info/1.0 - // - - // TODO: isrtl - // TODO: license - - model_info.id = options.model_id; - model_info.name = options.kmpJsonData.info.name.description; - - const author = options.kmpJsonData.info.author; - model_info.authorName = author?.description ?? ''; - - if (author?.url) { - // we strip the mailto: from the .kps file for the .model_info - const match = author.url.match(/^(mailto\:)?(.+)$/); - /* c8 ignore next 3 */ - if (match === null) { - callbacks.reportMessage(ModelInfoCompilerMessages.Error_InvalidAuthorEmail({email:author.url})); + /* + * Model info looks like this: + * + * { + * "name": "Example Template Model" + * "license": "mit", + * "version": "1.0.0", + * "languages": ["en"], + * "authorName": "Example Author", + * "authorEmail": "nobody@example.com", + * "description": "Example wordlist model" + * } + * + * For full documentation, see: + * https://help.keyman.com/developer/cloud/model_info/1.0/ + */ + + let jsFile: string = null; + + if(sources.modelFileName) { + jsFile = this.loadJsFile(sources.modelFileName); + if(!jsFile) { + return null; + } + } + + + let model_info: ModelInfoFile = { + languages: [], + }; + + // + // Build .model_info file -- some fields have "special" behaviours -- see below + // https://api.keyman.com/schemas/model_info.source.json and + // https://api.keyman.com/schemas/model_info.distribution.json + // https://help.keyman.com/developer/cloud/model_info/1.0 + // + + model_info.id = sources.model_id; + model_info.name = sources.kmpJsonData.info.name.description; + + // License + + if(!sources.kmpJsonData.options?.licenseFile) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_NoLicenseFound()); return null; } - model_info.authorEmail = match[2]; - } + if(!this.isLicenseMIT(this.callbacks.resolveFilename(sources.kpsFilename, sources.kmpJsonData.options.licenseFile))) { + return null; + } - // description + model_info.license = 'mit'; - if(options.kmpJsonData.info.description?.description) { - model_info.description = options.kmpJsonData.info.description.description.trim(); - } + const author = sources.kmpJsonData.info.author; + model_info.authorName = author?.description ?? ''; - // extract the language identifiers from the language metadata - // arrays for each of the lexical models in the kmp.json file, - // and merge into a single array of identifiers in the - // .model_info file. + if (author?.url) { + // we strip the mailto: from the .kps file for the .model_info + const match = author.url.match(/^(mailto\:)?(.+)$/); + /* c8 ignore next 3 */ + if (match === null) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_InvalidAuthorEmail({email:author.url})); + return null; + } - model_info.languages = options.kmpJsonData.lexicalModels.reduce((a, e) => [].concat(a, e.languages.map((f) => f.id)), []); + model_info.authorEmail = match[2]; + } - // TODO: use git date - model_info.lastModifiedDate = (new Date).toISOString(); + // description - model_info.packageFilename = callbacks.path.basename(options.kmpFileName); - model_info.packageFileSize = callbacks.fileSize(options.kmpFileName); - if(model_info.packageFileSize === undefined) { - callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileDoesNotExist({filename:options.kmpFileName})); - return null; - } + if(sources.kmpJsonData.info.description?.description) { + model_info.description = sources.kmpJsonData.info.description.description.trim(); + } + + // isRTL -- this is a little bit of a heuristic from a compiled .js + // which may need modification if compilers change - model_info.jsFilename = callbacks.path.basename(options.modelFileName); - model_info.jsFileSize = callbacks.fileSize(options.modelFileName); - if(model_info.jsFileSize === undefined) { - callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileDoesNotExist({filename:options.modelFileName})); - return null; + if(jsFile?.match(/("?)isRTL("?):\s*true/)) { + model_info.isRTL = true; + } + + // extract the language identifiers from the language metadata + // arrays for each of the lexical models in the kmp.json file, + // and merge into a single array of identifiers in the + // .model_info file. + + model_info.languages = sources.kmpJsonData.lexicalModels.reduce((a, e) => [].concat(a, e.languages.map((f) => f.id)), []); + + // If a last commit date is not given, then just use the current time + model_info.lastModifiedDate = sources.lastCommitDate ?? (new Date).toISOString(); + + model_info.packageFilename = this.callbacks.path.basename(sources.kmpFileName); + model_info.packageFileSize = this.callbacks.fileSize(sources.kmpFileName); + if(model_info.packageFileSize === undefined) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileDoesNotExist({filename:sources.kmpFileName})); + return null; + } + + model_info.jsFilename = this.callbacks.path.basename(sources.modelFileName); + model_info.jsFileSize = this.callbacks.fileSize(sources.modelFileName); + if(model_info.jsFileSize === undefined) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileDoesNotExist({filename:sources.modelFileName})); + return null; + } + + model_info.packageIncludes = sources.kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : []; + model_info.version = sources.kmpJsonData.info.version.description; + model_info.minKeymanVersion = minKeymanVersion; + model_info.helpLink = HelpRoot + model_info.id; + + if(sources.sourcePath) { + model_info.sourcePath = sources.sourcePath; + } + + const jsonOutput = JSON.stringify(model_info, null, 2); + return new TextEncoder().encode(jsonOutput); } - model_info.packageIncludes = options.kmpJsonData.files.filter((e) => !!e.name.match(/.[ot]tf$/i)).length ? ['fonts'] : []; - model_info.version = options.kmpJsonData.info.version.description; - model_info.minKeymanVersion = minKeymanVersion; - model_info.helpLink = HelpRoot + model_info.id; + private isLicenseMIT(filename: string) { + const data = this.callbacks.loadFile(filename); + if(!data) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_LicenseFileDoesNotExist({filename})); + return false; + } - if(options.sourcePath) { - model_info.sourcePath = options.sourcePath; + let license = null; + try { + license = new TextDecoder().decode(data); + } catch(e) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_LicenseFileIsDamaged({filename})); + return false; + } + if(!license) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_LicenseFileIsDamaged({filename})); + return false; + } + const message = validateMITLicense(license); + if(message != null) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_LicenseIsNotValid({filename, message})); + return false; + } + return true; } - const jsonOutput = JSON.stringify(model_info, null, 2); - return new TextEncoder().encode(jsonOutput); -} + private loadJsFile(filename: string) { + const data = this.callbacks.loadFile(filename); + if(!data) { + this.callbacks.reportMessage(ModelInfoCompilerMessages.Error_FileDoesNotExist({filename})); + return null; + } + const text = new TextDecoder('utf-8', {fatal: true}).decode(data); + return text; + } +} \ No newline at end of file diff --git a/developer/src/kmc-model-info/src/model-info-file.ts b/developer/src/kmc-model-info/src/model-info-file.ts index c993242e349..c038c67ca95 100644 --- a/developer/src/kmc-model-info/src/model-info-file.ts +++ b/developer/src/kmc-model-info/src/model-info-file.ts @@ -4,7 +4,7 @@ export interface ModelInfoFile { authorName?: string; authorEmail?: string; description?: string; - license: "mit"; + license?: "mit"; languages: Array; lastModifiedDate?: string; packageFilename?: string; diff --git a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/LICENSE.md b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/LICENSE.md index fefb033d6c1..3a58ba2ae32 100644 --- a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/LICENSE.md +++ b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/LICENSE.md @@ -1,6 +1,6 @@ The MIT License (MIT) -© 2021 - 2022 SIL International +Copyright © 2021 - 2022 SIL International Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/source/sil.cmo.bw.model.kps b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/source/sil.cmo.bw.model.kps index f53475848e7..57b610c83ab 100644 --- a/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/source/sil.cmo.bw.model.kps +++ b/developer/src/kmc-model-info/test/fixtures/sil.cmo.bw/source/sil.cmo.bw.model.kps @@ -7,6 +7,7 @@ readme.htm + ..\LICENSE.md @@ -28,6 +29,12 @@ 0 .js + + ..\LICENSE.md + + 0 + .md + welcome.htm diff --git a/developer/src/kmc-model-info/test/test-model-info-compiler.ts b/developer/src/kmc-model-info/test/test-model-info-compiler.ts index ebfb82f4f89..aa0c2ed65b7 100644 --- a/developer/src/kmc-model-info/test/test-model-info-compiler.ts +++ b/developer/src/kmc-model-info/test/test-model-info-compiler.ts @@ -3,7 +3,7 @@ import { assert } from 'chai'; import 'mocha'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { makePathToFixture } from './helpers/index.js'; -import { writeModelMetadataFile } from '../src/model-info-compiler.js'; +import { ModelInfoCompiler } from '../src/model-info-compiler.js'; import { KmpCompiler } from '@keymanapp/kmc-package'; const callbacks = new TestCompilerCallbacks(); @@ -14,21 +14,25 @@ beforeEach(function() { describe('model-info-compiler', function () { it('compile a .model_info file correctly', function() { - const kpsFileName = makePathToFixture('sil.cmo.bw', 'source', 'sil.cmo.bw.model.kps'); + const kpsFilename = makePathToFixture('sil.cmo.bw', 'source', 'sil.cmo.bw.model.kps'); const kmpFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.kmp'); const buildModelInfoFilename = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model_info'); const kmpCompiler = new KmpCompiler(callbacks); - const kmpJsonData = kmpCompiler.transformKpsToKmpObject(kpsFileName); + const kmpJsonData = kmpCompiler.transformKpsToKmpObject(kpsFilename); const modelFileName = makePathToFixture('sil.cmo.bw', 'build', 'sil.cmo.bw.model.js'); - const data = writeModelMetadataFile(callbacks, { + const data = (new ModelInfoCompiler(callbacks)).writeModelMetadataFile({ kmpFileName, kmpJsonData, model_id: 'sil.cmo.bw', modelFileName, - sourcePath: 'release/sil/sil.cmo.bw' + sourcePath: 'release/sil/sil.cmo.bw', + kpsFilename, }); + if(data == null) { + callbacks.printMessages(); + } assert.isNotNull(data); const actual = JSON.parse(new TextDecoder().decode(data)); diff --git a/developer/src/kmc-model-info/tsconfig.json b/developer/src/kmc-model-info/tsconfig.json index c7edb11615d..e751bfb5eaf 100644 --- a/developer/src/kmc-model-info/tsconfig.json +++ b/developer/src/kmc-model-info/tsconfig.json @@ -6,6 +6,7 @@ "rootDir": "src/", "paths": { "@keymanapp/common-types": ["../../../common/web/types/src/main"], + "@keymanapp/developer-utils": ["../developer/web/utils/index"], } }, "include": [ @@ -14,5 +15,6 @@ "references": [ { "path": "../../../common/web/types" }, { "path": "../../../common/models/types" }, + { "path": "../common/web/utils" }, ] } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 92a6a08c72a..5f9759aa54f 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -6,6 +6,7 @@ import { KmpCompiler } from '@keymanapp/kmc-package'; import { loadProject } from '../../util/projectLoader.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; import { calculateSourcePath } from '../../util/calculateSourcePath.js'; +import { getLastGitCommitDate } from 'src/util/getLastGitCommitDate.js'; export class BuildModelInfo extends BuildActivity { public get name(): string { return 'Lexical model metadata'; } @@ -55,6 +56,8 @@ export class BuildModelInfo extends BuildActivity { return false; } + const lastCommitDate = getLastGitCommitDate(project.projectPath); + const data = writeModelMetadataFile( callbacks, { @@ -63,6 +66,8 @@ export class BuildModelInfo extends BuildActivity { sourcePath: calculateSourcePath(infile), modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model), kmpFileName: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package), + kpsFilename: project.resolveInputFilePath(kps), + lastCommitDate } ); diff --git a/package.json b/package.json index 0afd9ade6eb..7f187b9a884 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "developer/src/kmc-package", "developer/src/kmc", "developer/src/server", + "developer/utils", "common/models/*", "common/test/resources", "common/tools/*", @@ -43,6 +44,7 @@ "dependencies": { "@keymanapp/common-types": "file:common/web/types", "@keymanapp/developer-test-helpers": "file:developer/src/common/web/test-helpers", + "@keymanapp/developer-utils": "file:developer/src/common/web/utils", "@keymanapp/hextobin": "file:common/tools/hextobin", "@keymanapp/keyman-version": "file:common/web/keyman-version", "@keymanapp/ldml-keyboard-constants": "file:core/include/ldml" diff --git a/tsconfig.esm.json b/tsconfig.esm.json index e0f72d8f8d9..d0df9dbdc55 100644 --- a/tsconfig.esm.json +++ b/tsconfig.esm.json @@ -9,6 +9,8 @@ //{ "path": "./developer/src/kmc/test/tsconfig.json" }, { "path": "./developer/src/common/web/test-helpers/tsconfig.json" }, + { "path": "./developer/src/common/web/utils/tsconfig.json" }, + { "path": "./developer/src/common/web/utils/test/tsconfig.json" }, { "path": "./developer/src/kmc/tsconfig.json" }, From 56d64da9a77930b2f7343cd9310534409ba60de3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 30 Aug 2023 16:31:43 +1000 Subject: [PATCH 3/8] chore(developer): npm install and fix import --- .../commands/buildClasses/BuildModelInfo.ts | 27 +++++++++---------- package-lock.json | 12 ++++++++- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 5f9759aa54f..490543c0e6a 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -1,12 +1,12 @@ import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; -import { writeModelMetadataFile } from '@keymanapp/kmc-model-info'; +import { ModelInfoCompiler } from '@keymanapp/kmc-model-info'; import { KmpCompiler } from '@keymanapp/kmc-package'; import { loadProject } from '../../util/projectLoader.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; import { calculateSourcePath } from '../../util/calculateSourcePath.js'; -import { getLastGitCommitDate } from 'src/util/getLastGitCommitDate.js'; +import { getLastGitCommitDate } from '../../util/getLastGitCommitDate.js'; export class BuildModelInfo extends BuildActivity { public get name(): string { return 'Lexical model metadata'; } @@ -57,19 +57,16 @@ export class BuildModelInfo extends BuildActivity { } const lastCommitDate = getLastGitCommitDate(project.projectPath); - - const data = writeModelMetadataFile( - callbacks, - { - model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project), - kmpJsonData, - sourcePath: calculateSourcePath(infile), - modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model), - kmpFileName: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package), - kpsFilename: project.resolveInputFilePath(kps), - lastCommitDate - } - ); + const compiler = new ModelInfoCompiler(callbacks); + const data = compiler.writeModelMetadataFile({ + model_id: callbacks.path.basename(project.projectPath, KeymanFileTypes.Source.Project), + kmpJsonData, + sourcePath: calculateSourcePath(infile), + modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model), + kmpFileName: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package), + kpsFilename: project.resolveInputFilePath(kps), + lastCommitDate + }); if(data == null) { // Error messages have already been emitted by writeModelMetadataFile diff --git a/package-lock.json b/package-lock.json index 7caa96d15ac..4b18a604978 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "developer/src/kmc-package", "developer/src/kmc", "developer/src/server", + "developer/utils", "common/models/*", "common/test/resources", "common/tools/*", @@ -30,6 +31,7 @@ "dependencies": { "@keymanapp/common-types": "file:common/web/types", "@keymanapp/developer-test-helpers": "file:developer/src/common/web/test-helpers", + "@keymanapp/developer-utils": "file:developer/src/common/web/utils", "@keymanapp/hextobin": "file:common/tools/hextobin", "@keymanapp/keyman-version": "file:common/web/keyman-version", "@keymanapp/ldml-keyboard-constants": "file:core/include/ldml" @@ -716,7 +718,6 @@ "bin": { "kmc": "build/src/kmc.js", "kmlmc": "build/src/kmlmc.js", - "kmlmi": "build/src/kmlmi.js", "kmlmp": "build/src/kmlmp.js" }, "devDependencies": { @@ -998,6 +999,7 @@ "license": "MIT", "dependencies": { "@keymanapp/common-types": "*", + "@keymanapp/developer-utils": "*", "@keymanapp/kmc-package": "*" }, "devDependencies": { @@ -1855,6 +1857,7 @@ "license": "MIT", "dependencies": { "@keymanapp/common-types": "*", + "@keymanapp/developer-utils": "*", "@keymanapp/models-types": "*" }, "devDependencies": { @@ -2960,6 +2963,13 @@ "resolved": "developer/src/common/web/test-helpers", "link": true }, + "node_modules/@keymanapp/developer-utils": { + "resolved": "file:developer/src/common/web/utils", + "license": "MIT", + "dependencies": { + "@keymanapp/common-types": "*" + } + }, "node_modules/@keymanapp/hextobin": { "resolved": "common/tools/hextobin", "link": true From 7f76de47811bdc8a8fcfe45e899aba8629300b25 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 1 Sep 2023 10:11:15 +0700 Subject: [PATCH 4/8] chore(developer): add developer-utils as workspace --- package-lock.json | 164 ++++++++++++++++++++++++++++++++++++++++++++-- package.json | 1 + 2 files changed, 160 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4b18a604978..1aed891ee7e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "resources/build/version", "core/include/ldml", "developer/src/common/web/test-helpers", + "developer/src/common/web/utils", "developer/src/kmc-analyze", "developer/src/kmc-keyboard-info", "developer/src/kmc-kmn", @@ -696,6 +697,162 @@ "typescript": ">=2.7" } }, + "developer/src/common/web/utils": { + "license": "MIT", + "dependencies": { + "@keymanapp/common-types": "*" + }, + "devDependencies": { + "@types/node": "^20.4.1", + "c8": "^7.12.0", + "chai": "^4.3.4", + "mocha": "^8.4.0", + "ts-node": "^9.1.1", + "typescript": "^4.9.5" + } + }, + "developer/src/common/web/utils/node_modules/@types/node": { + "version": "20.5.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.7.tgz", + "integrity": "sha512-dP7f3LdZIysZnmvP3ANJYTSwg+wLLl8p7RqniVlV7j+oXSXAbt9h0WIBFmJy5inWZoX9wZN6eXx+YXd9Rh3RBA==", + "dev": true + }, + "developer/src/common/web/utils/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "developer/src/common/web/utils/node_modules/js-yaml": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", + "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "developer/src/common/web/utils/node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "developer/src/common/web/utils/node_modules/mocha": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", + "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "dev": true, + "dependencies": { + "@ungap/promise-all-settled": "1.1.2", + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.1", + "debug": "4.3.1", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.1.6", + "growl": "1.10.5", + "he": "1.2.0", + "js-yaml": "4.0.0", + "log-symbols": "4.0.0", + "minimatch": "3.0.4", + "ms": "2.1.3", + "nanoid": "3.1.20", + "serialize-javascript": "5.0.1", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "which": "2.0.2", + "wide-align": "1.1.3", + "workerpool": "6.1.0", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" + }, + "bin": { + "_mocha": "bin/_mocha", + "mocha": "bin/mocha" + }, + "engines": { + "node": ">= 10.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mochajs" + } + }, + "developer/src/common/web/utils/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "developer/src/common/web/utils/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "developer/src/common/web/utils/node_modules/ts-node": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.1.1.tgz", + "integrity": "sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==", + "dev": true, + "dependencies": { + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.17", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "typescript": ">=2.7" + } + }, + "developer/src/common/web/utils/node_modules/ts-node/node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, "developer/src/kmc": { "name": "@keymanapp/kmc", "license": "MIT", @@ -2964,11 +3121,8 @@ "link": true }, "node_modules/@keymanapp/developer-utils": { - "resolved": "file:developer/src/common/web/utils", - "license": "MIT", - "dependencies": { - "@keymanapp/common-types": "*" - } + "resolved": "developer/src/common/web/utils", + "link": true }, "node_modules/@keymanapp/hextobin": { "resolved": "common/tools/hextobin", diff --git a/package.json b/package.json index 7f187b9a884..ac8c23dec5d 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "resources/build/version", "core/include/ldml", "developer/src/common/web/test-helpers", + "developer/src/common/web/utils", "developer/src/kmc-analyze", "developer/src/kmc-keyboard-info", "developer/src/kmc-kmn", From 6590658da400987f7792f2512bb5c98f8103c539 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 1 Sep 2023 10:12:12 +0700 Subject: [PATCH 5/8] chore(developer): add developer-utils package to publish --- developer/src/kmc/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index 2f8045f455e..4d7e50fd794 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -20,6 +20,7 @@ builder_describe "Build Keyman Keyboard Compiler kmc" \ "@/common/include" \ "@/common/web/keyman-version" \ "@/common/web/types" \ + "@/developer/src/common/web/utils" \ "@/developer/src/kmc-analyze" \ "@/developer/src/kmc-keyboard-info" \ "@/developer/src/kmc-kmn" \ @@ -93,6 +94,7 @@ readonly PACKAGES=( developer/src/kmc-model developer/src/kmc-model-info developer/src/kmc-package + developer/src/common/web/utils ) if builder_start_action bundle; then From 508588ef260a107aff36e18d17af00e84f708f06 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 1 Sep 2023 15:58:22 +0700 Subject: [PATCH 6/8] chore(developer): developer-utils as private package --- developer/src/common/web/utils/package.json | 2 +- developer/src/kmc-keyboard-info/package.json | 3 +++ developer/src/kmc-model-info/package.json | 3 +++ developer/src/kmc/build.sh | 1 - package-lock.json | 8 +++++++- 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index 277066b0464..b09b667e714 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -1,5 +1,6 @@ { "name": "@keymanapp/developer-utils", + "private": true, "description": "Keyman Developer utilities", "type": "module", "exports": { @@ -19,7 +20,6 @@ "scripts": { "build": "tsc -b" }, - "license": "MIT", "dependencies": { "@keymanapp/common-types": "*" }, diff --git a/developer/src/kmc-keyboard-info/package.json b/developer/src/kmc-keyboard-info/package.json index 7b028ec1294..f1f787b9cd2 100644 --- a/developer/src/kmc-keyboard-info/package.json +++ b/developer/src/kmc-keyboard-info/package.json @@ -28,6 +28,9 @@ "@keymanapp/kmc-package": "*", "@keymanapp/developer-utils": "*" }, + "bundleDependencies": [ + "@keymanapp/developer-utils" + ], "devDependencies": { "@types/chai": "^4.3.5", "@types/mocha": "^5.2.7", diff --git a/developer/src/kmc-model-info/package.json b/developer/src/kmc-model-info/package.json index e0499ad0e60..cddc61af63a 100644 --- a/developer/src/kmc-model-info/package.json +++ b/developer/src/kmc-model-info/package.json @@ -35,6 +35,9 @@ "@keymanapp/models-types": "*", "@keymanapp/developer-utils": "*" }, + "bundleDependencies": [ + "@keymanapp/developer-utils" + ], "devDependencies": { "@types/chai": "^4.1.7", "@types/mocha": "^5.2.7", diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index 4d7e50fd794..8b6a87de8a4 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -94,7 +94,6 @@ readonly PACKAGES=( developer/src/kmc-model developer/src/kmc-model-info developer/src/kmc-package - developer/src/common/web/utils ) if builder_start_action bundle; then diff --git a/package-lock.json b/package-lock.json index 1aed891ee7e..3e0c5a43c14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -698,7 +698,7 @@ } }, "developer/src/common/web/utils": { - "license": "MIT", + "name": "@keymanapp/developer-utils", "dependencies": { "@keymanapp/common-types": "*" }, @@ -1153,6 +1153,9 @@ }, "developer/src/kmc-keyboard-info": { "name": "@keymanapp/kmc-keyboard-info", + "bundleDependencies": [ + "@keymanapp/developer-utils" + ], "license": "MIT", "dependencies": { "@keymanapp/common-types": "*", @@ -2011,6 +2014,9 @@ }, "developer/src/kmc-model-info": { "name": "@keymanapp/kmc-model-info", + "bundleDependencies": [ + "@keymanapp/developer-utils" + ], "license": "MIT", "dependencies": { "@keymanapp/common-types": "*", From 0d96437cb8bf6efc38834b0c21318d977c3e8dce Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 1 Sep 2023 20:30:43 +0800 Subject: [PATCH 7/8] chore: workaround npm/cli#3466 when bundling internal deps Works around npm/cli#3466 when bundling internal dependencies using the bundleDependencies package.json property. This change works in tandem with the npm pack/publish process -- when we run `developer/src/kmc/build.sh publish` (or `pack`), we end up with `npm version` stomping on all our package.json files, so the repo is dirty after this. We need a copy of the top-level package.json before this stomping happens, in order to get a simple map of the location of each of our internal dependencies, from the `dependencies` property (it would be possible to figure this out with a lot more parsing of our package.json files, but this is simpler). This means, in future, we should avoid publishing our internal dependencies such as those under common/ to npm, as they serve no practical purpose there. --- developer/src/kmc/build.sh | 10 +++++ resources/build/build-utils-ci.inc.sh | 63 ++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index 8b6a87de8a4..b9f4ea32dd6 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -133,6 +133,10 @@ fi if builder_start_action publish; then . "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" + # To ensure that we cache the top-level package.json, we must call this before + # the global publish + builder_publish_cleanup + # For now, kmc will have responsibility for publishing keyman-version and # common-types, as well as all the other dependent modules. In the future, we # should probably have a top-level npm publish script that publishes all @@ -143,14 +147,20 @@ if builder_start_action publish; then # Finally, publish kmc builder_publish_to_npm + builder_publish_cleanup builder_finish_action success publish elif builder_start_action pack; then . "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" + # To ensure that we cache the top-level package.json, we must call this before + # the global pack + builder_publish_cleanup + for package in "${PACKAGES[@]}"; do "$KEYMAN_ROOT/$package/build.sh" pack $DRY_RUN done builder_publish_to_pack + builder_publish_cleanup builder_finish_action success pack fi diff --git a/resources/build/build-utils-ci.inc.sh b/resources/build/build-utils-ci.inc.sh index cab449a4636..973f6f8aca7 100644 --- a/resources/build/build-utils-ci.inc.sh +++ b/resources/build/build-utils-ci.inc.sh @@ -100,7 +100,9 @@ function _builder_publish_npm_package() { dry_run=--dry-run fi + _builder_publish_cache_package_json _builder_write_npm_version + _builder_prepublish # Note: In either case, npm publish MUST be given --access public to publish a # package in the @keymanapp scope on the public npm package index. @@ -141,9 +143,68 @@ function _builder_write_npm_version() { . + (try { dependencies: (.dependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + (try { devDependencies: (.devDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + - (try { bundleDependencies: (.bundleDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) + (try { optionalDependencies: (.optionalDependencies | to_entries | . + map(select(.key | match("@keymanapp/.*")) .value |= $VERSION_WITH_TAG) | from_entries) } catch {}) ' > "${line}_" mv -f "${line}_" "$line" done +} + +# +# Due to https://github.com/npm/cli/issues/3466, we manually create all +# bundleDependencies (__NOT__ bundledDependencies, beware typos) from +# the target's package.json in its node_modules folder. Must run from +# the target's folder. +# +function _builder_prepublish() { + mkdir -p node_modules/@keymanapp + local packages=($(cat package.json | "$JQ" --raw-output '.bundleDependencies | join(" ")')) + local package + + # For each @keymanapp/ package, we'll do a local symlink, note that Windows + # mklink is internal to cmd! + for package in "${packages[@]}"; do + if [[ $package =~ ^@keymanapp/ ]]; then + # Creating local symlink under node_modules + local link_source=node_modules/$package + + # lookup the link_target from top-level package.json/dependencies + local link_target="$(cat "$KEYMAN_ROOT/builder_package_publish.json" | jq -r .dependencies.\"$package\")" + + if [[ $link_target =~ ^file: ]]; then + link_target="$KEYMAN_ROOT"/${link_target#file:} + + builder_echo "Manually linking $link_source -> $link_target (see https://github.com/npm/cli/issues/3466)" + rm -rf $link_source + if [[ $BUILDER_OS == win ]]; then + link_source="$(cygpath -w "$link_source")" + link_target="$(cygpath -w "$link_target")" + cmd //c mklink //j "$link_source" "$link_target" + else + ln -sr "$link_target" "$link_source" + fi + fi + fi + done +} + +# +# We need to cache /package.json before npm version gets its sticky fingers on +# it, because afterwards, we lose the file: paths that help us to resolve +# dependencies easily. Part of the https://github.com/npm/cli/issues/3466 +# workaround. +# +function _builder_publish_cache_package_json() { + if [[ -f "$KEYMAN_ROOT/builder_package_publish.json" ]]; then + return 0 + fi + + if "$JQ" -e '.version' "$KEYMAN_ROOT/developer/src/kmc/package.json" > /dev/null; then + builder_die "npm version has already been run. Revert the version changes to all package.json files before re-running" + fi + + cp "$KEYMAN_ROOT/package.json" "$KEYMAN_ROOT/builder_package_publish.json" +} + +function builder_publish_cleanup() { + rm -f "$KEYMAN_ROOT/builder_package_publish.json" } \ No newline at end of file From a0bdb617a0f90877e8111ce99ce44478ceae8fc4 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 12 Sep 2023 17:15:35 +1000 Subject: [PATCH 8/8] chore(developer): Apply suggestions from code review Co-authored-by: Darcy Wong --- developer/src/kmc-model-info/src/model-info-compiler.ts | 2 +- resources/build/build-utils-ci.inc.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/developer/src/kmc-model-info/src/model-info-compiler.ts b/developer/src/kmc-model-info/src/model-info-compiler.ts index 4563963aa13..63644b9ef09 100644 --- a/developer/src/kmc-model-info/src/model-info-compiler.ts +++ b/developer/src/kmc-model-info/src/model-info-compiler.ts @@ -19,7 +19,7 @@ export class ModelInfoSources { /** The data from the .kps file, transformed to kmp.json */ kmpJsonData: KmpJsonFile.KmpJsonFile; - /** The path in the keymanapp/lexical-models repo where this model may be found (optional) */ + /** The path in the keymanapp/lexical-models repo where this model may be found */ sourcePath: string; /** The compiled model filename and relative path (.js) */ diff --git a/resources/build/build-utils-ci.inc.sh b/resources/build/build-utils-ci.inc.sh index 973f6f8aca7..47ba7a587c4 100644 --- a/resources/build/build-utils-ci.inc.sh +++ b/resources/build/build-utils-ci.inc.sh @@ -168,7 +168,7 @@ function _builder_prepublish() { local link_source=node_modules/$package # lookup the link_target from top-level package.json/dependencies - local link_target="$(cat "$KEYMAN_ROOT/builder_package_publish.json" | jq -r .dependencies.\"$package\")" + local link_target="$(cat "$KEYMAN_ROOT/builder_package_publish.json" | "$JQ" -r .dependencies.\"$package\")" if [[ $link_target =~ ^file: ]]; then link_target="$KEYMAN_ROOT"/${link_target#file:}