Merge branch 'epic/ldml-editor' into feat/developer/12756-vscode-build

HISTORY.md

@@ -1,5 +1,21 @@
 # Keyman Version History
 
+## 18.0.152 alpha 2024-12-04
+
+* refactor(mac): pass kmx data blob to keyman core instead of file path (#12760)
+* fix(developer): honour provided script when checking for matching scripts (#12768)
+* chore(common): rename test files (#12709)
+* fix(common): rename test file (#12770)
+
+## 18.0.151 alpha 2024-12-03
+
+* feat(android): Enhance how ENTER key is handled for FV and KMSample2 (#12745)
+* feat(developer): report on mismatching lang tag scripts when building keyboard-info (#12753)
+
+## 18.0.150 alpha 2024-12-02
+
+* fix(core,developer): use `NDEBUG` flag to disable assertions in release build (#12715)
+
 ## 18.0.149 alpha 2024-12-01
 
 * refactor(developer): unify test action (#12736)

VERSION.md

@@ -1 +1 @@
-18.0.150
+18.0.153

android/Samples/KMSample2/app/src/main/java/com/keyman/kmsample2/SystemKeyboard.java

@@ -130,6 +130,11 @@ public void onStartInput(EditorInfo attribute, boolean restarting) {
     super.onStartInput(attribute, restarting);
     KMManager.onStartInput(attribute, restarting);
     KMManager.resetContext(KeyboardType.KEYBOARD_TYPE_SYSTEM);
+
+    // Determine special handling for ENTER key
+    int inputType = attribute.inputType;
+    KMManager.setEnterMode(attribute.imeOptions, inputType);
+
     // User switched to a new input field so we should extract the text from input field
     // and pass it to Keyman Engine together with selection range
     InputConnection ic = getCurrentInputConnection();

common/web/types/.eslintrc.cjs

@@ -1,13 +1,13 @@
 module.exports = {
   parserOptions: {
-    project: ["./tsconfig.json", "./test/tsconfig.json"],
+    project: ["./tsconfig.json", "./tests/tsconfig.json"],
   },
   ignorePatterns: [
     ".*/*",
     "build/*",
     "coverage/*",
     "node_modules/*",
-    "test/fixtures/*",
+    "tests/fixtures/*",
     "tools/*",
     "src/schemas/*"
   ],

common/web/types/build.sh

@@ -88,7 +88,7 @@ function do_test() {
   fi
 
   eslint .
-  tsc --build test
+  tsc --build tests
 
   readonly C8_THRESHOLD=60
 

common/web/types/package.json

@@ -23,7 +23,7 @@
     "build": "tsc -b",
     "build:schema": "ajv compile",
     "lint": "eslint .",
-    "test": "npm run lint && cd test && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha"
+    "test": "npm run lint && cd tests && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha"
   },
   "author": "Marc Durdin <[email protected]> (https://github.com/mcdurdin)",
   "license": "MIT",
@@ -48,7 +48,7 @@
     "typescript": "^5.4.5"
   },
   "mocha": {
-    "spec": "build/test/**/test-*.js",
+    "spec": "build/tests/**/*.tests.js",
     "require": [
       "source-map-support/register"
     ]
@@ -76,7 +76,7 @@
       "src/keyman-touch-layout/keyman-touch-layout-file-writer.ts",
       "src/osk/osk.ts",
       "src/schemas/*",
-      "test/"
+      "tests/"
     ]
   },
   "sideEffects": false

common/web/types/src/kmx/kmx-plus/kmx-plus.ts

@@ -174,12 +174,7 @@ export class Strs extends Section {
    */
   allocString(s?: string, opts?: StrsOptions, sections?: DependencySections): StrsItem {
     // Run the string processing pipeline
-    s = Strs.processString(s, opts, sections);
-
-    // add to the set, for testing
-    if (s) {
-      this.allProcessedStrings.add(s);
-    }
+    s = this.processString(s, opts, sections);
 
     // if it's a single char, don't push it into the strs table
     if (opts?.singleOk && isOneChar(s)) {
@@ -196,8 +191,8 @@ export class Strs extends Section {
     return result;
   }
 
-  /** process everything according to opts */
-  static processString(s: string, opts: StrsOptions, sections: DependencySections) {
+  /** process everything according to opts, and add the string to this.allProcessedStrings */
+  private processString(s: string, opts: StrsOptions, sections: DependencySections) {
     s = s ?? '';
     // type check everything else
     if (typeof s !== 'string') {
@@ -215,6 +210,12 @@ export class Strs extends Section {
     if (opts?.unescape) {
       s = unescapeString(s);
     }
+
+    if (s) {
+      // add all processed strings here, so that we catch denormalized strings in the input
+      this.allProcessedStrings.add(s);
+    }
+
     // nfd
     if (opts?.nfd) {
       if (!sections?.meta?.normalizationDisabled) {

common/web/types/src/util/util.ts

@@ -305,6 +305,15 @@ export function isPUA(ch: number) {
     (ch >= Uni_PUA_16_START && ch <= Uni_PUA_16_END));
 }
 
+/** @returns false if s is NEITHER NFC nor NFD. (Returns true for falsy) */
+export function isNormalized(s: string) : boolean {
+  if(!s) return true; // empty or null
+  const nfc = s.normalize("NFC");
+  const nfd = s.normalize("NFD");
+  if (s !== nfc && s !== nfd) return false;
+  return true;
+}
+
 class BadStringMap extends Map<BadStringType, Set<number>> {
   public toString() : string {
     if (!this.size) {

common/web/types/test/helpers/index.ts → common/web/types/tests/helpers/index.ts

@@ -9,5 +9,5 @@ import { fileURLToPath } from "url";
  * @param components One or more path components.
  */
 export function makePathToFixture(...components: string[]): string {
-  return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url));
+  return fileURLToPath(new URL(path.join('..', '..', '..', 'tests', 'fixtures', ...components), import.meta.url));
 }

common/web/types/test/kvk/test-kvk-file.ts → common/web/types/tests/kvk/kvk-file.tests.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import 'mocha';
 import { makePathToFixture } from '../helpers/index.js';
 import KvkFileReader from "../../src/kvk/kvk-file-reader.js";
-import { verify_balochi_inpage, verify_khmer_angkor } from './test-kvk-utils.js';
+import { verify_balochi_inpage, verify_khmer_angkor } from './kvk-utils.tests.js';
 
 describe('kvk-file-reader', function () {
   it('kvk-file-reader should read a valid file', function() {

common/web/types/test/ldml-keyboard/test-unicodeset-parser-api.ts → common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts

@@ -1,8 +1,8 @@
 /*
  * Keyman is copyright (C) SIL Global. MIT License.
- * 
+ *
  * Created by Dr Mark C. Sinclair on 2024-11-29
- * 
+ *
  * Test code for unicodeset-parser-api.ts
  */
 

common/web/types/test/tsconfig.json → common/web/types/tests/tsconfig.json

@@ -4,13 +4,12 @@
   "compilerOptions": {
     "rootDir": ".",
     "rootDirs": ["./", "../src/"],
-    "outDir": "../build/test",
+    "outDir": "../build/tests",
     "baseUrl": ".",
     "strictNullChecks": false, // TODO: get rid of this as some point
     "allowSyntheticDefaultImports": true
   },
   "include": [
-    "**/test-*.ts",
     "**/*.tests.ts",
     "./helpers/*.ts",
   ],

common/web/types/test/util/test-unescape.ts → common/web/types/tests/util/unescape.tests.ts

@@ -1,6 +1,6 @@
 import 'mocha';
 import {assert} from 'chai';
-import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer} from '../../src/util/util.js';
+import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isNormalized} from '../../src/util/util.js';
 
 describe('test UTF32 functions()', function() {
   it('should properly categorize strings', () => {
@@ -186,6 +186,24 @@ describe('test bad char functions', () => {
       assert.isTrue(isPUA(ch), describeCodepoint(ch));
     }
   });
+  describe('test isDenormalized()', () => {
+    it('should correctly categorize strings', () => {
+      [
+        undefined,
+        null,
+        '',
+        'ABC',
+        'fa\u1E69cinating', // NFC
+        'fas\u0323\u0307cinating', // NFD
+        'd\u0323\u0307', // NFD
+        '\u1e0d\u0307', // NFC
+      ].map(s => assert.isTrue(isNormalized(s), `for string ${s}`));
+      [
+        'd\u0307\u0323', // NFD but reversed marks
+        'fas\u0307\u0323cinating', // not-NFD
+      ].map(s => assert.isFalse(isNormalized(s), `for string ${s}`));
+    });
+  });
 });
 
 describe('test BadStringAnalyzer', () => {

core/docs/api/index.md

@@ -75,6 +75,7 @@ modifiers such as Windows key are excluded from this set. Some modifiers are
 transient, such as Control, and others have long-lasting state, such as
 Caps Lock.
 
+- __See more in__   [Keyman Glossary](https://github.com/keymanapp/keyman/wiki/Keyman-glossary)
 
 [km_core_cp]: background#km_core_cp "km_core_cp type"
 [km_core_usv]: background#km_core_usv "km_core_usv type"

developer/src/kmc-keyboard-info/src/keyboard-info-compiler-messages.ts

@@ -2,7 +2,7 @@ import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m
 
 const Namespace = CompilerErrorNamespace.KeyboardInfoCompiler;
 // const SevInfo = CompilerErrorSeverity.Info | Namespace;
-// const SevHint = CompilerErrorSeverity.Hint | Namespace;
+const SevHint = CompilerErrorSeverity.Hint | Namespace;
 // const SevWarn = CompilerErrorSeverity.Warn | Namespace;
 const SevError = CompilerErrorSeverity.Error | Namespace;
 const SevFatal = CompilerErrorSeverity.Fatal | Namespace;
@@ -61,5 +61,10 @@ export class KeyboardInfoCompilerMessages {
   static Error_DescriptionIsMissing = (o:{filename:string}) => m(
     this.ERROR_DescriptionIsMissing,
     `The Info.Description field in the package ${def(o.filename)} is required, but is missing or empty.`);
+
+  static HINT_ScriptDoesNotMatch = SevHint | 0x0011;
+  static Hint_ScriptDoesNotMatch = (o:{script: string, bcp47:string, commonScript: string}) => m(
+    this.HINT_ScriptDoesNotMatch,
+    `The script '${def(o.script)}' associated with language tag '${def(o.bcp47)}' does not match the script '${def(o.commonScript)}' for the first language in the package.`);
 }
 

developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts

@@ -502,6 +502,8 @@ export class KeyboardInfoCompiler implements KeymanCompiler {
     const fontSource = [].concat(...kmpJsonData.keyboards.map(e => e.displayFont ? [e.displayFont] : []), ...kmpJsonData.keyboards.map(e => e.webDisplayFonts ?? []));
     const oskFontSource = [].concat(...kmpJsonData.keyboards.map(e => e.oskFont ? [e.oskFont] : []), ...kmpJsonData.keyboards.map(e => e.webOskFonts ?? []));
 
+    let commonScript = null;
+
     for(const bcp47 of Object.keys(keyboard_info.languages)) {
       const language = keyboard_info.languages[bcp47];
 
@@ -572,6 +574,15 @@ export class KeyboardInfoCompiler implements KeymanCompiler {
         ` (${language.regionName})` :
         ''
       );
+
+      const resolvedScript = locale.script ?? langtagsByTag[bcp47]?.script ?? langtagsByTag[locale.language]?.script ?? undefined;
+      if(commonScript === null) {
+        commonScript = resolvedScript;
+      } else {
+        if(resolvedScript !== commonScript) {
+          this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Hint_ScriptDoesNotMatch({commonScript, bcp47, script: resolvedScript}))
+        }
+      }
     }
     return true;
   }

developer/src/kmc-keyboard-info/test/fixtures/hint_script_does_not_match/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015-2022 SIL International
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

developer/src/kmc-keyboard-info/test/fixtures/hint_script_does_not_match/build/.gitattributes

@@ -0,0 +1 @@
+khmer_angkor.js -text