epic: shared fonts in Keyman Developer packages 🐻

common/schemas/kps/README.md

@@ -2,6 +2,19 @@
 
 Master version: https://github.com/keymanapp/api.keyman.com/blob/master/schemas/kps/17.0/kps.xsd
 
+See https://github.com/keymanapp/
+
+## 2024-11-29 18.0
+* Version 18.0 adds:
+  - Files/File/Source - a remote source for the file, or 'local'
+* Version 18.0 modifies:
+  - Files/File/Name - may be either a local file, relative to .kps (or absolute
+    path, not recommended) or a permanent URL on github.com
+* Version 18.0 deprecates (for later removal):
+  - Files/File/CopyLocation
+  - Files/File/Description
+  - Files/File/FileType
+
 ## 2023-10-19 17.0
 * Version 17.0 adds:
   - LicenseFile - a .md file, usually named LICENSE.md
@@ -10,7 +23,7 @@ Master version: https://github.com/keymanapp/api.keyman.com/blob/master/schemas/
   - RelatedPackages - a list of other packages which relate to this one, or are deprecated by it
   - Keyboards/Keyboard/Examples - a list of typing examples for the keyboard
   - Keyboarsd/Keyboard/WebOSKFonts - a list of font filenames (not necessarily in package) suitable for rendering the on screen keyboard
-  - Keyboarsd/Keyboard/WebDisplayFonts - a list of font filenames (not necessarily in package) suitable for use with the keyboard
+  - Keyboards/Keyboard/WebDisplayFonts - a list of font filenames (not necessarily in package) suitable for use with the keyboard
 * Version 17.0 removes:
   - LexicalModels/LexicalModel/Version - version information is not stored in the models, but only in the package metadata (was unused)
 

common/schemas/kps/kps.xsd

@@ -6,7 +6,10 @@
 
   Supports KPS files for Keyman Developer 7+
 
-  Expects FileVersion 7.0
+  Expects FileVersion:
+  - 7.0 - minimum version
+  - 12.0 - supports lexical models, kmp.json
+  - 18.0 - supports Files/File/Source
 -->
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
 
@@ -89,9 +92,10 @@
               <xs:complexType>
                 <xs:all>
                   <xs:element minOccurs="0" maxOccurs="1" name="Name" type="xs:string" />
-                  <xs:element minOccurs="0" maxOccurs="1" name="Description" type="xs:string" />
-                  <xs:element minOccurs="0" maxOccurs="1" name="CopyLocation" type="km-file-copy-location" />
-                  <xs:element minOccurs="0" maxOccurs="1" name="FileType" type="xs:string" />
+                  <xs:element minOccurs="0" maxOccurs="1" name="Description" type="xs:string" /> <!-- 18.0: deprecated -->
+                  <xs:element minOccurs="0" maxOccurs="1" name="CopyLocation" type="km-file-copy-location" /> <!-- 18.0: deprecated -->
+                  <xs:element minOccurs="0" maxOccurs="1" name="FileType" type="xs:string" /> <!-- 18.0: deprecated -->
+                  <xs:element minOccurs="0" maxOccurs="1" name="Source" type="xs:string" />
                 </xs:all>
               </xs:complexType>
             </xs:element>

developer/docs/help/reference/file-types/kps.md

@@ -17,11 +17,172 @@ Details:
 Distributed with keyboard:
 :   No. This is a development file and should not be distributed.
 
-## File format
+# File format
 
+## Schema
+
+The schema for .kps is documented at:
+https://github.com/keymanapp/keyman/tree/master/common/schemas/kps
+
+## Strings
 The optional `<Strings>` section of the file can be included to
 customise the text in the bootstrap installer. The default strings are
 found in the file:
 [strings.xml](https://github.com/keymanapp/keyman/blob/stable-14.0/windows/src/desktop/setup/locale/en/strings.xml).
 You can add your own strings for a given package which are used when
 compiling as a bundled installer.
+
+
+## File.Name
+
+The `Name` element may be either a local filename, or a GitHub raw permanent
+URL, conforming to the pattern:
+
+```
+<Name>https://github.com/{owner}/{repo}/raw/{commit}/[path/]{filename}</Name>
+```
+
+## File.Source
+
+The `<Source>` element documents the original source of the file, from
+one of several supported sources:
+* flo - fonts.languagetechnology.org
+* github
+* noto - in the future
+
+While the `<Name>` element must refer to either a local file or to a permanent
+GitHub file (as recognized by the presence of a commit hash), the `<Source>`
+element typically will refer to a file which may change over time.
+
+The `<Source>` element is supported in version 18.0 and later of .kps
+format.
+
+### Local file system
+
+The `<Source>` element must not be included for files sourced from the local
+file system, where `<Name>` references a file on the local filesystem.
+
+Note: the `<Source>` element is optional for other sources as well; the absence
+of the element does not mean that the `<Name>` element is necessarily
+referencing a local file -- that is determined by pattern match.
+
+```xml
+<File>
+  <Name>[relative-or-absolute-path/]{filename}</Name>
+</File>
+```
+
+Note: absolute paths are not recommended because they are not portable to
+other computers.
+
+#### Example
+
+```xml
+<File>
+  <Name>Alkalami-Regular.ttf</Name>
+</File>
+```
+
+```xml
+<File>
+  <Name>../../shared/fonts/Alkalami-Regular.ttf</Name>
+</File>
+```
+
+### fonts.languagetechnology.org `flo:<id>`
+
+References a font by ID from fonts.languagetechnology.org. The resource must be
+resolved to a permanent URL. If the `<Name>` field is missing, the compiler will
+return an error, reporting the URL it discovers, so the author should be able to
+paste that URL in.
+
+If the URL in the `<Name>` field differs from the resolved URL given by FLO,
+then a hint will be issued by the compiler, allowing the author to easily update
+to a new version of the resource at their convenience. It is anticipated that
+this may be scripted on the keyboards repository in the future.
+
+```xml
+<File>
+  <Name>https://github.com/{owner}/{repo}/raw/{commit}/[path/]{filename}</Name>
+  <Source>flo:{id}</Source>
+</File>
+```
+
+#### Example
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>flo:alkalami</Source>
+</File>
+```
+
+### GitHub
+
+A reference to a resource located on GitHub. The resource must be resolvable to
+a permanent URL. If the `<Name>` field is missing, the compiler will return an
+error, reporting the URL it discovers, so the author should be able to paste
+that URL in.
+
+If the URL in the `<Name>` field differs from the resolved URL from the
+`<Source>` element, then a hint will be issued by the compiler, allowing the
+author to easily update to a new version of the resource at their convenience.
+It is anticipated that this may be scripted on the keyboards repository in the
+future.
+
+```xml
+<File>
+  <Name>https://github.com/{owner}/{repo}/raw/{commit}/[path/]{filename}</Name>
+  <Source>https://github.com/{owner}/{repo}/(raw|blob)/([refs/heads/]{branch}|[refs/tags/]{tag}|{commit})/[path/]{filename}</Source>
+</File>
+```
+
+#### Examples
+
+A GitHub raw reference, from a branch:
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>https://github.com/silnrsi/fonts/raw/main/fonts/sil/alkalami/Alkalami-Regular.ttf</Source>
+</File>
+```
+
+Including `/refs/heads/` is also acceptable (as often found in GitHub UI):
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>https://github.com/silnrsi/fonts/raw/refs/heads/main/fonts/sil/alkalami/Alkalami-Regular.ttf</Source>
+</File>
+```
+
+A GitHub blob reference, as copied from a URL in GitHub:
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>https://github.com/silnrsi/fonts/blob/main/fonts/sil/alkalami/Alkalami-Regular.ttf</Source>
+</File>
+```
+
+A GitHub raw reference, from a commit, resolves identically. Note that as the
+`<Source>` element includes a commit hash, the referenced file can never change
+and so there is no real benefit in including it:
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Source>
+</File>
+```
+
+### noto
+
+In a future version.
+
+## Additional file format notes
+
+The `CopyLocation`, `Description`, and `FileType` elements are deprecated as of
+v18.0 and totally ignored by the compiler and end-user apps.
+

developer/src/common/web/utils/src/common-messages.ts

@@ -58,4 +58,10 @@ export class CommonTypesMessages {
     this.ERROR_InvalidPackageFile,
     `Package source file is invalid: ${(o.e ?? 'unknown error').toString()}`
   );
+
+  static ERROR_UnsupportedPackageFileVersion = SevError | 0x000A;
+  static Error_UnsupportedPackageFileVersion = (o:{version:string}) => m(
+    this.ERROR_UnsupportedPackageFileVersion,
+    `Package source file is an unsupported version '${def(o.version)}'`,
+  );
 };

developer/src/common/web/utils/src/types/kps/kps-file-reader.ts

@@ -12,7 +12,8 @@ import boxXmlArray = util.boxXmlArray;
 import { CommonTypesMessages } from "../../common-messages.js";
 import { CompilerCallbacks } from "../../compiler-callbacks.js";
 import { KeymanXMLReader } from "../../xml-utils.js";
-import { KpsPackage } from "./kps-file.js";
+import { KPS_FILE_VERSIONS, KpsPackage } from "./kps-file.js";
+
 
 export class KpsFileReader {
   constructor(private callbacks: CompilerCallbacks) {
@@ -38,6 +39,11 @@ export class KpsFileReader {
       return null;
     }
 
+    if(!KPS_FILE_VERSIONS.includes(kpsPackage.Package?.System?.FileVersion)) {
+      this.callbacks.reportMessage(CommonTypesMessages.Error_UnsupportedPackageFileVersion({version: kpsPackage.Package?.System?.FileVersion}));
+      return null;
+    }
+
     return this.boxArrays(kpsPackage);
   }
 

developer/src/common/web/utils/src/types/kps/kps-file.ts

@@ -16,6 +16,21 @@
 // * Strings element is not yet checked to be correct
 //
 
+/** minimum supported file version, for keyboard packages */
+export const KPS_FILE_MINIMUM_VERSION_KEYBOARD_SUPPORT = '7.0';
+
+/** minimum version for lexical model packages */
+export const KPS_FILE_MINIMUM_VERSION_LEXICAL_MODEL_SUPPORT = '12.0';
+
+/** minimum version that supports remote references */
+export const KPS_FILE_MINIMUM_VERSION_REMOTE_SUPPORT = '18.0';
+
+export const KPS_FILE_VERSIONS = [
+  KPS_FILE_MINIMUM_VERSION_KEYBOARD_SUPPORT,
+  KPS_FILE_MINIMUM_VERSION_LEXICAL_MODEL_SUPPORT,
+  KPS_FILE_MINIMUM_VERSION_REMOTE_SUPPORT,
+];
+
 export interface KpsPackage {
   /**
    * <Package> -- the root element.
@@ -77,6 +92,8 @@ export interface KpsFileContentFile {
   CopyLocation?: string;
   /** @deprecated */
   FileType?: string;
+  /** Source location, e.g. flo:<id>, URL */
+  Source?: string;
 }
 
 export interface KpsFileLexicalModel {

developer/src/common/web/utils/test/TestCompilerCallbacks.ts

@@ -0,0 +1,79 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { loadFile, resolveFilename } from './helpers/index.js';
+import { CompilerCallbacks, CompilerFileSystemAsyncCallbacks, CompilerFileSystemCallbacks, CompilerNetAsyncCallbacks, CompilerPathCallbacks } from '../src/compiler-callbacks.js';
+import { CompilerError, CompilerEvent } from '../src/compiler-interfaces.js';
+import { fileURLToPath } from 'url';
+
+// This is related to developer/src/common/web/test-helpers/index.ts but has a slightly different API surface
+// as this runs at a lower level than the compiler.
+/**
+ * A CompilerCallbacks implementation for testing
+ */
+export class TestCompilerCallbacks implements CompilerCallbacks {
+  clear() {
+    this.messages = [];
+  }
+  debug(msg: string): void {
+    console.debug(msg);
+  }
+
+  printMessages() {
+    process.stdout.write(CompilerError.formatEvent(this.messages));
+  }
+
+  messages: CompilerEvent[] = [];
+
+  get path(): CompilerPathCallbacks {
+    return {
+      ...path,
+      isAbsolute: path.win32.isAbsolute
+    };
+  }
+
+  get fs(): CompilerFileSystemCallbacks {
+    return fs;
+  }
+
+  get fsAsync(): CompilerFileSystemAsyncCallbacks {
+    return null; // Note: not currently used
+  }
+
+  get net(): CompilerNetAsyncCallbacks {
+    return null; // Note: not currently used
+  }
+
+  resolveFilename(baseFilename: string, filename: string): string {
+    return resolveFilename(baseFilename, filename);
+  }
+
+  fileURLToPath(url: string | URL): string {
+    return fileURLToPath(url);
+  }
+
+  loadFile(filename: string): Uint8Array {
+    // TODO: error management, does it belong here?
+    try {
+      return loadFile(filename);
+    } catch (e) {
+      if (e.code === 'ENOENT') {
+        return <Uint8Array><unknown>null;
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  fileSize(filename: string): number {
+    return fs.statSync(filename).size;
+  }
+
+  isDirectory(filename: string): boolean {
+    return fs.statSync(filename)?.isDirectory();
+  }
+
+  reportMessage(event: CompilerEvent): void {
+    // console.log(event.message);
+    this.messages.push(event);
+  }
+}