From 2d7784a13ec667138eb51d6a0846e63ab83a3b2b Mon Sep 17 00:00:00 2001 From: Dominic Griesel Date: Thu, 21 Nov 2024 21:29:26 +0100 Subject: [PATCH] refactor: add portable bindings for the filesystem --- packages/config/maintenance/importConfig.ts | 11 +- .../config/maintenance/lintConfigFiles.ts | 3 + packages/config/package.json | 1 + packages/config/src/ConfigManager.ts | 33 ++++- packages/config/src/JsonTemplate.ts | 20 ++- packages/config/src/Manufacturers.ts | 20 ++- packages/config/src/devices/DeviceConfig.ts | 64 ++++++--- packages/config/src/utils.ts | 33 +++-- packages/core/package.json | 6 + packages/core/src/bindings/fs/node.ts | 108 +++++++++++++++ .../crypto/primitives/primitives.browser.ts | 2 +- .../src/crypto/primitives/primitives.node.ts | 2 +- .../src/crypto/primitives/primitives.test.ts | 2 +- .../core/src/crypto/primitives/primitives.ts | 52 -------- packages/core/src/log/shared.ts | 2 +- packages/flash/package.json | 1 + packages/flash/src/cli.ts | 4 +- packages/maintenance/src/convert-json.ts | 8 +- packages/maintenance/src/generateTypedDocs.ts | 22 +-- .../maintenance/src/remove-unnecessary.ts | 8 +- packages/nvmedit/src/cli.ts | 24 ++-- packages/shared/package.json | 8 +- packages/shared/src/bindings.ts | 100 ++++++++++++++ packages/shared/src/docker.ts | 14 +- packages/shared/src/fs.ts | 59 +++++++-- packages/zwave-js/package.json | 1 + packages/zwave-js/src/lib/driver/Driver.ts | 125 ++++++++++++++---- .../zwave-js/src/lib/driver/DriverMock.ts | 8 +- .../zwave-js/src/lib/driver/NetworkCache.ts | 18 ++- .../zwave-js/src/lib/driver/UpdateConfig.ts | 32 +++-- .../zwave-js/src/lib/driver/ZWaveOptions.ts | 18 ++- packages/zwave-js/src/lib/node/Node.ts | 2 +- packages/zwave-js/src/lib/zniffer/Zniffer.ts | 21 ++- yarn.lock | 5 + 34 files changed, 641 insertions(+), 196 deletions(-) create mode 100644 packages/core/src/bindings/fs/node.ts delete mode 100644 packages/core/src/crypto/primitives/primitives.ts create mode 100644 packages/shared/src/bindings.ts diff --git a/packages/config/maintenance/importConfig.ts b/packages/config/maintenance/importConfig.ts index 0254b228f7ef..7801322318bf 100644 --- a/packages/config/maintenance/importConfig.ts +++ b/packages/config/maintenance/importConfig.ts @@ -9,6 +9,7 @@ process.on("unhandledRejection", (r) => { }); import { CommandClasses, getIntegerLimits } from "@zwave-js/core"; +import { fs as nodeFS } from "@zwave-js/core/bindings/fs/node"; import { enumFilesRecursive, formatId, @@ -852,6 +853,7 @@ async function parseZWAFiles(): Promise { let jsonData = []; const configFiles = await enumFilesRecursive( + nodeFS, zwaTempDir, (file) => file.endsWith(".json"), ); @@ -1602,8 +1604,9 @@ async function maintenanceParse(): Promise { const zwaData = []; // Load the zwa files - await fs.mkdir(zwaTempDir, { recursive: true }); + await nodeFS.ensureDir(zwaTempDir); const zwaFiles = await enumFilesRecursive( + nodeFS, zwaTempDir, (file) => file.endsWith(".json"), ); @@ -1611,7 +1614,7 @@ async function maintenanceParse(): Promise { // zWave Alliance numbering isn't always continuous and an html page is // returned when a device number doesn't. Test for and delete such files. try { - zwaData.push(await readJSON(file)); + zwaData.push(await readJSON(nodeFS, file)); } catch { await fs.unlink(file); } @@ -1619,6 +1622,7 @@ async function maintenanceParse(): Promise { // Build the list of device files const configFiles = await enumFilesRecursive( + nodeFS, processedDir, (file) => file.endsWith(".json"), ); @@ -2033,7 +2037,7 @@ async function importConfigFilesOH(): Promise { } } outFilename += ".json"; - await fs.ensureDir(path.dirname(outFilename)); + await nodeFS.ensureDir(path.dirname(outFilename)); const output = stringify(parsed, "\t") + "\n"; await fs.writeFile(outFilename, output, "utf8"); @@ -2305,6 +2309,7 @@ function getLatestConfigVersion( /** Changes the manufacturer names in all device config files to match manufacturers.json */ async function updateManufacturerNames(): Promise { const configFiles = await enumFilesRecursive( + nodeFS, processedDir, (file) => file.endsWith(".json") && !file.endsWith("index.json"), ); diff --git a/packages/config/maintenance/lintConfigFiles.ts b/packages/config/maintenance/lintConfigFiles.ts index ff4090e43851..268dc837c353 100644 --- a/packages/config/maintenance/lintConfigFiles.ts +++ b/packages/config/maintenance/lintConfigFiles.ts @@ -4,6 +4,7 @@ import { getLegalRangeForBitMask, getMinimumShiftForBitMask, } from "@zwave-js/core"; +import { fs } from "@zwave-js/core/bindings/fs/node"; import { reportProblem } from "@zwave-js/maintenance"; import { enumFilesRecursive, @@ -262,6 +263,7 @@ async function lintDevices(): Promise { const rootDir = path.join(configDir, "devices"); const forbiddenFiles = await enumFilesRecursive( + fs, rootDir, (filename) => !filename.endsWith(".json"), ); @@ -286,6 +288,7 @@ async function lintDevices(): Promise { let conditionalConfig: ConditionalDeviceConfig; try { conditionalConfig = await ConditionalDeviceConfig.from( + fs, filePath, true, { diff --git a/packages/config/package.json b/packages/config/package.json index 8d073e711150..65aa3d4bd924 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -64,6 +64,7 @@ "ansi-colors": "^4.1.3", "json-logic-js": "^2.0.5", "json5": "^2.2.3", + "pathe": "^1.1.2", "semver": "^7.6.3", "winston": "^3.15.0" }, diff --git a/packages/config/src/ConfigManager.ts b/packages/config/src/ConfigManager.ts index 04af72e7b56d..e563db2aeedc 100644 --- a/packages/config/src/ConfigManager.ts +++ b/packages/config/src/ConfigManager.ts @@ -5,7 +5,8 @@ import { isZWaveError, } from "@zwave-js/core"; import { getErrorMessage, pathExists } from "@zwave-js/shared"; -import path from "node:path"; +import { type FileSystem } from "@zwave-js/shared/bindings"; +import path from "pathe"; import { ConfigLogger } from "./Logger.js"; import { type ManufacturersMap, @@ -32,6 +33,7 @@ import { } from "./utils.js"; export interface ConfigManagerOptions { + bindings?: FileSystem; logContainer?: ZWaveLogContainer; deviceConfigPriorityDir?: string; deviceConfigExternalDir?: string; @@ -39,6 +41,7 @@ export interface ConfigManagerOptions { export class ConfigManager { public constructor(options: ConfigManagerOptions = {}) { + this._fs = options.bindings; this.logger = new ConfigLogger( options.logContainer ?? new ZWaveLogContainer({ enabled: false }), ); @@ -48,6 +51,12 @@ export class ConfigManager { this._configVersion = PACKAGE_VERSION; } + private _fs: FileSystem | undefined; + private async getFS(): Promise { + this._fs ??= (await import("@zwave-js/core/bindings/fs/node")).fs; + return this._fs; + } + private _configVersion: string; public get configVersion(): string { return this._configVersion; @@ -88,6 +97,7 @@ export class ConfigManager { const externalConfigDir = this.externalConfigDir; if (externalConfigDir) { syncResult = await syncExternalConfigDir( + await this.getFS(), externalConfigDir, this.logger, ); @@ -112,6 +122,7 @@ export class ConfigManager { public async loadManufacturers(): Promise { try { this._manufacturers = await loadManufacturersInternal( + await this.getFS(), this._useExternalConfig && this.externalConfigDir || undefined, ); } catch (e) { @@ -139,7 +150,10 @@ export class ConfigManager { ); } - await saveManufacturersInternal(this._manufacturers); + await saveManufacturersInternal( + await this.getFS(), + this._manufacturers, + ); } /** @@ -177,18 +191,21 @@ export class ConfigManager { } public async loadDeviceIndex(): Promise { + const fs = await this.getFS(); try { // The index of config files included in this package const embeddedIndex = await loadDeviceIndexInternal( + fs, this.logger, this._useExternalConfig && this.externalConfigDir || undefined, ); // A dynamic index of the user-defined priority device config files const priorityIndex: DeviceConfigIndex = []; if (this.deviceConfigPriorityDir) { - if (await pathExists(this.deviceConfigPriorityDir)) { + if (await pathExists(fs, this.deviceConfigPriorityDir)) { priorityIndex.push( ...(await generatePriorityDeviceIndex( + fs, this.deviceConfigPriorityDir, this.logger, )), @@ -230,7 +247,10 @@ export class ConfigManager { } public async loadFulltextDeviceIndex(): Promise { - this.fulltextIndex = await loadFulltextDeviceIndexInternal(this.logger); + this.fulltextIndex = await loadFulltextDeviceIndexInternal( + await this.getFS(), + this.logger, + ); } public getFulltextIndex(): FulltextDeviceConfigIndex | undefined { @@ -254,6 +274,8 @@ export class ConfigManager { // Load/regenerate the index if necessary if (!this.index) await this.loadDeviceIndex(); + const fs = await this.getFS(); + // Look up the device in the index const indexEntries = this.index!.filter( getDeviceEntryPredicate( @@ -274,7 +296,7 @@ export class ConfigManager { const filePath = path.isAbsolute(indexEntry.filename) ? indexEntry.filename : path.join(devicesDir, indexEntry.filename); - if (!(await pathExists(filePath))) return; + if (!(await pathExists(fs, filePath))) return; // A config file is treated as am embedded one when it is located under the devices root dir // or the external config dir @@ -291,6 +313,7 @@ export class ConfigManager { try { return await ConditionalDeviceConfig.from( + fs, filePath, isEmbedded, { rootDir, fallbackDirs }, diff --git a/packages/config/src/JsonTemplate.ts b/packages/config/src/JsonTemplate.ts index 1e42244415d5..c66438c783f0 100644 --- a/packages/config/src/JsonTemplate.ts +++ b/packages/config/src/JsonTemplate.ts @@ -1,10 +1,10 @@ import { ZWaveError, ZWaveErrorCodes } from "@zwave-js/core/safe"; -import { pathExists } from "@zwave-js/shared"; +import { pathExists, readTextFile } from "@zwave-js/shared"; +import { type FileSystem } from "@zwave-js/shared/bindings"; import { getErrorMessage } from "@zwave-js/shared/safe"; import { isArray, isObject } from "alcalzone-shared/typeguards"; import JSON5 from "json5"; -import fs from "node:fs/promises"; -import * as path from "node:path"; +import path from "pathe"; const IMPORT_KEY = "$import"; const importSpecifierRegex = @@ -22,10 +22,11 @@ export function clearTemplateCache(): void { /** Parses a JSON file with $import keys and replaces them with the selected objects */ export async function readJsonWithTemplate( + fs: FileSystem, filename: string, rootDirs?: string | string[], ): Promise> { - if (!(await pathExists(filename))) { + if (!(await pathExists(fs, filename))) { throw new ZWaveError( `Could not open config file ${filename}: not found!`, ZWaveErrorCodes.Config_NotFound, @@ -37,6 +38,7 @@ export async function readJsonWithTemplate( // Try to use the cached versions of the template files to speed up the loading const fileCache = new Map(templateCache); const ret = await readJsonWithTemplateInternal( + fs, filename, undefined, [], @@ -127,6 +129,7 @@ function getImportStack( } async function readJsonWithTemplateInternal( + fs: FileSystem, filename: string, selector: string | undefined, visited: string[], @@ -172,7 +175,7 @@ ${getImportStack(visited, selector)}`, json = fileCache.get(filename)!; } else { try { - const fileContent = await fs.readFile(filename, "utf8"); + const fileContent = await readTextFile(fs, filename, "utf8"); json = JSON5.parse(fileContent); fileCache.set(filename, json); } catch (e) { @@ -188,6 +191,7 @@ ${getImportStack(visited, selector)}`, } // Resolve the JSON imports for (a subset) of the file and return the compound file return resolveJsonImports( + fs, selector ? select(json, selector) : json, filename, [...visited, specifier], @@ -198,6 +202,7 @@ ${getImportStack(visited, selector)}`, /** Replaces all `$import` properties in a JSON object with object spreads of the referenced file/property */ async function resolveJsonImports( + fs: FileSystem, json: Record, filename: string, visited: string[], @@ -225,7 +230,7 @@ async function resolveJsonImports( rootDir, importFilename.slice(2), ); - if (await pathExists(newFilename)) { + if (await pathExists(fs, newFilename)) { break; } else { // Try the next @@ -275,6 +280,7 @@ async function resolveJsonImports( // const importFilename = path.join(path.dirname(filename), val); const imported = await readJsonWithTemplateInternal( + fs, newFilename, selector, visited, @@ -285,6 +291,7 @@ async function resolveJsonImports( } else if (isObject(val)) { // We're looking at an object, recurse into it ret[prop] = await resolveJsonImports( + fs, val, filename, visited, @@ -298,6 +305,7 @@ async function resolveJsonImports( if (isObject(v)) { vals.push( await resolveJsonImports( + fs, v, filename, visited, diff --git a/packages/config/src/Manufacturers.ts b/packages/config/src/Manufacturers.ts index de7a516fb0da..c3657274b589 100644 --- a/packages/config/src/Manufacturers.ts +++ b/packages/config/src/Manufacturers.ts @@ -1,9 +1,15 @@ import { ZWaveError, ZWaveErrorCodes, isZWaveError } from "@zwave-js/core"; -import { formatId, pathExists, stringify } from "@zwave-js/shared"; +import { + formatId, + pathExists, + readTextFile, + stringify, + writeTextFile, +} from "@zwave-js/shared"; +import { type FileSystem } from "@zwave-js/shared/bindings"; import { isObject } from "alcalzone-shared/typeguards"; import JSON5 from "json5"; -import fs from "node:fs/promises"; -import path from "node:path"; +import path from "pathe"; import { configDir } from "./utils.js"; import { hexKeyRegex4Digits, throwInvalidConfig } from "./utils_safe.js"; @@ -11,6 +17,7 @@ export type ManufacturersMap = Map; /** @internal */ export async function loadManufacturersInternal( + fs: FileSystem, externalConfigDir?: string, ): Promise { const configPath = path.join( @@ -18,14 +25,14 @@ export async function loadManufacturersInternal( "manufacturers.json", ); - if (!(await pathExists(configPath))) { + if (!(await pathExists(fs, configPath))) { throw new ZWaveError( "The manufacturer config file does not exist!", ZWaveErrorCodes.Config_Invalid, ); } try { - const fileContents = await fs.readFile(configPath, "utf8"); + const fileContents = await readTextFile(fs, configPath, "utf8"); const definition = JSON5.parse(fileContents); if (!isObject(definition)) { throwInvalidConfig( @@ -66,6 +73,7 @@ export async function loadManufacturersInternal( * Write current manufacturers map to json */ export async function saveManufacturersInternal( + fs: FileSystem, manufacturers: ManufacturersMap, ): Promise { const data: Record = {}; @@ -79,5 +87,5 @@ export async function saveManufacturersInternal( } const configPath = path.join(configDir, "manufacturers.json"); - await fs.writeFile(configPath, stringify(data, "\t") + "\n"); + await writeTextFile(fs, configPath, stringify(data, "\t") + "\n"); } diff --git a/packages/config/src/devices/DeviceConfig.ts b/packages/config/src/devices/DeviceConfig.ts index e74c541eacd5..dc66f14e4e09 100644 --- a/packages/config/src/devices/DeviceConfig.ts +++ b/packages/config/src/devices/DeviceConfig.ts @@ -9,12 +9,14 @@ import { padVersion, pathExists, pick, + readTextFile, stringify, + writeTextFile, } from "@zwave-js/shared"; +import { type FileSystem } from "@zwave-js/shared/bindings"; import { isArray, isObject } from "alcalzone-shared/typeguards"; import JSON5 from "json5"; -import fs from "node:fs/promises"; -import path from "node:path"; +import path from "pathe"; import semverGt from "semver/functions/gt.js"; import { clearTemplateCache, readJsonWithTemplate } from "../JsonTemplate.js"; import type { ConfigLogger } from "../Logger.js"; @@ -85,13 +87,14 @@ export type DeviceConfigIndex = DeviceConfigIndexEntry[]; export type FulltextDeviceConfigIndex = FulltextDeviceConfigIndexEntry[]; async function hasChangedDeviceFiles( + fs: FileSystem, devicesRoot: string, dir: string, lastChange: Date, ): Promise { // Check if there are any files BUT index.json that were changed // or directories that were modified - const filesAndDirs = await fs.readdir(dir); + const filesAndDirs = await fs.readDir(dir); for (const f of filesAndDirs) { const fullPath = path.join(dir, f); @@ -105,7 +108,12 @@ async function hasChangedDeviceFiles( } else if (stat.isDirectory()) { // we need to go deeper! if ( - await hasChangedDeviceFiles(devicesRoot, fullPath, lastChange) + await hasChangedDeviceFiles( + fs, + devicesRoot, + fullPath, + lastChange, + ) ) { return true; } @@ -119,6 +127,7 @@ async function hasChangedDeviceFiles( * Does not update the index itself. */ async function generateIndex>( + fs: FileSystem, devicesDir: string, isEmbedded: boolean, extractIndexEntries: (config: DeviceConfig) => T[], @@ -128,6 +137,7 @@ async function generateIndex>( clearTemplateCache(); const configFiles = await enumFilesRecursive( + fs, devicesDir, (file) => file.endsWith(".json") @@ -147,11 +157,16 @@ async function generateIndex>( .replaceAll("\\", "/"); // Try parsing the file try { - const config = await DeviceConfig.from(file, isEmbedded, { - rootDir: devicesDir, - fallbackDirs, - relative: true, - }); + const config = await DeviceConfig.from( + fs, + file, + isEmbedded, + { + rootDir: devicesDir, + fallbackDirs, + relative: true, + }, + ); // Add the file to the index index.push( ...extractIndexEntries(config).map((entry) => { @@ -184,19 +199,20 @@ async function generateIndex>( } async function loadDeviceIndexShared>( + fs: FileSystem, devicesDir: string, indexPath: string, extractIndexEntries: (config: DeviceConfig) => T[], logger?: ConfigLogger, ): Promise<(T & { filename: string })[]> { // The index file needs to be regenerated if it does not exist - let needsUpdate = !(await pathExists(indexPath)); + let needsUpdate = !(await pathExists(fs, indexPath)); let index: (T & { filename: string })[] | undefined; let mtimeIndex: Date | undefined; // ...or if cannot be parsed if (!needsUpdate) { try { - const fileContents = await fs.readFile(indexPath, "utf8"); + const fileContents = await readTextFile(fs, indexPath, "utf8"); index = JSON5.parse(fileContents); mtimeIndex = (await fs.stat(indexPath)).mtime; } catch { @@ -219,6 +235,7 @@ async function loadDeviceIndexShared>( // ...or if there were any changes in the file system if (!needsUpdate) { needsUpdate = await hasChangedDeviceFiles( + fs, devicesDir, devicesDir, mtimeIndex!, @@ -234,6 +251,7 @@ async function loadDeviceIndexShared>( if (needsUpdate) { // Read all files from disk and generate an index index = await generateIndex( + fs, devicesDir, true, extractIndexEntries, @@ -241,7 +259,8 @@ async function loadDeviceIndexShared>( ); // Save the index to disk try { - await fs.writeFile( + await writeTextFile( + fs, path.join(indexPath), `// This file is auto-generated. DO NOT edit it by hand if you don't know what you're doing!" ${stringify(index, "\t")} @@ -268,11 +287,13 @@ ${stringify(index, "\t")} * Transparently handles updating the index if necessary */ export async function generatePriorityDeviceIndex( + fs: FileSystem, deviceConfigPriorityDir: string, logger?: ConfigLogger, ): Promise { return ( await generateIndex( + fs, deviceConfigPriorityDir, false, (config) => @@ -304,6 +325,7 @@ export async function generatePriorityDeviceIndex( * Transparently handles updating the index if necessary */ export async function loadDeviceIndexInternal( + fs: FileSystem, logger?: ConfigLogger, externalConfigDir?: string, ): Promise { @@ -312,6 +334,7 @@ export async function loadDeviceIndexInternal( ); return loadDeviceIndexShared( + fs, devicesDir, indexPath, (config) => @@ -334,10 +357,12 @@ export async function loadDeviceIndexInternal( * Transparently handles updating the index if necessary */ export async function loadFulltextDeviceIndexInternal( + fs: FileSystem, logger?: ConfigLogger, ): Promise { // This method is not meant to operate with the external device index! return loadDeviceIndexShared( + fs, embeddedDevicesDir, fulltextIndexPath, (config) => @@ -375,6 +400,7 @@ function isFirmwareVersion(val: any): val is string { /** This class represents a device config entry whose conditional settings have not been evaluated yet */ export class ConditionalDeviceConfig { public static async from( + fs: FileSystem, filename: string, isEmbedded: boolean, options: { @@ -388,10 +414,14 @@ export class ConditionalDeviceConfig { const relativePath = relative ? path.relative(rootDir, filename).replaceAll("\\", "/") : filename; - const json = await readJsonWithTemplate(filename, [ - options.rootDir, - ...(options.fallbackDirs ?? []), - ]); + const json = await readJsonWithTemplate( + fs, + filename, + [ + options.rootDir, + ...(options.fallbackDirs ?? []), + ], + ); return new ConditionalDeviceConfig(relativePath, isEmbedded, json); } @@ -663,6 +693,7 @@ metadata is not an object`, export class DeviceConfig { public static async from( + fs: FileSystem, filename: string, isEmbedded: boolean, options: { @@ -673,6 +704,7 @@ export class DeviceConfig { }, ): Promise { const ret = await ConditionalDeviceConfig.from( + fs, filename, isEmbedded, options, diff --git a/packages/config/src/utils.ts b/packages/config/src/utils.ts index bbb56121f9cf..cff94737b52c 100644 --- a/packages/config/src/utils.ts +++ b/packages/config/src/utils.ts @@ -1,7 +1,13 @@ -import { copyFilesRecursive, formatId, padVersion } from "@zwave-js/shared"; -import fs from "node:fs/promises"; +import { + copyFilesRecursive, + formatId, + padVersion, + readTextFile, + writeTextFile, +} from "@zwave-js/shared"; +import { type FileSystem } from "@zwave-js/shared/bindings"; import { createRequire } from "node:module"; -import path from "node:path"; +import path from "pathe"; import semverGte from "semver/functions/gte.js"; import semverInc from "semver/functions/inc.js"; import semverLte from "semver/functions/lte.js"; @@ -64,6 +70,7 @@ export type SyncExternalConfigDirResult = * Synchronizes or updates the external config directory and returns whether the directory is in a state that can be used */ export async function syncExternalConfigDir( + fs: FileSystem, extConfigDir: string, logger: ConfigLogger, ): Promise { @@ -71,7 +78,7 @@ export async function syncExternalConfigDir( // Make sure the config dir exists try { - await fs.mkdir(extConfigDir, { recursive: true }); + await fs.ensureDir(extConfigDir); } catch { logger.print( `Synchronizing external config dir failed - directory could not be created`, @@ -98,7 +105,11 @@ export async function syncExternalConfigDir( let wipe = false; let externalVersion: string | undefined; try { - externalVersion = await fs.readFile(externalVersionFilename, "utf8"); + externalVersion = await readTextFile( + fs, + externalVersionFilename, + "utf8", + ); if (!semverValid(externalVersion)) { wipe = true; } else if ( @@ -118,14 +129,20 @@ export async function syncExternalConfigDir( // Wipe and override the external dir try { logger.print(`Synchronizing external config dir ${extConfigDir}...`); - await fs.rm(extConfigDir, { recursive: true, force: true }); - await fs.mkdir(extConfigDir, { recursive: true }); + await fs.deleteDir(extConfigDir); + await fs.ensureDir(extConfigDir); await copyFilesRecursive( + fs, configDir, extConfigDir, (src) => src.endsWith(".json"), ); - await fs.writeFile(externalVersionFilename, currentVersion, "utf8"); + await writeTextFile( + fs, + externalVersionFilename, + currentVersion, + "utf8", + ); externalVersion = currentVersion; } catch { // Something went wrong diff --git a/packages/core/package.json b/packages/core/package.json index d427d4dab8f9..ad52f3ca8a8b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -14,6 +14,11 @@ "import": "./build/esm/index.js", "require": "./build/cjs/index.js" }, + "./bindings/*": { + "@@dev": "./src/bindings/*.ts", + "import": "./build/esm/bindings/*.js", + "require": "./build/cjs/bindings/*.js" + }, "./safe": { "@@dev": "./src/index_safe.ts", "import": "./build/esm/index_safe.js", @@ -115,6 +120,7 @@ "fflate": "^0.8.2", "logform": "^2.6.1", "nrf-intel-hex": "^1.4.0", + "pathe": "^1.1.2", "reflect-metadata": "^0.2.2", "semver": "^7.6.3", "triple-beam": "*", diff --git a/packages/core/src/bindings/fs/node.ts b/packages/core/src/bindings/fs/node.ts new file mode 100644 index 000000000000..fb4b59631282 --- /dev/null +++ b/packages/core/src/bindings/fs/node.ts @@ -0,0 +1,108 @@ +import type { + FSStats, + FileHandle, + FileSystem, +} from "@zwave-js/shared/bindings"; +import fsp from "node:fs/promises"; + +/** An implementation of the FileSystem bindings for Node.js */ +export const fs: FileSystem = { + readDir(path: string): Promise { + return fsp.readdir(path); + }, + readFile(path: string): Promise { + return fsp.readFile(path); + }, + writeFile(path: string, data: Uint8Array): Promise { + return fsp.writeFile(path, data); + }, + copyFile(source: string, dest: string): Promise { + return fsp.copyFile(source, dest); + }, + async ensureDir(path: string): Promise { + await fsp.mkdir(path, { recursive: true }); + }, + deleteDir(path: string): Promise { + return fsp.rm(path, { recursive: true, force: true }); + }, + stat(path: string): Promise { + return fsp.stat(path); + }, + async open( + path: string, + flags: { + read: boolean; + write: boolean; + create: boolean; + truncate: boolean; + }, + ): Promise { + let mode = ""; + if (!flags.truncate && !flags.read) { + throw new Error( + "Cannot open a file writeonly without truncating it", + ); + } + if (!flags.write && flags.create) { + throw new Error("Cannot open a file readonly with create flag"); + } + + // FIXME: Figure out what the correct behavior is for each combination of flags + if (flags.read && !flags.write) { + mode = "r"; + } else if (flags.read && flags.write && !flags.create) { + mode = "r+"; + } else if (flags.write && flags.create && flags.truncate) { + mode = flags.read ? "w+" : "w"; + } + + return new NodeFileHandle(await fsp.open(path, mode)); + }, +}; + +export class NodeFileHandle implements FileHandle { + public constructor(handle: fsp.FileHandle) { + this.open = true; + this.handle = handle; + } + + private open: boolean; + private handle: fsp.FileHandle; + + async close(): Promise { + if (!this.open) return; + this.open = false; + await this.handle.close(); + } + + async read( + position?: number | null, + length?: number, + ): Promise<{ data: Uint8Array; bytesRead: number }> { + if (!this.open) throw new Error("File is not open"); + const ret = await this.handle.read({ + position, + length, + }); + return { + data: ret.buffer.subarray(0, ret.bytesRead), + bytesRead: ret.bytesRead, + }; + } + + async write( + data: Uint8Array, + position?: number | null, + ): Promise<{ bytesWritten: number }> { + if (!this.open) throw new Error("File is not open"); + const ret = await this.handle.write(data, null, null, position); + return { + bytesWritten: ret.bytesWritten, + }; + } + + stat(): Promise { + if (!this.open) throw new Error("File is not open"); + return this.handle.stat(); + } +} diff --git a/packages/core/src/crypto/primitives/primitives.browser.ts b/packages/core/src/crypto/primitives/primitives.browser.ts index f266da35b13a..b63a4281c9a0 100644 --- a/packages/core/src/crypto/primitives/primitives.browser.ts +++ b/packages/core/src/crypto/primitives/primitives.browser.ts @@ -1,6 +1,6 @@ +import { type CryptoPrimitives } from "@zwave-js/shared/bindings"; import { Bytes } from "@zwave-js/shared/safe"; import { BLOCK_SIZE, xor, zeroPad } from "../shared.js"; -import { type CryptoPrimitives } from "./primitives.js"; const webcrypto = typeof process !== "undefined" && (globalThis as any).crypto === undefined diff --git a/packages/core/src/crypto/primitives/primitives.node.ts b/packages/core/src/crypto/primitives/primitives.node.ts index 24d921890ab5..53f1cac35bb0 100644 --- a/packages/core/src/crypto/primitives/primitives.node.ts +++ b/packages/core/src/crypto/primitives/primitives.node.ts @@ -1,7 +1,7 @@ +import { type CryptoPrimitives } from "@zwave-js/shared/bindings"; import { Bytes } from "@zwave-js/shared/safe"; import crypto from "node:crypto"; import { BLOCK_SIZE, zeroPad } from "../shared.js"; -import { type CryptoPrimitives } from "./primitives.js"; // For Node.js, we use the built-in crypto module since it has better support // for some algorithms Z-Wave needs than the Web Crypto API, so we can implement diff --git a/packages/core/src/crypto/primitives/primitives.test.ts b/packages/core/src/crypto/primitives/primitives.test.ts index ca7e2f929af5..48e6271733fc 100644 --- a/packages/core/src/crypto/primitives/primitives.test.ts +++ b/packages/core/src/crypto/primitives/primitives.test.ts @@ -1,6 +1,6 @@ +import { type CryptoPrimitives } from "@zwave-js/shared/bindings"; import { Bytes } from "@zwave-js/shared/safe"; import { type ExpectStatic, test } from "vitest"; -import { type CryptoPrimitives } from "./primitives.js"; function assertBufferEquals( expect: ExpectStatic, diff --git a/packages/core/src/crypto/primitives/primitives.ts b/packages/core/src/crypto/primitives/primitives.ts deleted file mode 100644 index e7a399e2bb90..000000000000 --- a/packages/core/src/crypto/primitives/primitives.ts +++ /dev/null @@ -1,52 +0,0 @@ -export interface CryptoPrimitives { - randomBytes(length: number): Uint8Array; - /** Encrypts a payload using AES-128-ECB */ - encryptAES128ECB( - plaintext: Uint8Array, - key: Uint8Array, - ): Promise; - /** Encrypts a payload using AES-128-CBC */ - encryptAES128CBC( - plaintext: Uint8Array, - key: Uint8Array, - iv: Uint8Array, - ): Promise; - /** Encrypts a payload using AES-128-OFB */ - encryptAES128OFB( - plaintext: Uint8Array, - key: Uint8Array, - iv: Uint8Array, - ): Promise; - /** Decrypts a payload using AES-128-OFB */ - decryptAES128OFB( - ciphertext: Uint8Array, - key: Uint8Array, - iv: Uint8Array, - ): Promise; - /** Decrypts a payload using AES-256-CBC */ - decryptAES256CBC( - ciphertext: Uint8Array, - key: Uint8Array, - iv: Uint8Array, - ): Promise; - /** Encrypts and authenticates a payload using AES-128-CCM */ - encryptAES128CCM( - plaintext: Uint8Array, - key: Uint8Array, - iv: Uint8Array, - additionalData: Uint8Array, - authTagLength: number, - ): Promise<{ ciphertext: Uint8Array; authTag: Uint8Array }>; - /** Decrypts and verifies a payload using AES-128-CCM */ - decryptAES128CCM( - ciphertext: Uint8Array, - key: Uint8Array, - iv: Uint8Array, - additionalData: Uint8Array, - authTag: Uint8Array, - ): Promise<{ plaintext: Uint8Array; authOK: boolean }>; - digest( - algorithm: "md5" | "sha-1" | "sha-256", - data: Uint8Array, - ): Promise; -} diff --git a/packages/core/src/log/shared.ts b/packages/core/src/log/shared.ts index 19d1b97e84aa..0ece6aeb2af3 100644 --- a/packages/core/src/log/shared.ts +++ b/packages/core/src/log/shared.ts @@ -1,6 +1,6 @@ import { flatMap } from "@zwave-js/shared"; import type { Format, TransformFunction } from "logform"; -import * as path from "node:path"; +import path from "pathe"; import { MESSAGE, configs } from "triple-beam"; import winston from "winston"; import DailyRotateFile from "winston-daily-rotate-file"; diff --git a/packages/flash/package.json b/packages/flash/package.json index 37d20d519c32..87eee069c124 100644 --- a/packages/flash/package.json +++ b/packages/flash/package.json @@ -35,6 +35,7 @@ }, "dependencies": { "@zwave-js/core": "workspace:*", + "pathe": "^1.1.2", "yargs": "^17.7.2", "zwave-js": "workspace:*" }, diff --git a/packages/flash/src/cli.ts b/packages/flash/src/cli.ts index 63c03647a24f..831f9bcd5ed0 100644 --- a/packages/flash/src/cli.ts +++ b/packages/flash/src/cli.ts @@ -1,7 +1,7 @@ +import { fs } from "@zwave-js/core/bindings/fs/node"; import { ZWaveErrorCodes, isZWaveError } from "@zwave-js/core/safe"; import { wait } from "alcalzone-shared/async"; -import fs from "node:fs/promises"; -import path from "node:path"; +import path from "pathe"; import yargs from "yargs"; import { hideBin } from "yargs/helpers"; import { diff --git a/packages/maintenance/src/convert-json.ts b/packages/maintenance/src/convert-json.ts index e1f53efa5a51..bc47f82dfef3 100644 --- a/packages/maintenance/src/convert-json.ts +++ b/packages/maintenance/src/convert-json.ts @@ -3,9 +3,10 @@ * Execute with `yarn ts packages/maintenance/src/convert-json.ts` */ +import { fs } from "@zwave-js/core/bindings/fs/node"; import { enumFilesRecursive } from "@zwave-js/shared"; import esMain from "es-main"; -import fs from "node:fs/promises"; +import fsp from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { Project, ts } from "ts-morph"; @@ -19,6 +20,7 @@ async function main() { const devicesDir = path.join(__dirname, "../../config/config/devices"); const configFiles = await enumFilesRecursive( + fs, devicesDir, (file) => file.endsWith(".json") @@ -28,7 +30,7 @@ async function main() { ); for (const filename of configFiles) { - const content = await fs.readFile(filename, "utf8"); + const content = await fsp.readFile(filename, "utf8"); const sourceFile = project.createSourceFile(filename, content, { overwrite: true, scriptKind: ts.ScriptKind.JSON, @@ -102,7 +104,7 @@ async function main() { if (didChange) { let output = sourceFile.getFullText(); output = formatWithDprint(filename, output); - await fs.writeFile(filename, output, "utf8"); + await fsp.writeFile(filename, output, "utf8"); } } } diff --git a/packages/maintenance/src/generateTypedDocs.ts b/packages/maintenance/src/generateTypedDocs.ts index 00456ea7c111..6f54c8fabcd9 100644 --- a/packages/maintenance/src/generateTypedDocs.ts +++ b/packages/maintenance/src/generateTypedDocs.ts @@ -3,10 +3,11 @@ */ import { CommandClasses, getCCName } from "@zwave-js/core"; +import { fs } from "@zwave-js/core/bindings/fs/node"; import { enumFilesRecursive, num2hex } from "@zwave-js/shared"; import c from "ansi-colors"; import esMain from "es-main"; -import fs from "node:fs/promises"; +import fsp from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { isMainThread } from "node:worker_threads"; @@ -282,7 +283,7 @@ export async function processDocFile( docFile: string, ): Promise { console.log(`processing ${docFile}...`); - let fileContent = await fs.readFile(docFile, "utf8"); + let fileContent = await fsp.readFile(docFile, "utf8"); const ranges = findImportRanges(fileContent); let hasErrors = false; // Replace from back to start so we can reuse the indizes @@ -314,7 +315,7 @@ ${source} fileContent = fileContent.replaceAll("\r\n", "\n"); fileContent = formatWithDprint(docFile, fileContent); if (!hasErrors) { - await fs.writeFile(docFile, fileContent, "utf8"); + await fsp.writeFile(docFile, fileContent, "utf8"); } return hasErrors; } @@ -322,6 +323,7 @@ ${source} /** Processes all imports, returns true if there was an error */ async function processImports(piscina: Piscina): Promise { const files = await enumFilesRecursive( + fs, path.join(projectRoot, "docs"), (f) => !f.includes("/CCs/") && !f.includes("\\CCs\\") && f.endsWith(".md"), @@ -657,7 +659,7 @@ ${formatValueType(idType)} text = text.replaceAll("\r\n", "\n"); text = formatWithDprint(filename, text); - await fs.writeFile(path.join(ccDocsDir, filename), text, "utf8"); + await fsp.writeFile(path.join(ccDocsDir, filename), text, "utf8"); return { generatedIndex, generatedSidebar }; } @@ -671,7 +673,7 @@ async function generateCCDocs( // Load the index file before it gets deleted const indexFilename = path.join(ccDocsDir, "index.md"); - let indexFileContent = await fs.readFile(indexFilename, "utf8"); + let indexFileContent = await fsp.readFile(indexFilename, "utf8"); const indexAutoGenToken = ""; const indexAutoGenStart = indexFileContent.indexOf(indexAutoGenToken); if (indexAutoGenStart === -1) { @@ -681,8 +683,8 @@ async function generateCCDocs( return false; } - await fs.rm(ccDocsDir, { recursive: true, force: true }); - await fs.mkdir(ccDocsDir, { recursive: true }); + await fsp.rm(ccDocsDir, { recursive: true, force: true }); + await fsp.mkdir(ccDocsDir, { recursive: true }); // Find CC APIs const ccFiles = program.getSourceFiles("packages/cc/src/cc/**/*CC.ts"); @@ -712,10 +714,10 @@ async function generateCCDocs( indexAutoGenStart + indexAutoGenToken.length, ) + generatedIndex; indexFileContent = formatWithDprint("index.md", indexFileContent); - await fs.writeFile(indexFilename, indexFileContent, "utf8"); + await fsp.writeFile(indexFilename, indexFileContent, "utf8"); const sidebarInputFilename = path.join(docsDir, "_sidebar.md"); - let sidebarFileContent = await fs.readFile(sidebarInputFilename, "utf8"); + let sidebarFileContent = await fsp.readFile(sidebarInputFilename, "utf8"); const sidebarAutoGenToken = ""; const sidebarAutoGenStart = sidebarFileContent.indexOf(sidebarAutoGenToken); if (sidebarAutoGenStart === -1) { @@ -730,7 +732,7 @@ async function generateCCDocs( sidebarAutoGenStart + sidebarAutoGenToken.length, ); sidebarFileContent = formatWithDprint("_sidebar.md", sidebarFileContent); - await fs.writeFile( + await fsp.writeFile( path.join(ccDocsDir, "_sidebar.md"), sidebarFileContent, "utf8", diff --git a/packages/maintenance/src/remove-unnecessary.ts b/packages/maintenance/src/remove-unnecessary.ts index ef8c99a42d90..e679ed7fe1c9 100644 --- a/packages/maintenance/src/remove-unnecessary.ts +++ b/packages/maintenance/src/remove-unnecessary.ts @@ -1,9 +1,10 @@ // Script to remove unnecessary min/maxValue from config files +import { fs } from "@zwave-js/core/bindings/fs/node"; import { enumFilesRecursive } from "@zwave-js/shared"; import * as JSONC from "comment-json"; import esMain from "es-main"; -import fs from "node:fs/promises"; +import fsp from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { formatWithDprint } from "./dprint.js"; @@ -14,6 +15,7 @@ async function main() { const devicesDir = path.join(__dirname, "../../config/config/devices"); const configFiles = await enumFilesRecursive( + fs, devicesDir, (file) => file.endsWith(".json") @@ -24,7 +26,7 @@ async function main() { for (const filename of configFiles) { const config = JSONC.parse( - await fs.readFile(filename, "utf8"), + await fsp.readFile(filename, "utf8"), ) as JSONC.CommentObject; if (!config.paramInformation) continue; @@ -47,7 +49,7 @@ async function main() { let output = JSONC.stringify(config, null, "\t"); output = formatWithDprint(filename, output); - await fs.writeFile(filename, output, "utf8"); + await fsp.writeFile(filename, output, "utf8"); } } diff --git a/packages/nvmedit/src/cli.ts b/packages/nvmedit/src/cli.ts index 1d14aead6dc6..a8e99152c2b8 100644 --- a/packages/nvmedit/src/cli.ts +++ b/packages/nvmedit/src/cli.ts @@ -1,7 +1,7 @@ -import { readJSON } from "@zwave-js/shared"; +import { readJSON, writeTextFile } from "@zwave-js/shared"; import { isObject } from "alcalzone-shared/typeguards"; -import fs from "node:fs/promises"; import "reflect-metadata"; +import { fs } from "@zwave-js/core/bindings/fs/node"; import yargs from "yargs"; import { hideBin } from "yargs/helpers"; import { @@ -59,7 +59,7 @@ void yargsInstance process.exit(1); } } - await fs.writeFile(argv.out, JSON.stringify(json, null, "\t")); + await writeTextFile(fs, argv.out, JSON.stringify(json, null, "\t")); console.error(`NVM (JSON) written to ${argv.out}`); process.exit(0); @@ -96,7 +96,7 @@ void yargsInstance const { protocolVersion } = argv; const versionIs500 = /^\d\.\d+$/.test(protocolVersion); - const json = await readJSON(argv.in); + const json = await readJSON(fs, argv.in); const jsonIs500 = json.format === 500; if (versionIs500 && !jsonIs500) { console.error( @@ -156,9 +156,13 @@ Create a backup of the target stick, use the nvm2json command to convert it to J }, }), async (argv) => { - const json500 = await readJSON(argv.in); + const json500 = await readJSON(fs, argv.in); const json700 = json500To700(json500, argv.truncate); - await fs.writeFile(argv.out, JSON.stringify(json700, null, "\t")); + await writeTextFile( + fs, + argv.out, + JSON.stringify(json700, null, "\t"), + ); console.error(`700-series NVM (JSON) written to ${argv.out}`); process.exit(0); @@ -181,9 +185,13 @@ Create a backup of the target stick, use the nvm2json command to convert it to J }, }), async (argv) => { - const json700 = await readJSON(argv.in); + const json700 = await readJSON(fs, argv.in); const json500 = json700To500(json700); - await fs.writeFile(argv.out, JSON.stringify(json500, null, "\t")); + await writeTextFile( + fs, + argv.out, + JSON.stringify(json500, null, "\t"), + ); console.error(`500-series NVM (JSON) written to ${argv.out}`); process.exit(0); diff --git a/packages/shared/package.json b/packages/shared/package.json index c2e489351f2a..ad8ff053bdb5 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -19,6 +19,11 @@ "import": "./build/esm/index_safe.js", "require": "./build/cjs/index_safe.js" }, + "./bindings": { + "@@dev": "./src/bindings.ts", + "import": "./build/esm/bindings.js", + "require": "./build/cjs/bindings.js" + }, "./package.json": "./package.json" }, "sideEffects": false, @@ -46,7 +51,8 @@ "node": ">= 18" }, "dependencies": { - "alcalzone-shared": "^5.0.0" + "alcalzone-shared": "^5.0.0", + "pathe": "^1.1.2" }, "scripts": { "build": "tsc -b tsconfig.build.json --pretty && yarn postbuild", diff --git a/packages/shared/src/bindings.ts b/packages/shared/src/bindings.ts new file mode 100644 index 000000000000..94e7973187dd --- /dev/null +++ b/packages/shared/src/bindings.ts @@ -0,0 +1,100 @@ +// Definitions for runtime-agnostic low level bindings like cryptography, +// file system access, etc. + +export interface CryptoPrimitives { + randomBytes(length: number): Uint8Array; + /** Encrypts a payload using AES-128-ECB */ + encryptAES128ECB( + plaintext: Uint8Array, + key: Uint8Array, + ): Promise; + /** Encrypts a payload using AES-128-CBC */ + encryptAES128CBC( + plaintext: Uint8Array, + key: Uint8Array, + iv: Uint8Array, + ): Promise; + /** Encrypts a payload using AES-128-OFB */ + encryptAES128OFB( + plaintext: Uint8Array, + key: Uint8Array, + iv: Uint8Array, + ): Promise; + /** Decrypts a payload using AES-128-OFB */ + decryptAES128OFB( + ciphertext: Uint8Array, + key: Uint8Array, + iv: Uint8Array, + ): Promise; + /** Decrypts a payload using AES-256-CBC */ + decryptAES256CBC( + ciphertext: Uint8Array, + key: Uint8Array, + iv: Uint8Array, + ): Promise; + /** Encrypts and authenticates a payload using AES-128-CCM */ + encryptAES128CCM( + plaintext: Uint8Array, + key: Uint8Array, + iv: Uint8Array, + additionalData: Uint8Array, + authTagLength: number, + ): Promise<{ ciphertext: Uint8Array; authTag: Uint8Array }>; + /** Decrypts and verifies a payload using AES-128-CCM */ + decryptAES128CCM( + ciphertext: Uint8Array, + key: Uint8Array, + iv: Uint8Array, + additionalData: Uint8Array, + authTag: Uint8Array, + ): Promise<{ plaintext: Uint8Array; authOK: boolean }>; + digest( + algorithm: "md5" | "sha-1" | "sha-256", + data: Uint8Array, + ): Promise; +} + +export interface FSStats { + isDirectory(): boolean; + isFile(): boolean; + mtime: Date; + size: number; +} + +export interface FileHandle { + close(): Promise; + read( + position?: number | null, + length?: number, + ): Promise<{ data: Uint8Array; bytesRead: number }>; + write( + data: Uint8Array, + position?: number | null, + ): Promise<{ bytesWritten: number }>; + stat(): Promise; +} + +export interface FileSystem { + /** Lists files and subdirectories in the given directory */ + readDir(path: string): Promise; + /** Reads the given file */ + readFile(path: string): Promise; + /** Writes the given data to a file */ + writeFile(path: string, data: Uint8Array): Promise; + /** Copies a file */ + copyFile(source: string, dest: string): Promise; + /** Recursively creates a directory and all its parent directories that do not exist */ + ensureDir(path: string): Promise; + /** Deletes a directory and all its contents */ + deleteDir(path: string): Promise; + /** Returns information about a file or directory, or throws if it does not exist */ + stat(path: string): Promise; + /** Opens a file handle */ + open(path: string, flags: { + // FIXME: Define expected behavior for each flag + read: boolean; + write: boolean; + create: boolean; + truncate: boolean; + }): Promise; +} diff --git a/packages/shared/src/docker.ts b/packages/shared/src/docker.ts index a76b1c4e0cee..c5b41b0d3127 100644 --- a/packages/shared/src/docker.ts +++ b/packages/shared/src/docker.ts @@ -1,10 +1,11 @@ // Shamelessly copied from https://github.com/sindresorhus/is-docker -import fs from "node:fs"; +import { createRequire } from "node:module"; +const require = createRequire(import.meta.url); function hasDockerEnv(): boolean { try { - fs.statSync("/.dockerenv"); + require("node:fs").statSync("/.dockerenv"); return true; } catch { return false; @@ -13,7 +14,9 @@ function hasDockerEnv(): boolean { function hasDockerCGroup(): boolean { try { - return fs.readFileSync("/proc/self/cgroup", "utf8").includes("docker"); + return require("node:fs") + .readFileSync("/proc/self/cgroup", "utf8") + .includes("docker"); } catch { return false; } @@ -21,7 +24,10 @@ function hasDockerCGroup(): boolean { let _isDocker: boolean | undefined; -/** Check if the process is running inside a Docker container */ +/** + * Check if the process is running inside a Docker container + * @deprecated Use `is-docker` package instead, or copy this code into your project + */ export function isDocker(): boolean { if (_isDocker === undefined) { _isDocker = hasDockerEnv() || hasDockerCGroup(); diff --git a/packages/shared/src/fs.ts b/packages/shared/src/fs.ts index d45a3f48b16b..5961430370a2 100644 --- a/packages/shared/src/fs.ts +++ b/packages/shared/src/fs.ts @@ -1,19 +1,23 @@ -import fs from "node:fs/promises"; -import path from "node:path"; +import path from "pathe"; +import { Bytes } from "./Bytes.js"; +import { type FileSystem } from "./bindings.js"; import { getErrorMessage } from "./errors.js"; export async function enumFilesRecursive( + fs: FileSystem, rootDir: string, predicate?: (filename: string) => boolean, ): Promise { const ret: string[] = []; try { - const filesAndDirs = await fs.readdir(rootDir); + const filesAndDirs = await fs.readDir(rootDir); for (const f of filesAndDirs) { const fullPath = path.join(rootDir, f); if ((await fs.stat(fullPath)).isDirectory()) { - ret.push(...(await enumFilesRecursive(fullPath, predicate))); + ret.push( + ...(await enumFilesRecursive(fs, fullPath, predicate)), + ); } else if (predicate == undefined || predicate(fullPath)) { ret.push(fullPath); } @@ -28,26 +32,55 @@ export async function enumFilesRecursive( } export async function copyFilesRecursive( + fs: FileSystem, sourceDir: string, targetDir: string, predicate?: (filename: string) => boolean, ): Promise { - const files = await enumFilesRecursive(sourceDir, predicate); + const files = await enumFilesRecursive(fs, sourceDir, predicate); for (const file of files) { const relative = path.relative(sourceDir, file); const target = path.join(targetDir, relative); - await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.ensureDir(path.dirname(target)); await fs.copyFile(file, target); } } -export async function readJSON(filename: string): Promise { - const data = await fs.readFile(filename, "utf8"); - return JSON.parse(data); +export async function readTextFile( + fs: FileSystem, + filename: string, + encoding: BufferEncoding = "utf8", +): Promise { + const buffer = await fs.readFile(filename); + return Bytes.view(buffer).toString(encoding); } -export async function pathExists(filename: string): Promise { - return fs.access(filename) - .then(() => true) - .catch(() => false); +export async function writeTextFile( + fs: FileSystem, + filename: string, + content: string, + encoding: BufferEncoding = "utf8", +): Promise { + const buffer = Bytes.from(content, encoding); + await fs.writeFile(filename, buffer); +} + +export async function readJSON( + fs: FileSystem, + filename: string, +): Promise { + const content = await readTextFile(fs, filename); + return JSON.parse(content); +} + +export async function pathExists( + fs: FileSystem, + filename: string, +): Promise { + try { + await fs.stat(filename); + return true; + } catch { + return false; + } } diff --git a/packages/zwave-js/package.json b/packages/zwave-js/package.json index 34224c6f917d..83724441a436 100644 --- a/packages/zwave-js/package.json +++ b/packages/zwave-js/package.json @@ -113,6 +113,7 @@ "got": "^13.0.0", "mdns-server": "^1.0.11", "p-queue": "^8.0.1", + "pathe": "^1.1.2", "proper-lockfile": "^4.1.2", "reflect-metadata": "^0.2.2", "semver": "^7.6.3", diff --git a/packages/zwave-js/src/lib/driver/Driver.ts b/packages/zwave-js/src/lib/driver/Driver.ts index 360acdc5a33b..7de6e5aca50f 100644 --- a/packages/zwave-js/src/lib/driver/Driver.ts +++ b/packages/zwave-js/src/lib/driver/Driver.ts @@ -175,7 +175,6 @@ import { mergeDeep, noop, num2hex, - pathExists, pick, } from "@zwave-js/shared"; import { distinct } from "alcalzone-shared/arrays"; @@ -188,9 +187,8 @@ import { isArray, isObject } from "alcalzone-shared/typeguards"; import type { EventEmitter } from "node:events"; import fs from "node:fs/promises"; import os from "node:os"; -import path from "node:path"; -import { URL } from "node:url"; import * as util from "node:util"; +import path from "pathe"; import { SerialPort } from "serialport"; import { InterpreterStatus, interpret } from "xstate"; import { ZWaveController } from "../controller/Controller.js"; @@ -217,6 +215,7 @@ import { containsSerializedCC, isCommandRequest, } from "@zwave-js/serial/serialapi"; +import { type FileSystem } from "@zwave-js/shared/bindings"; import { PACKAGE_NAME, PACKAGE_VERSION } from "../_version.js"; import { type ZWaveNodeBase } from "../node/mixins/00_Base.js"; import { type NodeWakeup } from "../node/mixins/30_Wakeup.js"; @@ -315,20 +314,6 @@ const defaultOptions: ZWaveOptions = { queryAllUserCodes: false, }, storage: { - driver: { - async ensureDir(path) { - await fs.mkdir(path, { recursive: true }); - }, - pathExists(path) { - return pathExists(path); - }, - readFile(file, encoding) { - return fs.readFile(file, { encoding }); - }, - writeFile(file, data, options) { - return fs.writeFile(file, data, options); - }, - }, cacheDir: path.join(process.cwd(), "cache"), lockDir: process.env.ZWAVEJS_LOCK_DIRECTORY, throttle: "normal", @@ -616,6 +601,65 @@ function assertValidCCs(container: ContainsCC): void { } } +function wrapLegacyFSDriverForCacheMigrationOnly( + legacy: import("@zwave-js/core/traits").FileSystem, +): FileSystem { + // This usage only needs readFile and checking if a file exists + // Every other usage will throw! + return { + async readFile(path) { + const text = await legacy.readFile(path, "utf8"); + return Bytes.from(text, "utf8"); + }, + async stat(path) { + if (await legacy.pathExists(path)) { + return { + isDirectory() { + return false; + }, + isFile() { + return true; + }, + mtime: new Date(), + size: 0, + }; + } else { + throw new Error("File not found"); + } + }, + readDir(_path) { + return Promise.reject( + new Error("Not implemented for the legacy FS driver"), + ); + }, + deleteDir(_path) { + return Promise.reject( + new Error("Not implemented for the legacy FS driver"), + ); + }, + ensureDir(_path) { + return Promise.reject( + new Error("Not implemented for the legacy FS driver"), + ); + }, + open(_path, _flags) { + return Promise.reject( + new Error("Not implemented for the legacy FS driver"), + ); + }, + writeFile(_path, _data) { + return Promise.reject( + new Error("Not implemented for the legacy FS driver"), + ); + }, + copyFile(_source, _dest) { + return Promise.reject( + new Error("Not implemented for the legacy FS driver"), + ); + }, + }; +} + // Strongly type the event emitter events export interface DriverEventCallbacks extends PrefixedNodeEvents { "driver ready": () => void; @@ -1278,6 +1322,12 @@ export class Driver extends TypedEventEmitter return this._options; } + /** + * The bindings used to access file system etc. + */ + // This is set during `start()` and should not be accessed before + private bindings!: Required>; + private _wasStarted: boolean = false; private _isOpen: boolean = false; @@ -1304,6 +1354,13 @@ export class Driver extends TypedEventEmitter ); } + // Populate default bindings. This has to happen asynchronously, so the driver does not have a hard dependency + // on Node.js internals + this.bindings = { + fs: this._options.bindings?.fs + ?? (await import("@zwave-js/core/bindings/fs/node")).fs, + }; + const spOpenPromise = createDeferredPromise(); // Log which version is running @@ -1438,7 +1495,13 @@ export class Driver extends TypedEventEmitter // Try to create the cache directory. This can fail, in which case we should expose a good error message try { - await this._options.storage.driver.ensureDir(this.cacheDir); + // eslint-disable-next-line @typescript-eslint/no-deprecated + if (this._options.storage.driver) { + // eslint-disable-next-line @typescript-eslint/no-deprecated + await this._options.storage.driver.ensureDir(this.cacheDir); + } else { + await this.bindings.fs.ensureDir(this.cacheDir); + } } catch (e) { let message: string; if ( @@ -1587,8 +1650,8 @@ export class Driver extends TypedEventEmitter this._valueDB = new JsonlDB(valueDBFile, { ...options, enableTimestamps: true, - reviver: (key, value) => deserializeCacheValue(value), - serializer: (key, value) => serializeCacheValue(value), + reviver: (_key, value) => deserializeCacheValue(value), + serializer: (_key, value) => serializeCacheValue(value), }); await this._valueDB.open(); @@ -1628,7 +1691,13 @@ export class Driver extends TypedEventEmitter this.controller.homeId, this._networkCache, this._valueDB, - this._options.storage.driver, + // eslint-disable-next-line @typescript-eslint/no-deprecated + this._options.storage.driver + ? wrapLegacyFSDriverForCacheMigrationOnly( + // eslint-disable-next-line @typescript-eslint/no-deprecated + this._options.storage.driver, + ) + : this.bindings.fs, this.cacheDir, ); @@ -5872,7 +5941,7 @@ ${handlers.length} left`, }, notifyRetry: ( lastError, - message, + _message, attempts, maxAttempts, delay, @@ -7231,10 +7300,14 @@ ${handlers.length} left`, `Installing version ${newVersion} of configuration DB...`, ); try { - await installConfigUpdate(newVersion, { - cacheDir: this.cacheDir, - configDir: extConfigDir, - }); + await installConfigUpdate( + this.bindings.fs, + newVersion, + { + cacheDir: this.cacheDir, + configDir: extConfigDir, + }, + ); } catch (e) { this.driverLog.print(getErrorMessage(e), "error"); return false; diff --git a/packages/zwave-js/src/lib/driver/DriverMock.ts b/packages/zwave-js/src/lib/driver/DriverMock.ts index d2e48facd9ac..cdcb0ba7d995 100644 --- a/packages/zwave-js/src/lib/driver/DriverMock.ts +++ b/packages/zwave-js/src/lib/driver/DriverMock.ts @@ -4,10 +4,10 @@ import { type MockPortBinding, SerialPortMock, } from "@zwave-js/serial/mock"; +import { type FileSystem } from "@zwave-js/shared/bindings"; import { createDeferredPromise } from "alcalzone-shared/deferred-promise"; -import fs from "node:fs/promises"; import { tmpdir } from "node:os"; -import path from "node:path"; +import path from "pathe"; import type { SerialPort } from "serialport"; import { Driver } from "./Driver.js"; import type { PartialZWaveOptions, ZWaveOptions } from "./ZWaveOptions.js"; @@ -111,6 +111,7 @@ export interface CreateAndStartTestingDriverOptions { loadConfiguration?: boolean; portAddress: string; + fs?: FileSystem; } export async function createAndStartTestingDriver( @@ -124,6 +125,7 @@ export async function createAndStartTestingDriver( skipBootloaderCheck = true, skipNodeInterview = false, loadConfiguration = true, + fs = (await import("@zwave-js/core/bindings/fs/node")).fs, ...internalOptions } = options; @@ -167,7 +169,7 @@ export async function createAndStartTestingDriver( const originalDestroy = driver.destroy.bind(driver); driver.destroy = async () => { await originalDestroy(); - await fs.rm(cacheDir, { recursive: true, force: true }); + await fs.deleteDir(cacheDir); }; return new Promise((resolve) => { diff --git a/packages/zwave-js/src/lib/driver/NetworkCache.ts b/packages/zwave-js/src/lib/driver/NetworkCache.ts index e358070de94a..611d9c759c96 100644 --- a/packages/zwave-js/src/lib/driver/NetworkCache.ts +++ b/packages/zwave-js/src/lib/driver/NetworkCache.ts @@ -2,7 +2,6 @@ import type { JsonlDB } from "@alcalzone/jsonl-db"; import { type AssociationAddress } from "@zwave-js/cc"; import { type CommandClasses, - type FileSystem, NodeType, Protocols, SecurityClass, @@ -13,8 +12,9 @@ import { securityClassOrder, } from "@zwave-js/core"; import { Bytes, getEnumMemberName, num2hex, pickDeep } from "@zwave-js/shared"; +import type { FileSystem } from "@zwave-js/shared/bindings"; import { isArray, isObject } from "alcalzone-shared/typeguards"; -import path from "node:path"; +import path from "pathe"; import { ProvisioningEntryStatus, type SmartStartProvisioningEntry, @@ -623,12 +623,20 @@ export async function migrateLegacyNetworkCache( homeId: number, networkCache: JsonlDB, valueDB: JsonlDB, - storageDriver: FileSystem, + fs: FileSystem, cacheDir: string, ): Promise { const cacheFile = path.join(cacheDir, `${homeId.toString(16)}.json`); - if (!(await storageDriver.pathExists(cacheFile))) return; - const legacy = JSON.parse(await storageDriver.readFile(cacheFile, "utf8")); + try { + const stat = await fs.stat(cacheFile); + if (!stat.isFile()) return; + } catch { + // The file does not exist + return; + } + + const legacyContents = await fs.readFile(cacheFile); + const legacy = JSON.parse(Bytes.view(legacyContents).toString("utf8")); const jsonl = networkCache; function tryMigrate( diff --git a/packages/zwave-js/src/lib/driver/UpdateConfig.ts b/packages/zwave-js/src/lib/driver/UpdateConfig.ts index 8de169b6c944..5e91e55f7ddd 100644 --- a/packages/zwave-js/src/lib/driver/UpdateConfig.ts +++ b/packages/zwave-js/src/lib/driver/UpdateConfig.ts @@ -1,10 +1,16 @@ import { ZWaveError, ZWaveErrorCodes } from "@zwave-js/core"; -import { copyFilesRecursive, getErrorMessage } from "@zwave-js/shared"; +import { + copyFilesRecursive, + getErrorMessage, + noop, + writeTextFile, +} from "@zwave-js/shared"; +import { type FileSystem } from "@zwave-js/shared/bindings"; import { isObject } from "alcalzone-shared/typeguards"; import execa from "execa"; -import fs from "node:fs/promises"; +import fsp from "node:fs/promises"; import os from "node:os"; -import * as path from "node:path"; +import path from "pathe"; import * as lockfile from "proper-lockfile"; import semverInc from "semver/functions/inc.js"; import semverValid from "semver/functions/valid.js"; @@ -60,6 +66,7 @@ export async function checkForConfigUpdates( * This only works if an external configuation directory is used. */ export async function installConfigUpdate( + fs: FileSystem, newVersion: string, external: { configDir: string; @@ -120,7 +127,7 @@ export async function installConfigUpdate( // Download tarball to a temporary directory let tmpDir: string; try { - tmpDir = await fs.mkdtemp( + tmpDir = await fsp.mkdtemp( path.join(os.tmpdir(), "zjs-config-update-"), ); } catch (e) { @@ -138,7 +145,7 @@ export async function installConfigUpdate( // Download the package tarball into the temporary directory const tarFilename = path.join(tmpDir, "zjs-config-update.tgz"); try { - const handle = await fs.open(tarFilename, "w"); + const handle = await fsp.open(tarFilename, "w"); const fstream = handle.createWriteStream({ autoClose: true }); const response = got.stream.get(url); response.pipe(fstream); @@ -171,8 +178,8 @@ export async function installConfigUpdate( const extractedDir = path.join(tmpDir, "extracted"); try { // Extract the tarball in the temporary folder - await fs.rm(extractedDir, { recursive: true, force: true }); - await fs.mkdir(extractedDir, { recursive: true }); + await fs.deleteDir(extractedDir); + await fs.ensureDir(extractedDir); await execa("tar", [ "--strip-components=1", "-xzf", @@ -182,9 +189,10 @@ export async function installConfigUpdate( ]); // then overwrite the files in the external config directory - await fs.rm(external.configDir, { recursive: true, force: true }); - await fs.mkdir(external.configDir, { recursive: true }); + await fs.deleteDir(external.configDir); + await fs.ensureDir(external.configDir); await copyFilesRecursive( + fs, path.join(extractedDir, "config"), external.configDir, (src) => src.endsWith(".json"), @@ -193,7 +201,7 @@ export async function installConfigUpdate( external.configDir, "version", ); - await fs.writeFile(externalVersionFilename, newVersion, "utf8"); + await writeTextFile(fs, externalVersionFilename, newVersion); } catch { await freeLock(); throw new ZWaveError( @@ -203,9 +211,7 @@ export async function installConfigUpdate( } // Clean up the temp dir and ignore errors - void fs.rm(tmpDir, { recursive: true, force: true }).catch(() => { - // ignore - }); + await fs.deleteDir(tmpDir).catch(noop); // Free the lock await freeLock(); diff --git a/packages/zwave-js/src/lib/driver/ZWaveOptions.ts b/packages/zwave-js/src/lib/driver/ZWaveOptions.ts index 39c6646c37e9..10ed072ac528 100644 --- a/packages/zwave-js/src/lib/driver/ZWaveOptions.ts +++ b/packages/zwave-js/src/lib/driver/ZWaveOptions.ts @@ -7,6 +7,7 @@ import type { import type { ZWaveHostOptions } from "@zwave-js/host"; import type { ZWaveSerialPortBase } from "@zwave-js/serial"; import { type DeepPartial, type Expand } from "@zwave-js/shared"; +import type { FileSystem as FileSystemBindings } from "@zwave-js/shared/bindings"; import type { SerialPort } from "serialport"; import type { InclusionUserCallbacks, @@ -126,9 +127,20 @@ export interface ZWaveOptions extends ZWaveHostOptions { disableOnNodeAdded?: boolean; }; + bindings?: { + /** + * Specifies which bindings are used to access the file system when + * reading or writing the cache, or loading device configuration files. + */ + fs?: FileSystemBindings; + }; + storage: { - /** Allows you to replace the default file system driver used to store and read the cache */ - driver: FileSystem; + /** + * Allows you to replace the default file system driver used to store and read the cache + * @deprecated Use `bindings.fs` instead. + */ + driver?: FileSystem; /** Allows you to specify a different cache directory */ cacheDir: string; /** @@ -397,6 +409,7 @@ export type PartialZWaveOptions = Expand< | "joinNetworkUserCallbacks" | "logConfig" | "testingHooks" + | "bindings" > > & Partial< @@ -404,6 +417,7 @@ export type PartialZWaveOptions = Expand< ZWaveOptions, | "testingHooks" | "vendor" + | "bindings" > > & { diff --git a/packages/zwave-js/src/lib/node/Node.ts b/packages/zwave-js/src/lib/node/Node.ts index c83999c64457..d6117fb49618 100644 --- a/packages/zwave-js/src/lib/node/Node.ts +++ b/packages/zwave-js/src/lib/node/Node.ts @@ -234,7 +234,7 @@ import { import { roundTo } from "alcalzone-shared/math"; import { isArray, isObject } from "alcalzone-shared/typeguards"; import { EventEmitter } from "node:events"; -import path from "node:path"; +import path from "pathe"; import semverParse from "semver/functions/parse.js"; import { RemoveNodeReason } from "../controller/Inclusion.js"; import { determineNIF } from "../controller/NodeInformationFrame.js"; diff --git a/packages/zwave-js/src/lib/zniffer/Zniffer.ts b/packages/zwave-js/src/lib/zniffer/Zniffer.ts index 4044b013aaa7..8787ef6fbf8f 100644 --- a/packages/zwave-js/src/lib/zniffer/Zniffer.ts +++ b/packages/zwave-js/src/lib/zniffer/Zniffer.ts @@ -70,7 +70,6 @@ import { type DeferredPromise, createDeferredPromise, } from "alcalzone-shared/deferred-promise"; -import fs from "node:fs/promises"; import { type ZWaveOptions } from "../driver/ZWaveOptions.js"; import { ZnifferLogger } from "../log/Zniffer.js"; import { @@ -120,6 +119,8 @@ export interface ZnifferOptions { /** Security keys for decrypting Z-Wave Long Range traffic */ securityKeysLongRange?: ZWaveOptions["securityKeysLongRange"]; + bindings?: ZWaveOptions["bindings"]; + /** * The RSSI values reported by the Zniffer are not actual RSSI values. * They can be converted to dBm, but the conversion is chip dependent and not documented for 700/800 series Zniffers. @@ -265,6 +266,12 @@ export class Zniffer extends TypedEventEmitter { private _options: ZnifferOptions; + /** + * The bindings used to access file system etc. + */ + // This is set during `init()` and should not be accessed before + private bindings!: Required>; + /** The serial port instance */ private serial: ZnifferSerialPortBase | undefined; private parsingContext: Omit< @@ -330,6 +337,13 @@ export class Zniffer extends TypedEventEmitter { ); } + // Populate default bindings. This has to happen asynchronously, so the driver does not have a hard dependency + // on Node.js internals + this.bindings = { + fs: this._options.bindings?.fs + ?? (await import("@zwave-js/core/bindings/fs/node")).fs, + }; + // Open the serial port if (typeof this.port === "string") { if (this.port.startsWith("tcp://")) { @@ -984,7 +998,10 @@ supported frequencies: ${ filePath: string, frameFilter?: (frame: CapturedFrame) => boolean, ): Promise { - await fs.writeFile(filePath, this.getCaptureAsZLFBuffer(frameFilter)); + await this.bindings.fs.writeFile( + filePath, + this.getCaptureAsZLFBuffer(frameFilter), + ); } /** diff --git a/yarn.lock b/yarn.lock index 476ba031e44f..f33a8a4027d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2603,6 +2603,7 @@ __metadata: js-levenshtein: "npm:^1.1.6" json-logic-js: "npm:^2.0.5" json5: "npm:^2.2.3" + pathe: "npm:^1.1.2" peggy: "npm:^3.0.2" proxyquire: "npm:^2.1.3" semver: "npm:^7.6.3" @@ -2636,6 +2637,7 @@ __metadata: fflate: "npm:^0.8.2" logform: "npm:^2.6.1" nrf-intel-hex: "npm:^1.4.0" + pathe: "npm:^1.1.2" reflect-metadata: "npm:^0.2.2" semver: "npm:^7.6.3" sinon: "npm:^19.0.2" @@ -2672,6 +2674,7 @@ __metadata: "@types/yargs": "npm:^17.0.33" "@zwave-js/core": "workspace:*" del-cli: "npm:^6.0.0" + pathe: "npm:^1.1.2" typescript: "npm:5.6.2" yargs: "npm:^17.7.2" zwave-js: "workspace:*" @@ -2870,6 +2873,7 @@ __metadata: "@types/sinon": "npm:^17.0.3" alcalzone-shared: "npm:^5.0.0" del-cli: "npm:^6.0.0" + pathe: "npm:^1.1.2" sinon: "npm:^19.0.2" tsx: "npm:^4.19.2" typescript: "npm:5.6.2" @@ -10096,6 +10100,7 @@ __metadata: mdns-server: "npm:^1.0.11" mockdate: "npm:^3.0.5" p-queue: "npm:^8.0.1" + pathe: "npm:^1.1.2" proper-lockfile: "npm:^4.1.2" proxyquire: "npm:^2.1.3" reflect-metadata: "npm:^0.2.2"