From 1b0a76dd385667885b193455cb5a913daccd1123 Mon Sep 17 00:00:00 2001 From: Northword <44738481+northword@users.noreply.github.com> Date: Sun, 15 Dec 2024 11:00:42 +0800 Subject: [PATCH] prefs: reduce duplication io operations, use async method, drop replace-in-file, replace globby with tinyglobby (#68) * fix(deps): replace globby with tinyglobby * refactor: drop replace-in-file * pref: use async method * pref: ignore node_modules and write on demand * pref: tweaks * chore: clean * fix: module export error --- package.json | 3 +- pnpm-lock.yaml | 122 +---------------- src/config.ts | 5 +- src/core/builder.ts | 259 ++++++++++++++++-------------------- src/core/releaser/base.ts | 4 +- src/core/releaser/github.ts | 10 +- src/core/tester.ts | 10 +- src/types/config.ts | 1 - src/utils/crypto.ts | 12 +- src/utils/file.ts | 2 +- src/utils/manifest.ts | 12 -- src/utils/string.ts | 31 +++++ src/utils/zotero-runner.ts | 42 +++--- src/vendor/index.ts | 3 +- 14 files changed, 190 insertions(+), 326 deletions(-) delete mode 100644 src/utils/manifest.ts diff --git a/package.json b/package.json index 8ae5d52..96f49e6 100644 --- a/package.json +++ b/package.json @@ -79,12 +79,11 @@ "es-toolkit": "^1.29.0", "esbuild": "^0.24.0", "fs-extra": "^11.2.0", - "globby": "^14.0.2", "hookable": "^5.5.3", "mime": "^4.0.4", "octokit": "^4.0.2", - "replace-in-file": "^8.2.0", "std-env": "^3.8.0", + "tinyglobby": "^0.2.10", "update-notifier": "^7.3.1", "xvfb-ts": "^1.1.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9877d02..31700b5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -41,9 +41,6 @@ importers: fs-extra: specifier: ^11.2.0 version: 11.2.0 - globby: - specifier: ^14.0.2 - version: 14.0.2 hookable: specifier: ^5.5.3 version: 5.5.3 @@ -53,12 +50,12 @@ importers: octokit: specifier: ^4.0.2 version: 4.0.2 - replace-in-file: - specifier: ^8.2.0 - version: 8.2.0 std-env: specifier: ^3.8.0 version: 3.8.0 + tinyglobby: + specifier: ^0.2.10 + version: 0.2.10 update-notifier: specifier: ^7.3.1 version: 7.3.1 @@ -770,10 +767,6 @@ packages: resolution: {integrity: sha512-e5+YUKENATs1JgYHMzTr2MW/NDcXGfYFAuOQU8gJgF/kEh4EqKgfGrfLI67bMD4tbhZVlkigz/9YYwWcbOFthg==} engines: {node: '>=10.13.0'} - '@isaacs/cliui@8.0.2': - resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} - engines: {node: '>=12'} - '@jridgewell/gen-mapping@0.3.5': resolution: {integrity: sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==} engines: {node: '>=6.0.0'} @@ -911,10 +904,6 @@ packages: resolution: {integrity: sha512-TUkJLtI163Bz5+JK0O+zDkQpn4gKwN+BovclUvCj6pI/6RXrFqQvUMRS2M+Rt8Rv0qR3wjoMoOPmpJKeOh0nBg==} engines: {node: '>= 18'} - '@pkgjs/parseargs@0.11.0': - resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} - engines: {node: '>=14'} - '@pkgr/core@0.1.1': resolution: {integrity: sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} @@ -985,10 +974,6 @@ packages: rollup: optional: true - '@sindresorhus/merge-streams@2.3.0': - resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==} - engines: {node: '>=18'} - '@stylistic/eslint-plugin@2.11.0': resolution: {integrity: sha512-PNRHbydNG5EH8NK4c+izdJlxajIR6GxcUhzsYNRsn6Myep4dsZt0qFCz3rCPnkvgO5FYibDcMqgNHUT+zvjYZw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -1897,10 +1882,6 @@ packages: flatted@3.3.2: resolution: {integrity: sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA==} - foreground-child@3.3.0: - resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==} - engines: {node: '>=14'} - fraction.js@4.3.7: resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} @@ -1964,10 +1945,6 @@ packages: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} - glob@10.4.5: - resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} - hasBin: true - glob@8.1.0: resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} engines: {node: '>=12'} @@ -1997,10 +1974,6 @@ packages: resolution: {integrity: sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - globby@14.0.2: - resolution: {integrity: sha512-s3Fq41ZVh7vbbe2PN3nrW7yC7U7MFVc5c98/iTl9c2GawNMKx/J648KQRW6WKkuU8GIbbh2IXfIRQjOZnXcTnw==} - engines: {node: '>=18'} - graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} @@ -2132,9 +2105,6 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} - jackspeak@3.4.3: - resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} - jiti@1.21.6: resolution: {integrity: sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==} hasBin: true @@ -2434,10 +2404,6 @@ packages: resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==} engines: {node: '>=8'} - minipass@7.1.2: - resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} - engines: {node: '>=16 || 14 >=14.17'} - minizlib@2.1.2: resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} engines: {node: '>= 8'} @@ -2562,9 +2528,6 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} - package-json-from-dist@1.0.0: - resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==} - package-json@10.0.1: resolution: {integrity: sha512-ua1L4OgXSBdsu1FPb7F3tYH0F48a6kxvod4pLUlGY9COeJAJQNX/sNH2IiEmsxw7lqYiAwrdHMjz1FctOsyDQg==} engines: {node: '>=18'} @@ -2607,18 +2570,10 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} - path-scurry@1.11.1: - resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} - engines: {node: '>=16 || 14 >=14.18'} - path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} - path-type@5.0.0: - resolution: {integrity: sha512-5HviZNaZcfqP95rwpv+1HDgUamezbqdSYTyzjTvwtJSnIH+3vnbmWsItli8OFEndS984VT55M3jduxZbX351gg==} - engines: {node: '>=12'} - pathe@1.1.2: resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} @@ -2915,11 +2870,6 @@ packages: resolution: {integrity: sha512-qx+xQGZVsy55CH0a1hiVwHmqjLryfh7wQyF5HO07XJ9f7dQMY/gPQHhlyDkIzJKC+x2fUCpCcUODUUUFrm7SHA==} hasBin: true - replace-in-file@8.2.0: - resolution: {integrity: sha512-hMsQtdYHwWviQT5ZbNsgfu0WuCiNlcUSnnD+aHAL081kbU9dPkPocDaHlDvAHKydTWWpx1apfcEcmvIyQk3CpQ==} - engines: {node: '>=18'} - hasBin: true - require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -2993,10 +2943,6 @@ packages: resolution: {integrity: sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==} engines: {node: '>=12'} - slash@5.1.0: - resolution: {integrity: sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==} - engines: {node: '>=14.16'} - slashes@3.0.12: resolution: {integrity: sha512-Q9VME8WyGkc7pJf6QEkj3wE+2CnvZMI+XJhwdTPR8Z/kWQRXi7boAWLDibRPyHRTUTPx5FaU7MsyrjI3yLB4HA==} @@ -3824,15 +3770,6 @@ snapshots: '@hutson/parse-repository-url@5.0.0': {} - '@isaacs/cliui@8.0.2': - dependencies: - string-width: 5.1.2 - string-width-cjs: string-width@4.2.3 - strip-ansi: 7.1.0 - strip-ansi-cjs: strip-ansi@6.0.1 - wrap-ansi: 8.1.0 - wrap-ansi-cjs: wrap-ansi@7.0.0 - '@jridgewell/gen-mapping@0.3.5': dependencies: '@jridgewell/set-array': 1.2.1 @@ -4008,9 +3945,6 @@ snapshots: '@octokit/request-error': 6.1.4 '@octokit/webhooks-methods': 5.1.0 - '@pkgjs/parseargs@0.11.0': - optional: true - '@pkgr/core@0.1.1': {} '@pnpm/config.env-replace@1.1.0': {} @@ -4074,8 +4008,6 @@ snapshots: optionalDependencies: rollup: 3.29.4 - '@sindresorhus/merge-streams@2.3.0': {} - '@stylistic/eslint-plugin@2.11.0(eslint@9.16.0(jiti@2.3.3))(typescript@5.7.2)': dependencies: '@typescript-eslint/utils': 8.16.0(eslint@9.16.0(jiti@2.3.3))(typescript@5.7.2) @@ -5202,11 +5134,6 @@ snapshots: flatted@3.3.2: {} - foreground-child@3.3.0: - dependencies: - cross-spawn: 7.0.6 - signal-exit: 4.1.0 - fraction.js@4.3.7: {} fs-extra@11.2.0: @@ -5273,15 +5200,6 @@ snapshots: dependencies: is-glob: 4.0.3 - glob@10.4.5: - dependencies: - foreground-child: 3.3.0 - jackspeak: 3.4.3 - minimatch: 9.0.5 - minipass: 7.1.2 - package-json-from-dist: 1.0.0 - path-scurry: 1.11.1 - glob@8.1.0: dependencies: fs.realpath: 1.0.0 @@ -5312,15 +5230,6 @@ snapshots: merge2: 1.4.1 slash: 4.0.0 - globby@14.0.2: - dependencies: - '@sindresorhus/merge-streams': 2.3.0 - fast-glob: 3.3.2 - ignore: 5.3.2 - path-type: 5.0.0 - slash: 5.1.0 - unicorn-magic: 0.1.0 - graceful-fs@4.2.10: {} graceful-fs@4.2.11: {} @@ -5419,12 +5328,6 @@ snapshots: isexe@2.0.0: {} - jackspeak@3.4.3: - dependencies: - '@isaacs/cliui': 8.0.2 - optionalDependencies: - '@pkgjs/parseargs': 0.11.0 - jiti@1.21.6: {} jiti@2.3.3: {} @@ -5867,8 +5770,6 @@ snapshots: minipass@5.0.0: {} - minipass@7.1.2: {} - minizlib@2.1.2: dependencies: minipass: 3.3.6 @@ -6008,8 +5909,6 @@ snapshots: p-try@2.2.0: {} - package-json-from-dist@1.0.0: {} - package-json@10.0.1: dependencies: ky: 1.7.2 @@ -6051,15 +5950,8 @@ snapshots: path-parse@1.0.7: {} - path-scurry@1.11.1: - dependencies: - lru-cache: 10.4.3 - minipass: 7.1.2 - path-type@4.0.0: {} - path-type@5.0.0: {} - pathe@1.1.2: {} perfect-debounce@1.0.0: {} @@ -6346,12 +6238,6 @@ snapshots: dependencies: jsesc: 0.5.0 - replace-in-file@8.2.0: - dependencies: - chalk: 5.3.0 - glob: 10.4.5 - yargs: 17.7.2 - require-directory@2.1.1: {} resolve-from@4.0.0: {} @@ -6408,8 +6294,6 @@ snapshots: slash@4.0.0: {} - slash@5.1.0: {} - slashes@3.0.12: {} source-map-js@1.2.1: {} diff --git a/src/config.ts b/src/config.ts index fc69af0..dca90cc 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,9 +1,8 @@ import type { Config, Context, Hooks, OverrideConfig, UserConfig } from "./types/index.js"; -import path from "node:path"; import { env } from "node:process"; import { loadConfig as c12 } from "c12"; import { kebabCase, mapValues } from "es-toolkit"; -import fs from "fs-extra"; +import { readJsonSync } from "fs-extra/esm"; import { createHooks } from "hookable"; import { Log } from "./utils/log.js"; import { dateFormat, parseRepoUrl, template } from "./utils/string.js"; @@ -38,7 +37,7 @@ function resolveConfig(config: Config): Context { const logger = new Log(config); // Load user's package.json - const pkgUser = fs.readJsonSync(path.join("package.json"), { + const pkgUser = readJsonSync("package.json", { encoding: "utf-8", }); const { name, version } = pkgUser; diff --git a/src/core/builder.ts b/src/core/builder.ts index 3b0a888..743d6fe 100644 --- a/src/core/builder.ts +++ b/src/core/builder.ts @@ -1,17 +1,17 @@ import type { Context } from "../types/index.js"; import type { Manifest } from "../types/manifest.js"; import type { UpdateJSON } from "../types/update-json.js"; -import path from "node:path"; +import { readFile, writeFile } from "node:fs/promises"; +import { basename, dirname } from "node:path"; import { env } from "node:process"; import AdmZip from "adm-zip"; import chalk from "chalk"; import { toMerged } from "es-toolkit"; import { build as buildAsync } from "esbuild"; -import fs from "fs-extra"; -import { globbySync } from "globby"; -import { replaceInFileSync } from "replace-in-file"; -import { generateHashSync } from "../utils/crypto.js"; -import { dateFormat, toArray } from "../utils/string.js"; +import { copy, emptyDir, move, outputJSON, readJSON, writeJson } from "fs-extra/esm"; +import { glob } from "tinyglobby"; +import { generateHash } from "../utils/crypto.js"; +import { dateFormat, replaceInFile, toArray } from "../utils/string.js"; import { Base } from "./base.js"; export default class Build extends Base { @@ -37,25 +37,21 @@ export default class Build extends Base { ); await this.ctx.hooks.callHook("build:init", this.ctx); - fs.emptyDirSync(dist); + await emptyDir(dist); await this.ctx.hooks.callHook("build:mkdir", this.ctx); this.logger.tip("Preparing static assets"); - this.copyAssets(); + await this.makeAssets(); await this.ctx.hooks.callHook("build:copyAssets", this.ctx); this.logger.debug("Preparing manifest"); - this.makeManifest(); + await this.makeManifest(); await this.ctx.hooks.callHook("build:makeManifest", this.ctx); this.logger.debug("Preparing locale files"); - this.prepareLocaleFiles(); + await this.prepareLocaleFiles(); await this.ctx.hooks.callHook("build:fluent", this.ctx); - this.logger.debug("Replacing placeholders"); - this.replaceString(); - await this.ctx.hooks.callHook("build:replace", this.ctx); - this.logger.tip("Bundling scripts"); await this.esbuild(); await this.ctx.hooks.callHook("build:bundle", this.ctx); @@ -67,7 +63,7 @@ export default class Build extends Base { await this.pack(); await this.ctx.hooks.callHook("build:pack", this.ctx); - this.makeUpdateJson(); + await this.makeUpdateJson(); await this.ctx.hooks.callHook("build:makeUpdateJSON", this.ctx); } @@ -80,31 +76,47 @@ export default class Build extends Base { /** * Copys files in `Config.build.assets` to `Config.dist` */ - copyAssets() { + async makeAssets() { const { source, dist, build } = this.ctx; - const { assets } = build; + const { assets, define } = build; + + // We should ignore node_modules/ by default, glob this folder will be very slow + const paths = await glob(assets, { ignore: ["node_modules", ".git", dist] }); + const newPaths = paths.map(p => `${dist}/addon/${p.replace(new RegExp(toArray(source).join("|")), "")}`); - const files = globbySync(assets); - files.forEach((file) => { - const newPath = `${dist}/addon/${file.replace(new RegExp(toArray(source).join("|")), "")}`; - this.logger.debug(`Copy ${file} to ${newPath}`); - fs.copySync(file, newPath); + // Copys files in `Config.build.assets` to `Config.dist` + await Promise.all(paths.map(async (file, i) => { + await copy(file, newPaths[i]); + this.logger.debug(`Copy ${file} to ${newPaths[i]}`); + })); + + // Replace all `placeholder.key` to `placeholder.value` for all files in `dist` + const replaceMap = new Map( + Object.keys(define).map(key => [ + new RegExp(`__${key}__`, "g"), + define[key], + ]), + ); + this.logger.debug("replace map: ", replaceMap); + await replaceInFile({ + files: newPaths, + from: Array.from(replaceMap.keys()), + to: Array.from(replaceMap.values()), + isGlob: false, }); } /** * Override user's manifest * - * Write `applications.gecko` to `manifest.json` - * */ - makeManifest() { + async makeManifest() { if (!this.ctx.build.makeManifest.enable) return; const { name, id, updateURL, dist, version } = this.ctx; - const userData = fs.readJSONSync( + const userData = await readJSON( `${dist}/addon/manifest.json`, ) as Manifest; const template: Manifest = { @@ -124,120 +136,85 @@ export default class Build extends Base { const data: Manifest = toMerged(userData, template); this.logger.debug("manifest: ", JSON.stringify(data, null, 2)); - fs.outputJSONSync(`${dist}/addon/manifest.json`, data, { spaces: 2 }); + outputJSON(`${dist}/addon/manifest.json`, data, { spaces: 2 }); } - /** - * Replace all `placeholder.key` to `placeholder.value` for all files in `dist` - */ - replaceString() { - const { dist, build } = this.ctx; - const { assets, define } = build; - - const replaceMap = new Map( - Object.keys(define).map(key => [ - new RegExp(`__${key}__`, "g"), - define[key], - ]), - ); - this.logger.debug("replace map: ", replaceMap); - - const replaceResult = replaceInFileSync({ - files: toArray(assets).map( - asset => `${dist}/addon/${asset.split("/").slice(1).join("/")}`, - ), - from: Array.from(replaceMap.keys()), - to: Array.from(replaceMap.values()), - countMatches: true, - }); - - this.logger.debug( - "Run replace in ", - replaceResult - .filter(f => f.hasChanged) - .map(f => `${f.file} : ${f.numReplacements} / ${f.numMatches}`), - ); - } - - prepareLocaleFiles() { + async prepareLocaleFiles() { const { dist, namespace, build } = this.ctx; + // https://regex101.com/r/lQ9x5p/1 + // eslint-disable-next-line regexp/no-super-linear-backtracking + const FTL_MESSAGE_PATTERN = /^(?[a-z]\S*)( *= *)(?.*)$/gim; + const HTML_DATAI10NID_PATTERN = new RegExp(`(data-l10n-id)="((?!${namespace})\\S*)"`, "g"); + // Walk the sub folders of `build/addon/locale` - const localeNames = globbySync(`${dist}/addon/locale/**`, { onlyDirectories: true }) - .map(locale => path.basename(locale)); + const localeNames = (await glob(`${dist}/addon/locale/*`, { onlyDirectories: true })) + .map(locale => basename(locale)); this.logger.debug("locale names: ", localeNames); + const Messages = new Map>(); for (const localeName of localeNames) { - // rename *.ftl to addonRef-*.ftl - if (build.fluent.prefixLocaleFiles === true) { - globbySync(`${dist}/addon/locale/${localeName}/**/*.ftl`, {}) - .forEach((f) => { - fs.moveSync( - f, - `${path.dirname(f)}/${namespace}-${path.basename(f)}`, - ); - this.logger.debug(`Prefix filename: ${f}`); - }); - } - - // Prefix Fluent messages in each ftl - const MessageInThisLang = new Set(); - replaceInFileSync({ - files: [`${dist}/addon/locale/${localeName}/**/*.ftl`], - processor: (fltContent) => { - const lines = fltContent.split("\n"); - const prefixedLines = lines.map((line: string) => { - // https://regex101.com/r/lQ9x5p/1 - const match = line.match( - // eslint-disable-next-line regexp/no-super-linear-backtracking - /^(?[a-z]\S*)( *= *)(?.*)$/im, - ); - if (match && match.groups) { - MessageInThisLang.add(match.groups.message); - return build.fluent.prefixFluentMessages - ? `${namespace}-${line}` - : line; - } - else { - return line; - } - }); - return prefixedLines.join("\n"); - }, - }); + // Prefix Fluent messages in each ftl, add message to set. + const MessageInThisLang = new Set(); + const ftlPaths = await glob(`${dist}/addon/locale/${localeName}/**/*.ftl`); + await Promise.all(ftlPaths.map(async (ftlPath: string) => { + const ftlContent = await readFile(ftlPath, "utf-8"); + const matchs = [...ftlContent.matchAll(FTL_MESSAGE_PATTERN)]; + const newFtlContent = matchs.reduce((content, match) => { + if (!match.groups?.message) + return content; + MessageInThisLang.add(match.groups.message); + return content.replace(match.groups.message, `${namespace}-${match.groups.message}`); + }, ftlContent); + + // If prefixFluentMessages===true, we save the changed ftl file, + // otherwise discard the changes + if (build.fluent.prefixFluentMessages) + await writeFile(ftlPath, newFtlContent); + + // rename *.ftl to addonRef-*.ftl + if (build.fluent.prefixLocaleFiles === true) { + await move(ftlPath, `${dirname(ftlPath)}/${namespace}-${basename(ftlPath)}`); + this.logger.debug(`Prefix filename: ${ftlPath}`); + } + })); + Messages.set(localeName, MessageInThisLang); + } - // Prefix Fluent messages in xhtml - const MessagesInHTML = new Set(); - replaceInFileSync({ - files: [ - `${dist}/addon/**/*.xhtml`, - `${dist}/addon/**/*.html`, - ], - processor: (input) => { - const matches = [ - ...input.matchAll( - new RegExp(`(data-l10n-id)="((?!${namespace})\\S*)"`, "g"), - ), - ]; - matches.forEach((match) => { - const [matched, attrKey, attrVal] = match; - if (!MessageInThisLang.has(attrVal)) { - this.logger.warn(`${attrVal} don't exist in ${localeName}`); - return; - } - if (!this.ctx.build.fluent.prefixFluentMessages) { - return; - } - input = input.replace( - matched, - `${attrKey}="${namespace}-${attrVal}"`, - ); - MessagesInHTML.add(attrVal); - }); - return input; - }, + // Prefix Fluent messages in xhtml + const MessagesInHTML = new Set(); + const htmlPaths = await glob([ + `${dist}/addon/**/*.xhtml`, + `${dist}/addon/**/*.html`, + ]); + await Promise.all(htmlPaths.map(async (htmlPath) => { + const content = await readFile(htmlPath, "utf-8"); + const matches = [...content.matchAll(HTML_DATAI10NID_PATTERN)]; + const newHtmlContent = matches.reduce((result, match) => { + const [matched, attrKey, attrVal] = match; + MessagesInHTML.add(attrVal); + return result.replace( + matched, + `${attrKey}="${namespace}-${attrVal}"`, + ); + }, content); + + if (build.fluent.prefixFluentMessages) + await writeFile(htmlPath, newHtmlContent); + })); + + // Check miss 1: Cross check in diff locale + + // Check miss 2: Check ids in HTML but not in ftl + MessagesInHTML.forEach((messageInHTML) => { + const miss = new Set(); + Messages.forEach((messagesInThisLang, lang) => { + if (!messagesInThisLang.has(messageInHTML)) + miss.add(lang); }); - } + if (miss.size !== 0) + this.logger.warn(`FTL message "${messageInHTML}" don't exist in "${[...miss].join(", ")}"`); + }); } esbuild() { @@ -253,19 +230,16 @@ export default class Build extends Base { ); } - makeUpdateJson() { + async makeUpdateJson() { const { dist, xpiName, id, version, xpiDownloadLink, build } = this.ctx; - const manifest = fs.readJSONSync( + const manifest = await readJSON( `${dist}/addon/manifest.json`, ) as Manifest; const min = manifest.applications?.zotero?.strict_min_version; const max = manifest.applications?.zotero?.strict_max_version; - const updateHash = generateHashSync( - path.join(dist, `${xpiName}.xpi`), - "sha512", - ); + const updateHash = await generateHash(`${dist}/${xpiName}.xpi`, "sha512"); const data: UpdateJSON = { addons: { @@ -290,9 +264,9 @@ export default class Build extends Base { }, }; - fs.writeJsonSync(`${dist}/update-beta.json`, data, { spaces: 2 }); + await writeJson(`${dist}/update-beta.json`, data, { spaces: 2 }); if (!this.isPreRelease) - fs.writeJsonSync(`${dist}/update.json`, data, { spaces: 2 }); + await writeJson(`${dist}/update.json`, data, { spaces: 2 }); this.logger.debug( `Prepare Update.json for ${ @@ -305,15 +279,8 @@ export default class Build extends Base { async pack() { const { dist, xpiName } = this.ctx; - const zip = new AdmZip(); - - const paths = globbySync("**", { cwd: `${dist}/addon`, dot: true }); - paths.forEach((relativePath) => { - const absolutePath = path.resolve(`${dist}/addon`, relativePath); - zip.addLocalFile(absolutePath, path.dirname(relativePath)); - }); - + zip.addLocalFolder(`${dist}/addon`); zip.writeZip(`${dist}/${xpiName}.xpi`); } } diff --git a/src/core/releaser/base.ts b/src/core/releaser/base.ts index 050ab2e..007daa6 100644 --- a/src/core/releaser/base.ts +++ b/src/core/releaser/base.ts @@ -1,6 +1,6 @@ import type { Context } from "../../types/index.js"; -import { globbySync } from "globby"; import { isCI } from "std-env"; +import { globSync } from "tinyglobby"; import { Base } from "../base.js"; export abstract class ReleaseBase extends Base { @@ -15,7 +15,7 @@ export abstract class ReleaseBase extends Base { checkFiles() { const { dist } = this.ctx; - if (globbySync(`${dist}/*.xpi`).length === 0) { + if (globSync(`${dist}/*.xpi`).length === 0) { throw new Error("No xpi file found, are you sure you have run build?"); } } diff --git a/src/core/releaser/github.ts b/src/core/releaser/github.ts index 30bbc71..58fefec 100644 --- a/src/core/releaser/github.ts +++ b/src/core/releaser/github.ts @@ -1,10 +1,10 @@ import type { Context } from "../../types/index.js"; +import { readFile, stat } from "node:fs/promises"; import { basename, join } from "node:path"; import { env } from "node:process"; -import fs from "fs-extra"; -import { globbySync } from "globby"; import mime from "mime"; import { Octokit } from "octokit"; +import { glob } from "tinyglobby"; import { ReleaseBase } from "./base.js"; export default class GitHub extends ReleaseBase { @@ -92,10 +92,10 @@ export default class GitHub extends ReleaseBase { .uploadReleaseAsset({ ...this.remote, release_id: releaseID, - data: fs.readFileSync(asset) as unknown as string, + data: await readFile(asset) as unknown as string, headers: { "content-type": mime.getType(asset) || "application/octet-stream", - "content-length": fs.statSync(asset).size, + "content-length": (await stat(asset)).size, }, name: basename(asset), }) @@ -114,7 +114,7 @@ export default class GitHub extends ReleaseBase { const { dist, version } = this.ctx; - const assets = globbySync(`${dist}/update*.json`) + const assets = (await glob(`${dist}/update*.json`)) .map(p => basename(p)); const release diff --git a/src/core/tester.ts b/src/core/tester.ts index ffa229f..21651ff 100644 --- a/src/core/tester.ts +++ b/src/core/tester.ts @@ -4,9 +4,9 @@ import http from "node:http"; import { join, resolve } from "node:path"; import process, { cwd, env, exit } from "node:process"; import { build } from "esbuild"; -import { emptyDirSync, outputFile, outputFileSync, outputJSON, pathExists } from "fs-extra/esm"; -import { globbySync } from "globby"; +import { emptyDirSync, outputFile, outputJSON, pathExists } from "fs-extra/esm"; import { isCI } from "std-env"; +import { glob } from "tinyglobby"; import { Xvfb } from "xvfb-ts"; import { saveResource } from "../utils/file.js"; import { installXvfb, installZoteroLinux } from "../utils/headless.js"; @@ -193,7 +193,7 @@ function waitUtilAsync(condition, interval = 100, timeout = 1e4) { */ `; - outputFileSync(`${this.testPluginDir}/bootstrap.js`, code); + await outputFile(`${this.testPluginDir}/bootstrap.js`, code); this.logger.tip("Saved bootstrap.js for test"); } @@ -248,7 +248,7 @@ function waitUtilAsync(condition, interval = 100, timeout = 1e4) { } await build({ - entryPoints: globbySync(`${dir}/**/*.spec.{js,ts}`), + entryPoints: await glob(`${dir}/**/*.spec.{js,ts}`), outdir: this.testBuildDir, bundle: true, target: "firefox115", @@ -256,7 +256,7 @@ function waitUtilAsync(condition, interval = 100, timeout = 1e4) { }); } - const testFiles = globbySync(`${this.testBuildDir}/**/*.spec.js`); + const testFiles = await glob(`${this.testBuildDir}/**/*.spec.js`); // Sort the test files to ensure consistent test order testFiles.sort(); diff --git a/src/types/config.ts b/src/types/config.ts index 83e3772..f54560d 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -315,7 +315,6 @@ interface BuildHooks { "build:copyAssets": (ctx: Context) => void | Promise; "build:makeManifest": (ctx: Context) => void | Promise; "build:fluent": (ctx: Context) => void | Promise; - "build:replace": (ctx: Context) => void | Promise; "build:bundle": (ctx: Context) => void | Promise; "build:pack": (ctx: Context) => void | Promise; "build:makeUpdateJSON": (ctx: Context) => void | Promise; diff --git a/src/utils/crypto.ts b/src/utils/crypto.ts index d9c8430..9c4baa4 100644 --- a/src/utils/crypto.ts +++ b/src/utils/crypto.ts @@ -1,13 +1,13 @@ -import * as crypto from "node:crypto"; -import fs from "fs-extra"; +import { createHash } from "node:crypto"; +import { createReadStream, readFileSync } from "node:fs"; export function generateHash( filePath: string, algorithm: "sha256" | "sha512" | string, ): Promise { return new Promise((resolve, reject) => { - const hash = crypto.createHash(algorithm); - const stream = fs.createReadStream(filePath); + const hash = createHash(algorithm); + const stream = createReadStream(filePath); stream.on("data", (data) => { hash.update(data); @@ -28,7 +28,7 @@ export function generateHashSync( filePath: string, algorithm: "sha256" | "sha512" | string, ): string { - const data = fs.readFileSync(filePath); - const hash = crypto.createHash(algorithm).update(data).digest("hex"); + const data = readFileSync(filePath); + const hash = createHash(algorithm).update(data).digest("hex"); return `${algorithm}:${hash}`; } diff --git a/src/utils/file.ts b/src/utils/file.ts index 623ba91..677a954 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -1,4 +1,4 @@ -import { outputFile } from "fs-extra"; +import { outputFile } from "fs-extra/esm"; export async function saveResource(url: string, path: string) { const res = await fetch(url); diff --git a/src/utils/manifest.ts b/src/utils/manifest.ts deleted file mode 100644 index 41bd22e..0000000 --- a/src/utils/manifest.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Patch web-ext's getValidatedManifest - * @see https://github.com/mozilla/web-ext?tab=readme-ov-file#using-web-ext-in-nodejs-code - * @see https://github.com/mozilla/web-ext/blob/master/src/util/manifest.js#L15 - */ -export function getValidatedManifest(_sourceDir?: string) { - return { - manifest_version: 2, - name: "zoterp-plugin-scaffold-fake-name", - version: "0.0.0.0", - }; -} diff --git a/src/utils/string.ts b/src/utils/string.ts index ffdd758..0c18a29 100644 --- a/src/utils/string.ts +++ b/src/utils/string.ts @@ -1,3 +1,6 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { glob } from "tinyglobby"; + export function dateFormat(fmt: string, date: Date) { let ret; const opt: { [key: string]: string } = { @@ -50,3 +53,31 @@ export function parseRepoUrl(url?: string) { const [, owner, repo] = match; return { owner, repo }; } + +export function replace(contents: string, from: RegExp | RegExp[], to: string | string[]) { + const froms = Array.isArray(from) ? from : [from]; + const tos = Array.isArray(to) + ? to + : Array.from({ length: froms.length }, () => to); + + if (froms.length !== tos.length) { + throw new Error("The lengths of 'from' and 'to' must be equal"); + } + + return froms.reduce((result, pattern, index) => result.replace(pattern, tos[index]), contents); +} + +export async function replaceInFile({ files, from, to, isGlob = true }: { + files: string | string[]; + from: RegExp | RegExp[]; + to: string | string[]; + isGlob?: boolean; +}) { + const paths = isGlob ? await glob(files) : toArray(files); + await Promise.all(paths.map(async (path) => { + const contents = await readFile(path, "utf-8"); + const newContents = replace(contents, from, to); + if (contents !== newContents) + await writeFile(path, newContents); + })); +} diff --git a/src/utils/zotero-runner.ts b/src/utils/zotero-runner.ts index c89075e..f7cc9f6 100644 --- a/src/utils/zotero-runner.ts +++ b/src/utils/zotero-runner.ts @@ -1,10 +1,10 @@ import type { ChildProcessWithoutNullStreams } from "node:child_process"; import { execSync, spawn } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; +import { readFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { env } from "node:process"; import { delay } from "es-toolkit"; -import { outputFileSync, outputJSONSync, readJSONSync, removeSync } from "fs-extra/esm"; +import { outputFile, outputJSON, pathExists, readJSON, remove } from "fs-extra/esm"; import { isLinux, isMacOS, isWindows } from "std-env"; import { Log } from "./log.js"; import { isRunning } from "./process.js"; @@ -76,8 +76,8 @@ export class ZoteroRunner { let exsitedPrefs: string[] = []; const prefsPath = join(this.options.profilePath, "prefs.js"); - if (existsSync(prefsPath)) { - const PrefsLines = readFileSync(prefsPath, "utf-8").split("\n"); + if (await pathExists(prefsPath)) { + const PrefsLines = (await readFile(prefsPath, "utf-8")).split("\n"); exsitedPrefs = PrefsLines.map((line: string) => { if ( line.includes("extensions.lastAppBuildId") @@ -92,7 +92,7 @@ export class ZoteroRunner { }); } const updatedPrefs = [...defaultPrefs, ...exsitedPrefs, ...customPrefs].join("\n"); - outputFileSync(prefsPath, updatedPrefs, "utf-8"); + await outputFile(prefsPath, updatedPrefs, "utf-8"); logger.debug("The /prefs.js has been modified."); // Install plugins in proxy file mode @@ -158,31 +158,27 @@ export class ZoteroRunner { // Create a proxy file const addonProxyFilePath = join(this.options.profilePath, `extensions/${id}`); const buildPath = resolve(sourceDir); - if ( - !existsSync(addonProxyFilePath) - || readFileSync(addonProxyFilePath, "utf-8") !== buildPath - ) { - outputFileSync(addonProxyFilePath, buildPath); - logger.debug( - [ - `Addon proxy file has been updated.`, - ` File path: ${addonProxyFilePath}`, - ` Addon path: ${buildPath}`, - ].join("\n"), - ); - } + + await outputFile(addonProxyFilePath, buildPath); + logger.debug( + [ + `Addon proxy file has been updated.`, + ` File path: ${addonProxyFilePath}`, + ` Addon path: ${buildPath}`, + ].join("\n"), + ); // Delete XPI file const addonXpiFilePath = join(this.options.profilePath, `extensions/${id}.xpi`); - if (existsSync(addonXpiFilePath)) { - removeSync(addonXpiFilePath); + if (await pathExists(addonXpiFilePath)) { + await remove(addonXpiFilePath); logger.debug(`XPI file found, removed.`); } // Force enable plugin in extensions.json const addonInfoFilePath = join(this.options.profilePath, "extensions.json"); - if (existsSync(addonInfoFilePath)) { - const content = readJSONSync(addonInfoFilePath); + if (await pathExists(addonInfoFilePath)) { + const content = await readJSON(addonInfoFilePath); content.addons = content.addons.map((addon: any) => { if (addon.id === id && addon.active === false) { addon.active = true; @@ -191,7 +187,7 @@ export class ZoteroRunner { } return addon; }); - outputJSONSync(addonInfoFilePath, content); + await outputJSON(addonInfoFilePath, content); } } diff --git a/src/vendor/index.ts b/src/vendor/index.ts index 9f280c9..b73afcd 100644 --- a/src/vendor/index.ts +++ b/src/vendor/index.ts @@ -1,6 +1,7 @@ +export { replace, replaceInFile } from "../utils/string.js"; + /** * Export some dependencies of scaffold from this directory to make it easier for the user to invoke */ export * as esToolkit from "es-toolkit"; export * as fse from "fs-extra/esm"; -export * as replaceInFile from "replace-in-file";