Merge pull request #3342 from opral/parjs-309-very-very-slow-building…

…-of-paraglide-translation-files-since

speed up compiler

.changeset/cuddly-rockets-fry.md

@@ -0,0 +1,5 @@
+---
+"@inlang/sdk": patch
+---
+
+improve performance: only write db changes to lix on close

.changeset/loud-walls-hug.md

@@ -0,0 +1,5 @@
+---
+"@lix-js/sdk": patch
+---
+
+fix: account for db close via "driver has already been destroyed" in file queue

inlang/packages/paraglide-js/examples/cli/build.js

@@ -0,0 +1,15 @@
+import { compile } from "@inlang/paraglide-js/compiler";
+
+console.log("Compiling project...");
+
+const startTime = Date.now();
+
+await compile({
+	project: "./project.inlang",
+	outdir: "./src/paraglide",
+});
+
+const endTime = Date.now();
+
+console.log("Project compiled successfully!");
+console.log(`Time taken: ${endTime - startTime}ms`);

inlang/packages/paraglide-js/examples/cli/package.json

@@ -4,6 +4,7 @@
 	"version": "0.0.0",
 	"private": true,
 	"scripts": {
+		"bench": "node ./build.js",
 		"build": "paraglide-js compile --project ./project.inlang && tsc",
 		"pretest": "paraglide-js compile --project ./project.inlang",
 		"test": "tsc --noEmit && node --experimental-strip-types ./src/main.ts"

inlang/packages/paraglide-js/package.json

@@ -23,6 +23,7 @@
 	],
 	"scripts": {
 		"dev": "tsc --watch",
+		"bench": "vitest bench --run",
 		"build": "npm run env-variables && tsc --build",
 		"test": "npm run env-variables && tsc --noEmit && vitest run --coverage ./src/**/*",
 		"test:watch": "npm run env-variables && vitest --watch ./src/**/*",
@@ -37,18 +38,18 @@
 		"commander": "11.1.0",
 		"consola": "3.2.3",
 		"json5": "2.2.3",
-		"prettier": "^3.4.2",
-		"prettier-plugin-jsdoc": "^1.3.0",
 		"unplugin": "^2.1.2",
 		"typescript": "^5.7.3"
 	},
 	"devDependencies": {
+		"@inlang/plugin-message-format": "workspace:*",
 		"@eslint/js": "^9.12.0",
 		"@opral/tsconfig": "workspace:*",
 		"@rollup/plugin-virtual": "3.0.1",
 		"@ts-morph/bootstrap": "0.20.0",
 		"@types/node": "^22.10.5",
 		"@vitest/coverage-v8": "2.0.5",
+		"prettier": "^3.4.2",
 		"eslint": "^9.12.0",
 		"memfs": "4.6.0",
 		"rollup": "3.29.1",

inlang/packages/paraglide-js/src/compiler/compile.bench.ts

@@ -0,0 +1,57 @@
+import { bench } from "vitest";
+import nodeFs from "node:fs";
+import path from "node:path";
+import { memfs } from "memfs";
+import { compile } from "./compile.js";
+
+bench("compile 1000 messages", async () => {
+	const numMessages = 1000;
+
+	// the root of the repo
+	const repositoryRoot = import.meta.url
+		.slice(0, import.meta.url.lastIndexOf("inlang/packages"))
+		.replace("file://", "");
+
+	// load the inlang message format plugin to simulate a real project
+	const pluginAsText = nodeFs.readFileSync(
+		path.join(
+			repositoryRoot,
+			"inlang/packages/plugins/inlang-message-format/dist/index.js"
+		),
+		"utf8"
+	);
+
+	const mockMessages = Object.fromEntries(
+		Array.from({ length: numMessages }, (_, i) => [
+			`message_${i}`,
+			`Hello world {username}`,
+		])
+	);
+
+	const fs = memfs({
+		"/plugin.js": pluginAsText,
+		"/en.json": JSON.stringify(mockMessages),
+		"/project.inlang/settings.json": JSON.stringify({
+			baseLocale: "en",
+			locales: ["en"],
+			modules: ["/plugin.js"],
+			"plugin.inlang.messageFormat": {
+				pathPattern: "/{locale}.json",
+			},
+		}),
+	}).fs as unknown as typeof import("node:fs");
+
+	const start = performance.now();
+
+	await compile({
+		project: "/project.inlang",
+		outdir: "/output",
+		fs: fs,
+	});
+
+	const end = performance.now();
+
+	const duration = end - start;
+
+	console.log(`Compiled ${numMessages} messages in ${duration}ms`);
+});

inlang/packages/paraglide-js/src/compiler/compile.ts

@@ -27,16 +27,17 @@ import {
 export async function compile(args: {
 	project: string;
 	outdir: string;
-	fs: typeof import("node:fs");
+	fs?: typeof import("node:fs");
 	options?: ParaglideCompilerOptions;
 }): Promise<void> {
+	const fs = args.fs ?? (await import("node:fs"));
 	const absoluteOutdir = path.resolve(process.cwd(), args.outdir);
 
-	const localAccount = getLocalAccount({ fs: args.fs });
+	const localAccount = getLocalAccount({ fs });
 
 	const project = await loadProjectFromDirectory({
 		path: args.project,
-		fs: args.fs,
+		fs,
 		account: localAccount,
 		appId: ENV_VARIABLES.PARJS_APP_ID,
 	});
@@ -46,15 +47,15 @@ export async function compile(args: {
 		options: args.options,
 	});
 
-	await writeOutput(absoluteOutdir, output, args.fs.promises);
+	await writeOutput(absoluteOutdir, output, fs.promises);
 
 	if (!localAccount) {
 		const activeAccount = await project.lix.db
 			.selectFrom("active_account")
 			.selectAll()
 			.executeTakeFirstOrThrow();
 
-		saveLocalAccount({ fs: args.fs, account: activeAccount });
+		saveLocalAccount({ fs, account: activeAccount });
 	}
 
 	await project.close();

inlang/packages/paraglide-js/src/compiler/compileProject.ts

@@ -1,7 +1,6 @@
 import { compileBundle } from "./compileBundle.js";
 import { DEFAULT_REGISTRY } from "./registry.js";
 import { selectBundleNested, type InlangProject } from "@inlang/sdk";
-import * as prettier from "prettier";
 import { lookup } from "../services/lookup.js";
 import { emitDts } from "./emit-dts.js";
 import { generateLocaleModules } from "./output-structure/locale-modules.js";
@@ -117,46 +116,8 @@ export const compileProject = async (args: {
 		}
 	}
 
-	return await formatFiles(output);
-};
-
-async function formatFiles(
-	files: Record<string, string>
-): Promise<Record<string, string>> {
-	const output: Record<string, string> = {};
-	const promises: Promise<void>[] = [];
-
-	for (const [key, value] of Object.entries(files)) {
-		if (!key.endsWith(".js")) {
-			output[key] = value;
-			continue;
-		}
-
-		promises.push(
-			new Promise((resolve, reject) => {
-				fmt(value)
-					.then((formatted) => {
-						output[key] = formatted;
-						resolve();
-					})
-					.catch(reject);
-			})
-		);
-	}
-
-	await Promise.all(promises);
 	return output;
-}
-
-async function fmt(js: string): Promise<string> {
-	return await prettier.format(js, {
-		arrowParens: "always",
-		singleQuote: true,
-		printWidth: 100,
-		parser: "babel",
-		plugins: ["prettier-plugin-jsdoc"],
-	});
-}
+};
 
 export function getFallbackMap<T extends string>(
 	locales: T[],

inlang/packages/paraglide-js/vitest.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+	test: {
+		coverage: {
+			include: ["src/**/*"],
+			exclude: ["src/cli/**/*", "src/bundler-plugins/**/*"],
+		},
+	},
+});

inlang/packages/sdk/src/project/loadProject.ts

@@ -1,9 +1,8 @@
 import { toBlob, type Account, type Lix } from "@lix-js/sdk";
 import type { InlangPlugin } from "../plugin/schema.js";
 import type { ProjectSettings } from "../json-schema/settings.js";
-import { type SqliteDatabase } from "sqlite-wasm-kysely";
+import { contentFromDatabase, type SqliteDatabase } from "sqlite-wasm-kysely";
 import { initDb } from "../database/initDb.js";
-import { initHandleSaveToLixOnChange } from "./initHandleSaveToLixOnChange.js";
 import {
 	importPlugins,
 	type PreprocessPluginBeforeImportFunction,
@@ -98,18 +97,6 @@ export async function loadProject(args: {
 	// 	settings,
 	// });
 
-	// TODO implement garbage collection/a proper queue.
-	//      for the protoype and tests, it seems good enough
-	//      without garbage collection of old promises.
-	const pendingSaveToLixPromises: Promise<unknown>[] = [];
-
-	await initHandleSaveToLixOnChange({
-		sqlite: args.sqlite,
-		db,
-		lix: args.lix,
-		pendingPromises: pendingSaveToLixPromises,
-	});
-
 	// not awaiting to not block the load time of a project
 	maybeCaptureLoadedProject({
 		db,
@@ -210,18 +197,31 @@ export async function loadProject(args: {
 			).map((output) => ({ ...output, pluginKey }));
 		},
 		close: async () => {
+			await saveDbToLix({ sqlite: args.sqlite, lix: args.lix });
 			await db.destroy();
 			await args.lix.db.destroy();
 		},
 		_sqlite: args.sqlite,
 		toBlob: async () => {
-			await Promise.all(pendingSaveToLixPromises);
+			await saveDbToLix({ sqlite: args.sqlite, lix: args.lix });
 			return await toBlob({ lix: args.lix });
 		},
 		lix: args.lix,
 	};
 }
 
+async function saveDbToLix(args: {
+	sqlite: SqliteDatabase;
+	lix: Lix;
+}): Promise<void> {
+	const data = contentFromDatabase(args.sqlite);
+	await args.lix.db
+		.updateTable("file")
+		.set("data", data)
+		.where("path", "=", "/db.sqlite")
+		.execute();
+}
+
 /**
  * Old leftover migration from v1. Probably not needed anymore.
  *

packages/lix-sdk/src/file-queue/file-queue-process.ts

@@ -93,7 +93,11 @@ export async function initFileQueueProcess(args: {
 			}
 		} catch (e) {
 			// database has been closed, ignore
-			if (e instanceof Error && e.message.includes("DB has been closed")) {
+			if (
+				e instanceof Error &&
+				(e.message.includes("DB has been closed") ||
+					e.message.includes("driver has already been destroyed"))
+			) {
 				return;
 			}
 			console.error("file queue failed ", e);