Use new version of @vscode/l10n-dev that uses tree-sitter (#169668)

Use new version of @vscode/l10n-dev Required a bit more refactoring because an API had to be made async.
microsoft · Dec 20, 2022 · 0510da8 · 0510da8
1 parent 88c8146
commit 0510da8
Show file tree

Hide file tree

Showing 4 changed files with 123 additions and 94 deletions.
diff --git a/build/lib/i18n.js b/build/lib/i18n.js
diff --git a/build/lib/i18n.ts b/build/lib/i18n.ts
@@ -6,7 +6,8 @@
 import * as path from 'path';
 import * as fs from 'fs';
 
-import { merge, through, ThroughStream, writeArray } from 'event-stream';
+import { map, merge, through, ThroughStream } from 'event-stream';
+import * as jsonMerge from 'gulp-merge-json';
 import * as File from 'vinyl';
 import * as Is from 'is';
 import * as xml2js from 'xml2js';
@@ -579,57 +580,67 @@ export function createXlfFilesForCoreBundle(): ThroughStream {
 	});
 }
 
-function createL10nBundleForExtension(extensionFolderName: string): ThroughStream {
-	const result = through();
-	gulp.src([
-		// For source code of extensions
-		`extensions/${extensionFolderName}/src/**/*.{ts,tsx}`,
-		// For any dependencies pulled in (think vscode-css-languageservice or @vscode/emmet-helper)
-		`extensions/${extensionFolderName}/node_modules/**/*.{js,jsx}`,
-		// For any dependencies pulled in that bundle @vscode/l10n. They needed to export the bundle
-		`extensions/${extensionFolderName}/node_modules/**/bundle.l10n.json`,
-	]).pipe(writeArray((err, files: File[]) => {
-		if (err) {
-			result.emit('error', err);
-			return;
-		}
+function createL10nBundleForExtension(extensionFolderName: string): NodeJS.ReadWriteStream {
+	return gulp
+		.src([
+			// For source code of extensions
+			`extensions/${extensionFolderName}/{src,client,server}/**/*.{ts,tsx}`,
+			// // For any dependencies pulled in (think vscode-css-languageservice or @vscode/emmet-helper)
+			`extensions/${extensionFolderName}/**/node_modules/{@vscode,vscode-*}/**/*.{js,jsx}`,
+			// // For any dependencies pulled in that bundle @vscode/l10n. They needed to export the bundle
+			`extensions/${extensionFolderName}/**/node_modules/{@vscode,vscode-*}/**/bundle.l10n.json`,
+		])
+		.pipe(map(function (data, callback) {
+			const file = data as File;
+			if (!file.isBuffer()) {
+				// Not a buffer so we drop it
+				callback();
+				return;
+			}
+			const extension = path.extname(file.relative);
+			if (extension !== '.json') {
+				const contents = file.contents.toString('utf8');
+				getL10nJson([{ contents, extension }])
+					.then((json) => {
+						callback(undefined, new File({
+							path: `extensions/${extensionFolderName}/bundle.l10n.json`,
+							contents: Buffer.from(JSON.stringify(json), 'utf8')
+						}));
+					})
+					.catch((err) => {
+						callback(new Error(`File ${file.relative} threw an error when parsing: ${err}`));
+					});
+				// signal pause?
+				return false;
+			}
 
-		const buffers = files.filter(file => file.isBuffer());
-
-		const json = getL10nJson(buffers
-			.filter(file => path.extname(file.path) !== '.json')
-			.map(file => ({
-				contents: file.contents.toString('utf8'),
-				extension: path.extname(file.path)
-			})));
-
-		buffers
-			.filter(file => path.extname(file.path) === '.json')
-			.forEach(file => {
-				const bundleJson = JSON.parse(file.contents.toString('utf8'));
-				for (const key in bundleJson) {
-					if (
-						// some validation of the bundle.l10n.json format
-						typeof bundleJson[key] !== 'string' &&
-						(typeof bundleJson[key].message !== 'string' || !Array.isArray(bundleJson[key].comment))
-					) {
-						console.error(`Invalid bundle.l10n.json file. The value for key ${key} is not in the expected format. Skipping key...`);
-						continue;
-					}
-					json[key] = bundleJson[key];
-				}
-			});
+			// for bundle.l10n.jsons
+			let bundleJson;
+			try {
+				bundleJson = JSON.parse(file.contents.toString('utf8'));
+			} catch (err) {
+				callback(new Error(`File ${file.relative} threw an error when parsing: ${err}`));
+				return;
+			}
 
-		if (Object.keys(json).length > 0) {
-			result.emit('data', new File({
-				path: `extensions/${extensionFolderName}/bundle.l10n.json`,
-				contents: Buffer.from(JSON.stringify(json), 'utf8')
-			}));
-		}
-		result.emit('end');
-	}));
+			// some validation of the bundle.l10n.json format
+			for (const key in bundleJson) {
+				if (
+					typeof bundleJson[key] !== 'string' &&
+					(typeof bundleJson[key].message !== 'string' || !Array.isArray(bundleJson[key].comment))
+				) {
+					callback(new Error(`Invalid bundle.l10n.json file. The value for key ${key} is not in the expected format.`));
+					return;
+				}
+			}
 
-	return result;
+			callback(undefined, file);
+		}))
+		.pipe(jsonMerge({
+			fileName: `extensions/${extensionFolderName}/bundle.l10n.json`,
+			jsonSpace: '',
+			concatArrays: true
+		}));
 }
 
 export function createXlfFilesForExtensions(): ThroughStream {

diff --git a/package.json b/package.json
@@ -128,7 +128,7 @@
     "@typescript-eslint/eslint-plugin": "^5.39.0",
     "@typescript-eslint/experimental-utils": "^5.39.0",
     "@typescript-eslint/parser": "^5.39.0",
-    "@vscode/l10n-dev": "0.0.18",
+    "@vscode/l10n-dev": "0.0.21",
     "@vscode/telemetry-extractor": "^1.9.8",
     "@vscode/test-web": "^0.0.32",
     "ansi-colors": "^3.2.3",

diff --git a/yarn.lock b/yarn.lock
@@ -1252,15 +1252,15 @@
   resolved "https://registry.yarnpkg.com/@vscode/iconv-lite-umd/-/iconv-lite-umd-0.7.0.tgz#d2f1e0664ee6036408f9743fee264ea0699b0e48"
   integrity sha512-bRRFxLfg5dtAyl5XyiVWz/ZBPahpOpPrNYnnHpOpUZvam4tKH35wdhP4Kj6PbM0+KdliOsPzbGWpkxcdpNB/sg==
 
-"@vscode/[email protected].18":
-  version "0.0.18"
-  resolved "https://registry.yarnpkg.com/@vscode/l10n-dev/-/l10n-dev-0.0.18.tgz#80a8cf6ef13c7fe1796be7b0007d71993bd1832f"
-  integrity sha512-pEKLMnlg7hlxFrZLqcyJe08olmj6KVs2Rof7MVB5rN0D6NOKPBRtkQ176TuMUmW863EDV5WQUgNzOGa2nHBSSQ==
+"@vscode/[email protected].21":
+  version "0.0.21"
+  resolved "https://registry.yarnpkg.com/@vscode/l10n-dev/-/l10n-dev-0.0.21.tgz#c25a29c7d1484b1dacf4ba98fbd16c1b32a0087e"
+  integrity sha512-KR3OvvLs8AGCWM3LHAAKUfltltANdGjX4fn2jfY13Eb5z4c6smR81p7a9yTo6Kr///zSZuBHQ/hXMmMRtuPTBw==
   dependencies:
     deepmerge-json "^1.5.0"
     glob "^8.0.3"
     pseudo-localization "^2.4.0"
-    typescript "^4.7.4"
+    web-tree-sitter "^0.20.7"
     xml2js "^0.4.23"
     yargs "^17.5.1"
 
@@ -11338,6 +11338,11 @@ watchpack@^2.2.0:
     glob-to-regexp "^0.4.1"
     graceful-fs "^4.1.2"
 
+web-tree-sitter@^0.20.7:
+  version "0.20.7"
+  resolved "https://registry.yarnpkg.com/web-tree-sitter/-/web-tree-sitter-0.20.7.tgz#b0ddb78e8244221a3100f432c7e162516cd9cd09"
+  integrity sha512-flC9JJmTII9uAeeYpWF8hxDJ7bfY+leldQryetll8Nv4WgI+MXc6h7TiyAZASWl9uC9TvmfdgOjZn1DAQecb3A==
+
 webidl-conversions@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"