build: add watch/compile tasks for CLI (microsoft#182344)

* build: add watch/compile tasks for CLI

I spent time this morning working on the 'developer experience' of the
CLI in vscode, mainly getting the CLI to cross-compile chasing our
initial idea of having it auto-build in a devcontainer.

After some effort and disabling tunnels connections (to avoid having to
pull in OpenSSL which is a huge pain to cross compile), I was able to
get it to cross-compile from Linux to Windows, using the mingw linker.
I could probably figure out how to get macOS working as well with more
effort. However, I'm not a big fan of this, effectively it's one more
'platform' of build we need to support and test.

I think a better approach is downloading the latest compiled CLI from
the update server instead, as needed. That's what this PR does. It just
places the CLI where it _would_ normally get compiled to by cargo; so
far we don't need to do anything special outside of that.
A notice is shown to users if this fallback happens.

* update from review

.gitignore

@@ -16,4 +16,5 @@ vscode.lsif
 vscode.db
 /.profile-oss
 /cli/target
+/cli/openssl
 product.overrides.json

build/gulpfile.cli.js

@@ -0,0 +1,189 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+//@ts-check
+
+const es = require('event-stream');
+const gulp = require('gulp');
+const path = require('path');
+const fancyLog = require('fancy-log');
+const ansiColors = require('ansi-colors');
+const cp = require('child_process');
+const { tmpdir } = require('os');
+const { promises: fs, existsSync, mkdirSync, rmSync } = require('fs');
+
+const task = require('./lib/task');
+const watcher = require('./lib/watch');
+const { debounce } = require('./lib/util');
+const createReporter = require('./lib/reporter').createReporter;
+
+const root = 'cli';
+const rootAbs = path.resolve(__dirname, '..', root);
+const src = `${root}/src`;
+const targetCliPath = path.join(root, 'target', 'debug', process.platform === 'win32' ? 'code.exe' : 'code');
+
+const platformOpensslDirName =
+	process.platform === 'win32' ? (
+		process.arch === 'arm64'
+			? 'arm64-windows-static-md'
+			: process.arch === 'ia32'
+				? 'x86-windows-static-md'
+				: 'x64-windows-static-md')
+		: process.platform === 'darwin' ? (
+			process.arch === 'arm64'
+				? 'arm64-osx'
+				: 'x64-osx')
+			: (process.arch === 'arm64'
+				? 'arm64-linux'
+				: process.arch === 'arm'
+					? 'arm-linux'
+					: 'x64-linux');
+const platformOpensslDir = path.join(rootAbs, 'openssl', 'package', 'out', platformOpensslDirName);
+
+const hasLocalRust = (() => {
+	/** @type boolean | undefined */
+	let result = undefined;
+	return () => {
+		if (result !== undefined) {
+			return result;
+		}
+
+		try {
+			const r = cp.spawnSync('cargo', ['--version']);
+			result = r.status === 0;
+		} catch (e) {
+			result = false;
+		}
+
+		return result;
+	};
+})();
+
+const debounceEsStream = (fn, duration = 100) => {
+	let handle = undefined;
+	let pending = [];
+	const sendAll = (pending) => (event, ...args) => {
+		for (const stream of pending) {
+			pending.emit(event, ...args);
+		}
+	};
+
+	return es.map(function (_, callback) {
+		console.log('defer');
+		if (handle !== undefined) {
+			clearTimeout(handle);
+		}
+
+		handle = setTimeout(() => {
+			handle = undefined;
+
+			const previous = pending;
+			pending = [];
+			fn()
+				.on('error', sendAll('error'))
+				.on('data', sendAll('data'))
+				.on('end', sendAll('end'));
+		}, duration);
+
+		pending.push(this);
+	});
+};
+
+const compileFromSources = (callback) => {
+	const proc = cp.spawn('cargo', ['--color', 'always', 'build'], {
+		cwd: root,
+		stdio: ['ignore', 'pipe', 'pipe'],
+		env: existsSync(platformOpensslDir) ? { OPENSSL_DIR: platformOpensslDir, ...process.env } : process.env
+	});
+
+	/** @type Buffer[] */
+	const stdoutErr = [];
+	proc.stdout.on('data', d => stdoutErr.push(d));
+	proc.stderr.on('data', d => stdoutErr.push(d));
+	proc.on('error', callback);
+	proc.on('exit', code => {
+		if (code !== 0) {
+			callback(Buffer.concat(stdoutErr).toString());
+		} else {
+			callback();
+		}
+	});
+};
+
+const acquireBuiltOpenSSL = (callback) => {
+	const untar = require('gulp-untar');
+	const gunzip = require('gulp-gunzip');
+	const dir = path.join(tmpdir(), 'vscode-openssl-download');
+	mkdirSync(dir, { recursive: true });
+
+	cp.spawnSync(
+		process.platform === 'win32' ? 'npm.cmd' : 'npm',
+		['pack', '@vscode/openssl-prebuilt'],
+		{ stdio: ['ignore', 'ignore', 'inherit'], cwd: dir }
+	);
+
+	gulp.src('*.tgz', { cwd: dir })
+		.pipe(gunzip())
+		.pipe(untar())
+		.pipe(gulp.dest(`${root}/openssl`))
+		.on('error', callback)
+		.on('end', () => {
+			rmSync(dir, { recursive: true, force: true });
+			callback();
+		});
+};
+
+const compileWithOpenSSLCheck = (/** @type import('./lib/reporter').IReporter */ reporter) => es.map((_, callback) => {
+	compileFromSources(err => {
+		if (!err) {
+			// no-op
+		} else if (err.toString().includes('Could not find directory of OpenSSL installation') && !existsSync(platformOpensslDir)) {
+			fancyLog(ansiColors.yellow(`[cli]`), 'OpenSSL libraries not found, acquiring prebuilt bits...');
+			acquireBuiltOpenSSL(err => {
+				if (err) {
+					callback(err);
+				} else {
+					compileFromSources(err => {
+						if (err) {
+							reporter(err.toString());
+						}
+						callback(null, '');
+					});
+				}
+			});
+		} else {
+			reporter(err.toString());
+		}
+
+		callback(null, '');
+	});
+});
+
+const warnIfRustNotInstalled = () => {
+	if (!hasLocalRust()) {
+		fancyLog(ansiColors.yellow(`[cli]`), 'No local Rust install detected, compilation may fail.');
+		fancyLog(ansiColors.yellow(`[cli]`), 'Get rust from: https://rustup.rs/');
+	}
+};
+
+const compileCliTask = task.define('compile-cli', () => {
+	warnIfRustNotInstalled();
+	const reporter = createReporter('cli');
+	return gulp.src(`${root}/Cargo.toml`)
+		.pipe(compileWithOpenSSLCheck(reporter))
+		.pipe(reporter.end(true));
+});
+
+
+const watchCliTask = task.define('watch-cli', () => {
+	warnIfRustNotInstalled();
+	return watcher(`${src}/**`, { read: false })
+		.pipe(debounce(compileCliTask));
+});
+
+gulp.task(compileCliTask);
+gulp.task(watchCliTask);

build/lib/util.ts

@@ -77,7 +77,7 @@ export function incremental(streamProvider: IStreamProvider, initial: NodeJS.Rea
 	return es.duplex(input, output);
 }
 
-export function debounce(task: () => NodeJS.ReadWriteStream): NodeJS.ReadWriteStream {
+export function debounce(task: () => NodeJS.ReadWriteStream, duration = 500): NodeJS.ReadWriteStream {
 	const input = es.through();
 	const output = es.through();
 	let state = 'idle';
@@ -99,7 +99,7 @@ export function debounce(task: () => NodeJS.ReadWriteStream): NodeJS.ReadWriteSt
 
 	run();
 
-	const eventuallyRun = _debounce(() => run(), 500);
+	const eventuallyRun = _debounce(() => run(), duration);
 
 	input.on('data', () => {
 		if (state === 'idle') {

package.json

@@ -45,8 +45,10 @@
     "valid-layers-check": "node build/lib/layersChecker.js",
     "update-distro": "node build/npm/update-distro.mjs",
     "web": "echo 'yarn web' is replaced by './scripts/code-server' or './scripts/code-web'",
+    "compile-cli": "gulp compile-cli",
     "compile-web": "node --max_old_space_size=4095 ./node_modules/gulp/bin/gulp.js compile-web",
     "watch-web": "node --max_old_space_size=4095 ./node_modules/gulp/bin/gulp.js watch-web",
+    "watch-cli": "node --max_old_space_size=4095 ./node_modules/gulp/bin/gulp.js watch-cli",
     "eslint": "node build/eslint",
     "stylelint": "node build/stylelint",
     "playwright-install": "node build/azure-pipelines/common/installPlaywright.js",

test/smoke/package.json

@@ -11,7 +11,7 @@
     "mocha": "node ../node_modules/mocha/bin/mocha"
   },
   "dependencies": {
-    "@vscode/test-electron": "^2.2.1",
+    "@vscode/test-electron": "^2.3.2",
     "mkdirp": "^1.0.4",
     "ncp": "^2.0.0",
     "node-fetch": "^2.6.7",