feat: new approach for linking/installing externals

index.js

@@ -8,7 +8,7 @@ const toPlainObject       = require("es5-ext/object/normalize-options")
     , installPackage      = require("./lib/private/install-package/index");
 
 module.exports = (name, configuration, options = {}) => {
-	const progressData = ee({ done: new Set(), ongoingMap: new Map() });
+	const progressData = ee({ done: new Set(), ongoingMap: new Map(), externalsMap: new Map() });
 	const promise = ee(
 		installPackage(
 			{ name: ensureString(name) }, ensureConfiguration(configuration),

lib/private/install-package/npm-link-dependency/get-metadata.js

@@ -0,0 +1,22 @@
+"use strict";
+
+const memoizee = require("memoizee")
+    , got      = require("got")
+    , log      = require("log4").get("dev-package");
+
+module.exports = memoizee(
+	async dependencyName => {
+		log.info("resolve metadata for %s", dependencyName);
+		try {
+			return JSON.parse(
+				(await got(`https://registry.npmjs.org/${ dependencyName }`, {
+					headers: { accept: "application/vnd.npm.install-v1+json" }
+				})).body
+			);
+		} catch (error) {
+			log.error("Could not retrieve npm info for %s", dependencyName);
+			throw error;
+		}
+	},
+	{ promise: true }
+);

lib/private/install-package/npm-link-dependency/index.js

@@ -0,0 +1,82 @@
+"use strict";
+
+const optionalChaining    = require("es5-ext/optional-chaining")
+    , { resolve }         = require("path")
+    , log                 = require("log4").get("dev-package")
+    , isDirectory         = require("fs2/is-directory")
+    , isSymlink           = require("fs2/is-symlink")
+    , rm                  = require("fs2/rm")
+    , getNpmModulesPath   = require("../../../get-npm-modules-path")
+    , runProgram          = require("../../../run-program")
+    , getPackageJson      = require("../get-package-json")
+    , installExternal     = require("./install-external")
+    , resolveExternalMeta = require("./resolve-external-meta");
+
+const setupExternal = async (dependencyContext, progressData) => {
+	await resolveExternalMeta(dependencyContext, progressData);
+	const {
+		latestSupportedVersion,
+		dependencyName,
+		dependencyPath,
+		dependentName,
+		dependentPath,
+		externalMeta: { currentVersion, latestVersion },
+		versionRange
+	} = dependencyContext;
+	if (latestSupportedVersion !== latestVersion) {
+		if (!latestSupportedVersion) {
+			// Non semver version range, install in place
+			await rm(dependencyPath, { loose: true, recursive: true, force: true });
+			await runProgram("npm", ["install", dependencyName], {
+				cwd: dependentPath,
+				logger: log.levelRoot.get("npm:install")
+			});
+			return;
+		}
+		log.warn(
+			"%s references %s at outdated version %s", dependentName, dependencyName, versionRange
+		);
+		// Expects outdated version, therefore do not link but install in place (if needed)
+		if (
+			(await isDirectory(dependencyPath)) &&
+			optionalChaining(getPackageJson(dependencyPath), "version") === latestSupportedVersion
+		) {
+			// Up to date
+			return;
+		}
+		await installExternal(dependencyContext);
+		return;
+	}
+
+	if (currentVersion) return;
+	log.info("%s link external dependency %s", dependentName, dependencyName);
+	await rm(dependencyPath, { loose: true, recursive: true, force: true });
+	await runProgram("npm", ["link", `${ dependencyName }@${ latestVersion }`], {
+		cwd: dependentPath,
+		logger: log.levelRoot.get("npm:link")
+	});
+};
+
+module.exports = async (dependencyContext, progressData) => {
+	const { dependentName, dependentPath, dependencyName, isExternal } = dependencyContext;
+	const dependencyPath = (dependencyContext.dependencyPath = resolve(
+		dependentPath, "node_modules", dependencyName
+	));
+
+	const linkedPath = (dependencyContext.linkedPath = resolve(
+		await getNpmModulesPath(), dependencyName
+	));
+
+	if (isExternal) {
+		await setupExternal(dependencyContext, progressData);
+		return;
+	}
+
+	if (await isSymlink(dependencyPath, { linkPath: linkedPath })) return;
+	log.info("%s link dependency %s", dependentName, dependencyName);
+	await rm(dependencyPath, { loose: true, recursive: true, force: true });
+	await runProgram("npm", ["link", dependencyName], {
+		cwd: dependentPath,
+		logger: log.levelRoot.get("npm:link")
+	});
+};

lib/private/install-package/npm-link-dependency/install-external.js

@@ -0,0 +1,69 @@
+"use strict";
+
+const memoizee          = require("memoizee")
+    , { join, resolve } = require("path")
+    , copyDir           = require("fs2/copy-dir")
+    , mkdir             = require("fs2/mkdir")
+    , symlink           = require("fs2/symlink")
+    , rm                = require("fs2/rm")
+    , got               = require("got")
+    , tar               = require("tar")
+    , tmpdir            = require("os").tmpdir()
+    , log               = require("log4").get("dev-package")
+    , runProgram        = require("../../../run-program")
+    , getPackageJson    = require("../get-package-json");
+
+const prepareDependency = memoizee(
+	async (dependencyName, version, metadata) => {
+		const targetDirname = resolve(tmpdir, "dev-package", dependencyName, version);
+		log.notice("preparing install of %s", `${ dependencyName }@${ version }`);
+		await rm(targetDirname, { loose: true, recursive: true, force: true });
+		await mkdir(targetDirname, { intermediate: true });
+		await new Promise((promiseResolve, promiseReject) => {
+			const stream = got
+				.stream(metadata.versions[version].dist.tarball)
+				.pipe(tar.x({ cwd: targetDirname, strip: 1 }));
+			stream.on("error", promiseReject);
+			stream.on("end", promiseResolve);
+		});
+		await runProgram("npm", ["install", "--production"], {
+			cwd: targetDirname,
+			logger: log.levelRoot.get("npm:install")
+		});
+		await rm(resolve(targetDirname, "package-lock.json"), {
+			loose: true,
+			recursive: true,
+			force: true
+		});
+		return targetDirname;
+	},
+	{ promise: true, length: 2 }
+);
+
+module.exports = async dependencyContext => {
+	const {
+		dependencyPath,
+		dependencyName,
+		dependentName,
+		dependentPath,
+		latestSupportedVersion,
+		externalMeta: { metadata }
+	} = dependencyContext;
+	log.notice("%s updating %s to %s", dependentName, dependencyName, latestSupportedVersion);
+	const sourceDirname = await prepareDependency(dependencyName, latestSupportedVersion, metadata);
+	await rm(dependencyPath, { loose: true, recursive: true, force: true });
+	await copyDir(sourceDirname, dependencyPath);
+
+	// Ensure to map binaries
+	await Promise.all(
+		Object.entries(getPackageJson(dependencyPath).bin || {}).map(
+			async ([targetName, linkedPath]) => {
+				const targetPath = resolve(dependentPath, "node_modules/.bin", targetName);
+				await rm(targetPath, { loose: true, force: true, recursive: true });
+				await symlink(join("../", dependencyName, linkedPath), targetPath, {
+					intermediate: true
+				});
+			}
+		)
+	);
+};

lib/private/install-package/npm-link-dependency/resolve-external-meta.js

@@ -0,0 +1,74 @@
+"use strict";
+
+const optionalChaining = require("es5-ext/optional-chaining")
+    , log              = require("log4").get("dev-package")
+    , isDirectory      = require("fs2/is-directory")
+    , semver           = require("semver")
+    , getPackageJson   = require("../get-package-json")
+    , getMetadata      = require("./get-metadata");
+
+const getVersions = async dependencyName =>
+	Object.keys((await getMetadata(dependencyName)).versions);
+
+const resolveStableVersions = async dependencyName =>
+	Object.entries((await getMetadata(dependencyName)).versions)
+		.filter(([, meta]) => meta.deprecated)
+		.map(([version]) => version);
+
+const getVersionRange = (dependentPackageJson, dependencyName) => {
+	if (dependentPackageJson.dependencies && dependentPackageJson.dependencies[dependencyName]) {
+		return dependentPackageJson.dependencies[dependencyName];
+	}
+	if (
+		dependentPackageJson.devDependencies &&
+		dependentPackageJson.devDependencies[dependencyName]
+	) {
+		return dependentPackageJson.devDependencies[dependencyName];
+	}
+	return dependentPackageJson.optionalDependencies[dependencyName];
+};
+
+const getLinkCurrentVersion = async ({ linkedPath }) => {
+	// Accept installation only if in directory (not symlink)
+	if (!(await isDirectory(linkedPath))) return null;
+	return optionalChaining(getPackageJson(linkedPath), "version") || null;
+};
+
+const getLatestSupportedVersion = async ({ dependentName, dependencyName, versionRange }) => {
+	if (!semver.validRange(versionRange)) {
+		log.warning(
+			"%s references %s not by semver range %s", dependentName, dependencyName, versionRange
+		);
+		return null;
+	}
+	const latestSupportedVersion =
+		semver.maxSatisfying(await resolveStableVersions(dependencyName), versionRange) ||
+		semver.maxSatisfying(await getVersions(dependencyName), versionRange);
+	if (latestSupportedVersion) return latestSupportedVersion;
+
+	log.error(
+		"%s references %s with not satisfiable version range %s", dependentName, dependencyName,
+		versionRange
+	);
+	return null;
+};
+
+module.exports = async (dependencyContext, progressData) => {
+	const { dependentPackageJson, dependencyName } = dependencyContext;
+	const { externalsMap } = progressData;
+	if (!externalsMap.has(dependencyName)) {
+		const metadata = await getMetadata(dependencyName);
+		externalsMap.set(dependencyName, {
+			currentVersion: await getLinkCurrentVersion(dependencyContext),
+			latestVersion: metadata["dist-tags"].latest,
+			metadata
+		});
+		log.debug(
+			"resolve %s (external dependency) meta %o", dependencyName,
+			externalsMap.get(dependencyName)
+		);
+	}
+	dependencyContext.externalMeta = externalsMap.get(dependencyName);
+	dependencyContext.versionRange = getVersionRange(dependentPackageJson, dependencyName);
+	dependencyContext.latestSupportedVersion = await getLatestSupportedVersion(dependencyContext);
+};

lib/private/install-package/setup-dependencies.js

@@ -14,7 +14,7 @@ const setupDependency = async (
 	const { done, ongoingMap } = progressData;
 	if (!isExternal) {
 		if (ongoingMap.has(dependencyName)) {
-			ongoingMap.get(dependencyName).push(() => npmLink(dependencyContext));
+			ongoingMap.get(dependencyName).push(() => npmLink(dependencyContext, progressData));
 			return;
 		}
 		if (!done.has(dependencyName)) {
@@ -24,7 +24,7 @@ const setupDependency = async (
 			);
 		}
 	}
-	await npmLink(dependencyContext);
+	await npmLink(dependencyContext, progressData);
 };
 
 const resolveDependencyContext = (packageContext, dependencyName, userConfiguration) => {
@@ -33,6 +33,7 @@ const resolveDependencyContext = (packageContext, dependencyName, userConfigurat
 	return {
 		dependentName: name,
 		dependentPath: path,
+		dependentPackageJson: packageContext.packageJson,
 		dependencyName,
 		isExternal: !packagesMeta[dependencyName]
 	};
@@ -69,7 +70,7 @@ module.exports = async (packageContext, userConfiguration, inputOptions, progres
 			await setupDependency(dependencyContext, userConfiguration, inputOptions);
 			continue;
 		}
-		try { await npmLink(dependencyContext); }
+		try { await npmLink(dependencyContext, progressData); }
 		catch (error) {
 			log.error(
 				`Could not link optional dependency %s, crashed with:\n${ error.stack }`,