diff --git a/bin/setup-own-package b/bin/setup-own-package new file mode 100755 index 0000000..6f7e316 --- /dev/null +++ b/bin/setup-own-package @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +"use strict"; + +Error.stackTraceLimit = Infinity; + +process.on("unhandledRejection", reason => { throw reason; }); + +require("log4-nodejs")(); + +const meta = require("../package") + , argv = require("minimist")(process.argv.slice(2)); + +const usage = `setup-package v${ meta.version } - Setup package + +Usage: setup-package package-name + +Options: + + --version, -v Display version + --help, -h Show this message + +`; + +if (argv.h || argv.help) { + process.stdout.write(usage); + return; +} + +if (argv.v || argv.version) { + process.stdout.write(`${ meta.version }\n`); + return; +} + +const [packageName] = argv._; + +if (!packageName) { + process.stderr.write(`Provide package name to setup\n\n${ usage }`); + process.exit(1); +} + +require("..")(process.cwd(), packageName); diff --git a/index.js b/index.js index 0f42094..c7bb1f5 100644 --- a/index.js +++ b/index.js @@ -1,79 +1,6 @@ "use strict"; -const partial = require("es5-ext/function/#/partial") - , startsWith = require("es5-ext/string/#/starts-with") - , deferred = require("deferred") - , exec = require("exec-batch/exec") - , lstat = require("fs2/lstat") - , mkdir = require("fs2/mkdir") - , symlink = require("fs2/symlink") - , { resolve } = require("path") - , packages = require("./packages"); +const { resolve } = require("path") + , setupPackage = require("./lib/setup-package"); -const { keys } = Object - , root = process.cwd() - , dir = resolve(root, "modules") - , done = Object.create(null) - , done2 = Object.create(null); - -require("events").EventEmitter.defaultMaxListeners = Infinity; - -const setup = function (at, name) { - if (done2[`${ at }|${ name }`]) return null; - done2[`${ at }|${ name }`] = true; - if (!packages[name]) return exec(`npm install ${ name }`, { cwd: at }); - const path = resolve(at, "node_modules", name); - return lstat(resolve(path))( - stats => { - if (!stats.isSymbolicLink()) { - throw new Error(`Path '${ path }' is not a symbolic link`); - } - }, - err => { - if (err.code !== "ENOENT") throw err; - return deferred( - mkdir(resolve(at, "node_modules"), { intermediate: true }), setupGit(name) - )(() => symlink(resolve(dir, name), resolve(at, "node_modules", name))); - } - ); -}; - -const setupGit = function (name) { - if (done[name]) return null; - done[name] = true; - return lstat(resolve(dir, name))( - stats => { - if (!stats.isDirectory()) { - throw new Error(`Path '${ name }' is not adirectory`); - } - console.log("Do pull of", name); - return exec("git pull", { cwd: resolve(dir, name) }); - }, - err => { - let gitName, repoName; - if (err.code !== "ENOENT") throw err; - gitName = packages[name] === true ? name : packages[name]; - repoName = startsWith.call(gitName, "git@") - ? gitName - : `git@github.com:medikoo/${ gitName }.git`; - return exec(`git clone ${ repoName } ${ name }`, { cwd: dir }).aside(null, err => { - console.log(`Repository '${ repoName }' not found`); - }); - } - )(() => { - const dRequire = require - , conf = dRequire(resolve(dir, name, "package.json")) - , path = resolve(dir, name) - , lSetup = deferred.gate(partial.call(setup, path), 1); - - return deferred( - deferred.map(keys(conf.dependencies || {}) || [], lSetup), - deferred.map(keys(conf.devDependencies || {}) || [], lSetup), - deferred.map(keys(conf.peerDependencies || {}) || [], lSetup) - ); - }); -}; - -mkdir(dir, { intermediate: true })(() => - deferred.map(keys(packages), deferred.gate(setupGit, 1)) -).done(); +module.exports = (packagesPath, packageName) => setupPackage(resolve(packagesPath), packageName); diff --git a/lib/get-log-gatherer.js b/lib/get-log-gatherer.js new file mode 100644 index 0000000..7dbedb7 --- /dev/null +++ b/lib/get-log-gatherer.js @@ -0,0 +1,12 @@ +"use strict"; + +module.exports = () => { + const lines = []; + return { + lines, + logger: { + info(line) { lines.push(line); }, + error(line) { throw new Error(`Unexpected error line ${ line }`); } + } + }; +}; diff --git a/lib/is-directory.js b/lib/is-directory.js new file mode 100644 index 0000000..f122b8a --- /dev/null +++ b/lib/is-directory.js @@ -0,0 +1,8 @@ +"use strict"; + +const resolveStats = require("./resolve-stats"); + +module.exports = async path => { + const stats = await resolveStats(path); + return stats ? stats.isDirectory() : null; +}; diff --git a/lib/is-own-package.js b/lib/is-own-package.js new file mode 100644 index 0000000..2e7090e --- /dev/null +++ b/lib/is-own-package.js @@ -0,0 +1,24 @@ +"use strict"; + +const logger = require("log4").get("setup-own-package") + , memoize = require("memoizee") + , GithubApi = require("@octokit/rest"); + +const github = new GithubApi({ timeout: 5000 }); + +github.authenticate({ type: "token", token: "[HIDDEN]" }); + +const getReposPage = async ({ page }) => { + // eslint-disable-next-line camelcase + const { data } = await github.repos.getForUser({ username: "medikoo", page, per_page: 100 }); + const repos = data.filter(repo => !repo.fork).map(repo => repo.name); + if (data.length === 100) repos.push(...(await getReposPage({ page: page + 1 }))); + return repos; +}; + +const getRepos = memoize(async () => { + logger.notice("resolve own package names"); + return new Set(await getReposPage({ page: 1 }), { promise: true }); +}); + +module.exports = async packageName => (await getRepos()).has(packageName); diff --git a/lib/is-package-linked.js b/lib/is-package-linked.js new file mode 100644 index 0000000..8ec157a --- /dev/null +++ b/lib/is-package-linked.js @@ -0,0 +1,22 @@ +"use strict"; + +const logger = require("log4").get("setup-own-package") + , { resolve } = require("path") + , memoize = require("memoizee") + , getLogGatherer = require("./get-log-gatherer") + , isSymbolicLink = require("./is-symbolic-link") + , runProgram = require("./run-program"); + +const getNpmPathPrefix = memoize(async () => { + const { lines, logger: gatherer } = getLogGatherer(); + await runProgram("npm", ["config", "get", "prefix"], { cwd: process.cwd(), logger: gatherer }); + const [npmPathPrefix] = lines; + if (!npmPathPrefix) throw new Error("Could not resolve npm path prefix"); + logger.notice("resolved npm path prefix %s", npmPathPrefix); + return npmPathPrefix; +}); + +module.exports = async packageName => { + const npmPathPrefix = await getNpmPathPrefix(); + return isSymbolicLink(resolve(npmPathPrefix, "lib/node_modules", packageName)); +}; diff --git a/lib/is-symbolic-link.js b/lib/is-symbolic-link.js new file mode 100644 index 0000000..6c941f4 --- /dev/null +++ b/lib/is-symbolic-link.js @@ -0,0 +1,8 @@ +"use strict"; + +const resolveStats = require("./resolve-stats"); + +module.exports = async path => { + const stats = await resolveStats(path); + return stats ? stats.isSymbolicLink() : null; +}; diff --git a/lib/resolve-stats.js b/lib/resolve-stats.js new file mode 100644 index 0000000..82910e3 --- /dev/null +++ b/lib/resolve-stats.js @@ -0,0 +1,12 @@ +"use strict"; + +const lstat = require("fs2/lstat"); + +module.exports = async path => { + try { + return await lstat(path); + } catch (error) { + if (error.code === "ENOENT") return null; + throw error; + } +}; diff --git a/lib/rm-path.js b/lib/rm-path.js new file mode 100644 index 0000000..eee0e73 --- /dev/null +++ b/lib/rm-path.js @@ -0,0 +1,19 @@ +"use strict"; + +const rmdir = require("fs2/rmdir") + , unlink = require("fs2/unlink") + , logger = require("log4").get("setup-own-package") + , isDirectory = require("./is-directory"); + +module.exports = async path => { + const isDir = await isDirectory(path); + if (isDir) { + logger.notice("remove directory %s", path); + return rmdir(path, { recursive: true, force: true }); + } + if (isDir === false) { + logger.notice("remove file %s", path); + return unlink(path); + } + return null; +}; diff --git a/lib/run-program.js b/lib/run-program.js new file mode 100644 index 0000000..605714f --- /dev/null +++ b/lib/run-program.js @@ -0,0 +1,43 @@ +"use strict"; + +const { spawn } = require("child_process") + , Deferred = require("deferred"); + +const newLineRe = /[\n\r]/u; + +const processOutput = (stream, logger) => { + let buffer = ""; + const flush = isFinal => { + if (!buffer) return; + if (!buffer.match(newLineRe)) { + if (!isFinal) return; + logger(buffer); + return; + } + const lines = buffer.split(newLineRe); + buffer = isFinal ? "" : lines.pop(); + for (const line of lines) logger(line); + }; + + stream.on("data", data => { + buffer += data; + flush(); + }); + stream.on("error", flush); + stream.on("end", flush); +}; + +module.exports = (command, args, options) => { + const { logger } = options + , deferred = new Deferred() + , child = spawn(command, args, { cwd: options.cwd }); + + child.on("error", deferred.reject); + processOutput(child.stdout, logger.info); + processOutput(child.stderr, logger.notice); + child.on("close", code => { + if (code) deferred.reject(new Error(`Program exited with: ${ code }`)); + else deferred.resolve(); + }); + return deferred.promise; +}; diff --git a/lib/setup-npm-link.js b/lib/setup-npm-link.js new file mode 100644 index 0000000..65e03f3 --- /dev/null +++ b/lib/setup-npm-link.js @@ -0,0 +1,18 @@ +"use strict"; + +const { basename, resolve } = require("path") + , logger = require("log4").get("setup-own-package") + , isSymbolicLink = require("./is-symbolic-link") + , rmPath = require("./rm-path") + , runProgram = require("./run-program"); + +module.exports = async (packagePath, dependencyName) => { + const dependencyLinkPath = resolve(packagePath, "node_modules", dependencyName); + if (await isSymbolicLink(dependencyLinkPath)) return; + logger.notice("link %s in %s", dependencyName, basename(packagePath)); + await rmPath(dependencyLinkPath); + await runProgram("npm", ["link", dependencyName], { + cwd: packagePath, + logger: logger.levelRoot.get("npm:link") + }); +}; diff --git a/lib/setup-package.js b/lib/setup-package.js new file mode 100644 index 0000000..ca832fa --- /dev/null +++ b/lib/setup-package.js @@ -0,0 +1,101 @@ +"use strict"; + +const { resolve } = require("path") + , rmdir = require("fs2/rmdir") + , logger = require("log4").get("setup-own-package") + , isOwnPackage = require("./is-own-package") + , isPackageLinked = require("./is-package-linked") + , runProgram = require("./run-program") + , setupPrettier = require("./setup-prettier") + , setupRepository = require("./setup-repository") + , setupNpmLink = require("./setup-npm-link"); + +const exceptionsMap = new Map([["webmake", "modules-webmake"], ["next", "node-ext"]]); + +const ongoingMap = new Map(); +const done = new Set(); + +const setupDependencies = async (packagesPath, packageName) => { + const packagePath = resolve(packagesPath, packageName); + const packageMeta = require(resolve(packagePath, "package.json")); + const dependencies = new Set( + Object.keys(packageMeta.dependencies || {}).concat( + Object.keys(packageMeta.devDependencies || {}) + ) + ); + dependencies.delete(packageName); + + logger.notice("setup dependencies of %s", packageName); + for (const dependencyName of dependencies) { + if (!done.has(dependencyName)) { + if (ongoingMap.has(dependencyName)) { + ongoingMap + .get(dependencyName) + .push(() => setupNpmLink(packagePath, dependencyName)); + continue; + } else if (exceptionsMap.has(dependencyName) || await isOwnPackage(dependencyName)) { + await module.exports(packagesPath, dependencyName); + } + } + await setupNpmLink(packagePath, dependencyName); + } + + // Eventual optional dependencies + for (const dependencyName of Object.keys(packageMeta.optionalDependencies || {})) { + if (dependencyName === packageName) continue; + if (dependencies.has(dependencyName)) continue; + try { await setupNpmLink(packagePath, dependencyName); } + catch (error) { + logger.error( + `Could not link optional dependency %s, crashed with:\n${ error.stack }`, + dependencyName + ); + } + } +}; + +module.exports = async (packagesPath, packageName) => { + const packagePath = resolve(packagesPath, packageName); + + logger.notice("setup package %s", packageName); + const pendingJobs = []; + ongoingMap.set(packageName, pendingJobs); + + // Setup repository + await setupRepository( + packagePath, `git@github.com:medikoo/${ exceptionsMap.get(packageName) || packageName }.git` + ); + + // Cleanup eventual npm crashes + await rmdir(resolve(packagePath, "node_modules/.staging"), { + loose: true, + recursive: true, + force: true + }); + + // Setup dependencies + await setupDependencies(packagesPath, packageName); + + // Link package + if (!await isPackageLinked(packageName)) { + logger.notice("link %s", packageName); + await runProgram("npm", ["link"], { + cwd: packagePath, + logger: logger.levelRoot.get("npm:link") + }); + } + + // Setup prettier link + await setupPrettier(packagesPath, packagePath); + + // Done + logger.notice("done %s", packageName); + done.add(packageName); + ongoingMap.delete(packageName); + + // Run pending jobs + if (pendingJobs.length) { + logger.notice("run pending jobs of %s", packageName); + for (const pendingJob of pendingJobs) await pendingJob(); + } +}; diff --git a/lib/setup-prettier.js b/lib/setup-prettier.js new file mode 100644 index 0000000..9f48ad3 --- /dev/null +++ b/lib/setup-prettier.js @@ -0,0 +1,31 @@ +"use strict"; + +const { basename, resolve } = require("path") + , memoizee = require("memoizee") + , logger = require("log4").get("setup-own-package") + , resolveStats = require("./resolve-stats") + , runProgram = require("./run-program") + , setupRepository = require("./setup-repository") + , setupSymbolicLink = require("./setup-symbolic-link"); + +const setupPrettierPackage = memoizee( + async packagesPath => { + const prettierPath = resolve(packagesPath, "prettier-elastic"); + logger.notice("setup prettier package"); + await setupRepository(prettierPath, "git@github.com:medikoo/prettier-elastic.git"); + logger.notice("install %s", prettierPath); + await runProgram("yarn", [], { + cwd: prettierPath, + logger: logger.levelRoot.get("yarn:install") + }); + }, + { promise: true } +); + +module.exports = async (packagesPath, packagePath) => { + const symbolicLinkPath = resolve(packagePath, "node_modules/prettier"); + if (await resolveStats(symbolicLinkPath)) return; + await setupPrettierPackage(packagesPath); + logger.notice("link %s in %s", "prettier", basename(packagePath)); + await setupSymbolicLink(resolve(packagesPath, "prettier-elastic"), symbolicLinkPath); +}; diff --git a/lib/setup-repository.js b/lib/setup-repository.js new file mode 100644 index 0000000..267bf2e --- /dev/null +++ b/lib/setup-repository.js @@ -0,0 +1,29 @@ +"use strict"; + +const logger = require("log4").get("setup-own-package") + , isDirectory = require("./is-directory") + , runProgram = require("./run-program") + , getLogGatherer = require("./get-log-gatherer"); + +module.exports = async (repoPath, repoUrl) => { + if (await isDirectory(repoPath)) { + // Confirm directory is clean + const { lines, logger: gatherer } = getLogGatherer(); + await runProgram("git", ["status", "--porcelain"], { cwd: repoPath, logger: gatherer }); + if (lines.length) { + throw new Error(`Repository ${ repoPath } is not clean:\n${ lines.join("\n") }`); + } + + // Update + logger.notice("update repository %s", repoPath); + await runProgram("git", ["pull"], { + cwd: repoPath, + logger: logger.levelRoot.get("git:pull") + }); + return; + } + logger.notice("clone repository %s from %s", repoPath, repoUrl); + await runProgram("git", ["clone", repoUrl, repoPath], { + logger: logger.levelRoot.get("git:clone") + }); +}; diff --git a/lib/setup-symbolic-link.js b/lib/setup-symbolic-link.js new file mode 100644 index 0000000..28e912e --- /dev/null +++ b/lib/setup-symbolic-link.js @@ -0,0 +1,11 @@ +"use strict"; + +const logger = require("log4").get("setup-own-package") + , symlink = require("fs2/symlink") + , rmPath = require("./rm-path"); + +module.exports = async (target, path) => { + await rmPath(path); + logger.info("create symlink of %s at %s", target, path); + await symlink(target, path, { type: "dir", intermediate: true }); +}; diff --git a/package.json b/package.json index 1e1a19d..85a687e 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,16 @@ { "private": true, - "name": "modules-installer", + "name": "setup-own-package", "version": "0.0.0", "author": "Mariusz Nowak (http://www.medikoo.com/)", "dependencies": { - "es5-ext": "^0.10.46", - "source-map-support": "^0.5.9" + "deferred": "^0.7.9", + "fs2": "^0.2.8", + "log4": "^3.0.1", + "log4-nodejs": "^2.3.1", + "memoizee": "^0.4.14", + "minimist": "^1.2", + "@octokit/rest": "^15.11.1" }, "devDependencies": { "eslint": "^5.5", @@ -16,7 +21,15 @@ "root": true, "env": { "node": true - } + }, + "overrides": [ + { + "files": "lib/setup-package.js", + "rules": { + "no-await-in-loop": "off" + } + } + ] }, "scripts": { "lint": "eslint --ignore-path=.gitignore ." diff --git a/packages.json b/packages.json deleted file mode 100644 index a54f597..0000000 --- a/packages.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "cli-color": true, - "clock": true, - "controller-router": true, - "css-aid": true, - "csslint-next": true, - "d": true, - "dbjs": true, - "dbjs-copy": true, - "dbjs-dom": true, - "dbjs-domjs": true, - "dbjs-error-message": "git@github.com:kamsi/dbjs-error-message.git", - "dbjs-ext": true, - "dbjs-file": true, - "dbjs-fragment": true, - "dbjs-log": true, - "dbjs-reduce": true, - "deferred": true, - "dom-ext": true, - "domjs": true, - "domjs-ext": true, - "domjs-reactive-script": true, - "duration": true, - "engine-sniff": true, - "eregistrations": "git@github.com:egovernment/eregistrations.git", - "error-create": true, - "es3-ext": true, - "es5-ext": true, - "es5-fix": true, - "es6-iterator": true, - "es6-map": true, - "es6-set": true, - "es6-symbol": true, - "es6-template-strings": true, - "es6-weak-map": true, - "esniff": true, - "event-emitter": true, - "event-source": true, - "exec-batch": true, - "find-requires": true, - "fs2": true, - "git-branch-deploy": true, - "html-dom-event-ext": true, - "html-dom-ext": true, - "i18n2": true, - "i18n2-md-to-dom": true, - "i18n2-scanner": "git@github.com:kamsi/i18n2-scanner.git", - "lru-queue": true, - "memoizee": "memoize", - "microtime-x": true, - "next": "node-ext", - "next-tick": true, - "observable-array": true, - "observable-map": true, - "observable-multi-set": true, - "observable-set": true, - "observable-value": true, - "path2": true, - "post-controller-router": true, - "punycode2": true, - "querystring2": true, - "reactive-table": true, - "tad": true, - "time-uuid": true, - "timers-ext": true, - "url3": true, - "webmake": "modules-webmake", - "webmake-coffee": true, - "webmake-yaml": true, - "xlint": true, - "xlint-css-sublime": true, - "xlint-csslint": true, - "xlint-csslint-next": true, - "xlint-jslint": true, - "xlint-jslint-medikoo": true, - "xlint-sublime": true -}