From becf84370b2f1394dce2e49fa70fef4285d407ac Mon Sep 17 00:00:00 2001 From: Lance Ball Date: Fri, 1 Feb 2019 12:49:16 -0500 Subject: [PATCH 1/2] repl: add repl.setupHistory for programmatic repl Adds a `repl.setupHistory()` instance method so that programmatic REPLs can also write history to a file. This change also refactors all of the history file management to `lib/internal/repl/history.js`, cleaning up and simplifying `lib/internal/repl.js`. --- lib/internal/repl.js | 160 +----------- lib/internal/repl/history.js | 153 +++++++++++ lib/repl.js | 5 + node.gyp | 1 + .../test-repl-programmatic-history.js | 245 ++++++++++++++++++ 5 files changed, 406 insertions(+), 158 deletions(-) create mode 100644 lib/internal/repl/history.js create mode 100644 test/parallel/test-repl-programmatic-history.js diff --git a/lib/internal/repl.js b/lib/internal/repl.js index e58752e76eb890..321f4ab29eba59 100644 --- a/lib/internal/repl.js +++ b/lib/internal/repl.js @@ -1,24 +1,10 @@ 'use strict'; -const { Interface } = require('readline'); const REPL = require('repl'); -const path = require('path'); -const fs = require('fs'); -const os = require('os'); -const util = require('util'); -const debug = util.debuglog('repl'); + module.exports = Object.create(REPL); module.exports.createInternalRepl = createRepl; -// XXX(chrisdickinson): The 15ms debounce value is somewhat arbitrary. -// The debounce is to guard against code pasted into the REPL. -const kDebounceHistoryMS = 15; - -function _writeToOutput(repl, message) { - repl._writeToOutput(message); - repl._refreshLine(); -} - function createRepl(env, opts, cb) { if (typeof opts === 'function') { cb = opts; @@ -55,151 +41,9 @@ function createRepl(env, opts, cb) { if (!Number.isNaN(historySize) && historySize > 0) { opts.historySize = historySize; } else { - // XXX(chrisdickinson): set here to avoid affecting existing applications - // using repl instances. opts.historySize = 1000; } const repl = REPL.start(opts); - if (opts.terminal) { - return setupHistory(repl, env.NODE_REPL_HISTORY, cb); - } - - repl._historyPrev = _replHistoryMessage; - cb(null, repl); -} - -function setupHistory(repl, historyPath, ready) { - // Empty string disables persistent history - if (typeof historyPath === 'string') - historyPath = historyPath.trim(); - - if (historyPath === '') { - repl._historyPrev = _replHistoryMessage; - return ready(null, repl); - } - - if (!historyPath) { - try { - historyPath = path.join(os.homedir(), '.node_repl_history'); - } catch (err) { - _writeToOutput(repl, '\nError: Could not get the home directory.\n' + - 'REPL session history will not be persisted.\n'); - - debug(err.stack); - repl._historyPrev = _replHistoryMessage; - return ready(null, repl); - } - } - - var timer = null; - var writing = false; - var pending = false; - repl.pause(); - // History files are conventionally not readable by others: - // https://github.com/nodejs/node/issues/3392 - // https://github.com/nodejs/node/pull/3394 - fs.open(historyPath, 'a+', 0o0600, oninit); - - function oninit(err, hnd) { - if (err) { - // Cannot open history file. - // Don't crash, just don't persist history. - _writeToOutput(repl, '\nError: Could not open history file.\n' + - 'REPL session history will not be persisted.\n'); - debug(err.stack); - - repl._historyPrev = _replHistoryMessage; - repl.resume(); - return ready(null, repl); - } - fs.close(hnd, onclose); - } - - function onclose(err) { - if (err) { - return ready(err); - } - fs.readFile(historyPath, 'utf8', onread); - } - - function onread(err, data) { - if (err) { - return ready(err); - } - - if (data) { - repl.history = data.split(/[\n\r]+/, repl.historySize); - } else { - repl.history = []; - } - - fs.open(historyPath, 'r+', onhandle); - } - - function onhandle(err, hnd) { - if (err) { - return ready(err); - } - fs.ftruncate(hnd, 0, (err) => { - repl._historyHandle = hnd; - repl.on('line', online); - - // Reading the file data out erases it - repl.once('flushHistory', function() { - repl.resume(); - ready(null, repl); - }); - flushHistory(); - }); - } - - // ------ history listeners ------ - function online() { - repl._flushing = true; - - if (timer) { - clearTimeout(timer); - } - - timer = setTimeout(flushHistory, kDebounceHistoryMS); - } - - function flushHistory() { - timer = null; - if (writing) { - pending = true; - return; - } - writing = true; - const historyData = repl.history.join(os.EOL); - fs.write(repl._historyHandle, historyData, 0, 'utf8', onwritten); - } - - function onwritten(err, data) { - writing = false; - if (pending) { - pending = false; - online(); - } else { - repl._flushing = Boolean(timer); - if (!repl._flushing) { - repl.emit('flushHistory'); - } - } - } -} - - -function _replHistoryMessage() { - if (this.history.length === 0) { - _writeToOutput( - this, - '\nPersistent history support disabled. ' + - 'Set the NODE_REPL_HISTORY environment\nvariable to ' + - 'a valid, user-writable path to enable.\n' - ); - } - this._historyPrev = Interface.prototype._historyPrev; - return this._historyPrev(); + repl.setupHistory(opts.terminal ? env.NODE_REPL_HISTORY : '', cb); } diff --git a/lib/internal/repl/history.js b/lib/internal/repl/history.js new file mode 100644 index 00000000000000..a0ae07441eb912 --- /dev/null +++ b/lib/internal/repl/history.js @@ -0,0 +1,153 @@ +'use strict'; + +const { Interface } = require('readline'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); +const util = require('util'); +const debug = util.debuglog('repl'); + +// XXX(chrisdickinson): The 15ms debounce value is somewhat arbitrary. +// The debounce is to guard against code pasted into the REPL. +const kDebounceHistoryMS = 15; + +module.exports = setupHistory; + +function _writeToOutput(repl, message) { + repl._writeToOutput(message); + repl._refreshLine(); +} + +function setupHistory(repl, historyPath, ready) { + // Empty string disables persistent history + if (typeof historyPath === 'string') + historyPath = historyPath.trim(); + + if (historyPath === '') { + repl._historyPrev = _replHistoryMessage; + return ready(null, repl); + } + + if (!historyPath) { + try { + historyPath = path.join(os.homedir(), '.node_repl_history'); + } catch (err) { + _writeToOutput(repl, '\nError: Could not get the home directory.\n' + + 'REPL session history will not be persisted.\n'); + + debug(err.stack); + repl._historyPrev = _replHistoryMessage; + return ready(null, repl); + } + } + + var timer = null; + var writing = false; + var pending = false; + repl.pause(); + // History files are conventionally not readable by others: + // https://github.com/nodejs/node/issues/3392 + // https://github.com/nodejs/node/pull/3394 + fs.open(historyPath, 'a+', 0o0600, oninit); + + function oninit(err, hnd) { + if (err) { + // Cannot open history file. + // Don't crash, just don't persist history. + _writeToOutput(repl, '\nError: Could not open history file.\n' + + 'REPL session history will not be persisted.\n'); + debug(err.stack); + + repl._historyPrev = _replHistoryMessage; + repl.resume(); + return ready(null, repl); + } + fs.close(hnd, onclose); + } + + function onclose(err) { + if (err) { + return ready(err); + } + fs.readFile(historyPath, 'utf8', onread); + } + + function onread(err, data) { + if (err) { + return ready(err); + } + + if (data) { + repl.history = data.split(/[\n\r]+/, repl.historySize); + } else { + repl.history = []; + } + + fs.open(historyPath, 'r+', onhandle); + } + + function onhandle(err, hnd) { + if (err) { + return ready(err); + } + fs.ftruncate(hnd, 0, (err) => { + repl._historyHandle = hnd; + repl.on('line', online); + + // Reading the file data out erases it + repl.once('flushHistory', function() { + repl.resume(); + ready(null, repl); + }); + flushHistory(); + }); + } + + // ------ history listeners ------ + function online(line) { + repl._flushing = true; + + if (timer) { + clearTimeout(timer); + } + + timer = setTimeout(flushHistory, kDebounceHistoryMS); + } + + function flushHistory() { + timer = null; + if (writing) { + pending = true; + return; + } + writing = true; + const historyData = repl.history.join(os.EOL); + fs.write(repl._historyHandle, historyData, 0, 'utf8', onwritten); + } + + function onwritten(err, data) { + writing = false; + if (pending) { + pending = false; + online(); + } else { + repl._flushing = Boolean(timer); + if (!repl._flushing) { + repl.emit('flushHistory'); + } + } + } +} + +function _replHistoryMessage() { + if (this.history.length === 0) { + _writeToOutput( + this, + '\nPersistent history support disabled. ' + + 'Set the NODE_REPL_HISTORY environment\nvariable to ' + + 'a valid, user-writable path to enable.\n' + ); + } + this._historyPrev = Interface.prototype._historyPrev; + return this._historyPrev(); +} diff --git a/lib/repl.js b/lib/repl.js index 30812c232b7639..3844056b2da0a4 100644 --- a/lib/repl.js +++ b/lib/repl.js @@ -82,6 +82,7 @@ const { startSigintWatchdog, stopSigintWatchdog } = internalBinding('util'); +const history = require('internal/repl/history'); // Lazy-loaded. let processTopLevelAwait; @@ -774,6 +775,10 @@ exports.start = function(prompt, return repl; }; +REPLServer.prototype.setupHistory = function setupHistory(historyFile, cb) { + history(this, historyFile, cb); +}; + REPLServer.prototype.clearBufferedCommand = function clearBufferedCommand() { this[kBufferedCommandSymbol] = ''; }; diff --git a/node.gyp b/node.gyp index 6b7cbe26f34713..a7b94f218d0c27 100644 --- a/node.gyp +++ b/node.gyp @@ -171,6 +171,7 @@ 'lib/internal/readline.js', 'lib/internal/repl.js', 'lib/internal/repl/await.js', + 'lib/internal/repl/history.js', 'lib/internal/repl/recoverable.js', 'lib/internal/socket_list.js', 'lib/internal/test/binding.js', diff --git a/test/parallel/test-repl-programmatic-history.js b/test/parallel/test-repl-programmatic-history.js new file mode 100644 index 00000000000000..7eda401a7c386b --- /dev/null +++ b/test/parallel/test-repl-programmatic-history.js @@ -0,0 +1,245 @@ +'use strict'; + +const common = require('../common'); +const fixtures = require('../common/fixtures'); +const stream = require('stream'); +const REPL = require('repl'); +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +const tmpdir = require('../common/tmpdir'); +tmpdir.refresh(); + +// Mock os.homedir() +os.homedir = function() { + return tmpdir.path; +}; + +// Create an input stream specialized for testing an array of actions +class ActionStream extends stream.Stream { + run(data) { + const _iter = data[Symbol.iterator](); + const doAction = () => { + const next = _iter.next(); + if (next.done) { + // Close the repl. Note that it must have a clean prompt to do so. + setImmediate(() => { + this.emit('keypress', '', { ctrl: true, name: 'd' }); + }); + return; + } + const action = next.value; + + if (typeof action === 'object') { + this.emit('keypress', '', action); + } else { + this.emit('data', `${action}\n`); + } + setImmediate(doAction); + }; + setImmediate(doAction); + } + resume() {} + pause() {} +} +ActionStream.prototype.readable = true; + + +// Mock keys +const UP = { name: 'up' }; +const ENTER = { name: 'enter' }; +const CLEAR = { ctrl: true, name: 'u' }; + +// File paths +const historyFixturePath = fixtures.path('.node_repl_history'); +const historyPath = path.join(tmpdir.path, '.fixture_copy_repl_history'); +const historyPathFail = fixtures.path('nonexistent_folder', 'filename'); +const defaultHistoryPath = path.join(tmpdir.path, '.node_repl_history'); +const emptyHiddenHistoryPath = fixtures.path('.empty-hidden-repl-history-file'); +const devNullHistoryPath = path.join(tmpdir.path, + '.dev-null-repl-history-file'); +// Common message bits +const prompt = '> '; +const replDisabled = '\nPersistent history support disabled. Set the ' + + 'NODE_REPL_HISTORY environment\nvariable to a valid, ' + + 'user-writable path to enable.\n'; +const homedirErr = '\nError: Could not get the home directory.\n' + + 'REPL session history will not be persisted.\n'; +const replFailedRead = '\nError: Could not open history file.\n' + + 'REPL session history will not be persisted.\n'; + +const tests = [ + { + env: { NODE_REPL_HISTORY: '' }, + test: [UP], + expected: [prompt, replDisabled, prompt] + }, + { + env: { NODE_REPL_HISTORY: ' ' }, + test: [UP], + expected: [prompt, replDisabled, prompt] + }, + { + env: { NODE_REPL_HISTORY: historyPath }, + test: [UP, CLEAR], + expected: [prompt, `${prompt}'you look fabulous today'`, prompt] + }, + { + env: {}, + test: [UP, '\'42\'', ENTER], + expected: [prompt, '\'', '4', '2', '\'', '\'42\'\n', prompt, prompt], + clean: false + }, + { // Requires the above test case + env: {}, + test: [UP, UP, ENTER], + expected: [prompt, `${prompt}'42'`, '\'42\'\n', prompt] + }, + { + env: { NODE_REPL_HISTORY: historyPath, + NODE_REPL_HISTORY_SIZE: 1 }, + test: [UP, UP, CLEAR], + expected: [prompt, `${prompt}'you look fabulous today'`, prompt] + }, + { + env: { NODE_REPL_HISTORY: historyPathFail, + NODE_REPL_HISTORY_SIZE: 1 }, + test: [UP], + expected: [prompt, replFailedRead, prompt, replDisabled, prompt] + }, + { + before: function before() { + if (common.isWindows) { + const execSync = require('child_process').execSync; + execSync(`ATTRIB +H "${emptyHiddenHistoryPath}"`, (err) => { + assert.ifError(err); + }); + } + }, + env: { NODE_REPL_HISTORY: emptyHiddenHistoryPath }, + test: [UP], + expected: [prompt] + }, + { + before: function before() { + if (!common.isWindows) + fs.symlinkSync('/dev/null', devNullHistoryPath); + }, + env: { NODE_REPL_HISTORY: devNullHistoryPath }, + test: [UP], + expected: [prompt] + }, + { // Make sure this is always the last test, since we change os.homedir() + before: function before() { + // Mock os.homedir() failure + os.homedir = function() { + throw new Error('os.homedir() failure'); + }; + }, + env: {}, + test: [UP], + expected: [prompt, homedirErr, prompt, replDisabled, prompt] + } +]; +const numtests = tests.length; + + +function cleanupTmpFile() { + try { + // Write over the file, clearing any history + fs.writeFileSync(defaultHistoryPath, ''); + } catch (err) { + if (err.code === 'ENOENT') return true; + throw err; + } + return true; +} + +// Copy our fixture to the tmp directory +fs.createReadStream(historyFixturePath) + .pipe(fs.createWriteStream(historyPath)).on('unpipe', () => runTest()); + +const runTestWrap = common.mustCall(runTest, numtests); + +function runTest(assertCleaned) { + const opts = tests.shift(); + if (!opts) return; // All done + + if (assertCleaned) { + try { + assert.strictEqual(fs.readFileSync(defaultHistoryPath, 'utf8'), ''); + } catch (e) { + if (e.code !== 'ENOENT') { + console.error(`Failed test # ${numtests - tests.length}`); + throw e; + } + } + } + + const test = opts.test; + const expected = opts.expected; + const clean = opts.clean; + const before = opts.before; + const historySize = opts.env.NODE_REPL_HISTORY_SIZE; + const historyFile = opts.env.NODE_REPL_HISTORY; + + if (before) before(); + + const repl = REPL.start({ + input: new ActionStream(), + output: new stream.Writable({ + write(chunk, _, next) { + const output = chunk.toString(); + + // Ignore escapes and blank lines + if (output.charCodeAt(0) === 27 || /^[\r\n]+$/.test(output)) + return next(); + + try { + assert.strictEqual(output, expected.shift()); + } catch (err) { + console.error(`Failed test # ${numtests - tests.length}`); + throw err; + } + next(); + } + }), + prompt: prompt, + useColors: false, + terminal: true, + historySize: historySize + }); + + repl.setupHistory(historyFile, function(err, repl) { + if (err) { + console.error(`Failed test # ${numtests - tests.length}`); + throw err; + } + + repl.once('close', () => { + if (repl._flushing) { + repl.once('flushHistory', onClose); + return; + } + + onClose(); + }); + + function onClose() { + const cleaned = clean === false ? false : cleanupTmpFile(); + + try { + // Ensure everything that we expected was output + assert.strictEqual(expected.length, 0); + setImmediate(runTestWrap, cleaned); + } catch (err) { + console.error(`Failed test # ${numtests - tests.length}`); + throw err; + } + } + + repl.inputStream.run(test); + }); +} From 3352e6e04dd558d4858b8042663df7fc7c425d95 Mon Sep 17 00:00:00 2001 From: Lance Ball Date: Mon, 4 Feb 2019 12:27:07 -0500 Subject: [PATCH 2/2] doc: add replServer.setupHistory() to documentation --- doc/api/repl.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/api/repl.md b/doc/api/repl.md index 60adbf641d1592..4395193de20326 100644 --- a/doc/api/repl.md +++ b/doc/api/repl.md @@ -448,6 +448,22 @@ deprecated: v9.0.0 An internal method used to parse and execute `REPLServer` keywords. Returns `true` if `keyword` is a valid keyword, otherwise `false`. +### replServer.setupHistory(historyPath, callback) + + +* `historyPath` {string} the path to the history file +* `callback` {Function} called when history writes are ready or upon error + * `err` {Error} + * `repl` {repl.REPLServer} + +Initializes a history log file for the REPL instance. When executing the +Node.js binary and using the command line REPL, a history file is initialized +by default. However, this is not the case when creating a REPL +programmatically. Use this method to initialize a history log file when working +with REPL instances programmatically. + ## repl.start([options])