From 1aa3696e6243c4daab4b47ec9b1950e081943adb Mon Sep 17 00:00:00 2001 From: Andy Hayden Date: Tue, 9 Oct 2018 08:06:04 +0530 Subject: [PATCH] Add repl - Running repl from js side. - Add tests for repl behavior. - Handle ctrl-C and ctrl-D. --- BUILD.gn | 2 + Cargo.toml | 1 + build_extra/rust/BUILD.gn | 47 +++++++++++++++ js/main.ts | 13 ++-- js/repl.ts | 89 +++++++++++++++++++++++++++ src/main.rs | 2 + src/msg.fbs | 22 +++++++ src/ops.rs | 72 ++++++++++++++++++++++ src/repl.rs | 122 ++++++++++++++++++++++++++++++++++++++ src/resources.rs | 33 +++++++++-- third_party | 2 +- tools/repl_test.py | 110 ++++++++++++++++++++++++++++++++++ tools/test.py | 3 + 13 files changed, 505 insertions(+), 13 deletions(-) create mode 100644 js/repl.ts create mode 100644 src/repl.rs create mode 100644 tools/repl_test.py diff --git a/BUILD.gn b/BUILD.gn index b82c5e9a89d392..ce15a762d31475 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -63,6 +63,7 @@ main_extern = [ "$rust_build:rand", "$rust_build:remove_dir_all", "$rust_build:ring", + "$rust_build:rustyline", "$rust_build:tempfile", "$rust_build:tokio", "$rust_build:tokio_executor", @@ -114,6 +115,7 @@ ts_sources = [ "js/remove.ts", "js/rename.ts", "js/resources.ts", + "js/repl.ts", "js/stat.ts", "js/symlink.ts", "js/text_encoding.ts", diff --git a/Cargo.toml b/Cargo.toml index 869a63c83022da..88774a752424fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ libc = "0.2.43" log = "0.4.6" rand = "0.5.5" remove_dir_all = "0.5.1" +rustyline = "2.1.0" ring = "0.13.2" tempfile = "3.0.4" tokio = "0.1.11" diff --git a/build_extra/rust/BUILD.gn b/build_extra/rust/BUILD.gn index 0db09052b1d925..6740f3bebcaffd 100644 --- a/build_extra/rust/BUILD.gn +++ b/build_extra/rust/BUILD.gn @@ -11,6 +11,51 @@ import("rust.gni") crates = "//third_party/rust_crates" registry_github = "$crates/registry/src/github.aaakk.us.kg-1ecc6299db9ec823/" +rust_crate("nix") { + source_root = "$registry_github/nix-0.11.0/src/lib.rs" + extern = [ + ":cfg_if", + ":libc", + ":void", + ":bitflags", + ] +} + +rust_crate("rustyline") { + source_root = "$registry_github/rustyline-2.1.0/src/lib.rs" + extern = [ + ":dirs", + ":libc", + ":log", + ":memchr", + ":nix", + ":unicode_segmentation", + ":unicode_width", + ":utf8parse", + ":winapi", + ] +} + +rust_crate("bitflags") { + source_root = "$registry_github/bitflags-1.0.4/src/lib.rs" +} + +rust_crate("unicode_segmentation") { + source_root = "$registry_github/unicode-segmentation-1.2.1/src/lib.rs" +} + +rust_crate("memchr") { + source_root = "$registry_github/memchr-2.1.0/src/lib.rs" + extern = [ + ":cfg_if", + ":libc", + ] +} + +rust_crate("utf8parse") { + source_root = "$registry_github/utf8parse-0.1.1/src/lib.rs" +} + rust_crate("libc") { source_root = "$registry_github/libc-0.2.43/src/lib.rs" features = [ "use_std" ] @@ -127,6 +172,7 @@ rust_crate("winapi") { "knownfolders", "ktmtypes", "libloaderapi", + "limits", "lsalookup", "minwinbase", "minwindef", @@ -167,6 +213,7 @@ rust_crate("winapi") { "winnt", "winreg", "winsock2", + "winuser", "ws2def", "ws2ipdef", "ws2tcpip", diff --git a/js/main.ts b/js/main.ts index 50de2c26c31ed0..0ed45aaec50829 100644 --- a/js/main.ts +++ b/js/main.ts @@ -8,6 +8,7 @@ import { libdeno } from "./libdeno"; import { args } from "./deno"; import { sendSync, handleAsyncMsgFromRust } from "./dispatch"; import { promiseErrorExaminer, promiseRejectHandler } from "./promise_util"; +import { replLoop } from "./repl"; import { version } from "typescript"; function sendStart(): msg.StartRes { @@ -77,13 +78,13 @@ export default function denoMain() { } log("args", args); Object.freeze(args); - const inputFn = args[0]; - if (!inputFn) { - console.log("No input script specified."); - os.exit(1); - } compiler.recompile = startResMsg.recompileFlag(); - compiler.run(inputFn, `${cwd}/`); + + if (inputFn) { + compiler.run(inputFn, `${cwd}/`); + } else { + replLoop(); + } } diff --git a/js/repl.ts b/js/repl.ts new file mode 100644 index 00000000000000..b7c516110f6926 --- /dev/null +++ b/js/repl.ts @@ -0,0 +1,89 @@ +// Copyright 2018 the Deno authors. All rights reserved. MIT license. +import * as msg from "gen/msg_generated"; +import * as flatbuffers from "./flatbuffers"; +import { assert } from "./util"; +import * as deno from "./deno"; +import { close } from "./files"; +import * as dispatch from "./dispatch"; +import { exit } from "./os"; +import { window } from "./globals"; + +function startRepl(historyFile: string): number { + const builder = flatbuffers.createBuilder(); + const historyFile_ = builder.createString(historyFile); + + msg.ReplStart.startReplStart(builder); + msg.ReplStart.addHistoryFile(builder, historyFile_); + const inner = msg.ReplStart.endReplStart(builder); + + const baseRes = dispatch.sendSync(builder, msg.Any.ReplStart, inner); + assert(baseRes != null); + assert(msg.Any.ReplStartRes === baseRes!.innerType()); + const innerRes = new msg.ReplStartRes(); + assert(baseRes!.inner(innerRes) != null); + const rid = innerRes.rid(); + return rid; +} + +// @internal +export function readline(rid: number, prompt: string): string { + const builder = flatbuffers.createBuilder(); + const prompt_ = builder.createString(prompt); + msg.ReplReadline.startReplReadline(builder); + msg.ReplReadline.addRid(builder, rid); + msg.ReplReadline.addPrompt(builder, prompt_); + const inner = msg.ReplReadline.endReplReadline(builder); + + // TODO use async? + const baseRes = dispatch.sendSync(builder, msg.Any.ReplReadline, inner); + + assert(baseRes != null); + assert(msg.Any.ReplReadlineRes === baseRes!.innerType()); + const innerRes = new msg.ReplReadlineRes(); + assert(baseRes!.inner(innerRes) != null); + const line = innerRes.line(); + assert(line !== null); + return line || ""; +} + +// @internal +export function replLoop(): void { + window.deno = deno; // FIXME use a new scope (rather than window). + + const historyFile = "deno_history.txt"; + const prompt = "> "; + + const rid = startRepl(historyFile); + + let line = ""; + while (true) { + try { + line = readline(rid, prompt); + line = line.trim(); + } catch (err) { + if (err.message === "EOF") { + break; + } + console.error(err); + exit(1); + } + if (!line) { + continue; + } + if (line === ".exit") { + break; + } + try { + const result = eval.call(window, line); // FIXME use a new scope. + console.log(result); + } catch (err) { + if (err instanceof Error) { + console.error(`${err.constructor.name}: ${err.message}`); + } else { + console.error("Thrown:", err); + } + } + } + + close(rid); +} diff --git a/src/main.rs b/src/main.rs index af84b19be8e841..ca15d468b67672 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ extern crate libc; extern crate rand; extern crate remove_dir_all; extern crate ring; +extern crate rustyline; extern crate tempfile; extern crate tokio; extern crate tokio_executor; @@ -35,6 +36,7 @@ pub mod msg; pub mod msg_util; pub mod ops; pub mod permissions; +mod repl; pub mod resources; pub mod snapshot; mod tokio_util; diff --git a/src/msg.fbs b/src/msg.fbs index 5f86ad56a82ef7..9686c76cb6d954 100644 --- a/src/msg.fbs +++ b/src/msg.fbs @@ -24,6 +24,10 @@ union Any { Rename, Readlink, ReadlinkRes, + ReplStart, + ReplStartRes, + ReplReadline, + ReplReadlineRes, Resources, ResourcesRes, Symlink, @@ -273,6 +277,24 @@ table ReadlinkRes { path: string; } +table ReplStart { + history_file: string; + // TODO add config +} + +table ReplStartRes { + rid: int; +} + +table ReplReadline { + rid: int; + prompt: string; +} + +table ReplReadlineRes { + line: string; +} + table Resources {} table Resource { diff --git a/src/ops.rs b/src/ops.rs index 9266646ef4e522..f645f1d015508a 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -20,6 +20,7 @@ use futures::Poll; use hyper; use hyper::rt::{Future, Stream}; use remove_dir_all::remove_dir_all; +use repl; use resources::table_entries; use std; use std::fs; @@ -96,6 +97,8 @@ pub fn dispatch( msg::Any::Read => op_read, msg::Any::Remove => op_remove, msg::Any::Rename => op_rename, + msg::Any::ReplReadline => op_repl_readline, + msg::Any::ReplStart => op_repl_start, msg::Any::Resources => op_resources, msg::Any::SetEnv => op_set_env, msg::Any::Shutdown => op_shutdown, @@ -1086,6 +1089,75 @@ fn op_read_link( }) } +fn op_repl_start( + state: &Arc, + base: &msg::Base, + data: &'static mut [u8], +) -> Box { + assert_eq!(data.len(), 0); + let inner = base.inner_as_repl_start().unwrap(); + let cmd_id = base.cmd_id(); + let history_file = String::from(inner.history_file().unwrap()); + + debug!("op_repl_start {}", history_file); + let history_path = repl::history_path(&state.dir, &history_file); + let repl = repl::Repl::new(history_path); + let resource = resources::add_repl(repl); + + let builder = &mut FlatBufferBuilder::new(); + let inner = msg::ReplStartRes::create( + builder, + &msg::ReplStartResArgs { rid: resource.rid }, + ); + ok_future(serialize_response( + cmd_id, + builder, + msg::BaseArgs { + inner: Some(inner.as_union_value()), + inner_type: msg::Any::ReplStartRes, + ..Default::default() + }, + )) +} + +fn op_repl_readline( + _state: &Arc, + base: &msg::Base, + data: &'static mut [u8], +) -> Box { + assert_eq!(data.len(), 0); + let inner = base.inner_as_repl_readline().unwrap(); + let cmd_id = base.cmd_id(); + let rid = inner.rid(); + let prompt = inner.prompt().unwrap().to_owned(); + debug!("op_repl_readline {} {}", rid, prompt); + + // Ignore this clippy warning until this issue is addressed: + // https://github.com/rust-lang-nursery/rust-clippy/issues/1684 + #[cfg_attr(feature = "cargo-clippy", allow(redundant_closure_call))] + Box::new(futures::future::result((move || { + let line = resources::readline(rid, &prompt)?; + + let builder = &mut FlatBufferBuilder::new(); + let line_off = builder.create_string(&line); + let inner = msg::ReplReadlineRes::create( + builder, + &msg::ReplReadlineResArgs { + line: Some(line_off), + }, + ); + Ok(serialize_response( + cmd_id, + builder, + msg::BaseArgs { + inner: Some(inner.as_union_value()), + inner_type: msg::Any::ReplReadlineRes, + ..Default::default() + }, + )) + })())) +} + fn op_truncate( state: &Arc, base: &msg::Base, diff --git a/src/repl.rs b/src/repl.rs new file mode 100644 index 00000000000000..af1679194312c9 --- /dev/null +++ b/src/repl.rs @@ -0,0 +1,122 @@ +// Copyright 2018 the Deno authors. All rights reserved. MIT license. +extern crate rustyline; + +use rustyline::error::ReadlineError::Interrupted; + +use msg::ErrorKind; +use std::error::Error; + +use deno_dir::DenoDir; +use errors::new as deno_error; +use errors::DenoResult; +use std::path::PathBuf; +use std::process::exit; + +#[cfg(not(windows))] +use rustyline::Editor; + +// Work around the issue that on Windows, `struct Editor` does not implement the +// `Send` trait, because it embeds a windows HANDLE which is a type alias for +// *mut c_void. This value isn't actually a pointer and there's nothing that +// can be mutated through it, so hack around it. TODO: a prettier solution. +#[cfg(windows)] +use std::ops::{Deref, DerefMut}; + +#[cfg(windows)] +struct Editor { + inner: rustyline::Editor, +} + +#[cfg(windows)] +unsafe impl Send for Editor {} + +#[cfg(windows)] +impl Editor { + pub fn new() -> Editor { + Editor { + inner: rustyline::Editor::::new(), + } + } +} + +#[cfg(windows)] +impl Deref for Editor { + type Target = rustyline::Editor; + + fn deref(&self) -> &rustyline::Editor { + &self.inner + } +} + +#[cfg(windows)] +impl DerefMut for Editor { + fn deref_mut(&mut self) -> &mut rustyline::Editor { + &mut self.inner + } +} + +pub struct Repl { + editor: Editor<()>, + history_file: PathBuf, +} + +impl Repl { + pub fn new(history_file: PathBuf) -> Repl { + let mut repl = Repl { + editor: Editor::<()>::new(), + history_file, + }; + + repl.load_history(); + repl + } + + fn load_history(&mut self) -> () { + debug!("Loading REPL history: {:?}", self.history_file); + self + .editor + .load_history(&self.history_file.to_str().unwrap()) + .map_err(|e| debug!("Unable to load history file: {:?} {}", self.history_file, e)) + // ignore this error (e.g. it occurs on first load) + .unwrap_or(()) + } + + fn save_history(&mut self) -> DenoResult<()> { + self + .editor + .save_history(&self.history_file.to_str().unwrap()) + .map(|_| debug!("Saved REPL history to: {:?}", self.history_file)) + .map_err(|e| { + eprintln!("Unable to save REPL history: {:?} {}", self.history_file, e); + deno_error(ErrorKind::Other, e.description().to_string()) + }) + } + + pub fn readline(&mut self, prompt: &str) -> DenoResult { + self + .editor + .readline(&prompt) + .map(|line| { + self.editor.add_history_entry(line.as_ref()); + line + }).map_err(|e| match e { + Interrupted => { + self.save_history().unwrap(); + exit(1) + } + e => deno_error(ErrorKind::Other, e.description().to_string()), + }) + } +} + +impl Drop for Repl { + fn drop(&mut self) { + self.save_history().unwrap(); + } +} + +pub fn history_path(dir: &DenoDir, history_file: &str) -> PathBuf { + let mut p: PathBuf = dir.root.clone(); + p.push(history_file); + p +} diff --git a/src/resources.rs b/src/resources.rs index 1e3b0a9b7f2e39..5d472adc77ac1a 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -10,7 +10,10 @@ #[cfg(unix)] use eager_unix as eager; +use errors::bad_resource; use errors::DenoError; +use errors::DenoResult; +use repl::Repl; use tokio_util; use tokio_write; @@ -56,6 +59,7 @@ enum Repr { FsFile(tokio::fs::File), TcpListener(tokio::net::TcpListener), TcpStream(tokio::net::TcpStream), + Repl(Repl), } pub fn table_entries() -> Vec<(i32, String)> { @@ -85,6 +89,7 @@ fn inspect_repr(repr: &Repr) -> String { Repr::FsFile(_) => "fsFile", Repr::TcpListener(_) => "tcpListener", Repr::TcpStream(_) => "tcpStream", + Repr::Repl(_) => "repl", }; String::from(h_repr) @@ -150,10 +155,7 @@ impl AsyncRead for Resource { Repr::FsFile(ref mut f) => f.poll_read(buf), Repr::Stdin(ref mut f) => f.poll_read(buf), Repr::TcpStream(ref mut f) => f.poll_read(buf), - Repr::Stdout(_) | Repr::Stderr(_) => { - panic!("Cannot read from stdout/stderr") - } - Repr::TcpListener(_) => panic!("Cannot read"), + _ => panic!("Cannot read"), }, } } @@ -180,8 +182,7 @@ impl AsyncWrite for Resource { Repr::Stdout(ref mut f) => f.poll_write(buf), Repr::Stderr(ref mut f) => f.poll_write(buf), Repr::TcpStream(ref mut f) => f.poll_write(buf), - Repr::Stdin(_) => panic!("Cannot write to stdin"), - Repr::TcpListener(_) => panic!("Cannot write"), + _ => panic!("Cannot write"), }, } } @@ -221,6 +222,26 @@ pub fn add_tcp_stream(stream: tokio::net::TcpStream) -> Resource { Resource { rid } } +pub fn add_repl(repl: Repl) -> Resource { + let rid = new_rid(); + let mut tg = RESOURCE_TABLE.lock().unwrap(); + let r = tg.insert(rid, Repr::Repl(repl)); + assert!(r.is_none()); + Resource { rid } +} + +pub fn readline(rid: ResourceId, prompt: &str) -> DenoResult { + let mut table = RESOURCE_TABLE.lock().unwrap(); + let maybe_repr = table.get_mut(&rid); + match maybe_repr { + Some(Repr::Repl(ref mut r)) => { + let line = r.readline(&prompt)?; + Ok(line) + } + _ => Err(bad_resource()), + } +} + pub fn lookup(rid: ResourceId) -> Option { let table = RESOURCE_TABLE.lock().unwrap(); table.get(&rid).map(|_| Resource { rid }) diff --git a/third_party b/third_party index 96d35734a47e5b..d1447e6375ebdd 160000 --- a/third_party +++ b/third_party @@ -1 +1 @@ -Subproject commit 96d35734a47e5b63d98ba7f7cbd01dfe4cbc435f +Subproject commit d1447e6375ebddf590f1cd87219dadeca51cfec1 diff --git a/tools/repl_test.py b/tools/repl_test.py new file mode 100644 index 00000000000000..5b3172edbd2d19 --- /dev/null +++ b/tools/repl_test.py @@ -0,0 +1,110 @@ +# Copyright 2018 the Deno authors. All rights reserved. MIT license. +import os +from subprocess import PIPE, Popen +import sys +from time import sleep + +from util import build_path, executable_suffix, green_ok + + +class Repl(object): + def __init__(self, deno_exe): + self.deno_exe = deno_exe + self.warm_up() + + def input(self, *lines, **kwargs): + exit_ = kwargs.pop("exit", True) + p = Popen([self.deno_exe], stdout=PIPE, stderr=PIPE, stdin=PIPE) + try: + for line in lines: + p.stdin.write(line.encode("utf-8") + b'\n') + if exit_: + p.stdin.write(b'deno.exit(0)\n') + else: + sleep(1) # wait to be killed by js + out, err = p.communicate() + except Exception as e: # Should this be CalledProcessError? + p.kill() + p.wait() + raise + retcode = p.poll() + # Ignore Windows CRLF (\r\n). + return out.replace('\r\n', '\n'), err.replace('\r\n', '\n'), retcode + + def warm_up(self): + # This may output an error message about the history file (ignore it). + self.input("") + + def test_function(self): + out, err, code = self.input("deno.writeFileSync") + assertEqual(out, '[Function: writeFileSync]\n') + assertEqual(err, '') + assertEqual(code, 0) + + def test_console_log(self): + out, err, code = self.input("console.log('hello')", "'world'") + assertEqual(out, 'hello\nundefined\nworld\n') + assertEqual(err, '') + assertEqual(code, 0) + + def test_variable(self): + out, err, code = self.input("var a = 123;", "a") + assertEqual(out, 'undefined\n123\n') + assertEqual(err, '') + assertEqual(code, 0) + + def test_settimeout(self): + out, err, code = self.input( + "setTimeout(() => { console.log('b'); deno.exit(0); }, 10)", + "'a'", + exit=False) + assertEqual(out, '1\na\nb\n') + assertEqual(err, '') + assertEqual(code, 0) + + def test_reference_error(self): + out, err, code = self.input("not_a_variable") + assertEqual(out, '') + assertEqual(err, 'ReferenceError: not_a_variable is not defined\n') + assertEqual(code, 0) + + def test_syntax_error(self): + out, err, code = self.input("syntax error") + assertEqual(out, '') + assertEqual(err, "SyntaxError: Unexpected identifier\n") + assertEqual(code, 0) + + def test_type_error(self): + out, err, code = self.input("console()") + assertEqual(out, '') + assertEqual(err, 'TypeError: console is not a function\n') + assertEqual(code, 0) + + def test_exit_command(self): + out, err, code = self.input(".exit", "'ignored'", exit=False) + assertEqual(out, '') + assertEqual(err, '') + assertEqual(code, 0) + + def run(self): + print('repl_test.py') + test_names = [name for name in dir(self) if name.startswith("test_")] + for t in test_names: + self.__getattribute__(t)() + sys.stdout.write(".") + sys.stdout.flush() + print(' {}\n'.format(green_ok())) + + +def assertEqual(left, right): + if left != right: + raise AssertionError("{} != {}".format(repr(left), repr(right))) + + +def repl_tests(deno_exe): + Repl(deno_exe).run() + + +if __name__ == "__main__": + deno_exe = os.path.join(build_path(), "deno" + executable_suffix) + repl_tests(deno_exe) diff --git a/tools/test.py b/tools/test.py index 18fc23e5c463d3..41e811a6d3fb38 100755 --- a/tools/test.py +++ b/tools/test.py @@ -11,6 +11,7 @@ from unit_tests import unit_tests from util_test import util_test from benchmark_test import benchmark_test +from repl_test import repl_tests import subprocess import http_server @@ -67,6 +68,8 @@ def main(argv): from permission_prompt_test import permission_prompt_test permission_prompt_test(deno_exe) + repl_tests(deno_exe) + rmtree(deno_dir) deno_dir_test(deno_exe, deno_dir)