move shell_parse into parser (fixes #3150)

Enables the following:

    julia> macro example()
               f = "a"
               esc(quote
                   g = "b"
                   `echo $($f) $g`
               end)
           end

    julia> @example
    `echo a b`

base/REPLCompletions.jl

@@ -379,27 +379,25 @@ end
 
 function shell_completions(string, pos)
     # First parse everything up to the current position
-    scs = string[1:pos]
-    local args, last_parse
-    try
-        args, last_parse = Base.shell_parse(scs, true)
-    catch
-        return UTF8String[], 0:-1, false
+    partial = string[1:pos]
+    ex = Base.shell_parse(partial)
+
+    if isexpr(ex, :incomplete) || isexpr(ex, :error)
+        frange, method_name_end = find_start_brace(partial)
+        partial = partial[frange]
+        ret, range = completions(partial, endof(partial))
+        range += first(frange) - 1
+        return ret, range, true
     end
+
     # Now look at the last thing we parsed
-    isempty(args.args[end].args) && return UTF8String[], 0:-1, false
-    arg = args.args[end].args[end]
-    if all(s -> isa(s, AbstractString), args.args[end].args)
+    a = ex.args[end].args
+    if length(a) > 0 && all(s->isa(s,AbstractString), a)
         # Treat this as a path (perhaps give a list of commands in the future as well?)
-        return complete_path(join(args.args[end].args), pos)
-    elseif isexpr(arg, :escape) && (isexpr(arg.args[1], :incomplete) || isexpr(arg.args[1], :error))
-        r = first(last_parse):prevind(last_parse, last(last_parse))
-        partial = scs[r]
-        ret, range = completions(partial, endof(partial))
-        range += first(r) - 1
-        return ret, range, true
+        return complete_path(join(a), pos)
     end
-    return UTF8String[], 0:-1, false
+
+    UTF8String[], 0:-1, false
 end
 
 end # module

base/process.jl

@@ -649,7 +649,7 @@ function cmd_gen(parsed)
 end
 
 macro cmd(str)
-    :(cmd_gen($(shell_parse(str)[1])))
+    :(cmd_gen($(shell_parse(str))))
 end
 
 wait(x::Process)      = if !process_exited(x); stream_wait(x, x.exitnotify); end

base/shell.jl

@@ -2,127 +2,16 @@
 
 ## shell-like command parsing ##
 
-function shell_parse(raw::AbstractString, interp::Bool)
-    s = lstrip(raw)
-    #Strips the end but respects the space when the string endswith "\\ "
-    r = RevString(s)
-    i = start(r)
-    c_old = nothing
-    while !done(r,i)
-        c, j = next(r,i)
-        if c == '\\' && c_old == ' '
-            i -= 1
-            break
-        elseif !(c in _default_delims)
-            break
-        end
-        i = j
-        c_old = c
-    end
-    s = s[1:end-i+1]
-
-    last_parse = 0:-1
-    isempty(s) && return interp ? (Expr(:tuple,:()),last_parse) : ([],last_parse)
-
-    in_single_quotes = false
-    in_double_quotes = false
-
-    args::Vector{Any} = []
-    arg::Vector{Any} = []
-    i = start(s)
-    j = i
-
-    function update_arg(x)
-        if !isa(x,AbstractString) || !isempty(x)
-            push!(arg, x)
-        end
-    end
-    function append_arg()
-        if isempty(arg); arg = Any["",]; end
-        push!(args, arg)
-        arg = []
-    end
 
-    while !done(s,j)
-        c, k = next(s,j)
-        if !in_single_quotes && !in_double_quotes && isspace(c)
-            update_arg(s[i:j-1])
-            append_arg()
-            j = k
-            while !done(s,j)
-                c, k = next(s,j)
-                if !isspace(c)
-                    i = j
-                    break
-                end
-                j = k
-            end
-        elseif interp && !in_single_quotes && c == '$'
-            update_arg(s[i:j-1]); i = k; j = k
-            if done(s,k)
-                error("\$ right before end of command")
-            end
-            if isspace(s[k])
-                error("space not allowed right after \$")
-            end
-            stpos = j
-            ex, j = parse(s,j,greedy=false)
-            last_parse = stpos:j
-            update_arg(esc(ex)); i = j
-        else
-            if !in_double_quotes && c == '\''
-                in_single_quotes = !in_single_quotes
-                update_arg(s[i:j-1]); i = k
-            elseif !in_single_quotes && c == '"'
-                in_double_quotes = !in_double_quotes
-                update_arg(s[i:j-1]); i = k
-            elseif c == '\\'
-                if in_double_quotes
-                    if done(s,k)
-                        error("unterminated double quote")
-                    end
-                    if s[k] == '"' || s[k] == '$'
-                        update_arg(s[i:j-1]); i = k
-                        c, k = next(s,k)
-                    end
-                elseif !in_single_quotes
-                    if done(s,k)
-                        error("dangling backslash")
-                    end
-                    update_arg(s[i:j-1]); i = k
-                    c, k = next(s,k)
-                end
-            end
-            j = k
-        end
-    end
-
-    if in_single_quotes; error("unterminated single quote"); end
-    if in_double_quotes; error("unterminated double quote"); end
-
-    update_arg(s[i:end])
-    append_arg()
-
-    if !interp
-        return (args,last_parse)
-    end
-
-    # construct an expression
-    ex = Expr(:tuple)
-    for arg in args
-        push!(ex.args, Expr(:tuple, arg...))
-    end
-    (ex,last_parse)
+function shell_parse(s::ByteString, interpolate::Bool)
+    ccall(:jl_parse_shell_line, Any, (Cstring, Csize_t, Cint),
+                                     s, sizeof(s), interpolate)
 end
-shell_parse(s::AbstractString) = shell_parse(s,true)
+shell_parse(s::AbstractString) = shell_parse(bytestring(s), true)
 
 function shell_split(s::AbstractString)
-    parsed = shell_parse(s,false)[1]
-    args = AbstractString[]
-    for arg in parsed
-       push!(args, string(arg...))
-    end
-    args
+    parsed = shell_parse(bytestring(s), false)
+    [e.args[1]::AbstractString for e in parsed.args]
 end
 
 function print_shell_word(io::IO, word::AbstractString)

src/ast.c

@@ -516,6 +516,18 @@ DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len)
     return scm_to_julia(e,0);
 }
 
+// this is used to parse a line of shell input
+DLLEXPORT jl_value_t *jl_parse_shell_line(const char *str, size_t len,
+                                          int interpolate)
+{
+    value_t s = cvalue_static_cstrn(str, len);
+    value_t e = fl_applyn(2, symbol_value(symbol("jl-parse-shell-string")),
+                          s, interpolate?FL_T:FL_F);
+    if (e == FL_EOF)
+        return jl_nothing;
+    return scm_to_julia(e,0);
+}
+
 // this is for parsing one expression out of a string, keeping track of
 // the current position.
 DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,

src/jlfrontend.scm

@@ -156,6 +156,10 @@
                                (loop (nreconc (cdr expr) exprs))
                                (loop (cons expr exprs))))))))))
 
+(define (jl-parse-shell-string s interpolate)
+  (let ((inp (make-token-stream (open-input-string s))))
+    (parser-wrap (lambda () (parse-shell inp interpolate #f)))))
+
 ;; parse file-in-a-string
 (define (jl-parse-string-stream str filename)
   (jl-parser-set-stream filename (open-input-string str)))

src/julia-parser.scm

@@ -1637,22 +1637,86 @@
       (error "incomplete: invalid \"`\" syntax") ; NOTE: changing this may affect code in base/client.jl
       c))
 
+(define (filter-out-empty-str e)
+  (filter (lambda (s) (not (and (string? s) (= (length s) 0)))) e))
+
+(define (space? c) (memv c '(#\space #\newline #\tab)))
+
 (define (parse-backquote s)
-  (let ((b (open-output-string))
-        (p (ts:port s)))
-    (let loop ((c (read-char p)))
-      (if (eqv? c #\`)
-          #t
-          (begin (if (eqv? c #\\)
-                     (let ((nextch (read-char p)))
-                       (if (eqv? nextch #\`)
-                           (write-char nextch b)
-                           (begin (write-char #\\ b)
-                                  (write-char (not-eof-2 nextch) b))))
-                     (write-char (not-eof-2 c) b))
-                 (loop (read-char p)))))
-    (let ((str (io.tostring! b)))
-      `(macrocall @cmd ,str))))
+  `(call (top cmd_gen) ,(parse-shell s #t #t)))
+
+(define (parse-shell s interpolate in-backticks)
+  (define (push-str b e)
+    (cons (io.tostring! b) e))
+  (define (push-arg e es)
+    (cons (reverse (if (length> e 2) (filter-out-empty-str e) e)) es))
+  (let ((p (ts:port s)))
+    (let loop ((c (read-char p))
+               (b (open-output-string))
+               (e '(tuple))
+               (es '(tuple))
+               (mode 0)) ; 0:ignore-space, 1:normal, 2:"...", 3:'...'
+      (cond
+        ((if in-backticks (eqv? c #\`) (eof-object? c))
+         (let ((result
+                 (case mode
+                   ((0) es)
+                   ((1) (push-arg (push-str b e) es))
+                   ((2) (error "unterminated double quote"))
+                   ((3) (error "unterminated single quote")))))
+               (reverse result)))
+
+        ; escape
+        ((eqv? c #\\)
+         (let ((nextc (not-eof-2 (read-char p))))
+           (begin
+             (if (or (= mode 3)
+                     (and (= mode 2)
+                          (not (memv nextc '(#\$ #\" #\\)))))
+                 (write-char #\\ b))
+             (write-char nextc b)))
+         (loop (read-char p) b e es (max 1 mode)))
+
+        ; ignore space
+        ((and (space? c) (= mode 0))
+         (loop (read-char p) b e es 0))
+
+        ; end argument
+        ((and (space? c) (= mode 1))
+         (loop (read-char p)
+               (open-output-string)
+               '(tuple)
+               (push-arg (push-str b e) es)
+               0))
+
+        ; start double quotes
+        ((and (eqv? c #\") (< mode 2))
+         (loop (read-char p) b e es 2))
+
+        ; end double quotes
+        ((and (eqv? c #\") (= mode 2))
+         (loop (read-char p) b e es 1))
+
+        ; start single quotes
+        ((and (eqv? c #\') (< mode 2))
+         (loop (read-char p) b e es 3))
+
+        ; end single quotes
+        ((and (eqv? c #\') (= mode 3))
+         (loop (read-char p) b e es 1))
+
+        ; interpolate
+        ((and interpolate (eqv? c #\$) (< mode 3))
+         (let ((ex (parse-interpolate s)))
+           (loop (read-char p)
+                 (open-output-string)
+                 (cons ex (push-str b e))
+                 es
+                 (max 1 mode))))
+
+        (else
+         (write-char (not-eof-2 c) b)
+         (loop (read-char p) b e es (max 1 mode)))))))
 
 (define (not-eof-3 c)
   (if (eof-object? c)
@@ -2024,10 +2088,7 @@
            (take-token s)
            (let ((ps (parse-string-literal s #f)))
              (if (length> ps 1)
-                 `(string ,@(filter (lambda (s)
-                                      (not (and (string? s)
-                                                (= (length s) 0))))
-                                    ps))
+                 `(string ,@(filter-out-empty-str ps))
                  (car ps))))
 
           ;; macro call

src/julia.h

@@ -1208,6 +1208,7 @@ void jl_init_restored_modules(jl_array_t *init_order);
 
 // front end interface
 DLLEXPORT jl_value_t *jl_parse_input_line(const char *str, size_t len);
+DLLEXPORT jl_value_t *jl_parse_shell_line(const char *str, size_t len, int interpolate);
 DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
                                       int pos0, int greedy);
 DLLEXPORT int jl_parse_depwarn(int warn);

test/parse.jl

@@ -16,6 +16,13 @@ function parseall(str)
     end
 end
 
+# issue #3150
+macro test3150()
+    f="a"
+    esc(quote g="b" ; `echo $($f) $g` end)
+end
+@test @test3150().exec == ["echo","a","b"]
+
 # issue #9684
 let
     for (ex1, ex2) in [("5.≠x", "5.!=x"),

test/strings/io.jl

@@ -145,8 +145,8 @@ else
     primary_encoding = "UTF-32BE"
     primary_path = replace(joinpath(unicodedir, primary_encoding*".unicode"),"\\","\\\\\\\\")
     run(`perl -e "
-        $$fname = \"$primary_path\";
-        open(UNICODEF, \">\", \"$$fname\")         or die \"can\'t open $$fname: $$!\";
+        \$fname = \"$primary_path\";
+        open(UNICODEF, \">\", \"\$fname\")         or die \"can\'t open \$fname: \$!\";
         binmode(UNICODEF);
         print UNICODEF pack \"N*\", 0xfeff, 0..0xd7ff, 0xe000..0x10ffff;
         close(UNICODEF);"` )