Correctly parse and quote shell commands

The quoting is now safe for Posix sh-like shells.

The algorithm tries several ways to quote words (via single or double quotes or backslashes), and then uses a scoring method to choose the "best" result. The current scoring chooses the shortest string, and adds additional penalties for backslashes.

base/shell.jl

@@ -81,7 +81,7 @@ function shell_parse(raw::AbstractString, interp::Bool)
                     if done(s,k)
                         error("unterminated double quote")
                     end
-                    if s[k] == '"' || s[k] == '$'
+                    if s[k] == '"' || s[k] == '$' || s[k] == '\\'
                         update_arg(s[i:j-1]); i = k
                         c, k = next(s,k)
                     end
@@ -125,34 +125,64 @@ function shell_split(s::AbstractString)
     args
 end
 
-function print_shell_word(io::IO, word::AbstractString)
-    if isempty(word)
-        print(io, "''")
+"Quote by enclosing in single quotes"
+function _quote_via_single_quote(word::AbstractString)
+    # Quote by enclosing in single quotes, and replace all single
+    # quotes with backslash-quote
+    quoted = "'" * replace(word, r"'", "'\\''") * "'"
+    # Remove empty leading or trailing quotes
+    quoted = replace(quoted, r"^''", "")
+    quoted = replace(quoted, r"'''$(?!\n)", "'")
+    if isempty(quoted)
+        quoted = "''"
     end
-    has_single = false
-    has_special = false
-    for c in word
-        if isspace(c) || c=='\\' || c=='\'' || c=='"' || c=='$'
-            has_special = true
-            if c == '\''
-                has_single = true
-            end
-        end
+    quoted
+end
+"Quote by enclosing in double quotes"
+function _quote_via_double_quote(word::AbstractString)
+    # Quote by enclosing in double quotes and escaping certain special
+    # characters with a backslash, and by escaping all exclamation
+    # marks with a backslash
+    quoted = "\"" * replace(word, r"(?=[$`\"\\])", "\\") * "\""
+    quoted = replace(quoted, r"!", "\"\\!\"\"")
+    # Remove empty leading or trailing quotes
+    quoted = replace(quoted, r"^\"\"", "")
+    quoted = replace(quoted, r"!\"\"$(?!\n)", "!")
+    if isempty(quoted)
+        quoted = "''"
     end
-    if !has_special
-        print(io, word)
-    elseif !has_single
-        print(io, '\'', word, '\'')
-    else
-        print(io, '"')
-        for c in word
-            if c == '"' || c == '$'
-                print(io, '\\')
-            end
-            print(io, c)
-        end
-        print(io, '"')
+    quoted
+end
+"Quote by escaping with backslashes"
+function _quote_via_backslash(word::AbstractString)
+    # Quote by escaping all non-alphanumeric characters with a
+    # backslash, and by enclosing all newlines with single quotes
+    quoted = replace(word, r"(?=[^-0-9a-zA-Z_./\n])", "\\")
+    quoted = replace(quoted, r"\n", "'\\n'")
+    if isempty(quoted)
+        quoted = "''"
     end
+    quoted
+end
+"Score a quoted string (lower is better)"
+function _score_quoted(quoted::AbstractString)
+    # Prefer shorted strings, and penalize backslashes (which are
+    # arguably more confusing than quotes)
+    length(quoted) + count(c -> c=='\\', quoted)
+end
+"Quote a word so that it is safe to use with Posix sh"
+function _quote_shell_word(word::AbstractString)
+    # Return the "best" string
+    quotes = [_quote_via_single_quote(word),
+              _quote_via_double_quote(word),
+              _quote_via_backslash(word)]
+    scores = map(_score_quoted, quotes)
+    best = findmin(scores)[2]
+    return quotes[best]
+end
+
+function print_shell_word(io::IO, word::AbstractString)
+    print(io, _quote_shell_word(word))
 end
 
 function print_shell_escaped(io::IO, cmd::AbstractString, args::AbstractString...)

base/sysimg.jl

@@ -91,8 +91,8 @@ include("iobuffer.jl")
 include("string.jl")
 include("unicode.jl")
 include("parse.jl")
-include("shell.jl")
 include("regex.jl")
+include("shell.jl")
 include("base64.jl")
 importall .Base64
 

test/spawn.jl

@@ -328,4 +328,16 @@ let cmd = AbstractString[]
     # Backticks should automatically quote where necessary
     cmd = ["foo bar", "baz"]
     @test string(`$cmd`) == "`'foo bar' baz`"
+
+    # Test various difficult strings, and test multiple quoting levels
+    strs = ByteString["", "asdf", "'", "\"", "\\", " ", "\$",
+                      "a\"bc\"d", "'012'3", " a ", "\n", "\\n", "\t",
+                      "echo hello", " \${LANG}", "\"println(ARGS)\""]
+    for level in 1:3
+        cmd = Cmd(strs)
+        quoted = Base.shell_escape(cmd)
+        strs2 = Base.shell_split(quoted)
+        @test strs2 == strs
+        strs = ByteString[Base.shell_escape(str) for str in strs]
+    end
 end