Merge pull request #6385 from JuliaLang/cb/jlparsecompletion

Use julia parser in completion

base/REPLCompletions.jl

@@ -138,7 +138,8 @@ function complete_methods(input::String)
     UTF8String[string(m) for m in methods(fn)]
 end
 
-const non_word_chars = " \t\n\"\\'`@\$><=:;|&{}()[].,+-*/?%^~"
+const non_word_chars = [" \t\n\"\\'`@\$><=:;|&{}()[],+-*/?%^~"...]
+const non_filename_chars = [" \t\n\"\\'`@\$><=;|&{("...]
 
 # Aux function to detect whether we're right after a
 # using or import keyword
@@ -154,59 +155,25 @@ function afterusing(string::ByteString, startpos::Int)
 end
 
 function completions(string, pos)
-    startpos = min(pos, 1)
-    dotpos = 0
-    instring = false
-    incmd = false
-    infunc = false
-    escaped = false
-    nearquote = false
-    i = start(string)
-    while i <= pos
-        c,j = next(string, i)
-        if c == '\\'
-            instring && (escaped $= true)
-            nearquote = false
-        elseif c == '\''
-            !instring && (nearquote = true)
-        elseif c == '"'
-            (!escaped && !nearquote && !incmd) && (instring $= true)
-            escaped = nearquote = false
-        elseif c == '`'
-            (!escaped && !nearquote && !instring) && (incmd $= true)
-            escaped = nearquote = false
-        else
-            escaped = nearquote = false
-        end
-        if c < 0x80
-            if instring || incmd
-                c in " \t\n\"\\'`@\$><=;|&{(" && (startpos = j)
-            elseif c in non_word_chars
-                if c == '.'
-                    dotpos = i
-                elseif i == pos && c == '('
-                    infunc = true
-                else
-                    startpos = j
-                end
-            end
-        end
-        i = j
-    end
-
-    if instring || incmd
+    inc_tag = Base.incomplete_tag(parse(string[1:pos], raise=false))
+    if inc_tag in [:cmd, :string]
+        startpos = nextind(string, rsearch(string, non_filename_chars, pos))
         r = startpos:pos
         paths = complete_path(string[r])
-        if instring && length(paths) == 1
+        if inc_tag == :string && length(paths) == 1
             paths[1] *= "\""
         end
         return sort(paths), r, true
+    elseif inc_tag == :other && string[pos] == '('
+        endpos = prevind(string, pos)
+        startpos = nextind(string, rsearch(string, non_word_chars, endpos))
+        return complete_methods(string[startpos:endpos]), startpos:endpos, false
+    elseif inc_tag == :comment
+        return UTF8String[], 0:-1, false
     end
 
-    if infunc
-        # We're right after the start of a function call
-        return (complete_methods(string[startpos:pos-1]), startpos:pos, false)
-    end
+    dotpos = rsearch(string, '.', pos)
+    startpos = nextind(string, rsearch(string, non_word_chars, pos))
 
     ffunc = (mod,x)->true
     suggestions = UTF8String[]

base/client.jl

@@ -153,6 +153,21 @@ function parse_input_line(io::IO)
     end
 end
 
+# detect the reason which caused an :incomplete expression
+# from the error message
+# NOTE: the error messages are defined in src/julia-parser.scm
+incomplete_tag(ex) = :none
+function incomplete_tag(ex::Expr)
+    Meta.isexpr(ex, :incomplete) || return :none
+    msg = ex.args[1]
+    contains(msg, "string") && return :string
+    contains(msg, "comment") && return :comment
+    contains(msg, "requires end") && return :block
+    contains(msg, "\"`\"") && return :cmd
+    contains(msg, "character") && return :char
+    return :other
+end
+
 # try to include() a file, ignoring if not found
 try_include(path::String) = isfile(path) && include(path)
 

src/julia-parser.scm

@@ -360,7 +360,7 @@
   (define (skip-multiline-comment port count)
     (let ((c (read-char port)))
       (if (eof-object? c) 
-          (error "incomplete: unterminated multi-line comment #= ... =#")
+          (error "incomplete: unterminated multi-line comment #= ... =#") ; NOTE: changing this may affect code in base/client.jl
           (begin (if (eqv? c #\=)
                      (let ((c (peek-char port)))
                        (if (eqv? c #\#)
@@ -440,7 +440,7 @@
 (define (require-token s)
   (let ((t (or (ts:pbtok s) (ts:last-tok s) (next-token (ts:port s) s))))
     (if (eof-object? t)
-	(error "incomplete: premature end of input")
+	(error "incomplete: premature end of input") ; NOTE: changing this may affect code in base/client.jl
 	(if (newline? t)
 	    (begin (take-token s)
 		   (require-token s))
@@ -950,7 +950,7 @@
   (let ((t (peek-token s)))
     (cond ((eq? t 'end) (take-token s))
 	  ((eof-object? t)
-	   (error (string "incomplete: \"" word "\" at "
+	   (error (string "incomplete: \"" word "\" at " ; NOTE: changing this may affect code in base/client.jl
 			  current-filename ":" expect-end-current-line
 			  " requires end")))
 	  (else
@@ -1483,7 +1483,7 @@
 
 (define (not-eof-2 c)
   (if (eof-object? c)
-      (error "incomplete: invalid \"`\" syntax")
+      (error "incomplete: invalid \"`\" syntax") ; NOTE: changing this may affect code in base/client.jl
       c))
 
 (define (parse-backquote s)
@@ -1505,7 +1505,7 @@
 
 (define (not-eof-3 c)
   (if (eof-object? c)
-      (error "incomplete: invalid string syntax")
+      (error "incomplete: invalid string syntax") ; NOTE: changing this may affect code in base/client.jl
       c))
 
 (define (take-char p)
@@ -1592,7 +1592,7 @@
 
 (define (not-eof-1 c)
   (if (eof-object? c)
-      (error "incomplete: invalid character literal")
+      (error "incomplete: invalid character literal") ; NOTE: changing this may affect code in base/client.jl
       c))
 
 (define (unescape-string s)

test/core.jl

@@ -1412,6 +1412,21 @@ let
           ex == Expr(:error, "invalid assignment location \"#<julia_value>\"")
 end
 
+# make sure that incomplete tags are detected correctly
+# (i.e. error messages in src/julia-parser.scm must be matched correctly
+# by the code in base/client.jl)
+for (str, tag) in ["" => :none, "\"" => :string, "#=" => :comment, "'" => :char,
+                   "`" => :cmd, "begin;" => :block, "quote;" => :block,
+                   "let;" => :block, "for i=1;" => :block, "function f();" => :block,
+                   "f() do x;" => :block, "module X;" => :block, "type X;" => :block,
+                   "immutable X;" => :block, "(" => :other, "[" => :other,
+                   "{" => :other, "begin" => :other, "quote" => :other,
+                   "let" => :other, "for" => :other, "function" => :other,
+                   "f() do" => :other, "module" => :other, "type" => :other,
+                   "immutable" => :other]
+    @test Base.incomplete_tag(parse(str, raise=false)) == tag
+end
+
 # issue #6031
 macro m6031(x); x; end
 @test @m6031([2,4,6])[3] == 6

test/replcompletions.jl

@@ -14,7 +14,7 @@ test_scomplete(s) = shell_completions(s,endof(s))
 s = ""
 c,r = test_complete(s)
 @test in("CompletionFoo",c)
-@test r == 0:-1
+@test isempty(r)
 @test s[r] == ""
 
 s = "Comp"