Multi-byte character fixes for end-of-line operations

development.md

@@ -44,6 +44,13 @@ Once logging is enabled, you should see something like this from `nc`:
 └ @ VimBindings ~/.julia/dev/VimBindings/src/VimBindings.jl:161
 ```
 
+# Disabling precompile
+
+```
+using VimBindings, Preferences
+set_preferences!(VimBindings, "precompile_workload" => false; force=true)
+```
+
 # Useful References
 ### Vim plugins for other editors
 - https://github.com/JetBrains/ideavim

src/buffer.jl

@@ -1,7 +1,7 @@
 module Buffer
 import REPL.LineEdit as LE
 export VimBuffer, mode, VimMode, normal_mode, insert_mode, testbuf, readall, freeze,
-    BufferRecord, chars, peek_left, peek_right, read_left, read_right
+    BufferRecord, chars, peek_left, peek_right, peek_two_right, read_left, read_right
 
 @enum VimMode begin
     normal_mode
@@ -35,8 +35,11 @@ function testbuf(s::AbstractString)::VimBuffer
     end
     (a, mode, b) = (m[1], m[2], m[3])
     buf = IOBuffer(; read=true, write=true, append=true)
-    write(buf, a * b)
-    seek(buf, length(a))
+    cursor_index = write(buf, a)
+    after_cursor = write(buf, b)
+    # @debug "creating testbuf" cursor_index after_cursor
+
+    seek(buf, cursor_index)
     vim_mode = VimMode(mode)
     return VimBuffer(buf, vim_mode)
 end
@@ -46,10 +49,13 @@ struct VimBuffer <: IO
     mode::VimMode
 end
 
-VimBuffer(str::String)::VimBuffer = testbuf(str)
+VimBuffer() = VimBuffer(IOBuffer(), normal_mode)
 
 mode(vb::VimBuffer) = vb.mode
 # TODO modify VimBuffer operations to operate on Characters rather than bytes
+# Status: Removed uses of
+# - [ ] length()
+# - [ ] skip() in favor of skipchars, or else ensure skip operatoes on characters only
 Base.position(vb::VimBuffer) = position(vb.buf)
 Base.seek(vb::VimBuffer, n) = seek(vb.buf, n)
 Base.mark(vb::VimBuffer) = mark(vb.buf)
@@ -106,7 +112,7 @@ end
 """
 Read 1 valid UTF-8 character right of the current position and leave the buffer in the same position.
 """
-function peek_right(buf::IO)::Union{Char, Nothing}
+function peek_right(buf::IO)::Union{Char,Nothing}
     origin = position(buf)
     while !eof(buf)
         c = read(buf, Char)
@@ -119,12 +125,25 @@ function peek_right(buf::IO)::Union{Char, Nothing}
     return nothing
 end
 
+"""
+Read up to 2 valid UTF-8 character right of the current position and leave the buffer in the same position.
+
+Returns a tuple with each successful character (or nothing for a character not read successfully)
+"""
+function peek_two_right(buf::IO)
+    origin = position(buf)
+    c1 = read_right(buf)
+    c2 = read_right(buf)
+    seek(buf, origin)
+    return (c1, c2)
+end
+
 """
 Read 1 valid UTF-8 character right of the current position.
 
 Place the cursor after the char that is read, if any.
 """
-function read_right(buf::IO)::Union{Char, Nothing}
+function read_right(buf::IO)::Union{Char,Nothing}
     origin = position(buf)
     while !eof(buf)
         c = read(buf, Char)

src/changes.jl

@@ -61,7 +61,7 @@ struct Entry
 end
 
 
-Entry(s::String) = Entry(freeze(testbuf("|")))
+Entry(s::String) = Entry(freeze(VimBuffer()))
 # entry where both `prev` and `next` reference itself
 function Entry(record::BufferRecord)
     e = Entry(1, Ref{Entry}(), Ref{Entry}(), record)
@@ -81,7 +81,7 @@ end
 # blank entry. Both `prev` and `next` reference itself. id of 1.
 # for the root of a list (the first entry)
 function Entry()
-    record = VimBuffer("|") |> freeze
+    record = IOBuffer() |> freeze
     return Entry(record)
 end
 

src/execute.jl

@@ -37,8 +37,13 @@ function execute(buf, command::MotionCommand)::Union{VimMode,ReplAction,Nothing}
                 nothing
             end
         else
-            # execute the motion object
-            motion(buf)
+            k = key(command)
+            if !(k == 'l' && is_line_max(buf))
+                # we shouldn't move past the last character of the line
+
+                # execute the motion object
+                motion(buf)
+            end
         end
     end
     return repl_action
@@ -89,9 +94,6 @@ const insert_functions = Dict{Char,Function}(
             motion
         end
     end,
-    's' => buf -> begin
-        delete(buf, right(buf))
-    end
     # _ => gen_motion(buf, command)
 )
 
@@ -119,7 +121,7 @@ function execute(buf, command::OperatorCommand)::Union{VimMode,Nothing}
     # see vim help :h cw regarding this exception
     if command.operator == 'c' && command.action.name in ['w', 'W']
         # in the middle of a word
-        if at_junction_type(buf, In{>:Word}) || at_junction_type(buf, Start{>:Word})
+        if at_junction_type(In{>:Word}, buf) || at_junction_type(Start{>:Word}, buf)
             new_name = if command.action.name == 'w'
                 'e'
             else
@@ -172,7 +174,8 @@ function execute(buf, command::SynonymCommand)::Union{VimMode,Nothing}
         'X' => "dh",
         'C' => "c\$",
         'D' => "d\$",
-        'S' => "cc"
+        'S' => "cc",
+        's' => "cl",
     )
     new_command = parse_command("$(command.r1)$(synonyms[command.operator])")
 
@@ -182,7 +185,13 @@ end
 function execute(buf, command::ReplaceCommand)::Union{VimMode,Nothing}
     inserted = 0
     for r1 in 1:command.r1
+        move_right = is_line_max(buf)
         delete(buf, right(buf))
+        if move_right
+            let motion = right(buf)
+                motion(buf)
+            end
+        end
         inserted += LE.edit_insert(buf, command.replacement)
     end
     if inserted > 0

src/motion.jl

@@ -11,7 +11,7 @@ export Motion, MotionType, simple_motions, complex_motions, partial_complex_moti
     is_stationary, down, up, word_next, word_big_next, word_end, word_back,
     word_big_back, word_big_end, line_end, line_begin, line_zero,
     find_c, find_c_back, get_safe_name, all_keys, special_keys, exclusive, inclusive, linewise, endd,
-    left, right
+    left, right, snap_into_line
 
 # text objects
 export line
@@ -113,7 +113,7 @@ end
 
 function left(buf::IO)::Motion
     start = position(buf)
-    @loop_guard while position(buf) > 0
+    @loop_guard while position(buf) > 0 && !is_line_start(buf)
         seek(buf, position(buf) - 1)
         c = peek(buf)
         (((c & 0x80) == 0) || ((c & 0xc0) == 0xc0)) && break
@@ -323,6 +323,17 @@ function line_zero(buf::IO)::Motion
     return Motion(start, endd)
 end
 
+"""
+Ensure the cursor is within the current line. In other words,
+move left if that will put us on text that is within the line.
+"""
+snap_into_line(buf::IO)::Motion =
+    if is_line_end(buf) && !is_line_start(buf)
+        left(buf)
+    else
+        Motion(buf)
+    end
+
 function endd(buf::IO)::Motion
 
 end

src/operator.jl

@@ -23,8 +23,8 @@ end
 function change(buf::IO, motion::Motion) #, motion_type :: MotionType)
     text = String(take!(copy(buf)))
     left = min(motion)
-    right = min(max(motion), length(text))
-    @debug "change operator" buf motion text left right
+    right = min(max(motion), sizeof(text))
+    @debug "change operator" buf motion text left right max(motion) length(text)
     yank(buf, motion)
     move(buf, motion) #, motion_type)
     LE.edit_splice!(buf, left => right)
@@ -35,10 +35,10 @@ end
 # and "cw" only removes the inner word
 function delete(buf::IO, motion::Motion) #, motion_type :: MotionType)
     text = String(take!(copy(buf)))
+    @debug "delete range:" min(motion) max(motion) length(text) textwidth(text)
     left = min(motion)
-    right = min(max(motion), length(text))
-    # if motion.motiontype == linewise
-    #         end
+    right = max(motion)
+    @debug "delete range:" left right
     move_cursor = Motion(buf)
     if motion.motiontype == linewise
         # if we're deleting a line, include the '\n' at the beginning
@@ -68,7 +68,12 @@ function delete(buf::IO, motion::Motion) #, motion_type :: MotionType)
 
     if motion.motiontype == linewise
         move_cursor(buf)
+    elseif is_line_end(buf) && !is_line_start(buf)
+        let motion = snap_into_line(buf)
+            motion(buf)
+        end
     end
+
     return nothing
 end
 

src/parse.jl

@@ -53,8 +53,8 @@ end
 const UNDO_REDO = "(?|(u)|(\x12))"
 const TEXTOBJECT = "$REPEAT[ai][wWsp]"
 const PARTIALTEXTOBJECT = "$REPEAT[ai]([wWsp])?"
-const DELETECHARS = "[xXDCS]"
-const INSERTCHARS = "[aAiIoOs]"
+const DELETECHARS = "[xXDCSs]"
+const INSERTCHARS = "[aAiIoO]"
 const OPERATOR = "[ydc]"
 const RULES = TupleDict(
     "^(?<c>$INSERTCHARS)\$" |> Regex => InsertCommand, # insert commands

src/textutils.jl

@@ -1,3 +1,6 @@
+"""
+Getting carried away with types and multiple dispatch: text utility style
+"""
 module TextUtils
 import Base: *
 using REPL
@@ -9,33 +12,41 @@ export is_newline, is_whitespace, is_word_char, TextChar, WordChar, WhitespaceCh
     chars_by_cursor, junction_type, at_junction_type, Text, Object, Word, Line, Whitespace, Junction, Start, End, In,
     is_alphanumeric, is_alphabetic, is_uppercase, is_lowercase, is_punctuation, is_line_start, is_line_end,
     is_word_end, is_word_start, is_object_start, is_object_end, is_in_object, is_whitespace_end, is_whitespace_start, is_in_word, chars,
-    peek_left, peek_right, read_left, read_right
+    peek_left, peek_right, read_left, read_right, is_line_max
 
 """
     Determine whether the buffer is currently at the start of a text object.
     Whitespace is not included as a text object.
 """
-is_word_start(buf) = at_junction_type(buf, Start{>:Word})
-is_whitespace_start(buf) = at_junction_type(buf, Start{>:Whitespace})
+is_word_start(buf) = at_junction_type(Start{>:Word}, buf)
+is_whitespace_start(buf) = at_junction_type(Start{>:Whitespace}, buf)
 """
 Whether the buffer is currently at the start of a non-whitespace
 block
 """
-is_object_start(buf) = at_junction_type(buf, Start{>:Object})
+is_object_start(buf) = at_junction_type(Start{>:Object}, buf)
 
 """
     Whether the buffer is currently at the end of a text object. Whitespace is not included as a text object.
 """
-is_word_end(buf) = at_junction_type(buf, End{>:Word})
-is_object_end(buf) = at_junction_type(buf, End{>:Object})
-is_whitespace_end(buf) = at_junction_type(buf, End{>:Whitespace})
+is_word_end(buf) = at_junction_type(End{>:Word}, buf)
+is_object_end(buf) = at_junction_type(End{>:Object}, buf)
+is_whitespace_end(buf) = at_junction_type(End{>:Whitespace}, buf)
 
 """
     Whether the buffer is at the start of a line, in the literal sense
     In other words, is the char before the cursor a newline
 """
-is_line_start(buf) = at_junction_type(buf, Start{>:Line})
-is_line_end(buf) = at_junction_type(buf, End{>:Line})
+is_line_start(buf) = at_junction_type(Start{>:Line}, buf)
+is_line_end(buf) = at_junction_type(End{>:Line}, buf)
+
+"""
+Whether the buffer's location is the right-most that the cursor is allowed to go; the "end of the line" so that the cursor does not get placed at a newline where there is not actually a character to operate on.
+"""
+function is_line_max(buf)
+    c1, c2 = peek_two_right(buf)
+    return at_junction_type(End{>:Line}, c1, c2)
+end
 
 
 """
@@ -44,10 +55,10 @@ Whether the buffer is currently in an object of a continuous type (not between t
 function is_in_word(buf)
     a, b = chars_by_cursor(buf)
     typeof(a) == typeof(b) || return false
-    at_junction_type(buf, In{>:Word})
+    at_junction_type(In{>:Word}, buf)
 end
 
-is_in_object(buf) = at_junction_type(buf, In{<:Object})
+is_in_object(buf) = at_junction_type(In{<:Object}, buf)
 
 """
 Get the 
@@ -172,6 +183,8 @@ junction_type(char1::NewlineChar, char2::SpaceChar) = Set([Start{Line}(), Start{
 junction_type(char1::Nothing, char2::NewlineChar) = Set([Start{Line}(), End{Line}()])
 junction_type(char1::NewlineChar, char2::Nothing) = Set([Start{Line}(), End{Line}()])
 
+junction_type(char1::NewlineChar, char2::NewlineChar) = Set([Start{Line}(), End{Line}()])
+
 junction_type(char1::WordChar, char2::PunctuationChar) = Set([Start{Word}(), End{Word}(), In{Object}()])
 junction_type(char1::PunctuationChar, char2::WordChar) = Set([Start{Word}(), End{Word}(), In{Object}()])
 
@@ -181,8 +194,15 @@ junction_type(char1::T, char2::T) where {T<:SpaceChar} = Set([In{Whitespace}()])
 """
 Whether the given buffer is currently at a junction of type junc
 """
-function at_junction_type(buf, junc_type)
-    for junc in junction_type(buf)
+function at_junction_type(junc_type, obj::Vararg)
+    juncs = junction_type(obj...)
+    # return at_junction_type(junction_type(obj, junc_type...))
+    return at_junction_type(junc_type, juncs)
+end
+
+function at_junction_type(junc_type, juncs::Set)
+    # TODO nuke this whole thing
+    for junc in juncs
         if junc isa junc_type
             @debug "at_junction_type?" junc_type chars_by_cursor(buf) at_junction_type = true
             return true

test/changes.jl

@@ -5,13 +5,13 @@ import VimBindings.PkgTools: run as run_command
 
 @testset "BufferRecord and freeze" begin
     reset!()
-    a = freeze(VimBuffer("Hello world|!"))
-    b = freeze(VimBuffer("Hello worl|d"))
+    a = freeze(testbuf("Hello world|!"))
+    b = freeze(testbuf("Hello worl|d"))
 
     @test a != b
-    @test freeze(VimBuffer("Hello world|!")) != BufferRecord("Hello |!", 1)
-    @test freeze(VimBuffer("Hello world|!")) == BufferRecord("Hello world!", 11)
-    @test freeze(VimBuffer("Hello|i| world!")) == BufferRecord("Hello world!", 5)
+    @test freeze(testbuf("Hello world|!")) != BufferRecord("Hello |!", 1)
+    @test freeze(testbuf("Hello world|!")) == BufferRecord("Hello world!", 11)
+    @test freeze(testbuf("Hello|i| world!")) == BufferRecord("Hello world!", 5)
 
     # The record includes the cursor location but does not include it in equality
     @test BufferRecord("Hello world!", 7) == BufferRecord("Hello world!", 5)
@@ -153,22 +153,22 @@ end
 
 @testset "undo/redo cursor" begin
     reset!()
-    buf = testbuf("Hello worl|d")
+    buf = testbuf("Hello w|orld")
     record(buf)
-    run_command("daw", buf)
+    run_command("dw", buf)
     # test that `dw` records an entry
-    @test buf == testbuf("Hello |")
+    @test buf == testbuf("Hello |w")
     record(buf)
-    @test Changes.latest[].record == BufferRecord("Hello ", 6)
+    @test Changes.latest[].record == BufferRecord("Hello w", 6)
 
     run_command("u", buf)
     @test Changes.latest[].record == BufferRecord("Hello world", 6)
     # running undo records the record as `next`
-    @test Changes.latest[].next[].record == BufferRecord("Hello ", 6)
+    @test Changes.latest[].next[].record == BufferRecord("Hello w", 6)
 
     # \x12 == C-r for redo
     run_command("\x12", buf)
-    @test buf == testbuf("Hello |")
-    @test Changes.latest[].record == BufferRecord("Hello ", 6)
-    @test Changes.latest[].next[].record == BufferRecord("Hello ", 6)
+    @test buf == testbuf("Hello |w")
+    @test Changes.latest[].record == BufferRecord("Hello w", 6)
+    @test Changes.latest[].next[].record == BufferRecord("Hello w", 6)
 end