[completion] Follow LSP and treat positions as UTF16 code units

Makefile

@@ -91,7 +91,7 @@ opam-switch:
 # Install opam deps
 .PHONY: opam-deps
 opam-deps:
-	opam install ./coq-lsp.opam -y --deps-only
+	opam install ./coq-lsp.opam -y --deps-only --with-test
 
 # Install opam deps
 .PHONY: opam-dev-deps

controller/rq_completion.ml

@@ -79,8 +79,8 @@ let validate_line ~(doc : Fleche.Doc.t) ~line =
 let validate_position ~doc ~point =
   let line, char = point in
   Option.bind (validate_line ~doc ~line) (fun line ->
-      Option.bind (Coq.Utf8.index_of_char ~line ~char) (fun index ->
-          Some (String.get line index)))
+      let char = Coq.Utf8.get_byte_offset_from_utf16_pos line char in
+      Option.bind char (fun index -> Some (String.get line index)))
 
 let get_char_at_point ~(doc : Fleche.Doc.t) ~point =
   let line, char = point in

coq-lsp.opam

@@ -30,6 +30,9 @@ depends: [
   # waterproof parser
   "menhir"       { >= "20220210" }
 
+  # for unit testing
+  "alcotest"     { >= "1.7.0" & with-test }
+
   # Uncomment this for releases
   # "coq"          { >= "8.17" < "8.18"  }
   # "coq-serapi"   { >= "8.17" < "8.18"  }

coq/utf8.ml

@@ -69,3 +69,17 @@ let char_of_index ~line ~byte =
   (* | Some char -> Io.Log.trace "get_last_text" (Format.asprintf "char: %d"
      char)); *)
   char
+
+let get_byte_offset_from_utf16_pos line (char : int) =
+  let byte_idx = ref 0 in
+  let utf16_char_count = ref 0 in
+  while !byte_idx < String.length line && !utf16_char_count < char do
+    let ch = String.get_utf_8_uchar line !byte_idx in
+    byte_idx := next line !byte_idx;
+    let code_unit_count =
+      Uchar.utf_16_byte_length (Uchar.utf_decode_uchar ch) / 2
+    in
+    utf16_char_count := !utf16_char_count + code_unit_count;
+    ()
+  done;
+  if !byte_idx < String.length line then Some !byte_idx else None

coq/utf8.mli

@@ -46,3 +46,7 @@ val index_of_char : line:string -> char:char -> index option
 
 (** Lenght in utf-8 chars *)
 val length : string -> char
+
+(** Get the byte potition of a code point indexed in UTF-16 code units in a
+    UTF-8 encoded string. *)
+val get_byte_offset_from_utf16_pos : string -> int -> int option

flake.nix

@@ -89,8 +89,8 @@
 
           packages = l.attrValues {
             inherit (config.treefmt.build) wrapper;
-            inherit (pkgs) dune_3 nodejs;
-            inherit (ocamlPackages) ocaml ocaml-lsp;
+            inherit (pkgs) dune_3 nodejs dune-release;
+            inherit (ocamlPackages) ocaml ocaml-lsp alcotest;
           };
         };
       };

unit_test/dune

@@ -0,0 +1,4 @@
+(test
+ (name unittests)
+ (link_flags -linkall)
+ (libraries alcotest controller))

unit_test/unittests.ml

@@ -0,0 +1,30 @@
+(* Build with `ocamlbuild -pkg alcotest simple.byte` *)
+
+open Coq
+
+let testcases_x =
+  [ ("aax", 2, true)
+  ; ("  xoo", 2, true)
+  ; ("0123", 4, false)
+  ; ("  𝒞x", 4, true)
+  ; ("  𝒞x𝒞", 4, true)
+  ; ("  𝒞∫x", 5, true)
+  ; ("  𝒞", 4, false)
+  ; ("∫x.dy", 1, true)
+  ]
+
+(* The tests *)
+let test_x () =
+  List.iter
+    (fun (s, i, b) ->
+      let res = Utf8.get_byte_offset_from_utf16_pos s i in
+      if b then
+        let res = Option.map (fun i -> s.[i]) res in
+        Alcotest.(check (option char)) "x present" (Some 'x') res
+      else Alcotest.(check (option int)) "x present" None res)
+    testcases_x
+
+(* Run it *)
+let () =
+  let open Alcotest in
+  run "Controller" [ ("utf16", [ test_case "Find the x" `Quick test_x ]) ]