5.3 support: Backport utf8 identifiers (#2622)

Vendor required code from Misc.Utf8_lexeme, String, Bytes and Uchar.

Backport utf8 identifiers to standard and extended parser. This reduces the
diff with upstream.

* test: Add utf8_identifiers.ml

Taken from the compiler source.

* test_branch: Ignore parsing errors on 'infer'

The project contains a syntax error, that is new in OCaml 5.3:

    (** [x < min({x'|x'∊l})] *)

CHANGES.md

@@ -14,8 +14,9 @@ profile. This started with version 0.26.0.
   This might change the formatting of some functions due to the formatting code
   being completely rewritten.
 
-- Support OCaml 5.3 syntax (#2609, #2610, #2611, @Julow)
-  This adds support for short functor type arguments syntax.
+- Support OCaml 5.3 syntax (#2609, #2610, #2611, #2622, @Julow)
+  This adds support for short functor type arguments syntax and utf8
+  identifiers.
 
 - Documentation comments are now formatted by default (#2390, @Julow)
   Use the option `parse-docstrings = false` to restore the previous behavior.

test-extra/Makefile

@@ -13,11 +13,11 @@
 #       make DIRS=<directory> test
 # By default, test projects used as regression tests
 DIRS= \
-	code/ocamlformat code/infer code/js_of_ocaml code/dune code/irmin \
+	code/ocamlformat code/js_of_ocaml code/dune code/irmin \
 	code/dune-release code/mirage code/ppxlib code/base
 
 # Extra test directories, for which looser checking is done
-XDIRS=code/ocaml
+XDIRS=code/ocaml code/infer
 
 # Directories to ignore (given to find, compared literally)
 PRUNE_DIRS= \

test/failing/tests/unit_lex.ml.broken-ref

@@ -1,10 +1,5 @@
+ocamlformat: ignoring "tests/unit_lex.ml" (syntax error)
 File "tests/unit_lex.ml", line 18, characters 4-10:
 18 |     ������ (* this file must be iso-8859-1 *)
          ^^^^^^
-Alert deprecated: ISO-Latin1 characters in identifiers
-ocamlformat: ignoring "tests/unit_lex.ml" (syntax error)
-
-File "tests/unit_lex.ml", line 55, characters 2-8:
-55 |   '\999'; (* wrong, but yet... *)
-       ^^^^^^
-Error: Illegal backslash escape in string or character ('\999'): 999 is outside the range of legal characters (0-255).
+Error: Invalid encoding of identifier ������.

test/failing/tests/unit_values.ml.broken-ref

@@ -1,10 +1,5 @@
-File "tests/unit_values.ml", line 6, characters 10-11:
-6 | let i32 = −1073741824, 1073741823
-              ^
-Alert deprecated: ISO-Latin1 characters in identifiers
 ocamlformat: ignoring "tests/unit_values.ml" (syntax error)
-
-File "tests/unit_values.ml", line 6, characters 11-12:
+File "tests/unit_values.ml", line 6, characters 10-23:
 6 | let i32 = −1073741824, 1073741823
-               ^
-Error: Illegal character (\136)
+              ^^^^^^^^^^^^^
+Error: Invalid character U+2212 in identifier

test/passing/gen/dune.inc

@@ -4525,6 +4525,21 @@
  (alias runtest)
  (action (diff use_file.mlt.err use_file.mlt.stderr)))
 
+(rule
+ (deps .ocamlformat dune-project)
+ (action
+  (with-stdout-to utf8_identifiers.ml.stdout
+   (with-stderr-to utf8_identifiers.ml.stderr
+     (run %{bin:ocamlformat} --name utf8_identifiers.ml --margin-check %{dep:../tests/utf8_identifiers.ml})))))
+
+(rule
+ (alias runtest)
+ (action (diff utf8_identifiers.ml.ref utf8_identifiers.ml.stdout)))
+
+(rule
+ (alias runtest)
+ (action (diff utf8_identifiers.ml.err utf8_identifiers.ml.stderr)))
+
 (rule
  (deps .ocamlformat dune-project)
  (action

test/passing/refs.default/utf8_identifiers.ml.ref

@@ -0,0 +1,18 @@
+(* TEST
+readonly_files = "genfiles.ml";
+setup-ocamlc.byte-build-env;
+all_modules = "genfiles.ml";
+program = "./genfiles.byte.exe";
+ocamlc.byte;
+run;
+all_modules = "été.ml ça.ml test.ml";
+program = "./main.byte.exe";
+ocamlc.byte;
+run;
+*)
+
+let _ =
+  (* Source is NFC *)
+  assert (Été.x + Ça.x = 3);
+  (* Source is NFD *)
+  assert (Été.x + Ça.x = 3)

test/passing/refs.janestreet/utf8_identifiers.ml.ref

@@ -0,0 +1,19 @@
+(* TEST
+readonly_files = "genfiles.ml";
+setup-ocamlc.byte-build-env;
+all_modules = "genfiles.ml";
+program = "./genfiles.byte.exe";
+ocamlc.byte;
+run;
+all_modules = "été.ml ça.ml test.ml";
+program = "./main.byte.exe";
+ocamlc.byte;
+run;
+*)
+
+let _ =
+  (* Source is NFC *)
+  assert (Été.x + Ça.x = 3);
+  (* Source is NFD *)
+  assert (Été.x + Ça.x = 3)
+;;

test/passing/refs.ocamlformat/utf8_identifiers.ml.ref

@@ -0,0 +1,18 @@
+(* TEST
+readonly_files = "genfiles.ml";
+setup-ocamlc.byte-build-env;
+all_modules = "genfiles.ml";
+program = "./genfiles.byte.exe";
+ocamlc.byte;
+run;
+all_modules = "été.ml ça.ml test.ml";
+program = "./main.byte.exe";
+ocamlc.byte;
+run;
+*)
+
+let _ =
+  (* Source is NFC *)
+  assert (Été.x + Ça.x = 3) ;
+  (* Source is NFD *)
+  assert (Été.x + Ça.x = 3)

test/passing/tests/utf8_identifiers.ml

@@ -0,0 +1,18 @@
+(* TEST
+readonly_files = "genfiles.ml";
+setup-ocamlc.byte-build-env;
+all_modules = "genfiles.ml";
+program = "./genfiles.byte.exe";
+ocamlc.byte;
+run;
+all_modules = "été.ml ça.ml test.ml";
+program = "./main.byte.exe";
+ocamlc.byte;
+run;
+*)
+
+let _ =
+  (* Source is NFC *)
+  assert (Été.x + Ça.x = 3);
+  (* Source is NFD *)
+  assert (Été.x + Ça.x = 3)