Improve context management when parsing.

Implements #5453.

Syntax changes:
- Eliminate old non-decimal number syntax.
- Unspaced application like `f(x)` is now an error.
- Applying `:` to a non-QN in statement context is allowed and produces
  a `TypeAnnotated` (fixes #6152).

API changes:
- All fixed-content tokens are now distinct types.

Error improvements (fixes #5444), especially for:
- Out-of-context expressions/statements
- Statement syntaxes
- Named-app syntax
- Unspaced-application errors
- Number syntax
- Private annotations (fixes #10137)
- Parens (fixes #6741)
- Type defs (fixes #8633)
- Fix some panics caused by invalid expressions, found by parsing non-Enso code.
- Reject some operations in pattern context, e.g. `1 + 1 = 2`.
- Eliminate `export` with `all` or `hiding` (#10258).

Improve Rust parsing performance by 33%; now 20 MB/s on my bench machine:
- Stream lexer to parser.
- New, faster parser for type defs, statements, numbers.
- More efficient tree layout (fixes #5452).
Improve backend parsing performance additionally:
- Backend now uses optimized parser build (in debug builds,
  debug-assertions are still enabled).

Build improvements:
- Fix some redundancy between `sbt` and build script: now only `sbt`
  compiles JNI and generates parser's Java bindings for backend use.
  Build script generates Java to a different directory when parser
  serialization self-test is requested.
- Improve `sbt` detection of changed parser; this should reduce the need
  for clean builds in CI.

Testing:
- Add binary target for fuzzing with AFL.

Cargo.toml

@@ -23,6 +23,7 @@ members = [
     "lib/rust/parser/generate-java",
     "lib/rust/parser/schema",
     "lib/rust/parser/debug",
+    "lib/rust/parser/debug/fuzz",
     "tools/language-server/logstat",
     "tools/language-server/wstest",
 ]
@@ -47,6 +48,11 @@ incremental = true
 debug = false
 debug-assertions = false
 
+[profile.fuzz]
+inherits = "release"
+debug-assertions = true
+overflow-checks = true
+
 [profile.bench]
 opt-level = 3
 lto = true

app/gui2/rust-ffi/src/lib.rs

@@ -40,14 +40,14 @@ pub fn is_ident_or_operator(code: &str) -> u32 {
 #[wasm_bindgen]
 pub fn is_numeric_literal(code: &str) -> bool {
     let parsed = PARSER.with(|parser| parser.run(code));
-    let enso_parser::syntax::tree::Variant::BodyBlock(body) = *parsed.variant else { return false };
+    let enso_parser::syntax::tree::Variant::BodyBlock(body) = parsed.variant else { return false };
     let [stmt] = &body.statements[..] else { return false };
-    stmt.expression.as_ref().map_or(false, |expr| match &*expr.variant {
+    stmt.expression.as_ref().map_or(false, |expr| match &expr.variant {
         enso_parser::syntax::tree::Variant::Number(_) => true,
         enso_parser::syntax::tree::Variant::UnaryOprApp(app) =>
             app.opr.code == "-"
                 && app.rhs.as_ref().map_or(false, |rhs| {
-                    matches!(*rhs.variant, enso_parser::syntax::tree::Variant::Number(_))
+                    matches!(rhs.variant, enso_parser::syntax::tree::Variant::Number(_))
                 }),
         _ => false,
     })

build.sbt

@@ -639,10 +639,7 @@ lazy val rustParserTargetDirectory =
   SettingKey[File]("target directory for the Rust parser")
 
 (`syntax-rust-definition` / rustParserTargetDirectory) := {
-  // setting "debug" for release, because it isn't yet safely integrated into
-  // the parser definition
-  val versionName = if (BuildInfo.isReleaseMode) "debug" else "debug"
-  target.value / "rust" / versionName
+  target.value / "rust" / "parser-jni"
 }
 
 val generateRustParserLib =
@@ -668,31 +665,32 @@ val generateRustParserLib =
     target.foreach { t =>
       Cargo.rustUp(t, log)
     }
-    val baseArguments = Seq(
+    val profile = if (BuildInfo.isReleaseMode) "release" else "fuzz"
+    val arguments = Seq(
       "build",
       "-p",
       "enso-parser-jni",
+      "--profile",
+      profile,
       "-Z",
       "unstable-options"
     ) ++ target.map(t => Seq("--target", t)).getOrElse(Seq()) ++
       Seq(
         "--out-dir",
         (`syntax-rust-definition` / rustParserTargetDirectory).value.toString
       )
-    val adjustedArguments = baseArguments ++
-      (if (BuildInfo.isReleaseMode)
-         Seq("--release")
-       else Seq())
     val envVars = target
       .map(_ => Seq(("RUSTFLAGS", "-C target-feature=-crt-static")))
       .getOrElse(Seq())
-    Cargo.run(adjustedArguments, log, envVars)
+    Cargo.run(arguments, log, envVars)
   }
   FileTreeView.default.list(Seq(libGlob)).map(_._1.toFile)
 }
 
 `syntax-rust-definition` / generateRustParserLib / fileInputs +=
-  (`syntax-rust-definition` / baseDirectory).value.toGlob / "jni" / "src" / "*.rs"
+  (`syntax-rust-definition` / baseDirectory).value.toGlob / "jni" / "src" / ** / "*.rs"
+`syntax-rust-definition` / generateRustParserLib / fileInputs +=
+  (`syntax-rust-definition` / baseDirectory).value.toGlob / "src" / ** / "*.rs"
 
 val generateParserJavaSources = TaskKey[Seq[File]](
   "generateParserJavaSources",

build/build/src/engine.rs

@@ -247,7 +247,7 @@ impl Default for BuildConfigurationFlags {
             build_project_manager_package: false,
             build_launcher_bundle: false,
             build_project_manager_bundle: false,
-            generate_java_from_rust: true,
+            generate_java_from_rust: false,
             test_java_generated_from_rust: false,
             verify_packages: false,
         }

build/build/src/enso.rs

@@ -1,6 +1,6 @@
-use crate::engine::StandardLibraryTestsSelection;
 use crate::prelude::*;
 
+use crate::engine::StandardLibraryTestsSelection;
 use crate::paths::Paths;
 use crate::paths::ENSO_ENABLE_ASSERTIONS;
 use crate::paths::ENSO_META_TEST_ARGS;

build/build/src/project/backend.rs

@@ -155,11 +155,8 @@ impl IsTarget for Backend {
                 target_os == TARGET_OS,
                 "Enso Project Manager cannot be built on '{target_os}' for target '{TARGET_OS}'.",
             );
-            let config = BuildConfigurationFlags {
-                build_project_manager_bundle: true,
-                generate_java_from_rust: true,
-                ..default()
-            };
+            let config =
+                BuildConfigurationFlags { build_project_manager_bundle: true, ..default() };
             let context = inner.prepare_context(context, config)?;
             let artifacts = context.build().await?;
             let project_manager =

build/build/src/project/runtime.rs

@@ -43,11 +43,7 @@ impl IsTarget for Runtime {
         context: Context,
         job: WithDestination<Self::BuildInput>,
     ) -> BoxFuture<'static, Result<Self::Artifact>> {
-        let config = BuildConfigurationFlags {
-            build_engine_package: true,
-            generate_java_from_rust: true,
-            ..default()
-        };
+        let config = BuildConfigurationFlags { build_engine_package: true, ..default() };
         let this = *self;
         let WithDestination { inner, destination } = job;
         let triple = TargetTriple::new(inner.versions);

build/build/src/rust/enso_linter.rs

@@ -1,9 +1,11 @@
 use super::*;
 
+use crate::paths::generated::RepoRoot;
+
 use ide_ci::programs::cargo;
 use ide_ci::programs::Cargo;
 
-use crate::paths::generated::RepoRoot;
+
 
 const LINTER_CRATE_NAME: &str = "enso-parser-debug";
 const LINTER_BIN_NAME: &str = "check_syntax";

build/build/src/rust/parser.rs

@@ -12,7 +12,6 @@ use ide_ci::programs::Javac;
 
 
 const GENERATOR_CRATE_NAME: &str = "enso-parser-generate-java";
-const PARSER_JNI_CRATE_NAME: &str = "enso-parser-jni";
 const GENERATOR_BIN_NAME: &str = GENERATOR_CRATE_NAME;
 const TEST_GENERATOR_BIN_NAME: &str = "java-tests";
 const GENERATED_CODE_NAMESPACE: [&str; 3] = ["org", "enso", "syntax2"];
@@ -47,17 +46,8 @@ pub async fn generate_java(repo_root: &RepoRoot) -> Result {
     generate_java_to(repo_root, &output_path).await
 }
 
-fn cargo_build_parser_jni(repo_root: &Path) -> Result<Command> {
-    let mut ret = Cargo.cmd()?;
-    ret.current_dir(repo_root)
-        .apply(&cargo::Command::Build)
-        .apply(&cargo::Options::Package(PARSER_JNI_CRATE_NAME.into()));
-    Ok(ret)
-}
-
 #[context("Running self-tests for the generated Java sources failed.")]
 pub async fn run_self_tests(repo_root: &RepoRoot) -> Result {
-    cargo_build_parser_jni(repo_root)?.run_ok().await?;
     let base = &repo_root.target.generated_java;
     let lib = &repo_root.lib.rust.parser.generate_java.java;
     let package = repo_root.target.generated_java.join_iter(GENERATED_CODE_NAMESPACE);

distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso

@@ -1119,9 +1119,9 @@ type Integer
        of bits in the operands.
 
        > Example
-         Computing the bitwise conjunction of 2_01101101 and 2_11110000.
+         Computing the bitwise conjunction of 0b01101101 and 0b11110000.
 
-             2_01101101.bit_and 2_11110000
+             0b01101101.bit_and 0b11110000
     bit_and self that:Integer -> Integer = integer_bit_and self that
 
     ## GROUP Bitwise
@@ -1131,9 +1131,9 @@ type Integer
        The bitwise compliment negates the value of each bit in the operand.
 
        > Example
-         Bitwise negation of 2_0110.
+         Bitwise negation of 0b0110.
 
-             2_0110.bit_not
+             0b0110.bit_not
     bit_not self -> Integer = integer_bit_not self
 
     ## GROUP Bitwise
@@ -1148,9 +1148,9 @@ type Integer
        bits in the operands.
 
        > Example
-         Computing the bitwise disjunction of 2_01101101 and 2_11110000.
+         Computing the bitwise disjunction of 0b01101101 and 0b11110000.
 
-             2_01101101.bit_or 2_11110000
+             0b01101101.bit_or 0b11110000
     bit_or self that:Integer -> Integer = integer_bit_or self that
 
     ## GROUP Bitwise
@@ -1164,9 +1164,9 @@ type Integer
        corresponding bits in the operands.
 
        > Example
-         Computing the bitwise exclusive or of 2_01101101 and 2_11110000.
+         Computing the bitwise exclusive or of 0b01101101 and 0b11110000.
 
-             2_01101101.bit_xor 2_11110000
+             0b01101101.bit_xor 0b11110000
     bit_xor self that:Integer -> Integer = integer_bit_xor self that
 
     ## GROUP Bitwise

engine/runtime-integration-tests/src/test/java/org/enso/compiler/ErrorCompilerTest.java

@@ -44,17 +44,6 @@ public void spaceDotUnderscore() throws Exception {
         ir, Syntax.UnexpectedExpression$.MODULE$, "Unexpected expression", 14, 16);
   }
 
-  @Test
-  public void unaryMinus() throws Exception {
-    var ir = parse("""
-    from Standard.Base import all
-
-    main = Date.new day=-
-    """);
-
-    assertSingleSyntaxError(ir, Syntax.UnrecognizedToken$.MODULE$, "Unrecognized token", 51, 52);
-  }
-
   @Test
   public void dotUnderscore2() throws Exception {
     var ir = parse("""
@@ -403,22 +392,20 @@ public void malformedExport8() throws Exception {
   @Test
   public void malformedExport9() throws Exception {
     var ir = parse("from export all");
-    assertSingleSyntaxError(
-        ir, invalidExport("`all` not allowed in `export` statement"), null, 0, 15);
+    assertSingleSyntaxError(ir, Syntax.UnexpectedExpression$.MODULE$, "Unexpected expression", 0, 15);
   }
 
   @Test
   public void malformedExport10() throws Exception {
     var ir = parse("from Foo export all hiding");
-    assertSingleSyntaxError(
-        ir, invalidExport("`hiding` not allowed in `export` statement"), null, 0, 26);
+    assertSingleSyntaxError(ir, Syntax.UnexpectedExpression$.MODULE$, "Unexpected expression", 0, 26);
   }
 
   @Test
   public void malformedExport11() throws Exception {
     var ir = parse("from Foo export all hiding X.Y");
     assertSingleSyntaxError(
-        ir, invalidExport("`hiding` not allowed in `export` statement"), null, 0, 30);
+        ir, Syntax.UnexpectedExpression$.MODULE$, "Unexpected expression", 0, 30);
   }
 
   @Test
@@ -596,17 +583,17 @@ public void exportAllIsNotAllowed() {
     var ir = parse("""
         from project.Module export all
         """);
-    var expectedReason = new Syntax.InvalidExport("`all` not allowed in `export` statement");
-    assertSingleSyntaxError(ir, expectedReason, null, 0, 30);
+    assertSingleSyntaxError(
+            ir, Syntax.UnexpectedExpression$.MODULE$, "Unexpected expression", 0, 30);
   }
 
   @Test
   public void exportHidingIsNotAllowed() {
     var ir = parse("""
         from project.Module export all hiding Foo
         """);
-    var expectedReason = new Syntax.InvalidExport("`hiding` not allowed in `export` statement");
-    assertSingleSyntaxError(ir, expectedReason, null, 0, 41);
+    assertSingleSyntaxError(
+            ir, Syntax.UnexpectedExpression$.MODULE$, "Unexpected expression", 0, 41);
   }
 
   private void assertSingleSyntaxError(

engine/runtime-integration-tests/src/test/scala/org/enso/interpreter/test/semantic/DataflowErrorsTest.scala

@@ -93,7 +93,7 @@ class DataflowErrorsTest extends InterpreterTest {
           |
           |main =
           |    myErr = Error.throw (My_Error.Mk_My_Error 20)
-          |    IO.println(myErr.catch_primitive .recover)
+          |    IO.println (myErr.catch_primitive .recover)
           |""".stripMargin
       eval(code)
       consumeOut shouldEqual List("(Mk_My_Recovered 20)")