Lexer string interning (#1758)

This Pull Request is part of #279.

 It adds a string interner to Boa, which allows many types to not contain heap-allocated strings, and just contain a `NonZeroUsize` instead. This can move types to the stack (hopefully I'll be able to move `Token`, for example, maybe some `Node` types too.

Note that the internet is for now only available in the lexer. Next steps (in this PR or future ones) would include also using interning in the parser, and finally in execution. The idea is that strings should be represented with a `Sym` until they are displayed.

Talking about display. I have changed the `ParseError` type in order to not contain anything that could contain a `Sym` (basically tokens), which might be a bit faster, but what is important is that we don't depend on the interner when displaying errors.

The issue I have now is in order to display tokens. This requires the interner if we want to know identifiers, for example. The issue here is that Rust doesn't allow using a `fmt::Formatter` (only in nightly), which is making my head hurt. Maybe someone of you can find a better way of doing this.

Then, about `cursor.expect()`, this is the only place where we don't have the expected token type as a static string, so it's failing to compile. We have the option of changing the type definition of `ParseError` to contain an owned string, but maybe we can avoid this by having a `&'static str` come from a `TokenKind` with the default values, such as "identifier" for an identifier. I wanted for you to think about it and maybe we can just add that and avoid allocations there.

Oh, and this depends on the VM-only branch, so that has to be merged before :)

Another thing to check: should the interner be in its own module?

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "boa_wasm",
     "boa_tester",
     "boa_unicode",
+    "boa_interner",
 ]
 
 # The release profile, used for `cargo build --release`.

boa/Cargo.toml

@@ -13,15 +13,16 @@ rust-version = "1.57"
 
 [features]
 profiler = ["measureme"]
-deser = []
+deser = ["boa_interner/serde"]
 
 # Enable Boa's WHATWG console object implementation.
 console = []
 
 [dependencies]
 boa_unicode = { path = "../boa_unicode", version = "0.13.0" }
+boa_interner = { path = "../boa_interner", version = "0.13.0" }
 gc = { version = "0.4.1", features = ["derive"] }
-serde = { version = "1.0.132", features = ["derive", "rc"] }
+serde = { version = "1.0.134", features = ["derive", "rc"] }
 serde_json = "1.0.75"
 rand = "0.8.4"
 num-traits = "0.2.14"

boa/benches/full.rs

@@ -1,6 +1,7 @@
 //! Benchmarks of the whole execution engine in Boa.
 
-use boa::{parse, realm::Realm, Context};
+use boa::{realm::Realm, syntax::Parser, Context};
+use boa_interner::Interner;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
 #[cfg(all(target_arch = "x86_64", target_os = "linux", target_env = "gnu"))]
@@ -20,8 +21,9 @@ macro_rules! full_benchmarks {
             $(
                 {
                     static CODE: &str = include_str!(concat!("bench_scripts/", stringify!($name), ".js"));
+                    let mut interner = Interner::new();
                     c.bench_function(concat!($id, " (Parser)"), move |b| {
-                        b.iter(|| parse(black_box(CODE), false))
+                        b.iter(|| Parser::new(black_box(CODE.as_bytes()), false).parse_all(&mut interner))
                     });
                 }
             )*
@@ -30,7 +32,8 @@ macro_rules! full_benchmarks {
             $(
                 {
                     static CODE: &str = include_str!(concat!("bench_scripts/", stringify!($name), ".js"));
-                    let statement_list = parse(CODE, false).unwrap();
+                    let mut interner = Interner::new();
+                    let statement_list = Parser::new(CODE.as_bytes(), false).parse_all( &mut interner).expect("parsing failed");
                     c.bench_function(concat!($id, " (Compiler)"), move |b| {
                         b.iter(|| {
                             Context::compile(black_box(statement_list.clone()));
@@ -43,9 +46,10 @@ macro_rules! full_benchmarks {
             $(
                 {
                     static CODE: &str = include_str!(concat!("bench_scripts/", stringify!($name), ".js"));
-                    let statement_list = parse(CODE, false).unwrap();
+                    let mut interner = Interner::new();
+                    let statement_list = Parser::new(CODE.as_bytes(), false).parse_all( &mut interner).expect("parsing failed");
                     let code_block = Context::compile(statement_list);
-                    let mut context = Context::new();
+                    let mut context = Context::default();
                     c.bench_function(concat!($id, " (Execution)"), move |b| {
                         b.iter(|| {
                             context.execute(black_box(code_block.clone())).unwrap();

boa/examples/classes.rs

@@ -125,7 +125,7 @@ impl Class for Person {
 
 fn main() {
     // First we need to create a Javascript context.
-    let mut context = Context::new();
+    let mut context = Context::default();
 
     // Then we need to register the global class `Person` inside `context`.
     context.register_global_class::<Person>().unwrap();

boa/examples/closures.rs

@@ -10,7 +10,7 @@ use boa::{
 
 fn main() -> Result<(), JsValue> {
     // We create a new `Context` to create a new Javascript executor.
-    let mut context = Context::new();
+    let mut context = Context::default();
 
     // We make some operations in Rust that return a `Copy` value that we want
     // to pass to a Javascript function.