refactor: Use DSL v2 types in place of ScannerDefinitionNode (#1003)

Based on #1002
Part of #638

The DSL v2's `model::Scanner` is almost 1:1 usable with our current
parser codegen, with the exception of versioned scanners. In short we:
- introduce a helper type `VersionedScanner` that is used in place of
the old `ScannerDefinitionNode::Versioned`
- introduce a `ScannerExt` trait implemented for both `VersionedScanner`
and the `model::Scanner`, which is responsible for generating the main
scanning logic
- Repurposes `ScannerDefinition` slightly to surface more of the
scanner-related logic for trivia/fragment/token
- implements this directly for
`model::{TriviaItem,FragmentItem,TokenItem}` and stores it in the v1
Grammar struct
- similarly, uses `model::KeywordItem` directly in place of v1's
`KeywordScannerDefinition` as they shared the same functionality

crates/codegen/language/definition/src/model/terminals/keyword.rs

@@ -2,7 +2,7 @@ use codegen_language_internal_macros::{derive_spanned_type, ParseInputTokens, Wr
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
-use crate::model::{Identifier, VersionSpecifier};
+use crate::model::{Identifier, Scanner, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
@@ -32,6 +32,23 @@ pub enum KeywordValue {
     Atom { atom: String },
 }
 
+impl From<KeywordValue> for Scanner {
+    fn from(value: KeywordValue) -> Scanner {
+        match value {
+            KeywordValue::Optional { value } => Scanner::Optional {
+                scanner: Box::new((*value).into()),
+            },
+            KeywordValue::Sequence { values } => Scanner::Sequence {
+                scanners: values.into_iter().map(Into::into).collect(),
+            },
+            KeywordValue::Atom { atom } => Scanner::Atom { atom },
+            KeywordValue::Choice { values } => Scanner::Choice {
+                scanners: values.into_iter().map(Into::into).collect(),
+            },
+        }
+    }
+}
+
 impl KeywordValue {
     /// Collects all possible variations generated by this value.
     pub fn collect_variations(&self) -> Vec<String> {

crates/codegen/runtime/generator/src/parser/codegen.rs

@@ -5,8 +5,7 @@ mod scanner_definition;
 mod trie;
 mod versioned;
 
-pub use keyword_scanner_definition::KeywordScannerDefinitionCodegen;
+pub use keyword_scanner_definition::{KeywordItemAtom, KeywordScannerDefinitionCodegen};
 pub use parser_definition::ParserDefinitionCodegen;
 pub use precedence_parser_definition::PrecedenceParserDefinitionCodegen;
-pub use scanner_definition::ScannerDefinitionCodegen;
 pub use trie::Trie;

crates/codegen/runtime/generator/src/parser/codegen/keyword_scanner_definition.rs

@@ -1,22 +1,23 @@
+use std::rc::Rc;
+
 use codegen_language_definition::model;
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote};
 
-use crate::parser::codegen::scanner_definition::ScannerDefinitionNodeCodegen as _;
+use crate::parser::codegen::scanner_definition::ScannerCodegen as _;
 use crate::parser::codegen::versioned::VersionedQuote;
-use crate::parser::grammar::{KeywordScannerDefinitionRef, ScannerDefinitionNode};
 
 pub trait KeywordScannerDefinitionCodegen {
     fn to_scanner_code(&self) -> TokenStream;
 }
 
-impl KeywordScannerDefinitionCodegen for KeywordScannerDefinitionRef {
+impl KeywordScannerDefinitionCodegen for model::KeywordItem {
     fn to_scanner_code(&self) -> TokenStream {
-        let name_ident = format_ident!("{}", self.name());
+        let name_ident = format_ident!("{}", self.name);
         let terminal_kind = quote! { TerminalKind::#name_ident };
 
         let kw_scanners: Vec<_> = self
-            .definitions()
+            .definitions
             .iter()
             .map(|versioned_kw| {
                 let scanner = versioned_kw.value.to_scanner_code();
@@ -82,6 +83,55 @@ impl KeywordScannerDefinitionCodegen for KeywordScannerDefinitionRef {
 impl KeywordScannerDefinitionCodegen for model::KeywordValue {
     fn to_scanner_code(&self) -> TokenStream {
         // This is a subset; let's reuse that
-        ScannerDefinitionNode::from(self.clone()).to_scanner_code()
+        model::Scanner::from(self.clone()).to_scanner_code()
+    }
+}
+
+/// A newtype wrapper around [`model::KeywordItem`] that only has a single atom value.
+///
+/// The main usage for this type is to construct a keyword trie, as trie will
+/// only work with single atom values and keyword promotion needs to additionally account for
+/// keyword reservation, rather than just literal presence.
+#[derive(Clone)]
+pub struct KeywordItemAtom(Rc<model::KeywordItem>);
+
+impl KeywordItemAtom {
+    /// Wraps the keyword scanner definition if it is a single atom value.
+    pub fn try_from_def(def: &Rc<model::KeywordItem>) -> Option<Self> {
+        match def.definitions[..] {
+            [model::KeywordDefinition {
+                value: model::KeywordValue::Atom { .. },
+                ..
+            }] => Some(Self(Rc::clone(def))),
+            _ => None,
+        }
+    }
+}
+
+impl std::ops::Deref for KeywordItemAtom {
+    type Target = Rc<model::KeywordItem>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl KeywordItemAtom {
+    pub fn definition(&self) -> &model::KeywordDefinition {
+        self.0
+            .definitions
+            .first()
+            .expect("KeywordItemAtom should have exactly one definition")
+    }
+
+    /// The single atom value that this keyword item matches.
+    pub fn value(&self) -> &str {
+        match self.definition() {
+            model::KeywordDefinition {
+                value: model::KeywordValue::Atom { atom },
+                ..
+            } => atom,
+            _ => unreachable!("KeywordItemAtom should have a single atom value"),
+        }
     }
 }

crates/codegen/runtime/generator/src/parser/codegen/parser_definition.rs

@@ -144,7 +144,7 @@ impl ParserDefinitionNodeCodegen for ParserDefinitionNode {
 
             // Keyword scanner uses the promotion inside the parse_terminal
             Self::KeywordScannerDefinition(scanner_definition) => {
-                let kind = format_ident!("{name}", name = scanner_definition.name());
+                let kind = format_ident!("{name}", name = scanner_definition.name);
 
                 let parse_terminal = if is_trivia {
                     format_ident!("parse_terminal")