refactor!: Label the trivia terminal types accordingly

NomicFoundation · Mar 7, 2024 · 52db9c6 · 52db9c6
1 parent 59f5532
commit 52db9c6
Show file tree

Hide file tree

Showing 446 changed files with 2,688 additions and 2,607 deletions.
diff --git a/crates/codegen/grammar/src/constructor.rs b/crates/codegen/grammar/src/constructor.rs
@@ -45,12 +45,12 @@ impl GrammarConstructorDslV2 for Grammar {
 
         let leading_trivia = Rc::new(NamedTriviaParser {
             name: "LeadingTrivia",
-            def: resolve_trivia(lang.leading_trivia.clone(), &mut ctx),
+            def: resolve_trivia(lang.leading_trivia.clone(), TriviaKind::Leading, &mut ctx),
         }) as Rc<dyn TriviaParserDefinition>;
 
         let trailing_trivia = Rc::new(NamedTriviaParser {
             name: "TrailingTrivia",
-            def: resolve_trivia(lang.trailing_trivia.clone(), &mut ctx),
+            def: resolve_trivia(lang.trailing_trivia.clone(), TriviaKind::Trailing, &mut ctx),
         }) as Rc<dyn TriviaParserDefinition>;
 
         for (_lex_ctx, item) in items.values() {
@@ -507,35 +507,43 @@ fn resolve_keyword_value(value: model::KeywordValue) -> KeywordScannerDefinition
     }
 }
 
-fn resolve_trivia(parser: model::TriviaParser, ctx: &mut ResolveCtx<'_>) -> ParserDefinitionNode {
+fn resolve_trivia(
+    parser: model::TriviaParser,
+    kind: TriviaKind,
+    ctx: &mut ResolveCtx<'_>,
+) -> ParserDefinitionNode {
     match parser {
         model::TriviaParser::Optional { parser } => {
-            ParserDefinitionNode::Optional(Box::new(resolve_trivia(*parser, ctx)))
+            ParserDefinitionNode::Optional(Box::new(resolve_trivia(*parser, kind, ctx)))
         }
         model::TriviaParser::OneOrMore { parser } => ParserDefinitionNode::OneOrMore(
-            Labeled::anonymous(Box::new(resolve_trivia(*parser, ctx))),
+            Labeled::anonymous(Box::new(resolve_trivia(*parser, kind, ctx))),
         ),
         model::TriviaParser::ZeroOrMore { parser } => ParserDefinitionNode::ZeroOrMore(
-            Labeled::anonymous(Box::new(resolve_trivia(*parser, ctx))),
+            Labeled::anonymous(Box::new(resolve_trivia(*parser, kind, ctx))),
         ),
         model::TriviaParser::Sequence { parsers } => ParserDefinitionNode::Sequence(
             parsers
                 .into_iter()
-                .map(|scanner| Labeled::anonymous(resolve_trivia(scanner, ctx)))
+                .map(|scanner| Labeled::anonymous(resolve_trivia(scanner, kind, ctx)))
                 .collect(),
         ),
         model::TriviaParser::Choice { parsers } => {
             ParserDefinitionNode::Choice(Labeled::anonymous(
                 parsers
                     .into_iter()
-                    .map(|scanner| resolve_trivia(scanner, ctx))
+                    .map(|scanner| resolve_trivia(scanner, kind, ctx))
                     .collect(),
             ))
         }
         model::TriviaParser::Trivia { reference } => {
             match resolve_grammar_element(&reference, ctx) {
                 GrammarElement::ScannerDefinition(parser) => {
-                    ParserDefinitionNode::ScannerDefinition(parser)
+                    // Hack: This is a sequence of a single scanner in order to emit the names
+                    ParserDefinitionNode::Sequence(vec![Labeled::with_builtin_label(
+                        kind.label(),
+                        ParserDefinitionNode::ScannerDefinition(parser),
+                    )])
                 }
                 _ => panic!("Expected {reference} to be a ScannerDefinition"),
             }
@@ -793,6 +801,21 @@ fn resolve_precedence(
     }
 }
 
+#[derive(Clone, Copy)]
+enum TriviaKind {
+    Leading,
+    Trailing,
+}
+
+impl TriviaKind {
+    fn label(self) -> BuiltInLabel {
+        match self {
+            TriviaKind::Leading => BuiltInLabel::LeadingTrivia,
+            TriviaKind::Trailing => BuiltInLabel::TrailingTrivia,
+        }
+    }
+}
+
 trait IntoParserDefNode {
     fn into_parser_def_node(self) -> ParserDefinitionNode;
 }
@@ -861,6 +884,8 @@ enum BuiltInLabel {
     Operand,
     LeftOperand,
     RightOperand,
+    LeadingTrivia,
+    TrailingTrivia,
 }
 
 impl<T> LabeledExt<T> for Labeled<T> {

diff --git a/crates/codegen/parser/generator/src/rust_generator.rs b/crates/codegen/parser/generator/src/rust_generator.rs
@@ -280,6 +280,8 @@ impl GrammarVisitor for RustGenerator {
         self.labels.remove("operand");
         self.labels.remove("left_operand");
         self.labels.remove("right_operand");
+        self.labels.remove("leading_trivia");
+        self.labels.remove("trailing_trivia");
 
         // Just being anal about tidying up :)
         self.all_scanners.clear();

diff --git a/crates/codegen/parser/runtime/src/kinds.rs b/crates/codegen/parser/runtime/src/kinds.rs
@@ -76,6 +76,8 @@ pub enum NodeLabel {
     Operand,
     LeftOperand,
     RightOperand,
+    LeadingTrivia,
+    TrailingTrivia,
     // Used for testing this crate, this is generated in the client code
     Label1,
     Label2,

diff --git a/crates/codegen/parser/runtime/src/parser_support/parser_result.rs b/crates/codegen/parser/runtime/src/parser_support/parser_result.rs
@@ -82,6 +82,14 @@ impl ParserResult {
         {
             *prev_label = Some(label);
         }
+        // Also allow to name a single trivia token node
+        else if let ParserResult::Match(Match { nodes, .. }) = &mut self {
+            if let [node] = nodes.as_mut_slice() {
+                if node.as_token().is_some_and(|tok| tok.kind.is_trivia()) {
+                    node.label = Some(label);
+                }
+            }
+        }
 
         self
     }

diff --git a/crates/codegen/parser/runtime/src/templates/kinds.rs.jinja2 b/crates/codegen/parser/runtime/src/templates/kinds.rs.jinja2
@@ -46,6 +46,8 @@ pub enum NodeLabel {
     Operand,
     LeftOperand,
     RightOperand,
+    LeadingTrivia,
+    TrailingTrivia,
     // Generated
     {% for variant in generator.labels -%}
         {{ variant | pascal_case }},

diff --git a/crates/solidity/outputs/cargo/slang_solidity/src/generated/kinds.rs b/crates/solidity/outputs/cargo/slang_solidity/src/generated/kinds.rs
diff --git a/crates/solidity/outputs/cargo/slang_solidity/src/generated/language.rs b/crates/solidity/outputs/cargo/slang_solidity/src/generated/language.rs
diff --git a/crates/solidity/outputs/cargo/slang_solidity/src/generated/parser_support/parser_result.rs b/crates/solidity/outputs/cargo/slang_solidity/src/generated/parser_support/parser_result.rs
diff --git a/crates/solidity/outputs/npm/package/src/generated/index.d.ts b/crates/solidity/outputs/npm/package/src/generated/index.d.ts
diff --git a/...lidity/testing/snapshots/cst_output/AssemblyStatement/simple/generated/0.4.11-success.yml b/...lidity/testing/snapshots/cst_output/AssemblyStatement/simple/generated/0.4.11-success.yml
@@ -11,19 +11,19 @@ Tree:
   - (AssemblyStatement): # 'assembly "evmasm" ("memory-safe") {\n\n}\n' (0..39)
       - (assembly_keyword꞉ AssemblyKeyword): "assembly" # (0..8)
       - (label꞉ StringLiteral): # ' "evmasm"' (8..17)
-          - (Whitespace): " " # (8..9)
+          - (leading_trivia꞉ Whitespace): " " # (8..9)
           - (variant꞉ DoubleQuotedStringLiteral): '"evmasm"' # (9..17)
       - (flags꞉ AssemblyFlagsDeclaration): # ' ("memory-safe")' (17..33)
-          - (Whitespace): " " # (17..18)
+          - (leading_trivia꞉ Whitespace): " " # (17..18)
           - (open_paren꞉ OpenParen): "(" # (18..19)
           - (flags꞉ AssemblyFlags): # '"memory-safe"' (19..32)
               - (item꞉ StringLiteral) ► (variant꞉ DoubleQuotedStringLiteral): '"memory-safe"' # (19..32)
           - (close_paren꞉ CloseParen): ")" # (32..33)
       - (body꞉ YulBlock): # " {\n\n}\n" (33..39)
-          - (Whitespace): " " # (33..34)
+          - (leading_trivia꞉ Whitespace): " " # (33..34)
           - (open_brace꞉ OpenBrace): "{" # (34..35)
-          - (EndOfLine): "\n" # (35..36)
+          - (trailing_trivia꞉ EndOfLine): "\n" # (35..36)
           - (statements꞉ YulStatements): [] # (36..36)
-          - (EndOfLine): "\n" # (36..37)
+          - (leading_trivia꞉ EndOfLine): "\n" # (36..37)
           - (close_brace꞉ CloseBrace): "}" # (37..38)
-          - (EndOfLine): "\n" # (38..39)
+          - (trailing_trivia꞉ EndOfLine): "\n" # (38..39)
diff --git a/...sting/snapshots/cst_output/Block/postfix_recovery_regression/generated/0.4.11-failure.yml b/...sting/snapshots/cst_output/Block/postfix_recovery_regression/generated/0.4.11-failure.yml
@@ -21,20 +21,20 @@ Errors: # 1 total
 
 Tree:
   - (Block): # "// Make sure we don't panic when encountering an u..." (0..122)
-      - (SingleLineComment): "// Make sure we don't panic when encountering an u..." # (0..93)
-      - (EndOfLine): "\n" # (93..94)
-      - (EndOfLine): "\n" # (94..95)
+      - (leading_trivia꞉ SingleLineComment): "// Make sure we don't panic when encountering an u..." # (0..93)
+      - (leading_trivia꞉ EndOfLine): "\n" # (93..94)
+      - (leading_trivia꞉ EndOfLine): "\n" # (94..95)
       - (open_brace꞉ OpenBrace): "{" # (95..96)
-      - (EndOfLine): "\n" # (96..97)
+      - (trailing_trivia꞉ EndOfLine): "\n" # (96..97)
       - (statements꞉ Statements): # "    a.b('\n        }');\n" (97..120)
           - (item꞉ Statement) ► (variant꞉ VariableDeclarationStatement): # "    a.b('\n        }');\n" (97..120)
               - (variable_type꞉ VariableDeclarationType) ► (variant꞉ TypeName) ► (variant꞉ IdentifierPath): # "    a.b" (97..104)
-                  - (Whitespace): "    " # (97..101)
+                  - (leading_trivia꞉ Whitespace): "    " # (97..101)
                   - (item꞉ Identifier): "a" # (101..102)
                   - (separator꞉ Period): "." # (102..103)
                   - (item꞉ Identifier): "b" # (103..104)
               - (SKIPPED): "('\n        }')" # (104..118)
               - (semicolon꞉ Semicolon): ";" # (118..119)
-              - (EndOfLine): "\n" # (119..120)
+              - (trailing_trivia꞉ EndOfLine): "\n" # (119..120)
       - (close_brace꞉ CloseBrace): "}" # (120..121)
-      - (EndOfLine): "\n" # (121..122)
+      - (trailing_trivia꞉ EndOfLine): "\n" # (121..122)
diff --git a/...esting/snapshots/cst_output/Block/postfix_recovery_regression/generated/0.5.0-failure.yml b/...esting/snapshots/cst_output/Block/postfix_recovery_regression/generated/0.5.0-failure.yml
@@ -21,20 +21,20 @@ Errors: # 1 total
 
 Tree:
   - (Block): # "// Make sure we don't panic when encountering an u..." (0..122)
-      - (SingleLineComment): "// Make sure we don't panic when encountering an u..." # (0..93)
-      - (EndOfLine): "\n" # (93..94)
-      - (EndOfLine): "\n" # (94..95)
+      - (leading_trivia꞉ SingleLineComment): "// Make sure we don't panic when encountering an u..." # (0..93)
+      - (leading_trivia꞉ EndOfLine): "\n" # (93..94)
+      - (leading_trivia꞉ EndOfLine): "\n" # (94..95)
       - (open_brace꞉ OpenBrace): "{" # (95..96)
-      - (EndOfLine): "\n" # (96..97)
+      - (trailing_trivia꞉ EndOfLine): "\n" # (96..97)
       - (statements꞉ Statements): # "    a.b('\n        }');\n" (97..120)
           - (item꞉ Statement) ► (variant꞉ VariableDeclarationStatement): # "    a.b('\n        }');\n" (97..120)
               - (variable_type꞉ VariableDeclarationType) ► (variant꞉ TypeName) ► (variant꞉ IdentifierPath): # "    a.b" (97..104)
-                  - (Whitespace): "    " # (97..101)
+                  - (leading_trivia꞉ Whitespace): "    " # (97..101)
                   - (item꞉ Identifier): "a" # (101..102)
                   - (separator꞉ Period): "." # (102..103)
                   - (item꞉ Identifier): "b" # (103..104)
               - (SKIPPED): "('\n        }')" # (104..118)
               - (semicolon꞉ Semicolon): ";" # (118..119)
-              - (EndOfLine): "\n" # (119..120)
+              - (trailing_trivia꞉ EndOfLine): "\n" # (119..120)
       - (close_brace꞉ CloseBrace): "}" # (120..121)
-      - (EndOfLine): "\n" # (121..122)
+      - (trailing_trivia꞉ EndOfLine): "\n" # (121..122)
diff --git a/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.4.11-failure.yml b/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.4.11-failure.yml
@@ -19,8 +19,8 @@ Tree:
       - (statements꞉ Statements): # " unchecked " (1..12)
           - (item꞉ Statement) ► (variant꞉ ExpressionStatement): # " unchecked " (1..12)
               - (expression꞉ Expression): # " unchecked" (1..11)
-                  - (Whitespace): " " # (1..2)
+                  - (leading_trivia꞉ Whitespace): " " # (1..2)
                   - (variant꞉ Identifier): "unchecked" # (2..11)
-              - (Whitespace): " " # (11..12)
+              - (leading_trivia꞉ Whitespace): " " # (11..12)
       - (SKIPPED): "{ x = 1; } " # (12..23)
       - (close_brace꞉ CloseBrace): "}" # (23..24)
diff --git a/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.5.0-failure.yml b/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.5.0-failure.yml
@@ -17,6 +17,6 @@ Tree:
   - (Block): # "{ unchecked { x = 1; } }" (0..24)
       - (open_brace꞉ OpenBrace): "{" # (0..1)
       - (statements꞉ Statements): [] # (1..1)
-      - (Whitespace): " " # (1..2)
+      - (leading_trivia꞉ Whitespace): " " # (1..2)
       - (SKIPPED): "unchecked { x = 1; } " # (2..23)
       - (close_brace꞉ CloseBrace): "}" # (23..24)
diff --git a/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.5.3-failure.yml b/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.5.3-failure.yml
@@ -17,6 +17,6 @@ Tree:
   - (Block): # "{ unchecked { x = 1; } }" (0..24)
       - (open_brace꞉ OpenBrace): "{" # (0..1)
       - (statements꞉ Statements): [] # (1..1)
-      - (Whitespace): " " # (1..2)
+      - (leading_trivia꞉ Whitespace): " " # (1..2)
       - (SKIPPED): "unchecked { x = 1; } " # (2..23)
       - (close_brace꞉ CloseBrace): "}" # (23..24)
diff --git a/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.6.0-failure.yml b/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.6.0-failure.yml
@@ -17,6 +17,6 @@ Tree:
   - (Block): # "{ unchecked { x = 1; } }" (0..24)
       - (open_brace꞉ OpenBrace): "{" # (0..1)
       - (statements꞉ Statements): [] # (1..1)
-      - (Whitespace): " " # (1..2)
+      - (leading_trivia꞉ Whitespace): " " # (1..2)
       - (SKIPPED): "unchecked { x = 1; } " # (2..23)
       - (close_brace꞉ CloseBrace): "}" # (23..24)
diff --git a/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.7.0-failure.yml b/crates/solidity/testing/snapshots/cst_output/Block/unchecked/generated/0.7.0-failure.yml
@@ -17,6 +17,6 @@ Tree:
   - (Block): # "{ unchecked { x = 1; } }" (0..24)
       - (open_brace꞉ OpenBrace): "{" # (0..1)
       - (statements꞉ Statements): [] # (1..1)
-      - (Whitespace): " " # (1..2)
+      - (leading_trivia꞉ Whitespace): " " # (1..2)
       - (SKIPPED): "unchecked { x = 1; } " # (2..23)
       - (close_brace꞉ CloseBrace): "}" # (23..24)