parse line breaks without newlines (#854)

.changeset/famous-ants-heal.md

@@ -0,0 +1,5 @@
+---
+"@nomicfoundation/slang": patch
+---
+
+parse line breaks without newlines

crates/solidity/inputs/language/src/definition.rs

@@ -392,7 +392,8 @@ codegen_language_macros::compile!(Language(
                         ),
                         Trivia(
                             name = EndOfLine,
-                            scanner = Sequence([Optional(Atom("\r")), Atom("\n")])
+                            scanner =
+                                Choice([Atom("\n"), Sequence([Atom("\r"), Optional(Atom("\n"))])])
                         ),
                         Trivia(
                             name = SingleLineComment,

crates/solidity/outputs/cargo/tests/src/lib.rs

@@ -2,3 +2,4 @@
 
 mod cst_output;
 mod doc_examples;
+mod trivia;

crates/solidity/outputs/cargo/tests/src/trivia.rs

@@ -0,0 +1,47 @@
+use anyhow::Result;
+use semver::Version;
+use slang_solidity::cst::Node;
+use slang_solidity::kinds::{RuleKind, TokenKind};
+use slang_solidity::language::Language;
+
+#[test]
+fn end_of_line() -> Result<()> {
+    // Only one is valid as a single token:
+    compare_end_of_lines("\r", &["\r"])?;
+    compare_end_of_lines("\n", &["\n"])?;
+
+    // Two of the same are two tokens:
+    compare_end_of_lines("\n\n", &["\n", "\n"])?;
+    compare_end_of_lines("\r\r", &["\r", "\r"])?;
+
+    // A carriage return followed by a newline is one token, but the opposite is not:
+    compare_end_of_lines("\r\n", &["\r\n"])?;
+    compare_end_of_lines("\n\r", &["\n", "\r"])?;
+
+    Ok(())
+}
+
+fn compare_end_of_lines(input: &str, expected: &[&str]) -> Result<()> {
+    let version = Version::parse("0.8.0")?;
+    let language = &Language::new(version)?;
+
+    let output = language.parse(RuleKind::SourceUnit, input);
+    assert!(output.is_valid());
+
+    let actual = output
+        .create_tree_cursor()
+        .filter_map(|node| match node {
+            Node::Rule(_) => None,
+
+            Node::Token(token) => {
+                assert_eq!(token.kind, TokenKind::EndOfLine);
+                Some(token.text.to_owned())
+            }
+        })
+        .collect::<Vec<_>>();
+
+    let expected = expected.to_vec();
+    assert_eq!(actual, expected);
+
+    Ok(())
+}

crates/solidity/testing/utils/src/cst_snapshots/mod.rs

@@ -37,6 +37,9 @@ fn write_source(w: &mut String, source: &str) -> Result<()> {
         return Ok(());
     }
 
+    // "lines()" only handles "\n", so let's normalize all line endings:
+    let source = source.replace("\r\n", "\n").replace('\r', "\n");
+
     let line_data = source
         .lines()
         .enumerate()