From d722faf947ee32abb8f412b35a8bc49c0cf583bd Mon Sep 17 00:00:00 2001 From: Nadir Fejzic Date: Sun, 13 Oct 2024 18:29:30 +0200 Subject: [PATCH] feat: implement basic parsing of heading and paragraph --- frontend/Cargo.toml | 2 + frontend/src/lexer/mod.rs | 243 +++++++++--------- frontend/src/lib.rs | 1 + frontend/src/parser/block/bulletlist.rs | 34 +++ frontend/src/parser/block/heading.rs | 72 ++++++ frontend/src/parser/block/mod.rs | 22 ++ frontend/src/parser/block/paragraph.rs | 11 + frontend/src/parser/block/verbatim.rs | 19 ++ frontend/src/parser/mod.rs | 208 +++++++++++++++ frontend/tests/lexer/snapshot.rs | 20 +- frontend/tests/parser/mod.rs | 60 +++++ frontend/tests/parser/snapshot.rs | 83 ++++++ frontend/tests/snapshots.rs | 17 +- frontend/tests/spec/markup/block/heading.yml | 48 ++++ .../tests/spec/markup/block/paragraph.yml | 40 +++ .../block/heading/multi-line-level-1.snap | 49 ++++ .../block/heading/multi-line-level-6.snap | 49 ++++ .../block/heading/single-line-level-1.snap | 37 +++ .../block/heading/single-line-level-6.snap | 33 +++ .../lexer/block/paragraph/multi-line.snap | 49 ++++ .../lexer/block/paragraph/single-line.snap | 33 +++ .../lexer/block/paragraph/two-paragraphs.snap | 69 +++++ .../snapshots/lexer/bold/ambiguous-close.snap | 2 + .../snapshots/lexer/bold/ambiguous-end.snap | 2 + .../snapshots/lexer/bold/ambiguous-start.snap | 2 + .../snapshots/lexer/bold/bold-in-middle.snap | 2 + .../snapshots/lexer/bold/bold-not-bold.snap | 2 + .../snapshots/lexer/bold/escaped-bold.snap | 2 + .../lexer/bold/implicit-closed-bold.snap | 2 + .../spec/snapshots/lexer/bold/not-bold.snap | 2 + .../snapshots/lexer/bold/not-opened-bold.snap | 2 + .../snapshots/lexer/bold/simple-bold.snap | 2 + .../block/heading/multi-line-level-1.snap | 18 ++ .../block/heading/multi-line-level-6.snap | 18 ++ .../block/heading/single-line-level-1.snap | 15 ++ .../block/heading/single-line-level-6.snap | 15 ++ .../parser/block/paragraph/multi-line.snap | 21 ++ .../parser/block/paragraph/single-line.snap | 15 ++ .../block/paragraph/two-paragraphs.snap | 23 ++ .../parser/bold/ambiguous-close.snap | 15 ++ .../snapshots/parser/bold/ambiguous-end.snap | 15 ++ .../parser/bold/ambiguous-start.snap | 15 ++ .../snapshots/parser/bold/bold-in-middle.snap | 15 ++ .../snapshots/parser/bold/bold-not-bold.snap | 15 ++ .../snapshots/parser/bold/escaped-bold.snap | 15 ++ .../parser/bold/implicit-closed-bold.snap | 15 ++ .../spec/snapshots/parser/bold/not-bold.snap | 15 ++ .../parser/bold/not-opened-bold.snap | 15 ++ .../snapshots/parser/bold/simple-bold.snap | 15 ++ 49 files changed, 1355 insertions(+), 139 deletions(-) create mode 100644 frontend/src/parser/block/bulletlist.rs create mode 100644 frontend/src/parser/block/heading.rs create mode 100644 frontend/src/parser/block/mod.rs create mode 100644 frontend/src/parser/block/paragraph.rs create mode 100644 frontend/src/parser/block/verbatim.rs create mode 100644 frontend/src/parser/mod.rs create mode 100644 frontend/tests/parser/mod.rs create mode 100644 frontend/tests/parser/snapshot.rs create mode 100644 frontend/tests/spec/markup/block/heading.yml create mode 100644 frontend/tests/spec/markup/block/paragraph.yml create mode 100644 frontend/tests/spec/snapshots/lexer/block/heading/multi-line-level-1.snap create mode 100644 frontend/tests/spec/snapshots/lexer/block/heading/multi-line-level-6.snap create mode 100644 frontend/tests/spec/snapshots/lexer/block/heading/single-line-level-1.snap create mode 100644 frontend/tests/spec/snapshots/lexer/block/heading/single-line-level-6.snap create mode 100644 frontend/tests/spec/snapshots/lexer/block/paragraph/multi-line.snap create mode 100644 frontend/tests/spec/snapshots/lexer/block/paragraph/single-line.snap create mode 100644 frontend/tests/spec/snapshots/lexer/block/paragraph/two-paragraphs.snap create mode 100644 frontend/tests/spec/snapshots/parser/block/heading/multi-line-level-1.snap create mode 100644 frontend/tests/spec/snapshots/parser/block/heading/multi-line-level-6.snap create mode 100644 frontend/tests/spec/snapshots/parser/block/heading/single-line-level-1.snap create mode 100644 frontend/tests/spec/snapshots/parser/block/heading/single-line-level-6.snap create mode 100644 frontend/tests/spec/snapshots/parser/block/paragraph/multi-line.snap create mode 100644 frontend/tests/spec/snapshots/parser/block/paragraph/single-line.snap create mode 100644 frontend/tests/spec/snapshots/parser/block/paragraph/two-paragraphs.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/ambiguous-close.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/ambiguous-end.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/ambiguous-start.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/bold-in-middle.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/bold-not-bold.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/escaped-bold.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/implicit-closed-bold.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/not-bold.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/not-opened-bold.snap create mode 100644 frontend/tests/spec/snapshots/parser/bold/simple-bold.snap diff --git a/frontend/Cargo.toml b/frontend/Cargo.toml index 980f2f58..a0509540 100644 --- a/frontend/Cargo.toml +++ b/frontend/Cargo.toml @@ -17,6 +17,8 @@ harness=false [dependencies] icu_properties = "1.3.2" ribbon = "0.7.0" +strum = "0.22.0" +strum_macros = "0.22.0" [dev-dependencies] unimarkup-commons = { path ="../commons/", version = "0", features = ["test_runner"] } diff --git a/frontend/src/lexer/mod.rs b/frontend/src/lexer/mod.rs index 45851d43..dcd63623 100644 --- a/frontend/src/lexer/mod.rs +++ b/frontend/src/lexer/mod.rs @@ -11,7 +11,7 @@ use crate::symbol::{Symbol, SymbolKind}; /// Lexes the indentation token. Indentation is defined as some number of spaces at the beginning /// of a line. -fn indentation<'input>( +fn indentation_or_blankline<'input>( start_sym: &Symbol<'input>, sym_stream: &mut Tape>, ) -> Token<'input> { @@ -29,10 +29,24 @@ fn indentation<'input>( } } - Token { - input: start_sym.input, - kind: TokenKind::Indentation(indent), - span, + sym_stream.expand(); + + match sym_stream.peek_front() { + Some(sym) if sym.kind == SymbolKind::Newline => { + // indentation means there was a newline, then spaces + // and now again a newline, meaning we found a line that's blank + span.len += sym.len(); + Token { + input: start_sym.input, + kind: TokenKind::Blankline, + span, + } + } + _ => Token { + input: start_sym.input, + kind: TokenKind::Indentation(indent), + span, + }, } } @@ -96,25 +110,6 @@ fn whitespace<'input>( } } -fn title<'input>( - start_sym: &Symbol<'input>, - sym_stream: &mut Tape>, -) -> Token<'input> { - let mut span = start_sym.span; - - sym_stream.expand_while(|s| s.kind == SymbolKind::Hash); - - while let Some(sym) = sym_stream.pop_front() { - span.len += sym.len(); - } - - Token { - input: start_sym.input, - kind: TokenKind::Hash(span.len), - span, - } -} - fn plain<'input>( start_sym: &Symbol<'input>, sym_stream: &mut Tape>, @@ -171,6 +166,38 @@ fn escaped<'input>( } } +fn whitespace_or_blankline<'input>( + start_sym: &Symbol<'input>, + sym_stream: &mut Tape>, +) -> Token<'input> { + let mut span = start_sym.span; + + sym_stream.expand_while(|s| s.kind == start_sym.kind); + + while let Some(sym) = sym_stream.pop_front() { + span.len += sym.len(); + } + + sym_stream.expand(); + + match sym_stream.peek_front() { + Some(sym) if sym.kind == SymbolKind::Newline => { + span.len += sym.len(); + Token { + input: start_sym.input, + kind: TokenKind::Blankline, + span, + } + } + + _ => Token { + input: start_sym.input, + kind: TokenKind::Whitespace, + span, + }, + } +} + pub struct TokenStream<'input> { input: &'input str, sym_stream: Tape>, @@ -198,8 +225,8 @@ impl<'input> Iterator for TokenStream<'input> { match sym.kind { SymbolKind::Space => { - if sym.span.offs.saturating_sub(self.last_newline_offs) <= 1 { - return Some(indentation(&sym, &mut self.sym_stream)); + if self.is_start_of_line(&sym) { + return Some(indentation_or_blankline(&sym, &mut self.sym_stream)); } else { return Some(Token { input: self.input, @@ -210,7 +237,17 @@ impl<'input> Iterator for TokenStream<'input> { } SymbolKind::Newline => { + let input = self.input; + let span = sym.span; + + let kind = if self.is_start_of_line(&sym) { + TokenKind::Blankline + } else { + TokenKind::Newline + }; + self.last_newline_offs = sym.span.offs; + return Some(Token { input, kind, span }); } SymbolKind::Backslash => { @@ -234,7 +271,13 @@ impl<'input> Iterator for TokenStream<'input> { return Some(punctuation(&sym, &mut self.sym_stream)) } - SymbolKind::Whitespace => return Some(whitespace(&sym, &mut self.sym_stream)), + SymbolKind::Whitespace => { + if self.is_start_of_line(&sym) { + return Some(whitespace_or_blankline(&sym, &mut self.sym_stream)); + } else { + return Some(whitespace(&sym, &mut self.sym_stream)); + } + } SymbolKind::Eoi => return None, @@ -272,9 +315,19 @@ impl<'input> Iterator for TokenStream<'input> { } } +impl TokenStream<'_> { + fn is_start_of_line(&self, sym: &Symbol<'_>) -> bool { + if self.last_newline_offs == 0 { + sym.span.offs == 0 + } else { + sym.span.offs.saturating_sub(self.last_newline_offs) == 1 + } + } +} + #[cfg(test)] mod tests { - use crate::lexer::{token::Token, token_kind::TokenKind}; + use crate::lexer::token_kind::TokenKind; use super::TokenStream; @@ -284,24 +337,10 @@ mod tests { let tokens: Vec<_> = super::TokenStream::tokenize(input).collect(); assert_eq!(tokens.len(), 2); - assert!(matches!( - tokens.first(), - Some(&Token { - kind: TokenKind::Indentation(4), - .. - }) - )); - - assert!(matches!( - tokens.get(1), - Some(&Token { - kind: TokenKind::Plain, - .. - }) - )); + assert_eq!(tokens.first().unwrap().kind, TokenKind::Indentation(4)); let second = tokens.get(1).unwrap(); - + assert_eq!(second.kind, TokenKind::Plain); assert_eq!(second.as_input_str(), "hello"); } @@ -310,47 +349,25 @@ mod tests { let input = " hello\n there"; let tokens: Vec<_> = dbg!(super::TokenStream::tokenize(input).collect()); - assert_eq!(tokens.len(), 4); + assert_eq!(tokens.len(), 5); - assert!(matches!( - tokens.first(), - Some(&Token { - kind: TokenKind::Indentation(4), - .. - }) - )); + let first = tokens.first().unwrap(); + assert_eq!(first.kind, TokenKind::Indentation(4)); let second = tokens.get(1).unwrap(); - - assert!(matches!( - second, - &Token { - kind: TokenKind::Plain, - .. - } - )); - - assert!(matches!(second.as_input_str(), "hello")); + assert_eq!(second.kind, TokenKind::Plain); + assert_eq!(second.as_input_str(), "hello"); let third = tokens.get(2).unwrap(); - assert!(matches!( - third, - &Token { - kind: TokenKind::Indentation(6), - .. - } - )); + + assert_eq!(third.kind, TokenKind::Newline); let fourth = tokens.get(3).unwrap(); - assert!(matches!( - fourth, - &Token { - kind: TokenKind::Plain, - .. - } - )); + assert_eq!(fourth.kind, TokenKind::Indentation(6)); - assert!(matches!(fourth.as_input_str(), "there")); + let fifth = tokens.get(4).unwrap(); + assert_eq!(fifth.kind, TokenKind::Plain); + assert_eq!(fifth.as_input_str(), "there"); } #[test] @@ -359,30 +376,20 @@ mod tests { let tokens: Vec<_> = dbg!(TokenStream::tokenize(input).collect()); - assert_eq!(tokens.len(), 2); + assert_eq!(tokens.len(), 3); let first = tokens.first().unwrap(); - assert!(matches!( - first, - &Token { - kind: TokenKind::Plain, - .. - } - )); + assert_eq!(first.kind, TokenKind::Plain); - // newline is not present assert_eq!(first.as_input_str(), "hello"); let second = tokens.get(1).unwrap(); - assert!(matches!( - second, - &Token { - kind: TokenKind::Plain, - .. - } - )); + assert_eq!(second.kind, TokenKind::Newline); + + let third = tokens.get(2).unwrap(); + assert_eq!(third.kind, TokenKind::Plain); - assert_eq!(second.as_input_str(), "there"); + assert_eq!(third.as_input_str(), "there"); } #[test] @@ -391,30 +398,20 @@ mod tests { let tokens: Vec<_> = dbg!(TokenStream::tokenize(input).collect()); - assert_eq!(tokens.len(), 2); + assert_eq!(tokens.len(), 3); let first = tokens.first().unwrap(); - assert!(matches!( - first, - &Token { - kind: TokenKind::Plain, - .. - } - )); + assert!(first.kind == TokenKind::Plain); - // newline is not present assert_eq!(first.as_input_str(), "hello"); let second = tokens.get(1).unwrap(); - assert!(matches!( - second, - &Token { - kind: TokenKind::Plain, - .. - } - )); + assert!(second.kind == TokenKind::Newline); + + let third = tokens.get(2).unwrap(); + assert!(third.kind == TokenKind::Plain); - assert_eq!(second.as_input_str(), "there"); + assert_eq!(third.as_input_str(), "there"); } #[test] @@ -423,30 +420,20 @@ mod tests { let tokens: Vec<_> = dbg!(TokenStream::tokenize(input).collect()); - assert_eq!(tokens.len(), 2); + assert_eq!(tokens.len(), 3); let first = tokens.first().unwrap(); - assert!(matches!( - first, - &Token { - kind: TokenKind::Plain, - .. - } - )); + assert!(first.kind == TokenKind::Plain); - // newline is not present assert_eq!(first.as_input_str(), "hello"); let second = tokens.get(1).unwrap(); - assert!(matches!( - second, - &Token { - kind: TokenKind::Plain, - .. - } - )); + assert!(dbg!(second.kind) == TokenKind::Newline); + + let third = tokens.get(2).unwrap(); + assert!(third.kind == TokenKind::Plain); - assert_eq!(second.as_input_str(), "there"); + assert_eq!(third.as_input_str(), "there"); } #[test] diff --git a/frontend/src/lib.rs b/frontend/src/lib.rs index ddacd708..0437f0b0 100644 --- a/frontend/src/lib.rs +++ b/frontend/src/lib.rs @@ -1,4 +1,5 @@ pub mod lexer; +pub mod parser; mod scanner; pub mod span; mod symbol; diff --git a/frontend/src/parser/block/bulletlist.rs b/frontend/src/parser/block/bulletlist.rs new file mode 100644 index 00000000..fa7ff2db --- /dev/null +++ b/frontend/src/parser/block/bulletlist.rs @@ -0,0 +1,34 @@ +use crate::span::Span; + +/// Enum representing the keyword used to create a [`BulletListEntry`]. +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum BulletListEntryKeyword { + /// Minus keyword: `-` + Minus, + /// Plus keyword: `+` + Plus, + /// Star keyword: `*` + Star, +} + +/// Structure of a Unimarkup bullet list entry. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct BulletListEntry { + /// The [`BulletListEntryKeyword`] used to create this entry. + pub keyword: BulletListEntryKeyword, + /// The entry heading content of this entry. + pub heading: Vec, + /// The body of this entry. + pub body: Vec, + /// The span this element occupies in the Unimarkup input. + pub span: Span, +} + +/// Structure of a Unimarkup bullet list element. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct BulletList { + /// The list entries of this bullet list. + pub entries: Vec, + /// The span this element occupies in the Unimarkup input. + pub span: Span, +} diff --git a/frontend/src/parser/block/heading.rs b/frontend/src/parser/block/heading.rs new file mode 100644 index 00000000..18555e2b --- /dev/null +++ b/frontend/src/parser/block/heading.rs @@ -0,0 +1,72 @@ +use crate::span::Span; + +/// Enum of possible heading levels for unimarkup headings +#[derive(Eq, PartialEq, Debug, strum_macros::Display, strum_macros::EnumString, Clone, Copy)] +#[strum(serialize_all = "kebab-case")] +pub enum HeadingLevel { + /// Heading level 1, corresponds to `# ` in Unimarkup. + #[strum(serialize = "level-1")] + Level1 = 1, // start counting from 0 + + /// Heading level 2, corresponds to `## ` in Unimarkup. + #[strum(serialize = "level-2")] + Level2, + + /// Heading level 3, corresponds to `### ` in Unimarkup. + #[strum(serialize = "level-3")] + Level3, + + /// Heading level 4, corresponds to `#### ` in Unimarkup. + #[strum(serialize = "level-4")] + Level4, + + /// Heading level 5, corresponds to `##### ` in Unimarkup. + #[strum(serialize = "level-5")] + Level5, + + /// Heading level 6, corresponds to `###### ` in Unimarkup. + #[strum(serialize = "level-6")] + Level6, +} + +impl TryFrom for HeadingLevel { + type Error = String; + + fn try_from(value: u32) -> Result { + let level = match value { + 1 => HeadingLevel::Level1, + 2 => HeadingLevel::Level2, + 3 => HeadingLevel::Level3, + 4 => HeadingLevel::Level4, + 5 => HeadingLevel::Level5, + 6 => HeadingLevel::Level6, + other => return Err(format!("Invalid heading level: {other}")), + }; + + Ok(level) + } +} + +impl From for u8 { + fn from(value: HeadingLevel) -> Self { + value as u8 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Heading { + /// Unique identifier for a heading. + pub id: String, + + /// Heading level. + pub level: HeadingLevel, + + /// The content of the heading line. + pub content: Vec, + + /// Attributes of the heading. + pub attributes: Option, + + /// The span this element occupies in the Unimarkup input. + pub span: Span, +} diff --git a/frontend/src/parser/block/mod.rs b/frontend/src/parser/block/mod.rs new file mode 100644 index 00000000..4b8c27df --- /dev/null +++ b/frontend/src/parser/block/mod.rs @@ -0,0 +1,22 @@ +use crate::span::Span; + +pub mod bulletlist; +pub mod heading; +pub mod paragraph; +pub mod verbatim; + +/// Generic enum for all Unimarkup block elements. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Block { + /// Represents one blankline. + /// Needed in contexts where newlines must be kept. + Blankline(Span), + /// Represents the heading block + Heading(heading::Heading), + /// Represents the paragraph block + Paragraph(paragraph::Paragraph), + /// Represents the verbatim block + Verbatim(verbatim::Verbatim), + /// Represents the bullet list block + BulletList(bulletlist::BulletList), +} diff --git a/frontend/src/parser/block/paragraph.rs b/frontend/src/parser/block/paragraph.rs new file mode 100644 index 00000000..6843eee6 --- /dev/null +++ b/frontend/src/parser/block/paragraph.rs @@ -0,0 +1,11 @@ +use crate::span::Span; + +/// Structure of a Unimarkup paragraph element. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Paragraph { + /// The content of the paragraph. + pub content: Vec, + + /// The span this element occupies in the Unimarkup input. + pub span: Span, +} diff --git a/frontend/src/parser/block/verbatim.rs b/frontend/src/parser/block/verbatim.rs new file mode 100644 index 00000000..00be1ad7 --- /dev/null +++ b/frontend/src/parser/block/verbatim.rs @@ -0,0 +1,19 @@ +use crate::span::Span; + +/// Structure of a Unimarkup verbatim block element. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Verbatim { + /// The content of the verbatim block. + pub content: String, + /// The language used to highlight the content. + pub data_lang: Option, + /// Attributes of the verbatim block. + // TODO: make attributes data structure + pub attributes: Option, + /// Marks that this verbatim block was implicitly closed. + pub implicit_closed: bool, + /// The number of backticks this verbatim block was created with. + pub tick_len: usize, + /// The span this element occupies in the Unimarkup input. + pub span: Span, +} diff --git a/frontend/src/parser/mod.rs b/frontend/src/parser/mod.rs new file mode 100644 index 00000000..a8b2f59e --- /dev/null +++ b/frontend/src/parser/mod.rs @@ -0,0 +1,208 @@ +use crate::{ + lexer::{token::Token, token_kind::TokenKind, TokenStream}, + span::Span, +}; + +pub mod block; + +use block::{ + heading::{Heading, HeadingLevel}, + paragraph::Paragraph, + Block, +}; +use ribbon::{Enroll, Ribbon, Tape}; + +pub struct Parser<'input> { + /// Iterator that returns tokens found in the Unimarkup input. + tokens: Tape>, +} + +pub fn parse(input: &str) -> Parser<'_> { + Parser { + tokens: TokenStream::tokenize(input).tape(), + } +} + +impl Iterator for Parser<'_> { + type Item = Block; + + fn next(&mut self) -> Option { + loop { + let token = self.tokens.next()?; + + // make sure we can peek next token. + self.tokens.expand(); + + match token.kind { + TokenKind::Hash(count) => { + self.tokens.expand(); + + let is_next_whitespace = self + .tokens + .peek_front() + .map(|token| token.kind == TokenKind::Whitespace) + .unwrap_or(false); + + match HeadingLevel::try_from(count) { + Ok(level) if is_next_whitespace => { + return self.parse_heading(level).map(Block::Heading); + } + _ => return self.parse_paragraph(Some(token)), + } + } + + TokenKind::Blankline => { + continue; + } + + _other => return self.parse_paragraph(Some(token)), + // TokenKind::Star(_) => todo!(), + // TokenKind::Minus(_) => todo!(), + // TokenKind::Plus(_) => todo!(), + // TokenKind::Underline(_) => todo!(), + // TokenKind::Caret(_) => todo!(), + // TokenKind::Tick(_) => todo!(), + // TokenKind::Pipe(_) => todo!(), + // TokenKind::Tilde(_) => todo!(), + // TokenKind::Quote(_) => todo!(), + // TokenKind::Dollar(_) => todo!(), + // TokenKind::Colon(_) => todo!(), + // TokenKind::Dot(_) => todo!(), + // TokenKind::Ampersand(_) => todo!(), + // TokenKind::Comma(_) => todo!(), + // TokenKind::OpenParenthesis => todo!(), + // TokenKind::CloseParenthesis => todo!(), + // TokenKind::OpenBracket => todo!(), + // TokenKind::CloseBracket => todo!(), + // TokenKind::OpenBrace => todo!(), + // TokenKind::CloseBrace => todo!(), + // TokenKind::Whitespace => todo!(), + // TokenKind::Newline => todo!(), + // TokenKind::Blankline => todo!(), + // TokenKind::Eoi => todo!(), + // TokenKind::Indentation(_) => todo!(), + // TokenKind::EscapedPlain => todo!(), + // TokenKind::EscapedWhitespace => todo!(), + // TokenKind::EscapedNewline => todo!(), + // TokenKind::Plain => todo!(), + // TokenKind::TerminalPunctuation => todo!(), + // TokenKind::Comment { implicit_close } => todo!(), + // TokenKind::DirectUri => todo!(), + // TokenKind::Any => todo!(), + // TokenKind::Space => todo!(), + // TokenKind::EnclosedBlockEnd => todo!(), + // TokenKind::PossibleAttributes => todo!(), + // TokenKind::PossibleDecorator => todo!(), + } + } + } +} + +impl Parser<'_> { + fn parse_heading(&mut self, level: HeadingLevel) -> Option { + let expected_indentation = (u8::from(level) + 1) as u32; + + self.tokens.expand_while(|token| match token.kind { + TokenKind::Indentation(indent_level) => indent_level == expected_indentation, + TokenKind::Blankline => false, + _other => true, + }); + + let mut content = String::with_capacity(self.tokens.len()); + let mut span: Option = None; + + let mut is_start_of_line = true; + + while let Some(token) = self.tokens.pop_front() { + if let Some(span) = span.as_mut() { + span.len += token.span.len; + } else { + span = Some(token.span); + } + + match token.kind { + TokenKind::Whitespace | TokenKind::Space | TokenKind::Indentation(_) + if is_start_of_line => + { + continue; + } + _ => content += token.as_input_str(), + } + + is_start_of_line = matches!(token.kind, TokenKind::Newline | TokenKind::Blankline) + } + + let span = span?; + + Some(Heading { + id: String::from("placeholder-id"), + level, + content: vec![content], + attributes: None, + span, + }) + } + + fn parse_paragraph(&mut self, first_token: Option>) -> Option { + self.tokens + .expand_while(|token| token.kind != TokenKind::Blankline); + + let tape_len = self.tokens.len(); + let tape_iter = std::iter::from_fn(|| self.tokens.pop_front()); + + let mut content = String::with_capacity(tape_len); + let mut span: Option = None; + + for token in std::iter::once(first_token).flatten().chain(tape_iter) { + if let Some(span) = &mut span { + span.len += token.span.len; + } else { + span = Some(token.span); + } + + content += token.as_input_str(); + } + + let span = span?; + + Some(Block::Paragraph(Paragraph { + content: vec![content], + span, + })) + } +} + +#[cfg(test)] +mod tests { + use crate::parser::block::{heading::HeadingLevel, Block}; + + use super::parse; + + #[test] + fn parse_heading() { + let input = "## hello there!"; + + let heading_block = parse(input) + .next() + .expect("Should correctly parse heading!"); + + let Block::Heading(heading) = heading_block else { + panic!("Should correctly parse heading."); + }; + + assert_eq!(heading.level, HeadingLevel::Level2); + } + + #[test] + fn invalid_heading() { + let input = "##hello there!"; + + let block = parse(input) + .next() + .expect("Should correctly parse heading!"); + + let Block::Paragraph(_paragraph) = block else { + panic!("Should correctly parse heading."); + }; + } +} diff --git a/frontend/tests/lexer/snapshot.rs b/frontend/tests/lexer/snapshot.rs index 46b40d73..e18ba952 100644 --- a/frontend/tests/lexer/snapshot.rs +++ b/frontend/tests/lexer/snapshot.rs @@ -9,9 +9,16 @@ impl AsSnapshot for Snapshot> { fn as_snapshot(&self) -> String { let token = self.0; - let indent_len = get_indent(token.input, token.span.offs as usize); + let indent_len = crate::get_indent(token.input, token.span.offs); + + let mut orig_input = token.as_input_str(); + + if orig_input == "\n" { + orig_input = "␊"; + } else if orig_input == "\r\n" { + orig_input = "␍"; + } - let orig_input = token.as_input_str(); let marker = "^".repeat(token.span.len as usize); let indent = " ".repeat(indent_len); let kind = Snapshot(token.kind).as_snapshot(); @@ -30,6 +37,7 @@ impl AsSnapshot for Snapshot> { impl AsSnapshot for Snapshot { fn as_snapshot(&self) -> String { + #[allow(clippy::useless_format)] match self.0 { TokenKind::Star(r) => format!("Star({r})"), TokenKind::Hash(r) => format!("Hash({r})"), @@ -78,11 +86,3 @@ impl AsSnapshot for Snapshot { } } } - -fn get_indent(input: &str, offs: usize) -> usize { - input[0..offs] - .bytes() - .rev() - .position(|byte| byte == b'\n') - .unwrap_or(offs) -} diff --git a/frontend/tests/parser/mod.rs b/frontend/tests/parser/mod.rs new file mode 100644 index 00000000..88ae3a64 --- /dev/null +++ b/frontend/tests/parser/mod.rs @@ -0,0 +1,60 @@ +use std::{fmt::Write, panic}; + +use libtest_mimic::Trial; +use unimarkup_commons::test_runner::{ + self, as_snapshot::AsSnapshot, snap_test_runner::SnapTestRunner, +}; + +use crate::snapshot::Snapshot; + +mod snapshot; + +pub(crate) fn collect_snapshot_tests() -> Vec { + let tests_path = unimarkup_commons::crate_tests_path!(); + let test_cases = test_runner::collect_tests( + tests_path.join("spec/markup"), + tests_path.join("spec/snapshots/parser"), + "markup", + ); + + let mut test_runs = Vec::with_capacity(test_cases.len()); + + for case in test_cases { + let snap_test_name = format!("{}::snap::{}", module_path!(), case.test.name.as_str()); + + let snap_test_run = move || { + panic::catch_unwind(|| run_snap_test(case)).map_err(|err| { + let panic_msg = err + .downcast_ref::<&str>() + .unwrap_or(&"Panic message not available"); + + format!("Test case panicked: {}", panic_msg).into() + }) + }; + + test_runs.push(Trial::test(snap_test_name, snap_test_run)); + } + + test_runs +} + +fn run_snap_test(case: test_runner::test_file::TestCase) { + let runner = SnapTestRunner::with_fn(&case.test.name, &case.test.input, |input_str| { + let block_stream = unimarkup_frontend::parser::parse(input_str); + + let block_snaps = block_stream + .map(|block| Snapshot((input_str.as_str(), block))) + .fold(String::new(), |mut agg, snap| { + let _ = writeln!(&mut agg, "{}", snap.as_snapshot()); + agg + }); + + format!("{input_str}\n{block_snaps}") + }) + .with_info(format!( + "Test '{}' from '{}'", + case.test.name, case.file_path + )); + + unimarkup_commons::run_snap_test!(runner, &case.out_path); +} diff --git a/frontend/tests/parser/snapshot.rs b/frontend/tests/parser/snapshot.rs new file mode 100644 index 00000000..6b73a15e --- /dev/null +++ b/frontend/tests/parser/snapshot.rs @@ -0,0 +1,83 @@ +use std::fmt::Write; + +use unimarkup_commons::test_runner::as_snapshot::AsSnapshot; +use unimarkup_frontend::parser::block::{ + bulletlist::BulletList, heading::Heading, paragraph::Paragraph, verbatim::Verbatim, Block, +}; + +use crate::snapshot::Snapshot; + +impl AsSnapshot for Snapshot<(&str, Block)> { + fn as_snapshot(&self) -> String { + let (input, block) = &self.0; + + match block { + Block::Blankline(span) => { + format!("Blankline @ ({} -> {})", span.offs, span.offs + span.len) + } + Block::Heading(heading) => Snapshot((*input, heading)).as_snapshot(), + Block::Paragraph(paragraph) => Snapshot((*input, paragraph)).as_snapshot(), + Block::Verbatim(verbatim) => Snapshot((*input, verbatim)).as_snapshot(), + Block::BulletList(bullet_list) => Snapshot((*input, bullet_list)).as_snapshot(), + } + } +} + +impl AsSnapshot for Snapshot<(&str, &Heading)> { + fn as_snapshot(&self) -> String { + let (_, heading) = self.0; + + let mut output = String::with_capacity(heading.content.iter().map(|c| c.len()).sum()); + + let _ = writeln!( + &mut output, + "Heading({}) @ ({} -> {}) {{", + heading.level, + heading.span.offs, + heading.span.offs + heading.span.len + ); + + for line in heading.content.iter().flat_map(|s| s.lines()) { + let _ = writeln!(&mut output, " {line}"); + } + + let _ = writeln!(&mut output, "}}"); + + output + } +} + +impl AsSnapshot for Snapshot<(&str, &Paragraph)> { + fn as_snapshot(&self) -> String { + let (_, paragraph) = self.0; + + let mut output = String::with_capacity(paragraph.content.iter().map(|s| s.len()).sum()); + + let _ = writeln!( + &mut output, + "Paragraph @ ({} -> {}) {{", + paragraph.span.offs, + paragraph.span.offs + paragraph.span.len + ); + + for line in paragraph.content.iter().flat_map(|s| s.lines()) { + let _ = writeln!(&mut output, " {line}"); + } + + let _ = writeln!(&mut output, "}}"); + + output + } +} + +impl AsSnapshot for Snapshot<(&str, &Verbatim)> { + fn as_snapshot(&self) -> String { + todo!() + } +} + +impl AsSnapshot for Snapshot<(&str, &BulletList)> { + fn as_snapshot(&self) -> String { + todo!() + } +} diff --git a/frontend/tests/snapshots.rs b/frontend/tests/snapshots.rs index 317a3a04..d5732f4d 100644 --- a/frontend/tests/snapshots.rs +++ b/frontend/tests/snapshots.rs @@ -1,13 +1,28 @@ mod lexer; +mod parser; mod snapshot; use libtest_mimic::Arguments; // pub(crate) use snapshot::*; +fn get_indent(input: &str, offs: u32) -> usize { + input[0..offs as usize] + .bytes() + .rev() + .position(|byte| byte == b'\n') + .unwrap_or(offs as usize) +} + fn main() { let args = Arguments::from_args(); let lexer_tests = lexer::collect_snapshot_tests(); + let _parser_tests = parser::collect_snapshot_tests(); + + let tests = lexer_tests + .into_iter() + .chain(_parser_tests) + .collect::>(); - libtest_mimic::run(&args, lexer_tests).exit(); + libtest_mimic::run(&args, tests).exit(); } diff --git a/frontend/tests/spec/markup/block/heading.yml b/frontend/tests/spec/markup/block/heading.yml new file mode 100644 index 00000000..c1a9bff6 --- /dev/null +++ b/frontend/tests/spec/markup/block/heading.yml @@ -0,0 +1,48 @@ +# Unimarkup specification version +spec: "0.0.1" + +name: heading-block +description: Test parsing of heading. + +tests: + - name: single-line-level-1 + description: | + Single line level 1 heading. + + input: | + # This is a simple heading. + + html: | +

This is a simple paragraph.

+ + - name: multi-line-level-1 + description: | + Heading over multiple lines. + + input: | + # This is a heading + in two lines. + + html: | +

This is a heading in two lines.

+ + - name: single-line-level-6 + description: | + Level 6 heading in a single line. + + input: | + ###### This is a heading. + + html: | +

This is a heading

+ + - name: multi-line-level-6 + description: | + Level 6 heading in multiple lines. + + input: | + ###### This is a heading + in multiple lines. + + html: | +

This is a heading in multiple lines.

diff --git a/frontend/tests/spec/markup/block/paragraph.yml b/frontend/tests/spec/markup/block/paragraph.yml new file mode 100644 index 00000000..7d7cedad --- /dev/null +++ b/frontend/tests/spec/markup/block/paragraph.yml @@ -0,0 +1,40 @@ +# Unimarkup specification version +spec: "0.0.1" + +name: paragraph-block +description: Test parsing of paragraph. + +tests: + - name: single-line + description: | + Single line paragraph. + + input: | + This is a simple paragraph. + + html: | +

This is a simple paragraph.

+ + - name: multi-line + description: | + Paragraph over multiple lines. + + input: | + This is a paragraph + that spans multiple + lines. + + html: | +

This is a paragraph that spans multiple lines.

+ + - name: two-paragraphs + description: | + Two paragraphs separated by a blank line. + + input: | + This is the first paragraph. + + And this should be the second one. + + html: | +

This is the first paragraph.

And this should be the second one.

diff --git a/frontend/tests/spec/snapshots/lexer/block/heading/multi-line-level-1.snap b/frontend/tests/spec/snapshots/lexer/block/heading/multi-line-level-1.snap new file mode 100644 index 00000000..4a1bae52 --- /dev/null +++ b/frontend/tests/spec/snapshots/lexer/block/heading/multi-line-level-1.snap @@ -0,0 +1,49 @@ +--- +source: frontend/tests/lexer/mod.rs +info: "Test 'multi-line-level-1' from 'markup/block/heading.yml'" +--- +# This is a heading + in two lines. + +# +^ - Hash(1) @ (0 -> 1) + + ^ - Whitespace @ (1 -> 2) + This + ^^^^ - Plain @ (2 -> 6) + + ^ - Whitespace @ (6 -> 7) + is + ^^ - Plain @ (7 -> 9) + + ^ - Whitespace @ (9 -> 10) + a + ^ - Plain @ (10 -> 11) + + ^ - Whitespace @ (11 -> 12) + heading + ^^^^^^^ - Plain @ (12 -> 19) + ␊ + ^ - Newline @ (19 -> 20) + +^^ - Indentation(2) @ (20 -> 22) + in + ^^ - Plain @ (22 -> 24) + + ^ - Whitespace @ (24 -> 25) + two + ^^^ - Plain @ (25 -> 28) + + ^ - Whitespace @ (28 -> 29) + lines + ^^^^^ - Plain @ (29 -> 34) + . + ^ - Dot(1) @ (34 -> 35) + ␊ + ^ - Newline @ (35 -> 36) + +--- +With input: + +# This is a heading + in two lines. diff --git a/frontend/tests/spec/snapshots/lexer/block/heading/multi-line-level-6.snap b/frontend/tests/spec/snapshots/lexer/block/heading/multi-line-level-6.snap new file mode 100644 index 00000000..706559ac --- /dev/null +++ b/frontend/tests/spec/snapshots/lexer/block/heading/multi-line-level-6.snap @@ -0,0 +1,49 @@ +--- +source: frontend/tests/lexer/mod.rs +info: "Test 'multi-line-level-6' from 'markup/block/heading.yml'" +--- +###### This is a heading + in multiple lines. + +###### +^^^^^^ - Hash(6) @ (0 -> 6) + + ^ - Whitespace @ (6 -> 7) + This + ^^^^ - Plain @ (7 -> 11) + + ^ - Whitespace @ (11 -> 12) + is + ^^ - Plain @ (12 -> 14) + + ^ - Whitespace @ (14 -> 15) + a + ^ - Plain @ (15 -> 16) + + ^ - Whitespace @ (16 -> 17) + heading + ^^^^^^^ - Plain @ (17 -> 24) + ␊ + ^ - Newline @ (24 -> 25) + +^^^^^^^ - Indentation(7) @ (25 -> 32) + in + ^^ - Plain @ (32 -> 34) + + ^ - Whitespace @ (34 -> 35) + multiple + ^^^^^^^^ - Plain @ (35 -> 43) + + ^ - Whitespace @ (43 -> 44) + lines + ^^^^^ - Plain @ (44 -> 49) + . + ^ - Dot(1) @ (49 -> 50) + ␊ + ^ - Newline @ (50 -> 51) + +--- +With input: + +###### This is a heading + in multiple lines. diff --git a/frontend/tests/spec/snapshots/lexer/block/heading/single-line-level-1.snap b/frontend/tests/spec/snapshots/lexer/block/heading/single-line-level-1.snap new file mode 100644 index 00000000..36cfdc51 --- /dev/null +++ b/frontend/tests/spec/snapshots/lexer/block/heading/single-line-level-1.snap @@ -0,0 +1,37 @@ +--- +source: frontend/tests/lexer/mod.rs +info: "Test 'single-line-level-1' from 'markup/block/heading.yml'" +--- +# This is a simple heading. + +# +^ - Hash(1) @ (0 -> 1) + + ^ - Whitespace @ (1 -> 2) + This + ^^^^ - Plain @ (2 -> 6) + + ^ - Whitespace @ (6 -> 7) + is + ^^ - Plain @ (7 -> 9) + + ^ - Whitespace @ (9 -> 10) + a + ^ - Plain @ (10 -> 11) + + ^ - Whitespace @ (11 -> 12) + simple + ^^^^^^ - Plain @ (12 -> 18) + + ^ - Whitespace @ (18 -> 19) + heading + ^^^^^^^ - Plain @ (19 -> 26) + . + ^ - Dot(1) @ (26 -> 27) + ␊ + ^ - Newline @ (27 -> 28) + +--- +With input: + +# This is a simple heading. diff --git a/frontend/tests/spec/snapshots/lexer/block/heading/single-line-level-6.snap b/frontend/tests/spec/snapshots/lexer/block/heading/single-line-level-6.snap new file mode 100644 index 00000000..805050b0 --- /dev/null +++ b/frontend/tests/spec/snapshots/lexer/block/heading/single-line-level-6.snap @@ -0,0 +1,33 @@ +--- +source: frontend/tests/lexer/mod.rs +info: "Test 'single-line-level-6' from 'markup/block/heading.yml'" +--- +###### This is a heading. + +###### +^^^^^^ - Hash(6) @ (0 -> 6) + + ^ - Whitespace @ (6 -> 7) + This + ^^^^ - Plain @ (7 -> 11) + + ^ - Whitespace @ (11 -> 12) + is + ^^ - Plain @ (12 -> 14) + + ^ - Whitespace @ (14 -> 15) + a + ^ - Plain @ (15 -> 16) + + ^ - Whitespace @ (16 -> 17) + heading + ^^^^^^^ - Plain @ (17 -> 24) + . + ^ - Dot(1) @ (24 -> 25) + ␊ + ^ - Newline @ (25 -> 26) + +--- +With input: + +###### This is a heading. diff --git a/frontend/tests/spec/snapshots/lexer/block/paragraph/multi-line.snap b/frontend/tests/spec/snapshots/lexer/block/paragraph/multi-line.snap new file mode 100644 index 00000000..f0703879 --- /dev/null +++ b/frontend/tests/spec/snapshots/lexer/block/paragraph/multi-line.snap @@ -0,0 +1,49 @@ +--- +source: frontend/tests/lexer/mod.rs +info: "Test 'multi-line' from 'markup/block/paragraph.yml'" +--- +This is a paragraph +that spans multiple +lines. + +This +^^^^ - Plain @ (0 -> 4) + + ^ - Whitespace @ (4 -> 5) + is + ^^ - Plain @ (5 -> 7) + + ^ - Whitespace @ (7 -> 8) + a + ^ - Plain @ (8 -> 9) + + ^ - Whitespace @ (9 -> 10) + paragraph + ^^^^^^^^^ - Plain @ (10 -> 19) + ␊ + ^ - Newline @ (19 -> 20) +that +^^^^ - Plain @ (20 -> 24) + + ^ - Whitespace @ (24 -> 25) + spans + ^^^^^ - Plain @ (25 -> 30) + + ^ - Whitespace @ (30 -> 31) + multiple + ^^^^^^^^ - Plain @ (31 -> 39) + ␊ + ^ - Newline @ (39 -> 40) +lines +^^^^^ - Plain @ (40 -> 45) + . + ^ - Dot(1) @ (45 -> 46) + ␊ + ^ - Newline @ (46 -> 47) + +--- +With input: + +This is a paragraph +that spans multiple +lines. diff --git a/frontend/tests/spec/snapshots/lexer/block/paragraph/single-line.snap b/frontend/tests/spec/snapshots/lexer/block/paragraph/single-line.snap new file mode 100644 index 00000000..a7408917 --- /dev/null +++ b/frontend/tests/spec/snapshots/lexer/block/paragraph/single-line.snap @@ -0,0 +1,33 @@ +--- +source: frontend/tests/lexer/mod.rs +info: "Test 'single-line' from 'markup/block/paragraph.yml'" +--- +This is a simple paragraph. + +This +^^^^ - Plain @ (0 -> 4) + + ^ - Whitespace @ (4 -> 5) + is + ^^ - Plain @ (5 -> 7) + + ^ - Whitespace @ (7 -> 8) + a + ^ - Plain @ (8 -> 9) + + ^ - Whitespace @ (9 -> 10) + simple + ^^^^^^ - Plain @ (10 -> 16) + + ^ - Whitespace @ (16 -> 17) + paragraph + ^^^^^^^^^ - Plain @ (17 -> 26) + . + ^ - Dot(1) @ (26 -> 27) + ␊ + ^ - Newline @ (27 -> 28) + +--- +With input: + +This is a simple paragraph. diff --git a/frontend/tests/spec/snapshots/lexer/block/paragraph/two-paragraphs.snap b/frontend/tests/spec/snapshots/lexer/block/paragraph/two-paragraphs.snap new file mode 100644 index 00000000..8eb8e138 --- /dev/null +++ b/frontend/tests/spec/snapshots/lexer/block/paragraph/two-paragraphs.snap @@ -0,0 +1,69 @@ +--- +source: frontend/tests/lexer/mod.rs +info: "Test 'two-paragraphs' from 'markup/block/paragraph.yml'" +--- +This is the first paragraph. + +And this should be the second one. + +This +^^^^ - Plain @ (0 -> 4) + + ^ - Whitespace @ (4 -> 5) + is + ^^ - Plain @ (5 -> 7) + + ^ - Whitespace @ (7 -> 8) + the + ^^^ - Plain @ (8 -> 11) + + ^ - Whitespace @ (11 -> 12) + first + ^^^^^ - Plain @ (12 -> 17) + + ^ - Whitespace @ (17 -> 18) + paragraph + ^^^^^^^^^ - Plain @ (18 -> 27) + . + ^ - Dot(1) @ (27 -> 28) + ␊ + ^ - Newline @ (28 -> 29) +␊ +^ - Blankline @ (29 -> 30) +And +^^^ - Plain @ (30 -> 33) + + ^ - Whitespace @ (33 -> 34) + this + ^^^^ - Plain @ (34 -> 38) + + ^ - Whitespace @ (38 -> 39) + should + ^^^^^^ - Plain @ (39 -> 45) + + ^ - Whitespace @ (45 -> 46) + be + ^^ - Plain @ (46 -> 48) + + ^ - Whitespace @ (48 -> 49) + the + ^^^ - Plain @ (49 -> 52) + + ^ - Whitespace @ (52 -> 53) + second + ^^^^^^ - Plain @ (53 -> 59) + + ^ - Whitespace @ (59 -> 60) + one + ^^^ - Plain @ (60 -> 63) + . + ^ - Dot(1) @ (63 -> 64) + ␊ + ^ - Newline @ (64 -> 65) + +--- +With input: + +This is the first paragraph. + +And this should be the second one. diff --git a/frontend/tests/spec/snapshots/lexer/bold/ambiguous-close.snap b/frontend/tests/spec/snapshots/lexer/bold/ambiguous-close.snap index b5ab4609..6d5c1659 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/ambiguous-close.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/ambiguous-close.snap @@ -22,6 +22,8 @@ info: "Test 'ambiguous-close' from 'markup/bold.yml'" ^ - Whitespace @ (18 -> 19) plain ^^^^^ - Plain @ (19 -> 24) + ␊ + ^ - Newline @ (24 -> 25) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/ambiguous-end.snap b/frontend/tests/spec/snapshots/lexer/bold/ambiguous-end.snap index 52c000f7..63b1f52d 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/ambiguous-end.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/ambiguous-end.snap @@ -28,6 +28,8 @@ The ^^^^ - Plain @ (22 -> 26) . ^ - Dot(1) @ (26 -> 27) + ␊ + ^ - Newline @ (27 -> 28) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/ambiguous-start.snap b/frontend/tests/spec/snapshots/lexer/bold/ambiguous-start.snap index b3600c3c..2cd7af69 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/ambiguous-start.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/ambiguous-start.snap @@ -28,6 +28,8 @@ The ^^^^^^^^^^ - Plain @ (22 -> 32) . ^ - Dot(1) @ (32 -> 33) + ␊ + ^ - Newline @ (33 -> 34) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/bold-in-middle.snap b/frontend/tests/spec/snapshots/lexer/bold/bold-in-middle.snap index 79947b14..a1fd7db9 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/bold-in-middle.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/bold-in-middle.snap @@ -28,6 +28,8 @@ The ^^^^ - Plain @ (21 -> 25) . ^ - Dot(1) @ (25 -> 26) + ␊ + ^ - Newline @ (26 -> 27) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/bold-not-bold.snap b/frontend/tests/spec/snapshots/lexer/bold/bold-not-bold.snap index 07d63b88..beff2e2e 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/bold-not-bold.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/bold-not-bold.snap @@ -20,6 +20,8 @@ info: "Test 'bold-not-bold' from 'markup/bold.yml'" ^^^^ - Plain @ (13 -> 17) . ^ - Dot(1) @ (17 -> 18) + ␊ + ^ - Newline @ (18 -> 19) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/escaped-bold.snap b/frontend/tests/spec/snapshots/lexer/bold/escaped-bold.snap index b817fd7e..accaa4bb 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/escaped-bold.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/escaped-bold.snap @@ -18,6 +18,8 @@ info: "Test 'escaped-bold' from 'markup/bold.yml'" ^^ - Plain @ (12 -> 14) \* ^^ - Plain @ (14 -> 16) + ␊ + ^ - Newline @ (16 -> 17) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/implicit-closed-bold.snap b/frontend/tests/spec/snapshots/lexer/bold/implicit-closed-bold.snap index 0b54b420..cd3fde1e 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/implicit-closed-bold.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/implicit-closed-bold.snap @@ -18,6 +18,8 @@ info: "Test 'implicit-closed-bold' from 'markup/bold.yml'" ^^ - Star(2) @ (16 -> 18) close ^^^^^ - Plain @ (18 -> 23) + ␊ + ^ - Newline @ (23 -> 24) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/not-bold.snap b/frontend/tests/spec/snapshots/lexer/bold/not-bold.snap index 33b9ed45..d856065e 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/not-bold.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/not-bold.snap @@ -18,6 +18,8 @@ info: "Test 'not-bold' from 'markup/bold.yml'" ^ - Whitespace @ (11 -> 12) ** ^^ - Star(2) @ (12 -> 14) + ␊ + ^ - Newline @ (14 -> 15) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/not-opened-bold.snap b/frontend/tests/spec/snapshots/lexer/bold/not-opened-bold.snap index 2f38fa06..99052678 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/not-opened-bold.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/not-opened-bold.snap @@ -16,6 +16,8 @@ info: "Test 'not-opened-bold' from 'markup/bold.yml'" ^^^^ - Plain @ (7 -> 11) ** ^^ - Star(2) @ (11 -> 13) + ␊ + ^ - Newline @ (13 -> 14) --- With input: diff --git a/frontend/tests/spec/snapshots/lexer/bold/simple-bold.snap b/frontend/tests/spec/snapshots/lexer/bold/simple-bold.snap index 37acd96b..63c5fd54 100644 --- a/frontend/tests/spec/snapshots/lexer/bold/simple-bold.snap +++ b/frontend/tests/spec/snapshots/lexer/bold/simple-bold.snap @@ -10,6 +10,8 @@ info: "Test 'simple-bold' from 'markup/bold.yml'" ^^^^ - Plain @ (2 -> 6) ** ^^ - Star(2) @ (6 -> 8) + ␊ + ^ - Newline @ (8 -> 9) --- With input: diff --git a/frontend/tests/spec/snapshots/parser/block/heading/multi-line-level-1.snap b/frontend/tests/spec/snapshots/parser/block/heading/multi-line-level-1.snap new file mode 100644 index 00000000..f531607b --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/block/heading/multi-line-level-1.snap @@ -0,0 +1,18 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'multi-line-level-1' from 'markup/block/heading.yml'" +--- +# This is a heading + in two lines. + +Heading(level-1) @ (1 -> 36) { + This is a heading + in two lines. +} + + +--- +With input: + +# This is a heading + in two lines. diff --git a/frontend/tests/spec/snapshots/parser/block/heading/multi-line-level-6.snap b/frontend/tests/spec/snapshots/parser/block/heading/multi-line-level-6.snap new file mode 100644 index 00000000..dc6fb884 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/block/heading/multi-line-level-6.snap @@ -0,0 +1,18 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'multi-line-level-6' from 'markup/block/heading.yml'" +--- +###### This is a heading + in multiple lines. + +Heading(level-6) @ (6 -> 51) { + This is a heading + in multiple lines. +} + + +--- +With input: + +###### This is a heading + in multiple lines. diff --git a/frontend/tests/spec/snapshots/parser/block/heading/single-line-level-1.snap b/frontend/tests/spec/snapshots/parser/block/heading/single-line-level-1.snap new file mode 100644 index 00000000..6f5f816a --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/block/heading/single-line-level-1.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'single-line-level-1' from 'markup/block/heading.yml'" +--- +# This is a simple heading. + +Heading(level-1) @ (1 -> 28) { + This is a simple heading. +} + + +--- +With input: + +# This is a simple heading. diff --git a/frontend/tests/spec/snapshots/parser/block/heading/single-line-level-6.snap b/frontend/tests/spec/snapshots/parser/block/heading/single-line-level-6.snap new file mode 100644 index 00000000..7738a08b --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/block/heading/single-line-level-6.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'single-line-level-6' from 'markup/block/heading.yml'" +--- +###### This is a heading. + +Heading(level-6) @ (6 -> 26) { + This is a heading. +} + + +--- +With input: + +###### This is a heading. diff --git a/frontend/tests/spec/snapshots/parser/block/paragraph/multi-line.snap b/frontend/tests/spec/snapshots/parser/block/paragraph/multi-line.snap new file mode 100644 index 00000000..7128fdd1 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/block/paragraph/multi-line.snap @@ -0,0 +1,21 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'multi-line' from 'markup/block/paragraph.yml'" +--- +This is a paragraph +that spans multiple +lines. + +Paragraph @ (0 -> 47) { + This is a paragraph + that spans multiple + lines. +} + + +--- +With input: + +This is a paragraph +that spans multiple +lines. diff --git a/frontend/tests/spec/snapshots/parser/block/paragraph/single-line.snap b/frontend/tests/spec/snapshots/parser/block/paragraph/single-line.snap new file mode 100644 index 00000000..9bad6a31 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/block/paragraph/single-line.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'single-line' from 'markup/block/paragraph.yml'" +--- +This is a simple paragraph. + +Paragraph @ (0 -> 28) { + This is a simple paragraph. +} + + +--- +With input: + +This is a simple paragraph. diff --git a/frontend/tests/spec/snapshots/parser/block/paragraph/two-paragraphs.snap b/frontend/tests/spec/snapshots/parser/block/paragraph/two-paragraphs.snap new file mode 100644 index 00000000..a7528790 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/block/paragraph/two-paragraphs.snap @@ -0,0 +1,23 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'two-paragraphs' from 'markup/block/paragraph.yml'" +--- +This is the first paragraph. + +And this should be the second one. + +Paragraph @ (0 -> 29) { + This is the first paragraph. +} + +Paragraph @ (30 -> 65) { + And this should be the second one. +} + + +--- +With input: + +This is the first paragraph. + +And this should be the second one. diff --git a/frontend/tests/spec/snapshots/parser/bold/ambiguous-close.snap b/frontend/tests/spec/snapshots/parser/bold/ambiguous-close.snap new file mode 100644 index 00000000..864107d9 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/ambiguous-close.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'ambiguous-close' from 'markup/bold.yml'" +--- +**bold *+italic*** plain + +Paragraph @ (0 -> 25) { + **bold *+italic*** plain +} + + +--- +With input: + +**bold *+italic*** plain diff --git a/frontend/tests/spec/snapshots/parser/bold/ambiguous-end.snap b/frontend/tests/spec/snapshots/parser/bold/ambiguous-end.snap new file mode 100644 index 00000000..df6489a8 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/ambiguous-end.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'ambiguous-end' from 'markup/bold.yml'" +--- +The next **word*** is bold. + +Paragraph @ (0 -> 28) { + The next **word*** is bold. +} + + +--- +With input: + +The next **word*** is bold. diff --git a/frontend/tests/spec/snapshots/parser/bold/ambiguous-start.snap b/frontend/tests/spec/snapshots/parser/bold/ambiguous-start.snap new file mode 100644 index 00000000..7f0c2772 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/ambiguous-start.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'ambiguous-start' from 'markup/bold.yml'" +--- +The next ***word** is bolditalic. + +Paragraph @ (0 -> 34) { + The next ***word** is bolditalic. +} + + +--- +With input: + +The next ***word** is bolditalic. diff --git a/frontend/tests/spec/snapshots/parser/bold/bold-in-middle.snap b/frontend/tests/spec/snapshots/parser/bold/bold-in-middle.snap new file mode 100644 index 00000000..bbd33c97 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/bold-in-middle.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'bold-in-middle' from 'markup/bold.yml'" +--- +The next **word** is bold. + +Paragraph @ (0 -> 27) { + The next **word** is bold. +} + + +--- +With input: + +The next **word** is bold. diff --git a/frontend/tests/spec/snapshots/parser/bold/bold-not-bold.snap b/frontend/tests/spec/snapshots/parser/bold/bold-not-bold.snap new file mode 100644 index 00000000..4a50af94 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/bold-not-bold.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'bold-not-bold' from 'markup/bold.yml'" +--- +**Bold** not bold. + +Paragraph @ (0 -> 19) { + **Bold** not bold. +} + + +--- +With input: + +**Bold** not bold. diff --git a/frontend/tests/spec/snapshots/parser/bold/escaped-bold.snap b/frontend/tests/spec/snapshots/parser/bold/escaped-bold.snap new file mode 100644 index 00000000..8a5d41cc --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/escaped-bold.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'escaped-bold' from 'markup/bold.yml'" +--- +\*\*not bold\*\* + +Paragraph @ (0 -> 17) { + \*\*not bold\*\* +} + + +--- +With input: + +\*\*not bold\*\* diff --git a/frontend/tests/spec/snapshots/parser/bold/implicit-closed-bold.snap b/frontend/tests/spec/snapshots/parser/bold/implicit-closed-bold.snap new file mode 100644 index 00000000..c645780a --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/implicit-closed-bold.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'implicit-closed-bold' from 'markup/bold.yml'" +--- +**implicit bold **close + +Paragraph @ (0 -> 24) { + **implicit bold **close +} + + +--- +With input: + +**implicit bold **close diff --git a/frontend/tests/spec/snapshots/parser/bold/not-bold.snap b/frontend/tests/spec/snapshots/parser/bold/not-bold.snap new file mode 100644 index 00000000..72f54506 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/not-bold.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'not-bold' from 'markup/bold.yml'" +--- +** not bold ** + +Paragraph @ (0 -> 15) { + ** not bold ** +} + + +--- +With input: + +** not bold ** diff --git a/frontend/tests/spec/snapshots/parser/bold/not-opened-bold.snap b/frontend/tests/spec/snapshots/parser/bold/not-opened-bold.snap new file mode 100644 index 00000000..31f7dfe7 --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/not-opened-bold.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'not-opened-bold' from 'markup/bold.yml'" +--- +** not bold** + +Paragraph @ (0 -> 14) { + ** not bold** +} + + +--- +With input: + +** not bold** diff --git a/frontend/tests/spec/snapshots/parser/bold/simple-bold.snap b/frontend/tests/spec/snapshots/parser/bold/simple-bold.snap new file mode 100644 index 00000000..2e20f13e --- /dev/null +++ b/frontend/tests/spec/snapshots/parser/bold/simple-bold.snap @@ -0,0 +1,15 @@ +--- +source: frontend/tests/parser/mod.rs +info: "Test 'simple-bold' from 'markup/bold.yml'" +--- +**Bold** + +Paragraph @ (0 -> 9) { + **Bold** +} + + +--- +With input: + +**Bold**