README.md

rustc_lexer_zig

Zig port of rustc_lexer

Rust 🦀	Zig ⚡
char	u21
&str	std.unicode.Utf8View
std::str::Chars	std.unicode.Utf8Iterator

Highlights

comptime + @unionInit

Rust 🦀	match first_char { // ... 'b' => self.c_or_byte_string( |terminated| ByteStr { terminated }, |n_hashes| RawByteStr { n_hashes }, Some(|terminated| Byte { terminated }), ), 'c' => self.c_or_byte_string( |terminated| CStr { terminated }, |n_hashes| RawCStr { n_hashes }, None, ), // ... }
Zig ⚡	switch (c) { // ... 'b' => self.cOrByteString("byte_str", "raw_byte_str", "byte"), 'c' => self.cOrByteString("c_str", "raw_c_str", null), // ... }

Rust 🦀	fn c_or_byte_string( &mut self, mk_kind: impl FnOnce(bool) -> LiteralKind, mk_kind_raw: impl FnOnce(Option<u8>) -> LiteralKind, single_quoted: Option<fn(bool) -> LiteralKind>, ) -> TokenKind {
Zig ⚡	fn cOrByteString( self: *Self, comptime kind_name: []const u8, comptime raw_kind_name: []const u8, comptime single_quoted: ?[]const u8, ) TokenKind {

Rust 🦀	let kind = mk_kind(terminated);
Zig ⚡	const kind = @unionInit(LiteralKind, kind_name, .{ .terminated = terminated });

Multiline String Literals

Rust 🦀	#[test] fn comment_flavors() { check_lexing( r" // line //// line as well /// outer doc line //! inner doc line /* block */ /**/ /*** also block */ /** outer doc block */ /*! inner doc block */ ", // ... ) }
Zig ⚡	test "comment flavors" { try checkLexing( \\// line \\//// line as well \\/// outer doc line \\//! inner doc line \\/* block */ \\/**/ \\/*** also block */ \\/** outer doc block */ \\/*! inner doc block */ , // ... ); }

Anonymous Struct Literals

Rust 🦀	#[test] fn smoke_test() { check_lexing( "/* my source file */ fn main() { println!(\"zebra\"); }\n", expect![[r#" Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 } Token { kind: Whitespace, len: 1 } ... "#]], ) }
Zig ⚡	test "smoke test" { try checkLexing( "/* my source file */ fn main() { println!(\"zebra\"); }\n", &.{ .{ .kind = .{ .block_comment = .{ .doc_style = null, .terminated = true } }, .len = 20 }, .{ .kind = .whitespace, .len = 1 }, // ... }, ); }