Skip to content

Commit

Permalink
Handle files with BOM in lexer (#285)
Browse files Browse the repository at this point in the history
This came up in OpenDream (blame VS...). Figured it wouldn't be too 
hard to fix here.
  • Loading branch information
PJB3005 authored Sep 18, 2021
1 parent 78bf0ee commit 7bff6f7
Showing 1 changed file with 13 additions and 1 deletion.
14 changes: 13 additions & 1 deletion src/dreammaker/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,10 @@ enum Directive {
Stringy,
}

fn has_bom(slice: &[u8]) -> bool {
slice.starts_with(b"\xEF\xBB\xBF")
}

fn buffer_read<R: Read>(file: FileId, mut read: R) -> Result<Vec<u8>, DMError> {
let mut buffer = Vec::new();

Expand Down Expand Up @@ -630,9 +634,17 @@ impl<'ctx> HasLocation for Lexer<'ctx> {
impl<'ctx> Lexer<'ctx> {
/// Create a new lexer from a byte stream.
pub fn new<I: Into<Cow<'ctx, [u8]>>>(context: &'ctx Context, file_number: FileId, input: I) -> Self {
let mut cow = input.into();
if has_bom(&cow) {
cow = match cow {
Cow::Borrowed(b) => Cow::from(&b[3..]),
Cow::Owned(mut o) => { o.drain(..3); Cow::Owned(o) }
};
}

Lexer {
context,
input: LocationTracker::new(file_number, input.into()),
input: LocationTracker::new(file_number, cow),
next: None,
final_newline: false,
at_line_head: true,
Expand Down

0 comments on commit 7bff6f7

Please sign in to comment.