barf_commonlang_scanner.reflex

// 2006.11.09 - Copyright Victor Dods - Licensed under Apache 2.0

%targets cpp

/* ***************************************************************************
/* cpp scanner header-file-related directives
/* **************************************************************************/

%target.cpp.header_filename "barf_commonlang_scanner.hpp"
%target.cpp.implementation_filename "barf_commonlang_scanner.cpp"
%target.cpp.top_of_header_file %{
#if !defined(BARF_COMMONLANG_SCANNER_HPP_)
#define BARF_COMMONLANG_SCANNER_HPP_

#include "barf_commonlang.hpp"

#include <ostream>

#include "barf_inputbase.hpp"

namespace Barf {
namespace Ast {

class Base;

} // end of namespace Ast

namespace CommonLang {
%}
%target.cpp.class_name Scanner
%target.cpp.class_inheritance { protected InputBase }
%target.cpp.top_of_class {
    struct Token
    {
        enum Type
        {
            BAD_END_OF_FILE = 0x100,
            BAD_TOKEN,
            CHAR_LITERAL,
            DIRECTIVE_ADD_CODESPEC,
            DIRECTIVE_ADD_OPTIONAL_DIRECTIVE,
            DIRECTIVE_ADD_REQUIRED_DIRECTIVE,
            DIRECTIVE_CASE_INSENSITIVE,
            DIRECTIVE_DEFAULT,
            DIRECTIVE_DEFAULT_PARSE_NONTERMINAL,
            DIRECTIVE_DUMB_CODE_BLOCK,
            DIRECTIVE_EMPTY,
            DIRECTIVE_END,
            DIRECTIVE_ERROR,
            DIRECTIVE_ID,
            DIRECTIVE_LOOKAHEAD,
            DIRECTIVE_MACRO,
            DIRECTIVE_NONTERMINAL,
            DIRECTIVE_PREC,
            DIRECTIVE_START_IN_STATE_MACHINE,
            DIRECTIVE_STATE_MACHINE,
            DIRECTIVE_STRICT_CODE_BLOCK,
            DIRECTIVE_STRING,
            DIRECTIVE_TARGET,
            DIRECTIVE_TARGETS,
            DIRECTIVE_TERMINAL,
            DIRECTIVE_TYPE,
            DIRECTIVE_UNGREEDY,
            DUMB_CODE_BLOCK,
            END_OF_FILE,
            END_PREAMBLE,
            ID,
            NEWLINE,
            REGEX,
            STRICT_CODE_BLOCK,
            STRING_LITERAL,

            COUNT_PLUS_0x100
        }; // end of enum Scanner::Token::Type
    }; // end of struct Scanner::Token
}
%target.cpp.scan_method_parameters { Ast::Base *&token }
%target.cpp.bottom_of_class {
    using InputBase::IsOpen;
    using InputBase::GetFiLoc;
    using InputBase::InputName;

    bool OpenFile (string const &input_filename);
    void OpenString (string const &input_string, string const &input_name, bool use_line_numbers = false);
    void OpenUsingStream (istream *input_stream, string const &input_name, bool use_line_numbers);

    using InputBase::Close;

private:

    Token::Type ParseDirective (string const &accepted_string, Ast::Base *&token);

    bool m_is_in_preamble;
    Uint32 m_regex_paren_level;
    Uint32 m_regex_bracket_level;
    Uint32 m_code_block_bracket_level;
    StateMachine::Name m_return_state;
}
%target.cpp.bottom_of_header_file %{
ostream &operator << (ostream &stream, Scanner::Token::Type scanner_token_type);

} // end of namespace CommonLang
} // end of namespace Barf

#endif // !defined(BARF_COMMONLANG_SCANNER_HPP_)
%}

/* ***************************************************************************
/* cpp scanner implementation-file-related directives
/* **************************************************************************/

%target.cpp.top_of_implementation_file %{
#include <cstdlib>
#include <sstream>

#include "barf_ast.hpp"
#include "barf_util.hpp"

namespace Barf {
namespace CommonLang {

ostream &operator << (ostream &stream, Scanner::Token::Type scanner_token_type)
{
    static string const s_scanner_token_type_string[Scanner::Token::COUNT_PLUS_0x100 - 0x100] =
    {
        "BAD_END_OF_FILE",
        "BAD_TOKEN",
        "CHAR_LITERAL",
        "DIRECTIVE_ADD_CODESPEC",
        "DIRECTIVE_ADD_OPTIONAL_DIRECTIVE",
        "DIRECTIVE_ADD_REQUIRED_DIRECTIVE",
        "DIRECTIVE_CASE_INSENSITIVE",
        "DIRECTIVE_DEFAULT",
        "DIRECTIVE_DEFAULT_PARSE_NONTERMINAL",
        "DIRECTIVE_DUMB_CODE_BLOCK",
        "DIRECTIVE_EMPTY",
        "DIRECTIVE_END",
        "DIRECTIVE_ERROR",
        "DIRECTIVE_ID",
        "DIRECTIVE_LOOKAHEAD",
        "DIRECTIVE_MACRO",
        "DIRECTIVE_NONTERMINAL",
        "DIRECTIVE_PREC",
        "DIRECTIVE_START_IN_STATE_MACHINE",
        "DIRECTIVE_STATE_MACHINE",
        "DIRECTIVE_STRICT_CODE_BLOCK",
        "DIRECTIVE_STRING",
        "DIRECTIVE_TARGET",
        "DIRECTIVE_TARGETS",
        "DIRECTIVE_TERMINAL",
        "DIRECTIVE_TYPE",
        "DIRECTIVE_UNGREEDY",
        "DUMB_CODE_BLOCK",
        "END_OF_FILE",
        "END_PREAMBLE",
        "ID",
        "NEWLINE",
        "REGEX",
        "STRICT_CODE_BLOCK",
        "STRING_LITERAL"
    };
    if (scanner_token_type < 0x100)
    {
        assert(scanner_token_type >= 0);
        return stream << CharLiteral(Uint8(scanner_token_type));
    }
    else
    {
        assert(scanner_token_type < Scanner::Token::COUNT_PLUS_0x100);
        return stream << s_scanner_token_type_string[scanner_token_type - 0x100];
    }
}

bool Scanner::OpenFile (string const &input_filename)
{
    bool open_success = InputBase::OpenFile(input_filename);
    if (open_success)
    {
        ResetForNewInput();
        IstreamIterator(istream_iterator<char>(In()));
        // unlimited readahead (read the whole file immediately)
        InputReadahead(0);
    }
    return open_success;
}

void Scanner::OpenString (string const &input_string, string const &input_name, bool use_line_numbers)
{
    InputBase::OpenString(input_string, input_name, use_line_numbers);
    ResetForNewInput();
    IstreamIterator(istream_iterator<char>(In()));
    // unlimited readahead (read the whole file immediately)
    InputReadahead(0);
}

void Scanner::OpenUsingStream (istream *input_stream, string const &input_name, bool use_line_numbers)
{
    InputBase::OpenUsingStream(input_stream, input_name, use_line_numbers);
    ResetForNewInput();
    IstreamIterator(istream_iterator<char>(In()));
    // unlimited readahead (read the whole file immediately)
    InputReadahead(0);
}

Scanner::Token::Type Scanner::ParseDirective (string const &accepted_string, Ast::Base *&token)
{
    assert(token == NULL);

    if (accepted_string == "%add_codespec") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_ADD_CODESPEC; }
    if (accepted_string == "%add_optional_directive") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_ADD_OPTIONAL_DIRECTIVE; }
    if (accepted_string == "%add_required_directive") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_ADD_REQUIRED_DIRECTIVE; }
    if (accepted_string == "%case_insensitive") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_CASE_INSENSITIVE; }
    if (accepted_string == "%default") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_DEFAULT; }
    if (accepted_string == "%default_parse_nonterminal") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_DEFAULT_PARSE_NONTERMINAL; }
    if (accepted_string == "%dumb_code_block") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_DUMB_CODE_BLOCK; }
    if (accepted_string == "%empty") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_EMPTY; }
    if (accepted_string == "%end") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_END; }
    if (accepted_string == "%error") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_ERROR; }
    if (accepted_string == "%identifier") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_ID; }
    if (accepted_string == "%lookahead") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_LOOKAHEAD; }
    if (accepted_string == "%macro") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_MACRO; }
    if (accepted_string == "%nonterminal") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_NONTERMINAL; }
    if (accepted_string == "%prec") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_PREC; }
    if (accepted_string == "%start_with_state_machine") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_START_IN_STATE_MACHINE; }
    if (accepted_string == "%state_machine") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_STATE_MACHINE; }
    if (accepted_string == "%strict_code_block") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_STRICT_CODE_BLOCK; }
    if (accepted_string == "%string") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_STRING; }
    if (accepted_string == "%target") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_TARGET; }
    if (accepted_string == "%targets") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_TARGETS; }
    if (accepted_string == "%terminal") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_TERMINAL; }
    if (accepted_string == "%type") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_TYPE; }
    if (accepted_string == "%ungreedy") { token = new Ast::ThrowAway(GetFiLoc()); return Token::DIRECTIVE_UNGREEDY; }

    EmitError("invalid directive \"" + accepted_string + "\"", GetFiLoc());
    return Token::BAD_TOKEN;
}
%}
%target.cpp.top_of_scan_method_actions %{
    assert(token == NULL);
%}
%target.cpp.bottom_of_scan_method_actions %{
    assert(false && "you didn't handle EOF properly");
    return Token::BAD_END_OF_FILE;
%}
%target.cpp.bottom_of_implementation_file %{
} // end of namespace CommonLang
} // end of namespace Barf
%}

/* ***************************************************************************
/* cpp scanner I/O parameters
/* **************************************************************************/

%target.cpp.return_type "Scanner::Token::Type"
%target.cpp.rejection_actions {
    EmitError("unrecognized character " + CharLiteral(rejected_atom), GetFiLoc());
}
%target.cpp.reset_for_new_input_actions {
    m_is_in_preamble = true;
    m_regex_paren_level = 0;
    m_code_block_bracket_level = 0;
}

/* ***************************************************************************
/* cpp scanner misc directives
/* **************************************************************************/

%target.cpp.generate_debug_spew_code
%target.cpp.debug_spew_prefix {"CommonLang::Scanner" << (GetFiLoc().IsValid() ? " ("+GetFiLoc().AsString()+")" : g_empty_string) << ":"}

/* ***************************************************************************
/* target-independent scanner directives
/* **************************************************************************/

// a single octal (base 8) digit
%macro OCT_DIGIT            ([0-7])
// an octal escape char (e.g. \42)
%macro OCT_CHAR             (\\{OCT_DIGIT}+)

// a single hexadecimal (base 16) digit
%macro HEX_DIGIT            ([0-9A-Fa-f])
// a hexadecimal escape char (e.g. \xF9)
%macro HEX_CHAR             (\\x{HEX_DIGIT}+)

// an escaped char literal (e.g. '\t', '\0' or even '\j')
%macro CHAR_ESC_CHAR        (\\[[:print:]])
// everything that doesn't need to be escaped in a char literal (e.g. 'p')
%macro CHAR_NORMAL_CHAR     ([^[:cntrl:]"'\\])

// single-char escape codes for a string literal (e.g. \n is newline)
%macro STRING_ESC_CHAR      (\\[[:print:]])
// malformed single-char escape codes for a string literal (e.g. a backslash followed by a tab)
%macro STRING_BAD_ESC_CHAR  (\\[^[:print:]])
// everything that doesn't need to be escaped in a string literal
%macro STRING_NORMAL_CHAR   ([^[:cntrl:]"\\]|\t|\n)

// id (same as C-style, but can't start or end with an underscore)
%macro ID                   ([A-Za-z]([_A-Za-z0-9]*[A-Za-z0-9])?)

// all operators that should be returned by ascii value
%macro OPERATOR             ([.:;|\[\]!])
// non-newline whitespace
%macro WHITESPACE           ([ \t])
// newline
%macro NEWLINE              (\n)
// any char
%macro ANY                  (\n|.)
// end-of-file condition
%macro END_OF_FILE          (\z)

%start_with_state_machine MAIN

%%

/* ***************************************************************************
/* state machines and constituent regex rules
/* **************************************************************************/

%state_machine MAIN
:
    (/[*]) // opening delimiter for a block comment
    %target.cpp {
        m_return_state = StateMachine::MAIN;
        SwitchToStateMachine(StateMachine::BLOCK_COMMENT);
    }
|
    (//.*) // line comment (not including the newline)
    %target.cpp { }
|
    (') // opening quote for a char literal
    %target.cpp {
        SwitchToStateMachine(StateMachine::CHAR_LITERAL_GUTS);
    }
|
    (") // opening quote for a string literal
    %target.cpp {
        token = new Ast::String(GetFiLoc());
        SwitchToStateMachine(StateMachine::STRING_LITERAL_GUTS);
    }
|
    (\() // opening delimiter for a regular expression
    %target.cpp {
        token = new Ast::String(GetFiLoc());
        SwitchToStateMachine(StateMachine::REGULAR_EXPRESSION);
    }
|
    (%\{) // opening delimiter for a dumb code block
    %target.cpp {
        token = new Ast::DumbCodeBlock(GetFiLoc());
        SwitchToStateMachine(StateMachine::DUMB_CODE_BLOCK);
    }
|
    (\{) // opening delimiter for a strict code block
    %target.cpp {
        token = new Ast::StrictCodeBlock(GetFiLoc());
        SwitchToStateMachine(StateMachine::STRICT_CODE_BLOCK);
    }
|
    (%{ID}) // a directive
    %target.cpp {
        return Scanner::ParseDirective(accepted_string, token);
    }
|
    (%%)
    %target.cpp {
        token = new Ast::ThrowAway(GetFiLoc());
        m_is_in_preamble = false;
        return Token::END_PREAMBLE;
    }
|
    ({ID})
    %target.cpp {
        token = new Ast::Id(accepted_string, GetFiLoc());
        return Token::ID;
    }
|
    ({OPERATOR})
    %target.cpp {
        return Token::Type(accepted_string[0]);
    }
|
    ({WHITESPACE}) %target.cpp { /* ignore all whitespace */ }
|
    ({NEWLINE})
    %target.cpp {
        IncrementLineNumber();
        if (m_is_in_preamble)
            return Token::NEWLINE;
    }
|
    ({END_OF_FILE})
    %target.cpp {
        return Token::END_OF_FILE;
    }
|
    (.)
    %target.cpp {
        EmitError("unexpected character " + CharLiteral(accepted_string[0]), GetFiLoc());
        return Token::BAD_TOKEN;
    }
;

%state_machine BLOCK_COMMENT %ungreedy
:
    ({ANY}*[*]/) // everything up through the closing delimiter of a block comment
    %target.cpp {
        IncrementLineNumber(NewlineCount(accepted_string));
        if (token != NULL)
            Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
        SwitchToStateMachine(m_return_state);
    }
|
    ({ANY}*{END_OF_FILE}) // everything up through EOF
    %target.cpp {
        IncrementLineNumber(NewlineCount(accepted_string));
        EmitWarning("unterminated block comment", GetFiLoc());
        delete token;
        token = NULL;
        return Token::END_OF_FILE;
    }
;

%state_machine CHAR_LITERAL_GUTS
:
    ({OCT_CHAR})
    %target.cpp {
        assert(accepted_string.length() >= 2);
        assert(accepted_string[0] == '\\');
        Uint32 value = strtol(accepted_string.c_str()+1, NULL, 8);
        if (value >= 0x100)
            EmitError("octal character literal value out of range (" + accepted_string + ")", GetFiLoc());
        token = new Ast::Char(Uint8(value), GetFiLoc());
        SwitchToStateMachine(StateMachine::CHAR_LITERAL_END);
    }
|
    ({HEX_CHAR})
    %target.cpp {
        assert(accepted_string.length() >= 3);
        assert(accepted_string[0] == '\\');
        assert(accepted_string[1] == 'x');
        Uint32 value = strtol(accepted_string.c_str()+2, NULL, 16);
        if (value >= 0x100)
            EmitError("hexadecimal character literal value out of range (" + accepted_string + ")", GetFiLoc());
        token = new Ast::Char(Uint8(value), GetFiLoc());
        SwitchToStateMachine(StateMachine::CHAR_LITERAL_END);
    }
|
    ({CHAR_ESC_CHAR})
    %target.cpp {
        assert(accepted_string.length() == 2);
        assert(accepted_string[0] == '\\');
        Ast::Char *ch = new Ast::Char(Uint8(accepted_string[1]), GetFiLoc());
        ch->Escape();
        token = ch;
        SwitchToStateMachine(StateMachine::CHAR_LITERAL_END);
    }
|
    ({CHAR_NORMAL_CHAR})
    %target.cpp {
        assert(accepted_string.length() == 1);
        token = new Ast::Char(Uint8(accepted_string[0]), GetFiLoc());
        SwitchToStateMachine(StateMachine::CHAR_LITERAL_END);
    }
|
    (\\?{END_OF_FILE}) // end of file (which may be preceded by a backslash)
    %target.cpp {
        EmitError("unterminated character literal", GetFiLoc());
        return Token::END_OF_FILE;
    }
|
    ({ANY}) // anything else just means it's malformed
    %target.cpp {
        EmitError("unexpected character " + CharLiteral(accepted_string[0]) + " in character literal", GetFiLoc());
        if (accepted_string[0] == '\n')
            IncrementLineNumber();
        token = new Ast::Char(Uint8(accepted_string[0]), GetFiLoc());
        SwitchToStateMachine(StateMachine::CHAR_LITERAL_END);
    }
;

%state_machine CHAR_LITERAL_END
:
    (') // closing single-quote
    %target.cpp {
        assert(token != NULL);
        SwitchToStateMachine(StateMachine::MAIN);
        return Token::CHAR_LITERAL;
    }
|
    (\\?{END_OF_FILE}) // end of file (which may be preceded by a backslash)
    %target.cpp {
        EmitError("unterminated character literal", GetFiLoc());
        assert(token != NULL);
        delete token;
        token = NULL;
        SwitchToStateMachine(StateMachine::MAIN);
        return Token::END_OF_FILE;
    }
|
    ({ANY}) // anything else just means it's malformed
    %target.cpp {
        EmitError("malformed character literal", GetFiLoc());
        if (accepted_string[0] == '\n')
            IncrementLineNumber();
        assert(token != NULL);
        delete token;
        token = NULL;
        SwitchToStateMachine(StateMachine::MAIN);
        return Token::BAD_TOKEN;
    }
;

%state_machine STRING_LITERAL_GUTS %ungreedy
:
    (([^\\]|\\{ANY})*") // everything up through the first unescaped double quote
    %target.cpp {
        assert(token != NULL);
        IncrementLineNumber(NewlineCount(accepted_string));
        // get rid of the trailing endquote
        assert(accepted_string.length() >= 1);
        accepted_string.resize(accepted_string.length()-1);
        // escape the string in-place and handle the return code
        EscapeStringStatus status = EscapeString(accepted_string);
        switch (status.m_return_code)
        {
            case ESRC_SUCCESS:
                // awesome
                break;

            case ESRC_UNEXPECTED_EOI:
                assert(false && "the formulation of the regex should prevent this");
                break;

            case ESRC_MALFORMED_HEX_CHAR:
                EmitError(
                    "\\x with no trailing hex digits",
                    FiLoc(token->GetFiLoc().Filename(),
                          token->GetFiLoc().LineNumber() + status.m_line_number_offset));
                break;

            case ESRC_HEX_ESCAPE_SEQUENCE_OUT_OF_RANGE:
            case ESRC_OCTAL_ESCAPE_SEQUENCE_OUT_OF_RANGE:
                EmitError(
                    "hex/octal escape sequence out of range",
                    FiLoc(token->GetFiLoc().Filename(),
                          token->GetFiLoc().LineNumber() + status.m_line_number_offset));
                break;
        }
        Dsc<Ast::String *>(token)->AppendText(accepted_string);
        SwitchToStateMachine(StateMachine::MAIN);
        return Token::STRING_LITERAL;
    }
|
    (([^\\]|\\{ANY})*\\?{END_OF_FILE}) // everything up through EOF (which may be preceded by a hanging backslash)
    %target.cpp {
        EmitError("unterminated string literal", GetFiLoc());
        IncrementLineNumber(NewlineCount(accepted_string));
        assert(token != NULL);
        delete token;
        token = NULL;
        return Token::END_OF_FILE;
    }
;

%state_machine REGULAR_EXPRESSION
:
    (\)) // closing delimiter for a regular expression
    %target.cpp {
        assert(token != NULL);
        if (m_regex_paren_level == 0)
        {
            SwitchToStateMachine(StateMachine::MAIN);
            return Token::REGEX;
        }
        --m_regex_paren_level;
        Dsc<Ast::String *>(token)->AppendText(accepted_string);
    }
|
    (\() // opening delimeter for a sub-expression
    %target.cpp {
        assert(token != NULL);
        ++m_regex_paren_level;
        Dsc<Ast::String *>(token)->AppendText(accepted_string);
    }
|
    (\[) // opening delimiter for a bracket expression
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::String *>(token)->AppendText(accepted_string);
        m_regex_bracket_level = 0;
        SwitchToStateMachine(StateMachine::REGULAR_EXPRESSION_BRACKET_EXPRESSION);
    }
|
    (([^()\[\]\\]|\\{ANY})+) // any not-()[]-or-backslash, or any escaped char
    %target.cpp {
        assert(token != NULL);
        IncrementLineNumber(NewlineCount(accepted_string));
        Dsc<Ast::String *>(token)->AppendText(accepted_string);
    }
|
    (\\?{END_OF_FILE}) // end of file (which may be preceded by a backslash)
    %target.cpp {
        assert(token != NULL);
        delete token;
        token = NULL;
        EmitError("unterminated regular expression", GetFiLoc());
        return Token::END_OF_FILE;
    }
;

%state_machine REGULAR_EXPRESSION_BRACKET_EXPRESSION
:
    (\]) // closing delimiter for bracket expression (or char class)
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::String *>(token)->AppendText(accepted_string);
        if (m_regex_bracket_level == 0)
            SwitchToStateMachine(StateMachine::REGULAR_EXPRESSION);
        else
            --m_regex_bracket_level;
    }
|
    (\[) // opening delimiter for char class
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::String *>(token)->AppendText(accepted_string);
        ++m_regex_bracket_level;
    }
|
    (([^\[\]\\]|\\{ANY})+) // any non-delimiter non-backslash or any escaped char
    %target.cpp {
        assert(token != NULL);
        IncrementLineNumber(NewlineCount(accepted_string));
        Dsc<Ast::String *>(token)->AppendText(accepted_string);
    }
|
    (\\?{END_OF_FILE}) // end of file (which may be preceded by a backslash)
    %target.cpp {
        assert(token != NULL);
        delete token;
        token = NULL;
        EmitError("unterminated bracket expression inside regular expression", GetFiLoc());
        return Token::END_OF_FILE;
    }
;

%state_machine DUMB_CODE_BLOCK %ungreedy
:
    ({ANY}*%\}) // everything up through the closing %}
    %target.cpp {
        assert(token != NULL);
        assert(accepted_string.length() >= 2);
        // take off the %} at the end
        accepted_string.resize(accepted_string.length()-2);
        IncrementLineNumber(NewlineCount(accepted_string));
        Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
        SwitchToStateMachine(StateMachine::MAIN);
        return Token::DUMB_CODE_BLOCK;
    }
|
    ({ANY}*{END_OF_FILE}) // everything up through EOF
    %target.cpp {
        EmitError("unterminated dumb code block (looking for %} delimiter)", GetFiLoc());
        IncrementLineNumber(NewlineCount(accepted_string));
        assert(token != NULL);
        delete token;
        token = NULL;
        return Token::END_OF_FILE;
    }
;

%state_machine STRICT_CODE_BLOCK
:
    (\}) // closing delimiter for a code block
    %target.cpp {
        assert(token != NULL);
        if (m_code_block_bracket_level == 0)
        {
            SwitchToStateMachine(StateMachine::MAIN);
            return Token::STRICT_CODE_BLOCK;
        }
        --m_code_block_bracket_level;
        Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
    }
|
    (\{) // opening delimiter for a code block
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
        ++m_code_block_bracket_level;
    }
|
    (') // opening quote for a char literal
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
        SwitchToStateMachine(StateMachine::CHAR_LITERAL_INSIDE_STRICT_CODE_BLOCK);
    }
|
    (") // opening quote for a string literal
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
        SwitchToStateMachine(StateMachine::STRING_LITERAL_INSIDE_STRICT_CODE_BLOCK);
    }
|
    (/[*]) // opening delimiter for a block comment
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
        m_return_state = StateMachine::STRICT_CODE_BLOCK;
        SwitchToStateMachine(StateMachine::BLOCK_COMMENT);
    }
|
    (//.*) // line comment (not including the newline)
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
    }
|
    ([^{}'"/]+|.) // any non-delimiter char or just any single char
    %target.cpp {
        assert(token != NULL);
        IncrementLineNumber(NewlineCount(accepted_string));
        Dsc<Ast::CodeBlock *>(token)->AppendText(accepted_string);
    }
|
    ({END_OF_FILE}) // end of file
    %target.cpp {
        assert(token != NULL);
        delete token;
        token = NULL;
        EmitError("unterminated strict code block", GetFiLoc());
        return Token::END_OF_FILE;
    }
;

%state_machine CHAR_LITERAL_INSIDE_STRICT_CODE_BLOCK %ungreedy
:
    (([^\\]|\\{ANY})*') // everything up through the first unescaped single quote
    %target.cpp {
        assert(token != NULL);
        IncrementLineNumber(NewlineCount(accepted_string));
        Dsc<Ast::StrictCodeBlock *>(token)->AppendText(accepted_string);
        SwitchToStateMachine(StateMachine::STRICT_CODE_BLOCK);
    }
|
    (([^\\]|\\{ANY})*\\?{END_OF_FILE}) // everything up through EOF (which may be preceded by a hanging backslash)
    %target.cpp {
        assert(token != NULL);
        delete token;
        token = NULL;
        EmitError("unterminated character literal in strict code block", GetFiLoc());
        IncrementLineNumber(NewlineCount(accepted_string));
        return Token::END_OF_FILE;
    }
;

%state_machine STRING_LITERAL_INSIDE_STRICT_CODE_BLOCK %ungreedy
:
    (([^\\]|\\{ANY})*") // everything up through the first unescaped double quote
    %target.cpp {
        assert(token != NULL);
        Dsc<Ast::StrictCodeBlock *>(token)->AppendText(accepted_string);
        IncrementLineNumber(NewlineCount(accepted_string));
        SwitchToStateMachine(StateMachine::STRICT_CODE_BLOCK);
    }
|
    (([^\\]|\\{ANY})*\\?{END_OF_FILE}) // everything up through EOF (which may be preceded by a hanging backslash)
    %target.cpp {
        assert(token != NULL);
        delete token;
        token = NULL;
        EmitError("unterminated string literal in strict code block", GetFiLoc());
        IncrementLineNumber(NewlineCount(accepted_string));
        return Token::END_OF_FILE;
    }
;