barf_preprocessor_scanner.reflex

// 2006.11.09 - Copyright Victor Dods - Licensed under Apache 2.0

%targets cpp

/* ***************************************************************************
/* cpp scanner header-file-related directives
/* **************************************************************************/

%target.cpp.header_filename "barf_preprocessor_scanner.hpp"
%target.cpp.implementation_filename "barf_preprocessor_scanner.cpp"
%target.cpp.top_of_header_file %{
#if !defined(BARF_PREPROCESSOR_SCANNER_HPP_)
#define BARF_PREPROCESSOR_SCANNER_HPP_

#include "barf_preprocessor.hpp"

#include <ostream>

#include "barf_inputbase.hpp"
#include "barf_preprocessor_parser.hpp"

namespace Barf {
namespace Preprocessor {

class Text;
%}
%target.cpp.class_name Scanner
%target.cpp.class_inheritance { protected InputBase }
%target.cpp.top_of_class {
}
%target.cpp.bottom_of_class {
    using InputBase::IsOpen;
    using InputBase::GetFiLoc;
    using InputBase::InputName;

    bool OpenFile (string const &input_filename);
    void OpenString (string const &input_string, string const &input_name, bool use_line_numbers = false);
    void OpenUsingStream (istream *input_stream, string const &input_name, bool use_line_numbers);

    using InputBase::Close;

private:

    Parser::Token ParseKeyword (string const &accepted_string);

    bool m_is_reading_newline_sensitive_code;
    Text *m_text;
}
%target.cpp.bottom_of_header_file %{
} // end of namespace Preprocessor
} // end of namespace Barf

#endif // !defined(BARF_PREPROCESSOR_SCANNER_HPP_)
%}

/* ***************************************************************************
/* cpp scanner implementation-file-related directives
/* **************************************************************************/

%target.cpp.top_of_implementation_file %{
#include "barf_preprocessor_ast.hpp"

namespace Barf {
namespace Preprocessor {

bool Scanner::OpenFile (string const &input_filename)
{
    bool open_succeeded = InputBase::OpenFile(input_filename);
    if (open_succeeded)
    {
        ResetForNewInput();
        IstreamIterator(istream_iterator<char>(In()));
        // unlimited readahead (read the whole file immediately)
        InputReadahead(0);
    }
    return open_succeeded;
}

void Scanner::OpenString (string const &input_string, string const &input_name, bool use_line_numbers)
{
    InputBase::OpenString(input_string, input_name, use_line_numbers);
    ResetForNewInput();
    IstreamIterator(istream_iterator<char>(In()));
    // unlimited readahead (read the whole file immediately)
    InputReadahead(0);
}

void Scanner::OpenUsingStream (istream *input_stream, string const &input_name, bool use_line_numbers)
{
    InputBase::OpenUsingStream(input_stream, input_name, use_line_numbers);
    ResetForNewInput();
    IstreamIterator(istream_iterator<char>(In()));
    // unlimited readahead (read the whole file immediately)
    InputReadahead(0);
}

Parser::Token Scanner::ParseKeyword (string const &accepted_string)
{
    if (accepted_string == "declare_array")         return Parser::Token(Parser::Terminal::DECLARE_ARRAY);
    if (accepted_string == "declare_map")           return Parser::Token(Parser::Terminal::DECLARE_MAP);
    if (accepted_string == "define")                return Parser::Token(Parser::Terminal::DEFINE);
    if (accepted_string == "dump_symbol_table")     return Parser::Token(Parser::Terminal::DUMP_SYMBOL_TABLE);
    if (accepted_string == "else")                  return Parser::Token(Parser::Terminal::ELSE);
    if (accepted_string == "else_if")               return Parser::Token(Parser::Terminal::ELSE_IF);
    if (accepted_string == "end_define")            return Parser::Token(Parser::Terminal::END_DEFINE);
    if (accepted_string == "end_for_each")          return Parser::Token(Parser::Terminal::END_FOR_EACH);
    if (accepted_string == "end_if")                return Parser::Token(Parser::Terminal::END_IF);
    if (accepted_string == "end_loop")              return Parser::Token(Parser::Terminal::END_LOOP);
    if (accepted_string == "error")                 return Parser::Token(Parser::Terminal::ERROR);
    if (accepted_string == "fatal_error")           return Parser::Token(Parser::Terminal::FATAL_ERROR);
    if (accepted_string == "for_each")              return Parser::Token(Parser::Terminal::FOR_EACH);
    if (accepted_string == "if")                    return Parser::Token(Parser::Terminal::IF);
    if (accepted_string == "include")               return Parser::Token(Parser::Terminal::INCLUDE);
    if (accepted_string == "int")                   return Parser::Token(Parser::Terminal::KEYWORD_INT);
    if (accepted_string == "is_defined")            return Parser::Token(Parser::Terminal::IS_DEFINED);
    if (accepted_string == "loop")                  return Parser::Token(Parser::Terminal::LOOP);
    if (accepted_string == "sandbox_include")       return Parser::Token(Parser::Terminal::SANDBOX_INCLUDE);
    if (accepted_string == "sizeof")                return Parser::Token(Parser::Terminal::SIZEOF);
    if (accepted_string == "string")                return Parser::Token(Parser::Terminal::KEYWORD_STRING);
    if (accepted_string == "string_length")         return Parser::Token(Parser::Terminal::STRING_LENGTH);
    if (accepted_string == "to_character_literal")  return Parser::Token(Parser::Terminal::TO_CHARACTER_LITERAL);
    if (accepted_string == "to_string_literal")     return Parser::Token(Parser::Terminal::TO_STRING_LITERAL);
    if (accepted_string == "undefine")              return Parser::Token(Parser::Terminal::UNDEFINE);
    if (accepted_string == "warning")               return Parser::Token(Parser::Terminal::WARNING);

    return Parser::Token(Parser::Terminal::ID, new Ast::Id(accepted_string, GetFiLoc()));
}
%}
%target.cpp.constructor_actions {
    m_text = NULL;
}
%target.cpp.destructor_actions {
    delete m_text;
    m_text = NULL;
}
%target.cpp.bottom_of_scan_method_actions %{
    assert(false && "you didn't handle EOF properly");
    return Parser::Token(Parser::Terminal::END_);
%}
%target.cpp.bottom_of_implementation_file %{
} // end of namespace Preprocessor
} // end of namespace Barf
%}

/* ***************************************************************************
/* cpp scanner I/O parameters
/* **************************************************************************/

%target.cpp.return_type "Parser::Token"
%target.cpp.rejection_actions {
    EmitError("unrecognized character " + CharLiteral(rejected_atom), GetFiLoc());
}
%target.cpp.reset_for_new_input_actions {
    delete m_text;
    m_text = NULL;
}

/* ***************************************************************************
/* cpp scanner misc directives
/* **************************************************************************/

%target.cpp.generate_debug_spew_code
%target.cpp.debug_spew_prefix {"Preprocessor::Scanner" << (GetFiLoc().IsValid() ? " ("+GetFiLoc().AsString()+")" : g_empty_string) << ":"}

/* ***************************************************************************
/* target-independent scanner directives
/* **************************************************************************/

// a single octal (base 8) digit
%macro OCT_DIGIT            ([0-7])
// an octal escape char (e.g. \42)
%macro OCT_CHAR             (\\{OCT_DIGIT}+)

// a single hexadecimal (base 16) digit
%macro HEX_DIGIT            ([0-9A-Fa-f])
// a hexadecimal escape char (e.g. \xF9)
%macro HEX_CHAR             (\\x{HEX_DIGIT}+)

// a base-10 integer -- 0 or a sequence of decimal digits not beginning with 0
%macro INTEGER_LITERAL      (0|[1-9][0-9]*)

// single-char escape codes for a string literal (e.g. \n is newline)
%macro STRING_ESC_CHAR      (\\[[:print:]])
// malformed single-char escape codes for a string literal (e.g. a backslash followed by a tab)
%macro STRING_BAD_ESC_CHAR  (\\[^[:print:]])
// everything that doesn't need to be escaped in a string literal
%macro STRING_NORMAL_CHAR   ([^[:cntrl:]"\\]|\t|\n)

// C-style identifier
%macro ID                   ([_A-Za-z][_A-Za-z0-9]*)

// all operators that should be returned by ascii value
%macro OPERATOR             ([()\[\],.+\-*/%!=<>|&?])
// non-newline whitespace
%macro WHITESPACE           ([ \t])
// newline
%macro NEWLINE              (\n)
// any char
%macro ANY                  (\n|.)
// end-of-file condition
%macro END_OF_FILE          (\z)

%start_with_state_machine READING_BODY

%%

/* ***************************************************************************
/* state machines and constituent regex rules
/* **************************************************************************/

%state_machine READING_BODY %ungreedy
:
    ({ANY}*(<\||<\{))
    %target.cpp {
        IncrementLineNumber(NewlineCount(accepted_string));

        // save whether or not we got <|
        m_is_reading_newline_sensitive_code = *accepted_string.rbegin() == '|';

        // take off the <| or <{ at the end
        assert(accepted_string.length() >= 2);
        accepted_string.resize(accepted_string.length()-2);

        // if there's already a body text in progress, continue it.
        if (m_text != NULL)
            m_text->AppendText(accepted_string);
        // otherwise start a new one.
        else
            m_text = new Text(accepted_string, GetFiLoc());

        SwitchToStateMachine(StateMachine::TRANSITION_TO_CODE);
        Ast::Base *token = m_text;
        m_text = NULL;
        return Parser::Token(Parser::Terminal::TEXT, token);
    }
|
    ({ANY}*{END_OF_FILE})
    %target.cpp {
        // if there's already a body text in progress, continue it.
        if (m_text != NULL)
            m_text->AppendText(accepted_string);
        // otherwise start a new one.
        else
            m_text = new Text(accepted_string, GetFiLoc());

        SwitchToStateMachine(StateMachine::EXPECTING_END_OF_FILE);
        Ast::Base *token = m_text;
        m_text = NULL;
        return Parser::Token(Parser::Terminal::TEXT, token);
    }
;

// this is a transitional state machine which immediately matches
// the empty string and then transitions to StateMachine::READING_CODE
%state_machine TRANSITION_TO_CODE
:
    () // always matches immediately
    %target.cpp {
        assert(m_text == NULL);
        SwitchToStateMachine(StateMachine::READING_CODE);
        if (m_is_reading_newline_sensitive_code)
            return Parser::Token(Parser::Terminal::CODE_LINE);
        else
            return Parser::Token(Parser::Terminal::START_CODE);
    }
;

%state_machine READING_CODE
:
    ({WHITESPACE})
    %target.cpp {
        // ignore whitespace
    }
|
    ({NEWLINE})
    %target.cpp {
        IncrementLineNumber(1);
        // we only return from the scanner on a newline if we're scanning
        // a newline-sensitive code line (i.e. one that starts with "<|").
        if (m_is_reading_newline_sensitive_code)
        {
            SwitchToStateMachine(StateMachine::READING_BODY);
            return Parser::Token(Parser::Terminal::CODE_NEWLINE);
        }
    }
|
    ({END_OF_FILE})
    %target.cpp {
        SwitchToStateMachine(StateMachine::EXPECTING_END_OF_FILE);
        if (m_is_reading_newline_sensitive_code)
            return Parser::Token(Parser::Terminal::CODE_NEWLINE);
        else
            EmitError("unexpected end of file encountered within preprocessor code section", GetFiLoc());
    }
|
    (\})
    %target.cpp {
        if (!m_is_reading_newline_sensitive_code)
        {
            SwitchToStateMachine(StateMachine::READING_BODY);
            return Parser::Token(Parser::Terminal::END_CODE);
        }
        else
        {
            EmitError("unexpected '}' encountered", GetFiLoc());
            return Parser::Token(Parser::Terminal::BAD_TOKEN);
        }
    }
|
    ({OPERATOR})
    %target.cpp {
        return Parser::Token(Parser::Token::Id(accepted_string[0]));
    }
|
    ({ID})
    %target.cpp {
        return ParseKeyword(accepted_string);
    }
|
    ({INTEGER_LITERAL})
    %target.cpp {
        Sint32 value = 0;
        istringstream in(accepted_string);
        in >> value;
        return Parser::Token(Parser::Terminal::INTEGER_LITERAL, new Integer(value, GetFiLoc()));
    }
|
    (")
    %target.cpp {
        assert(m_text == NULL);
        m_text = new Text("", GetFiLoc());
        SwitchToStateMachine(StateMachine::READING_CODE_STRING_LITERAL_GUTS);
    }
|
    ({ANY}) // catch-all
    %target.cpp {
        EmitError("unrecognized character encountered within preprocessor code section", GetFiLoc());
        return Parser::Token(Parser::Terminal::BAD_TOKEN);
    }
;

%state_machine READING_CODE_STRING_LITERAL_GUTS %ungreedy
:
    (([^\\]|\\{ANY})*") // everything up through the first unescaped double quote
    %target.cpp {
        assert(m_text != NULL);
        IncrementLineNumber(NewlineCount(accepted_string));
        // get rid of the trailing endquote
        assert(accepted_string.length() >= 1);
        accepted_string.resize(accepted_string.length()-1);
        // escape the string in-place and handle the return code
        EscapeStringStatus status = EscapeString(accepted_string);
        switch (status.m_return_code)
        {
            case ESRC_SUCCESS:
                // awesome
                break;

            case ESRC_UNEXPECTED_EOI:
                assert(false && "the formulation of the regex should prevent this");
                break;

            case ESRC_MALFORMED_HEX_CHAR:
                EmitError(
                    "\\x with no trailing hex digits",
                    FiLoc(m_text->GetFiLoc().Filename(),
                          m_text->GetFiLoc().LineNumber() + status.m_line_number_offset));
                break;

            case ESRC_HEX_ESCAPE_SEQUENCE_OUT_OF_RANGE:
            case ESRC_OCTAL_ESCAPE_SEQUENCE_OUT_OF_RANGE:
                EmitError(
                    "hex/octal escape sequence out of range",
                    FiLoc(m_text->GetFiLoc().Filename(),
                          m_text->GetFiLoc().LineNumber() + status.m_line_number_offset));
                break;
        }
        m_text->AppendText(accepted_string);
        SwitchToStateMachine(StateMachine::READING_CODE);
        Ast::Base *token = m_text;
        m_text = NULL;
        return Parser::Token(Parser::Terminal::STRING_LITERAL, token);
    }
|
    (([^\\]|\\{ANY})*\\?{END_OF_FILE}) // everything up through EOF (which may be preceded by a hanging backslash)
    %target.cpp {
        EmitError("unterminated string literal", GetFiLoc());
        IncrementLineNumber(NewlineCount(accepted_string));
        assert(m_text != NULL);
        delete m_text;
        m_text = NULL;
        return Parser::Token(Parser::Terminal::END_);
    }
;

%state_machine EXPECTING_END_OF_FILE
:
    ({ANY}+)
    %target.cpp {
        assert(false && "this should never happen");
        return Parser::Token(Parser::Terminal::BAD_TOKEN);
    }
|
    ({END_OF_FILE})
    %target.cpp {
        return Parser::Token(Parser::Terminal::END_);
    }
;