refactor: Preprocessors

parser_library/src/CMakeLists.txt

@@ -26,6 +26,8 @@ target_sources(parser_library PRIVATE
 	diagnostic.h
 	diagnostic_consumer.h
 	diagnostic_adder.h
+	document.cpp
+	document.h
 	ebcdic_encoding.cpp
 	ebcdic_encoding.h
 	error_messages.h

parser_library/src/context/hlasm_context.cpp

@@ -401,11 +401,10 @@ hlasm_context::~hlasm_context() = default;
 
 void hlasm_context::set_source_position(position pos) { source_stack_.back().current_instruction.pos = pos; }
 
-void hlasm_context::set_source_indices(size_t begin_index, size_t end_index, size_t end_line)
+void hlasm_context::set_source_indices(size_t begin_index, size_t end_index)
 {
     source_stack_.back().begin_index = begin_index;
     source_stack_.back().end_index = end_index;
-    source_stack_.back().end_line = end_line;
 }
 
 std::pair<source_position, source_snapshot> hlasm_context::get_begin_snapshot(bool ignore_macros) const
@@ -416,13 +415,11 @@ std::pair<source_position, source_snapshot> hlasm_context::get_begin_snapshot(bo
 
     if (!is_in_macros && current_copy_stack().empty())
     {
-        statement_position.file_offset = current_source().begin_index;
-        statement_position.file_line = current_source().current_instruction.pos.line;
+        statement_position.rewind_target = current_source().begin_index;
     }
     else
     {
-        statement_position.file_offset = current_source().end_index;
-        statement_position.file_line = current_source().end_line + 1;
+        statement_position.rewind_target = current_source().end_index;
     }
 
     context::source_snapshot snapshot = current_source().create_snapshot();
@@ -436,8 +433,7 @@ std::pair<source_position, source_snapshot> hlasm_context::get_begin_snapshot(bo
 std::pair<source_position, source_snapshot> hlasm_context::get_end_snapshot() const
 {
     context::source_position statement_position;
-    statement_position.file_offset = current_source().end_index;
-    statement_position.file_line = current_source().end_line + 1;
+    statement_position.rewind_target = current_source().end_index;
 
     context::source_snapshot snapshot = current_source().create_snapshot();
 
@@ -896,7 +892,6 @@ void hlasm_context::apply_source_snapshot(source_snapshot snapshot)
     source_stack_.back().current_instruction = std::move(snapshot.instruction);
     source_stack_.back().begin_index = snapshot.begin_index;
     source_stack_.back().end_index = snapshot.end_index;
-    source_stack_.back().end_line = snapshot.end_line;
 
     source_stack_.back().copy_stack.clear();
 

parser_library/src/context/hlasm_context.h

@@ -120,7 +120,7 @@ class hlasm_context
     // sets current source position
     void set_source_position(position pos);
     // sets current source file indices
-    void set_source_indices(size_t begin_index, size_t end_index, size_t end_line);
+    void set_source_indices(size_t begin_index, size_t end_index);
 
     std::pair<source_position, source_snapshot> get_begin_snapshot(bool ignore_macros) const;
     std::pair<source_position, source_snapshot> get_end_snapshot() const;

parser_library/src/context/source_context.cpp

@@ -32,7 +32,7 @@ source_snapshot source_context::create_snapshot() const
     if (!copy_frames.empty())
         --copy_frames.back().statement_offset;
 
-    return source_snapshot { current_instruction, begin_index, end_index, end_line, std::move(copy_frames) };
+    return source_snapshot { current_instruction, begin_index, end_index, std::move(copy_frames) };
 }
 
 processing_frame::processing_frame(

parser_library/src/context/source_context.h

@@ -29,7 +29,6 @@ struct source_context
     // location in the file
     size_t begin_index = 0;
     size_t end_index = 0;
-    size_t end_line = 0;
 
     // stack of copy nests
     std::vector<copy_member_invocation> copy_stack;

parser_library/src/context/source_snapshot.h

@@ -26,19 +26,14 @@ namespace hlasm_plugin::parser_library::context {
 struct source_position
 {
     // line in the file
-    size_t file_line;
-    // character offset in the file
-    size_t file_offset;
+    size_t rewind_target = 0;
 
-    source_position(size_t file_line = 0, size_t file_offset = 0)
-        : file_line(file_line)
-        , file_offset(file_offset)
+    source_position() = default;
+    explicit source_position(size_t rewind_target)
+        : rewind_target(rewind_target)
     {}
 
-    bool operator==(const source_position& oth) const
-    {
-        return file_line == oth.file_line && file_offset == oth.file_offset;
-    }
+    bool operator==(const source_position& oth) const noexcept = default;
 };
 
 // helper structure representing a copy member invocation
@@ -64,27 +59,20 @@ struct source_snapshot
     location instruction;
     size_t begin_index = 0;
     size_t end_index = 0;
-    size_t end_line = 0;
     std::vector<copy_frame> copy_frames;
 
     source_snapshot() = default;
 
-    source_snapshot(location instruction,
-        size_t begin_index,
-        size_t end_index,
-        size_t end_line,
-        std::vector<copy_frame> copy_frames)
+    source_snapshot(location instruction, size_t begin_index, size_t end_index, std::vector<copy_frame> copy_frames)
         : instruction(std::move(instruction))
         , begin_index(begin_index)
         , end_index(end_index)
-        , end_line(end_line)
         , copy_frames(std::move(copy_frames))
     {}
 
     bool operator==(const source_snapshot& oth) const
     {
-        return end_line == oth.end_line && begin_index == oth.begin_index && end_index == oth.end_index
-            && copy_frames == oth.copy_frames;
+        return begin_index == oth.begin_index && end_index == oth.end_index && copy_frames == oth.copy_frames;
     }
 };
 

parser_library/src/diagnostic.cpp

@@ -19,67 +19,12 @@
 #include <type_traits>
 #include <utility>
 
+#include "utils/concat.h"
 #include "utils/utf8text.h"
 
 namespace hlasm_plugin::parser_library {
 
-namespace {
-struct concat_helper
-{
-    void operator()(std::string& s, std::string_view t) const { s.append(t); }
-    template<typename T>
-    std::enable_if_t<!std::is_convertible_v<T&&, std::string_view>> operator()(std::string& s, T&& t) const
-    {
-        s.append(std::to_string(std::forward<T>(t)));
-    }
-
-    constexpr static std::string_view span_sep = ", ";
-    template<typename T>
-    void operator()(std::string& s, typename std::span<T> span) const
-    {
-        bool first = true;
-        for (const auto& e : span)
-        {
-            if (!first)
-                s.append(span_sep);
-            else
-                first = false;
-
-            operator()(s, e);
-        }
-    }
-
-    size_t len(std::string_view t) const { return t.size(); }
-    template<typename T>
-    std::enable_if_t<!std::is_convertible_v<T&&, std::string_view>, size_t> len(const T&) const
-    {
-        return 8; // arbitrary estimate for the length of the stringified argument (typically small numbers)
-    }
-    template<typename T>
-    size_t len(const typename std::span<T>& span) const
-    {
-        size_t result = 0;
-        for (const auto& e : span)
-            result += span_sep.size() + len(e);
-
-        return result - (result ? span_sep.size() : 0);
-    }
-};
-
-template<typename... Args>
-std::string concat(Args&&... args)
-{
-    std::string result;
-
-    concat_helper h;
-
-    result.reserve((... + h.len(std::as_const(args))));
-
-    (h(result, std::forward<Args>(args)), ...);
-
-    return result;
-}
-} // namespace
+using hlasm_plugin::utils::concat;
 
 // diagnostic_op errors
 

parser_library/src/document.cpp

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2022 Broadcom.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Broadcom, Inc. - initial API and implementation
+ */
+
+#include "document.h"
+
+#include <numeric>
+
+namespace hlasm_plugin::parser_library {
+document::document(std::string_view text)
+{
+    if (text.empty())
+    {
+        m_lines.emplace_back(original_line());
+        return;
+    }
+    size_t line_no = 0;
+    while (!text.empty())
+    {
+        auto p = text.find_first_of("\r\n");
+        if (p == std::string_view::npos)
+            break;
+        if (text.substr(p, 2) == "\r\n")
+            ++p;
+
+        m_lines.emplace_back(original_line { text.substr(0, p + 1), line_no });
+
+        text.remove_prefix(p + 1);
+        ++line_no;
+    }
+    if (!text.empty())
+        m_lines.emplace_back(original_line { text, line_no });
+}
+
+std::string document::text() const
+{
+    return std::accumulate(m_lines.begin(), m_lines.end(), std::string(), [](std::string&& result, const auto& l) {
+        auto t = l.text();
+        result.append(t);
+        if (t.empty() || t.back() != '\n')
+            result.push_back('\n');
+        return std::move(result);
+    });
+}
+
+void document::convert_to_replaced()
+{
+    for (auto& line : m_lines)
+    {
+        if (line.is_original())
+        {
+            line = document_line(replaced_line { std::string(line.text()) });
+        }
+    }
+}
+
+} // namespace hlasm_plugin::parser_library

parser_library/src/document.h

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2022 Broadcom.
+ * The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.
+ *
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *   Broadcom, Inc. - initial API and implementation
+ */
+
+#ifndef HLASMPLUGIN_PARSERLIBRARY_DOCUMENT_H
+#define HLASMPLUGIN_PARSERLIBRARY_DOCUMENT_H
+
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <variant>
+#include <vector>
+
+namespace hlasm_plugin::parser_library {
+
+struct replaced_line
+{
+    std::string m_text;
+};
+
+struct original_line
+{
+    std::string_view m_text;
+    size_t m_lineno = 0;
+};
+
+class document_line
+{
+    std::variant<original_line, replaced_line> m_line;
+
+public:
+    explicit document_line(original_line l) noexcept
+        : m_line(std::move(l))
+    {}
+    explicit document_line(replaced_line l) noexcept
+        : m_line(std::move(l))
+    {}
+
+    std::string_view text() const noexcept
+    {
+        return std::visit([](const auto& s) -> std::string_view { return s.m_text; }, m_line);
+    }
+
+    std::optional<size_t> lineno() const noexcept
+    {
+        if (std::holds_alternative<original_line>(m_line))
+            return std::get<original_line>(m_line).m_lineno;
+        else
+            return std::nullopt;
+    }
+
+    bool is_original() const noexcept { return std::holds_alternative<original_line>(m_line); }
+
+    bool same_type(const document_line& d) const noexcept { return m_line.index() == d.m_line.index(); }
+};
+
+class document
+{
+    std::vector<document_line> m_lines;
+
+public:
+    document() = default;
+    explicit document(std::string_view text);
+    explicit document(std::vector<document_line> lines) noexcept
+        : m_lines(std::move(lines))
+    {}
+
+    auto begin() const { return m_lines.begin(); }
+
+    auto end() const { return m_lines.end(); }
+
+    auto size() const { return m_lines.size(); }
+
+    std::string text() const;
+
+    const auto& at(size_t idx) const { return m_lines.at(idx); }
+
+    void convert_to_replaced();
+};
+
+} // namespace hlasm_plugin::parser_library
+
+#endif

parser_library/src/processing/CMakeLists.txt

@@ -18,6 +18,7 @@ target_sources(parser_library PRIVATE
 	op_code.h
 	opencode_provider.cpp
 	opencode_provider.h
+	preprocessor.cpp
 	preprocessor.h
 	processing_format.h
 	processing_manager.cpp