re #476: handle byte order marks

README.md

@@ -762,6 +762,11 @@ following situations:
   reflects the usual practice of having at most 1 or 2 tag directives;
   also, be aware that this feature is under consideration for removal
   in YAML 1.3.
+* Byte Order Marks: while ryml correctly handles BOMs at the beginning
+  of the stream or documents (as per the standard), BOMs inside
+  scalars are ignored. The [standard mandates that they should be
+  quoted](https://yaml.org/spec/1.2.2/#52-character-encodings) when
+  emitted, this is not done.
 * ryml tends to be on the permissive side in several cases where the
   YAML standard dictates that there should be an error; in many of these
   cases, ryml will tolerate the input. This may be good or bad, but in

changelog/current.md

@@ -10,6 +10,7 @@
   - add workarounds for problems with codegen of gcc 11,12,13.
   - improve CI coverage of gcc and clang optimization levels.
 - [BREAKING] Fix [#477](https://github.com/biojppm/rapidyaml/issues/477) ([PR#479](https://github.com/biojppm/rapidyaml/pull/479)): changed `read<std::map>()` to overwrite existing entries. The provided implementations had an inconsistency between `std::map` (which wasn't overwriting) and `std::vector` (which *was* overwriting).
+- Fix [#476](https://github.com/biojppm/rapidyaml/issues/476) [PR#493](https://github.com/biojppm/rapidyaml/pull/493): add handling of Byte Order Marks.
 - [PR#492](https://github.com/biojppm/rapidyaml/pull/492): fix emit of explicit keys when indented:
   ```yaml
   fixed:

src/c4/yml/common.hpp

@@ -419,6 +419,20 @@ struct RYML_EXPORT Callbacks
 /** @} */
 
 
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+
+typedef enum {
+    NOBOM,
+    UTF8,
+    UTF16LE,
+    UTF16BE,
+    UTF32LE,
+    UTF32BE,
+} Encoding_e;
+
+
 //-----------------------------------------------------------------------------
 //-----------------------------------------------------------------------------
 //-----------------------------------------------------------------------------

src/c4/yml/parse_engine.def.hpp

@@ -5,13 +5,13 @@
 #include "c4/error.hpp"
 #include "c4/charconv.hpp"
 #include "c4/utf.hpp"
-#include <c4/dump.hpp>
 
 #include <ctype.h>
 
 #include "c4/yml/detail/parser_dbg.hpp"
 #include "c4/yml/filter_processor.hpp"
 #ifdef RYML_DBG
+#include <c4/dump.hpp>
 #include "c4/yml/detail/print.hpp"
 #endif
 
@@ -261,6 +261,9 @@ ParseEngine<EventHandler>::ParseEngine(EventHandler *evt_handler, ParserOptions
     , m_evt_handler(evt_handler)
     , m_pending_anchors()
     , m_pending_tags()
+    , m_was_inside_qmrk(false)
+    , m_doc_empty(false)
+    , m_encoding(NOBOM)
     , m_newline_offsets()
     , m_newline_offsets_size(0)
     , m_newline_offsets_capacity(0)
@@ -277,6 +280,9 @@ ParseEngine<EventHandler>::ParseEngine(ParseEngine &&that) noexcept
     , m_evt_handler(that.m_evt_handler)
     , m_pending_anchors(that.m_pending_anchors)
     , m_pending_tags(that.m_pending_tags)
+    , m_was_inside_qmrk(false)
+    , m_doc_empty(false)
+    , m_encoding(NOBOM)
     , m_newline_offsets(that.m_newline_offsets)
     , m_newline_offsets_size(that.m_newline_offsets_size)
     , m_newline_offsets_capacity(that.m_newline_offsets_capacity)
@@ -293,6 +299,9 @@ ParseEngine<EventHandler>::ParseEngine(ParseEngine const& that)
     , m_evt_handler(that.m_evt_handler)
     , m_pending_anchors(that.m_pending_anchors)
     , m_pending_tags(that.m_pending_tags)
+    , m_was_inside_qmrk(false)
+    , m_doc_empty(false)
+    , m_encoding(NOBOM)
     , m_newline_offsets()
     , m_newline_offsets_size()
     , m_newline_offsets_capacity()
@@ -317,6 +326,9 @@ ParseEngine<EventHandler>& ParseEngine<EventHandler>::operator=(ParseEngine &&th
     m_evt_handler = that.m_evt_handler;
     m_pending_anchors = that.m_pending_anchors;
     m_pending_tags = that.m_pending_tags;
+    m_was_inside_qmrk = that.m_was_inside_qmrk;
+    m_doc_empty = that.m_doc_empty;
+    m_encoding = that.m_encoding;
     m_newline_offsets = (that.m_newline_offsets);
     m_newline_offsets_size = (that.m_newline_offsets_size);
     m_newline_offsets_capacity = (that.m_newline_offsets_capacity);
@@ -337,6 +349,9 @@ ParseEngine<EventHandler>& ParseEngine<EventHandler>::operator=(ParseEngine cons
         m_evt_handler = that.m_evt_handler;
         m_pending_anchors = that.m_pending_anchors;
         m_pending_tags = that.m_pending_tags;
+        m_was_inside_qmrk = that.m_was_inside_qmrk;
+        m_doc_empty = that.m_doc_empty;
+        m_encoding = that.m_encoding;
         if(that.m_newline_offsets_capacity > m_newline_offsets_capacity)
             _resize_locations(that.m_newline_offsets_capacity);
         _RYML_CB_CHECK(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_capacity >= that.m_newline_offsets_capacity);
@@ -357,6 +372,9 @@ void ParseEngine<EventHandler>::_clr()
     m_evt_handler = {};
     m_pending_anchors = {};
     m_pending_tags = {};
+    m_was_inside_qmrk = false;
+    m_doc_empty = true;
+    m_encoding = NOBOM;
     m_newline_offsets = {};
     m_newline_offsets_size = {};
     m_newline_offsets_capacity = {};
@@ -385,11 +403,12 @@ void ParseEngine<EventHandler>::_reset()
     m_pending_anchors = {};
     m_pending_tags = {};
     m_doc_empty = true;
+    m_was_inside_qmrk = false;
+    m_encoding = NOBOM;
     if(m_options.locations())
     {
         _prepare_locations();
     }
-    m_was_inside_qmrk = false;
 }
 
 
@@ -4351,6 +4370,72 @@ void ParseEngine<EventHandler>::_handle_directive(csubstr rem)
     }
 }
 
+template<class EventHandler>
+bool ParseEngine<EventHandler>::_handle_bom()
+{
+    const csubstr rem = m_evt_handler->m_curr->line_contents.rem;
+    if(rem.len)
+    {
+        const csubstr rest = rem.sub(1);
+        // https://yaml.org/spec/1.2.2/#52-character-encodings
+        #define _rymlisascii(c) ((c) <= '\x7f') // is the character ASCII?
+        if(rem.begins_with("\x00\x00\xfe\xff") || (rem.begins_with("\x00\x00\x00") && rem.len >= 4u && _rymlisascii(rem.str[3])))
+        {
+            _c4dbgp("byte order mark: UTF32BE");
+            _handle_bom(UTF32BE);
+            _line_progressed(4);
+            return true;
+        }
+        else if(rem.begins_with("\xff\xfe\x00\x00") || (rest.begins_with("\x00\x00\x00") && rem.len >= 4u && _rymlisascii(rem.str[0])))
+        {
+            _c4dbgp("byte order mark: UTF32LE");
+            _handle_bom(UTF32LE);
+            _line_progressed(4);
+            return true;
+        }
+        else if(rem.begins_with("\xfe\xff") || (rem.begins_with('\x00') && rem.len >= 2u && _rymlisascii(rem.str[1])))
+        {
+            _c4dbgp("byte order mark: UTF16BE");
+            _handle_bom(UTF16BE);
+            _line_progressed(2);
+            return true;
+        }
+        else if(rem.begins_with("\xff\xfe") || (rest.begins_with('\x00') && rem.len >= 2u && _rymlisascii(rem.str[0])))
+        {
+            _c4dbgp("byte order mark: UTF16LE");
+            _handle_bom(UTF16LE);
+            _line_progressed(2);
+            return true;
+        }
+        else if(rem.begins_with("\xef\xbb\xbf"))
+        {
+            _c4dbgp("byte order mark: UTF8");
+            _handle_bom(UTF8);
+            _line_progressed(3);
+            return true;
+        }
+        #undef _rymlisascii
+    }
+    return false;
+}
+
+template<class EventHandler>
+void ParseEngine<EventHandler>::_handle_bom(Encoding_e enc)
+{
+    if(m_encoding == NOBOM)
+    {
+        const bool is_beginning_of_file = m_evt_handler->m_curr->line_contents.rem.str == m_buf.str;
+        if(enc == UTF8 || is_beginning_of_file)
+            m_encoding = enc;
+        else
+            _c4err("non-UTF8 byte order mark can appear only at the beginning of the file");
+    }
+    else if(enc != m_encoding)
+    {
+        _c4err("byte order mark can only be set once");
+    }
+}
+
 
 //-----------------------------------------------------------------------------
 
@@ -7202,6 +7287,10 @@ void ParseEngine<EventHandler>::_handle_unk_json()
         _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
         _line_progressed(1);
     }
+    else if(_handle_bom())
+    {
+        _c4dbgp("byte order mark");
+    }
     else
     {
         _RYML_CB_ASSERT(m_evt_handler->m_stack.m_callbacks,  ! has_any(SSCL));
@@ -7288,8 +7377,13 @@ void ParseEngine<EventHandler>::_handle_unk()
 
     if(m_evt_handler->m_curr->line_contents.indentation == 0u && _at_line_begin())
     {
-        const char first = rem.str[0];
         _c4dbgp("rtop: zero indent + at line begin");
+        if(_handle_bom())
+        {
+            _c4dbgp("byte order mark!");
+            rem = m_evt_handler->m_curr->line_contents.rem;
+        }
+        const char first = rem.str[0];
         if(first == '-')
         {
             _c4dbgp("rtop: suspecting doc");

src/c4/yml/parse_engine.hpp

@@ -366,6 +366,10 @@ class ParseEngine
     /** Get the latest YAML buffer parsed by this object. */
     csubstr source() const { return m_buf; }
 
+    /** Get the encoding of the latest YAML buffer parsed by this object.
+     * If no encoding was specified, UTF8 is assumed as per the YAML standard. */
+    Encoding_e encoding() const { return m_encoding != NOBOM ? m_encoding : UTF8; }
+
     id_type stack_capacity() const { RYML_ASSERT(m_evt_handler); return m_evt_handler->m_stack.capacity(); }
     size_t locations_capacity() const { return m_newline_offsets_capacity; }
 
@@ -714,6 +718,8 @@ class ParseEngine
     void _handle_annotations_and_indentation_after_start_mapblck(size_t key_indentation, size_t key_line);
     size_t _select_indentation_from_annotations(size_t val_indentation, size_t val_line);
     void _handle_directive(csubstr rem);
+    bool _handle_bom();
+    void _handle_bom(Encoding_e enc);
 
     void _check_tag(csubstr tag);
 
@@ -738,6 +744,8 @@ class ParseEngine
     bool m_was_inside_qmrk;
     bool m_doc_empty = true;
 
+    Encoding_e m_encoding = UTF8;
+
 private:
 
     size_t *m_newline_offsets;