diff --git a/include/boost/json/detail/handler.hpp b/include/boost/json/detail/handler.hpp index 6880d723d..2cf2f58d4 100644 --- a/include/boost/json/detail/handler.hpp +++ b/include/boost/json/detail/handler.hpp @@ -35,6 +35,7 @@ struct handler max_string_size = string::max_size(); value_stack st; + bool ignore_duplicate_keys = true; template explicit diff --git a/include/boost/json/detail/impl/handler.ipp b/include/boost/json/detail/impl/handler.ipp index 20b40efa3..e7426524d 100644 --- a/include/boost/json/detail/impl/handler.ipp +++ b/include/boost/json/detail/impl/handler.ipp @@ -51,10 +51,10 @@ bool handler:: on_object_end( std::size_t n, - error_code&) + error_code& ec) { - st.push_object(n); - return true; + ec = st.push_object(n, ignore_duplicate_keys); + return !ec.failed(); } bool @@ -85,7 +85,7 @@ on_key_part( st.push_chars(s); return true; } - + bool handler:: on_key( @@ -96,12 +96,12 @@ on_key( st.push_key(s); return true; } - + bool handler:: on_string_part( string_view s, - std::size_t, + std::size_t, error_code&) { st.push_chars(s); @@ -112,7 +112,7 @@ bool handler:: on_string( string_view s, - std::size_t, + std::size_t, error_code&) { st.push_string(s); @@ -138,7 +138,7 @@ on_int64( st.push_int64(i); return true; } - + bool handler:: on_uint64( @@ -160,7 +160,7 @@ on_double( st.push_double(d); return true; } - + bool handler:: on_bool( @@ -187,7 +187,7 @@ on_comment_part( { return true; } - + bool handler:: on_comment( diff --git a/include/boost/json/detail/object.hpp b/include/boost/json/detail/object.hpp index 73c1956a5..0044cceec 100644 --- a/include/boost/json/detail/object.hpp +++ b/include/boost/json/detail/object.hpp @@ -28,30 +28,29 @@ class unchecked_object // first one is a string key, // second one is the value. value* data_; - std::size_t size_; + value* end_; storage_ptr const& sp_; + bool ignore_duplicates_; public: inline ~unchecked_object(); + inline unchecked_object( value* data, std::size_t size, // # of kv-pairs - storage_ptr const& sp) noexcept - : data_(data) - , size_(size) - , sp_(sp) - { - } + storage_ptr const& sp, + bool ignore_duplicates) noexcept; unchecked_object( unchecked_object&& other) noexcept : data_(other.data_) - , size_(other.size_) + , end_(other.end_) , sp_(other.sp_) + , ignore_duplicates_(other.ignore_duplicates_) { - other.data_ = nullptr; + other.data_ = other.end_ = nullptr; } storage_ptr const& @@ -60,19 +59,25 @@ class unchecked_object return sp_; } + inline std::size_t - size() const noexcept + size() const noexcept; + + bool + ignore_duplicate_keys() const noexcept { - return size_; + return ignore_duplicates_; } value* - release() noexcept + front() noexcept { - auto const data = data_; - data_ = nullptr; - return data; + return data_; } + + inline + void + pop_front() noexcept; }; template @@ -88,6 +93,9 @@ find_in_object( object const&, string_view key) noexcept; +template< bool SmallTable, bool IgnoreDuplicates > +void init_from_unchecked( object& obj, unchecked_object& uo ); + } // detail BOOST_JSON_NS_END diff --git a/include/boost/json/error.hpp b/include/boost/json/error.hpp index 6b19ee039..863ca8e5c 100644 --- a/include/boost/json/error.hpp +++ b/include/boost/json/error.hpp @@ -66,6 +66,9 @@ enum class error /// error occured when trying to read input input_error, + /// duplicate object key + duplicate_key, + // // generic errors // diff --git a/include/boost/json/impl/error.ipp b/include/boost/json/impl/error.ipp index 09d57fcfb..ec508ac96 100644 --- a/include/boost/json/impl/error.ipp +++ b/include/boost/json/impl/error.ipp @@ -45,6 +45,7 @@ case error::array_too_large: return "array too large"; case error::key_too_large: return "key too large"; case error::string_too_large: return "string too large"; case error::input_error: return "input error"; +case error::duplicate_key: return "duplicate key"; case error::exception: return "got exception"; case error::test_failure: return "test failure"; @@ -93,6 +94,7 @@ case error::array_too_large: case error::key_too_large: case error::string_too_large: case error::input_error: +case error::duplicate_key: return condition::parse_error; case error::missing_slash: diff --git a/include/boost/json/impl/object.hpp b/include/boost/json/impl/object.hpp index 628f4479b..82d67f430 100644 --- a/include/boost/json/impl/object.hpp +++ b/include/boost/json/impl/object.hpp @@ -518,19 +518,43 @@ namespace detail { unchecked_object:: ~unchecked_object() { - if(! data_) - return; if(sp_.is_not_shared_and_deallocate_is_trivial()) return; - value* p = data_; - while(size_--) + + for( value* p = data_; p != end_; p += 2 ) { p[0].~value(); p[1].~value(); - p += 2; } } +unchecked_object:: +unchecked_object( + value* data, + std::size_t size, + storage_ptr const& sp, + bool ignore_duplicates) noexcept + : data_(data) + , end_(data + 2 * size) + , sp_(sp) + , ignore_duplicates_(ignore_duplicates) +{ +} + +std::size_t +unchecked_object:: +size() const noexcept +{ + return std::size_t(end_ - data_) / 2; +} + +void +unchecked_object:: +pop_front() noexcept +{ + data_ += 2; +} + } // detail BOOST_JSON_NS_END diff --git a/include/boost/json/impl/object.ipp b/include/boost/json/impl/object.ipp index 28f732db1..bfdbea4b0 100644 --- a/include/boost/json/impl/object.ipp +++ b/include/boost/json/impl/object.ipp @@ -70,6 +70,75 @@ find_in_object( object const& obj, string_view key) noexcept; +template< bool SmallTable, bool IgnoreDuplicates > +void init_from_unchecked( object& obj, unchecked_object& uo ) +{ + // insert all elements, keeping + // the last of any duplicate keys, unless IgnoreDuplicates is false. + auto const begin = obj.begin(); + auto dest = begin; + for( ; uo.size(); uo.pop_front() ) + { + auto src = uo.front(); + access::construct_key_value_pair( + dest, pilfer(src[0]), pilfer(src[1])); + + string_view const key = dest->key(); + key_value_pair* duplicate = nullptr; + BOOST_IF_CONSTEXPR ( SmallTable ) + duplicate = find_in_object( obj, key ).first; + else + { + auto& head = obj.t_->bucket( key ); + auto i = head; + while( true ) + { + if( i == object::null_index_ ) + { + // end of bucket + access::next(*dest) = head; + head = static_cast( dest - begin ); + break; + } + auto& v = begin[i]; + if( v.key() != key ) + { + i = access::next(v); + continue; + } + + // handle duplicate + access::next(*dest) = access::next(v); + duplicate = &v; + break; + } + } + + if( !duplicate ) + { + ++dest; + ++obj.t_->size; + continue; + } + + // handle duplicate + BOOST_IF_CONSTEXPR ( IgnoreDuplicates ) + { + // don't bother to check if + // storage deallocate is trivial + duplicate->~key_value_pair(); + // trivial relocate + std::memcpy( + static_cast(duplicate), dest, sizeof(key_value_pair) ); + } + else + { + dest->~key_value_pair(); + return; + } + } +} + } // namespace detail //---------------------------------------------------------- @@ -197,7 +266,7 @@ destroy() noexcept //---------------------------------------------------------- object:: -object(detail::unchecked_object&& uo) +object(detail::unchecked_object& uo) : sp_(uo.storage()) { if(uo.size() == 0) @@ -210,80 +279,18 @@ object(detail::unchecked_object&& uo) uo.size() <= max_size()); t_ = table::allocate( uo.size(), 0, sp_); + t_->size = 0; - // insert all elements, keeping - // the last of any duplicate keys. - auto dest = begin(); - auto src = uo.release(); - auto const end = src + 2 * uo.size(); if(t_->is_small()) - { - t_->size = 0; - while(src != end) - { - access::construct_key_value_pair( - dest, pilfer(src[0]), pilfer(src[1])); - src += 2; - auto result = detail::find_in_object(*this, dest->key()); - if(! result.first) - { - ++dest; - ++t_->size; - continue; - } - // handle duplicate - auto& v = *result.first; - // don't bother to check if - // storage deallocate is trivial - v.~key_value_pair(); - // trivial relocate - std::memcpy( - static_cast(&v), - dest, sizeof(v)); - } - return; - } - while(src != end) - { - access::construct_key_value_pair( - dest, pilfer(src[0]), pilfer(src[1])); - src += 2; - auto& head = t_->bucket(dest->key()); - auto i = head; - for(;;) - { - if(i == null_index_) - { - // end of bucket - access::next( - *dest) = head; - head = static_cast( - dest - begin()); - ++dest; - break; - } - auto& v = (*t_)[i]; - if(v.key() != dest->key()) - { - i = access::next(v); - continue; - } - - // handle duplicate - access::next(*dest) = - access::next(v); - // don't bother to check if - // storage deallocate is trivial - v.~key_value_pair(); - // trivial relocate - std::memcpy( - static_cast(&v), - dest, sizeof(v)); - break; - } - } - t_->size = static_cast< - index_t>(dest - begin()); + if( uo.ignore_duplicate_keys() ) + detail::init_from_unchecked< true, true >( *this, uo ); + else + detail::init_from_unchecked< true, false >( *this, uo ); + else + if( uo.ignore_duplicate_keys() ) + detail::init_from_unchecked< false, true >( *this, uo ); + else + detail::init_from_unchecked< false, false >( *this, uo ); } object:: diff --git a/include/boost/json/impl/parser.ipp b/include/boost/json/impl/parser.ipp index 6c17bd42c..399b5f8cd 100644 --- a/include/boost/json/impl/parser.ipp +++ b/include/boost/json/impl/parser.ipp @@ -32,20 +32,19 @@ parser( size) { reset(); + p_.handler().ignore_duplicate_keys = opt.ignore_duplicate_keys; } parser:: parser( storage_ptr sp, parse_options const& opt) noexcept - : p_( - opt, + : parser( std::move(sp), - nullptr, + opt, + static_cast(nullptr), 0) -{ - reset(); -} +{ } void parser:: diff --git a/include/boost/json/impl/stream_parser.ipp b/include/boost/json/impl/stream_parser.ipp index b5b798ac2..37fe6c0ea 100644 --- a/include/boost/json/impl/stream_parser.ipp +++ b/include/boost/json/impl/stream_parser.ipp @@ -32,20 +32,19 @@ stream_parser( size) { reset(); + p_.handler().ignore_duplicate_keys = opt.ignore_duplicate_keys; } stream_parser:: stream_parser( storage_ptr sp, parse_options const& opt) noexcept - : p_( - opt, + : stream_parser( std::move(sp), - nullptr, + opt, + static_cast(nullptr), 0) -{ - reset(); -} +{ } void stream_parser:: diff --git a/include/boost/json/impl/value_stack.ipp b/include/boost/json/impl/value_stack.ipp index 1e23b0042..11ad65cf3 100644 --- a/include/boost/json/impl/value_stack.ipp +++ b/include/boost/json/impl/value_stack.ipp @@ -289,7 +289,7 @@ exchange(Unchecked&& u) // which belongs to `u`. detail::access:: construct_value( - &jv.v, std::move(u)); + &jv.v, static_cast(u)); std::memcpy( reinterpret_cast< char*>(top_), @@ -364,16 +364,25 @@ push_array(std::size_t n) st_.exchange(std::move(ua)); } -void +error_code value_stack:: -push_object(std::size_t n) +push_object(std::size_t n, bool ignore_duplicates) { // we already have room if n > 0 if(BOOST_JSON_UNLIKELY(n == 0)) st_.maybe_grow(); + detail::unchecked_object uo( - st_.release(n * 2), n, sp_); - st_.exchange(std::move(uo)); + st_.release(n * 2), n, sp_, ignore_duplicates); + st_.exchange(uo); + + error_code ec; + // constructed object should have consumed all of uo's data + if( uo.size() ) + { + BOOST_JSON_FAIL( ec, error::duplicate_key ); + } + return ec; } void diff --git a/include/boost/json/object.hpp b/include/boost/json/object.hpp index a4ef9351f..516fc2558 100644 --- a/include/boost/json/object.hpp +++ b/include/boost/json/object.hpp @@ -90,7 +90,7 @@ class object BOOST_JSON_DECL explicit - object(detail::unchecked_object&& uo); + object(detail::unchecked_object& uo); public: /** The type of _Allocator_ returned by @ref get_allocator @@ -1646,6 +1646,11 @@ class object reindex_relocate( key_value_pair* src, key_value_pair* dst) noexcept; + + template< bool SmallTable, bool IgnoreDuplicates > + friend + void + detail::init_from_unchecked( object& obj, detail::unchecked_object& uo ); }; BOOST_JSON_NS_END diff --git a/include/boost/json/parse_options.hpp b/include/boost/json/parse_options.hpp index 2b3fee182..206a29eb5 100644 --- a/include/boost/json/parse_options.hpp +++ b/include/boost/json/parse_options.hpp @@ -76,6 +76,20 @@ struct parse_options @ref stream_parser. */ bool allow_invalid_utf8 = false; + + + /** Unique keys restriction setting + + Forbid duplicate keys to appear in objects. + + @note Since @ref basic_parser doesn't store parsed elements directly, + this option has to be taken account by implementers of handlers. + + @see + @ref basic_parser, + @ref stream_parser. + */ + bool ignore_duplicate_keys = true; }; BOOST_JSON_NS_END diff --git a/include/boost/json/value.hpp b/include/boost/json/value.hpp index c5527d4d6..f4efb7b73 100644 --- a/include/boost/json/value.hpp +++ b/include/boost/json/value.hpp @@ -81,8 +81,8 @@ class value explicit value( - detail::unchecked_object&& uo) - : obj_(std::move(uo)) + detail::unchecked_object& uo) + : obj_(uo) { } diff --git a/include/boost/json/value_stack.hpp b/include/boost/json/value_stack.hpp index 885f8d2b3..67061aa51 100644 --- a/include/boost/json/value_stack.hpp +++ b/include/boost/json/value_stack.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -355,8 +356,8 @@ class value_stack top of the stack to form the array. */ BOOST_JSON_DECL - void - push_object(std::size_t n); + error_code + push_object(std::size_t n, bool ignore_duplicates = true); /** Push part of a key or string onto the stack. diff --git a/test/error.cpp b/test/error.cpp index ad57dc3d0..d2f37b24b 100644 --- a/test/error.cpp +++ b/test/error.cpp @@ -61,6 +61,7 @@ class error_test check(condition::parse_error, error::key_too_large); check(condition::parse_error, error::string_too_large); check(condition::parse_error, error::input_error); + check(condition::parse_error, error::duplicate_key); check(condition::pointer_parse_error, error::missing_slash); check(condition::pointer_parse_error, error::invalid_escape); diff --git a/test/parse.cpp b/test/parse.cpp index 41405f49b..48cd3ce6a 100644 --- a/test/parse.cpp +++ b/test/parse.cpp @@ -203,6 +203,32 @@ class parse_test BOOST_TEST_THROWS( parse(ss), system_error ); } + void + testDuplicates() + { + value jv = parse( R"( {"a": 1, "a": 2} )" ); + BOOST_TEST( jv.as_object().size() == 1 ); + + parse_options opt; + opt.ignore_duplicate_keys = false; + + error_code ec; + jv = parse( R"( {"a": 1, "a": 2} )", ec, {}, opt ); + BOOST_TEST( ec == error::duplicate_key ); + + ec.clear(); + std::string s; + s = "{\"999\":null"; + for(int i = 1; i < 1000; ++i) + s += + ",\"" + + std::to_string(i) + + "\":null"; + s.append("}"); + jv = parse( s, ec, {}, opt ); + BOOST_TEST( ec == error::duplicate_key ); + } + void run() { @@ -210,6 +236,7 @@ class parse_test testMemoryUsage(); testIssue726(); testIstream(); + testDuplicates(); } };