Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JSON parsing fix from steem PR 2178 #15

Merged
merged 23 commits into from
Mar 13, 2018
Merged

JSON parsing fix from steem PR 2178 #15

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
bbe2a8b
JSON parsing fix from steem PR 2178
pmconrad Mar 1, 2018
b12205c
Minor optimization
pmconrad Mar 2, 2018
0c75709
Added unit tests for stringstream, [io]fstream and buffered_[io]stream
pmconrad Mar 3, 2018
72e96e3
Added first json test case
pmconrad Mar 3, 2018
52f6810
Fixed from_file
pmconrad Mar 3, 2018
4d782e8
Deduplicate some code
pmconrad Mar 3, 2018
8c09ff0
Added json write/validate/read test
pmconrad Mar 5, 2018
e37d9a5
Code cleanups and simplifications
pmconrad Mar 5, 2018
43fabf6
Code deduplication
pmconrad Mar 5, 2018
cb9c61f
Avoid legacy_generator
pmconrad Mar 6, 2018
1ae3cc2
Make unused parsers compile-time optional
pmconrad Mar 6, 2018
b4f3947
Include variants_from_string in parser test
pmconrad Mar 6, 2018
4bb8bf7
Fixed relaxed parser wrt "" input
pmconrad Mar 6, 2018
d336af8
Applied variant_from_stream fix from regular to relaxed
pmconrad Mar 6, 2018
a7e0c88
Added test case for recursion depth 240
pmconrad Mar 7, 2018
90137d4
Fix for recursion depth limitation
pmconrad Mar 7, 2018
2bacd5f
Added broken_nul_parser to preserve previous behaviour
pmconrad Mar 7, 2018
66ed9fc
Minor fixes
pmconrad Mar 7, 2018
1412df1
Make broken_nul_parser usable
pmconrad Mar 8, 2018
f9802f6
Added max_depth parameter to all to_/from_ methods
pmconrad Mar 8, 2018
1331485
Stringify numbers >MAXINT or <MININT
pmconrad Mar 8, 2018
d24bee8
Added test case for exception in exception handling
pmconrad Mar 9, 2018
527daab
Moved format_string from variant.cpp to string.cpp (it is declared in…
pmconrad Mar 9, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion include/fc/io/json.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,19 @@ namespace fc
enum parse_type
{
legacy_parser = 0,
#ifdef WITH_EXOTIC_JSON_PARSERS
strict_parser = 1,
relaxed_parser = 2,
legacy_parser_with_string_doubles = 3
legacy_parser_with_string_doubles = 3,
#endif
broken_nul_parser = 4
};
enum output_formatting
{
stringify_large_ints_and_doubles = 0,
#ifdef WITH_EXOTIC_JSON_PARSERS
legacy_generator = 1
#endif
};

static ostream& to_stream( ostream& out, const fc::string&);
Expand Down
187 changes: 59 additions & 128 deletions include/fc/io/json_relaxed.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
namespace fc { namespace json_relaxed
{
template<typename T, bool strict>
variant variant_from_stream( T& in );
variant variant_from_stream( T& in, uint32_t depth );

template<typename T>
fc::string tokenFromStream( T& in )
Expand Down Expand Up @@ -104,8 +104,15 @@ namespace fc { namespace json_relaxed
if( in.peek() == q )
{
in.get();
if( in.peek() != q )
return fc::string();
try
{
if( in.peek() != q )
return fc::string();
}
catch( const fc::eof_exception& e )
{
return fc::string();
}

// triple quote processing
if( strict )
Expand Down Expand Up @@ -562,86 +569,18 @@ namespace fc { namespace json_relaxed
} FC_CAPTURE_AND_RETHROW( (token) ) }

template<typename T, bool strict>
variant_object objectFromStream( T& in )
variant_object objectFromStream( T& in, uint32_t depth )
{
mutable_variant_object obj;
try
{
char c = in.peek();
if( c != '{' )
FC_THROW_EXCEPTION( parse_error_exception,
"Expected '{', but read '${char}'",
("char",string(&c, &c + 1)) );
in.get();
skip_white_space(in);
while( in.peek() != '}' )
{
if( in.peek() == ',' )
{
in.get();
continue;
}
if( skip_white_space(in) ) continue;
string key = json_relaxed::stringFromStream<T, strict>( in );
skip_white_space(in);
if( in.peek() != ':' )
{
FC_THROW_EXCEPTION( parse_error_exception, "Expected ':' after key \"${key}\"",
("key", key) );
}
in.get();
auto val = json_relaxed::variant_from_stream<T, strict>( in );

obj(std::move(key),std::move(val));
skip_white_space(in);
}
if( in.peek() == '}' )
{
in.get();
return obj;
}
FC_THROW_EXCEPTION( parse_error_exception, "Expected '}' after ${variant}", ("variant", obj ) );
}
catch( const fc::eof_exception& e )
{
FC_THROW_EXCEPTION( parse_error_exception, "Unexpected EOF: ${e}", ("e", e.to_detail_string() ) );
}
catch( const std::ios_base::failure& e )
{
FC_THROW_EXCEPTION( parse_error_exception, "Unexpected EOF: ${e}", ("e", e.what() ) );
} FC_RETHROW_EXCEPTIONS( warn, "Error parsing object" );
std::function<std::string(T&)> get_key = []( T& in ){ return json_relaxed::stringFromStream<T, strict>( in ); };
std::function<variant(T&)> get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream<T, strict>( in, depth ); };
return objectFromStreamBase<T>( in, get_key, get_value );
}

template<typename T, bool strict>
variants arrayFromStream( T& in )
variants arrayFromStream( T& in, uint32_t depth )
{
variants ar;
try
{
if( in.peek() != '[' )
FC_THROW_EXCEPTION( parse_error_exception, "Expected '['" );
in.get();
skip_white_space(in);

while( in.peek() != ']' )
{
if( in.peek() == ',' )
{
in.get();
continue;
}
if( skip_white_space(in) ) continue;
ar.push_back( json_relaxed::variant_from_stream<T, strict>(in) );
skip_white_space(in);
}
if( in.peek() != ']' )
FC_THROW_EXCEPTION( parse_error_exception, "Expected ']' after parsing ${variant}",
("variant", ar) );

in.get();
} FC_RETHROW_EXCEPTIONS( warn, "Attempting to parse array ${array}",
("array", ar ) );
return ar;
std::function<variant(T&)> get_value = [depth]( T& in ){ return json_relaxed::variant_from_stream<T, strict>( in, depth ); };
return arrayFromStreamBase<T>( in, get_value );
}

template<typename T, bool strict>
Expand Down Expand Up @@ -686,60 +625,52 @@ namespace fc { namespace json_relaxed
}

template<typename T, bool strict>
variant variant_from_stream( T& in )
variant variant_from_stream( T& in, uint32_t depth )
{
FC_ASSERT( depth < MAX_RECURSION_DEPTH, "Too many nested items in JSON string!" );
skip_white_space(in);
variant var;
while( signed char c = in.peek() )
signed char c = in.peek();
switch( c )
{
switch( c )
{
case ' ':
case '\t':
case '\n':
case '\r':
in.get();
continue;
case '"':
return json_relaxed::stringFromStream<T, strict>( in );
case '{':
return json_relaxed::objectFromStream<T, strict>( in );
case '[':
return json_relaxed::arrayFromStream<T, strict>( in );
case '-':
case '+':
case '.':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return json_relaxed::numberFromStream<T, strict>( in );
// null, true, false, or 'warning' / string
case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h':
case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p':
case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x':
case 'y': case 'z':
case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H':
case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P':
case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
case 'Y': case 'Z':
case '_': case '/':
return json_relaxed::wordFromStream<T, strict>( in );
case 0x04: // ^D end of transmission
case EOF:
FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" );
default:
FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"",
("c", c)("s", stringFromToken(in)) );
}
case '"':
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about single quotes? According to stringFromStream(...), If strict is false, need to support it and raw strings etc.

return json_relaxed::stringFromStream<T, strict>( in );
case '{':
return json_relaxed::objectFromStream<T, strict>( in, depth + 1 );
case '[':
return json_relaxed::arrayFromStream<T, strict>( in, depth + 1 );
case '-':
case '+':
case '.':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return json_relaxed::numberFromStream<T, strict>( in );
// null, true, false, or 'warning' / string
case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h':
case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p':
case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x':
case 'y': case 'z':
case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H':
case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P':
case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X':
case 'Y': case 'Z':
case '_': case '/':
return json_relaxed::wordFromStream<T, strict>( in );
case 0x04: // ^D end of transmission
case EOF:
FC_THROW_EXCEPTION( eof_exception, "unexpected end of file" );
case 0:
default:
FC_THROW_EXCEPTION( parse_error_exception, "Unexpected char '${c}' in \"${s}\"",
("c", c)("s", stringFromToken(in)) );
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about '\'? According to other code, unquoted strings are allowed in non-strict mode.

}
return variant();
}

} } // fc::json_relaxed
2 changes: 1 addition & 1 deletion include/fc/smart_ref_impl.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ namespace fc {
template<typename T>
T& smart_ref<T>::operator = ( smart_ref<T>&& u ) {
if( &u == this ) return *impl;
if( impl ) delete impl;
delete impl;
impl = u.impl;
u.impl = nullptr;
return *impl;
Expand Down
Loading