From cfa0384d642ee2623502792d53c6c06020bd2eb5 Mon Sep 17 00:00:00 2001 From: isle2983 Date: Sat, 10 Sep 2016 16:15:39 -0600 Subject: [PATCH 01/20] Convenience wrappers for push_back-ing integer types --- include/univalue.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/include/univalue.h b/include/univalue.h index e48b905bfb8571..1d215d570736e7 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -92,6 +92,18 @@ class UniValue { std::string s(val_); return push_back(s); } + bool push_back(uint64_t val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } + bool push_back(int64_t val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } + bool push_back(int val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } bool push_backV(const std::vector& vec); bool pushKV(const std::string& key, const UniValue& val); From 26ef3fff1551bf0c322d88608c086c9fb5083b55 Mon Sep 17 00:00:00 2001 From: BtcDrak Date: Wed, 26 Oct 2016 19:02:20 +0100 Subject: [PATCH 02/20] Remove trailing whitespace from JSON export --- lib/univalue_write.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/univalue_write.cpp b/lib/univalue_write.cpp index cfbdad3284ed53..cf27835991162b 100644 --- a/lib/univalue_write.cpp +++ b/lib/univalue_write.cpp @@ -79,8 +79,6 @@ void UniValue::writeArray(unsigned int prettyIndent, unsigned int indentLevel, s s += values[i].write(prettyIndent, indentLevel + 1); if (i != (values.size() - 1)) { s += ","; - if (prettyIndent) - s += " "; } if (prettyIndent) s += "\n"; From 839ccd71f3e547842319065d71a185eade5e4453 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 1 Nov 2016 15:15:10 -0400 Subject: [PATCH 03/20] Add test driver for JSONTestSuite --- Makefile.am | 7 ++++++- test/.gitignore | 1 + test/test_json.cpp | 24 ++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 test/test_json.cpp diff --git a/Makefile.am b/Makefile.am index 6c1ec81e63fb7c..caa74bf41a84e2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -33,7 +33,7 @@ gen: lib/univalue_escapes.h $(GENBIN) @echo Updating $< $(AM_V_at)$(GENBIN) > lib/univalue_escapes.h -noinst_PROGRAMS = $(TESTS) +noinst_PROGRAMS = $(TESTS) test/test_json TEST_DATA_DIR=test @@ -42,6 +42,11 @@ test_unitester_LDADD = libunivalue.la test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(TEST_DATA_DIR)\" test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) +test_test_json_SOURCES = test/test_json.cpp +test_test_json_LDADD = libunivalue.la +test_test_json_CXXFLAGS = -I$(top_srcdir)/include +test_test_json_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + TEST_FILES = \ $(TEST_DATA_DIR)/fail10.json \ $(TEST_DATA_DIR)/fail11.json \ diff --git a/test/.gitignore b/test/.gitignore index 3d9347fe7e52b4..599875d26d2877 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,4 +1,5 @@ unitester +test_json *.trs *.log diff --git a/test/test_json.cpp b/test/test_json.cpp new file mode 100644 index 00000000000000..2943bae2b15e92 --- /dev/null +++ b/test/test_json.cpp @@ -0,0 +1,24 @@ +// Test program that can be called by the JSON test suite at +// https://github.com/nst/JSONTestSuite. +// +// It reads JSON input from stdin and exits with code 0 if it can be parsed +// successfully. It also pretty prints the parsed JSON value to stdout. + +#include +#include +#include "univalue.h" + +using namespace std; + +int main (int argc, char *argv[]) +{ + UniValue val; + if (val.read(string(istreambuf_iterator(cin), + istreambuf_iterator()))) { + cout << val.write(1 /* prettyIndent */, 4 /* indentLevel */) << endl; + return 0; + } else { + cerr << "JSON Parse Error." << endl; + return 1; + } +} From 0bb1439d0dc649371e5d2aaee0c881d73e7a9483 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 1 Nov 2016 15:19:24 -0400 Subject: [PATCH 04/20] Support parsing raw literals in UniValue Fixes following test failures in https://github.com/nst/JSONTestSuite: bitcoin SHOULD_HAVE_PASSED y_structure_lonely_negative_real.json bitcoin SHOULD_HAVE_PASSED y_structure_lonely_string.json bitcoin SHOULD_HAVE_PASSED y_structure_lonely_false.json bitcoin SHOULD_HAVE_PASSED y_structure_lonely_null.json bitcoin SHOULD_HAVE_PASSED y_string_space.json bitcoin SHOULD_HAVE_PASSED y_structure_string_empty.json bitcoin SHOULD_HAVE_PASSED y_structure_lonely_int.json bitcoin SHOULD_HAVE_PASSED y_structure_lonely_true.json --- Makefile.am | 7 ++++++- lib/univalue_read.cpp | 29 +++++++++++++++++------------ test/fail1.json | 2 +- test/round3.json | 1 + test/round4.json | 1 + test/round5.json | 1 + test/round6.json | 1 + test/round7.json | 1 + test/unitester.cpp | 5 +++++ 9 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 test/round3.json create mode 100644 test/round4.json create mode 100644 test/round5.json create mode 100644 test/round6.json create mode 100644 test/round7.json diff --git a/Makefile.am b/Makefile.am index 6c1ec81e63fb7c..122aa218f7c112 100644 --- a/Makefile.am +++ b/Makefile.am @@ -88,6 +88,11 @@ TEST_FILES = \ $(TEST_DATA_DIR)/pass2.json \ $(TEST_DATA_DIR)/pass3.json \ $(TEST_DATA_DIR)/round1.json \ - $(TEST_DATA_DIR)/round2.json + $(TEST_DATA_DIR)/round2.json \ + $(TEST_DATA_DIR)/round3.json \ + $(TEST_DATA_DIR)/round4.json \ + $(TEST_DATA_DIR)/round5.json \ + $(TEST_DATA_DIR)/round6.json \ + $(TEST_DATA_DIR)/round7.json EXTRA_DIST=$(TEST_FILES) $(GEN_SRCS) diff --git a/lib/univalue_read.cpp b/lib/univalue_read.cpp index 95bac6958d0fa7..fd9f4aeb537413 100644 --- a/lib/univalue_read.cpp +++ b/lib/univalue_read.cpp @@ -177,7 +177,7 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, string valStr; JSONUTF8StringFilter writer(valStr); - while (*raw) { + while (true) { if ((unsigned char)*raw < 0x20) return JTOK_ERR; @@ -371,9 +371,6 @@ bool UniValue::read(const char *raw) case JTOK_KW_NULL: case JTOK_KW_TRUE: case JTOK_KW_FALSE: { - if (!stack.size()) - return false; - UniValue tmpVal; switch (tok) { case JTOK_KW_NULL: @@ -388,6 +385,11 @@ bool UniValue::read(const char *raw) default: /* impossible */ break; } + if (!stack.size()) { + *this = tmpVal; + break; + } + UniValue *top = stack.back(); top->values.push_back(tmpVal); @@ -396,10 +398,12 @@ bool UniValue::read(const char *raw) } case JTOK_NUMBER: { - if (!stack.size()) - return false; - UniValue tmpVal(VNUM, tokenVal); + if (!stack.size()) { + *this = tmpVal; + break; + } + UniValue *top = stack.back(); top->values.push_back(tmpVal); @@ -408,17 +412,18 @@ bool UniValue::read(const char *raw) } case JTOK_STRING: { - if (!stack.size()) - return false; - - UniValue *top = stack.back(); - if (expect(OBJ_NAME)) { + UniValue *top = stack.back(); top->keys.push_back(tokenVal); clearExpect(OBJ_NAME); setExpect(COLON); } else { UniValue tmpVal(VSTR, tokenVal); + if (!stack.size()) { + *this = tmpVal; + break; + } + UniValue *top = stack.back(); top->values.push_back(tmpVal); } diff --git a/test/fail1.json b/test/fail1.json index 6216b865f10219..8feb01a6d0db2a 100644 --- a/test/fail1.json +++ b/test/fail1.json @@ -1 +1 @@ -"A JSON payload should be an object or array, not a string." \ No newline at end of file +"This is a string that never ends, yes it goes on and on, my friends. diff --git a/test/round3.json b/test/round3.json new file mode 100644 index 00000000000000..7182dc2f9b8e47 --- /dev/null +++ b/test/round3.json @@ -0,0 +1 @@ +"abcdefghijklmnopqrstuvwxyz" diff --git a/test/round4.json b/test/round4.json new file mode 100644 index 00000000000000..7f8f011eb73d60 --- /dev/null +++ b/test/round4.json @@ -0,0 +1 @@ +7 diff --git a/test/round5.json b/test/round5.json new file mode 100644 index 00000000000000..27ba77ddaf6153 --- /dev/null +++ b/test/round5.json @@ -0,0 +1 @@ +true diff --git a/test/round6.json b/test/round6.json new file mode 100644 index 00000000000000..c508d5366f70bb --- /dev/null +++ b/test/round6.json @@ -0,0 +1 @@ +false diff --git a/test/round7.json b/test/round7.json new file mode 100644 index 00000000000000..19765bd501b636 --- /dev/null +++ b/test/round7.json @@ -0,0 +1 @@ +null diff --git a/test/unitester.cpp b/test/unitester.cpp index 05f3842cd1eb63..bf597fe5e5b9b4 100644 --- a/test/unitester.cpp +++ b/test/unitester.cpp @@ -125,6 +125,11 @@ static const char *filenames[] = { "pass3.json", "round1.json", // round-trip test "round2.json", // unicode + "round3.json", // bare string + "round4.json", // bare number + "round5.json", // bare true + "round6.json", // bare false + "round7.json", // bare null }; // Test \u handling From fd32d1ab85c82356f5e8a848514a60e65b03cfe5 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 1 Nov 2016 15:49:43 -0400 Subject: [PATCH 05/20] Don't require nul-terminated string inputs Avoid giving '\0' characters special treatment in JSON parsing code. This way the parser will reject input strings like "[1,2,3]\0-extra-nonsense" instead of returning a partially parsed interpretation of the input. It also allows the parser to be called with char pointers that might not be nul terminated (like the char pointers returned by std::string::data()). Fixes following test failures in https://github.com/nst/JSONTestSuite: bitcoin SHOULD_HAVE_FAILED n_multidigit_number_then_00.json --- Makefile.am | 8 +++++++- include/univalue.h | 5 +++-- lib/univalue.cpp | 2 +- lib/univalue_read.cpp | 46 +++++++++++++++++++++++++----------------- test/.gitignore | 1 + test/fail42.json | Bin 0 -> 37 bytes test/no_nul.cpp | 8 ++++++++ test/unitester.cpp | 1 + 8 files changed, 48 insertions(+), 23 deletions(-) create mode 100644 test/fail42.json create mode 100644 test/no_nul.cpp diff --git a/Makefile.am b/Makefile.am index 6c1ec81e63fb7c..ee3b33e4a597d0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -20,7 +20,7 @@ libunivalue_la_LDFLAGS = \ -no-undefined libunivalue_la_CXXFLAGS = -I$(top_srcdir)/include -TESTS = test/unitester +TESTS = test/unitester test/no_nul GENBIN = gen/gen$(BUILD_EXEEXT) GEN_SRCS = gen/gen.cpp @@ -42,6 +42,11 @@ test_unitester_LDADD = libunivalue.la test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(TEST_DATA_DIR)\" test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) +test_no_nul_SOURCES = test/no_nul.cpp +test_no_nul_LDADD = libunivalue.la +test_no_nul_CXXFLAGS = -I$(top_srcdir)/include +test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + TEST_FILES = \ $(TEST_DATA_DIR)/fail10.json \ $(TEST_DATA_DIR)/fail11.json \ @@ -77,6 +82,7 @@ TEST_FILES = \ $(TEST_DATA_DIR)/fail39.json \ $(TEST_DATA_DIR)/fail40.json \ $(TEST_DATA_DIR)/fail41.json \ + $(TEST_DATA_DIR)/fail42.json \ $(TEST_DATA_DIR)/fail3.json \ $(TEST_DATA_DIR)/fail4.json \ $(TEST_DATA_DIR)/fail5.json \ diff --git a/include/univalue.h b/include/univalue.h index e8ce283519fed9..362bd6f28dc288 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -124,9 +124,10 @@ class UniValue { std::string write(unsigned int prettyIndent = 0, unsigned int indentLevel = 0) const; + bool read(const char *raw, size_t len); bool read(const char *raw); bool read(const std::string& rawStr) { - return read(rawStr.c_str()); + return read(rawStr.data(), rawStr.size()); } private: @@ -240,7 +241,7 @@ enum jtokentype { }; extern enum jtokentype getJsonToken(std::string& tokenVal, - unsigned int& consumed, const char *raw); + unsigned int& consumed, const char *raw, const char *end); extern const char *uvTypeName(UniValue::VType t); static inline bool jsonTokenIsValue(enum jtokentype jtt) diff --git a/lib/univalue.cpp b/lib/univalue.cpp index 5a2860c13f8d6c..6058622a549c87 100644 --- a/lib/univalue.cpp +++ b/lib/univalue.cpp @@ -104,7 +104,7 @@ static bool validNumStr(const string& s) { string tokenVal; unsigned int consumed; - enum jtokentype tt = getJsonToken(tokenVal, consumed, s.c_str()); + enum jtokentype tt = getJsonToken(tokenVal, consumed, s.data(), s.data() + s.size()); return (tt == JTOK_NUMBER); } diff --git a/lib/univalue_read.cpp b/lib/univalue_read.cpp index 95bac6958d0fa7..81cde4c19dd8ad 100644 --- a/lib/univalue_read.cpp +++ b/lib/univalue_read.cpp @@ -43,21 +43,21 @@ static const char *hatoui(const char *first, const char *last, } enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, - const char *raw) + const char *raw, const char *end) { tokenVal.clear(); consumed = 0; const char *rawStart = raw; - while ((*raw) && (json_isspace(*raw))) // skip whitespace + while (raw < end && (json_isspace(*raw))) // skip whitespace raw++; - switch (*raw) { - - case 0: + if (raw >= end) return JTOK_NONE; + switch (*raw) { + case '{': raw++; consumed = (raw - rawStart); @@ -127,40 +127,40 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, numStr += *raw; // copy first char raw++; - if ((*first == '-') && (!json_isdigit(*raw))) + if ((*first == '-') && (raw < end) && (!json_isdigit(*raw))) return JTOK_ERR; - while ((*raw) && json_isdigit(*raw)) { // copy digits + while (raw < end && json_isdigit(*raw)) { // copy digits numStr += *raw; raw++; } // part 2: frac - if (*raw == '.') { + if (raw < end && *raw == '.') { numStr += *raw; // copy . raw++; - if (!json_isdigit(*raw)) + if (raw >= end || !json_isdigit(*raw)) return JTOK_ERR; - while ((*raw) && json_isdigit(*raw)) { // copy digits + while (raw < end && json_isdigit(*raw)) { // copy digits numStr += *raw; raw++; } } // part 3: exp - if (*raw == 'e' || *raw == 'E') { + if (raw < end && (*raw == 'e' || *raw == 'E')) { numStr += *raw; // copy E raw++; - if (*raw == '-' || *raw == '+') { // copy +/- + if (raw < end && (*raw == '-' || *raw == '+')) { // copy +/- numStr += *raw; raw++; } - if (!json_isdigit(*raw)) + if (raw >= end || !json_isdigit(*raw)) return JTOK_ERR; - while ((*raw) && json_isdigit(*raw)) { // copy digits + while (raw < end && json_isdigit(*raw)) { // copy digits numStr += *raw; raw++; } @@ -177,13 +177,16 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, string valStr; JSONUTF8StringFilter writer(valStr); - while (*raw) { + while (raw < end) { if ((unsigned char)*raw < 0x20) return JTOK_ERR; else if (*raw == '\\') { raw++; // skip backslash + if (raw >= end) + return JTOK_ERR; + switch (*raw) { case '"': writer.push_back('\"'); break; case '\\': writer.push_back('\\'); break; @@ -196,7 +199,8 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, case 'u': { unsigned int codepoint; - if (hatoui(raw + 1, raw + 1 + 4, codepoint) != + if (raw + 1 + 4 >= end || + hatoui(raw + 1, raw + 1 + 4, codepoint) != raw + 1 + 4) return JTOK_ERR; writer.push_back_u(codepoint); @@ -246,7 +250,7 @@ enum expect_bits { #define setExpect(bit) (expectMask |= EXP_##bit) #define clearExpect(bit) (expectMask &= ~EXP_##bit) -bool UniValue::read(const char *raw) +bool UniValue::read(const char *raw, size_t size) { clear(); @@ -257,10 +261,11 @@ bool UniValue::read(const char *raw) unsigned int consumed; enum jtokentype tok = JTOK_NONE; enum jtokentype last_tok = JTOK_NONE; + const char* end = raw + size; do { last_tok = tok; - tok = getJsonToken(tokenVal, consumed, raw); + tok = getJsonToken(tokenVal, consumed, raw, end); if (tok == JTOK_NONE || tok == JTOK_ERR) return false; raw += consumed; @@ -432,10 +437,13 @@ bool UniValue::read(const char *raw) } while (!stack.empty ()); /* Check that nothing follows the initial construct (parsed above). */ - tok = getJsonToken(tokenVal, consumed, raw); + tok = getJsonToken(tokenVal, consumed, raw, end); if (tok != JTOK_NONE) return false; return true; } +bool UniValue::read(const char *raw) { + return read(raw, strlen(raw)); +} diff --git a/test/.gitignore b/test/.gitignore index 3d9347fe7e52b4..aea12376e5234f 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,4 +1,5 @@ unitester +no_nul *.trs *.log diff --git a/test/fail42.json b/test/fail42.json new file mode 100644 index 0000000000000000000000000000000000000000..9c7565adbddf645df5edfbdcd630c7a0f94aa2eb GIT binary patch literal 37 kcma!6N=i-3FG^L&E6q_zsw_!Wie*qrOe;w(LWpny0Pvs;RsaA1 literal 0 HcmV?d00001 diff --git a/test/no_nul.cpp b/test/no_nul.cpp new file mode 100644 index 00000000000000..83d292200bf875 --- /dev/null +++ b/test/no_nul.cpp @@ -0,0 +1,8 @@ +#include "univalue.h" + +int main (int argc, char *argv[]) +{ + char buf[] = "___[1,2,3]___"; + UniValue val; + return val.read(buf + 3, 7) ? 0 : 1; +} diff --git a/test/unitester.cpp b/test/unitester.cpp index 05f3842cd1eb63..489ae0593054d3 100644 --- a/test/unitester.cpp +++ b/test/unitester.cpp @@ -113,6 +113,7 @@ static const char *filenames[] = { "fail39.json", // invalid unicode: only second half of surrogate pair "fail40.json", // invalid unicode: broken UTF-8 "fail41.json", // invalid unicode: unfinished UTF-8 + "fail42.json", // valid json with garbage following a nul byte "fail3.json", "fail4.json", // extra comma "fail5.json", From a38fcd35564d6590cb25af7c63d1b38c6c49021e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Jan=C3=ADk?= Date: Wed, 16 Nov 2016 18:19:10 +0100 Subject: [PATCH 06/20] Do not shadow member variable codepoint. --- lib/univalue_utffilter.h | 42 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/univalue_utffilter.h b/lib/univalue_utffilter.h index 0e330dce9cd04a..b4a9ddc0a31003 100644 --- a/lib/univalue_utffilter.h +++ b/lib/univalue_utffilter.h @@ -46,19 +46,19 @@ class JSONUTF8StringFilter } } // Write codepoint directly, possibly collating surrogate pairs - void push_back_u(unsigned int codepoint) + void push_back_u(unsigned int codepoint_) { if (state) // Only accept full codepoints in open state is_valid = false; - if (codepoint >= 0xD800 && codepoint < 0xDC00) { // First half of surrogate pair + if (codepoint_ >= 0xD800 && codepoint_ < 0xDC00) { // First half of surrogate pair if (surpair) // Two subsequent surrogate pair openers - fail is_valid = false; else - surpair = codepoint; - } else if (codepoint >= 0xDC00 && codepoint < 0xE000) { // Second half of surrogate pair + surpair = codepoint_; + } else if (codepoint_ >= 0xDC00 && codepoint_ < 0xE000) { // Second half of surrogate pair if (surpair) { // Open surrogate pair, expect second half // Compute code point from UTF-16 surrogate pair - append_codepoint(0x10000 | ((surpair - 0xD800)<<10) | (codepoint - 0xDC00)); + append_codepoint(0x10000 | ((surpair - 0xD800)<<10) | (codepoint_ - 0xDC00)); surpair = 0; } else // Second half doesn't follow a first half - fail is_valid = false; @@ -66,7 +66,7 @@ class JSONUTF8StringFilter if (surpair) // First half of surrogate pair not followed by second - fail is_valid = false; else - append_codepoint(codepoint); + append_codepoint(codepoint_); } } // Check that we're in a state where the string can be ended @@ -96,22 +96,22 @@ class JSONUTF8StringFilter // Two subsequent \u.... may have to be replaced with one actual codepoint. unsigned int surpair; // First half of open UTF-16 surrogate pair, or 0 - void append_codepoint(unsigned int codepoint) + void append_codepoint(unsigned int codepoint_) { - if (codepoint <= 0x7f) - str.push_back((char)codepoint); - else if (codepoint <= 0x7FF) { - str.push_back((char)(0xC0 | (codepoint >> 6))); - str.push_back((char)(0x80 | (codepoint & 0x3F))); - } else if (codepoint <= 0xFFFF) { - str.push_back((char)(0xE0 | (codepoint >> 12))); - str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); - str.push_back((char)(0x80 | (codepoint & 0x3F))); - } else if (codepoint <= 0x1FFFFF) { - str.push_back((char)(0xF0 | (codepoint >> 18))); - str.push_back((char)(0x80 | ((codepoint >> 12) & 0x3F))); - str.push_back((char)(0x80 | ((codepoint >> 6) & 0x3F))); - str.push_back((char)(0x80 | (codepoint & 0x3F))); + if (codepoint_ <= 0x7f) + str.push_back((char)codepoint_); + else if (codepoint_ <= 0x7FF) { + str.push_back((char)(0xC0 | (codepoint_ >> 6))); + str.push_back((char)(0x80 | (codepoint_ & 0x3F))); + } else if (codepoint_ <= 0xFFFF) { + str.push_back((char)(0xE0 | (codepoint_ >> 12))); + str.push_back((char)(0x80 | ((codepoint_ >> 6) & 0x3F))); + str.push_back((char)(0x80 | (codepoint_ & 0x3F))); + } else if (codepoint_ <= 0x1FFFFF) { + str.push_back((char)(0xF0 | (codepoint_ >> 18))); + str.push_back((char)(0x80 | ((codepoint_ >> 12) & 0x3F))); + str.push_back((char)(0x80 | ((codepoint_ >> 6) & 0x3F))); + str.push_back((char)(0x80 | (codepoint_ & 0x3F))); } } }; From 4fd5444d182704d714a39658db9a76677077b677 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Tue, 1 Nov 2016 15:49:43 -0400 Subject: [PATCH 07/20] Reject unterminated strings Fixes following test failures in https://github.com/nst/JSONTestSuite: bitcoin SHOULD_HAVE_FAILED n_string_single_doublequote.json --- Makefile.am | 1 + lib/univalue_read.cpp | 4 ++-- test/fail44.json | 1 + test/unitester.cpp | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 test/fail44.json diff --git a/Makefile.am b/Makefile.am index 532fa1949dc0b4..a8e08557901c93 100644 --- a/Makefile.am +++ b/Makefile.am @@ -88,6 +88,7 @@ TEST_FILES = \ $(TEST_DATA_DIR)/fail40.json \ $(TEST_DATA_DIR)/fail41.json \ $(TEST_DATA_DIR)/fail42.json \ + $(TEST_DATA_DIR)/fail44.json \ $(TEST_DATA_DIR)/fail3.json \ $(TEST_DATA_DIR)/fail4.json \ $(TEST_DATA_DIR)/fail5.json \ diff --git a/lib/univalue_read.cpp b/lib/univalue_read.cpp index 7a9acdd75fdf7e..2bd83238b48602 100644 --- a/lib/univalue_read.cpp +++ b/lib/univalue_read.cpp @@ -177,8 +177,8 @@ enum jtokentype getJsonToken(string& tokenVal, unsigned int& consumed, string valStr; JSONUTF8StringFilter writer(valStr); - while (raw < end) { - if ((unsigned char)*raw < 0x20) + while (true) { + if (raw >= end || (unsigned char)*raw < 0x20) return JTOK_ERR; else if (*raw == '\\') { diff --git a/test/fail44.json b/test/fail44.json new file mode 100644 index 00000000000000..80edceddf1ef6c --- /dev/null +++ b/test/fail44.json @@ -0,0 +1 @@ +"This file ends without a newline or close-quote. \ No newline at end of file diff --git a/test/unitester.cpp b/test/unitester.cpp index aa6f91c1b2fea9..2c37794a4bdb8f 100644 --- a/test/unitester.cpp +++ b/test/unitester.cpp @@ -114,6 +114,7 @@ static const char *filenames[] = { "fail40.json", // invalid unicode: broken UTF-8 "fail41.json", // invalid unicode: unfinished UTF-8 "fail42.json", // valid json with garbage following a nul byte + "fail44.json", // unterminated string "fail3.json", "fail4.json", // extra comma "fail5.json", From a31231b51965effa6f2d7be1ba8392d652b9f971 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sun, 29 Jan 2017 14:55:43 -0500 Subject: [PATCH 08/20] Version 1.0.3 --- configure.ac | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 93d3ba945d143b..8298332ac132cb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,7 +1,7 @@ m4_define([libunivalue_major_version], [1]) m4_define([libunivalue_minor_version], [1]) -m4_define([libunivalue_micro_version], [2]) -m4_define([libunivalue_interface_age], [2]) +m4_define([libunivalue_micro_version], [3]) +m4_define([libunivalue_interface_age], [3]) # If you need a modifier for the version number. # Normally empty, but can be used to make "fixup" releases. m4_define([libunivalue_extraversion], []) @@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()]) -AC_INIT([univalue], [1.0.2], +AC_INIT([univalue], [1.0.3], [http://github.com/jgarzik/univalue/]) dnl make the compilation flags quiet unless V=1 is used From 640158fa26bcba551855782bf5bd73900a7e7c61 Mon Sep 17 00:00:00 2001 From: kozyilmaz Date: Mon, 1 May 2017 21:43:36 -0400 Subject: [PATCH 09/20] Private findKey() method becomes size_t clean, and returns bool on failure. --- include/univalue.h | 4 ++-- lib/univalue.cpp | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/include/univalue.h b/include/univalue.h index 8f671888b52974..4efc64aba211f5 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -72,7 +72,7 @@ class UniValue { bool checkObject(const std::map& memberTypes); const UniValue& operator[](const std::string& key) const; const UniValue& operator[](unsigned int index) const; - bool exists(const std::string& key) const { return (findKey(key) >= 0); } + bool exists(const std::string& key) const { size_t i; return findKey(key, i); } bool isNull() const { return (typ == VNULL); } bool isTrue() const { return (typ == VBOOL) && (val == "1"); } @@ -148,7 +148,7 @@ class UniValue { std::vector keys; std::vector values; - int findKey(const std::string& key) const; + bool findKey(const std::string& key, size_t& retIdx) const; void writeArray(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const; void writeObject(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const; diff --git a/lib/univalue.cpp b/lib/univalue.cpp index 6058622a549c87..a973d2680cc630 100644 --- a/lib/univalue.cpp +++ b/lib/univalue.cpp @@ -212,22 +212,24 @@ bool UniValue::pushKVs(const UniValue& obj) return true; } -int UniValue::findKey(const std::string& key) const +bool UniValue::findKey(const std::string& key, size_t& retIdx) const { - for (unsigned int i = 0; i < keys.size(); i++) { - if (keys[i] == key) - return (int) i; + for (size_t i = 0; i < keys.size(); i++) { + if (keys[i] == key) { + retIdx = i; + return true; + } } - return -1; + return false; } bool UniValue::checkObject(const std::map& t) { for (std::map::const_iterator it = t.begin(); it != t.end(); ++it) { - int idx = findKey(it->first); - if (idx < 0) + size_t idx = 0; + if (!findKey(it->first, idx)) return false; if (values.at(idx).getType() != it->second) @@ -242,8 +244,8 @@ const UniValue& UniValue::operator[](const std::string& key) const if (typ != VOBJ) return NullUniValue; - int index = findKey(key); - if (index < 0) + size_t index = 0; + if (!findKey(key, index)) return NullUniValue; return values.at(index); From 0d3e74dd1e8c8898a4db5cafd1fe4b833bfb5f16 Mon Sep 17 00:00:00 2001 From: kozyilmaz Date: Mon, 1 May 2017 21:58:40 -0400 Subject: [PATCH 10/20] operator[] takes size_t index parameter (versus unsigned int) --- include/univalue.h | 2 +- lib/univalue.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/univalue.h b/include/univalue.h index 4efc64aba211f5..cc92cd2a9c551d 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -71,7 +71,7 @@ class UniValue { bool getBool() const { return isTrue(); } bool checkObject(const std::map& memberTypes); const UniValue& operator[](const std::string& key) const; - const UniValue& operator[](unsigned int index) const; + const UniValue& operator[](size_t index) const; bool exists(const std::string& key) const { size_t i; return findKey(key, i); } bool isNull() const { return (typ == VNULL); } diff --git a/lib/univalue.cpp b/lib/univalue.cpp index a973d2680cc630..b9491a967b7d9b 100644 --- a/lib/univalue.cpp +++ b/lib/univalue.cpp @@ -251,7 +251,7 @@ const UniValue& UniValue::operator[](const std::string& key) const return values.at(index); } -const UniValue& UniValue::operator[](unsigned int index) const +const UniValue& UniValue::operator[](size_t index) const { if (typ != VOBJ && typ != VARR) return NullUniValue; From 1dfe464eff0689579caffa0a81f55c47765204de Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Tue, 2 May 2017 19:23:33 -0400 Subject: [PATCH 11/20] Import UniValue class unit tests from bitcoin project. --- Makefile.am | 7 +- test/.gitignore | 2 + test/object.cpp | 365 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 test/object.cpp diff --git a/Makefile.am b/Makefile.am index a8e08557901c93..287bb97fec507f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -20,7 +20,7 @@ libunivalue_la_LDFLAGS = \ -no-undefined libunivalue_la_CXXFLAGS = -I$(top_srcdir)/include -TESTS = test/unitester test/no_nul +TESTS = test/object test/unitester test/no_nul GENBIN = gen/gen$(BUILD_EXEEXT) GEN_SRCS = gen/gen.cpp @@ -52,6 +52,11 @@ test_no_nul_LDADD = libunivalue.la test_no_nul_CXXFLAGS = -I$(top_srcdir)/include test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) +test_object_SOURCES = test/object.cpp +test_object_LDADD = libunivalue.la +test_object_CXXFLAGS = -I$(top_srcdir)/include +test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + TEST_FILES = \ $(TEST_DATA_DIR)/fail10.json \ $(TEST_DATA_DIR)/fail11.json \ diff --git a/test/.gitignore b/test/.gitignore index 9d0ed57019655b..7b27cf0da2901b 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -1,3 +1,5 @@ + +object unitester test_json no_nul diff --git a/test/object.cpp b/test/object.cpp new file mode 100644 index 00000000000000..f1f2e63b3d6f00 --- /dev/null +++ b/test/object.cpp @@ -0,0 +1,365 @@ +// Copyright (c) 2014 BitPay Inc. +// Copyright (c) 2014-2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include + +#define BOOST_FIXTURE_TEST_SUITE(a, b) +#define BOOST_AUTO_TEST_CASE(funcName) void funcName() +#define BOOST_AUTO_TEST_SUITE_END() +#define BOOST_CHECK(expr) assert(expr) +#define BOOST_CHECK_EQUAL(v1, v2) assert((v1) == (v2)) +#define BOOST_CHECK_THROW(stmt, excMatch) { \ + try { \ + (stmt); \ + } catch (excMatch & e) { \ + } catch (...) { \ + assert(0); \ + } \ + } +#define BOOST_CHECK_NO_THROW(stmt) { \ + try { \ + (stmt); \ + } catch (...) { \ + assert(0); \ + } \ + } + +BOOST_FIXTURE_TEST_SUITE(univalue_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(univalue_constructor) +{ + UniValue v1; + BOOST_CHECK(v1.isNull()); + + UniValue v2(UniValue::VSTR); + BOOST_CHECK(v2.isStr()); + + UniValue v3(UniValue::VSTR, "foo"); + BOOST_CHECK(v3.isStr()); + BOOST_CHECK_EQUAL(v3.getValStr(), "foo"); + + UniValue numTest; + BOOST_CHECK(numTest.setNumStr("82")); + BOOST_CHECK(numTest.isNum()); + BOOST_CHECK_EQUAL(numTest.getValStr(), "82"); + + uint64_t vu64 = 82; + UniValue v4(vu64); + BOOST_CHECK(v4.isNum()); + BOOST_CHECK_EQUAL(v4.getValStr(), "82"); + + int64_t vi64 = -82; + UniValue v5(vi64); + BOOST_CHECK(v5.isNum()); + BOOST_CHECK_EQUAL(v5.getValStr(), "-82"); + + int vi = -688; + UniValue v6(vi); + BOOST_CHECK(v6.isNum()); + BOOST_CHECK_EQUAL(v6.getValStr(), "-688"); + + double vd = -7.21; + UniValue v7(vd); + BOOST_CHECK(v7.isNum()); + BOOST_CHECK_EQUAL(v7.getValStr(), "-7.21"); + + std::string vs("yawn"); + UniValue v8(vs); + BOOST_CHECK(v8.isStr()); + BOOST_CHECK_EQUAL(v8.getValStr(), "yawn"); + + const char *vcs = "zappa"; + UniValue v9(vcs); + BOOST_CHECK(v9.isStr()); + BOOST_CHECK_EQUAL(v9.getValStr(), "zappa"); +} + +BOOST_AUTO_TEST_CASE(univalue_typecheck) +{ + UniValue v1; + BOOST_CHECK(v1.setNumStr("1")); + BOOST_CHECK(v1.isNum()); + BOOST_CHECK_THROW(v1.get_bool(), std::runtime_error); + + UniValue v2; + BOOST_CHECK(v2.setBool(true)); + BOOST_CHECK_EQUAL(v2.get_bool(), true); + BOOST_CHECK_THROW(v2.get_int(), std::runtime_error); + + UniValue v3; + BOOST_CHECK(v3.setNumStr("32482348723847471234")); + BOOST_CHECK_THROW(v3.get_int64(), std::runtime_error); + BOOST_CHECK(v3.setNumStr("1000")); + BOOST_CHECK_EQUAL(v3.get_int64(), 1000); + + UniValue v4; + BOOST_CHECK(v4.setNumStr("2147483648")); + BOOST_CHECK_EQUAL(v4.get_int64(), 2147483648); + BOOST_CHECK_THROW(v4.get_int(), std::runtime_error); + BOOST_CHECK(v4.setNumStr("1000")); + BOOST_CHECK_EQUAL(v4.get_int(), 1000); + BOOST_CHECK_THROW(v4.get_str(), std::runtime_error); + BOOST_CHECK_EQUAL(v4.get_real(), 1000); + BOOST_CHECK_THROW(v4.get_array(), std::runtime_error); + BOOST_CHECK_THROW(v4.getKeys(), std::runtime_error); + BOOST_CHECK_THROW(v4.getValues(), std::runtime_error); + BOOST_CHECK_THROW(v4.get_obj(), std::runtime_error); + + UniValue v5; + BOOST_CHECK(v5.read("[true, 10]")); + BOOST_CHECK_NO_THROW(v5.get_array()); + std::vector vals = v5.getValues(); + BOOST_CHECK_THROW(vals[0].get_int(), std::runtime_error); + BOOST_CHECK_EQUAL(vals[0].get_bool(), true); + + BOOST_CHECK_EQUAL(vals[1].get_int(), 10); + BOOST_CHECK_THROW(vals[1].get_bool(), std::runtime_error); +} + +BOOST_AUTO_TEST_CASE(univalue_set) +{ + UniValue v(UniValue::VSTR, "foo"); + v.clear(); + BOOST_CHECK(v.isNull()); + BOOST_CHECK_EQUAL(v.getValStr(), ""); + + BOOST_CHECK(v.setObject()); + BOOST_CHECK(v.isObject()); + BOOST_CHECK_EQUAL(v.size(), 0); + BOOST_CHECK_EQUAL(v.getType(), UniValue::VOBJ); + BOOST_CHECK(v.empty()); + + BOOST_CHECK(v.setArray()); + BOOST_CHECK(v.isArray()); + BOOST_CHECK_EQUAL(v.size(), 0); + + BOOST_CHECK(v.setStr("zum")); + BOOST_CHECK(v.isStr()); + BOOST_CHECK_EQUAL(v.getValStr(), "zum"); + + BOOST_CHECK(v.setFloat(-1.01)); + BOOST_CHECK(v.isNum()); + BOOST_CHECK_EQUAL(v.getValStr(), "-1.01"); + + BOOST_CHECK(v.setInt((int)1023)); + BOOST_CHECK(v.isNum()); + BOOST_CHECK_EQUAL(v.getValStr(), "1023"); + + BOOST_CHECK(v.setInt((int64_t)-1023LL)); + BOOST_CHECK(v.isNum()); + BOOST_CHECK_EQUAL(v.getValStr(), "-1023"); + + BOOST_CHECK(v.setInt((uint64_t)1023ULL)); + BOOST_CHECK(v.isNum()); + BOOST_CHECK_EQUAL(v.getValStr(), "1023"); + + BOOST_CHECK(v.setNumStr("-688")); + BOOST_CHECK(v.isNum()); + BOOST_CHECK_EQUAL(v.getValStr(), "-688"); + + BOOST_CHECK(v.setBool(false)); + BOOST_CHECK_EQUAL(v.isBool(), true); + BOOST_CHECK_EQUAL(v.isTrue(), false); + BOOST_CHECK_EQUAL(v.isFalse(), true); + BOOST_CHECK_EQUAL(v.getBool(), false); + + BOOST_CHECK(v.setBool(true)); + BOOST_CHECK_EQUAL(v.isBool(), true); + BOOST_CHECK_EQUAL(v.isTrue(), true); + BOOST_CHECK_EQUAL(v.isFalse(), false); + BOOST_CHECK_EQUAL(v.getBool(), true); + + BOOST_CHECK(!v.setNumStr("zombocom")); + + BOOST_CHECK(v.setNull()); + BOOST_CHECK(v.isNull()); +} + +BOOST_AUTO_TEST_CASE(univalue_array) +{ + UniValue arr(UniValue::VARR); + + UniValue v((int64_t)1023LL); + BOOST_CHECK(arr.push_back(v)); + + std::string vStr("zippy"); + BOOST_CHECK(arr.push_back(vStr)); + + const char *s = "pippy"; + BOOST_CHECK(arr.push_back(s)); + + std::vector vec; + v.setStr("boing"); + vec.push_back(v); + + v.setStr("going"); + vec.push_back(v); + + BOOST_CHECK(arr.push_backV(vec)); + + BOOST_CHECK_EQUAL(arr.empty(), false); + BOOST_CHECK_EQUAL(arr.size(), 5); + + BOOST_CHECK_EQUAL(arr[0].getValStr(), "1023"); + BOOST_CHECK_EQUAL(arr[1].getValStr(), "zippy"); + BOOST_CHECK_EQUAL(arr[2].getValStr(), "pippy"); + BOOST_CHECK_EQUAL(arr[3].getValStr(), "boing"); + BOOST_CHECK_EQUAL(arr[4].getValStr(), "going"); + + BOOST_CHECK_EQUAL(arr[999].getValStr(), ""); + + arr.clear(); + BOOST_CHECK(arr.empty()); + BOOST_CHECK_EQUAL(arr.size(), 0); +} + +BOOST_AUTO_TEST_CASE(univalue_object) +{ + UniValue obj(UniValue::VOBJ); + std::string strKey, strVal; + UniValue v; + + strKey = "age"; + v.setInt(100); + BOOST_CHECK(obj.pushKV(strKey, v)); + + strKey = "first"; + strVal = "John"; + BOOST_CHECK(obj.pushKV(strKey, strVal)); + + strKey = "last"; + const char *cVal = "Smith"; + BOOST_CHECK(obj.pushKV(strKey, cVal)); + + strKey = "distance"; + BOOST_CHECK(obj.pushKV(strKey, (int64_t) 25)); + + strKey = "time"; + BOOST_CHECK(obj.pushKV(strKey, (uint64_t) 3600)); + + strKey = "calories"; + BOOST_CHECK(obj.pushKV(strKey, (int) 12)); + + strKey = "temperature"; + BOOST_CHECK(obj.pushKV(strKey, (double) 90.012)); + + UniValue obj2(UniValue::VOBJ); + BOOST_CHECK(obj2.pushKV("cat1", 9000)); + BOOST_CHECK(obj2.pushKV("cat2", 12345)); + + BOOST_CHECK(obj.pushKVs(obj2)); + + BOOST_CHECK_EQUAL(obj.empty(), false); + BOOST_CHECK_EQUAL(obj.size(), 9); + + BOOST_CHECK_EQUAL(obj["age"].getValStr(), "100"); + BOOST_CHECK_EQUAL(obj["first"].getValStr(), "John"); + BOOST_CHECK_EQUAL(obj["last"].getValStr(), "Smith"); + BOOST_CHECK_EQUAL(obj["distance"].getValStr(), "25"); + BOOST_CHECK_EQUAL(obj["time"].getValStr(), "3600"); + BOOST_CHECK_EQUAL(obj["calories"].getValStr(), "12"); + BOOST_CHECK_EQUAL(obj["temperature"].getValStr(), "90.012"); + BOOST_CHECK_EQUAL(obj["cat1"].getValStr(), "9000"); + BOOST_CHECK_EQUAL(obj["cat2"].getValStr(), "12345"); + + BOOST_CHECK_EQUAL(obj["nyuknyuknyuk"].getValStr(), ""); + + BOOST_CHECK(obj.exists("age")); + BOOST_CHECK(obj.exists("first")); + BOOST_CHECK(obj.exists("last")); + BOOST_CHECK(obj.exists("distance")); + BOOST_CHECK(obj.exists("time")); + BOOST_CHECK(obj.exists("calories")); + BOOST_CHECK(obj.exists("temperature")); + BOOST_CHECK(obj.exists("cat1")); + BOOST_CHECK(obj.exists("cat2")); + + BOOST_CHECK(!obj.exists("nyuknyuknyuk")); + + std::map objTypes; + objTypes["age"] = UniValue::VNUM; + objTypes["first"] = UniValue::VSTR; + objTypes["last"] = UniValue::VSTR; + objTypes["distance"] = UniValue::VNUM; + objTypes["time"] = UniValue::VNUM; + objTypes["calories"] = UniValue::VNUM; + objTypes["temperature"] = UniValue::VNUM; + objTypes["cat1"] = UniValue::VNUM; + objTypes["cat2"] = UniValue::VNUM; + BOOST_CHECK(obj.checkObject(objTypes)); + + objTypes["cat2"] = UniValue::VSTR; + BOOST_CHECK(!obj.checkObject(objTypes)); + + obj.clear(); + BOOST_CHECK(obj.empty()); + BOOST_CHECK_EQUAL(obj.size(), 0); +} + +static const char *json1 = +"[1.10000000,{\"key1\":\"str\\u0000\",\"key2\":800,\"key3\":{\"name\":\"martian http://test.com\"}}]"; + +BOOST_AUTO_TEST_CASE(univalue_readwrite) +{ + UniValue v; + BOOST_CHECK(v.read(json1)); + + std::string strJson1(json1); + BOOST_CHECK(v.read(strJson1)); + + BOOST_CHECK(v.isArray()); + BOOST_CHECK_EQUAL(v.size(), 2); + + BOOST_CHECK_EQUAL(v[0].getValStr(), "1.10000000"); + + UniValue obj = v[1]; + BOOST_CHECK(obj.isObject()); + BOOST_CHECK_EQUAL(obj.size(), 3); + + BOOST_CHECK(obj["key1"].isStr()); + std::string correctValue("str"); + correctValue.push_back('\0'); + BOOST_CHECK_EQUAL(obj["key1"].getValStr(), correctValue); + BOOST_CHECK(obj["key2"].isNum()); + BOOST_CHECK_EQUAL(obj["key2"].getValStr(), "800"); + BOOST_CHECK(obj["key3"].isObject()); + + BOOST_CHECK_EQUAL(strJson1, v.write()); + + /* Check for (correctly reporting) a parsing error if the initial + JSON construct is followed by more stuff. Note that whitespace + is, of course, exempt. */ + + BOOST_CHECK(v.read(" {}\n ")); + BOOST_CHECK(v.isObject()); + BOOST_CHECK(v.read(" []\n ")); + BOOST_CHECK(v.isArray()); + + BOOST_CHECK(!v.read("@{}")); + BOOST_CHECK(!v.read("{} garbage")); + BOOST_CHECK(!v.read("[]{}")); + BOOST_CHECK(!v.read("{}[]")); + BOOST_CHECK(!v.read("{} 42")); +} + +BOOST_AUTO_TEST_SUITE_END() + +int main (int argc, char *argv[]) +{ + univalue_constructor(); + univalue_typecheck(); + univalue_set(); + univalue_array(); + univalue_object(); + univalue_readwrite(); + return 0; +} + From f1b86edb4cb923ea42abc58bbc9fd2fe6c40a906 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Tue, 2 May 2017 22:44:34 -0400 Subject: [PATCH 12/20] Convert README to markdown style. --- README | 7 ------- README.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) delete mode 100644 README create mode 100644 README.md diff --git a/README b/README deleted file mode 100644 index 48167b083b0e82..00000000000000 --- a/README +++ /dev/null @@ -1,7 +0,0 @@ - - UniValue - -A universal value object, with JSON encoding (output) and decoding (input). - -Built as a single dynamic RAII C++ object class, and no templates. - diff --git a/README.md b/README.md new file mode 100644 index 00000000000000..2f28e062daa81d --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ + +# UniValue + +## Summary + +A universal value class, with JSON encoding and decoding. + +UniValue is an abstract data type that may be a null, boolean, string, +number, array container, or a key/value dictionary container, nested to +an arbitrary depth. + +This class is aligned with the JSON standard, [RFC +7159](https://tools.ietf.org/html/rfc7159.html). + +## Installation + +This project is a standard GNU +[autotools](https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html) +project. Build and install instructions are available in the `INSTALL` +file following the `autogen.sh` step. + +```$ ./autogen.sh +$ ./configure +$ make + +## Design + +UniValue provides a single dynamic RAII C++ object class, +and minimizes template use (contra json_spirit). + From d09b8429dab1090e6ace5e290ee98af19025f436 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Tue, 2 May 2017 22:45:36 -0400 Subject: [PATCH 13/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f28e062daa81d..aaa839c1f8f935 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ This class is aligned with the JSON standard, [RFC This project is a standard GNU [autotools](https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html) project. Build and install instructions are available in the `INSTALL` -file following the `autogen.sh` step. +file provided with GNU autotools. ```$ ./autogen.sh $ ./configure From 3e31dcffbe6d7fd19b30312cc11fd07f1fdaff64 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Tue, 2 May 2017 22:46:33 -0400 Subject: [PATCH 14/20] README.md: close code quote --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aaa839c1f8f935..0361de37d6357f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ file provided with GNU autotools. ```$ ./autogen.sh $ ./configure -$ make +$ make``` ## Design From dac529675973c5da1e32414e1274908836399d9b Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Tue, 2 May 2017 22:47:09 -0400 Subject: [PATCH 15/20] README.md: update code quotes --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0361de37d6357f..36aa786a4c5de1 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,11 @@ This project is a standard GNU project. Build and install instructions are available in the `INSTALL` file provided with GNU autotools. -```$ ./autogen.sh +``` +$ ./autogen.sh $ ./configure -$ make``` +$ make +``` ## Design From 52e85b35b8107e305fc5344ee73ac254e7c8a082 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 4 May 2017 19:45:34 -0400 Subject: [PATCH 16/20] Move exception-throwing get_* methods into separate implementation module. --- Makefile.am | 1 + lib/univalue.cpp | 135 --------------------------------------- lib/univalue_get.cpp | 147 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 135 deletions(-) create mode 100644 lib/univalue_get.cpp diff --git a/Makefile.am b/Makefile.am index 287bb97fec507f..e283fc890e232d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,6 +12,7 @@ pkgconfig_DATA = pc/libunivalue.pc libunivalue_la_SOURCES = \ lib/univalue.cpp \ + lib/univalue_get.cpp \ lib/univalue_read.cpp \ lib/univalue_write.cpp diff --git a/lib/univalue.cpp b/lib/univalue.cpp index b9491a967b7d9b..af8f2203350dbc 100644 --- a/lib/univalue.cpp +++ b/lib/univalue.cpp @@ -4,75 +4,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include #include -#include #include -#include #include -#include #include "univalue.h" -namespace -{ -static bool ParsePrechecks(const std::string& str) -{ - if (str.empty()) // No empty string allowed - return false; - if (str.size() >= 1 && (json_isspace(str[0]) || json_isspace(str[str.size()-1]))) // No padding allowed - return false; - if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed - return false; - return true; -} - -bool ParseInt32(const std::string& str, int32_t *out) -{ - if (!ParsePrechecks(str)) - return false; - char *endp = NULL; - errno = 0; // strtol will not set errno if valid - long int n = strtol(str.c_str(), &endp, 10); - if(out) *out = (int32_t)n; - // Note that strtol returns a *long int*, so even if strtol doesn't report a over/underflow - // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit - // platforms the size of these types may be different. - return endp && *endp == 0 && !errno && - n >= std::numeric_limits::min() && - n <= std::numeric_limits::max(); -} - -bool ParseInt64(const std::string& str, int64_t *out) -{ - if (!ParsePrechecks(str)) - return false; - char *endp = NULL; - errno = 0; // strtoll will not set errno if valid - long long int n = strtoll(str.c_str(), &endp, 10); - if(out) *out = (int64_t)n; - // Note that strtoll returns a *long long int*, so even if strtol doesn't report a over/underflow - // we still have to check that the returned value is within the range of an *int64_t*. - return endp && *endp == 0 && !errno && - n >= std::numeric_limits::min() && - n <= std::numeric_limits::max(); -} - -bool ParseDouble(const std::string& str, double *out) -{ - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed - return false; - std::istringstream text(str); - text.imbue(std::locale::classic()); - double result; - text >> result; - if(out) *out = result; - return text.eof() && !text.fail(); -} -} - using namespace std; const UniValue NullUniValue; @@ -285,75 +222,3 @@ const UniValue& find_value(const UniValue& obj, const std::string& name) return NullUniValue; } -const std::vector& UniValue::getKeys() const -{ - if (typ != VOBJ) - throw std::runtime_error("JSON value is not an object as expected"); - return keys; -} - -const std::vector& UniValue::getValues() const -{ - if (typ != VOBJ && typ != VARR) - throw std::runtime_error("JSON value is not an object or array as expected"); - return values; -} - -bool UniValue::get_bool() const -{ - if (typ != VBOOL) - throw std::runtime_error("JSON value is not a boolean as expected"); - return getBool(); -} - -const std::string& UniValue::get_str() const -{ - if (typ != VSTR) - throw std::runtime_error("JSON value is not a string as expected"); - return getValStr(); -} - -int UniValue::get_int() const -{ - if (typ != VNUM) - throw std::runtime_error("JSON value is not an integer as expected"); - int32_t retval; - if (!ParseInt32(getValStr(), &retval)) - throw std::runtime_error("JSON integer out of range"); - return retval; -} - -int64_t UniValue::get_int64() const -{ - if (typ != VNUM) - throw std::runtime_error("JSON value is not an integer as expected"); - int64_t retval; - if (!ParseInt64(getValStr(), &retval)) - throw std::runtime_error("JSON integer out of range"); - return retval; -} - -double UniValue::get_real() const -{ - if (typ != VNUM) - throw std::runtime_error("JSON value is not a number as expected"); - double retval; - if (!ParseDouble(getValStr(), &retval)) - throw std::runtime_error("JSON double out of range"); - return retval; -} - -const UniValue& UniValue::get_obj() const -{ - if (typ != VOBJ) - throw std::runtime_error("JSON value is not an object as expected"); - return *this; -} - -const UniValue& UniValue::get_array() const -{ - if (typ != VARR) - throw std::runtime_error("JSON value is not an array as expected"); - return *this; -} - diff --git a/lib/univalue_get.cpp b/lib/univalue_get.cpp new file mode 100644 index 00000000000000..eabcf2dad1acc6 --- /dev/null +++ b/lib/univalue_get.cpp @@ -0,0 +1,147 @@ +// Copyright 2014 BitPay Inc. +// Copyright 2015 Bitcoin Core Developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "univalue.h" + +namespace +{ +static bool ParsePrechecks(const std::string& str) +{ + if (str.empty()) // No empty string allowed + return false; + if (str.size() >= 1 && (json_isspace(str[0]) || json_isspace(str[str.size()-1]))) // No padding allowed + return false; + if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed + return false; + return true; +} + +bool ParseInt32(const std::string& str, int32_t *out) +{ + if (!ParsePrechecks(str)) + return false; + char *endp = NULL; + errno = 0; // strtol will not set errno if valid + long int n = strtol(str.c_str(), &endp, 10); + if(out) *out = (int32_t)n; + // Note that strtol returns a *long int*, so even if strtol doesn't report a over/underflow + // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit + // platforms the size of these types may be different. + return endp && *endp == 0 && !errno && + n >= std::numeric_limits::min() && + n <= std::numeric_limits::max(); +} + +bool ParseInt64(const std::string& str, int64_t *out) +{ + if (!ParsePrechecks(str)) + return false; + char *endp = NULL; + errno = 0; // strtoll will not set errno if valid + long long int n = strtoll(str.c_str(), &endp, 10); + if(out) *out = (int64_t)n; + // Note that strtoll returns a *long long int*, so even if strtol doesn't report a over/underflow + // we still have to check that the returned value is within the range of an *int64_t*. + return endp && *endp == 0 && !errno && + n >= std::numeric_limits::min() && + n <= std::numeric_limits::max(); +} + +bool ParseDouble(const std::string& str, double *out) +{ + if (!ParsePrechecks(str)) + return false; + if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed + return false; + std::istringstream text(str); + text.imbue(std::locale::classic()); + double result; + text >> result; + if(out) *out = result; + return text.eof() && !text.fail(); +} +} + +const std::vector& UniValue::getKeys() const +{ + if (typ != VOBJ) + throw std::runtime_error("JSON value is not an object as expected"); + return keys; +} + +const std::vector& UniValue::getValues() const +{ + if (typ != VOBJ && typ != VARR) + throw std::runtime_error("JSON value is not an object or array as expected"); + return values; +} + +bool UniValue::get_bool() const +{ + if (typ != VBOOL) + throw std::runtime_error("JSON value is not a boolean as expected"); + return getBool(); +} + +const std::string& UniValue::get_str() const +{ + if (typ != VSTR) + throw std::runtime_error("JSON value is not a string as expected"); + return getValStr(); +} + +int UniValue::get_int() const +{ + if (typ != VNUM) + throw std::runtime_error("JSON value is not an integer as expected"); + int32_t retval; + if (!ParseInt32(getValStr(), &retval)) + throw std::runtime_error("JSON integer out of range"); + return retval; +} + +int64_t UniValue::get_int64() const +{ + if (typ != VNUM) + throw std::runtime_error("JSON value is not an integer as expected"); + int64_t retval; + if (!ParseInt64(getValStr(), &retval)) + throw std::runtime_error("JSON integer out of range"); + return retval; +} + +double UniValue::get_real() const +{ + if (typ != VNUM) + throw std::runtime_error("JSON value is not a number as expected"); + double retval; + if (!ParseDouble(getValStr(), &retval)) + throw std::runtime_error("JSON double out of range"); + return retval; +} + +const UniValue& UniValue::get_obj() const +{ + if (typ != VOBJ) + throw std::runtime_error("JSON value is not an object as expected"); + return *this; +} + +const UniValue& UniValue::get_array() const +{ + if (typ != VARR) + throw std::runtime_error("JSON value is not an array as expected"); + return *this; +} + From d4153003166ea6d15464306e414f240b82b6e7e6 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 4 May 2017 23:03:10 -0400 Subject: [PATCH 17/20] Move one-line implementation of UniValue::read() to header. --- include/univalue.h | 3 ++- lib/univalue_read.cpp | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/univalue.h b/include/univalue.h index cc92cd2a9c551d..f10f78d432864c 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -7,6 +7,7 @@ #define __UNIVALUE_H__ #include +#include #include #include @@ -137,7 +138,7 @@ class UniValue { unsigned int indentLevel = 0) const; bool read(const char *raw, size_t len); - bool read(const char *raw); + bool read(const char *raw) { return read(raw, strlen(raw)); } bool read(const std::string& rawStr) { return read(rawStr.data(), rawStr.size()); } diff --git a/lib/univalue_read.cpp b/lib/univalue_read.cpp index 2bd83238b48602..ae75cb462a0188 100644 --- a/lib/univalue_read.cpp +++ b/lib/univalue_read.cpp @@ -449,6 +449,3 @@ bool UniValue::read(const char *raw, size_t size) return true; } -bool UniValue::read(const char *raw) { - return read(raw, strlen(raw)); -} From 107db9829950a8cf9080bf64f224445bf9f66f20 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 4 May 2017 23:23:57 -0400 Subject: [PATCH 18/20] Add ::push_back(double) method for feature parity. Add tests for this and other push_back()'s. --- include/univalue.h | 4 ++++ test/object.cpp | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/include/univalue.h b/include/univalue.h index f10f78d432864c..c9a5690eed43d6 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -105,6 +105,10 @@ class UniValue { UniValue tmpVal(val_); return push_back(tmpVal); } + bool push_back(double val_) { + UniValue tmpVal(val_); + return push_back(tmpVal); + } bool push_backV(const std::vector& vec); bool pushKV(const std::string& key, const UniValue& val); diff --git a/test/object.cpp b/test/object.cpp index f1f2e63b3d6f00..7c4f21c85f57a5 100644 --- a/test/object.cpp +++ b/test/object.cpp @@ -205,14 +205,23 @@ BOOST_AUTO_TEST_CASE(univalue_array) BOOST_CHECK(arr.push_backV(vec)); + BOOST_CHECK(arr.push_back((uint64_t) 400ULL)); + BOOST_CHECK(arr.push_back((int64_t) -400LL)); + BOOST_CHECK(arr.push_back((int) -401)); + BOOST_CHECK(arr.push_back(-40.1)); + BOOST_CHECK_EQUAL(arr.empty(), false); - BOOST_CHECK_EQUAL(arr.size(), 5); + BOOST_CHECK_EQUAL(arr.size(), 9); BOOST_CHECK_EQUAL(arr[0].getValStr(), "1023"); BOOST_CHECK_EQUAL(arr[1].getValStr(), "zippy"); BOOST_CHECK_EQUAL(arr[2].getValStr(), "pippy"); BOOST_CHECK_EQUAL(arr[3].getValStr(), "boing"); BOOST_CHECK_EQUAL(arr[4].getValStr(), "going"); + BOOST_CHECK_EQUAL(arr[5].getValStr(), "400"); + BOOST_CHECK_EQUAL(arr[6].getValStr(), "-400"); + BOOST_CHECK_EQUAL(arr[7].getValStr(), "-401"); + BOOST_CHECK_EQUAL(arr[8].getValStr(), "-40.1"); BOOST_CHECK_EQUAL(arr[999].getValStr(), ""); From ceb1194137421afba01e8c1042bf75f3a2cdddc3 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 3 Aug 2017 10:18:59 -0400 Subject: [PATCH 19/20] Handle .pushKV() and .checkObject() edge cases. pushKV: Behave properly in the face of duplicate keys. checkObject: return error, if not object. Inspired-by: Charles Yin in PR #37 --- include/univalue.h | 1 + lib/univalue.cpp | 22 ++++++++++++++++------ test/object.cpp | 13 +++++++++++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/include/univalue.h b/include/univalue.h index c9a5690eed43d6..e732e27b532a7b 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -111,6 +111,7 @@ class UniValue { } bool push_backV(const std::vector& vec); + void __pushKV(const std::string& key, const UniValue& val); bool pushKV(const std::string& key, const UniValue& val); bool pushKV(const std::string& key, const std::string& val_) { UniValue tmpVal(VSTR, val_); diff --git a/lib/univalue.cpp b/lib/univalue.cpp index af8f2203350dbc..97a38c9b97be69 100644 --- a/lib/univalue.cpp +++ b/lib/univalue.cpp @@ -126,13 +126,22 @@ bool UniValue::push_backV(const std::vector& vec) return true; } +void UniValue::__pushKV(const std::string& key, const UniValue& val_) +{ + keys.push_back(key); + values.push_back(val_); +} + bool UniValue::pushKV(const std::string& key, const UniValue& val_) { if (typ != VOBJ) return false; - keys.push_back(key); - values.push_back(val_); + size_t idx; + if (findKey(key, idx)) + values[idx] = val_; + else + __pushKV(key, val_); return true; } @@ -141,10 +150,8 @@ bool UniValue::pushKVs(const UniValue& obj) if (typ != VOBJ || obj.typ != VOBJ) return false; - for (unsigned int i = 0; i < obj.keys.size(); i++) { - keys.push_back(obj.keys[i]); - values.push_back(obj.values.at(i)); - } + for (size_t i = 0; i < obj.keys.size(); i++) + __pushKV(obj.keys[i], obj.values.at(i)); return true; } @@ -163,6 +170,9 @@ bool UniValue::findKey(const std::string& key, size_t& retIdx) const bool UniValue::checkObject(const std::map& t) { + if (typ != VOBJ) + return false; + for (std::map::const_iterator it = t.begin(); it != t.end(); ++it) { size_t idx = 0; diff --git a/test/object.cpp b/test/object.cpp index 7c4f21c85f57a5..c47f1048b354b6 100644 --- a/test/object.cpp +++ b/test/object.cpp @@ -311,6 +311,19 @@ BOOST_AUTO_TEST_CASE(univalue_object) obj.clear(); BOOST_CHECK(obj.empty()); BOOST_CHECK_EQUAL(obj.size(), 0); + BOOST_CHECK_EQUAL(obj.getType(), UniValue::VNULL); + + BOOST_CHECK_EQUAL(obj.setObject(), true); + UniValue uv; + uv.setInt(42); + obj.__pushKV("age", uv); + BOOST_CHECK_EQUAL(obj.size(), 1); + BOOST_CHECK_EQUAL(obj["age"].getValStr(), "42"); + + uv.setInt(43); + obj.pushKV("age", uv); + BOOST_CHECK_EQUAL(obj.size(), 1); + BOOST_CHECK_EQUAL(obj["age"].getValStr(), "43"); } static const char *json1 = From ba341a20d75c62795fea8ff8e1af2a5661ee3f4d Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Thu, 3 Aug 2017 14:19:44 -0400 Subject: [PATCH 20/20] Add getObjMap() helper method. Also, constify checkObject(). --- include/univalue.h | 3 ++- lib/univalue.cpp | 12 +++++++++++- test/object.cpp | 8 ++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/include/univalue.h b/include/univalue.h index e732e27b532a7b..4fd2223b302e7a 100644 --- a/include/univalue.h +++ b/include/univalue.h @@ -70,7 +70,8 @@ class UniValue { size_t size() const { return values.size(); } bool getBool() const { return isTrue(); } - bool checkObject(const std::map& memberTypes); + void getObjMap(std::map& kv) const; + bool checkObject(const std::map& memberTypes) const; const UniValue& operator[](const std::string& key) const; const UniValue& operator[](size_t index) const; bool exists(const std::string& key) const { size_t i; return findKey(key, i); } diff --git a/lib/univalue.cpp b/lib/univalue.cpp index 97a38c9b97be69..d8ad7c4b90c6b1 100644 --- a/lib/univalue.cpp +++ b/lib/univalue.cpp @@ -156,6 +156,16 @@ bool UniValue::pushKVs(const UniValue& obj) return true; } +void UniValue::getObjMap(std::map& kv) const +{ + if (typ != VOBJ) + return; + + kv.clear(); + for (size_t i = 0; i < keys.size(); i++) + kv[keys[i]] = values[i]; +} + bool UniValue::findKey(const std::string& key, size_t& retIdx) const { for (size_t i = 0; i < keys.size(); i++) { @@ -168,7 +178,7 @@ bool UniValue::findKey(const std::string& key, size_t& retIdx) const return false; } -bool UniValue::checkObject(const std::map& t) +bool UniValue::checkObject(const std::map& t) const { if (typ != VOBJ) return false; diff --git a/test/object.cpp b/test/object.cpp index c47f1048b354b6..02446292a1aa4a 100644 --- a/test/object.cpp +++ b/test/object.cpp @@ -324,6 +324,14 @@ BOOST_AUTO_TEST_CASE(univalue_object) obj.pushKV("age", uv); BOOST_CHECK_EQUAL(obj.size(), 1); BOOST_CHECK_EQUAL(obj["age"].getValStr(), "43"); + + obj.pushKV("name", "foo bar"); + + std::map kv; + obj.getObjMap(kv); + BOOST_CHECK_EQUAL(kv["age"].getValStr(), "43"); + BOOST_CHECK_EQUAL(kv["name"].getValStr(), "foo bar"); + } static const char *json1 =