From c9938ea8389cd723357f6d7a7e5216775e6a5b92 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 7 Jan 2018 22:09:12 +0100 Subject: [PATCH 01/16] :construction: started UBJSON implementation --- src/json.hpp | 405 ++++- test/Makefile | 1 + .../CouchDB4k.formatted.json | 275 +++ .../universal-binary-json-java/CouchDB4k.ubj | Bin 0 -> 2724 bytes test/data/universal-binary-json-java/LICENSE | 202 +++ .../MediaContent.formatted.json | 34 + .../MediaContent.ubj | Bin 0 -> 442 bytes test/data/universal-binary-json-java/README | 357 ++++ .../TwitterTimeline.formatted.json | 80 + .../TwitterTimeline.ubj | Bin 0 -> 1792 bytes test/data/universal-binary-json-java/url.txt | 1 + test/src/unit-ubjson.cpp | 1593 +++++++++++++++++ 12 files changed, 2929 insertions(+), 19 deletions(-) create mode 100644 test/data/universal-binary-json-java/CouchDB4k.formatted.json create mode 100644 test/data/universal-binary-json-java/CouchDB4k.ubj create mode 100644 test/data/universal-binary-json-java/LICENSE create mode 100644 test/data/universal-binary-json-java/MediaContent.formatted.json create mode 100644 test/data/universal-binary-json-java/MediaContent.ubj create mode 100644 test/data/universal-binary-json-java/README create mode 100644 test/data/universal-binary-json-java/TwitterTimeline.formatted.json create mode 100644 test/data/universal-binary-json-java/TwitterTimeline.ubj create mode 100644 test/data/universal-binary-json-java/url.txt create mode 100644 test/src/unit-ubjson.cpp diff --git a/src/json.hpp b/src/json.hpp index a12fcbe3..a356bdef 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -4534,6 +4534,27 @@ class binary_reader return res; } + /*! + @brief create a JSON value from UBJSON input + + @param[in] strict whether to expect the input to be consumed completed + @return JSON value created from UBJSON input + + @throw parse_error.110 if input ended unexpectedly or the end of file was + not reached when @a strict was set to true + @throw parse_error.112 if unsupported byte was read + */ + BasicJsonType parse_ubjson(const bool strict) + { + const auto res = parse_ubjson_internal(); + if (strict) + { + get(); + check_eof(true); + } + return res; + } + /*! @brief determine system byte order @@ -5195,6 +5216,68 @@ class binary_reader } } + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + */ + BasicJsonType parse_ubjson_internal(const bool get_char = true) + { + switch (get_char ? get() : current) + { + case std::char_traits::eof(): // EOF + JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + + case 'T': // true + return true; + case 'F': // false + return false; + + case 'Z': // null + return nullptr; + + case 'N': // no-op + return parse_ubjson_internal(); // read next byte + + case 'U': + return get_number(); + case 'i': + return get_number(); + case 'I': + return get_number(); + case 'l': + return get_number(); + case 'L': + return get_number(); + case 'd': + return get_number(); + case 'D': + return get_number(); + + case 'C': // char + { + get(); + check_eof(); + return std::string(1, static_cast(current)); + } + + case 'S': // string + return get_ubjson_string(); + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(112, chars_read, + "error reading UBJSON; last byte: 0x" + ss.str())); + } + } + /*! @brief get next character from the input @@ -5495,6 +5578,80 @@ class binary_reader return result; } + /*! + @brief reads a UBJSON string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return string + + @throw parse_error.110 if input ended + @throw parse_error.113 if an unexpected byte is read + */ + std::string get_ubjson_string(const bool get_char = true) + { + if (get_char) + { + get(); + } + + check_eof(); + + switch (current) + { + case 'U': + return get_string(get_number()); + case 'i': + return get_string(get_number()); + case 'I': + return get_string(get_number()); + case 'l': + return get_string(get_number()); + case 'L': + return get_string(get_number()); + default: + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(113, chars_read, + "expected a UBJSON string; last byte: 0x" + ss.str())); + } + } + + BasicJsonType get_ubjson_array() + { + BasicJsonType result = value_t::array; + + while (get() != ']') + { + // skip no-op + if (current == 'N') + { + continue; + } + result.push_back(parse_ubjson_internal(false)); + } + + return result; + } + + BasicJsonType get_ubjson_object() + { + BasicJsonType result = value_t::object; + + while (get() != '}') + { + auto key = get_ubjson_string(false); + result[std::move(key)] = parse_ubjson_internal(); + } + + return result; + } + /*! @brief check if input ended @throw parse_error.110 if input ended @@ -5678,23 +5835,23 @@ class binary_writer { write_number(static_cast(0x60 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x78)); write_number(static_cast(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x79)); write_number(static_cast(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x7A)); write_number(static_cast(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x7B)); write_number(static_cast(N)); @@ -5716,23 +5873,23 @@ class binary_writer { write_number(static_cast(0x80 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x98)); write_number(static_cast(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x99)); write_number(static_cast(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x9A)); write_number(static_cast(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x9B)); write_number(static_cast(N)); @@ -5755,23 +5912,23 @@ class binary_writer { write_number(static_cast(0xA0 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xB8)); write_number(static_cast(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xB9)); write_number(static_cast(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xBA)); write_number(static_cast(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xBB)); write_number(static_cast(N)); @@ -5939,19 +6096,19 @@ class binary_writer // fixstr write_number(static_cast(0xA0 | N)); } - else if (N <= 255) + else if (N <= (std::numeric_limits::max)()) { // str 8 oa->write_character(static_cast(0xD9)); write_number(static_cast(N)); } - else if (N <= 65535) + else if (N <= (std::numeric_limits::max)()) { // str 16 oa->write_character(static_cast(0xDA)); write_number(static_cast(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits::max)()) { // str 32 oa->write_character(static_cast(0xDB)); @@ -5974,13 +6131,13 @@ class binary_writer // fixarray write_number(static_cast(0x90 | N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits::max)()) { // array 16 oa->write_character(static_cast(0xDC)); write_number(static_cast(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { // array 32 oa->write_character(static_cast(0xDD)); @@ -6004,13 +6161,13 @@ class binary_writer // fixmap write_number(static_cast(0x80 | (N & 0xF))); } - else if (N <= 65535) + else if (N <= (std::numeric_limits::max)()) { // map 16 oa->write_character(static_cast(0xDE)); write_number(static_cast(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits::max)()) { // map 32 oa->write_character(static_cast(0xDF)); @@ -6031,6 +6188,110 @@ class binary_writer } } + /*! + @brief[in] j JSON value to serialize + */ + void write_ubjson(const BasicJsonType& j, const bool use_count = false) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(static_cast('Z')); + break; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? static_cast('T') + : static_cast('F')); + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float); + break; + } + + case value_t::string: + { + oa->write_character(static_cast('S')); + write_number_with_ubjson_prefix(j.m_value.string->size()); + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + oa->write_character(static_cast('[')); + + if (use_count) + { + oa->write_character(static_cast('#')); + write_number_with_ubjson_prefix(j.m_value.array->size()); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count); + } + + if (not use_count) + { + oa->write_character(static_cast(']')); + } + + break; + } + + case value_t::object: + { + oa->write_character(static_cast('{')); + + if (use_count) + { + oa->write_character(static_cast('#')); + write_number_with_ubjson_prefix(j.m_value.object->size()); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size()); + oa->write_characters( + reinterpret_cast(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count); + } + + if (not use_count) + { + oa->write_character(static_cast('}')); + } + + break; + } + + default: + break; + } + } + private: /* @brief write a number to output input @@ -6058,6 +6319,82 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + template + void write_number_with_ubjson_prefix(const NumberType n) + { + if (std::is_floating_point::value) + { + oa->write_character(static_cast('D')); // float64 + write_number(n); + } + else if (std::is_unsigned::value) + { + if (n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('i')); // uint8 + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('U')); // uint8 + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('I')); // int16 + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('l')); // int32 + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('L')); // int64 + write_number(static_cast(n)); + } + else + { + // TODO: replace by exception + assert(false); + } + } + else + { + if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('i')); // int8 + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('U')); // uint8 + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('I')); // int16 + write_number(static_cast(n)); + } + else if (-(std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('l')); // int32 + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + oa->write_character(static_cast('L')); // int64 + write_number(static_cast(n)); + } + else + { + // TODO: replace by exception + assert(false); + } + } + } + private: /// whether we can assume little endianess const bool is_little_endian = binary_reader::little_endianess(); @@ -13511,6 +13848,23 @@ class basic_json binary_writer(o).write_msgpack(j); } + static std::vector to_ubjson(const basic_json& j) + { + std::vector result; + to_ubjson(j, result); + return result; + } + + static void to_ubjson(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_ubjson(j); + } + + static void to_ubjson(const basic_json& j, detail::output_adapter o) + { + binary_writer(o).write_ubjson(j); + } + /*! @brief create a JSON value from an input in CBOR format @@ -13705,6 +14059,19 @@ class basic_json return binary_reader(detail::input_adapter(std::forward(a1), std::forward(a2))).parse_msgpack(strict); } + static basic_json from_ubjson(detail::input_adapter i, + const bool strict = true) + { + return binary_reader(i).parse_ubjson(strict); + } + + template::value, int> = 0> + static basic_json from_ubjson(A1 && a1, A2 && a2, const bool strict = true) + { + return binary_reader(detail::input_adapter(std::forward(a1), std::forward(a2))).parse_ubjson(strict); + } + /// @} ////////////////////////// diff --git a/test/Makefile b/test/Makefile index a170acb9..d13fa1e8 100644 --- a/test/Makefile +++ b/test/Makefile @@ -39,6 +39,7 @@ SOURCES = src/unit.cpp \ src/unit-regression.cpp \ src/unit-serialization.cpp \ src/unit-testsuites.cpp \ + src/unit-ubjson.cpp \ src/unit-unicode.cpp OBJECTS = $(SOURCES:.cpp=.o) diff --git a/test/data/universal-binary-json-java/CouchDB4k.formatted.json b/test/data/universal-binary-json-java/CouchDB4k.formatted.json new file mode 100644 index 00000000..d12387da --- /dev/null +++ b/test/data/universal-binary-json-java/CouchDB4k.formatted.json @@ -0,0 +1,275 @@ +{ + "data3":"ColreUHAtuYoUOx1N4ZloouQt2o6ugnUT6eYtS10gu7niM8i0vEiNufpk1RlMQXaHXlIwQBDsMFDFUQcFeg2vW5eD259Xm", + "data4":"zCxriJhL726WNNTdJJzurgSA8vKT6rHA0cFCb9koZcLUMXg4rmoXVPqIHWYaCV0ovl2t6xm7I1Hm36jXpLlXEb8fRfbwBeTW2V0OAsVqYH8eAT", + "data0":"9EVqHm5ARqcEB5jq2D14U2bCJPyBY0JWDr1Tjh8gTB0sWUNjqYiWDxFzlx6S", + "data7":"Bi1ujcgEvfADfBeyZudE7nwxc3Ik8qpYjsJIfKmwOMEbV2L3Bi0x2tcRpGuf4fiyvIbypDvJN1PPdQtfQW1Gv6zccXHwwZwKzUq6", + "data5":{ + "integers":[ + 756509, + 116117, + 776378, + 275045, + 703447, + 50156, + 685803, + 147958, + 941747, + 905651, + 57367, + 530248, + 312888, + 740951, + 988947, + 450154 + ], + "float1":76.572, + "float2":83.5299, + "nested1":{ + "integers":[ + 756509, + 116117, + 776378, + 275045, + 703447, + 50156, + 685803, + 147958, + 941747, + 905651, + 57367, + 530248, + 312888, + 740951, + 988947, + 450154 + ], + "floats":[ + 43121609.5543, + 99454976.3019, + 32945584.756, + 18122905.9212, + 45893183.44, + 63052200.6225, + 69032152.6897, + 3748217.6946, + 75449850.474, + 37111527.415, + 84852536.859, + 32906366.487, + 27027600.417, + 63874310.5614, + 39440408.51, + 97176857.1716, + 97438252.1171, + 49728043.5056, + 40818570.245, + 41415831.8949, + 24796297.4251, + 2819085.3449, + 84263963.4848, + 74503228.6878, + 67925677.403, + 4758851.9417, + 75227407.9214, + 76946667.8403, + 72518275.9469, + 94167085.9588, + 75883067.8321, + 27389831.6101, + 57987075.5053, + 1298995.2674, + 14590614.6939, + 45292214.2242, + 3332166.364, + 53784167.729, + 25193846.1867, + 81456965.477, + 68532032.39, + 73820009.7952, + 57736110.5717, + 37304166.7363, + 20054244.864, + 29746392.7397, + 86467624.6, + 45192685.8793, + 44008816.5186, + 1861872.8736, + 14595859.467, + 87795257.6703, + 57768720.8303, + 18290154.3126, + 45893183.44, + 63052200.6225, + 69032152.6897, + 3748217.6946, + 75449850.474, + 37111527.415, + 84852536.859, + 32906366.487, + 27027600.417, + 63874310.5614, + 39440408.51, + 97176857.1716, + 97438252.1171, + 49728043.5056, + 40818570.245, + 41415831.8949, + 24796297.4251, + 2819085.3449, + 84263963.4848, + 74503228.6878, + 67925677.403, + 4758851.9417, + 75227407.9214, + 76946667.8403, + 72518275.9469, + 94167085.9588, + 75883067.8321, + 27389831.6101, + 57987075.5053, + 1298995.2674, + 80858801.2712, + 98262252.4656, + 51612877.944, + 33397812.7835, + 36089655.3049, + 50164685.8153, + 16852105.5192, + 61171929.752, + 86376339.7175, + 73009014.5521, + 7397302.331, + 34345128.9589, + 98343269.4418, + 95039116.9058, + 44833102.5752, + 51052997.8873, + 22719195.6783, + 64883244.8699 + ] + }, + "nested2":{ + "integers":[ + 756509, + 116117, + 776378, + 275045, + 703447, + 50156, + 685803, + 147958, + 941747, + 905651, + 57367, + 530248, + 312888, + 740951, + 988947, + 450154 + ], + "float1":76.572, + "float2":83.5299 + } + }, + "strings":[ + "edx5XzRkPVeEW2MBQzQMcUSuMI4FntjhlJ9VGhQaBHKPEazAaT", + "2fQUbzRUax4A", + "jURcBZ0vrJcmf2roZUMzZJQoTsKZDIdj7KhO7itskKvM80jBU9", + "8jKLmo3N2zYdKyTyfTczfr2x6bPaarorlnTNJ7r8lIkiZyBvrP", + "jbUeAVOdBSPzYmYhH0sabUHUH39O5e", + "I8yAQKZsyZhMfpzWjArQU9pQ6PfU6b14q2eWvQjtCUdgAUxFjg", + "97N8ZmGcxRZO4ZabzRRcY4KVHqxJwQ8qY", + "0DtY1aWXmUfJENt9rYW9", + "DtpBUEppPwMnWexi8eIIxlXRO3GUpPgeNFG9ONpWJYvk8xBkVj", + "YsX8V2xOrTw6LhNIMMhO4F4VXFyXUXFr66L3sTkLWgFA9NZuBV", + "fKYYthv8iFvaYoFoYZyB", + "zGuLsPXoJqMbO4PcePteZfDMYFXdWtvNF8WvaplXypsd6" + ], + "data1":"9EVqHm5ARqcEB5jq21v2g0jVcG9CXB0Abk7uAF4NHYyTzeF3TnHhpZBECD14U2bCJPyBY0JWDr1Tjh8gTB0sWUNjqYiWDxFzlx6S", + "integers":[ + 756509, + 116117, + 776378, + 275045, + 703447, + 50156, + 685803, + 147958, + 941747, + 905651, + 57367, + 530248, + 312888, + 740951, + 988947, + 450154 + ], + "more_nested":{ + "integers":[ + 756509, + 116117, + 776378, + 275045, + 703447, + 50156, + 685803, + 147958, + 941747, + 905651, + 57367, + 530248, + 312888, + 740951, + 988947, + 450154 + ], + "float1":76.572, + "float2":83.5299, + "nested1":{ + "integers":[ + 756509, + 116117, + 776378, + 275045, + 703447, + 50156, + 685803, + 147958, + 941747, + 905651, + 57367, + 530248, + 312888, + 740951, + 988947, + 450154 + ] + }, + "nested2":{ + "strings":[ + "2fQUbzRUax4A", + "jURcBZ0vrJcmf2roZUMzZJQoTsKZDIdj7KhO7itskKvM80jBU9", + "8jKLmo3N2zYdKyTyfTczfr2x6bPaarorlnTNJ7r8lIkiZyBvrP", + "jbUeAVOdBSPzYmYhH0sabUHUH39O5e", + "I8yAQKZsyZhMfpzWjArQU9pQ6PfU6b14q2eWvQjtCUdgAUxFjg", + "97N8ZmGcxRZO4ZabzRRcY4KVHqxJwQ8qY", + "0DtY1aWXmUfJENt9rYW9", + "DtpBUEppPwMnWexi8eIIxlXRO3GUpPgeNFG9ONpWJYvk8xBkVj", + "YsX8V2xOrTw6LhNIMMhO4F4VXFyXUXFr66L3sTkLWgFA9NZuBV", + "fKYYthv8iFvaYoFoYZyB", + "zGuLsPXoJqMbO4PcePteZfDMYFXdWtvNF8WvaplXypsd6" + ], + "integers":[ + 756509, + 116117, + 776378, + 57367, + 530248, + 312888, + 740951, + 988947, + 450154 + ] + } + } +} \ No newline at end of file diff --git a/test/data/universal-binary-json-java/CouchDB4k.ubj b/test/data/universal-binary-json-java/CouchDB4k.ubj new file mode 100644 index 0000000000000000000000000000000000000000..75a6465bc74dcbec4c856640935b331b9e6fa0c3 GIT binary patch literal 2724 zcmeHJS;!kz6i!{Lty(JHh+svO#X1O!TnIw~ClDU(~ zB#OA;LVr|(3b3oHj|lF=@NgTQFk$2_N-_F^*|#C;+nlR=Tsaf3YR z8za;=DlN)uc&#bpsyflk)@a-un;hlR>cOyCd`YR&9CL#Ox-oVt%z{5A%DI9@v=)W> zj4`uC~Evb8So>47Ly)jgRdPBe(5_*j~$Lj$1yR&#ZXu&Q#j_0-rgGaQV#YUGMRk%qs^{ zDgJ`b96^7_XVz?aiqD+zcDiIfUe9NaUC*V;uT!a+tn--@*PqO1R(|B;m0mxvqZ}|h z&i?Swh5+V~6KnXGp^GCIa={UQ!v(PI0XqBYuK~UEWCp|^ti2#^ElYqdI_Y*mqZ@rd z*FSa|2=LmcfNuM2H=q-?y#VNe_dZRP*IW+bo&pC5Kgs|)C0YsSdGn!kd7lMz>66QV zPJ4PkpwoXm2gH{ytblO-+66%OKX*N#TQ}Sc=-OM~1!3F8sdU>_X93#&%DsTLDs@0l z-IKby{lzMXZ+-tNpv{F9z~8rRCx}}fybI9zuRj5TXrBs5+L)fbOzr}7_vX(5UH8mq zfNpv7OF)}0I{;{n^Z|%FG9ieceBB51@PQjrTlY#p=kD7C{Ql}qslR7W2J}euMryI; zRS=iAe)})Y9PKQ<3ussGK0xLDc@TfvvjNbm$MyiaeA71|UbDRdsB4jFmb=rxT)yHf z5ccgJ06Oxh^HQ@rE&%j&@y$$~L&^y*Ewq7xFM%oWMemL?KkV26?4=?kT1wWHV!C)O+WoVHbbJmD!?Yu?c8 zq(EcB*BTCEW#dqA1|AUxmL|{kE+%+tMk}I)XI4k@MswY#Lq#i%6teDVWGhz~6RsXBezd4To7NK6Z{ssdMVYb& zRVQg!wSr}*Ni`g^AU3&iB1{!(Y~ppj^m4>>T zbSanTlfKcY<*VAL-gafSTB^w-T`=Pwl`y@gj|meSRFg<*p*kf|N9HA|Q!B8Aroqk( z&0s^4M0u?CkltqLl59rWti&s>{Z<8L)wwz6QQvUY#IZHv(ZSwQ7aB&1p!HK`1g|8k2p)3E=tp9nkSn-#l_Yfx$DJLCs7_P_t554~w^nRUZ MR{dt4`J-XwXD4K^8UO$Q literal 0 HcmV?d00001 diff --git a/test/data/universal-binary-json-java/LICENSE b/test/data/universal-binary-json-java/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/test/data/universal-binary-json-java/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/test/data/universal-binary-json-java/MediaContent.formatted.json b/test/data/universal-binary-json-java/MediaContent.formatted.json new file mode 100644 index 00000000..fe547871 --- /dev/null +++ b/test/data/universal-binary-json-java/MediaContent.formatted.json @@ -0,0 +1,34 @@ +{ + "Media":{ + "uri":"http://javaone.com/keynote.mpg", + "title":"Javaone Keynote", + "width":640, + "height":480, + "format":"video/mpg4", + "duration":18000000, + "size":58982400, + "bitrate":262144, + "persons":[ + "Bill Gates", + "Steve Jobs" + ], + "player":"JAVA", + "copyright":null + }, + "Images":[ + { + "uri":"http://javaone.com/keynote_large.jpg", + "title":"Javaone Keynote", + "width":1024, + "height":768, + "size":"LARGE" + }, + { + "uri":"http://javaone.com/keynote_small.jpg", + "title":"Javaone Keynote", + "width":320, + "height":240, + "size":"SMALL" + } + ] +} \ No newline at end of file diff --git a/test/data/universal-binary-json-java/MediaContent.ubj b/test/data/universal-binary-json-java/MediaContent.ubj new file mode 100644 index 0000000000000000000000000000000000000000..616531ce4938f44ec1dfac12194504e76cea6ca0 GIT binary patch literal 442 zcma)%Jx;?w5QW#jfGLnT0CGWXH&j#@2}L$elSmXaXcG_ihPBt)w=ptR3J$}uI08pt zBMT`FQp%?{Gv9p5MKGsIaCHuEs|BtaWBS`P-SB~{j*_)%(>EGBWhiO;8i5f;5-y6V zj~}NF!MmtTlQZV-5H=+0#@OI|BYai5pB1*@2+PrtcAy*<$u^3<@om@;H=pcYtC)f2id=E6ZO%@s+y( literal 0 HcmV?d00001 diff --git a/test/data/universal-binary-json-java/README b/test/data/universal-binary-json-java/README new file mode 100644 index 00000000..b4fc4059 --- /dev/null +++ b/test/data/universal-binary-json-java/README @@ -0,0 +1,357 @@ +Universal Binary JSON Java Library +http://ubjson.org + + +About this project... +--------------------- +This code base is actively under development and implements the latest +specification of Universal Binary JSON (Draft 8). + +I/O is handled through the following core classes: + + * UBJOutputStream + * UBJInputStream + * UBJInputStreamParser + +Additionally, if you are working with Java's NIO and need byte[]-based +results, you can wrap any of the above I/O classes around one of the highly +optimized custom byte[]-stream impls: + + * ByteArrayInputStream (optimized for reuse, not from JDK) + * ByteArrayOutputStream (optimized for reuse, not from JDK) + +If you are working with NIO and want maximum performance by using (and reusing) +direct ByteBuffers along with the UBJSON stream impls, take a look at the: + + * ByteBufferInputStream + * ByteBufferOutputStream + +classes. You can wrap any ByteBuffer source or destination with this stream type, +then wrap that stream type with a UBJSON stream impl and utilize the full +potential of Java's NIO with Universal Binary JSON without giving yourself an +ulcer. + +This allows you to re-use the streams over and over again in a pool of reusable +streams for high-performance I/O with no object creation and garbage collection +overhead; a perfect match for high frequency NIO-based communication. + +All of the core I/O classes have been stable for a while, with tweaks to make the +performance tighter and the error messages more informative over the last few +months. + +More Java-convenient reflection-based I/O classes are available in the +org.ubjson.io.reflect package, but they are under active development. + +There are other efforts (like utilities) in other sub portions of the source +tree. This project intends to eventually contain a multitude of UBJSON +abstraction layers, I/O methods and utilities. + + +Changelog +--------- +02-10-12 + * Added ByteBuffer input and output stream impls as compliments to the + re-usable byte[] stream impls. + + Provides a fast translation layer between standard Java stream-IO and the + new Buffer-based I/O in NIO (including transparent support for using + ultra-fast direct ByteBuffers). + + * Optimized some of the read/write methods by removing unnecessary bounds + checks that are done by subsequent Input or Output stream impls themselves. + +02-09-12 + * Fixed bug with readHugeAsBigInteger returning an invalid value and not + treating the underlying bytes as a string-encoded value. + + * Removed implicit buffer.flip() at the end of StreamDecoder; there is no + way to know what the caller had planned for the buffer before reading all + the data back out. Also the flip was in the wrong place and in the case of + an empty decode request (length=0) the flip would not have been performed, + providing the caller with a "full" buffer of nothing. + + * Rewrote all readHugeXXX method impls; now huge's can be read in as a + simple Number (BigInteger or BigDecimal) as well as raw bytes and even + decoded chars. Additionally the methods can even accept and use existing + buffers to write into to allow for tighter optimizations. + + * Rewrote all readStringXXX methods using the same optimizations and + flexibility that readHuge methods now use. + +02-07-12 + More Memory and CPU optimizations across all the I/O impls. + + * StreamDecoder was rewritten to no longer create a ByteBuffer on every + invocation and instead re-use the same one to decode from on every single call. + + * StreamDecoder now requires the call to pass in a CharBuffer instance to hold + the result of the decode operation. This avoids the creation of a CharBuffer + and allows for large-scale optimization by re-using existing buffers between + calls. + + * StreamEncoder was rewritten to no longer create a ByteBuffer on every + invocation either and now re-uses the same single instance over and over + again. + + * UBJOutputStream writeHuge and writeString series of methods were all + rewritten to accept a CharBuffer in the rawest form (no longer char[]) to stop + hiding the fact that the underlying encode operation required one. + + This gives the caller an opportunity to cache and re-use CharBuffers over + and over again if they can; otherwise this just pushes the CharBuffer.wrap() + call up to the caller instead of hiding it secretly in the method impl under + the guise of accepting a raw char[] (that it couldn't use directly). + + For callers that can re-use buffers, this will lead to big performance gains + now that were previously impossible. + + * UBJInputStream added readHuge and readString methods that accept an existing + CharBuffer argument to make use of the optimizations made in the Stream encoder + and decoder impls. + +01-15-12 + Huge performance boost for deserialization! + + StreamDecoder previously used separate read and write buffers for decoding + bytes to chars including the resulting char[] that was returned to the caller. + This design required at least 1 full array copy before returning a result in + the best case and 2x full array copies before returning the result in the + worst case. + + The rewrite removed the need for a write buffer entire as well as ALL array + copies; in the best OR worse case they never occur anymore. + + Raw performance boost of roughly 25% in all UBJ I/O classes as a result. + +12-01-11 through 01-24-12 + A large amount of work has continued on the core I/O classes (stream impls) + to help make them not only faster and more robust, but also more helpful. + When errors are encountered in the streams, they are reported along with the + stream positions. This is critical for debugging problems with corrupt + formats. + + Also provided ByteArray I/O stream classes that have the potential to provide + HUGE performance boosts for high frequency systems. + + Both these classes (ByteArrayInputStream and ByteArrayOutputStream) are + reusable and when wrapped by a UBJInputStream or UBJOutputStream, the top + level UBJ streams implicitly become reusable as well. + + Reusing the streams not only saves on object creation/GC cleanup but also + allows the caller to re-use the temporary byte[] used to translate to and + from the UBJ format, avoiding object churn entirely! + + This optimized design was chosen to be intentionally performant when combined + with NIO implementations as the ByteBuffer's can be used to wrap() existing + outbound buffers (avoiding the most expensive part of a buffer) or use + array() to get access to the underlying buffer that needs to be written to + the stream. + + In the case of direct ByteBuffers, there is no additional overhead added + because the calls to get or put are required anyway to pull or push the + values from the native memory location. + + This approach allows the fastest implementation of Universal Binary JSON + I/O possible in the JVM whether you are using the standard IO (stream) + classes or the NIO (ByteBuffer) classes in the JDK. + + Some ancillary work on UBJ-based command line utilities (viewers, converters, + etc.) has begun as well. + +11-28-11 + * Fixed UBJInputStreamParser implementation; nextType correctly implements + logic to skip existing element (if called back to back) as well as validate + the marker type it encounters before returning it to the caller. + + * Modified IObjectReader contract; a Parser implementation is required to + make traversing the UBJ stream possible without knowing what type of element + is next. + +11-27-11 + * Streamlined ByteArrayOutputStream implementation to ensure the capacity + of the underlying byte[] is never grown unless absolutely necessary. + + * Rewrote class Javadoc for ByteArrayOutputStream to include a code snippet + on how to use it. + +11-26-11 + * Fixed major bug in how 16, 32 and 64-bit integers are re-assembled when + read back from binary representations. + + * Added a numeric test to specifically catch this error if it ever pops up + again. + + * Optimized reading and writing of numeric values in Input and Output + stream implementations. + + * Optimized ObjectWriter implementation by streamlining the numeric read/write + logic and removing the sub-optimal force-compression of on-disk storage. + + * Fixed ObjectWriter to match exactly with the output written by + UBJOutputStream. + + * Normalized all testing between I/O classes so they use the same classes + to ensure parity. + +11-10-11 + * DRAFT 8 Spec Support Added. + + * Added support for the END marker (readEnd) to the InputStreams which is + required for proper unbounded-container support. + + * Added support for the END marker (writeEnd) to UBJOutputStream. + + UBJInputStreamParser must be used for properly support unbounded-containers + because you never know when the 'E' will be encountered marking the end; + so the caller needs to pay attention to the type marker that nextType() + returns and respond accordingly. + + * Added readHugeAsBytes to InputStream implementations, allowing the bytes + used to represent a HUGE to be read in their raw form with no decoding. + + This can save on processing as BigInteger and BigDecimal do their own decoding + of byte[] directly. + + * Added readHugeAsChars to InputStream implementations, allowing a HUGE + value to be read in as a raw char[] without trying to decode it to a + BigInteger or BigDecimal. + + * Added writeHuge(char[]) to support writing out HUGE values directly from + their raw char[] form without trying to decode from a BigInteger or + BigDecimal. + + * readArrayLength and readObjectLenght were modified to return -1 when an + unbounded container length is encountered (255). + + * Fixed UBJInputStreamParser.nextType to correctly skip past any NOOP + markers found in the underlying stream before returning a valid next type. + + * Normalized all reading of "next byte" to the singular + UBJInputStream.nextMarker method -- correctly skips over NOOP until end of + stream OR until the next valid marker byte, then returns it. + + * Modified readNull behavior to have no return type. It is already designed + to throw an exception when 'NULL' isn't found, no need for the additional + return type. + + * Began work on a simple abstract representation of the UBJ data types as + objects that can be assembled into maps and lists and written/read easily + using the IO package. + + This is intended to be a higher level of abstraction than the IO streams, + but lower level (and faster) than the reflection-based work that inspects + user-provided classes. + + * Refactored StreamDecoder and StreamEncoder into the core IO package, + because they are part of core IO. + + * Refactored StreamParser into the io.parser package to more clearly denote + its relationship to the core IO classes. It is a slightly higher level + abstraction ontop of the core IO, having it along side the core IO classes + while .reflect was a subpackage was confusing and suggested that + StreamParser was somehow intended as a swapable replacement for + UBJInputStream which is not how it is intended to be used. + + * Refactored org.ubjson.reflect to org.ubjson.io.reflect to more correctly + communicate the relationship -- the reflection-based classes are built on + the core IO classes and are just a higher abstraction to interact with UBJSON + with. + + * Renamed IDataType to IMarkerType to follow the naming convention for the + marker bytes set forth by the spec doc. + + +10-14-11 + * ObjectWriter rewritten and works correctly. Tested with the example test + data and wrote out the compressed and uncompressed formats files correctly + from their original object representation. + + * Added support for reading and writing huge values as BigInteger as well + as BigDecimal. + + * Added automatic numeric storage compression support to ObjectWriter - based + on the numeric value (regardless of type) the value will be stored as the + smallest possible representation in the UBJ format if requested. + + * Added mapping support for BigDecimal, BigInteger, AtomicInteger and + AtomicLong to ObjectWriter. + + * Added readNull and readBoolean to the UBJInputStream and + UBJInputStreamParser implementations to make the API feel complete and feel + more natural to use. + +10-10-11 + * com.ubjson.io AND com.ubjson.io.charset are finalized against the + Draft 8 specification. + + * The lowest level UBJInput/OuputStream classes were tightened up to run as + fast as possible showing an 800ns-per-op improvement in speed. + + * Profiled core UBJInput/OuputStream classes using HPROF for a few million + iterations and found no memory leaks and no performance traps; everything at + that low level is as tight as it can be. + + * Stream-parsing facilities were moved out of the overloaded UBJInputStream + class and into their own subclass called UBJInputStreamParser which operates + exactly like a pull-parsing scenario (calling nextType then switching on the + value and pulling the appropriate value out of the stream). + + * More example testing/benchmarking data checked into /test/java/com/ubjson/data + + Will begin reworking the Reflection based Object mapping in the + org.ubjson.reflect package now that the core IO classes are finalized. + + * Removed all old scratch test files from the org.ubjson package, this was + confusing. + +09-27-11 + * Initial check-in of core IO classes to read/write spec. + + +Status +------ +Using the standard UBJInputStream, UBJInputStreamParser and UBJOutputStream +implementations to manually read/write UBJ objects is stable and tuned for +optimal performance. + +Automatic mapping of objects to/from UBJ format via the reflection-based +implementation is not tuned yet. Writing is implemented, but not tuned for +optimal performance and reading still has to be written. + + * org.ubjson.io - STABLE + * org.ubjson.io.parser - STABLE + * org.ubjson.io.reflect - ALPHA + * org.ubjson.model - BETA + + +License +------- +This library is released under the Apache 2 License. See LICENSE. + + +Description +----------- +This project represents (the official?) Java implementations of the +Universal Binary JSON specification: http://ubjson.org + + +Example +------- +Comming soon... + + +Performance +----------- +Comming soon... + + +Reference +--------- +Universal Binary JSON Specification - http://ubjson.org +JSON Specification - http://json.org + + +Contact +------- +If you have questions, comments or bug reports for this software please contact +us at: software@thebuzzmedia.com \ No newline at end of file diff --git a/test/data/universal-binary-json-java/TwitterTimeline.formatted.json b/test/data/universal-binary-json-java/TwitterTimeline.formatted.json new file mode 100644 index 00000000..f56dd9b2 --- /dev/null +++ b/test/data/universal-binary-json-java/TwitterTimeline.formatted.json @@ -0,0 +1,80 @@ +{ + "id_str":"121769183821312000", + "retweet_count":0, + "in_reply_to_screen_name":null, + "in_reply_to_user_id":null, + "truncated":false, + "retweeted":false, + "possibly_sensitive":false, + "in_reply_to_status_id_str":null, + "entities":{ + "urls":[ + { + "url":"http:\/\/t.co\/wtioKkFS", + "display_url":"dlvr.it\/pWQy2", + "indices":[ + 33, + 53 + ], + "expanded_url":"http:\/\/dlvr.it\/pWQy2" + } + ], + "hashtags":[ + + ], + "user_mentions":[ + + ] + }, + "geo":null, + "place":null, + "coordinates":null, + "created_at":"Thu Oct 06 02:10:10 +0000 2011", + "in_reply_to_user_id_str":null, + "user":{ + "id_str":"77029015", + "profile_link_color":"009999", + "protected":false, + "url":"http:\/\/www.techday.co.nz\/", + "screen_name":"techdaynz", + "statuses_count":5144, + "profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1479058408\/techday_48_normal.jpg", + "name":"TechDay", + "default_profile_image":false, + "default_profile":false, + "profile_background_color":"131516", + "lang":"en", + "profile_background_tile":false, + "utc_offset":43200, + "description":"", + "is_translator":false, + "show_all_inline_media":false, + "contributors_enabled":false, + "profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/75893948\/Techday_Background.jpg", + "created_at":"Thu Sep 24 20:02:01 +0000 2009", + "profile_sidebar_fill_color":"efefef", + "follow_request_sent":false, + "friends_count":3215, + "followers_count":3149, + "time_zone":"Auckland", + "favourites_count":0, + "profile_sidebar_border_color":"eeeeee", + "profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1479058408\/techday_48_normal.jpg", + "following":false, + "geo_enabled":false, + "notifications":false, + "profile_use_background_image":true, + "listed_count":151, + "verified":false, + "profile_text_color":"333333", + "location":"Ponsonby, Auckland, NZ", + "id":77029015, + "profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/75893948\/Techday_Background.jpg" + }, + "contributors":null, + "source":"\u003Ca href=\"http:\/\/dlvr.it\" rel=\"nofollow\"\u003Edlvr.it\u003C\/a\u003E", + "in_reply_to_status_id":null, + "favorited":false, + "id":121769183821312000, + "text":"Apple CEO's message to employees http:\/\/t.co\/wtioKkFS" +} \ No newline at end of file diff --git a/test/data/universal-binary-json-java/TwitterTimeline.ubj b/test/data/universal-binary-json-java/TwitterTimeline.ubj new file mode 100644 index 0000000000000000000000000000000000000000..19702d4bb147d330ce7bfbc9d819e065c83cafad GIT binary patch literal 1792 zcmai#-)|d55XVDeQYWz~v94=c8hA&jM8tr6*PZEWcWGk<-lbe6kmM^C zPB#~p#~jC&kC5+RH9z9S?Mf^6U~dGvj~`Rem}nfuTAM^DIyv_7kU%*Gj9dKbVw`#B zvGx2@&u^RsK7GCKh_~-G{2N>)swUkos=9B6gKbdC(aH)mkhxpkd zs+jN*V8}CR*@mAG?`}O0@!`T^z~jr2A=M@e7^LZTaEc|#)d+VFo%WI_>tU=v6L5AitG${I1q$O;)K zK9ElNwOax&0W!xA@jSngs8QSoa_xBeN{l+~Zm-#Ebt4Cj^2uH-3FJusC&0ImdW{w- z?u<0(2TK9^J^W&^K@b%JoUUn*C&ovI70NP(^uENJZ9VEGCR zd6@UbqLMAVbB#Bz. +Copyright (c) 2013-2017 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#include "catch.hpp" + +#include "json.hpp" +using nlohmann::json; + +#include + +TEST_CASE("UBJSON") +{ + SECTION("individual values") + { + SECTION("discarded") + { + // discarded values are not serialized + json j = json::value_t::discarded; + const auto result = json::to_ubjson(j); + CHECK(result.empty()); + } + + SECTION("null") + { + json j = nullptr; + std::vector expected = {'Z'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("boolean") + { + SECTION("true") + { + json j = true; + std::vector expected = {'T'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("false") + { + json j = false; + std::vector expected = {'F'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("number") + { + SECTION("signed") + { + SECTION("-9223372036854775808..-2147483649 (int64)") + { + std::vector numbers; + numbers.push_back(INT64_MIN); + numbers.push_back(-1000000000000000000); + numbers.push_back(-100000000000000000); + numbers.push_back(-10000000000000000); + numbers.push_back(-1000000000000000); + numbers.push_back(-100000000000000); + numbers.push_back(-10000000000000); + numbers.push_back(-1000000000000); + numbers.push_back(-100000000000); + numbers.push_back(-10000000000); + numbers.push_back(-2147483649); + for (auto i : numbers) + { + CAPTURE(i); + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('L')); + expected.push_back(static_cast((i >> 56) & 0xff)); + expected.push_back(static_cast((i >> 48) & 0xff)); + expected.push_back(static_cast((i >> 40) & 0xff)); + expected.push_back(static_cast((i >> 32) & 0xff)); + expected.push_back(static_cast((i >> 24) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 9); + + // check individual bytes + CHECK(result[0] == 'L'); + uint64_t restored = (static_cast(result[1]) << 070) + + (static_cast(result[2]) << 060) + + (static_cast(result[3]) << 050) + + (static_cast(result[4]) << 040) + + (static_cast(result[5]) << 030) + + (static_cast(result[6]) << 020) + + (static_cast(result[7]) << 010) + + static_cast(result[8]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("-2147483648..-32769 (int32)") + { + std::vector numbers; + numbers.push_back(-32769); + numbers.push_back(-100000); + numbers.push_back(-1000000); + numbers.push_back(-10000000); + numbers.push_back(-100000000); + numbers.push_back(-1000000000); + numbers.push_back(-2147483648); + for (auto i : numbers) + { + CAPTURE(i); + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('l')); + expected.push_back(static_cast((i >> 24) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 5); + + // check individual bytes + CHECK(result[0] == 'l'); + int32_t restored = (static_cast(result[1]) << 030) + + (static_cast(result[2]) << 020) + + (static_cast(result[3]) << 010) + + static_cast(result[4]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("-32768..-129 (int16)") + { + for (int32_t i = -32768; i <= -129; ++i) + { + CAPTURE(i); + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('I')); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'I'); + int16_t restored = static_cast(((result[1] << 8) + result[2])); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("-9263 (int16)") + { + json j = -9263; + std::vector expected = {'I', 0xdb, 0xd1}; + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'I'); + int16_t restored = static_cast(((result[1] << 8) + result[2])); + CHECK(restored == -9263); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("-128..-1 (int8)") + { + for (auto i = -128; i <= -1; ++i) + { + CAPTURE(i); + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back('i'); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'i'); + CHECK(static_cast(result[1]) == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("0..127 (int8)") + { + for (size_t i = 0; i <= 127; ++i) + { + CAPTURE(i); + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('i')); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'i'); + CHECK(result[1] == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("128..255 (uint8)") + { + for (size_t i = 128; i <= 255; ++i) + { + CAPTURE(i); + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('U')); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'U'); + CHECK(result[1] == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("256..32767 (int16)") + { + for (size_t i = 256; i <= 32767; ++i) + { + CAPTURE(i); + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('I')); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'I'); + uint16_t restored = static_cast(static_cast(result[1]) * 256 + static_cast(result[2])); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("65536..2147483647 (int32)") + { + for (uint32_t i : + { + 65536u, 77777u, 1048576u + }) + { + CAPTURE(i); + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back('l'); + expected.push_back(static_cast((i >> 24) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 5); + + // check individual bytes + CHECK(result[0] == 'l'); + uint32_t restored = (static_cast(result[1]) << 030) + + (static_cast(result[2]) << 020) + + (static_cast(result[3]) << 010) + + static_cast(result[4]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("2147483648..9223372036854775807 (int64)") + { + for (uint64_t i : + { + 2147483648ul, 9223372036854775807ul + }) + { + CAPTURE(i); + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back('L'); + expected.push_back(static_cast((i >> 070) & 0xff)); + expected.push_back(static_cast((i >> 060) & 0xff)); + expected.push_back(static_cast((i >> 050) & 0xff)); + expected.push_back(static_cast((i >> 040) & 0xff)); + expected.push_back(static_cast((i >> 030) & 0xff)); + expected.push_back(static_cast((i >> 020) & 0xff)); + expected.push_back(static_cast((i >> 010) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 9); + + // check individual bytes + CHECK(result[0] == 'L'); + uint64_t restored = (static_cast(result[1]) << 070) + + (static_cast(result[2]) << 060) + + (static_cast(result[3]) << 050) + + (static_cast(result[4]) << 040) + + (static_cast(result[5]) << 030) + + (static_cast(result[6]) << 020) + + (static_cast(result[7]) << 010) + + static_cast(result[8]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + } + + SECTION("unsigned") + { + SECTION("0..127 (int8)") + { + for (size_t i = 0; i <= 127; ++i) + { + CAPTURE(i); + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('i'); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'i'); + uint8_t restored = static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("128..255 (uint8)") + { + for (size_t i = 128; i <= 255; ++i) + { + CAPTURE(i); + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('U'); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'U'); + uint8_t restored = static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("256..32767 (int16)") + { + for (size_t i = 256; i <= 32767; ++i) + { + CAPTURE(i); + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('I'); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'I'); + uint16_t restored = static_cast(static_cast(result[1]) * 256 + static_cast(result[2])); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("65536..2147483647 (int32)") + { + for (uint32_t i : + { + 65536u, 77777u, 1048576u + }) + { + CAPTURE(i); + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('l'); + expected.push_back(static_cast((i >> 24) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 5); + + // check individual bytes + CHECK(result[0] == 'l'); + uint32_t restored = (static_cast(result[1]) << 030) + + (static_cast(result[2]) << 020) + + (static_cast(result[3]) << 010) + + static_cast(result[4]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("2147483648..9223372036854775807 (int64)") + { + for (uint64_t i : + { + 2147483648ul, 9223372036854775807ul + }) + { + CAPTURE(i); + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('L'); + expected.push_back(static_cast((i >> 070) & 0xff)); + expected.push_back(static_cast((i >> 060) & 0xff)); + expected.push_back(static_cast((i >> 050) & 0xff)); + expected.push_back(static_cast((i >> 040) & 0xff)); + expected.push_back(static_cast((i >> 030) & 0xff)); + expected.push_back(static_cast((i >> 020) & 0xff)); + expected.push_back(static_cast((i >> 010) & 0xff)); + expected.push_back(static_cast(i & 0xff)); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == 9); + + // check individual bytes + CHECK(result[0] == 'L'); + uint64_t restored = (static_cast(result[1]) << 070) + + (static_cast(result[2]) << 060) + + (static_cast(result[3]) << 050) + + (static_cast(result[4]) << 040) + + (static_cast(result[5]) << 030) + + (static_cast(result[6]) << 020) + + (static_cast(result[7]) << 010) + + static_cast(result[8]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + } + + SECTION("float64") + { + SECTION("3.1415925") + { + double v = 3.1415925; + json j = v; + std::vector expected = + { + 'D', 0x40, 0x09, 0x21, 0xfb, 0x3f, 0xa6, 0xde, 0xfc + }; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + CHECK(json::from_ubjson(result) == v); + } + } + } + + SECTION("string") + { + SECTION("N = 0..127") + { + for (size_t N = 0; N <= 127; ++N) + { + CAPTURE(N); + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector + std::vector expected; + expected.push_back('S'); + expected.push_back('i'); + expected.push_back(static_cast(N)); + for (size_t i = 0; i < N; ++i) + { + expected.push_back('x'); + } + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == N + 3); + // check that no null byte is appended + if (N > 0) + { + CHECK(result.back() != '\x00'); + } + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("N = 128..255") + { + for (size_t N = 128; N <= 255; ++N) + { + CAPTURE(N); + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector + std::vector expected; + expected.push_back('S'); + expected.push_back('U'); + expected.push_back(static_cast(N)); + for (size_t i = 0; i < N; ++i) + { + expected.push_back('x'); + } + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == N + 3); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("N = 256..32767") + { + for (size_t N : + { + 256u, 999u, 1025u, 3333u, 2048u, 32767u + }) + { + CAPTURE(N); + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector (hack: create string first) + std::vector expected(N, 'x'); + // reverse order of commands, because we insert at begin() + expected.insert(expected.begin(), static_cast(N & 0xff)); + expected.insert(expected.begin(), static_cast((N >> 8) & 0xff)); + expected.insert(expected.begin(), 'I'); + expected.insert(expected.begin(), 'S'); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == N + 4); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("N = 65536..2147483647") + { + for (size_t N : + { + 65536u, 77777u, 1048576u + }) + { + CAPTURE(N); + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector (hack: create string first) + std::vector expected(N, 'x'); + // reverse order of commands, because we insert at begin() + expected.insert(expected.begin(), static_cast(N & 0xff)); + expected.insert(expected.begin(), static_cast((N >> 8) & 0xff)); + expected.insert(expected.begin(), static_cast((N >> 16) & 0xff)); + expected.insert(expected.begin(), static_cast((N >> 24) & 0xff)); + expected.insert(expected.begin(), 'l'); + expected.insert(expected.begin(), 'S'); + + // compare result + size + const auto result = json::to_ubjson(j); + CHECK(result == expected); + CHECK(result.size() == N + 6); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + } + + SECTION("array") + { + SECTION("empty") + { + json j = json::array(); + std::vector expected = {'[', ']'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("[null]") + { + json j = {nullptr}; + std::vector expected = {'[', 'Z', ']'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("[1,2,3,4,5]") + { + json j = json::parse("[1,2,3,4,5]"); + std::vector expected = {'[', 'i', 1, 'i', 2, 'i', 3, 'i', 4, 'i', 5, ']'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("[[[[]]]]") + { + json j = json::parse("[[[[]]]]"); + std::vector expected = {'[', '[', '[', '[', ']', ']', ']', ']'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("array with uint16_t elements") + { + json j(257, nullptr); + std::vector expected(j.size() + 2, 'Z'); // all null + expected[0] = '['; // opening array + expected[258] = ']'; // closing array + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("array with uint32_t elements") + { + json j(65793, nullptr); + std::vector expected(j.size() + 2, 'Z'); // all null + expected[0] = '['; // opening array + expected[65794] = ']'; // closing array + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + + SECTION("object") + { + SECTION("empty") + { + json j = json::object(); + std::vector expected = {'{', '}'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("{\"\":null}") + { + json j = {{"", nullptr}}; + std::vector expected = {'{', 'i', 0, 'Z', '}'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("{\"a\": {\"b\": {\"c\": {}}}}") + { + json j = json::parse("{\"a\": {\"b\": {\"c\": {}}}}"); + std::vector expected = + { + '{', 'i', 1, 'a', '{', 'i', 1, 'b', '{', 'i', 1, 'c', '{', '}', '}', '}', '}' + }; + const auto result = json::to_ubjson(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + } + } + + SECTION("errors") + { + SECTION("empty byte vector") + { + CHECK_THROWS_AS(json::from_ubjson(std::vector()), json::parse_error&); + CHECK_THROWS_WITH(json::from_ubjson(std::vector()), + "[json.exception.parse_error.110] parse error at 1: unexpected end of input"); + } + + SECTION("too short byte vector") + { + } + + SECTION("unsupported bytes") + { + SECTION("concrete examples") + { + CHECK_THROWS_AS(json::from_cbor(std::vector({0x1c})), json::parse_error&); + CHECK_THROWS_WITH(json::from_cbor(std::vector({0x1c})), + "[json.exception.parse_error.112] parse error at 1: error reading CBOR; last byte: 0x1C"); + CHECK_THROWS_AS(json::from_cbor(std::vector({0xf8})), json::parse_error&); + CHECK_THROWS_WITH(json::from_cbor(std::vector({0xf8})), + "[json.exception.parse_error.112] parse error at 1: error reading CBOR; last byte: 0xF8"); + } + + SECTION("all unsupported bytes") + { + for (auto byte : + { + // ? + 0x1c, 0x1d, 0x1e, 0x1f, + // ? + 0x3c, 0x3d, 0x3e, 0x3f, + // byte strings + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + // byte strings + 0x58, 0x59, 0x5a, 0x5b, + // ? + 0x5c, 0x5d, 0x5e, + // byte string + 0x5f, + // ? + 0x7c, 0x7d, 0x7e, + // ? + 0x9c, 0x9d, 0x9e, + // ? + 0xbc, 0xbd, 0xbe, + // date/time + 0xc0, 0xc1, + // bignum + 0xc2, 0xc3, + // fraction + 0xc4, + // bigfloat + 0xc5, + // tagged item + 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, + // expected conversion + 0xd5, 0xd6, 0xd7, + // more tagged items + 0xd8, 0xd9, 0xda, 0xdb, + // ? + 0xdc, 0xdd, 0xde, 0xdf, + // (simple value) + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, + // undefined + 0xf7, + // simple value + 0xf8 + }) + { + CHECK_THROWS_AS(json::from_cbor(std::vector({static_cast(byte)})), json::parse_error&); + } + } + } + + SECTION("invalid string in map") + { + CHECK_THROWS_AS(json::from_cbor(std::vector({0xa1, 0xff, 0x01})), json::parse_error&); + CHECK_THROWS_WITH(json::from_cbor(std::vector({0xa1, 0xff, 0x01})), + "[json.exception.parse_error.113] parse error at 2: expected a CBOR string; last byte: 0xFF"); + } + + SECTION("strict mode") + { + std::vector vec = {0xf6, 0xf6}; + SECTION("non-strict mode") + { + const auto result = json::from_cbor(vec, false); + CHECK(result == json()); + } + + SECTION("strict mode") + { + CHECK_THROWS_AS(json::from_cbor(vec), json::parse_error&); + CHECK_THROWS_WITH(json::from_cbor(vec), + "[json.exception.parse_error.110] parse error at 2: expected end of input"); + } + } + } +} + +// use this testcase outside [hide] to run it with Valgrind +TEST_CASE("single CBOR roundtrip") +{ + SECTION("sample.json") + { + std::string filename = "test/data/json_testsuite/sample.json"; + + // parse JSON file + std::ifstream f_json(filename); + json j1 = json::parse(f_json); + + // parse MessagePack file + std::ifstream f_cbor(filename + ".cbor", std::ios::binary); + std::vector packed((std::istreambuf_iterator(f_cbor)), + std::istreambuf_iterator()); + json j2; + CHECK_NOTHROW(j2 = json::from_cbor(packed)); + + // compare parsed JSON values + CHECK(j1 == j2); + + SECTION("roundtrips") + { + SECTION("std::ostringstream") + { + std::ostringstream ss; + json::to_cbor(j1, ss); + json j3 = json::from_cbor(ss.str()); + CHECK(j1 == j3); + } + + SECTION("std::string") + { + std::string s; + json::to_cbor(j1, s); + json j3 = json::from_cbor(s); + CHECK(j1 == j3); + } + } + + // check with different start index + packed.insert(packed.begin(), 5, 0xff); + CHECK(j1 == json::from_cbor(packed.begin() + 5, packed.end())); + } +} + +TEST_CASE("CBOR roundtrips", "[hide]") +{ + SECTION("input from flynn") + { + for (std::string filename : + { + "test/data/json_nlohmann_tests/all_unicode.json", + "test/data/json.org/1.json", + "test/data/json.org/2.json", + "test/data/json.org/3.json", + "test/data/json.org/4.json", + "test/data/json.org/5.json", + "test/data/json_roundtrip/roundtrip01.json", + "test/data/json_roundtrip/roundtrip02.json", + "test/data/json_roundtrip/roundtrip03.json", + "test/data/json_roundtrip/roundtrip04.json", + "test/data/json_roundtrip/roundtrip05.json", + "test/data/json_roundtrip/roundtrip06.json", + "test/data/json_roundtrip/roundtrip07.json", + "test/data/json_roundtrip/roundtrip08.json", + "test/data/json_roundtrip/roundtrip09.json", + "test/data/json_roundtrip/roundtrip10.json", + "test/data/json_roundtrip/roundtrip11.json", + "test/data/json_roundtrip/roundtrip12.json", + "test/data/json_roundtrip/roundtrip13.json", + "test/data/json_roundtrip/roundtrip14.json", + "test/data/json_roundtrip/roundtrip15.json", + "test/data/json_roundtrip/roundtrip16.json", + "test/data/json_roundtrip/roundtrip17.json", + "test/data/json_roundtrip/roundtrip18.json", + "test/data/json_roundtrip/roundtrip19.json", + "test/data/json_roundtrip/roundtrip20.json", + "test/data/json_roundtrip/roundtrip21.json", + "test/data/json_roundtrip/roundtrip22.json", + "test/data/json_roundtrip/roundtrip23.json", + "test/data/json_roundtrip/roundtrip24.json", + "test/data/json_roundtrip/roundtrip25.json", + "test/data/json_roundtrip/roundtrip26.json", + "test/data/json_roundtrip/roundtrip27.json", + "test/data/json_roundtrip/roundtrip28.json", + "test/data/json_roundtrip/roundtrip29.json", + "test/data/json_roundtrip/roundtrip30.json", + "test/data/json_roundtrip/roundtrip31.json", + "test/data/json_roundtrip/roundtrip32.json", + "test/data/json_testsuite/sample.json", // kills AppVeyor + "test/data/json_tests/pass1.json", + "test/data/json_tests/pass2.json", + "test/data/json_tests/pass3.json", + "test/data/regression/floats.json", + "test/data/regression/signed_ints.json", + "test/data/regression/unsigned_ints.json", + "test/data/regression/working_file.json", + "test/data/nst_json_testsuite/test_parsing/y_array_arraysWithSpaces.json", + "test/data/nst_json_testsuite/test_parsing/y_array_empty-string.json", + "test/data/nst_json_testsuite/test_parsing/y_array_empty.json", + "test/data/nst_json_testsuite/test_parsing/y_array_ending_with_newline.json", + "test/data/nst_json_testsuite/test_parsing/y_array_false.json", + "test/data/nst_json_testsuite/test_parsing/y_array_heterogeneous.json", + "test/data/nst_json_testsuite/test_parsing/y_array_null.json", + "test/data/nst_json_testsuite/test_parsing/y_array_with_1_and_newline.json", + "test/data/nst_json_testsuite/test_parsing/y_array_with_leading_space.json", + "test/data/nst_json_testsuite/test_parsing/y_array_with_several_null.json", + "test/data/nst_json_testsuite/test_parsing/y_array_with_trailing_space.json", + "test/data/nst_json_testsuite/test_parsing/y_number.json", + "test/data/nst_json_testsuite/test_parsing/y_number_0e+1.json", + "test/data/nst_json_testsuite/test_parsing/y_number_0e1.json", + "test/data/nst_json_testsuite/test_parsing/y_number_after_space.json", + "test/data/nst_json_testsuite/test_parsing/y_number_double_close_to_zero.json", + "test/data/nst_json_testsuite/test_parsing/y_number_double_huge_neg_exp.json", + //"test/data/nst_json_testsuite/test_parsing/y_number_huge_exp.json", + "test/data/nst_json_testsuite/test_parsing/y_number_int_with_exp.json", + "test/data/nst_json_testsuite/test_parsing/y_number_minus_zero.json", + "test/data/nst_json_testsuite/test_parsing/y_number_negative_int.json", + "test/data/nst_json_testsuite/test_parsing/y_number_negative_one.json", + "test/data/nst_json_testsuite/test_parsing/y_number_negative_zero.json", + "test/data/nst_json_testsuite/test_parsing/y_number_real_capital_e.json", + "test/data/nst_json_testsuite/test_parsing/y_number_real_capital_e_neg_exp.json", + "test/data/nst_json_testsuite/test_parsing/y_number_real_capital_e_pos_exp.json", + "test/data/nst_json_testsuite/test_parsing/y_number_real_exponent.json", + "test/data/nst_json_testsuite/test_parsing/y_number_real_fraction_exponent.json", + "test/data/nst_json_testsuite/test_parsing/y_number_real_neg_exp.json", + //"test/data/nst_json_testsuite/test_parsing/y_number_real_neg_overflow.json", + "test/data/nst_json_testsuite/test_parsing/y_number_real_pos_exponent.json", + //"test/data/nst_json_testsuite/test_parsing/y_number_real_pos_overflow.json", + "test/data/nst_json_testsuite/test_parsing/y_number_real_underflow.json", + "test/data/nst_json_testsuite/test_parsing/y_number_simple_int.json", + "test/data/nst_json_testsuite/test_parsing/y_number_simple_real.json", + //"test/data/nst_json_testsuite/test_parsing/y_number_too_big_neg_int.json", + //"test/data/nst_json_testsuite/test_parsing/y_number_too_big_pos_int.json", + //"test/data/nst_json_testsuite/test_parsing/y_number_very_big_negative_int.json", + "test/data/nst_json_testsuite/test_parsing/y_object.json", + "test/data/nst_json_testsuite/test_parsing/y_object_basic.json", + "test/data/nst_json_testsuite/test_parsing/y_object_duplicated_key.json", + "test/data/nst_json_testsuite/test_parsing/y_object_duplicated_key_and_value.json", + "test/data/nst_json_testsuite/test_parsing/y_object_empty.json", + "test/data/nst_json_testsuite/test_parsing/y_object_empty_key.json", + "test/data/nst_json_testsuite/test_parsing/y_object_escaped_null_in_key.json", + "test/data/nst_json_testsuite/test_parsing/y_object_extreme_numbers.json", + "test/data/nst_json_testsuite/test_parsing/y_object_long_strings.json", + "test/data/nst_json_testsuite/test_parsing/y_object_simple.json", + "test/data/nst_json_testsuite/test_parsing/y_object_string_unicode.json", + "test/data/nst_json_testsuite/test_parsing/y_object_with_newlines.json", + "test/data/nst_json_testsuite/test_parsing/y_string_1_2_3_bytes_UTF-8_sequences.json", + "test/data/nst_json_testsuite/test_parsing/y_string_UTF-16_Surrogates_U+1D11E_MUSICAL_SYMBOL_G_CLEF.json", + "test/data/nst_json_testsuite/test_parsing/y_string_accepted_surrogate_pair.json", + "test/data/nst_json_testsuite/test_parsing/y_string_accepted_surrogate_pairs.json", + "test/data/nst_json_testsuite/test_parsing/y_string_allowed_escapes.json", + "test/data/nst_json_testsuite/test_parsing/y_string_backslash_and_u_escaped_zero.json", + "test/data/nst_json_testsuite/test_parsing/y_string_backslash_doublequotes.json", + "test/data/nst_json_testsuite/test_parsing/y_string_comments.json", + "test/data/nst_json_testsuite/test_parsing/y_string_double_escape_a.json", + "test/data/nst_json_testsuite/test_parsing/y_string_double_escape_n.json", + "test/data/nst_json_testsuite/test_parsing/y_string_escaped_control_character.json", + "test/data/nst_json_testsuite/test_parsing/y_string_escaped_noncharacter.json", + "test/data/nst_json_testsuite/test_parsing/y_string_in_array.json", + "test/data/nst_json_testsuite/test_parsing/y_string_in_array_with_leading_space.json", + "test/data/nst_json_testsuite/test_parsing/y_string_last_surrogates_1_and_2.json", + "test/data/nst_json_testsuite/test_parsing/y_string_newline_uescaped.json", + "test/data/nst_json_testsuite/test_parsing/y_string_nonCharacterInUTF-8_U+10FFFF.json", + "test/data/nst_json_testsuite/test_parsing/y_string_nonCharacterInUTF-8_U+1FFFF.json", + "test/data/nst_json_testsuite/test_parsing/y_string_nonCharacterInUTF-8_U+FFFF.json", + "test/data/nst_json_testsuite/test_parsing/y_string_null_escape.json", + "test/data/nst_json_testsuite/test_parsing/y_string_one-byte-utf-8.json", + "test/data/nst_json_testsuite/test_parsing/y_string_pi.json", + "test/data/nst_json_testsuite/test_parsing/y_string_simple_ascii.json", + "test/data/nst_json_testsuite/test_parsing/y_string_space.json", + "test/data/nst_json_testsuite/test_parsing/y_string_three-byte-utf-8.json", + "test/data/nst_json_testsuite/test_parsing/y_string_two-byte-utf-8.json", + "test/data/nst_json_testsuite/test_parsing/y_string_u+2028_line_sep.json", + "test/data/nst_json_testsuite/test_parsing/y_string_u+2029_par_sep.json", + "test/data/nst_json_testsuite/test_parsing/y_string_uEscape.json", + "test/data/nst_json_testsuite/test_parsing/y_string_unescaped_char_delete.json", + "test/data/nst_json_testsuite/test_parsing/y_string_unicode.json", + "test/data/nst_json_testsuite/test_parsing/y_string_unicodeEscapedBackslash.json", + "test/data/nst_json_testsuite/test_parsing/y_string_unicode_2.json", + "test/data/nst_json_testsuite/test_parsing/y_string_unicode_U+200B_ZERO_WIDTH_SPACE.json", + "test/data/nst_json_testsuite/test_parsing/y_string_unicode_U+2064_invisible_plus.json", + "test/data/nst_json_testsuite/test_parsing/y_string_unicode_escaped_double_quote.json", + // "test/data/nst_json_testsuite/test_parsing/y_string_utf16.json", + "test/data/nst_json_testsuite/test_parsing/y_string_utf8.json", + "test/data/nst_json_testsuite/test_parsing/y_string_with_del_character.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_lonely_false.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_lonely_int.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_lonely_negative_real.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_lonely_null.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_lonely_string.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_lonely_true.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_string_empty.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_trailing_newline.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_true_in_array.json", + "test/data/nst_json_testsuite/test_parsing/y_structure_whitespace_array.json" + }) + { + CAPTURE(filename); + + // parse JSON file + std::ifstream f_json(filename); + json j1 = json::parse(f_json); + + SECTION("std::vector") + { + // parse CBOR file + std::ifstream f_cbor(filename + ".cbor", std::ios::binary); + std::vector packed( + (std::istreambuf_iterator(f_cbor)), + std::istreambuf_iterator()); + json j2; + CHECK_NOTHROW(j2 = json::from_cbor(packed)); + + // compare parsed JSON values + CHECK(j1 == j2); + } + + SECTION("std::ifstream") + { + // parse CBOR file + std::ifstream f_cbor(filename + ".cbor", std::ios::binary); + json j2; + CHECK_NOTHROW(j2 = json::from_cbor(f_cbor)); + + // compare parsed JSON values + CHECK(j1 == j2); + } + + SECTION("uint8_t* and size") + { + // parse CBOR file + std::ifstream f_cbor(filename + ".cbor", std::ios::binary); + std::vector packed( + (std::istreambuf_iterator(f_cbor)), + std::istreambuf_iterator()); + json j2; + CHECK_NOTHROW(j2 = json::from_cbor({packed.data(), packed.size()})); + + // compare parsed JSON values + CHECK(j1 == j2); + } + + SECTION("output to output adapters") + { + // parse CBOR file + std::ifstream f_cbor(filename + ".cbor", std::ios::binary); + std::vector packed( + (std::istreambuf_iterator(f_cbor)), + std::istreambuf_iterator()); + + SECTION("std::vector") + { + std::vector vec; + json::to_cbor(j1, vec); + CHECK(vec == packed); + } + } + } + } +} + +/* +TEST_CASE("Universal Binary JSON Specification Examples") +{ + for (std::string prefix : + { + "test/data/universal-binary-json-java/CouchDB4k", + "test/data/universal-binary-json-java/MediaContent", + "test/data/universal-binary-json-java/TwitterTimeline" + }) + { + CAPTURE(prefix); + + // parse JSON file + std::ifstream f_json(prefix + ".formatted.json"); + json j1 = json::parse(f_json); + + SECTION("std::vector") + { + // parse UBJSON file + std::ifstream f_ubjson(prefix + ".ubj", std::ios::binary); + std::vector packed( + (std::istreambuf_iterator(f_ubjson)), + std::istreambuf_iterator()); + json j2; + CHECK_NOTHROW(j2 = json::from_ubjson(packed)); + + // compare parsed JSON values + CHECK(j1 == j2); + } + } +} +*/ + + +/* +TEST_CASE("all first bytes", "[!throws]") +{ + // these bytes will fail immediately with exception parse_error.112 + std::set unsupported = + { + //// types not supported by this library + + // byte strings + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + // byte strings + 0x58, 0x59, 0x5a, 0x5b, 0x5f, + // date/time + 0xc0, 0xc1, + // bignum + 0xc2, 0xc3, + // decimal fracion + 0xc4, + // bigfloat + 0xc5, + // tagged item + 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, + 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd8, + 0xd9, 0xda, 0xdb, + // expected conversion + 0xd5, 0xd6, 0xd7, + // simple value + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xef, 0xf0, + 0xf1, 0xf2, 0xf3, + 0xf8, + // undefined + 0xf7, + + //// bytes not specified by CBOR + + 0x1c, 0x1d, 0x1e, 0x1f, + 0x3c, 0x3d, 0x3e, 0x3f, + 0x5c, 0x5d, 0x5e, + 0x7c, 0x7d, 0x7e, + 0x9c, 0x9d, 0x9e, + 0xbc, 0xbd, 0xbe, + 0xdc, 0xdd, 0xde, 0xdf, + 0xee, + 0xfc, 0xfe, 0xfd, + + /// break cannot be the first byte + + 0xff + }; + + for (auto i = 0; i < 256; ++i) + { + const auto byte = static_cast(i); + + try + { + json::from_cbor(std::vector(1, byte)); + } + catch (const json::parse_error& e) + { + // check that parse_error.112 is only thrown if the + // first byte is in the unsupported set + CAPTURE(e.what()); + if (std::find(unsupported.begin(), unsupported.end(), byte) != unsupported.end()) + { + CHECK(e.id == 112); + } + else + { + CHECK(e.id != 112); + } + } + } +} +*/ + +/* +TEST_CASE("examples from RFC 7049 Appendix A") +{ + SECTION("numbers") + { + CHECK(json::to_cbor(json::parse("0")) == std::vector({0x00})); + CHECK(json::parse("0") == json::from_cbor(std::vector({0x00}))); + + CHECK(json::to_cbor(json::parse("1")) == std::vector({0x01})); + CHECK(json::parse("1") == json::from_cbor(std::vector({0x01}))); + + CHECK(json::to_cbor(json::parse("10")) == std::vector({0x0a})); + CHECK(json::parse("10") == json::from_cbor(std::vector({0x0a}))); + + CHECK(json::to_cbor(json::parse("23")) == std::vector({0x17})); + CHECK(json::parse("23") == json::from_cbor(std::vector({0x17}))); + + CHECK(json::to_cbor(json::parse("24")) == std::vector({0x18, 0x18})); + CHECK(json::parse("24") == json::from_cbor(std::vector({0x18, 0x18}))); + + CHECK(json::to_cbor(json::parse("25")) == std::vector({0x18, 0x19})); + CHECK(json::parse("25") == json::from_cbor(std::vector({0x18, 0x19}))); + + CHECK(json::to_cbor(json::parse("100")) == std::vector({0x18, 0x64})); + CHECK(json::parse("100") == json::from_cbor(std::vector({0x18, 0x64}))); + + CHECK(json::to_cbor(json::parse("1000")) == std::vector({0x19, 0x03, 0xe8})); + CHECK(json::parse("1000") == json::from_cbor(std::vector({0x19, 0x03, 0xe8}))); + + CHECK(json::to_cbor(json::parse("1000000")) == std::vector({0x1a, 0x00, 0x0f, 0x42, 0x40})); + CHECK(json::parse("1000000") == json::from_cbor(std::vector({0x1a, 0x00, 0x0f, 0x42, 0x40}))); + + CHECK(json::to_cbor(json::parse("1000000000000")) == std::vector({0x1b, 0x00, 0x00, 0x00, 0xe8, 0xd4, 0xa5, 0x10, 0x00})); + CHECK(json::parse("1000000000000") == json::from_cbor(std::vector({0x1b, 0x00, 0x00, 0x00, 0xe8, 0xd4, 0xa5, 0x10, 0x00}))); + + CHECK(json::to_cbor(json::parse("18446744073709551615")) == std::vector({0x1b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff})); + CHECK(json::parse("18446744073709551615") == json::from_cbor(std::vector({0x1b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}))); + + // positive bignum is not supported + //CHECK(json::to_cbor(json::parse("18446744073709551616")) == std::vector({0xc2, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})); + //CHECK(json::parse("18446744073709551616") == json::from_cbor(std::vector({0xc2, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}))); + + //CHECK(json::to_cbor(json::parse("-18446744073709551616")) == std::vector({0x3b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff})); + //CHECK(json::parse("-18446744073709551616") == json::from_cbor(std::vector({0x3b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}))); + + // negative bignum is not supported + //CHECK(json::to_cbor(json::parse("-18446744073709551617")) == std::vector({0xc3, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})); + //CHECK(json::parse("-18446744073709551617") == json::from_cbor(std::vector({0xc3, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}))); + + CHECK(json::to_cbor(json::parse("-1")) == std::vector({0x20})); + CHECK(json::parse("-1") == json::from_cbor(std::vector({0x20}))); + + CHECK(json::to_cbor(json::parse("-10")) == std::vector({0x29})); + CHECK(json::parse("-10") == json::from_cbor(std::vector({0x29}))); + + CHECK(json::to_cbor(json::parse("-100")) == std::vector({0x38, 0x63})); + CHECK(json::parse("-100") == json::from_cbor(std::vector({0x38, 0x63}))); + + CHECK(json::to_cbor(json::parse("-1000")) == std::vector({0x39, 0x03, 0xe7})); + CHECK(json::parse("-1000") == json::from_cbor(std::vector({0x39, 0x03, 0xe7}))); + + // half-precision float + //CHECK(json::to_cbor(json::parse("0.0")) == std::vector({0xf9, 0x00, 0x00})); + CHECK(json::parse("0.0") == json::from_cbor(std::vector({0xf9, 0x00, 0x00}))); + + // half-precision float + //CHECK(json::to_cbor(json::parse("-0.0")) == std::vector({0xf9, 0x80, 0x00})); + CHECK(json::parse("-0.0") == json::from_cbor(std::vector({0xf9, 0x80, 0x00}))); + + // half-precision float + //CHECK(json::to_cbor(json::parse("1.0")) == std::vector({0xf9, 0x3c, 0x00})); + CHECK(json::parse("1.0") == json::from_cbor(std::vector({0xf9, 0x3c, 0x00}))); + + CHECK(json::to_cbor(json::parse("1.1")) == std::vector({0xfb, 0x3f, 0xf1, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9a})); + CHECK(json::parse("1.1") == json::from_cbor(std::vector({0xfb, 0x3f, 0xf1, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9a}))); + + // half-precision float + //CHECK(json::to_cbor(json::parse("1.5")) == std::vector({0xf9, 0x3e, 0x00})); + CHECK(json::parse("1.5") == json::from_cbor(std::vector({0xf9, 0x3e, 0x00}))); + + // half-precision float + //CHECK(json::to_cbor(json::parse("65504.0")) == std::vector({0xf9, 0x7b, 0xff})); + CHECK(json::parse("65504.0") == json::from_cbor(std::vector({0xf9, 0x7b, 0xff}))); + + //CHECK(json::to_cbor(json::parse("100000.0")) == std::vector({0xfa, 0x47, 0xc3, 0x50, 0x00})); + CHECK(json::parse("100000.0") == json::from_cbor(std::vector({0xfa, 0x47, 0xc3, 0x50, 0x00}))); + + //CHECK(json::to_cbor(json::parse("3.4028234663852886e+38")) == std::vector({0xfa, 0x7f, 0x7f, 0xff, 0xff})); + CHECK(json::parse("3.4028234663852886e+38") == json::from_cbor(std::vector({0xfa, 0x7f, 0x7f, 0xff, 0xff}))); + + CHECK(json::to_cbor(json::parse("1.0e+300")) == std::vector({0xfb, 0x7e, 0x37, 0xe4, 0x3c, 0x88, 0x00, 0x75, 0x9c})); + CHECK(json::parse("1.0e+300") == json::from_cbor(std::vector({0xfb, 0x7e, 0x37, 0xe4, 0x3c, 0x88, 0x00, 0x75, 0x9c}))); + + // half-precision float + //CHECK(json::to_cbor(json::parse("5.960464477539063e-8")) == std::vector({0xf9, 0x00, 0x01})); + CHECK(json::parse("-4.0") == json::from_cbor(std::vector({0xf9, 0xc4, 0x00}))); + + // half-precision float + //CHECK(json::to_cbor(json::parse("0.00006103515625")) == std::vector({0xf9, 0x04, 0x00})); + CHECK(json::parse("-4.0") == json::from_cbor(std::vector({0xf9, 0xc4, 0x00}))); + + // half-precision float + //CHECK(json::to_cbor(json::parse("-4.0")) == std::vector({0xf9, 0xc4, 0x00})); + CHECK(json::parse("-4.0") == json::from_cbor(std::vector({0xf9, 0xc4, 0x00}))); + + CHECK(json::to_cbor(json::parse("-4.1")) == std::vector({0xfb, 0xc0, 0x10, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66})); + CHECK(json::parse("-4.1") == json::from_cbor(std::vector({0xfb, 0xc0, 0x10, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66}))); + } + + SECTION("simple values") + { + CHECK(json::to_cbor(json::parse("false")) == std::vector({0xf4})); + CHECK(json::parse("false") == json::from_cbor(std::vector({0xf4}))); + + CHECK(json::to_cbor(json::parse("true")) == std::vector({0xf5})); + CHECK(json::parse("true") == json::from_cbor(std::vector({0xf5}))); + + CHECK(json::to_cbor(json::parse("true")) == std::vector({0xf5})); + CHECK(json::parse("true") == json::from_cbor(std::vector({0xf5}))); + } + + SECTION("strings") + { + CHECK(json::to_cbor(json::parse("\"\"")) == std::vector({0x60})); + CHECK(json::parse("\"\"") == json::from_cbor(std::vector({0x60}))); + + CHECK(json::to_cbor(json::parse("\"a\"")) == std::vector({0x61, 0x61})); + CHECK(json::parse("\"a\"") == json::from_cbor(std::vector({0x61, 0x61}))); + + CHECK(json::to_cbor(json::parse("\"IETF\"")) == std::vector({0x64, 0x49, 0x45, 0x54, 0x46})); + CHECK(json::parse("\"IETF\"") == json::from_cbor(std::vector({0x64, 0x49, 0x45, 0x54, 0x46}))); + + CHECK(json::to_cbor(json::parse("\"\\u00fc\"")) == std::vector({0x62, 0xc3, 0xbc})); + CHECK(json::parse("\"\\u00fc\"") == json::from_cbor(std::vector({0x62, 0xc3, 0xbc}))); + + CHECK(json::to_cbor(json::parse("\"\\u6c34\"")) == std::vector({0x63, 0xe6, 0xb0, 0xb4})); + CHECK(json::parse("\"\\u6c34\"") == json::from_cbor(std::vector({0x63, 0xe6, 0xb0, 0xb4}))); + + CHECK(json::to_cbor(json::parse("\"\\ud800\\udd51\"")) == std::vector({0x64, 0xf0, 0x90, 0x85, 0x91})); + CHECK(json::parse("\"\\ud800\\udd51\"") == json::from_cbor(std::vector({0x64, 0xf0, 0x90, 0x85, 0x91}))); + + // indefinite length strings + CHECK(json::parse("\"streaming\"") == json::from_cbor(std::vector({0x7f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0xff}))); + } + + SECTION("arrays") + { + CHECK(json::to_cbor(json::parse("[]")) == std::vector({0x80})); + CHECK(json::parse("[]") == json::from_cbor(std::vector({0x80}))); + + CHECK(json::to_cbor(json::parse("[1, 2, 3]")) == std::vector({0x83, 0x01, 0x02, 0x03})); + CHECK(json::parse("[1, 2, 3]") == json::from_cbor(std::vector({0x83, 0x01, 0x02, 0x03}))); + + CHECK(json::to_cbor(json::parse("[1, [2, 3], [4, 5]]")) == std::vector({0x83, 0x01, 0x82, 0x02, 0x03, 0x82, 0x04, 0x05})); + CHECK(json::parse("[1, [2, 3], [4, 5]]") == json::from_cbor(std::vector({0x83, 0x01, 0x82, 0x02, 0x03, 0x82, 0x04, 0x05}))); + + CHECK(json::to_cbor(json::parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]")) == std::vector({0x98, 0x19, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x18, 0x18, 0x19})); + CHECK(json::parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]") == json::from_cbor(std::vector({0x98, 0x19, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x18, 0x18, 0x19}))); + + // indefinite length arrays + CHECK(json::parse("[]") == json::from_cbor(std::vector({0x9f, 0xff}))); + CHECK(json::parse("[1, [2, 3], [4, 5]] ") == json::from_cbor(std::vector({0x9f, 0x01, 0x82, 0x02, 0x03, 0x9f, 0x04, 0x05, 0xff, 0xff}))); + CHECK(json::parse("[1, [2, 3], [4, 5]]") == json::from_cbor(std::vector({0x9f, 0x01, 0x82, 0x02, 0x03, 0x82, 0x04, 0x05, 0xff}))); + CHECK(json::parse("[1, [2, 3], [4, 5]]") == json::from_cbor(std::vector({0x83, 0x01, 0x82, 0x02, 0x03, 0x9f, 0x04, 0x05, 0xff}))); + CHECK(json::parse("[1, [2, 3], [4, 5]]") == json::from_cbor(std::vector({0x83, 0x01, 0x9f, 0x02, 0x03, 0xff, 0x82, 0x04, 0x05}))); + CHECK(json::parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]") == json::from_cbor(std::vector({0x9f, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x18, 0x18, 0x19, 0xff}))); + } + + SECTION("objects") + { + CHECK(json::to_cbor(json::parse("{}")) == std::vector({0xa0})); + CHECK(json::parse("{}") == json::from_cbor(std::vector({0xa0}))); + + CHECK(json::to_cbor(json::parse("{\"a\": 1, \"b\": [2, 3]}")) == std::vector({0xa2, 0x61, 0x61, 0x01, 0x61, 0x62, 0x82, 0x02, 0x03})); + CHECK(json::parse("{\"a\": 1, \"b\": [2, 3]}") == json::from_cbor(std::vector({0xa2, 0x61, 0x61, 0x01, 0x61, 0x62, 0x82, 0x02, 0x03}))); + + CHECK(json::to_cbor(json::parse("[\"a\", {\"b\": \"c\"}]")) == std::vector({0x82, 0x61, 0x61, 0xa1, 0x61, 0x62, 0x61, 0x63})); + CHECK(json::parse("[\"a\", {\"b\": \"c\"}]") == json::from_cbor(std::vector({0x82, 0x61, 0x61, 0xa1, 0x61, 0x62, 0x61, 0x63}))); + + CHECK(json::to_cbor(json::parse("{\"a\": \"A\", \"b\": \"B\", \"c\": \"C\", \"d\": \"D\", \"e\": \"E\"}")) == std::vector({0xa5, 0x61, 0x61, 0x61, 0x41, 0x61, 0x62, 0x61, 0x42, 0x61, 0x63, 0x61, 0x43, 0x61, 0x64, 0x61, 0x44, 0x61, 0x65, 0x61, 0x45})); + CHECK(json::parse("{\"a\": \"A\", \"b\": \"B\", \"c\": \"C\", \"d\": \"D\", \"e\": \"E\"}") == json::from_cbor(std::vector({0xa5, 0x61, 0x61, 0x61, 0x41, 0x61, 0x62, 0x61, 0x42, 0x61, 0x63, 0x61, 0x43, 0x61, 0x64, 0x61, 0x44, 0x61, 0x65, 0x61, 0x45}))); + + // indefinite length objects + CHECK(json::parse("{\"a\": 1, \"b\": [2, 3]}") == json::from_cbor(std::vector({0xbf, 0x61, 0x61, 0x01, 0x61, 0x62, 0x9f, 0x02, 0x03, 0xff, 0xff}))); + CHECK(json::parse("[\"a\", {\"b\": \"c\"}]") == json::from_cbor(std::vector({0x82, 0x61, 0x61, 0xbf, 0x61, 0x62, 0x61, 0x63, 0xff}))); + CHECK(json::parse("{\"Fun\": true, \"Amt\": -2}") == json::from_cbor(std::vector({0xbf, 0x63, 0x46, 0x75, 0x6e, 0xf5, 0x63, 0x41, 0x6d, 0x74, 0x21, 0xff}))); + } +} +*/ From 126ce2e56c8084a480937e2ec887ecc19d7872ab Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 8 Jan 2018 18:54:17 +0100 Subject: [PATCH 02/16] :construction: further UBJSON --- src/json.hpp | 79 ++++++++++++++++++++++++++++++++++++++++ test/src/unit-ubjson.cpp | 12 ++---- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index a356bdef..2aef1594 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -6395,6 +6395,85 @@ class binary_writer } } + char ubjson_prefix(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'i'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'U'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'I'; + } + else if (-(std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'l'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'L'; + } + break; + } + + case value_t::number_unsigned: + { + if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'i'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'U'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'I'; + } + else if (-(std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'l'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'L'; + } + break; + } + + case value_t::number_float: + return 'D'; + + case value_t::string: + return 'S'; + + case value_t::array: + return '['; + + case value_t::object: + return '{'; + + default: + break; + } + + return '\0'; + } + private: /// whether we can assume little endianess const bool is_little_endian = binary_reader::little_endianess(); diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index ce1b3c38..7e1ffefc 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -414,10 +414,8 @@ TEST_CASE("UBJSON") SECTION("2147483648..9223372036854775807 (int64)") { - for (uint64_t i : - { - 2147483648ul, 9223372036854775807ul - }) + std::vector v = {2147483648ul, 9223372036854775807ul}; + for (uint64_t i : v) { CAPTURE(i); @@ -605,10 +603,8 @@ TEST_CASE("UBJSON") SECTION("2147483648..9223372036854775807 (int64)") { - for (uint64_t i : - { - 2147483648ul, 9223372036854775807ul - }) + std::vector v = {2147483648ul, 9223372036854775807ul}; + for (uint64_t i : v) { CAPTURE(i); From 3d7658da891c435312ab781ebd8ff0e70f8d3cf0 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 8 Jan 2018 19:48:18 +0100 Subject: [PATCH 03/16] :construction: working on AppVeyor's errors --- test/src/unit-ubjson.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index 7e1ffefc..b7daa1a1 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -88,7 +88,7 @@ TEST_CASE("UBJSON") SECTION("-9223372036854775808..-2147483649 (int64)") { std::vector numbers; - numbers.push_back(INT64_MIN); + numbers.push_back((std::numeric_limits::min)()); numbers.push_back(-1000000000000000000); numbers.push_back(-100000000000000000); numbers.push_back(-10000000000000000); @@ -128,14 +128,14 @@ TEST_CASE("UBJSON") // check individual bytes CHECK(result[0] == 'L'); - uint64_t restored = (static_cast(result[1]) << 070) + - (static_cast(result[2]) << 060) + - (static_cast(result[3]) << 050) + - (static_cast(result[4]) << 040) + - (static_cast(result[5]) << 030) + - (static_cast(result[6]) << 020) + - (static_cast(result[7]) << 010) + - static_cast(result[8]); + int64_t restored = (static_cast(result[1]) << 070) + + (static_cast(result[2]) << 060) + + (static_cast(result[3]) << 050) + + (static_cast(result[4]) << 040) + + (static_cast(result[5]) << 030) + + (static_cast(result[6]) << 020) + + (static_cast(result[7]) << 010) + + static_cast(result[8]); CHECK(restored == i); // roundtrip @@ -145,7 +145,7 @@ TEST_CASE("UBJSON") SECTION("-2147483648..-32769 (int32)") { - std::vector numbers; + std::vector numbers; numbers.push_back(-32769); numbers.push_back(-100000); numbers.push_back(-1000000); From ebf28a26ee3b17448aceb27de6e48a85fc7ed426 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 8 Jan 2018 20:49:02 +0100 Subject: [PATCH 04/16] :construction: another try --- test/src/unit-ubjson.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index b7daa1a1..d65f5a14 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -89,16 +89,16 @@ TEST_CASE("UBJSON") { std::vector numbers; numbers.push_back((std::numeric_limits::min)()); - numbers.push_back(-1000000000000000000); - numbers.push_back(-100000000000000000); - numbers.push_back(-10000000000000000); - numbers.push_back(-1000000000000000); - numbers.push_back(-100000000000000); - numbers.push_back(-10000000000000); - numbers.push_back(-1000000000000); - numbers.push_back(-100000000000); - numbers.push_back(-10000000000); - numbers.push_back(-2147483649); + numbers.push_back(-1000000000000000000l); + numbers.push_back(-100000000000000000l); + numbers.push_back(-10000000000000000l); + numbers.push_back(-1000000000000000l); + numbers.push_back(-100000000000000l); + numbers.push_back(-10000000000000l); + numbers.push_back(-1000000000000l); + numbers.push_back(-100000000000l); + numbers.push_back(-10000000000l); + numbers.push_back(-2147483649l); for (auto i : numbers) { CAPTURE(i); From 97e0d20ce9223cc12a44ab0da31e72eead9353a5 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 8 Jan 2018 21:16:20 +0100 Subject: [PATCH 05/16] :construction: debug --- test/src/unit-ubjson.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index d65f5a14..e4c06091 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -99,9 +99,12 @@ TEST_CASE("UBJSON") numbers.push_back(-100000000000l); numbers.push_back(-10000000000l); numbers.push_back(-2147483649l); + size_t index = 0; for (auto i : numbers) { CAPTURE(i); + CAPTURE(index); + ++index; // create JSON value with integer number json j = i; From fb1154c237cdbd21094e5cf526201b466efdc7b2 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 8 Jan 2018 21:26:41 +0100 Subject: [PATCH 06/16] :construction: debug --- test/src/unit-ubjson.cpp | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index e4c06091..bda8224a 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -89,22 +89,19 @@ TEST_CASE("UBJSON") { std::vector numbers; numbers.push_back((std::numeric_limits::min)()); - numbers.push_back(-1000000000000000000l); - numbers.push_back(-100000000000000000l); - numbers.push_back(-10000000000000000l); - numbers.push_back(-1000000000000000l); - numbers.push_back(-100000000000000l); - numbers.push_back(-10000000000000l); - numbers.push_back(-1000000000000l); - numbers.push_back(-100000000000l); - numbers.push_back(-10000000000l); - numbers.push_back(-2147483649l); - size_t index = 0; + numbers.push_back(-1000000000000000000ll); + numbers.push_back(-100000000000000000ll); + numbers.push_back(-10000000000000000ll); + numbers.push_back(-1000000000000000ll); + numbers.push_back(-100000000000000ll); + numbers.push_back(-10000000000000ll); + numbers.push_back(-1000000000000ll); + numbers.push_back(-100000000000ll); + numbers.push_back(-10000000000ll); + numbers.push_back(-2147483649ll); for (auto i : numbers) { CAPTURE(i); - CAPTURE(index); - ++index; // create JSON value with integer number json j = i; From fd04967676f9224226d304945b51187e94c23253 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 8 Jan 2018 21:37:51 +0100 Subject: [PATCH 07/16] :bug: fixed copy-paste error --- src/json.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 2aef1594..e7465e29 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -6377,7 +6377,7 @@ class binary_writer oa->write_character(static_cast('I')); // int16 write_number(static_cast(n)); } - else if (-(std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) { oa->write_character(static_cast('l')); // int32 write_number(static_cast(n)); @@ -6419,7 +6419,7 @@ class binary_writer { return 'I'; } - else if (-(std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) { return 'l'; } @@ -6444,7 +6444,7 @@ class binary_writer { return 'I'; } - else if (-(std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'l'; } From 965a70e38d666520bdbcbf3aafb01312ba7bc833 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 10 Jan 2018 11:22:19 +0100 Subject: [PATCH 08/16] :hammer: optimized output format --- src/json.hpp | 148 +++++++++++++++++++++++++++++++-------- test/src/unit-ubjson.cpp | 14 ++-- 2 files changed, 125 insertions(+), 37 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index e7465e29..9dc0b24a 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -6189,48 +6189,61 @@ class binary_writer } /*! - @brief[in] j JSON value to serialize + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value */ - void write_ubjson(const BasicJsonType& j, const bool use_count = false) + void write_ubjson(const BasicJsonType& j, + const bool use_count = false, + const bool use_type = false, + const bool add_prefix = true) { switch (j.type()) { case value_t::null: { - oa->write_character(static_cast('Z')); + if (add_prefix) + { + oa->write_character(static_cast('Z')); + } break; } case value_t::boolean: { - oa->write_character(j.m_value.boolean - ? static_cast('T') - : static_cast('F')); + if (add_prefix) + oa->write_character(j.m_value.boolean + ? static_cast('T') + : static_cast('F')); break; } case value_t::number_integer: { - write_number_with_ubjson_prefix(j.m_value.number_integer); + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); break; } case value_t::number_unsigned: { - write_number_with_ubjson_prefix(j.m_value.number_unsigned); + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); break; } case value_t::number_float: { - write_number_with_ubjson_prefix(j.m_value.number_float); + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); break; } case value_t::string: { - oa->write_character(static_cast('S')); - write_number_with_ubjson_prefix(j.m_value.string->size()); + if (add_prefix) + { + oa->write_character(static_cast('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); oa->write_characters( reinterpret_cast(j.m_value.string->c_str()), j.m_value.string->size()); @@ -6239,17 +6252,38 @@ class binary_writer case value_t::array: { - oa->write_character(static_cast('[')); + if (add_prefix) + { + oa->write_character(static_cast('[')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.array->empty()) + { + const char first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(static_cast('$')); + oa->write_character(static_cast(first_prefix)); + } + } if (use_count) { oa->write_character(static_cast('#')); - write_number_with_ubjson_prefix(j.m_value.array->size()); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); } for (const auto& el : *j.m_value.array) { - write_ubjson(el, use_count); + write_ubjson(el, use_count, use_type, prefix_required); } if (not use_count) @@ -6262,21 +6296,42 @@ class binary_writer case value_t::object: { - oa->write_character(static_cast('{')); + if (add_prefix) + { + oa->write_character(static_cast('{')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.object->empty()) + { + const char first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(static_cast('$')); + oa->write_character(static_cast(first_prefix)); + } + } if (use_count) { oa->write_character(static_cast('#')); - write_number_with_ubjson_prefix(j.m_value.object->size()); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); } for (const auto& el : *j.m_value.object) { - write_number_with_ubjson_prefix(el.first.size()); + write_number_with_ubjson_prefix(el.first.size(), true); oa->write_characters( reinterpret_cast(el.first.c_str()), el.first.size()); - write_ubjson(el.second, use_count); + write_ubjson(el.second, use_count, use_type, prefix_required); } if (not use_count) @@ -6320,38 +6375,56 @@ class binary_writer } template - void write_number_with_ubjson_prefix(const NumberType n) + void write_number_with_ubjson_prefix(const NumberType n, const bool add_prefix) { if (std::is_floating_point::value) { - oa->write_character(static_cast('D')); // float64 + if (add_prefix) + { + oa->write_character(static_cast('D')); // float64 + } write_number(n); } else if (std::is_unsigned::value) { if (n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('i')); // uint8 + if (add_prefix) + { + oa->write_character(static_cast('i')); // uint8 + } write_number(static_cast(n)); } else if (n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('U')); // uint8 + if (add_prefix) + { + oa->write_character(static_cast('U')); // uint8 + } write_number(static_cast(n)); } else if (n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('I')); // int16 + if (add_prefix) + { + oa->write_character(static_cast('I')); // int16 + } write_number(static_cast(n)); } else if (n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('l')); // int32 + if (add_prefix) + { + oa->write_character(static_cast('l')); // int32 + } write_number(static_cast(n)); } else if (n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('L')); // int64 + if (add_prefix) + { + oa->write_character(static_cast('L')); // int64 + } write_number(static_cast(n)); } else @@ -6364,27 +6437,42 @@ class binary_writer { if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('i')); // int8 + if (add_prefix) + { + oa->write_character(static_cast('i')); // int8 + } write_number(static_cast(n)); } else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('U')); // uint8 + if (add_prefix) + { + oa->write_character(static_cast('U')); // uint8 + } write_number(static_cast(n)); } else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('I')); // int16 + if (add_prefix) + { + oa->write_character(static_cast('I')); // int16 + } write_number(static_cast(n)); } else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('l')); // int32 + if (add_prefix) + { + oa->write_character(static_cast('l')); // int32 + } write_number(static_cast(n)); } else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) { - oa->write_character(static_cast('L')); // int64 + if (add_prefix) + { + oa->write_character(static_cast('L')); // int64 + } write_number(static_cast(n)); } else diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index bda8224a..43c461cc 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -129,13 +129,13 @@ TEST_CASE("UBJSON") // check individual bytes CHECK(result[0] == 'L'); int64_t restored = (static_cast(result[1]) << 070) + - (static_cast(result[2]) << 060) + - (static_cast(result[3]) << 050) + - (static_cast(result[4]) << 040) + - (static_cast(result[5]) << 030) + - (static_cast(result[6]) << 020) + - (static_cast(result[7]) << 010) + - static_cast(result[8]); + (static_cast(result[2]) << 060) + + (static_cast(result[3]) << 050) + + (static_cast(result[4]) << 040) + + (static_cast(result[5]) << 030) + + (static_cast(result[6]) << 020) + + (static_cast(result[7]) << 010) + + static_cast(result[8]); CHECK(restored == i); // roundtrip From 31bfabc4c0868e7d2bf4a97cdafdeb9b966c4fab Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 10 Jan 2018 16:14:43 +0100 Subject: [PATCH 09/16] :hammer: optimized input format --- src/json.hpp | 210 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 149 insertions(+), 61 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 9dc0b24a..9b27d3d9 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -5223,59 +5223,7 @@ class binary_reader */ BasicJsonType parse_ubjson_internal(const bool get_char = true) { - switch (get_char ? get() : current) - { - case std::char_traits::eof(): // EOF - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); - - case 'T': // true - return true; - case 'F': // false - return false; - - case 'Z': // null - return nullptr; - - case 'N': // no-op - return parse_ubjson_internal(); // read next byte - - case 'U': - return get_number(); - case 'i': - return get_number(); - case 'I': - return get_number(); - case 'l': - return get_number(); - case 'L': - return get_number(); - case 'd': - return get_number(); - case 'D': - return get_number(); - - case 'C': // char - { - get(); - check_eof(); - return std::string(1, static_cast(current)); - } - - case 'S': // string - return get_ubjson_string(); - - case '[': // array - return get_ubjson_array(); - - case '{': // object - return get_ubjson_object(); - - default: // anything else - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(112, chars_read, - "error reading UBJSON; last byte: 0x" + ss.str())); - } + return get_ubjson_value(get_char ? get_ignore_noop() : current); } /*! @@ -5293,6 +5241,20 @@ class binary_reader return (current = ia->get_character()); } + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + /* @brief read a number from the input @@ -5622,18 +5584,124 @@ class binary_reader } } + std::pair get_ubjson_size_type() + { + std::size_t sz = std::string::npos; + int tc = 0; + + get_ignore_noop(); + + switch (current) + { + case '$': + { + get_ignore_noop(); + check_eof(); + tc = current; + + get_ignore_noop(); + if (current != '#') + { + assert(false); + } + sz = parse_ubjson_internal(); + break; + } + + case '#': + { + sz = parse_ubjson_internal(); + break; + } + + default: + break; + } + + return std::make_pair(sz, tc); + + } + + BasicJsonType get_ubjson_value(const int prefix) + { + switch (prefix) + { + case std::char_traits::eof(): // EOF + JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + + case 'T': // true + return true; + case 'F': // false + return false; + + case 'Z': // null + return nullptr; + + case 'U': + return get_number(); + case 'i': + return get_number(); + case 'I': + return get_number(); + case 'l': + return get_number(); + case 'L': + return get_number(); + case 'd': + return get_number(); + case 'D': + return get_number(); + + case 'C': // char + { + get(); + check_eof(); + return std::string(1, static_cast(current)); + } + + case 'S': // string + return get_ubjson_string(); + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(112, chars_read, + "error reading UBJSON; last byte: 0x" + ss.str())); + } + } + BasicJsonType get_ubjson_array() { BasicJsonType result = value_t::array; + const auto size_and_type = get_ubjson_size_type(); - while (get() != ']') + if (size_and_type.first != std::string::npos) { - // skip no-op - if (current == 'N') + for (std::size_t i = 0; i < size_and_type.first; ++i) { - continue; + if (size_and_type.second != 0) + { + result.push_back(get_ubjson_value(size_and_type.second)); + } + else + { + result.push_back(parse_ubjson_internal()); + } + } + } + else + { + while (current != ']') + { + result.push_back(parse_ubjson_internal(false)); + get_ignore_noop(); } - result.push_back(parse_ubjson_internal(false)); } return result; @@ -5642,11 +5710,31 @@ class binary_reader BasicJsonType get_ubjson_object() { BasicJsonType result = value_t::object; + const auto size_and_type = get_ubjson_size_type(); - while (get() != '}') + if (size_and_type.first != std::string::npos) { - auto key = get_ubjson_string(false); - result[std::move(key)] = parse_ubjson_internal(); + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + auto key = get_ubjson_string(); + if (size_and_type.second != 0) + { + result[std::move(key)] = get_ubjson_value(size_and_type.second); + } + else + { + result[std::move(key)] = parse_ubjson_internal(); + } + } + } + else + { + while (current != '}') + { + auto key = get_ubjson_string(false); + result[std::move(key)] = parse_ubjson_internal(); + get_ignore_noop(); + } } return result; From 3a7585e7385efe26ff4346936c731269fe8c4221 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 10 Jan 2018 22:47:07 +0100 Subject: [PATCH 10/16] :white_check_mark: added more tests --- src/json.hpp | 144 ++++--- test/src/unit-ubjson.cpp | 864 +++++++++++++++++++++++++++------------ 2 files changed, 691 insertions(+), 317 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 9b27d3d9..5f872e52 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -4549,7 +4549,7 @@ class binary_reader const auto res = parse_ubjson_internal(); if (strict) { - get(); + get_ignore_noop(); check_eof(true); } return res; @@ -5591,35 +5591,25 @@ class binary_reader get_ignore_noop(); - switch (current) + if (current == '$') { - case '$': + get(); // must not ignore 'N', because 'N' maybe the type + check_eof(); + tc = current; + + get_ignore_noop(); + if (current != '#') { - get_ignore_noop(); - check_eof(); - tc = current; - - get_ignore_noop(); - if (current != '#') - { - assert(false); - } - sz = parse_ubjson_internal(); - break; + assert(false); } - - case '#': - { - sz = parse_ubjson_internal(); - break; - } - - default: - break; + sz = parse_ubjson_internal(); + } + else if (current == '#') + { + sz = parse_ubjson_internal(); } return std::make_pair(sz, tc); - } BasicJsonType get_ubjson_value(const int prefix) @@ -5656,6 +5646,7 @@ class binary_reader { get(); check_eof(); + assert(0 <= current and current <= 127); return std::string(1, static_cast(current)); } @@ -5683,16 +5674,22 @@ class binary_reader if (size_and_type.first != std::string::npos) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (size_and_type.second != 0) { - if (size_and_type.second != 0) + if (size_and_type.second != 'N') + std::generate_n(std::back_inserter(*result.m_value.array), + size_and_type.first, [this, size_and_type]() { - result.push_back(get_ubjson_value(size_and_type.second)); - } - else + return get_ubjson_value(size_and_type.second); + }); + } + else + { + std::generate_n(std::back_inserter(*result.m_value.array), + size_and_type.first, [this]() { - result.push_back(parse_ubjson_internal()); - } + return parse_ubjson_internal(); + }); } } else @@ -5714,17 +5711,28 @@ class binary_reader if (size_and_type.first != std::string::npos) { - for (std::size_t i = 0; i < size_and_type.first; ++i) + if (size_and_type.second != 0) { - auto key = get_ubjson_string(); - if (size_and_type.second != 0) + if (size_and_type.second != 'N') + std::generate_n(std::inserter(*result.m_value.object, + result.m_value.object->end()), + size_and_type.first, [this, size_and_type]() { - result[std::move(key)] = get_ubjson_value(size_and_type.second); - } - else + auto key = get_ubjson_string(); + auto val = get_ubjson_value(size_and_type.second); + return std::make_pair(std::move(key), std::move(val)); + }); + } + else + { + std::generate_n(std::inserter(*result.m_value.object, + result.m_value.object->end()), + size_and_type.first, [this]() { - result[std::move(key)] = parse_ubjson_internal(); - } + auto key = get_ubjson_string(); + auto val = parse_ubjson_internal(); + return std::make_pair(std::move(key), std::move(val)); + }); } } else @@ -6282,10 +6290,8 @@ class binary_writer @param[in] use_type whether to use '$' prefixes (optimized format) @param[in] add_prefix whether prefixes need to be used for this value */ - void write_ubjson(const BasicJsonType& j, - const bool use_count = false, - const bool use_type = false, - const bool add_prefix = true) + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) { switch (j.type()) { @@ -6348,6 +6354,7 @@ class binary_writer bool prefix_required = true; if (use_type and not j.m_value.array->empty()) { + assert(use_count); const char first_prefix = ubjson_prefix(j.front()); const bool same_prefix = std::all_of(j.begin() + 1, j.end(), [this, first_prefix](const BasicJsonType & v) @@ -6392,6 +6399,7 @@ class binary_writer bool prefix_required = true; if (use_type and not j.m_value.object->empty()) { + assert(use_count); const char first_prefix = ubjson_prefix(j.front()); const bool same_prefix = std::all_of(j.begin(), j.end(), [this, first_prefix](const BasicJsonType & v) @@ -6463,13 +6471,14 @@ class binary_writer } template - void write_number_with_ubjson_prefix(const NumberType n, const bool add_prefix) + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) { if (std::is_floating_point::value) { if (add_prefix) { - oa->write_character(static_cast('D')); // float64 + oa->write_character(static_cast('D')); // float64 } write_number(n); } @@ -6479,7 +6488,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('i')); // uint8 + oa->write_character(static_cast('i')); // uint8 } write_number(static_cast(n)); } @@ -6487,7 +6496,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('U')); // uint8 + oa->write_character(static_cast('U')); // uint8 } write_number(static_cast(n)); } @@ -6495,7 +6504,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('I')); // int16 + oa->write_character(static_cast('I')); // int16 } write_number(static_cast(n)); } @@ -6503,7 +6512,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('l')); // int32 + oa->write_character(static_cast('l')); // int32 } write_number(static_cast(n)); } @@ -6511,7 +6520,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('L')); // int64 + oa->write_character(static_cast('L')); // int64 } write_number(static_cast(n)); } @@ -6527,7 +6536,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('i')); // int8 + oa->write_character(static_cast('i')); // int8 } write_number(static_cast(n)); } @@ -6535,7 +6544,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('U')); // uint8 + oa->write_character(static_cast('U')); // uint8 } write_number(static_cast(n)); } @@ -6543,7 +6552,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('I')); // int16 + oa->write_character(static_cast('I')); // int16 } write_number(static_cast(n)); } @@ -6551,7 +6560,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('l')); // int32 + oa->write_character(static_cast('l')); // int32 } write_number(static_cast(n)); } @@ -6559,7 +6568,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(static_cast('L')); // int64 + oa->write_character(static_cast('L')); // int64 } write_number(static_cast(n)); } @@ -6608,23 +6617,23 @@ class binary_writer case value_t::number_unsigned: { - if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'i'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'U'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'I'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'l'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_unsigned and j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) { return 'L'; } @@ -14103,21 +14112,24 @@ class basic_json binary_writer(o).write_msgpack(j); } - static std::vector to_ubjson(const basic_json& j) + static std::vector to_ubjson(const basic_json& j, + const bool use_size = false, const bool use_type = false) { std::vector result; - to_ubjson(j, result); + to_ubjson(j, result, use_size, use_type); return result; } - static void to_ubjson(const basic_json& j, detail::output_adapter o) + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) { - binary_writer(o).write_ubjson(j); + binary_writer(o).write_ubjson(j, use_size, use_type); } - static void to_ubjson(const basic_json& j, detail::output_adapter o) + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) { - binary_writer(o).write_ubjson(j); + binary_writer(o).write_ubjson(j, use_size, use_type); } /*! diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index 43c461cc..38912761 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -811,72 +811,234 @@ TEST_CASE("UBJSON") { SECTION("empty") { - json j = json::array(); - std::vector expected = {'[', ']'}; - const auto result = json::to_ubjson(j); - CHECK(result == expected); + SECTION("size=false type=false") + { + json j = json::array(); + std::vector expected = {'[', ']'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j = json::array(); + std::vector expected = {'[', '#', 'i', 0}; + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j = json::array(); + std::vector expected = {'[', '#', 'i', 0}; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } SECTION("[null]") { - json j = {nullptr}; - std::vector expected = {'[', 'Z', ']'}; - const auto result = json::to_ubjson(j); - CHECK(result == expected); + SECTION("size=false type=false") + { + json j = {nullptr}; + std::vector expected = {'[', 'Z', ']'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j = {nullptr}; + std::vector expected = {'[', '#', 'i', 1, 'Z'}; + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j = {nullptr}; + std::vector expected = {'[', '$', 'Z', '#', 'i', 1}; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } SECTION("[1,2,3,4,5]") { - json j = json::parse("[1,2,3,4,5]"); - std::vector expected = {'[', 'i', 1, 'i', 2, 'i', 3, 'i', 4, 'i', 5, ']'}; - const auto result = json::to_ubjson(j); - CHECK(result == expected); + SECTION("size=false type=false") + { + json j = json::parse("[1,2,3,4,5]"); + std::vector expected = {'[', 'i', 1, 'i', 2, 'i', 3, 'i', 4, 'i', 5, ']'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j = json::parse("[1,2,3,4,5]"); + std::vector expected = {'[', '#', 'i', 5, 'i', 1, 'i', 2, 'i', 3, 'i', 4, 'i', 5}; + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j = json::parse("[1,2,3,4,5]"); + std::vector expected = {'[', '$', 'i', '#', 'i', 5, 1, 2, 3, 4, 5}; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } SECTION("[[[[]]]]") { - json j = json::parse("[[[[]]]]"); - std::vector expected = {'[', '[', '[', '[', ']', ']', ']', ']'}; - const auto result = json::to_ubjson(j); - CHECK(result == expected); + SECTION("size=false type=false") + { + json j = json::parse("[[[[]]]]"); + std::vector expected = {'[', '[', '[', '[', ']', ']', ']', ']'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j = json::parse("[[[[]]]]"); + std::vector expected = {'[', '#', 'i', 1, '[', '#', 'i', 1, '[', '#', 'i', 1, '[', '#', 'i', 0}; + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j = json::parse("[[[[]]]]"); + std::vector expected = {'[', '$', '[', '#', 'i', 1, '$', '[', '#', 'i', 1, '$', '[', '#', 'i', 1, '#', 'i', 0}; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } SECTION("array with uint16_t elements") { - json j(257, nullptr); - std::vector expected(j.size() + 2, 'Z'); // all null - expected[0] = '['; // opening array - expected[258] = ']'; // closing array - const auto result = json::to_ubjson(j); - CHECK(result == expected); + SECTION("size=false type=false") + { + json j(257, nullptr); + std::vector expected(j.size() + 2, 'Z'); // all null + expected[0] = '['; // opening array + expected[258] = ']'; // closing array + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j(257, nullptr); + std::vector expected(j.size() + 5, 'Z'); // all null + expected[0] = '['; // opening array + expected[1] = '#'; // array size + expected[2] = 'I'; // int16 + expected[3] = 0x01; // 0x0101, first byte + expected[4] = 0x01; // 0x0101, second byte + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j(257, nullptr); + std::vector expected = {'[', '$', 'Z', '#', 'I', 0x01, 0x01}; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } SECTION("array with uint32_t elements") { - json j(65793, nullptr); - std::vector expected(j.size() + 2, 'Z'); // all null - expected[0] = '['; // opening array - expected[65794] = ']'; // closing array - const auto result = json::to_ubjson(j); - CHECK(result == expected); + SECTION("size=false type=false") + { + json j(65793, nullptr); + std::vector expected(j.size() + 2, 'Z'); // all null + expected[0] = '['; // opening array + expected[65794] = ']'; // closing array + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j(65793, nullptr); + std::vector expected(j.size() + 7, 'Z'); // all null + expected[0] = '['; // opening array + expected[1] = '#'; // array size + expected[2] = 'l'; // int32 + expected[3] = 0x00; // 0x00010101, first byte + expected[4] = 0x01; // 0x00010101, second byte + expected[5] = 0x01; // 0x00010101, third byte + expected[6] = 0x01; // 0x00010101, fourth byte + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j(65793, nullptr); + std::vector expected = {'[', '$', 'Z', '#', 'l', 0x00, 0x01, 0x01, 0x01}; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } } @@ -884,38 +1046,119 @@ TEST_CASE("UBJSON") { SECTION("empty") { - json j = json::object(); - std::vector expected = {'{', '}'}; - const auto result = json::to_ubjson(j); - CHECK(result == expected); + SECTION("size=false type=false") + { + json j = json::object(); + std::vector expected = {'{', '}'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j = json::object(); + std::vector expected = {'{', '#', 'i', 0}; + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j = json::object(); + std::vector expected = {'{', '#', 'i', 0}; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } SECTION("{\"\":null}") { - json j = {{"", nullptr}}; - std::vector expected = {'{', 'i', 0, 'Z', '}'}; - const auto result = json::to_ubjson(j); - CHECK(result == expected); + SECTION("size=false type=false") + { + json j = {{"", nullptr}}; + std::vector expected = {'{', 'i', 0, 'Z', '}'}; + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j = {{"", nullptr}}; + std::vector expected = {'{', '#', 'i', 1, 'i', 0, 'Z'}; + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j = {{"", nullptr}}; + std::vector expected = {'{', '$', 'Z', '#', 'i', 1, 'i', 0}; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } SECTION("{\"a\": {\"b\": {\"c\": {}}}}") { - json j = json::parse("{\"a\": {\"b\": {\"c\": {}}}}"); - std::vector expected = + SECTION("size=false type=false") { - '{', 'i', 1, 'a', '{', 'i', 1, 'b', '{', 'i', 1, 'c', '{', '}', '}', '}', '}' - }; - const auto result = json::to_ubjson(j); - CHECK(result == expected); + json j = json::parse("{\"a\": {\"b\": {\"c\": {}}}}"); + std::vector expected = + { + '{', 'i', 1, 'a', '{', 'i', 1, 'b', '{', 'i', 1, 'c', '{', '}', '}', '}', '}' + }; + const auto result = json::to_ubjson(j); + CHECK(result == expected); - // roundtrip - CHECK(json::from_ubjson(result) == j); + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=false") + { + json j = json::parse("{\"a\": {\"b\": {\"c\": {}}}}"); + std::vector expected = + { + '{', '#', 'i', 1, 'i', 1, 'a', '{', '#', 'i', 1, 'i', 1, 'b', '{', '#', 'i', 1, 'i', 1, 'c', '{', '#', 'i', 0 + }; + const auto result = json::to_ubjson(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } + + SECTION("size=true type=true") + { + json j = json::parse("{\"a\": {\"b\": {\"c\": {}}}}"); + std::vector expected = + { + '{', '$', '{', '#', 'i', 1, 'i', 1, 'a', '$', '{', '#', 'i', 1, 'i', 1, 'b', '$', '{', '#', 'i', 1, 'i', 1, 'c', '#', 'i', 0 + }; + const auto result = json::to_ubjson(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_ubjson(result) == j); + } } } } @@ -1285,8 +1528,317 @@ TEST_CASE("CBOR roundtrips", "[hide]") } } +TEST_CASE("Universal Binary JSON Specification Examples 1") +{ + SECTION("Null Value") + { + json j = {{"passcode", nullptr}}; + std::vector v = {'{', 'i', 8, 'p', 'a', 's', 's', 'c', 'o', 'd', 'e', 'Z', '}'}; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("No-Op Value") + { + json j = {"foo", "bar", "baz"}; + std::vector v = {'[', 'S', 'i', 3, 'f', 'o', 'o', + 'S', 'i', 3, 'b', 'a', 'r', + 'S', 'i', 3, 'b', 'a', 'z', ']' + }; + std::vector v2 = {'[', 'S', 'i', 3, 'f', 'o', 'o', 'N', + 'S', 'i', 3, 'b', 'a', 'r', 'N', 'N', 'N', + 'S', 'i', 3, 'b', 'a', 'z', 'N', 'N', ']' + }; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + CHECK(json::from_ubjson(v2) == j); + } + + SECTION("Boolean Types") + { + json j = {{"authorized", true}, {"verified", false}}; + std::vector v = {'{', 'i', 10, 'a', 'u', 't', 'h', 'o', 'r', 'i', 'z', 'e', 'd', 'T', + 'i', 8, 'v', 'e', 'r', 'i', 'f', 'i', 'e', 'd', 'F', '}' + }; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("Numeric Types") + { + json j = + { + {"int8", 16}, + {"uint8", 255}, + {"int16", 32767}, + {"int32", 2147483647}, + {"int64", 9223372036854775807}, + {"float64", 113243.7863123} + }; + std::vector v = {'{', + 'i', 7, 'f', 'l', 'o', 'a', 't', '6', '4', 'D', 0x40, 0xfb, 0xa5, 0xbc, 0x94, 0xbc, 0x34, 0xcf, + 'i', 5, 'i', 'n', 't', '1', '6', 'I', 0x7f, 0xff, + 'i', 5, 'i', 'n', 't', '3', '2', 'l', 0x7f, 0xff, 0xff, 0xff, + 'i', 5, 'i', 'n', 't', '6', '4', 'L', 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 'i', 4, 'i', 'n', 't', '8', 'i', 16, + 'i', 5, 'u', 'i', 'n', 't', '8', 'U', 0xff, + '}' + }; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("Char Type") + { + json j = {{"rolecode", "a"}, {"delim", ";"}}; + std::vector v = {'{', 'i', 5, 'd', 'e', 'l', 'i', 'm', 'C', ';', 'i', 8, 'r', 'o', 'l', 'e', 'c', 'o', 'd', 'e', 'C', 'a', '}'}; + //CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("String Type") + { + SECTION("English") + { + json j = "hello"; + std::vector v = {'S', 'i', 5, 'h', 'e', 'l', 'l', 'o'}; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("Russian") + { + json j = "привет"; + std::vector v = {'S', 'i', 12, 0xD0, 0xBF, 0xD1, 0x80, 0xD0, 0xB8, 0xD0, 0xB2, 0xD0, 0xB5, 0xD1, 0x82}; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("Russian") + { + json j = "مرحبا"; + std::vector v = {'S', 'i', 10, 0xD9, 0x85, 0xD8, 0xB1, 0xD8, 0xAD, 0xD8, 0xA8, 0xD8, 0xA7}; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + } + + SECTION("Array Type") + { + SECTION("size=false type=false") + { + // note the float has been replaced by a double + json j = {nullptr, true, false, 4782345193, 153.132, "ham"}; + std::vector v = {'[', 'Z', 'T', 'F', 'L', 0x00, 0x00, 0x00, 0x01, 0x1D, 0x0C, 0xCB, 0xE9, 'D', 0x40, 0x63, 0x24, 0x39, 0x58, 0x10, 0x62, 0x4e, 'S', 'i', 3, 'h', 'a', 'm', ']'}; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("size=true type=false") + { + // note the float has been replaced by a double + json j = {nullptr, true, false, 4782345193, 153.132, "ham"}; + std::vector v = {'[', '#', 'i', 6, 'Z', 'T', 'F', 'L', 0x00, 0x00, 0x00, 0x01, 0x1D, 0x0C, 0xCB, 0xE9, 'D', 0x40, 0x63, 0x24, 0x39, 0x58, 0x10, 0x62, 0x4e, 'S', 'i', 3, 'h', 'a', 'm'}; + CHECK(json::to_ubjson(j, true) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("size=true type=true") + { + // note the float has been replaced by a double + json j = {nullptr, true, false, 4782345193, 153.132, "ham"}; + std::vector v = {'[', '#', 'i', 6, 'Z', 'T', 'F', 'L', 0x00, 0x00, 0x00, 0x01, 0x1D, 0x0C, 0xCB, 0xE9, 'D', 0x40, 0x63, 0x24, 0x39, 0x58, 0x10, 0x62, 0x4e, 'S', 'i', 3, 'h', 'a', 'm'}; + CHECK(json::to_ubjson(j, true, true) == v); + CHECK(json::from_ubjson(v) == j); + } + } + + SECTION("Object Type") + { + SECTION("size=false type=false") + { + json j = + { + { + "post", { + {"id", 1137}, + {"author", "rkalla"}, + {"timestamp", 1364482090592}, + {"body", "I totally agree!"} + } + } + }; + std::vector v = {'{', 'i', 4, 'p', 'o', 's', 't', '{', + 'i', 6, 'a', 'u', 't', 'h', 'o', 'r', 'S', 'i', 6, 'r', 'k', 'a', 'l', 'l', 'a', + 'i', 4, 'b', 'o', 'd', 'y', 'S', 'i', 16, 'I', ' ', 't', 'o', 't', 'a', 'l', 'l', 'y', ' ', 'a', 'g', 'r', 'e', 'e', '!', + 'i', 2, 'i', 'd', 'I', 0x04, 0x71, + 'i', 9, 't', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', 'L', 0x00, 0x00, 0x01, 0x3D, 0xB1, 0x78, 0x66, 0x60, + '}', '}' + }; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("size=true type=false") + { + json j = + { + { + "post", { + {"id", 1137}, + {"author", "rkalla"}, + {"timestamp", 1364482090592}, + {"body", "I totally agree!"} + } + } + }; + std::vector v = {'{', '#', 'i', 1, 'i', 4, 'p', 'o', 's', 't', '{', '#', 'i', 4, + 'i', 6, 'a', 'u', 't', 'h', 'o', 'r', 'S', 'i', 6, 'r', 'k', 'a', 'l', 'l', 'a', + 'i', 4, 'b', 'o', 'd', 'y', 'S', 'i', 16, 'I', ' ', 't', 'o', 't', 'a', 'l', 'l', 'y', ' ', 'a', 'g', 'r', 'e', 'e', '!', + 'i', 2, 'i', 'd', 'I', 0x04, 0x71, + 'i', 9, 't', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', 'L', 0x00, 0x00, 0x01, 0x3D, 0xB1, 0x78, 0x66, 0x60 + }; + CHECK(json::to_ubjson(j, true) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("size=true type=true") + { + json j = + { + { + "post", { + {"id", 1137}, + {"author", "rkalla"}, + {"timestamp", 1364482090592}, + {"body", "I totally agree!"} + } + } + }; + std::vector v = {'{', '$', '{', '#', 'i', 1, 'i', 4, 'p', 'o', 's', 't', '#', 'i', 4, + 'i', 6, 'a', 'u', 't', 'h', 'o', 'r', 'S', 'i', 6, 'r', 'k', 'a', 'l', 'l', 'a', + 'i', 4, 'b', 'o', 'd', 'y', 'S', 'i', 16, 'I', ' ', 't', 'o', 't', 'a', 'l', 'l', 'y', ' ', 'a', 'g', 'r', 'e', 'e', '!', + 'i', 2, 'i', 'd', 'I', 0x04, 0x71, + 'i', 9, 't', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', 'L', 0x00, 0x00, 0x01, 0x3D, 0xB1, 0x78, 0x66, 0x60 + }; + CHECK(json::to_ubjson(j, true, true) == v); + CHECK(json::from_ubjson(v) == j); + } + } + + SECTION("Optimized Format") + { + SECTION("Array Example") + { + SECTION("No Optimization") + { + // note the floats have been replaced by doubles + json j = {29.97, 31.13, 67.0, 2.113, 23.888}; + std::vector v = {'[', + 'D', 0x40, 0x3d, 0xf8, 0x51, 0xeb, 0x85, 0x1e, 0xb8, + 'D', 0x40, 0x3f, 0x21, 0x47, 0xae, 0x14, 0x7a, 0xe1, + 'D', 0x40, 0x50, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 'D', 0x40, 0x00, 0xe7, 0x6c, 0x8b, 0x43, 0x95, 0x81, + 'D', 0x40, 0x37, 0xe3, 0x53, 0xf7, 0xce, 0xd9, 0x17, + ']' + }; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("Optimized with count") + { + // note the floats have been replaced by doubles + json j = {29.97, 31.13, 67.0, 2.113, 23.888}; + std::vector v = {'[', '#', 'i', 5, + 'D', 0x40, 0x3d, 0xf8, 0x51, 0xeb, 0x85, 0x1e, 0xb8, + 'D', 0x40, 0x3f, 0x21, 0x47, 0xae, 0x14, 0x7a, 0xe1, + 'D', 0x40, 0x50, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 'D', 0x40, 0x00, 0xe7, 0x6c, 0x8b, 0x43, 0x95, 0x81, + 'D', 0x40, 0x37, 0xe3, 0x53, 0xf7, 0xce, 0xd9, 0x17 + }; + CHECK(json::to_ubjson(j, true) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("Optimized with type & count") + { + // note the floats have been replaced by doubles + json j = {29.97, 31.13, 67.0, 2.113, 23.888}; + std::vector v = {'[', '$', 'D', '#', 'i', 5, + 0x40, 0x3d, 0xf8, 0x51, 0xeb, 0x85, 0x1e, 0xb8, + 0x40, 0x3f, 0x21, 0x47, 0xae, 0x14, 0x7a, 0xe1, + 0x40, 0x50, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0xe7, 0x6c, 0x8b, 0x43, 0x95, 0x81, + 0x40, 0x37, 0xe3, 0x53, 0xf7, 0xce, 0xd9, 0x17 + }; + CHECK(json::to_ubjson(j, true, true) == v); + CHECK(json::from_ubjson(v) == j); + } + } + + SECTION("Object Example") + { + SECTION("No Optimization") + { + // note the floats have been replaced by doubles + json j = { {"lat", 29.976}, {"long", 31.131}, {"alt", 67.0} }; + std::vector v = {'{', + 'i', 3, 'a', 'l', 't', 'D', 0x40, 0x50, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 'i', 3, 'l', 'a', 't', 'D', 0x40, 0x3d, 0xf9, 0xdb, 0x22, 0xd0, 0xe5, 0x60, + 'i', 4, 'l', 'o', 'n', 'g', 'D', 0x40, 0x3f, 0x21, 0x89, 0x37, 0x4b, 0xc6, 0xa8, + '}' + }; + CHECK(json::to_ubjson(j) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("Optimized with count") + { + // note the floats have been replaced by doubles + json j = { {"lat", 29.976}, {"long", 31.131}, {"alt", 67.0} }; + std::vector v = {'{', '#', 'i', 3, + 'i', 3, 'a', 'l', 't', 'D', 0x40, 0x50, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 'i', 3, 'l', 'a', 't', 'D', 0x40, 0x3d, 0xf9, 0xdb, 0x22, 0xd0, 0xe5, 0x60, + 'i', 4, 'l', 'o', 'n', 'g', 'D', 0x40, 0x3f, 0x21, 0x89, 0x37, 0x4b, 0xc6, 0xa8 + }; + CHECK(json::to_ubjson(j, true) == v); + CHECK(json::from_ubjson(v) == j); + } + + SECTION("Optimized with type & count") + { + // note the floats have been replaced by doubles + json j = { {"lat", 29.976}, {"long", 31.131}, {"alt", 67.0} }; + std::vector v = {'{', '$', 'D', '#', 'i', 3, + 'i', 3, 'a', 'l', 't', 0x40, 0x50, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 'i', 3, 'l', 'a', 't', 0x40, 0x3d, 0xf9, 0xdb, 0x22, 0xd0, 0xe5, 0x60, + 'i', 4, 'l', 'o', 'n', 'g', 0x40, 0x3f, 0x21, 0x89, 0x37, 0x4b, 0xc6, 0xa8 + }; + CHECK(json::to_ubjson(j, true, true) == v); + CHECK(json::from_ubjson(v) == j); + } + } + + SECTION("Special Cases (Null, No-Op and Boolean)") + { + SECTION("Array") + { + std::vector v = {'[', '$', 'N', '#', 'I', 0x02, 0x00}; + CHECK(json::from_ubjson(v) == json::array()); + } + + SECTION("Object") + { + std::vector v = {'{', '$', 'Z', '#', 'i', 3, 'i', 4, 'n', 'a', 'm', 'e', 'i', 8, 'p', 'a', 's', 's', 'w', 'o', 'r', 'd', 'i', 5, 'e', 'm', 'a', 'i', 'l'}; + CHECK(json::from_ubjson(v) == json({ {"name", nullptr}, {"password", nullptr}, {"email", nullptr} })); + } + } + } +} + /* -TEST_CASE("Universal Binary JSON Specification Examples") +TEST_CASE("Universal Binary JSON Specification Examples 2") { for (std::string prefix : { @@ -1397,193 +1949,3 @@ TEST_CASE("all first bytes", "[!throws]") } } */ - -/* -TEST_CASE("examples from RFC 7049 Appendix A") -{ - SECTION("numbers") - { - CHECK(json::to_cbor(json::parse("0")) == std::vector({0x00})); - CHECK(json::parse("0") == json::from_cbor(std::vector({0x00}))); - - CHECK(json::to_cbor(json::parse("1")) == std::vector({0x01})); - CHECK(json::parse("1") == json::from_cbor(std::vector({0x01}))); - - CHECK(json::to_cbor(json::parse("10")) == std::vector({0x0a})); - CHECK(json::parse("10") == json::from_cbor(std::vector({0x0a}))); - - CHECK(json::to_cbor(json::parse("23")) == std::vector({0x17})); - CHECK(json::parse("23") == json::from_cbor(std::vector({0x17}))); - - CHECK(json::to_cbor(json::parse("24")) == std::vector({0x18, 0x18})); - CHECK(json::parse("24") == json::from_cbor(std::vector({0x18, 0x18}))); - - CHECK(json::to_cbor(json::parse("25")) == std::vector({0x18, 0x19})); - CHECK(json::parse("25") == json::from_cbor(std::vector({0x18, 0x19}))); - - CHECK(json::to_cbor(json::parse("100")) == std::vector({0x18, 0x64})); - CHECK(json::parse("100") == json::from_cbor(std::vector({0x18, 0x64}))); - - CHECK(json::to_cbor(json::parse("1000")) == std::vector({0x19, 0x03, 0xe8})); - CHECK(json::parse("1000") == json::from_cbor(std::vector({0x19, 0x03, 0xe8}))); - - CHECK(json::to_cbor(json::parse("1000000")) == std::vector({0x1a, 0x00, 0x0f, 0x42, 0x40})); - CHECK(json::parse("1000000") == json::from_cbor(std::vector({0x1a, 0x00, 0x0f, 0x42, 0x40}))); - - CHECK(json::to_cbor(json::parse("1000000000000")) == std::vector({0x1b, 0x00, 0x00, 0x00, 0xe8, 0xd4, 0xa5, 0x10, 0x00})); - CHECK(json::parse("1000000000000") == json::from_cbor(std::vector({0x1b, 0x00, 0x00, 0x00, 0xe8, 0xd4, 0xa5, 0x10, 0x00}))); - - CHECK(json::to_cbor(json::parse("18446744073709551615")) == std::vector({0x1b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff})); - CHECK(json::parse("18446744073709551615") == json::from_cbor(std::vector({0x1b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}))); - - // positive bignum is not supported - //CHECK(json::to_cbor(json::parse("18446744073709551616")) == std::vector({0xc2, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})); - //CHECK(json::parse("18446744073709551616") == json::from_cbor(std::vector({0xc2, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}))); - - //CHECK(json::to_cbor(json::parse("-18446744073709551616")) == std::vector({0x3b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff})); - //CHECK(json::parse("-18446744073709551616") == json::from_cbor(std::vector({0x3b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}))); - - // negative bignum is not supported - //CHECK(json::to_cbor(json::parse("-18446744073709551617")) == std::vector({0xc3, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})); - //CHECK(json::parse("-18446744073709551617") == json::from_cbor(std::vector({0xc3, 0x49, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}))); - - CHECK(json::to_cbor(json::parse("-1")) == std::vector({0x20})); - CHECK(json::parse("-1") == json::from_cbor(std::vector({0x20}))); - - CHECK(json::to_cbor(json::parse("-10")) == std::vector({0x29})); - CHECK(json::parse("-10") == json::from_cbor(std::vector({0x29}))); - - CHECK(json::to_cbor(json::parse("-100")) == std::vector({0x38, 0x63})); - CHECK(json::parse("-100") == json::from_cbor(std::vector({0x38, 0x63}))); - - CHECK(json::to_cbor(json::parse("-1000")) == std::vector({0x39, 0x03, 0xe7})); - CHECK(json::parse("-1000") == json::from_cbor(std::vector({0x39, 0x03, 0xe7}))); - - // half-precision float - //CHECK(json::to_cbor(json::parse("0.0")) == std::vector({0xf9, 0x00, 0x00})); - CHECK(json::parse("0.0") == json::from_cbor(std::vector({0xf9, 0x00, 0x00}))); - - // half-precision float - //CHECK(json::to_cbor(json::parse("-0.0")) == std::vector({0xf9, 0x80, 0x00})); - CHECK(json::parse("-0.0") == json::from_cbor(std::vector({0xf9, 0x80, 0x00}))); - - // half-precision float - //CHECK(json::to_cbor(json::parse("1.0")) == std::vector({0xf9, 0x3c, 0x00})); - CHECK(json::parse("1.0") == json::from_cbor(std::vector({0xf9, 0x3c, 0x00}))); - - CHECK(json::to_cbor(json::parse("1.1")) == std::vector({0xfb, 0x3f, 0xf1, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9a})); - CHECK(json::parse("1.1") == json::from_cbor(std::vector({0xfb, 0x3f, 0xf1, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9a}))); - - // half-precision float - //CHECK(json::to_cbor(json::parse("1.5")) == std::vector({0xf9, 0x3e, 0x00})); - CHECK(json::parse("1.5") == json::from_cbor(std::vector({0xf9, 0x3e, 0x00}))); - - // half-precision float - //CHECK(json::to_cbor(json::parse("65504.0")) == std::vector({0xf9, 0x7b, 0xff})); - CHECK(json::parse("65504.0") == json::from_cbor(std::vector({0xf9, 0x7b, 0xff}))); - - //CHECK(json::to_cbor(json::parse("100000.0")) == std::vector({0xfa, 0x47, 0xc3, 0x50, 0x00})); - CHECK(json::parse("100000.0") == json::from_cbor(std::vector({0xfa, 0x47, 0xc3, 0x50, 0x00}))); - - //CHECK(json::to_cbor(json::parse("3.4028234663852886e+38")) == std::vector({0xfa, 0x7f, 0x7f, 0xff, 0xff})); - CHECK(json::parse("3.4028234663852886e+38") == json::from_cbor(std::vector({0xfa, 0x7f, 0x7f, 0xff, 0xff}))); - - CHECK(json::to_cbor(json::parse("1.0e+300")) == std::vector({0xfb, 0x7e, 0x37, 0xe4, 0x3c, 0x88, 0x00, 0x75, 0x9c})); - CHECK(json::parse("1.0e+300") == json::from_cbor(std::vector({0xfb, 0x7e, 0x37, 0xe4, 0x3c, 0x88, 0x00, 0x75, 0x9c}))); - - // half-precision float - //CHECK(json::to_cbor(json::parse("5.960464477539063e-8")) == std::vector({0xf9, 0x00, 0x01})); - CHECK(json::parse("-4.0") == json::from_cbor(std::vector({0xf9, 0xc4, 0x00}))); - - // half-precision float - //CHECK(json::to_cbor(json::parse("0.00006103515625")) == std::vector({0xf9, 0x04, 0x00})); - CHECK(json::parse("-4.0") == json::from_cbor(std::vector({0xf9, 0xc4, 0x00}))); - - // half-precision float - //CHECK(json::to_cbor(json::parse("-4.0")) == std::vector({0xf9, 0xc4, 0x00})); - CHECK(json::parse("-4.0") == json::from_cbor(std::vector({0xf9, 0xc4, 0x00}))); - - CHECK(json::to_cbor(json::parse("-4.1")) == std::vector({0xfb, 0xc0, 0x10, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66})); - CHECK(json::parse("-4.1") == json::from_cbor(std::vector({0xfb, 0xc0, 0x10, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66}))); - } - - SECTION("simple values") - { - CHECK(json::to_cbor(json::parse("false")) == std::vector({0xf4})); - CHECK(json::parse("false") == json::from_cbor(std::vector({0xf4}))); - - CHECK(json::to_cbor(json::parse("true")) == std::vector({0xf5})); - CHECK(json::parse("true") == json::from_cbor(std::vector({0xf5}))); - - CHECK(json::to_cbor(json::parse("true")) == std::vector({0xf5})); - CHECK(json::parse("true") == json::from_cbor(std::vector({0xf5}))); - } - - SECTION("strings") - { - CHECK(json::to_cbor(json::parse("\"\"")) == std::vector({0x60})); - CHECK(json::parse("\"\"") == json::from_cbor(std::vector({0x60}))); - - CHECK(json::to_cbor(json::parse("\"a\"")) == std::vector({0x61, 0x61})); - CHECK(json::parse("\"a\"") == json::from_cbor(std::vector({0x61, 0x61}))); - - CHECK(json::to_cbor(json::parse("\"IETF\"")) == std::vector({0x64, 0x49, 0x45, 0x54, 0x46})); - CHECK(json::parse("\"IETF\"") == json::from_cbor(std::vector({0x64, 0x49, 0x45, 0x54, 0x46}))); - - CHECK(json::to_cbor(json::parse("\"\\u00fc\"")) == std::vector({0x62, 0xc3, 0xbc})); - CHECK(json::parse("\"\\u00fc\"") == json::from_cbor(std::vector({0x62, 0xc3, 0xbc}))); - - CHECK(json::to_cbor(json::parse("\"\\u6c34\"")) == std::vector({0x63, 0xe6, 0xb0, 0xb4})); - CHECK(json::parse("\"\\u6c34\"") == json::from_cbor(std::vector({0x63, 0xe6, 0xb0, 0xb4}))); - - CHECK(json::to_cbor(json::parse("\"\\ud800\\udd51\"")) == std::vector({0x64, 0xf0, 0x90, 0x85, 0x91})); - CHECK(json::parse("\"\\ud800\\udd51\"") == json::from_cbor(std::vector({0x64, 0xf0, 0x90, 0x85, 0x91}))); - - // indefinite length strings - CHECK(json::parse("\"streaming\"") == json::from_cbor(std::vector({0x7f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0xff}))); - } - - SECTION("arrays") - { - CHECK(json::to_cbor(json::parse("[]")) == std::vector({0x80})); - CHECK(json::parse("[]") == json::from_cbor(std::vector({0x80}))); - - CHECK(json::to_cbor(json::parse("[1, 2, 3]")) == std::vector({0x83, 0x01, 0x02, 0x03})); - CHECK(json::parse("[1, 2, 3]") == json::from_cbor(std::vector({0x83, 0x01, 0x02, 0x03}))); - - CHECK(json::to_cbor(json::parse("[1, [2, 3], [4, 5]]")) == std::vector({0x83, 0x01, 0x82, 0x02, 0x03, 0x82, 0x04, 0x05})); - CHECK(json::parse("[1, [2, 3], [4, 5]]") == json::from_cbor(std::vector({0x83, 0x01, 0x82, 0x02, 0x03, 0x82, 0x04, 0x05}))); - - CHECK(json::to_cbor(json::parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]")) == std::vector({0x98, 0x19, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x18, 0x18, 0x19})); - CHECK(json::parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]") == json::from_cbor(std::vector({0x98, 0x19, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x18, 0x18, 0x19}))); - - // indefinite length arrays - CHECK(json::parse("[]") == json::from_cbor(std::vector({0x9f, 0xff}))); - CHECK(json::parse("[1, [2, 3], [4, 5]] ") == json::from_cbor(std::vector({0x9f, 0x01, 0x82, 0x02, 0x03, 0x9f, 0x04, 0x05, 0xff, 0xff}))); - CHECK(json::parse("[1, [2, 3], [4, 5]]") == json::from_cbor(std::vector({0x9f, 0x01, 0x82, 0x02, 0x03, 0x82, 0x04, 0x05, 0xff}))); - CHECK(json::parse("[1, [2, 3], [4, 5]]") == json::from_cbor(std::vector({0x83, 0x01, 0x82, 0x02, 0x03, 0x9f, 0x04, 0x05, 0xff}))); - CHECK(json::parse("[1, [2, 3], [4, 5]]") == json::from_cbor(std::vector({0x83, 0x01, 0x9f, 0x02, 0x03, 0xff, 0x82, 0x04, 0x05}))); - CHECK(json::parse("[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]") == json::from_cbor(std::vector({0x9f, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x18, 0x18, 0x19, 0xff}))); - } - - SECTION("objects") - { - CHECK(json::to_cbor(json::parse("{}")) == std::vector({0xa0})); - CHECK(json::parse("{}") == json::from_cbor(std::vector({0xa0}))); - - CHECK(json::to_cbor(json::parse("{\"a\": 1, \"b\": [2, 3]}")) == std::vector({0xa2, 0x61, 0x61, 0x01, 0x61, 0x62, 0x82, 0x02, 0x03})); - CHECK(json::parse("{\"a\": 1, \"b\": [2, 3]}") == json::from_cbor(std::vector({0xa2, 0x61, 0x61, 0x01, 0x61, 0x62, 0x82, 0x02, 0x03}))); - - CHECK(json::to_cbor(json::parse("[\"a\", {\"b\": \"c\"}]")) == std::vector({0x82, 0x61, 0x61, 0xa1, 0x61, 0x62, 0x61, 0x63})); - CHECK(json::parse("[\"a\", {\"b\": \"c\"}]") == json::from_cbor(std::vector({0x82, 0x61, 0x61, 0xa1, 0x61, 0x62, 0x61, 0x63}))); - - CHECK(json::to_cbor(json::parse("{\"a\": \"A\", \"b\": \"B\", \"c\": \"C\", \"d\": \"D\", \"e\": \"E\"}")) == std::vector({0xa5, 0x61, 0x61, 0x61, 0x41, 0x61, 0x62, 0x61, 0x42, 0x61, 0x63, 0x61, 0x43, 0x61, 0x64, 0x61, 0x44, 0x61, 0x65, 0x61, 0x45})); - CHECK(json::parse("{\"a\": \"A\", \"b\": \"B\", \"c\": \"C\", \"d\": \"D\", \"e\": \"E\"}") == json::from_cbor(std::vector({0xa5, 0x61, 0x61, 0x61, 0x41, 0x61, 0x62, 0x61, 0x42, 0x61, 0x63, 0x61, 0x43, 0x61, 0x64, 0x61, 0x44, 0x61, 0x65, 0x61, 0x45}))); - - // indefinite length objects - CHECK(json::parse("{\"a\": 1, \"b\": [2, 3]}") == json::from_cbor(std::vector({0xbf, 0x61, 0x61, 0x01, 0x61, 0x62, 0x9f, 0x02, 0x03, 0xff, 0xff}))); - CHECK(json::parse("[\"a\", {\"b\": \"c\"}]") == json::from_cbor(std::vector({0x82, 0x61, 0x61, 0xbf, 0x61, 0x62, 0x61, 0x63, 0xff}))); - CHECK(json::parse("{\"Fun\": true, \"Amt\": -2}") == json::from_cbor(std::vector({0xbf, 0x63, 0x46, 0x75, 0x6e, 0xf5, 0x63, 0x41, 0x6d, 0x74, 0x21, 0xff}))); - } -} -*/ From 10bad9381daffa6cedf2a4d14feabe11c961edc2 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 10 Jan 2018 23:13:48 +0100 Subject: [PATCH 11/16] :construction: added size benchmark for binary formats --- benchmarks/Makefile | 5 ++++- benchmarks/src/formats.cpp | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 benchmarks/src/formats.cpp diff --git a/benchmarks/Makefile b/benchmarks/Makefile index ef2de8a3..463a1fbb 100644 --- a/benchmarks/Makefile +++ b/benchmarks/Makefile @@ -17,5 +17,8 @@ json_benchmarks_simple: src/benchmarks_simple.cpp ../src/json.hpp number_jsons: (test -e files/numbers/floats.json -a -e files/numbers/signed_ints.json -a -e files/numbers/unsigned_ints.json) || (cd files/numbers ; python generate.py) +json_formats: src/formats.cpp ../src/json.hpp number_jsons + $(CXX) -std=c++11 $(CXXFLAGS) -DNDEBUG -O3 -flto -I../src $(<) $(LDFLAGS) -o $@ + clean: - rm -f json_benchmarks json_benchmarks_simple files/numbers/*.json + rm -f json_benchmarks json_benchmarks_simple json_formats files/numbers/*.json diff --git a/benchmarks/src/formats.cpp b/benchmarks/src/formats.cpp new file mode 100644 index 00000000..6070e1b3 --- /dev/null +++ b/benchmarks/src/formats.cpp @@ -0,0 +1,35 @@ +#include +#include +#include + +using json = nlohmann::json; + +int main() +{ + std::vector files = { + "files/jeopardy/jeopardy.json", + "files/nativejson-benchmark/canada.json", + "files/nativejson-benchmark/citm_catalog.json", + "files/nativejson-benchmark/twitter.json", + "files/numbers/floats.json", + "files/numbers/signed_ints.json", + "files/numbers/unsigned_ints.json" + }; + + for (const auto& file: files) + { + std::ifstream f(file); + json j = json::parse(f); + auto v_cbor = json::to_cbor(j); + auto v_msgpack = json::to_msgpack(j); + auto v_ubjson = json::to_ubjson(j, true, true); + + double baseline = j.dump().size(); + + std::cout << file << ", JSON: " << j.dump(2).size() << " " << j.dump(2).size()/baseline << std::endl; + std::cout << file << ", JSON (minified): " << j.dump().size() << std::endl; + std::cout << file << ", CBOR: " << v_cbor.size() << " " << v_cbor.size()/baseline << std::endl; + std::cout << file << ", MessagePack: " << v_msgpack.size() << " " << v_msgpack.size()/baseline << std::endl; + std::cout << file << ", UBJSON (optimized): " << v_ubjson.size() << " " << v_ubjson.size()/baseline << std::endl; + } +} From 241eea0c0e64f87a4b5a84ee19f1cde8623dae7f Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 12 Jan 2018 09:23:33 +0100 Subject: [PATCH 12/16] :memo: documentation --- src/json.hpp | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/json.hpp b/src/json.hpp index 5f872e52..097a580e 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -5543,8 +5543,9 @@ class binary_reader /*! @brief reads a UBJSON string - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. @param[in] get_char whether a new character should be retrieved from the input (true, default) or whether the last read @@ -5559,7 +5560,7 @@ class binary_reader { if (get_char) { - get(); + get(); // TODO: may we ignore N here? } check_eof(); @@ -5584,6 +5585,14 @@ class binary_reader } } + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @return pair of the size and the type + */ std::pair get_ubjson_size_type() { std::size_t sz = std::string::npos; @@ -5593,14 +5602,16 @@ class binary_reader if (current == '$') { - get(); // must not ignore 'N', because 'N' maybe the type + tc = get(); // must not ignore 'N', because 'N' maybe the type check_eof(); - tc = current; get_ignore_noop(); if (current != '#') { - assert(false); + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(112, chars_read, + "expected '#' after UBJSON type information; last byte: 0x" + ss.str())); } sz = parse_ubjson_internal(); } @@ -5646,7 +5657,13 @@ class binary_reader { get(); check_eof(); - assert(0 <= current and current <= 127); + if (JSON_UNLIKELY(not(0 <= current and current <= 127))) + { + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(113, chars_read, + "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + ss.str())); + } return std::string(1, static_cast(current)); } From c772c01a48b36740fdaa7d3b4e62bdf72ec320f9 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 14 Jan 2018 10:27:30 +0100 Subject: [PATCH 13/16] :recycle: refactored code to split headers --- develop/detail/parsing/binary_reader.hpp | 270 ++++++++++++++++ develop/detail/parsing/binary_writer.hpp | 386 +++++++++++++++++++++-- develop/json.hpp | 33 ++ src/json.hpp | 2 +- 4 files changed, 671 insertions(+), 20 deletions(-) diff --git a/develop/detail/parsing/binary_reader.hpp b/develop/detail/parsing/binary_reader.hpp index bacf3d46..1ba961c9 100644 --- a/develop/detail/parsing/binary_reader.hpp +++ b/develop/detail/parsing/binary_reader.hpp @@ -90,6 +90,27 @@ class binary_reader return res; } + /*! + @brief create a JSON value from UBJSON input + + @param[in] strict whether to expect the input to be consumed completed + @return JSON value created from UBJSON input + + @throw parse_error.110 if input ended unexpectedly or the end of file was + not reached when @a strict was set to true + @throw parse_error.112 if unsupported byte was read + */ + BasicJsonType parse_ubjson(const bool strict) + { + const auto res = parse_ubjson_internal(); + if (strict) + { + get_ignore_noop(); + check_eof(true); + } + return res; + } + /*! @brief determine system byte order @@ -751,6 +772,16 @@ class binary_reader } } + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + */ + BasicJsonType parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + /*! @brief get next character from the input @@ -766,6 +797,20 @@ class binary_reader return (current = ia->get_character()); } + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + /* @brief read a number from the input @@ -1051,6 +1096,231 @@ class binary_reader return result; } + /*! + @brief reads a UBJSON string + + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. + + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return string + + @throw parse_error.110 if input ended + @throw parse_error.113 if an unexpected byte is read + */ + std::string get_ubjson_string(const bool get_char = true) + { + if (get_char) + { + get(); // TODO: may we ignore N here? + } + + check_eof(); + + switch (current) + { + case 'U': + return get_string(get_number()); + case 'i': + return get_string(get_number()); + case 'I': + return get_string(get_number()); + case 'l': + return get_string(get_number()); + case 'L': + return get_string(get_number()); + default: + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(113, chars_read, + "expected a UBJSON string; last byte: 0x" + ss.str())); + } + } + + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @return pair of the size and the type + */ + std::pair get_ubjson_size_type() + { + std::size_t sz = std::string::npos; + int tc = 0; + + get_ignore_noop(); + + if (current == '$') + { + tc = get(); // must not ignore 'N', because 'N' maybe the type + check_eof(); + + get_ignore_noop(); + if (current != '#') + { + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(112, chars_read, + "expected '#' after UBJSON type information; last byte: 0x" + ss.str())); + } + sz = parse_ubjson_internal(); + } + else if (current == '#') + { + sz = parse_ubjson_internal(); + } + + return std::make_pair(sz, tc); + } + + BasicJsonType get_ubjson_value(const int prefix) + { + switch (prefix) + { + case std::char_traits::eof(): // EOF + JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + + case 'T': // true + return true; + case 'F': // false + return false; + + case 'Z': // null + return nullptr; + + case 'U': + return get_number(); + case 'i': + return get_number(); + case 'I': + return get_number(); + case 'l': + return get_number(); + case 'L': + return get_number(); + case 'd': + return get_number(); + case 'D': + return get_number(); + + case 'C': // char + { + get(); + check_eof(); + if (JSON_UNLIKELY(not(0 <= current and current <= 127))) + { + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(113, chars_read, + "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + ss.str())); + } + return std::string(1, static_cast(current)); + } + + case 'S': // string + return get_ubjson_string(); + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + std::stringstream ss; + ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; + JSON_THROW(parse_error::create(112, chars_read, + "error reading UBJSON; last byte: 0x" + ss.str())); + } + } + + BasicJsonType get_ubjson_array() + { + BasicJsonType result = value_t::array; + const auto size_and_type = get_ubjson_size_type(); + + if (size_and_type.first != std::string::npos) + { + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + std::generate_n(std::back_inserter(*result.m_value.array), + size_and_type.first, [this, size_and_type]() + { + return get_ubjson_value(size_and_type.second); + }); + } + else + { + std::generate_n(std::back_inserter(*result.m_value.array), + size_and_type.first, [this]() + { + return parse_ubjson_internal(); + }); + } + } + else + { + while (current != ']') + { + result.push_back(parse_ubjson_internal(false)); + get_ignore_noop(); + } + } + + return result; + } + + BasicJsonType get_ubjson_object() + { + BasicJsonType result = value_t::object; + const auto size_and_type = get_ubjson_size_type(); + + if (size_and_type.first != std::string::npos) + { + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + std::generate_n(std::inserter(*result.m_value.object, + result.m_value.object->end()), + size_and_type.first, [this, size_and_type]() + { + auto key = get_ubjson_string(); + auto val = get_ubjson_value(size_and_type.second); + return std::make_pair(std::move(key), std::move(val)); + }); + } + else + { + std::generate_n(std::inserter(*result.m_value.object, + result.m_value.object->end()), + size_and_type.first, [this]() + { + auto key = get_ubjson_string(); + auto val = parse_ubjson_internal(); + return std::make_pair(std::move(key), std::move(val)); + }); + } + } + else + { + while (current != '}') + { + auto key = get_ubjson_string(false); + result[std::move(key)] = parse_ubjson_internal(); + get_ignore_noop(); + } + } + + return result; + } + /*! @brief check if input ended @throw parse_error.110 if input ended diff --git a/develop/detail/parsing/binary_writer.hpp b/develop/detail/parsing/binary_writer.hpp index 68034fc7..a7722d2b 100644 --- a/develop/detail/parsing/binary_writer.hpp +++ b/develop/detail/parsing/binary_writer.hpp @@ -164,23 +164,23 @@ class binary_writer { write_number(static_cast(0x60 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x78)); write_number(static_cast(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x79)); write_number(static_cast(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x7A)); write_number(static_cast(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x7B)); write_number(static_cast(N)); @@ -202,23 +202,23 @@ class binary_writer { write_number(static_cast(0x80 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x98)); write_number(static_cast(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x99)); write_number(static_cast(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x9A)); write_number(static_cast(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0x9B)); write_number(static_cast(N)); @@ -241,23 +241,23 @@ class binary_writer { write_number(static_cast(0xA0 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xB8)); write_number(static_cast(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xB9)); write_number(static_cast(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xBA)); write_number(static_cast(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xBB)); write_number(static_cast(N)); @@ -425,19 +425,19 @@ class binary_writer // fixstr write_number(static_cast(0xA0 | N)); } - else if (N <= 255) + else if (N <= (std::numeric_limits::max)()) { // str 8 oa->write_character(static_cast(0xD9)); write_number(static_cast(N)); } - else if (N <= 65535) + else if (N <= (std::numeric_limits::max)()) { // str 16 oa->write_character(static_cast(0xDA)); write_number(static_cast(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits::max)()) { // str 32 oa->write_character(static_cast(0xDB)); @@ -460,13 +460,13 @@ class binary_writer // fixarray write_number(static_cast(0x90 | N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits::max)()) { // array 16 oa->write_character(static_cast(0xDC)); write_number(static_cast(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits::max)()) { // array 32 oa->write_character(static_cast(0xDD)); @@ -490,13 +490,13 @@ class binary_writer // fixmap write_number(static_cast(0x80 | (N & 0xF))); } - else if (N <= 65535) + else if (N <= (std::numeric_limits::max)()) { // map 16 oa->write_character(static_cast(0xDE)); write_number(static_cast(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits::max)()) { // map 32 oa->write_character(static_cast(0xDF)); @@ -517,6 +517,165 @@ class binary_writer } } + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(static_cast('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + oa->write_character(j.m_value.boolean + ? static_cast('T') + : static_cast('F')); + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(static_cast('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(static_cast('[')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.array->empty()) + { + assert(use_count); + const char first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(static_cast('$')); + oa->write_character(static_cast(first_prefix)); + } + } + + if (use_count) + { + oa->write_character(static_cast('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(static_cast(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(static_cast('{')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.object->empty()) + { + assert(use_count); + const char first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(static_cast('$')); + oa->write_character(static_cast(first_prefix)); + } + } + + if (use_count) + { + oa->write_character(static_cast('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(static_cast('}')); + } + + break; + } + + default: + break; + } + } + private: /* @brief write a number to output input @@ -544,6 +703,195 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + template + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (std::is_floating_point::value) + { + if (add_prefix) + { + oa->write_character(static_cast('D')); // float64 + } + write_number(n); + } + else if (std::is_unsigned::value) + { + if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('i')); // uint8 + } + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('U')); // uint8 + } + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('I')); // int16 + } + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('l')); // int32 + } + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('L')); // int64 + } + write_number(static_cast(n)); + } + else + { + // TODO: replace by exception + assert(false); + } + } + else + { + if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('i')); // int8 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('U')); // uint8 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('I')); // int16 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('l')); // int32 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(static_cast('L')); // int64 + } + write_number(static_cast(n)); + } + else + { + // TODO: replace by exception + assert(false); + } + } + } + + char ubjson_prefix(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'i'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'U'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'I'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'l'; + } + else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'L'; + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'i'; + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'U'; + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'I'; + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'l'; + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'L'; + } + break; + } + + case value_t::number_float: + return 'D'; + + case value_t::string: + return 'S'; + + case value_t::array: + return '['; + + case value_t::object: + return '{'; + + default: + break; + } + + return '\0'; + } + private: /// whether we can assume little endianess const bool is_little_endian = binary_reader::little_endianess(); diff --git a/develop/json.hpp b/develop/json.hpp index f4c139e4..6982a3f4 100644 --- a/develop/json.hpp +++ b/develop/json.hpp @@ -6630,6 +6630,26 @@ class basic_json binary_writer(o).write_msgpack(j); } + static std::vector to_ubjson(const basic_json& j, + const bool use_size = false, const bool use_type = false) + { + std::vector result; + to_ubjson(j, result, use_size, use_type); + return result; + } + + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type); + } + + static void to_ubjson(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type); + } + /*! @brief create a JSON value from an input in CBOR format @@ -6824,6 +6844,19 @@ class basic_json return binary_reader(detail::input_adapter(std::forward(a1), std::forward(a2))).parse_msgpack(strict); } + static basic_json from_ubjson(detail::input_adapter i, + const bool strict = true) + { + return binary_reader(i).parse_ubjson(strict); + } + + template::value, int> = 0> + static basic_json from_ubjson(A1 && a1, A2 && a2, const bool strict = true) + { + return binary_reader(detail::input_adapter(std::forward(a1), std::forward(a2))).parse_ubjson(strict); + } + /// @} ////////////////////////// diff --git a/src/json.hpp b/src/json.hpp index 09253359..09da4d16 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -6409,7 +6409,7 @@ class binary_writer write_number(static_cast(N)); } // LCOV_EXCL_START - else if (N <= (std::numeric_limits::max)()) + else if (N <= (std::numeric_limits::max)()) { oa->write_character(static_cast(0xBB)); write_number(static_cast(N)); From f85f4967feb06173d52460ddc596b6f91b53e7b3 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 14 Jan 2018 17:22:06 +0100 Subject: [PATCH 14/16] :white_check_mark: improved test coverage --- .travis.yml | 5 - develop/detail/exceptions.hpp | 1 + develop/detail/parsing/binary_reader.hpp | 9 +- develop/detail/parsing/binary_writer.hpp | 8 +- src/json.hpp | 18 +- test/src/unit-ubjson.cpp | 409 ++++++++++++++++------- 6 files changed, 298 insertions(+), 152 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f7e9b21..6391c088 100644 --- a/.travis.yml +++ b/.travis.yml @@ -278,11 +278,6 @@ script: - ctest -C Release -V -j - cd .. - # check if header was correctly amalgamated - - if [ `which python` ]; then - make check-amalgamation ; - fi - # check if homebrew works (only checks develop branch) - if [ `which brew` ]; then brew update ; diff --git a/develop/detail/exceptions.hpp b/develop/detail/exceptions.hpp index 8eee702a..f9d89dad 100644 --- a/develop/detail/exceptions.hpp +++ b/develop/detail/exceptions.hpp @@ -263,6 +263,7 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON only supports integers numbers up to 9223372036854775807. | @liveexample{The following code shows how an `out_of_range` exception can be caught.,out_of_range} diff --git a/develop/detail/parsing/binary_reader.hpp b/develop/detail/parsing/binary_reader.hpp index 1ba961c9..174528c5 100644 --- a/develop/detail/parsing/binary_reader.hpp +++ b/develop/detail/parsing/binary_reader.hpp @@ -1213,7 +1213,7 @@ class binary_reader { get(); check_eof(); - if (JSON_UNLIKELY(not(0 <= current and current <= 127))) + if (JSON_UNLIKELY(current > 127)) { std::stringstream ss; ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; @@ -1286,10 +1286,9 @@ class binary_reader { if (size_and_type.second != 0) { - if (size_and_type.second != 'N') - std::generate_n(std::inserter(*result.m_value.object, - result.m_value.object->end()), - size_and_type.first, [this, size_and_type]() + std::generate_n(std::inserter(*result.m_value.object, + result.m_value.object->end()), + size_and_type.first, [this, size_and_type]() { auto key = get_ubjson_string(); auto val = get_ubjson_value(size_and_type.second); diff --git a/develop/detail/parsing/binary_writer.hpp b/develop/detail/parsing/binary_writer.hpp index a7722d2b..1d2cf601 100644 --- a/develop/detail/parsing/binary_writer.hpp +++ b/develop/detail/parsing/binary_writer.hpp @@ -759,8 +759,7 @@ class binary_writer } else { - // TODO: replace by exception - assert(false); + JSON_THROW(out_of_range::create(407, "number overflow serializing " + std::to_string(n))); } } else @@ -805,11 +804,12 @@ class binary_writer } write_number(static_cast(n)); } + // LCOV_EXCL_START else { - // TODO: replace by exception - assert(false); + JSON_THROW(out_of_range::create(407, "number overflow serializing " + std::to_string(n))); } + // LCOV_EXCL_STOP } } diff --git a/src/json.hpp b/src/json.hpp index 09da4d16..9644eed6 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -715,6 +715,7 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON only supports integers numbers up to 9223372036854775807. | @liveexample{The following code shows how an `out_of_range` exception can be caught.,out_of_range} @@ -6001,7 +6002,7 @@ class binary_reader { get(); check_eof(); - if (JSON_UNLIKELY(not(0 <= current and current <= 127))) + if (JSON_UNLIKELY(current > 127)) { std::stringstream ss; ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; @@ -6074,10 +6075,9 @@ class binary_reader { if (size_and_type.second != 0) { - if (size_and_type.second != 'N') - std::generate_n(std::inserter(*result.m_value.object, - result.m_value.object->end()), - size_and_type.first, [this, size_and_type]() + std::generate_n(std::inserter(*result.m_value.object, + result.m_value.object->end()), + size_and_type.first, [this, size_and_type]() { auto key = get_ubjson_string(); auto val = get_ubjson_value(size_and_type.second); @@ -6911,8 +6911,7 @@ class binary_writer } else { - // TODO: replace by exception - assert(false); + JSON_THROW(out_of_range::create(407, "number overflow serializing " + std::to_string(n))); } } else @@ -6957,11 +6956,12 @@ class binary_writer } write_number(static_cast(n)); } + // LCOV_EXCL_START else { - // TODO: replace by exception - assert(false); + JSON_THROW(out_of_range::create(407, "number overflow serializing " + std::to_string(n))); } + // LCOV_EXCL_STOP } } diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index 38912761..d7b7cde1 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -1172,96 +1172,291 @@ TEST_CASE("UBJSON") "[json.exception.parse_error.110] parse error at 1: unexpected end of input"); } - SECTION("too short byte vector") - { - } - - SECTION("unsupported bytes") - { - SECTION("concrete examples") - { - CHECK_THROWS_AS(json::from_cbor(std::vector({0x1c})), json::parse_error&); - CHECK_THROWS_WITH(json::from_cbor(std::vector({0x1c})), - "[json.exception.parse_error.112] parse error at 1: error reading CBOR; last byte: 0x1C"); - CHECK_THROWS_AS(json::from_cbor(std::vector({0xf8})), json::parse_error&); - CHECK_THROWS_WITH(json::from_cbor(std::vector({0xf8})), - "[json.exception.parse_error.112] parse error at 1: error reading CBOR; last byte: 0xF8"); - } - - SECTION("all unsupported bytes") - { - for (auto byte : - { - // ? - 0x1c, 0x1d, 0x1e, 0x1f, - // ? - 0x3c, 0x3d, 0x3e, 0x3f, - // byte strings - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - // byte strings - 0x58, 0x59, 0x5a, 0x5b, - // ? - 0x5c, 0x5d, 0x5e, - // byte string - 0x5f, - // ? - 0x7c, 0x7d, 0x7e, - // ? - 0x9c, 0x9d, 0x9e, - // ? - 0xbc, 0xbd, 0xbe, - // date/time - 0xc0, 0xc1, - // bignum - 0xc2, 0xc3, - // fraction - 0xc4, - // bigfloat - 0xc5, - // tagged item - 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, - // expected conversion - 0xd5, 0xd6, 0xd7, - // more tagged items - 0xd8, 0xd9, 0xda, 0xdb, - // ? - 0xdc, 0xdd, 0xde, 0xdf, - // (simple value) - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, - // undefined - 0xf7, - // simple value - 0xf8 - }) - { - CHECK_THROWS_AS(json::from_cbor(std::vector({static_cast(byte)})), json::parse_error&); - } - } - } - - SECTION("invalid string in map") - { - CHECK_THROWS_AS(json::from_cbor(std::vector({0xa1, 0xff, 0x01})), json::parse_error&); - CHECK_THROWS_WITH(json::from_cbor(std::vector({0xa1, 0xff, 0x01})), - "[json.exception.parse_error.113] parse error at 2: expected a CBOR string; last byte: 0xFF"); - } - SECTION("strict mode") { - std::vector vec = {0xf6, 0xf6}; + std::vector vec = {'Z', 'Z'}; SECTION("non-strict mode") { - const auto result = json::from_cbor(vec, false); + const auto result = json::from_ubjson(vec, false); CHECK(result == json()); } SECTION("strict mode") { - CHECK_THROWS_AS(json::from_cbor(vec), json::parse_error&); - CHECK_THROWS_WITH(json::from_cbor(vec), + CHECK_THROWS_AS(json::from_ubjson(vec), json::parse_error&); + CHECK_THROWS_WITH(json::from_ubjson(vec), "[json.exception.parse_error.110] parse error at 2: expected end of input"); } } + + SECTION("number out of range") + { + // larger than max int64 + json j = 9223372036854775808llu; + CHECK_THROWS_AS(json::to_ubjson(j), json::out_of_range); + CHECK_THROWS_WITH(json::to_ubjson(j), "[json.exception.out_of_range.407] number overflow serializing 9223372036854775808"); + } + } + + SECTION("parsing values") + { + SECTION("strings") + { + // create a single-character string for all number types + std::vector s_i = {'S', 'i', 1, 'a'}; + std::vector s_U = {'S', 'U', 1, 'a'}; + std::vector s_I = {'S', 'I', 0, 1, 'a'}; + std::vector s_l = {'S', 'l', 0, 0, 0, 1, 'a'}; + std::vector s_L = {'S', 'L', 0, 0, 0, 0, 0, 0, 0, 1, 'a'}; + + // check if string is parsed correctly to "a" + CHECK(json::from_ubjson(s_i) == "a"); + CHECK(json::from_ubjson(s_U) == "a"); + CHECK(json::from_ubjson(s_I) == "a"); + CHECK(json::from_ubjson(s_l) == "a"); + CHECK(json::from_ubjson(s_L) == "a"); + + // roundtrip: output should be optimized + CHECK(json::to_ubjson(json::from_ubjson(s_i)) == s_i); + CHECK(json::to_ubjson(json::from_ubjson(s_U)) == s_i); + CHECK(json::to_ubjson(json::from_ubjson(s_I)) == s_i); + CHECK(json::to_ubjson(json::from_ubjson(s_l)) == s_i); + CHECK(json::to_ubjson(json::from_ubjson(s_L)) == s_i); + } + + SECTION("number") + { + SECTION("float") + { + // float32 + std::vector v_d = {'d', 0x40, 0x49, 0x0f, 0xd0}; + CHECK(json::from_ubjson(v_d) == 3.14159f); + + // float64 + std::vector v_D = {'D', 0x40, 0x09, 0x21, 0xf9, 0xf0, 0x1b, 0x86, 0x6e}; + CHECK(json::from_ubjson(v_D) == 3.14159); + + // float32 is serialized as float64 as the library does not support float32 + CHECK(json::to_ubjson(json::from_ubjson(v_d)) == json::to_ubjson(3.14159f)); + } + } + + SECTION("array") + { + SECTION("optimized version (length only)") + { + // create vector with two elements of the same type + std::vector v_T = {'[', '#', 'i', 2, 'T', 'T'}; + std::vector v_F = {'[', '#', 'i', 2, 'F', 'F'}; + std::vector v_Z = {'[', '#', 'i', 2, 'Z', 'Z'}; + std::vector v_i = {'[', '#', 'i', 2, 'i', 0x7F, 'i', 0x7F}; + std::vector v_U = {'[', '#', 'i', 2, 'U', 0xFF, 'U', 0xFF}; + std::vector v_I = {'[', '#', 'i', 2, 'I', 0x7F, 0xFF, 'I', 0x7F, 0xFF}; + std::vector v_l = {'[', '#', 'i', 2, 'l', 0x7F, 0xFF, 0xFF, 0xFF, 'l', 0x7F, 0xFF, 0xFF, 0xFF}; + std::vector v_L = {'[', '#', 'i', 2, 'L', 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 'L', 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + std::vector v_D = {'[', '#', 'i', 2, 'D', 0x40, 0x09, 0x21, 0xfb, 0x4d, 0x12, 0xd8, 0x4a, 'D', 0x40, 0x09, 0x21, 0xfb, 0x4d, 0x12, 0xd8, 0x4a}; + std::vector v_S = {'[', '#', 'i', 2, 'S', 'i', 1, 'a', 'S', 'i', 1, 'a'}; + std::vector v_C = {'[', '#', 'i', 2, 'C', 'a', 'C', 'a'}; + + // check if vector is parsed correctly + CHECK(json::from_ubjson(v_T) == json({true, true})); + CHECK(json::from_ubjson(v_F) == json({false, false})); + CHECK(json::from_ubjson(v_Z) == json({nullptr, nullptr})); + CHECK(json::from_ubjson(v_i) == json({127, 127})); + CHECK(json::from_ubjson(v_U) == json({255, 255})); + CHECK(json::from_ubjson(v_I) == json({32767, 32767})); + CHECK(json::from_ubjson(v_l) == json({2147483647, 2147483647})); + CHECK(json::from_ubjson(v_L) == json({9223372036854775807, 9223372036854775807})); + CHECK(json::from_ubjson(v_D) == json({3.1415926, 3.1415926})); + CHECK(json::from_ubjson(v_S) == json({"a", "a"})); + CHECK(json::from_ubjson(v_C) == json({"a", "a"})); + + // roundtrip: output should be optimized + CHECK(json::to_ubjson(json::from_ubjson(v_T), true) == v_T); + CHECK(json::to_ubjson(json::from_ubjson(v_F), true) == v_F); + CHECK(json::to_ubjson(json::from_ubjson(v_Z), true) == v_Z); + CHECK(json::to_ubjson(json::from_ubjson(v_i), true) == v_i); + CHECK(json::to_ubjson(json::from_ubjson(v_U), true) == v_U); + CHECK(json::to_ubjson(json::from_ubjson(v_I), true) == v_I); + CHECK(json::to_ubjson(json::from_ubjson(v_l), true) == v_l); + CHECK(json::to_ubjson(json::from_ubjson(v_L), true) == v_L); + CHECK(json::to_ubjson(json::from_ubjson(v_D), true) == v_D); + CHECK(json::to_ubjson(json::from_ubjson(v_S), true) == v_S); + CHECK(json::to_ubjson(json::from_ubjson(v_C), true) == v_S); // char is serialized to string + } + + SECTION("optimized version (type and length)") + { + // create vector with two elements of the same type + std::vector v_N = {'[', '$', 'N', '#', 'i', 2}; + std::vector v_T = {'[', '$', 'T', '#', 'i', 2}; + std::vector v_F = {'[', '$', 'F', '#', 'i', 2}; + std::vector v_Z = {'[', '$', 'Z', '#', 'i', 2}; + std::vector v_i = {'[', '$', 'i', '#', 'i', 2, 0x7F, 0x7F}; + std::vector v_U = {'[', '$', 'U', '#', 'i', 2, 0xFF, 0xFF}; + std::vector v_I = {'[', '$', 'I', '#', 'i', 2, 0x7F, 0xFF, 0x7F, 0xFF}; + std::vector v_l = {'[', '$', 'l', '#', 'i', 2, 0x7F, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF}; + std::vector v_L = {'[', '$', 'L', '#', 'i', 2, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + std::vector v_D = {'[', '$', 'D', '#', 'i', 2, 0x40, 0x09, 0x21, 0xfb, 0x4d, 0x12, 0xd8, 0x4a, 0x40, 0x09, 0x21, 0xfb, 0x4d, 0x12, 0xd8, 0x4a}; + std::vector v_S = {'[', '$', 'S', '#', 'i', 2, 'i', 1, 'a', 'i', 1, 'a'}; + std::vector v_C = {'[', '$', 'C', '#', 'i', 2, 'a', 'a'}; + + // check if vector is parsed correctly + CHECK(json::from_ubjson(v_N) == json::array()); + CHECK(json::from_ubjson(v_T) == json({true, true})); + CHECK(json::from_ubjson(v_F) == json({false, false})); + CHECK(json::from_ubjson(v_Z) == json({nullptr, nullptr})); + CHECK(json::from_ubjson(v_i) == json({127, 127})); + CHECK(json::from_ubjson(v_U) == json({255, 255})); + CHECK(json::from_ubjson(v_I) == json({32767, 32767})); + CHECK(json::from_ubjson(v_l) == json({2147483647, 2147483647})); + CHECK(json::from_ubjson(v_L) == json({9223372036854775807, 9223372036854775807})); + CHECK(json::from_ubjson(v_D) == json({3.1415926, 3.1415926})); + CHECK(json::from_ubjson(v_S) == json({"a", "a"})); + CHECK(json::from_ubjson(v_C) == json({"a", "a"})); + + // roundtrip: output should be optimized + std::vector v_empty = {'[', '#', 'i', 0}; + CHECK(json::to_ubjson(json::from_ubjson(v_N), true, true) == v_empty); + CHECK(json::to_ubjson(json::from_ubjson(v_T), true, true) == v_T); + CHECK(json::to_ubjson(json::from_ubjson(v_F), true, true) == v_F); + CHECK(json::to_ubjson(json::from_ubjson(v_Z), true, true) == v_Z); + CHECK(json::to_ubjson(json::from_ubjson(v_i), true, true) == v_i); + CHECK(json::to_ubjson(json::from_ubjson(v_U), true, true) == v_U); + CHECK(json::to_ubjson(json::from_ubjson(v_I), true, true) == v_I); + CHECK(json::to_ubjson(json::from_ubjson(v_l), true, true) == v_l); + CHECK(json::to_ubjson(json::from_ubjson(v_L), true, true) == v_L); + CHECK(json::to_ubjson(json::from_ubjson(v_D), true, true) == v_D); + CHECK(json::to_ubjson(json::from_ubjson(v_S), true, true) == v_S); + CHECK(json::to_ubjson(json::from_ubjson(v_C), true, true) == v_S); // char is serialized to string + } + } + } + + SECTION("parse errors") + { + SECTION("char") + { + SECTION("eof after C byte") + { + std::vector v = {'C'}; + CHECK_THROWS_AS(json::from_ubjson(v), json::parse_error); + CHECK_THROWS_WITH(json::from_ubjson(v), "[json.exception.parse_error.110] parse error at 2: unexpected end of input"); + } + + SECTION("byte out of range") + { + std::vector v = {'C', 130}; + CHECK_THROWS_AS(json::from_ubjson(v), json::parse_error); + CHECK_THROWS_WITH(json::from_ubjson(v), "[json.exception.parse_error.113] parse error at 2: byte after 'C' must be in range 0x00..0x7F; last byte: 0x82"); + } + } + + SECTION("strings") + { + SECTION("eof after S byte") + { + std::vector v = {'S'}; + CHECK_THROWS_AS(json::from_ubjson(v), json::parse_error); + CHECK_THROWS_WITH(json::from_ubjson(v), "[json.exception.parse_error.110] parse error at 2: unexpected end of input"); + } + + SECTION("invalid byte") + { + std::vector v = {'S', '1', 'a'}; + CHECK_THROWS_AS(json::from_ubjson(v), json::parse_error); + CHECK_THROWS_WITH(json::from_ubjson(v), "[json.exception.parse_error.113] parse error at 2: expected a UBJSON string; last byte: 0x31"); + } + } + + SECTION("array") + { + SECTION("optimized array: no size following type") + { + std::vector v = {'[', '$', 'i', 2}; + CHECK_THROWS_AS(json::from_ubjson(v), json::parse_error); + CHECK_THROWS_WITH(json::from_ubjson(v), "[json.exception.parse_error.112] parse error at 4: expected '#' after UBJSON type information; last byte: 0x02"); + } + } + } + + SECTION("writing optimized values") + { + SECTION("integer") + { + SECTION("array of i") + { + json j = {1, 2}; + std::vector expected = {'[', '$', 'i', '#', 'i', 2, 1, 2}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + + SECTION("array of U") + { + json j = {200, 201}; + std::vector expected = {'[', '$', 'U', '#', 'i', 2, 0xC8, 0xC9}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + + SECTION("array of I") + { + json j = {30000, 30001}; + std::vector expected = {'[', '$', 'I', '#', 'i', 2, 0x75, 0x30, 0x75, 0x31}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + + SECTION("array of l") + { + json j = {70000, 70001}; + std::vector expected = {'[', '$', 'l', '#', 'i', 2, 0x00, 0x01, 0x11, 0x70, 0x00, 0x01, 0x11, 0x71}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + + SECTION("array of L") + { + json j = {5000000000, 5000000001}; + std::vector expected = {'[', '$', 'L', '#', 'i', 2, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x01}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + } + + SECTION("unsigned integer") + { + SECTION("array of i") + { + json j = {1u, 2u}; + std::vector expected = {'[', '$', 'i', '#', 'i', 2, 1, 2}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + + SECTION("array of U") + { + json j = {200u, 201u}; + std::vector expected = {'[', '$', 'U', '#', 'i', 2, 0xC8, 0xC9}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + + SECTION("array of I") + { + json j = {30000u, 30001u}; + std::vector expected = {'[', '$', 'I', '#', 'i', 2, 0x75, 0x30, 0x75, 0x31}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + + SECTION("array of l") + { + json j = {70000u, 70001u}; + std::vector expected = {'[', '$', 'l', '#', 'i', 2, 0x00, 0x01, 0x11, 0x70, 0x00, 0x01, 0x11, 0x71}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + + SECTION("array of L") + { + json j = {5000000000u, 5000000001u}; + std::vector expected = {'[', '$', 'L', '#', 'i', 2, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x01}; + CHECK(json::to_ubjson(j, true, true) == expected); + } + } } } @@ -1871,73 +2066,29 @@ TEST_CASE("Universal Binary JSON Specification Examples 2") */ -/* TEST_CASE("all first bytes", "[!throws]") { // these bytes will fail immediately with exception parse_error.112 - std::set unsupported = + std::set supported = { - //// types not supported by this library - - // byte strings - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - // byte strings - 0x58, 0x59, 0x5a, 0x5b, 0x5f, - // date/time - 0xc0, 0xc1, - // bignum - 0xc2, 0xc3, - // decimal fracion - 0xc4, - // bigfloat - 0xc5, - // tagged item - 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, - 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd8, - 0xd9, 0xda, 0xdb, - // expected conversion - 0xd5, 0xd6, 0xd7, - // simple value - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xef, 0xf0, - 0xf1, 0xf2, 0xf3, - 0xf8, - // undefined - 0xf7, - - //// bytes not specified by CBOR - - 0x1c, 0x1d, 0x1e, 0x1f, - 0x3c, 0x3d, 0x3e, 0x3f, - 0x5c, 0x5d, 0x5e, - 0x7c, 0x7d, 0x7e, - 0x9c, 0x9d, 0x9e, - 0xbc, 0xbd, 0xbe, - 0xdc, 0xdd, 0xde, 0xdf, - 0xee, - 0xfc, 0xfe, 0xfd, - - /// break cannot be the first byte - - 0xff + 'T', 'F', 'Z', 'U', 'i', 'I', 'l', 'L', 'd', 'D', 'C', 'S', '[', '{', 'N' }; for (auto i = 0; i < 256; ++i) { const auto byte = static_cast(i); + CAPTURE(byte); try { - json::from_cbor(std::vector(1, byte)); + json::from_ubjson(std::vector(1, byte)); } catch (const json::parse_error& e) { // check that parse_error.112 is only thrown if the - // first byte is in the unsupported set + // first byte is not in the supported set CAPTURE(e.what()); - if (std::find(unsupported.begin(), unsupported.end(), byte) != unsupported.end()) + if (std::find(supported.begin(), supported.end(), byte) == supported.end()) { CHECK(e.id == 112); } @@ -1948,4 +2099,4 @@ TEST_CASE("all first bytes", "[!throws]") } } } -*/ + From 9d6b3731b9a15996e8cee7462d997691063b9939 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 14 Jan 2018 20:07:38 +0100 Subject: [PATCH 15/16] :white_check_mark: improved test coverage --- Makefile | 2 +- develop/detail/parsing/binary_writer.hpp | 23 ++++++++++------- src/json.hpp | 23 ++++++++++------- test/src/unit-ubjson.cpp | 33 ++++++++++++++++++------ 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 6756182e..d4b3336f 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ coverage: mkdir build_coverage cd build_coverage ; CXX=g++-5 cmake .. -GNinja -DJSON_Coverage=ON cd build_coverage ; ninja - cd build_coverage ; ctest + cd build_coverage ; ctest -j10 cd build_coverage ; ninja lcov_html open build_coverage/test/html/index.html diff --git a/develop/detail/parsing/binary_writer.hpp b/develop/detail/parsing/binary_writer.hpp index 1d2cf601..7bf894f6 100644 --- a/develop/detail/parsing/binary_writer.hpp +++ b/develop/detail/parsing/binary_writer.hpp @@ -813,7 +813,16 @@ class binary_writer } } - char ubjson_prefix(const BasicJsonType& j) + /*! + @brief determine the type prefix of container values + + @note This function does not need to be 100% accurate when it comes to + integer limits. In case a number exceeds the limits of int64_t, + this will be detected by a later call to function + write_number_with_ubjson_prefix. Therefore, we return 'L' for any + value that does not fit the previous limits. + */ + char ubjson_prefix(const BasicJsonType& j) const noexcept { switch (j.type()) { @@ -841,11 +850,10 @@ class binary_writer { return 'l'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + else // no check and assume int64_t (see note above) { return 'L'; } - break; } case value_t::number_unsigned: @@ -866,11 +874,10 @@ class binary_writer { return 'l'; } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else // no check and assume int64_t (see note above) { return 'L'; } - break; } case value_t::number_float: @@ -885,11 +892,9 @@ class binary_writer case value_t::object: return '{'; - default: - break; + default: // discarded values + return 'N'; } - - return '\0'; } private: diff --git a/src/json.hpp b/src/json.hpp index 9644eed6..731feb95 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -6965,7 +6965,16 @@ class binary_writer } } - char ubjson_prefix(const BasicJsonType& j) + /*! + @brief determine the type prefix of container values + + @note This function does not need to be 100% accurate when it comes to + integer limits. In case a number exceeds the limits of int64_t, + this will be detected by a later call to function + write_number_with_ubjson_prefix. Therefore, we return 'L' for any + value that does not fit the previous limits. + */ + char ubjson_prefix(const BasicJsonType& j) const noexcept { switch (j.type()) { @@ -6993,11 +7002,10 @@ class binary_writer { return 'l'; } - else if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + else // no check and assume int64_t (see note above) { return 'L'; } - break; } case value_t::number_unsigned: @@ -7018,11 +7026,10 @@ class binary_writer { return 'l'; } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + else // no check and assume int64_t (see note above) { return 'L'; } - break; } case value_t::number_float: @@ -7037,11 +7044,9 @@ class binary_writer case value_t::object: return '{'; - default: - break; + default: // discarded values + return 'N'; } - - return '\0'; } private: diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index d7b7cde1..20c846cb 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -1386,8 +1386,8 @@ TEST_CASE("UBJSON") { SECTION("array of i") { - json j = {1, 2}; - std::vector expected = {'[', '$', 'i', '#', 'i', 2, 1, 2}; + json j = {1, -1}; + std::vector expected = {'[', '$', 'i', '#', 'i', 2, 1, 0xff}; CHECK(json::to_ubjson(j, true, true) == expected); } @@ -1400,22 +1400,22 @@ TEST_CASE("UBJSON") SECTION("array of I") { - json j = {30000, 30001}; - std::vector expected = {'[', '$', 'I', '#', 'i', 2, 0x75, 0x30, 0x75, 0x31}; + json j = {30000, -30000}; + std::vector expected = {'[', '$', 'I', '#', 'i', 2, 0x75, 0x30, 0x8a, 0xd0}; CHECK(json::to_ubjson(j, true, true) == expected); } SECTION("array of l") { - json j = {70000, 70001}; - std::vector expected = {'[', '$', 'l', '#', 'i', 2, 0x00, 0x01, 0x11, 0x70, 0x00, 0x01, 0x11, 0x71}; + json j = {70000, -70000}; + std::vector expected = {'[', '$', 'l', '#', 'i', 2, 0x00, 0x01, 0x11, 0x70, 0xFF, 0xFE, 0xEE, 0x90}; CHECK(json::to_ubjson(j, true, true) == expected); } SECTION("array of L") { - json j = {5000000000, 5000000001}; - std::vector expected = {'[', '$', 'L', '#', 'i', 2, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x01}; + json j = {5000000000, -5000000000}; + std::vector expected = {'[', '$', 'L', '#', 'i', 2, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x00, 0xFF, 0xFF, 0xFF, 0xFE, 0xD5, 0xFA, 0x0E, 0x00}; CHECK(json::to_ubjson(j, true, true) == expected); } } @@ -1426,37 +1426,54 @@ TEST_CASE("UBJSON") { json j = {1u, 2u}; std::vector expected = {'[', '$', 'i', '#', 'i', 2, 1, 2}; + std::vector expected_size = {'[', '#', 'i', 2, 'i', 1, 'i', 2}; CHECK(json::to_ubjson(j, true, true) == expected); + CHECK(json::to_ubjson(j, true) == expected_size); } SECTION("array of U") { json j = {200u, 201u}; std::vector expected = {'[', '$', 'U', '#', 'i', 2, 0xC8, 0xC9}; + std::vector expected_size = {'[', '#', 'i', 2, 'U', 0xC8, 'U', 0xC9}; CHECK(json::to_ubjson(j, true, true) == expected); + CHECK(json::to_ubjson(j, true) == expected_size); } SECTION("array of I") { json j = {30000u, 30001u}; std::vector expected = {'[', '$', 'I', '#', 'i', 2, 0x75, 0x30, 0x75, 0x31}; + std::vector expected_size = {'[', '#', 'i', 2, 'I', 0x75, 0x30, 'I', 0x75, 0x31}; CHECK(json::to_ubjson(j, true, true) == expected); + CHECK(json::to_ubjson(j, true) == expected_size); } SECTION("array of l") { json j = {70000u, 70001u}; std::vector expected = {'[', '$', 'l', '#', 'i', 2, 0x00, 0x01, 0x11, 0x70, 0x00, 0x01, 0x11, 0x71}; + std::vector expected_size = {'[', '#', 'i', 2, 'l', 0x00, 0x01, 0x11, 0x70, 'l', 0x00, 0x01, 0x11, 0x71}; CHECK(json::to_ubjson(j, true, true) == expected); + CHECK(json::to_ubjson(j, true) == expected_size); } SECTION("array of L") { json j = {5000000000u, 5000000001u}; std::vector expected = {'[', '$', 'L', '#', 'i', 2, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x01}; + std::vector expected_size = {'[', '#', 'i', 2, 'L', 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x00, 'L', 0x00, 0x00, 0x00, 0x01, 0x2A, 0x05, 0xF2, 0x01}; CHECK(json::to_ubjson(j, true, true) == expected); + CHECK(json::to_ubjson(j, true) == expected_size); } } + + SECTION("discarded") + { + json j = {json::value_t::discarded, json::value_t::discarded}; + std::vector expected = {'[', '$', 'N', '#', 'i', 2}; + CHECK(json::to_ubjson(j, true, true) == expected); + } } } From 92f729506380e5023e02d5f0243e5e0a965ba006 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 15 Jan 2018 13:21:33 +0100 Subject: [PATCH 16/16] :hammer: cleanup --- .../CouchDB4k.formatted.json | 275 -------------- .../universal-binary-json-java/CouchDB4k.ubj | Bin 2724 -> 0 bytes test/data/universal-binary-json-java/LICENSE | 202 ---------- .../MediaContent.formatted.json | 34 -- .../MediaContent.ubj | Bin 442 -> 0 bytes test/data/universal-binary-json-java/README | 357 ------------------ .../TwitterTimeline.formatted.json | 80 ---- .../TwitterTimeline.ubj | Bin 1792 -> 0 bytes test/data/universal-binary-json-java/url.txt | 1 - test/src/unit-ubjson.cpp | 80 ++-- 10 files changed, 34 insertions(+), 995 deletions(-) delete mode 100644 test/data/universal-binary-json-java/CouchDB4k.formatted.json delete mode 100644 test/data/universal-binary-json-java/CouchDB4k.ubj delete mode 100644 test/data/universal-binary-json-java/LICENSE delete mode 100644 test/data/universal-binary-json-java/MediaContent.formatted.json delete mode 100644 test/data/universal-binary-json-java/MediaContent.ubj delete mode 100644 test/data/universal-binary-json-java/README delete mode 100644 test/data/universal-binary-json-java/TwitterTimeline.formatted.json delete mode 100644 test/data/universal-binary-json-java/TwitterTimeline.ubj delete mode 100644 test/data/universal-binary-json-java/url.txt diff --git a/test/data/universal-binary-json-java/CouchDB4k.formatted.json b/test/data/universal-binary-json-java/CouchDB4k.formatted.json deleted file mode 100644 index d12387da..00000000 --- a/test/data/universal-binary-json-java/CouchDB4k.formatted.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "data3":"ColreUHAtuYoUOx1N4ZloouQt2o6ugnUT6eYtS10gu7niM8i0vEiNufpk1RlMQXaHXlIwQBDsMFDFUQcFeg2vW5eD259Xm", - "data4":"zCxriJhL726WNNTdJJzurgSA8vKT6rHA0cFCb9koZcLUMXg4rmoXVPqIHWYaCV0ovl2t6xm7I1Hm36jXpLlXEb8fRfbwBeTW2V0OAsVqYH8eAT", - "data0":"9EVqHm5ARqcEB5jq2D14U2bCJPyBY0JWDr1Tjh8gTB0sWUNjqYiWDxFzlx6S", - "data7":"Bi1ujcgEvfADfBeyZudE7nwxc3Ik8qpYjsJIfKmwOMEbV2L3Bi0x2tcRpGuf4fiyvIbypDvJN1PPdQtfQW1Gv6zccXHwwZwKzUq6", - "data5":{ - "integers":[ - 756509, - 116117, - 776378, - 275045, - 703447, - 50156, - 685803, - 147958, - 941747, - 905651, - 57367, - 530248, - 312888, - 740951, - 988947, - 450154 - ], - "float1":76.572, - "float2":83.5299, - "nested1":{ - "integers":[ - 756509, - 116117, - 776378, - 275045, - 703447, - 50156, - 685803, - 147958, - 941747, - 905651, - 57367, - 530248, - 312888, - 740951, - 988947, - 450154 - ], - "floats":[ - 43121609.5543, - 99454976.3019, - 32945584.756, - 18122905.9212, - 45893183.44, - 63052200.6225, - 69032152.6897, - 3748217.6946, - 75449850.474, - 37111527.415, - 84852536.859, - 32906366.487, - 27027600.417, - 63874310.5614, - 39440408.51, - 97176857.1716, - 97438252.1171, - 49728043.5056, - 40818570.245, - 41415831.8949, - 24796297.4251, - 2819085.3449, - 84263963.4848, - 74503228.6878, - 67925677.403, - 4758851.9417, - 75227407.9214, - 76946667.8403, - 72518275.9469, - 94167085.9588, - 75883067.8321, - 27389831.6101, - 57987075.5053, - 1298995.2674, - 14590614.6939, - 45292214.2242, - 3332166.364, - 53784167.729, - 25193846.1867, - 81456965.477, - 68532032.39, - 73820009.7952, - 57736110.5717, - 37304166.7363, - 20054244.864, - 29746392.7397, - 86467624.6, - 45192685.8793, - 44008816.5186, - 1861872.8736, - 14595859.467, - 87795257.6703, - 57768720.8303, - 18290154.3126, - 45893183.44, - 63052200.6225, - 69032152.6897, - 3748217.6946, - 75449850.474, - 37111527.415, - 84852536.859, - 32906366.487, - 27027600.417, - 63874310.5614, - 39440408.51, - 97176857.1716, - 97438252.1171, - 49728043.5056, - 40818570.245, - 41415831.8949, - 24796297.4251, - 2819085.3449, - 84263963.4848, - 74503228.6878, - 67925677.403, - 4758851.9417, - 75227407.9214, - 76946667.8403, - 72518275.9469, - 94167085.9588, - 75883067.8321, - 27389831.6101, - 57987075.5053, - 1298995.2674, - 80858801.2712, - 98262252.4656, - 51612877.944, - 33397812.7835, - 36089655.3049, - 50164685.8153, - 16852105.5192, - 61171929.752, - 86376339.7175, - 73009014.5521, - 7397302.331, - 34345128.9589, - 98343269.4418, - 95039116.9058, - 44833102.5752, - 51052997.8873, - 22719195.6783, - 64883244.8699 - ] - }, - "nested2":{ - "integers":[ - 756509, - 116117, - 776378, - 275045, - 703447, - 50156, - 685803, - 147958, - 941747, - 905651, - 57367, - 530248, - 312888, - 740951, - 988947, - 450154 - ], - "float1":76.572, - "float2":83.5299 - } - }, - "strings":[ - "edx5XzRkPVeEW2MBQzQMcUSuMI4FntjhlJ9VGhQaBHKPEazAaT", - "2fQUbzRUax4A", - "jURcBZ0vrJcmf2roZUMzZJQoTsKZDIdj7KhO7itskKvM80jBU9", - "8jKLmo3N2zYdKyTyfTczfr2x6bPaarorlnTNJ7r8lIkiZyBvrP", - "jbUeAVOdBSPzYmYhH0sabUHUH39O5e", - "I8yAQKZsyZhMfpzWjArQU9pQ6PfU6b14q2eWvQjtCUdgAUxFjg", - "97N8ZmGcxRZO4ZabzRRcY4KVHqxJwQ8qY", - "0DtY1aWXmUfJENt9rYW9", - "DtpBUEppPwMnWexi8eIIxlXRO3GUpPgeNFG9ONpWJYvk8xBkVj", - "YsX8V2xOrTw6LhNIMMhO4F4VXFyXUXFr66L3sTkLWgFA9NZuBV", - "fKYYthv8iFvaYoFoYZyB", - "zGuLsPXoJqMbO4PcePteZfDMYFXdWtvNF8WvaplXypsd6" - ], - "data1":"9EVqHm5ARqcEB5jq21v2g0jVcG9CXB0Abk7uAF4NHYyTzeF3TnHhpZBECD14U2bCJPyBY0JWDr1Tjh8gTB0sWUNjqYiWDxFzlx6S", - "integers":[ - 756509, - 116117, - 776378, - 275045, - 703447, - 50156, - 685803, - 147958, - 941747, - 905651, - 57367, - 530248, - 312888, - 740951, - 988947, - 450154 - ], - "more_nested":{ - "integers":[ - 756509, - 116117, - 776378, - 275045, - 703447, - 50156, - 685803, - 147958, - 941747, - 905651, - 57367, - 530248, - 312888, - 740951, - 988947, - 450154 - ], - "float1":76.572, - "float2":83.5299, - "nested1":{ - "integers":[ - 756509, - 116117, - 776378, - 275045, - 703447, - 50156, - 685803, - 147958, - 941747, - 905651, - 57367, - 530248, - 312888, - 740951, - 988947, - 450154 - ] - }, - "nested2":{ - "strings":[ - "2fQUbzRUax4A", - "jURcBZ0vrJcmf2roZUMzZJQoTsKZDIdj7KhO7itskKvM80jBU9", - "8jKLmo3N2zYdKyTyfTczfr2x6bPaarorlnTNJ7r8lIkiZyBvrP", - "jbUeAVOdBSPzYmYhH0sabUHUH39O5e", - "I8yAQKZsyZhMfpzWjArQU9pQ6PfU6b14q2eWvQjtCUdgAUxFjg", - "97N8ZmGcxRZO4ZabzRRcY4KVHqxJwQ8qY", - "0DtY1aWXmUfJENt9rYW9", - "DtpBUEppPwMnWexi8eIIxlXRO3GUpPgeNFG9ONpWJYvk8xBkVj", - "YsX8V2xOrTw6LhNIMMhO4F4VXFyXUXFr66L3sTkLWgFA9NZuBV", - "fKYYthv8iFvaYoFoYZyB", - "zGuLsPXoJqMbO4PcePteZfDMYFXdWtvNF8WvaplXypsd6" - ], - "integers":[ - 756509, - 116117, - 776378, - 57367, - 530248, - 312888, - 740951, - 988947, - 450154 - ] - } - } -} \ No newline at end of file diff --git a/test/data/universal-binary-json-java/CouchDB4k.ubj b/test/data/universal-binary-json-java/CouchDB4k.ubj deleted file mode 100644 index 75a6465bc74dcbec4c856640935b331b9e6fa0c3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2724 zcmeHJS;!kz6i!{Lty(JHh+svO#X1O!TnIw~ClDU(~ zB#OA;LVr|(3b3oHj|lF=@NgTQFk$2_N-_F^*|#C;+nlR=Tsaf3YR z8za;=DlN)uc&#bpsyflk)@a-un;hlR>cOyCd`YR&9CL#Ox-oVt%z{5A%DI9@v=)W> zj4`uC~Evb8So>47Ly)jgRdPBe(5_*j~$Lj$1yR&#ZXu&Q#j_0-rgGaQV#YUGMRk%qs^{ zDgJ`b96^7_XVz?aiqD+zcDiIfUe9NaUC*V;uT!a+tn--@*PqO1R(|B;m0mxvqZ}|h z&i?Swh5+V~6KnXGp^GCIa={UQ!v(PI0XqBYuK~UEWCp|^ti2#^ElYqdI_Y*mqZ@rd z*FSa|2=LmcfNuM2H=q-?y#VNe_dZRP*IW+bo&pC5Kgs|)C0YsSdGn!kd7lMz>66QV zPJ4PkpwoXm2gH{ytblO-+66%OKX*N#TQ}Sc=-OM~1!3F8sdU>_X93#&%DsTLDs@0l z-IKby{lzMXZ+-tNpv{F9z~8rRCx}}fybI9zuRj5TXrBs5+L)fbOzr}7_vX(5UH8mq zfNpv7OF)}0I{;{n^Z|%FG9ieceBB51@PQjrTlY#p=kD7C{Ql}qslR7W2J}euMryI; zRS=iAe)})Y9PKQ<3ussGK0xLDc@TfvvjNbm$MyiaeA71|UbDRdsB4jFmb=rxT)yHf z5ccgJ06Oxh^HQ@rE&%j&@y$$~L&^y*Ewq7xFM%oWMemL?KkV26?4=?kT1wWHV!C)O+WoVHbbJmD!?Yu?c8 zq(EcB*BTCEW#dqA1|AUxmL|{kE+%+tMk}I)XI4k@MswY#Lq#i%6teDVWGhz~6RsXBezd4To7NK6Z{ssdMVYb& zRVQg!wSr}*Ni`g^AU3&iB1{!(Y~ppj^m4>>T zbSanTlfKcY<*VAL-gafSTB^w-T`=Pwl`y@gj|meSRFg<*p*kf|N9HA|Q!B8Aroqk( z&0s^4M0u?CkltqLl59rWti&s>{Z<8L)wwz6QQvUY#IZHv(ZSwQ7aB&1p!HK`1g|8k2p)3E=tp9nkSn-#l_Yfx$DJLCs7_P_t554~w^nRUZ MR{dt4`J-XwXD4K^8UO$Q diff --git a/test/data/universal-binary-json-java/LICENSE b/test/data/universal-binary-json-java/LICENSE deleted file mode 100644 index d6456956..00000000 --- a/test/data/universal-binary-json-java/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/test/data/universal-binary-json-java/MediaContent.formatted.json b/test/data/universal-binary-json-java/MediaContent.formatted.json deleted file mode 100644 index fe547871..00000000 --- a/test/data/universal-binary-json-java/MediaContent.formatted.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "Media":{ - "uri":"http://javaone.com/keynote.mpg", - "title":"Javaone Keynote", - "width":640, - "height":480, - "format":"video/mpg4", - "duration":18000000, - "size":58982400, - "bitrate":262144, - "persons":[ - "Bill Gates", - "Steve Jobs" - ], - "player":"JAVA", - "copyright":null - }, - "Images":[ - { - "uri":"http://javaone.com/keynote_large.jpg", - "title":"Javaone Keynote", - "width":1024, - "height":768, - "size":"LARGE" - }, - { - "uri":"http://javaone.com/keynote_small.jpg", - "title":"Javaone Keynote", - "width":320, - "height":240, - "size":"SMALL" - } - ] -} \ No newline at end of file diff --git a/test/data/universal-binary-json-java/MediaContent.ubj b/test/data/universal-binary-json-java/MediaContent.ubj deleted file mode 100644 index 616531ce4938f44ec1dfac12194504e76cea6ca0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 442 zcma)%Jx;?w5QW#jfGLnT0CGWXH&j#@2}L$elSmXaXcG_ihPBt)w=ptR3J$}uI08pt zBMT`FQp%?{Gv9p5MKGsIaCHuEs|BtaWBS`P-SB~{j*_)%(>EGBWhiO;8i5f;5-y6V zj~}NF!MmtTlQZV-5H=+0#@OI|BYai5pB1*@2+PrtcAy*<$u^3<@om@;H=pcYtC)f2id=E6ZO%@s+y( diff --git a/test/data/universal-binary-json-java/README b/test/data/universal-binary-json-java/README deleted file mode 100644 index b4fc4059..00000000 --- a/test/data/universal-binary-json-java/README +++ /dev/null @@ -1,357 +0,0 @@ -Universal Binary JSON Java Library -http://ubjson.org - - -About this project... ---------------------- -This code base is actively under development and implements the latest -specification of Universal Binary JSON (Draft 8). - -I/O is handled through the following core classes: - - * UBJOutputStream - * UBJInputStream - * UBJInputStreamParser - -Additionally, if you are working with Java's NIO and need byte[]-based -results, you can wrap any of the above I/O classes around one of the highly -optimized custom byte[]-stream impls: - - * ByteArrayInputStream (optimized for reuse, not from JDK) - * ByteArrayOutputStream (optimized for reuse, not from JDK) - -If you are working with NIO and want maximum performance by using (and reusing) -direct ByteBuffers along with the UBJSON stream impls, take a look at the: - - * ByteBufferInputStream - * ByteBufferOutputStream - -classes. You can wrap any ByteBuffer source or destination with this stream type, -then wrap that stream type with a UBJSON stream impl and utilize the full -potential of Java's NIO with Universal Binary JSON without giving yourself an -ulcer. - -This allows you to re-use the streams over and over again in a pool of reusable -streams for high-performance I/O with no object creation and garbage collection -overhead; a perfect match for high frequency NIO-based communication. - -All of the core I/O classes have been stable for a while, with tweaks to make the -performance tighter and the error messages more informative over the last few -months. - -More Java-convenient reflection-based I/O classes are available in the -org.ubjson.io.reflect package, but they are under active development. - -There are other efforts (like utilities) in other sub portions of the source -tree. This project intends to eventually contain a multitude of UBJSON -abstraction layers, I/O methods and utilities. - - -Changelog ---------- -02-10-12 - * Added ByteBuffer input and output stream impls as compliments to the - re-usable byte[] stream impls. - - Provides a fast translation layer between standard Java stream-IO and the - new Buffer-based I/O in NIO (including transparent support for using - ultra-fast direct ByteBuffers). - - * Optimized some of the read/write methods by removing unnecessary bounds - checks that are done by subsequent Input or Output stream impls themselves. - -02-09-12 - * Fixed bug with readHugeAsBigInteger returning an invalid value and not - treating the underlying bytes as a string-encoded value. - - * Removed implicit buffer.flip() at the end of StreamDecoder; there is no - way to know what the caller had planned for the buffer before reading all - the data back out. Also the flip was in the wrong place and in the case of - an empty decode request (length=0) the flip would not have been performed, - providing the caller with a "full" buffer of nothing. - - * Rewrote all readHugeXXX method impls; now huge's can be read in as a - simple Number (BigInteger or BigDecimal) as well as raw bytes and even - decoded chars. Additionally the methods can even accept and use existing - buffers to write into to allow for tighter optimizations. - - * Rewrote all readStringXXX methods using the same optimizations and - flexibility that readHuge methods now use. - -02-07-12 - More Memory and CPU optimizations across all the I/O impls. - - * StreamDecoder was rewritten to no longer create a ByteBuffer on every - invocation and instead re-use the same one to decode from on every single call. - - * StreamDecoder now requires the call to pass in a CharBuffer instance to hold - the result of the decode operation. This avoids the creation of a CharBuffer - and allows for large-scale optimization by re-using existing buffers between - calls. - - * StreamEncoder was rewritten to no longer create a ByteBuffer on every - invocation either and now re-uses the same single instance over and over - again. - - * UBJOutputStream writeHuge and writeString series of methods were all - rewritten to accept a CharBuffer in the rawest form (no longer char[]) to stop - hiding the fact that the underlying encode operation required one. - - This gives the caller an opportunity to cache and re-use CharBuffers over - and over again if they can; otherwise this just pushes the CharBuffer.wrap() - call up to the caller instead of hiding it secretly in the method impl under - the guise of accepting a raw char[] (that it couldn't use directly). - - For callers that can re-use buffers, this will lead to big performance gains - now that were previously impossible. - - * UBJInputStream added readHuge and readString methods that accept an existing - CharBuffer argument to make use of the optimizations made in the Stream encoder - and decoder impls. - -01-15-12 - Huge performance boost for deserialization! - - StreamDecoder previously used separate read and write buffers for decoding - bytes to chars including the resulting char[] that was returned to the caller. - This design required at least 1 full array copy before returning a result in - the best case and 2x full array copies before returning the result in the - worst case. - - The rewrite removed the need for a write buffer entire as well as ALL array - copies; in the best OR worse case they never occur anymore. - - Raw performance boost of roughly 25% in all UBJ I/O classes as a result. - -12-01-11 through 01-24-12 - A large amount of work has continued on the core I/O classes (stream impls) - to help make them not only faster and more robust, but also more helpful. - When errors are encountered in the streams, they are reported along with the - stream positions. This is critical for debugging problems with corrupt - formats. - - Also provided ByteArray I/O stream classes that have the potential to provide - HUGE performance boosts for high frequency systems. - - Both these classes (ByteArrayInputStream and ByteArrayOutputStream) are - reusable and when wrapped by a UBJInputStream or UBJOutputStream, the top - level UBJ streams implicitly become reusable as well. - - Reusing the streams not only saves on object creation/GC cleanup but also - allows the caller to re-use the temporary byte[] used to translate to and - from the UBJ format, avoiding object churn entirely! - - This optimized design was chosen to be intentionally performant when combined - with NIO implementations as the ByteBuffer's can be used to wrap() existing - outbound buffers (avoiding the most expensive part of a buffer) or use - array() to get access to the underlying buffer that needs to be written to - the stream. - - In the case of direct ByteBuffers, there is no additional overhead added - because the calls to get or put are required anyway to pull or push the - values from the native memory location. - - This approach allows the fastest implementation of Universal Binary JSON - I/O possible in the JVM whether you are using the standard IO (stream) - classes or the NIO (ByteBuffer) classes in the JDK. - - Some ancillary work on UBJ-based command line utilities (viewers, converters, - etc.) has begun as well. - -11-28-11 - * Fixed UBJInputStreamParser implementation; nextType correctly implements - logic to skip existing element (if called back to back) as well as validate - the marker type it encounters before returning it to the caller. - - * Modified IObjectReader contract; a Parser implementation is required to - make traversing the UBJ stream possible without knowing what type of element - is next. - -11-27-11 - * Streamlined ByteArrayOutputStream implementation to ensure the capacity - of the underlying byte[] is never grown unless absolutely necessary. - - * Rewrote class Javadoc for ByteArrayOutputStream to include a code snippet - on how to use it. - -11-26-11 - * Fixed major bug in how 16, 32 and 64-bit integers are re-assembled when - read back from binary representations. - - * Added a numeric test to specifically catch this error if it ever pops up - again. - - * Optimized reading and writing of numeric values in Input and Output - stream implementations. - - * Optimized ObjectWriter implementation by streamlining the numeric read/write - logic and removing the sub-optimal force-compression of on-disk storage. - - * Fixed ObjectWriter to match exactly with the output written by - UBJOutputStream. - - * Normalized all testing between I/O classes so they use the same classes - to ensure parity. - -11-10-11 - * DRAFT 8 Spec Support Added. - - * Added support for the END marker (readEnd) to the InputStreams which is - required for proper unbounded-container support. - - * Added support for the END marker (writeEnd) to UBJOutputStream. - - UBJInputStreamParser must be used for properly support unbounded-containers - because you never know when the 'E' will be encountered marking the end; - so the caller needs to pay attention to the type marker that nextType() - returns and respond accordingly. - - * Added readHugeAsBytes to InputStream implementations, allowing the bytes - used to represent a HUGE to be read in their raw form with no decoding. - - This can save on processing as BigInteger and BigDecimal do their own decoding - of byte[] directly. - - * Added readHugeAsChars to InputStream implementations, allowing a HUGE - value to be read in as a raw char[] without trying to decode it to a - BigInteger or BigDecimal. - - * Added writeHuge(char[]) to support writing out HUGE values directly from - their raw char[] form without trying to decode from a BigInteger or - BigDecimal. - - * readArrayLength and readObjectLenght were modified to return -1 when an - unbounded container length is encountered (255). - - * Fixed UBJInputStreamParser.nextType to correctly skip past any NOOP - markers found in the underlying stream before returning a valid next type. - - * Normalized all reading of "next byte" to the singular - UBJInputStream.nextMarker method -- correctly skips over NOOP until end of - stream OR until the next valid marker byte, then returns it. - - * Modified readNull behavior to have no return type. It is already designed - to throw an exception when 'NULL' isn't found, no need for the additional - return type. - - * Began work on a simple abstract representation of the UBJ data types as - objects that can be assembled into maps and lists and written/read easily - using the IO package. - - This is intended to be a higher level of abstraction than the IO streams, - but lower level (and faster) than the reflection-based work that inspects - user-provided classes. - - * Refactored StreamDecoder and StreamEncoder into the core IO package, - because they are part of core IO. - - * Refactored StreamParser into the io.parser package to more clearly denote - its relationship to the core IO classes. It is a slightly higher level - abstraction ontop of the core IO, having it along side the core IO classes - while .reflect was a subpackage was confusing and suggested that - StreamParser was somehow intended as a swapable replacement for - UBJInputStream which is not how it is intended to be used. - - * Refactored org.ubjson.reflect to org.ubjson.io.reflect to more correctly - communicate the relationship -- the reflection-based classes are built on - the core IO classes and are just a higher abstraction to interact with UBJSON - with. - - * Renamed IDataType to IMarkerType to follow the naming convention for the - marker bytes set forth by the spec doc. - - -10-14-11 - * ObjectWriter rewritten and works correctly. Tested with the example test - data and wrote out the compressed and uncompressed formats files correctly - from their original object representation. - - * Added support for reading and writing huge values as BigInteger as well - as BigDecimal. - - * Added automatic numeric storage compression support to ObjectWriter - based - on the numeric value (regardless of type) the value will be stored as the - smallest possible representation in the UBJ format if requested. - - * Added mapping support for BigDecimal, BigInteger, AtomicInteger and - AtomicLong to ObjectWriter. - - * Added readNull and readBoolean to the UBJInputStream and - UBJInputStreamParser implementations to make the API feel complete and feel - more natural to use. - -10-10-11 - * com.ubjson.io AND com.ubjson.io.charset are finalized against the - Draft 8 specification. - - * The lowest level UBJInput/OuputStream classes were tightened up to run as - fast as possible showing an 800ns-per-op improvement in speed. - - * Profiled core UBJInput/OuputStream classes using HPROF for a few million - iterations and found no memory leaks and no performance traps; everything at - that low level is as tight as it can be. - - * Stream-parsing facilities were moved out of the overloaded UBJInputStream - class and into their own subclass called UBJInputStreamParser which operates - exactly like a pull-parsing scenario (calling nextType then switching on the - value and pulling the appropriate value out of the stream). - - * More example testing/benchmarking data checked into /test/java/com/ubjson/data - - Will begin reworking the Reflection based Object mapping in the - org.ubjson.reflect package now that the core IO classes are finalized. - - * Removed all old scratch test files from the org.ubjson package, this was - confusing. - -09-27-11 - * Initial check-in of core IO classes to read/write spec. - - -Status ------- -Using the standard UBJInputStream, UBJInputStreamParser and UBJOutputStream -implementations to manually read/write UBJ objects is stable and tuned for -optimal performance. - -Automatic mapping of objects to/from UBJ format via the reflection-based -implementation is not tuned yet. Writing is implemented, but not tuned for -optimal performance and reading still has to be written. - - * org.ubjson.io - STABLE - * org.ubjson.io.parser - STABLE - * org.ubjson.io.reflect - ALPHA - * org.ubjson.model - BETA - - -License -------- -This library is released under the Apache 2 License. See LICENSE. - - -Description ------------ -This project represents (the official?) Java implementations of the -Universal Binary JSON specification: http://ubjson.org - - -Example -------- -Comming soon... - - -Performance ------------ -Comming soon... - - -Reference ---------- -Universal Binary JSON Specification - http://ubjson.org -JSON Specification - http://json.org - - -Contact -------- -If you have questions, comments or bug reports for this software please contact -us at: software@thebuzzmedia.com \ No newline at end of file diff --git a/test/data/universal-binary-json-java/TwitterTimeline.formatted.json b/test/data/universal-binary-json-java/TwitterTimeline.formatted.json deleted file mode 100644 index f56dd9b2..00000000 --- a/test/data/universal-binary-json-java/TwitterTimeline.formatted.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "id_str":"121769183821312000", - "retweet_count":0, - "in_reply_to_screen_name":null, - "in_reply_to_user_id":null, - "truncated":false, - "retweeted":false, - "possibly_sensitive":false, - "in_reply_to_status_id_str":null, - "entities":{ - "urls":[ - { - "url":"http:\/\/t.co\/wtioKkFS", - "display_url":"dlvr.it\/pWQy2", - "indices":[ - 33, - 53 - ], - "expanded_url":"http:\/\/dlvr.it\/pWQy2" - } - ], - "hashtags":[ - - ], - "user_mentions":[ - - ] - }, - "geo":null, - "place":null, - "coordinates":null, - "created_at":"Thu Oct 06 02:10:10 +0000 2011", - "in_reply_to_user_id_str":null, - "user":{ - "id_str":"77029015", - "profile_link_color":"009999", - "protected":false, - "url":"http:\/\/www.techday.co.nz\/", - "screen_name":"techdaynz", - "statuses_count":5144, - "profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1479058408\/techday_48_normal.jpg", - "name":"TechDay", - "default_profile_image":false, - "default_profile":false, - "profile_background_color":"131516", - "lang":"en", - "profile_background_tile":false, - "utc_offset":43200, - "description":"", - "is_translator":false, - "show_all_inline_media":false, - "contributors_enabled":false, - "profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/75893948\/Techday_Background.jpg", - "created_at":"Thu Sep 24 20:02:01 +0000 2009", - "profile_sidebar_fill_color":"efefef", - "follow_request_sent":false, - "friends_count":3215, - "followers_count":3149, - "time_zone":"Auckland", - "favourites_count":0, - "profile_sidebar_border_color":"eeeeee", - "profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1479058408\/techday_48_normal.jpg", - "following":false, - "geo_enabled":false, - "notifications":false, - "profile_use_background_image":true, - "listed_count":151, - "verified":false, - "profile_text_color":"333333", - "location":"Ponsonby, Auckland, NZ", - "id":77029015, - "profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/75893948\/Techday_Background.jpg" - }, - "contributors":null, - "source":"\u003Ca href=\"http:\/\/dlvr.it\" rel=\"nofollow\"\u003Edlvr.it\u003C\/a\u003E", - "in_reply_to_status_id":null, - "favorited":false, - "id":121769183821312000, - "text":"Apple CEO's message to employees http:\/\/t.co\/wtioKkFS" -} \ No newline at end of file diff --git a/test/data/universal-binary-json-java/TwitterTimeline.ubj b/test/data/universal-binary-json-java/TwitterTimeline.ubj deleted file mode 100644 index 19702d4bb147d330ce7bfbc9d819e065c83cafad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1792 zcmai#-)|d55XVDeQYWz~v94=c8hA&jM8tr6*PZEWcWGk<-lbe6kmM^C zPB#~p#~jC&kC5+RH9z9S?Mf^6U~dGvj~`Rem}nfuTAM^DIyv_7kU%*Gj9dKbVw`#B zvGx2@&u^RsK7GCKh_~-G{2N>)swUkos=9B6gKbdC(aH)mkhxpkd zs+jN*V8}CR*@mAG?`}O0@!`T^z~jr2A=M@e7^LZTaEc|#)d+VFo%WI_>tU=v6L5AitG${I1q$O;)K zK9ElNwOax&0W!xA@jSngs8QSoa_xBeN{l+~Zm-#Ebt4Cj^2uH-3FJusC&0ImdW{w- z?u<0(2TK9^J^W&^K@b%JoUUn*C&ovI70NP(^uENJZ9VEGCR zd6@UbqLMAVbB#Bz()), json::parse_error&); - CHECK_THROWS_WITH(json::from_ubjson(std::vector()), - "[json.exception.parse_error.110] parse error at 1: unexpected end of input"); - } - SECTION("strict mode") { std::vector vec = {'Z', 'Z'}; @@ -1335,6 +1328,13 @@ TEST_CASE("UBJSON") SECTION("parse errors") { + SECTION("empty byte vector") + { + CHECK_THROWS_AS(json::from_ubjson(std::vector()), json::parse_error&); + CHECK_THROWS_WITH(json::from_ubjson(std::vector()), + "[json.exception.parse_error.110] parse error at 1: unexpected end of input"); + } + SECTION("char") { SECTION("eof after C byte") @@ -1477,8 +1477,9 @@ TEST_CASE("UBJSON") } } +/* // use this testcase outside [hide] to run it with Valgrind -TEST_CASE("single CBOR roundtrip") +TEST_CASE("single UBJSON roundtrip") { SECTION("sample.json") { @@ -1489,11 +1490,11 @@ TEST_CASE("single CBOR roundtrip") json j1 = json::parse(f_json); // parse MessagePack file - std::ifstream f_cbor(filename + ".cbor", std::ios::binary); - std::vector packed((std::istreambuf_iterator(f_cbor)), + std::ifstream f_ubjson(filename + ".ubj", std::ios::binary); + std::vector packed((std::istreambuf_iterator(f_ubjson)), std::istreambuf_iterator()); json j2; - CHECK_NOTHROW(j2 = json::from_cbor(packed)); + CHECK_NOTHROW(j2 = json::from_ubjson(packed)); // compare parsed JSON values CHECK(j1 == j2); @@ -1503,29 +1504,31 @@ TEST_CASE("single CBOR roundtrip") SECTION("std::ostringstream") { std::ostringstream ss; - json::to_cbor(j1, ss); - json j3 = json::from_cbor(ss.str()); + json::to_ubjson(j1, ss); + json j3 = json::from_ubjson(ss.str()); CHECK(j1 == j3); } SECTION("std::string") { std::string s; - json::to_cbor(j1, s); - json j3 = json::from_cbor(s); + json::to_ubjson(j1, s); + json j3 = json::from_ubjson(s); CHECK(j1 == j3); } } // check with different start index - packed.insert(packed.begin(), 5, 0xff); - CHECK(j1 == json::from_cbor(packed.begin() + 5, packed.end())); + // packed.insert(packed.begin(), 5, 0xff); + // CHECK(j1 == json::from_ubjson(packed.begin() + 5, packed.end())); } } +*/ -TEST_CASE("CBOR roundtrips", "[hide]") +/* +TEST_CASE("UBJSON roundtrips", "[hide]") { - SECTION("input from flynn") + SECTION("input from py-ubjson") { for (std::string filename : { @@ -1684,13 +1687,13 @@ TEST_CASE("CBOR roundtrips", "[hide]") SECTION("std::vector") { - // parse CBOR file - std::ifstream f_cbor(filename + ".cbor", std::ios::binary); + // parse UBJSON file + std::ifstream f_ubjson(filename + ".ubj", std::ios::binary); std::vector packed( - (std::istreambuf_iterator(f_cbor)), + (std::istreambuf_iterator(f_ubjson)), std::istreambuf_iterator()); json j2; - CHECK_NOTHROW(j2 = json::from_cbor(packed)); + CHECK_NOTHROW(j2 = json::from_ubjson(packed)); // compare parsed JSON values CHECK(j1 == j2); @@ -1698,10 +1701,10 @@ TEST_CASE("CBOR roundtrips", "[hide]") SECTION("std::ifstream") { - // parse CBOR file - std::ifstream f_cbor(filename + ".cbor", std::ios::binary); + // parse UBJSON file + std::ifstream f_ubjson(filename + ".ubj", std::ios::binary); json j2; - CHECK_NOTHROW(j2 = json::from_cbor(f_cbor)); + CHECK_NOTHROW(j2 = json::from_ubjson(f_ubjson)); // compare parsed JSON values CHECK(j1 == j2); @@ -1709,36 +1712,21 @@ TEST_CASE("CBOR roundtrips", "[hide]") SECTION("uint8_t* and size") { - // parse CBOR file - std::ifstream f_cbor(filename + ".cbor", std::ios::binary); + // parse UBJSON file + std::ifstream f_ubjson(filename + ".ubj", std::ios::binary); std::vector packed( - (std::istreambuf_iterator(f_cbor)), + (std::istreambuf_iterator(f_ubjson)), std::istreambuf_iterator()); json j2; - CHECK_NOTHROW(j2 = json::from_cbor({packed.data(), packed.size()})); + CHECK_NOTHROW(j2 = json::from_ubjson({packed.data(), packed.size()})); // compare parsed JSON values CHECK(j1 == j2); } - - SECTION("output to output adapters") - { - // parse CBOR file - std::ifstream f_cbor(filename + ".cbor", std::ios::binary); - std::vector packed( - (std::istreambuf_iterator(f_cbor)), - std::istreambuf_iterator()); - - SECTION("std::vector") - { - std::vector vec; - json::to_cbor(j1, vec); - CHECK(vec == packed); - } - } } } } +*/ TEST_CASE("Universal Binary JSON Specification Examples 1") {