From f4c4bab600c80746e2fd7e0c03a715d3a9ce0288 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 16 Jun 2020 12:55:36 +0200 Subject: [PATCH 1/7] :sparkles: add option JSON_TestDataDirectory to set path with test data #2189 --- cmake/download_test_data.cmake | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/cmake/download_test_data.cmake b/cmake/download_test_data.cmake index fe95235f..7d73c6d3 100644 --- a/cmake/download_test_data.cmake +++ b/cmake/download_test_data.cmake @@ -3,15 +3,21 @@ find_package(Git) set(JSON_TEST_DATA_URL https://github.com/nlohmann/json_test_data) set(JSON_TEST_DATA_VERSION 2.0.0) -# target to download test data -add_custom_target(download_test_data - COMMAND test -d json_test_data || ${GIT_EXECUTABLE} clone -c advice.detachedHead=false --branch v${JSON_TEST_DATA_VERSION} ${JSON_TEST_DATA_URL}.git --quiet --depth 1 - COMMENT "Downloading test data from ${JSON_TEST_DATA_URL} (v${JSON_TEST_DATA_VERSION})" - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} -) - -# create a header with the path to the downloaded test data -file(WRITE ${CMAKE_BINARY_DIR}/include/test_data.hpp "#define TEST_DATA_DIRECTORY \"${CMAKE_BINARY_DIR}/json_test_data\"\n") +# if variable is set, use test data from given directory rather than downloading them +if(JSON_TestDataDirectory) + message(STATUS "Using test data in ${JSON_TestDataDirectory}.") + add_custom_target(download_test_data) + file(WRITE ${CMAKE_BINARY_DIR}/include/test_data.hpp "#define TEST_DATA_DIRECTORY \"${JSON_TestDataDirectory}\"\n") +else() + # target to download test data + add_custom_target(download_test_data + COMMAND test -d json_test_data || ${GIT_EXECUTABLE} clone -c advice.detachedHead=false --branch v${JSON_TEST_DATA_VERSION} ${JSON_TEST_DATA_URL}.git --quiet --depth 1 + COMMENT "Downloading test data from ${JSON_TEST_DATA_URL} (v${JSON_TEST_DATA_VERSION})" + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + # create a header with the path to the downloaded test data + file(WRITE ${CMAKE_BINARY_DIR}/include/test_data.hpp "#define TEST_DATA_DIRECTORY \"${CMAKE_BINARY_DIR}/json_test_data\"\n") +endif() # determine the operating system (for debug and support purposes) find_program(UNAME_COMMAND uname) From 4d96f4cf6aadd3e5d7b0c0f8bb1472644617e1cb Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Tue, 16 Jun 2020 20:23:01 +0200 Subject: [PATCH 2/7] :wrench: overwork CMake files --- cmake/download_test_data.cmake | 3 +-- test/cmake_fetch_content/project/CMakeLists.txt | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmake/download_test_data.cmake b/cmake/download_test_data.cmake index 7d73c6d3..7d7ff178 100644 --- a/cmake/download_test_data.cmake +++ b/cmake/download_test_data.cmake @@ -1,5 +1,3 @@ -find_package(Git) - set(JSON_TEST_DATA_URL https://github.com/nlohmann/json_test_data) set(JSON_TEST_DATA_VERSION 2.0.0) @@ -9,6 +7,7 @@ if(JSON_TestDataDirectory) add_custom_target(download_test_data) file(WRITE ${CMAKE_BINARY_DIR}/include/test_data.hpp "#define TEST_DATA_DIRECTORY \"${JSON_TestDataDirectory}\"\n") else() + find_package(Git) # target to download test data add_custom_target(download_test_data COMMAND test -d json_test_data || ${GIT_EXECUTABLE} clone -c advice.detachedHead=false --branch v${JSON_TEST_DATA_VERSION} ${JSON_TEST_DATA_URL}.git --quiet --depth 1 diff --git a/test/cmake_fetch_content/project/CMakeLists.txt b/test/cmake_fetch_content/project/CMakeLists.txt index 742a112b..fd8fbdd5 100644 --- a/test/cmake_fetch_content/project/CMakeLists.txt +++ b/test/cmake_fetch_content/project/CMakeLists.txt @@ -4,9 +4,8 @@ project(DummyImport CXX) include(FetchContent) -FetchContent_Declare(json - GIT_REPOSITORY ${CMAKE_CURRENT_SOURCE_DIR}/../../.. - GIT_TAG HEAD) +get_filename_component(GIT_REPOSITORY_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../.. ABSOLUTE) +FetchContent_Declare(json GIT_REPOSITORY ${GIT_REPOSITORY_DIRECTORY} GIT_TAG HEAD) FetchContent_GetProperties(json) if(NOT json_POPULATED) From e86b3fae98854312fc6e59c62e35919a8d8012b1 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 17 Jun 2020 12:35:59 +0200 Subject: [PATCH 3/7] :wrench: add label to tests that require a git checkout --- test/cmake_fetch_content/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/cmake_fetch_content/CMakeLists.txt b/test/cmake_fetch_content/CMakeLists.txt index 6d92d149..0b0d9f65 100644 --- a/test/cmake_fetch_content/CMakeLists.txt +++ b/test/cmake_fetch_content/CMakeLists.txt @@ -11,8 +11,10 @@ if (${CMAKE_VERSION} VERSION_GREATER "3.11.0") ) set_tests_properties(cmake_fetch_content_configure PROPERTIES FIXTURES_SETUP cmake_fetch_content + LABELS git_required ) set_tests_properties(cmake_fetch_content_build PROPERTIES FIXTURES_REQUIRED cmake_fetch_content + LABELS git_required ) endif() From 88a37010d6cfa808ddfa9559a15285b8291ad187 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 17 Jun 2020 21:14:23 +0200 Subject: [PATCH 4/7] :bug: serialize 32-bit floating-point numbers as float 32 in MessagePack (0xCA) #2196 --- .../features/binary_formats/messagepack.md | 7 ++-- .../nlohmann/detail/output/binary_writer.hpp | 14 ++++++-- include/nlohmann/json.hpp | 6 ++-- single_include/nlohmann/json.hpp | 20 +++++++---- test/src/unit-msgpack.cpp | 34 +++++++++++++++++++ 5 files changed, 64 insertions(+), 17 deletions(-) diff --git a/doc/mkdocs/docs/features/binary_formats/messagepack.md b/doc/mkdocs/docs/features/binary_formats/messagepack.md index ed061056..3e041bb7 100644 --- a/doc/mkdocs/docs/features/binary_formats/messagepack.md +++ b/doc/mkdocs/docs/features/binary_formats/messagepack.md @@ -31,7 +31,8 @@ number_unsigned | 128..255 | uint 8 | 0xCC number_unsigned | 256..65535 | uint 16 | 0xCD number_unsigned | 65536..4294967295 | uint 32 | 0xCE number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF -number_float | *any value* | float 64 | 0xCB +number_float | *any value representable by a float* | float 32 | 0xCA +number_float | *any value NOT representable by a float* | float 64 | 0xCB string | *length*: 0..31 | fixstr | 0xA0..0xBF string | *length*: 32..255 | str 8 | 0xD9 string | *length*: 256..65535 | str 16 | 0xDA @@ -61,10 +62,6 @@ binary | *size*: 65536..4294967295 | bin 32 | 0xC6 - arrays with more than 4294967295 elements - objects with more than 4294967295 elements -!!! info "Unused MessagePack types" - - The following MessagePack types are not used in the conversion: float 32 (0xCA) - !!! info "NaN/infinity handling" If NaN or Infinity are stored inside a JSON number, they are serialized properly. function which serializes NaN or Infinity to `null`. diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 269df0d5..f3221191 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -504,8 +504,18 @@ class binary_writer case value_t::number_float: { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); + if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and + static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and + static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) + { + oa->write_character(get_msgpack_float_prefix(static_cast(j.m_value.number_float))); + write_number(static_cast(j.m_value.number_float)); + } + else + { + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + } break; } diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 409a6e79..790ecd4b 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -7040,7 +7040,8 @@ class basic_json number_unsigned | 256..65535 | uint 16 | 0xCD number_unsigned | 65536..4294967295 | uint 32 | 0xCE number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF - number_float | *any value* | float 64 | 0xCB + number_float | *any value representable by a float* | float 32 | 0xCA + number_float | *any value NOT representable by a float* | float 64 | 0xCB string | *length*: 0..31 | fixstr | 0xA0..0xBF string | *length*: 32..255 | str 8 | 0xD9 string | *length*: 256..65535 | str 16 | 0xDA @@ -7064,9 +7065,6 @@ class basic_json - arrays with more than 4294967295 elements - objects with more than 4294967295 elements - @note The following MessagePack types are not used in the conversion: - - float 32 (0xCA) - @note Any MessagePack output created @ref to_msgpack can be successfully parsed by @ref from_msgpack. diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index cc822a54..85fffc7c 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12714,8 +12714,18 @@ class binary_writer case value_t::number_float: { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); + if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and + static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and + static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) + { + oa->write_character(get_msgpack_float_prefix(static_cast(j.m_value.number_float))); + write_number(static_cast(j.m_value.number_float)); + } + else + { + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + } break; } @@ -22821,7 +22831,8 @@ class basic_json number_unsigned | 256..65535 | uint 16 | 0xCD number_unsigned | 65536..4294967295 | uint 32 | 0xCE number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF - number_float | *any value* | float 64 | 0xCB + number_float | *any value representable by a float* | float 32 | 0xCA + number_float | *any value NOT representable by a float* | float 64 | 0xCB string | *length*: 0..31 | fixstr | 0xA0..0xBF string | *length*: 32..255 | str 8 | 0xD9 string | *length*: 256..65535 | str 16 | 0xDA @@ -22845,9 +22856,6 @@ class basic_json - arrays with more than 4294967295 elements - objects with more than 4294967295 elements - @note The following MessagePack types are not used in the conversion: - - float 32 (0xCA) - @note Any MessagePack output created @ref to_msgpack can be successfully parsed by @ref from_msgpack. diff --git a/test/src/unit-msgpack.cpp b/test/src/unit-msgpack.cpp index 2744a67b..5baecf02 100644 --- a/test/src/unit-msgpack.cpp +++ b/test/src/unit-msgpack.cpp @@ -783,6 +783,40 @@ TEST_CASE("MessagePack") CHECK(json::from_msgpack(result) == v); CHECK(json::from_msgpack(result, true, false) == j); } + + SECTION("1.0") + { + double v = 1.0; + json j = v; + std::vector expected = + { + 0xca, 0x3f, 0x80, 0x00, 0x00 + }; + const auto result = json::to_msgpack(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_msgpack(result) == j); + CHECK(json::from_msgpack(result) == v); + CHECK(json::from_msgpack(result, true, false) == j); + } + + SECTION("128.128") + { + double v = 128.1280059814453125; + json j = v; + std::vector expected = + { + 0xca, 0x43, 0x00, 0x20, 0xc5 + }; + const auto result = json::to_msgpack(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_msgpack(result) == j); + CHECK(json::from_msgpack(result) == v); + CHECK(json::from_msgpack(result, true, false) == j); + } } } From cd115cbc33d074e231d3a703c8120b72223cb1e4 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 18 Jun 2020 12:50:32 +0200 Subject: [PATCH 5/7] :white_check_mark: update test suite --- cmake/download_test_data.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/download_test_data.cmake b/cmake/download_test_data.cmake index fe95235f..b023d8b4 100644 --- a/cmake/download_test_data.cmake +++ b/cmake/download_test_data.cmake @@ -1,7 +1,7 @@ find_package(Git) set(JSON_TEST_DATA_URL https://github.com/nlohmann/json_test_data) -set(JSON_TEST_DATA_VERSION 2.0.0) +set(JSON_TEST_DATA_VERSION 3.0.0) # target to download test data add_custom_target(download_test_data From b64002bbca497e79f56466ced8f99f90ca9cf347 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 19 Jun 2020 13:24:08 +0200 Subject: [PATCH 6/7] :recycle: extract common code to function --- .../nlohmann/detail/output/binary_writer.hpp | 47 +++++++++---------- single_include/nlohmann/json.hpp | 47 +++++++++---------- 2 files changed, 46 insertions(+), 48 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index f3221191..7289da3d 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -28,6 +28,7 @@ class binary_writer { using string_t = typename BasicJsonType::string_t; using binary_t = typename BasicJsonType::binary_t; + using number_float_t = typename BasicJsonType::number_float_t; public: /*! @@ -194,18 +195,7 @@ class binary_writer } else { - if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and - static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and - static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) - { - oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); - write_number(static_cast(j.m_value.number_float)); - } - else - { - oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - } + write_compact_float(j.m_value.number_float, detail::input_format_t::cbor); } break; } @@ -504,18 +494,7 @@ class binary_writer case value_t::number_float: { - if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and - static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and - static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) - { - oa->write_character(get_msgpack_float_prefix(static_cast(j.m_value.number_float))); - write_number(static_cast(j.m_value.number_float)); - } - else - { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - } + write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack); break; } @@ -1528,6 +1507,26 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + void write_compact_float(const number_float_t n, detail::input_format_t format) + { + if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) and + static_cast(n) <= static_cast((std::numeric_limits::max)()) and + static_cast(static_cast(n)) == static_cast(n)) + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); + write_number(static_cast(n)); + } + else + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); + write_number(n); + } + } + public: // The following to_char_type functions are implement the conversion // between uint8_t and CharType. In case CharType is not unsigned, diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 85fffc7c..33d0c357 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12238,6 +12238,7 @@ class binary_writer { using string_t = typename BasicJsonType::string_t; using binary_t = typename BasicJsonType::binary_t; + using number_float_t = typename BasicJsonType::number_float_t; public: /*! @@ -12404,18 +12405,7 @@ class binary_writer } else { - if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and - static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and - static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) - { - oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); - write_number(static_cast(j.m_value.number_float)); - } - else - { - oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - } + write_compact_float(j.m_value.number_float, detail::input_format_t::cbor); } break; } @@ -12714,18 +12704,7 @@ class binary_writer case value_t::number_float: { - if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and - static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and - static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) - { - oa->write_character(get_msgpack_float_prefix(static_cast(j.m_value.number_float))); - write_number(static_cast(j.m_value.number_float)); - } - else - { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - } + write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack); break; } @@ -13738,6 +13717,26 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + void write_compact_float(const number_float_t n, detail::input_format_t format) + { + if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) and + static_cast(n) <= static_cast((std::numeric_limits::max)()) and + static_cast(static_cast(n)) == static_cast(n)) + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); + write_number(static_cast(n)); + } + else + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); + write_number(n); + } + } + public: // The following to_char_type functions are implement the conversion // between uint8_t and CharType. In case CharType is not unsigned, From 24992003d99fefdd9fb5a605f0eb58a05a91985b Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sat, 20 Jun 2020 09:55:11 +0200 Subject: [PATCH 7/7] :memo: add notes from #2189 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d7492da3..42d057a3 100644 --- a/README.md +++ b/README.md @@ -1557,4 +1557,6 @@ $ cmake --build . $ ctest --output-on-failure ``` -For more information, have a look at the file [.travis.yml](https://github.com/nlohmann/json/blob/master/.travis.yml). +Note that during the `ctest` stage, several JSON test files are downloaded from an [external repository](https://github.com/nlohmann/json_test_data). If policies forbid downloading artifacts during testing, you can download the files yourself and pass the directory with the test files via `-DJSON_TestDataDirectory=path` to CMake. Then, no Internet connectivity is required. See [issue #2189](https://github.com/nlohmann/json/issues/2189) for more information. + +In case you have downloaded the library rather than checked out the code via Git, test `cmake_fetch_content_configure`. Please execute `ctest -LE git_required` to skip these tests. See [issue #2189](https://github.com/nlohmann/json/issues/2189) for more information.