j3d1/json
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

🚧 some changes to the BSON code

Browse source 
- added fuzz testers
- added some reference files
- made an exception text more clear
This commit is contained in:
Niels Lohmann 2018-10-24 15:43:37 +02:00
parent bba159121f
commit e2c5913a50
No known key found for this signature in database
GPG key ID: 7F3CEA63AE251B69
13 changed files with 218 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Makefile
View file

@ -48,6 +48,7 @@ all:
@echo "cppcheck - analyze code with cppcheck"
@echo "doctest - compile example files and check their output"
@echo "fuzz_testing - prepare fuzz testing of the JSON parser"
@echo "fuzz_testing_bson - prepare fuzz testing of the BSON parser"
@echo "fuzz_testing_cbor - prepare fuzz testing of the CBOR parser"
@echo "fuzz_testing_msgpack - prepare fuzz testing of the MessagePack parser"
@echo "fuzz_testing_ubjson - prepare fuzz testing of the UBJSON parser"
@ -220,6 +221,14 @@ fuzz_testing:
find test/data/json_tests -size -5k -name *json | xargs -I{} cp "{}" fuzz-testing/testcases
@echo "Execute: afl-fuzz -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer"
fuzz_testing_bson:
rm -fr fuzz-testing
mkdir -p fuzz-testing fuzz-testing/testcases fuzz-testing/out
$(MAKE) parse_bson_fuzzer -C test CXX=afl-clang++
mv test/parse_bson_fuzzer fuzz-testing/fuzzer
find test/data -size -5k -name *.bson | xargs -I{} cp "{}" fuzz-testing/testcases
@echo "Execute: afl-fuzz -i fuzz-testing/testcases -o fuzz-testing/out fuzz-testing/fuzzer"
fuzz_testing_cbor:
rm -fr fuzz-testing
mkdir -p fuzz-testing fuzz-testing/testcases fuzz-testing/out

4
include/nlohmann/detail/meta/type_traits.hpp
View file

@ -193,13 +193,13 @@ struct is_constructible_object_type_impl <
static constexpr bool value =
std::is_constructible<typename ConstructibleObjectType::key_type,
typename object_t::key_type>::value and
std::is_same<typename object_t::mapped_type,
(std::is_same<typename object_t::mapped_type,
typename ConstructibleObjectType::mapped_type>::value or
(has_from_json<BasicJsonType,
typename ConstructibleObjectType::mapped_type>::value or
has_non_default_from_json <
BasicJsonType,
typename ConstructibleObjectType::mapped_type >::value);
typename ConstructibleObjectType::mapped_type >::value));
};
template <typename BasicJsonType, typename ConstructibleObjectType>

20
include/nlohmann/detail/output/binary_writer.hpp
View file

@ -976,15 +976,23 @@ class binary_writer
{
switch (j.type())
{
default:
JSON_THROW(type_error::create(317, "JSON value of type " + std::to_string(static_cast<std::uint8_t>(j.type())) + " cannot be serialized to requested format"));
break;
case value_t::discarded:
break;
case value_t::object:
{
write_bson_object(*j.m_value.object);
break;
}
case value_t::discarded:
{
break;
}
default:
{
JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name())));
break;
}
}
}
@ -1009,7 +1017,7 @@ class binary_writer
std::memcpy(vec.data(), &n, sizeof(NumberType));
// step 2: write array to output (with possible reordering)
if (is_little_endian && !OutputIsLittleEndian)
if (is_little_endian and not OutputIsLittleEndian)
{
// reverse byte order prior to conversion if necessary
std::reverse(vec.begin(), vec.end());

24
single_include/nlohmann/json.hpp
View file

@ -553,13 +553,13 @@ struct is_constructible_object_type_impl <
static constexpr bool value =
std::is_constructible<typename ConstructibleObjectType::key_type,
typename object_t::key_type>::value and
std::is_same<typename object_t::mapped_type,
(std::is_same<typename object_t::mapped_type,
typename ConstructibleObjectType::mapped_type>::value or
(has_from_json<BasicJsonType,
typename ConstructibleObjectType::mapped_type>::value or
has_non_default_from_json <
BasicJsonType,
typename ConstructibleObjectType::mapped_type >::value);
typename ConstructibleObjectType::mapped_type >::value));
};
template <typename BasicJsonType, typename ConstructibleObjectType>
@ -9234,15 +9234,23 @@ class binary_writer
{
switch (j.type())
{
default:
JSON_THROW(type_error::create(317, "JSON value of type " + std::to_string(static_cast<std::uint8_t>(j.type())) + " cannot be serialized to requested format"));
break;
case value_t::discarded:
break;
case value_t::object:
{
write_bson_object(*j.m_value.object);
break;
}
case value_t::discarded:
{
break;
}
default:
{
JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name())));
break;
}
}
}
@ -9267,7 +9275,7 @@ class binary_writer
std::memcpy(vec.data(), &n, sizeof(NumberType));
// step 2: write array to output (with possible reordering)
if (is_little_endian && !OutputIsLittleEndian)
if (is_little_endian and not OutputIsLittleEndian)
{
// reverse byte order prior to conversion if necessary
std::reverse(vec.begin(), vec.end());

6
test/Makefile
View file

@ -10,6 +10,7 @@ SOURCES = src/unit.cpp \
src/unit-algorithms.cpp \
src/unit-allocator.cpp \
src/unit-alt-string.cpp \
src/unit-bson.cpp \
src/unit-capacity.cpp \
src/unit-cbor.cpp \
src/unit-class_const_iterator.cpp \
@ -90,12 +91,15 @@ check: $(OBJECTS) $(TESTCASES)
##############################################################################
FUZZER_ENGINE = src/fuzzer-driver_afl.cpp
FUZZERS = parse_afl_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer
FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer
fuzzers: $(FUZZERS)
parse_afl_fuzzer:
$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_json.cpp -o $@
parse_bson_fuzzer:
$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_bson.cpp -o $@
parse_cbor_fuzzer:
$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_cbor.cpp -o $@

BIN
test/data/json.org/1.json.bson Normal file
View file

Binary file not shown.

BIN
test/data/json.org/2.json.bson Normal file
View file

Binary file not shown.

BIN
test/data/json.org/3.json.bson Normal file
View file

Binary file not shown.

BIN
test/data/json.org/4.json.bson Normal file
View file

Binary file not shown.

BIN
test/data/json.org/5.json.bson Normal file
View file

Binary file not shown.

BIN
test/data/json_tests/pass3.json.bson Normal file
View file

Binary file not shown.

68
test/src/fuzzer-parse_bson.cpp Normal file
View file

@ -0,0 +1,68 @@
/*
__ _____ _____ _____
__| | __| | | | JSON for Modern C++ (fuzz test support)
| | |__ | | | | | | version 3.3.0
|_____|_____|_____|_|___| https://github.com/nlohmann/json
This file implements a parser test suitable for fuzz testing. Given a byte
array data, it performs the following steps:
- j1 = from_bson(data)
- vec = to_bson(j1)
- j2 = from_bson(vec)
- assert(j1 == j2)
The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer
drivers.
Licensed under the MIT License <http://opensource.org/licenses/MIT>.
*/
#include <iostream>
#include <sstream>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
// see http://llvm.org/docs/LibFuzzer.html
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
{
try
{
// step 1: parse input
std::vector<uint8_t> vec1(data, data + size);
json j1 = json::from_bson(vec1);
try
{
// step 2: round trip
std::vector<uint8_t> vec2 = json::to_bson(j1);
// parse serialization
json j2 = json::from_bson(vec2);
// serializations must match
assert(json::to_bson(j2) == vec2);
}
catch (const json::parse_error&)
{
// parsing a CBOR serialization must not fail
assert(false);
}
}
catch (const json::parse_error&)
{
// parse errors are ok, because input may be random bytes
}
catch (const json::type_error&)
{
// type errors can occur during parsing, too
}
catch (const json::out_of_range&)
{
// out of range errors can occur during parsing, too
}
// return 0 - non-zero return values are reserved for future use
return 0;
}

103
test/src/unit-bson.cpp
View file

@ -49,7 +49,7 @@ TEST_CASE("BSON")
{
json j = nullptr;
REQUIRE_THROWS_AS(json::to_bson(j), json::type_error&);
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] JSON value of type 0 cannot be serialized to requested format");
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] to serialize to BSON, top-level type must be object, but is null");
}
SECTION("boolean")
@ -58,14 +58,14 @@ TEST_CASE("BSON")
{
json j = true;
REQUIRE_THROWS_AS(json::to_bson(j), json::type_error&);
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] JSON value of type 4 cannot be serialized to requested format");
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] to serialize to BSON, top-level type must be object, but is boolean");
}
SECTION("false")
{
json j = false;
REQUIRE_THROWS_AS(json::to_bson(j), json::type_error&);
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] JSON value of type 4 cannot be serialized to requested format");
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] to serialize to BSON, top-level type must be object, but is boolean");
}
}
@ -73,28 +73,28 @@ TEST_CASE("BSON")
{
json j = 42;
REQUIRE_THROWS_AS(json::to_bson(j), json::type_error&);
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] JSON value of type 5 cannot be serialized to requested format");
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] to serialize to BSON, top-level type must be object, but is number");
}
SECTION("float")
{
json j = 4.2;
REQUIRE_THROWS_AS(json::to_bson(j), json::type_error&);
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] JSON value of type 7 cannot be serialized to requested format");
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] to serialize to BSON, top-level type must be object, but is number");
}
SECTION("string")
{
json j = "not supported";
REQUIRE_THROWS_AS(json::to_bson(j), json::type_error&);
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] JSON value of type 3 cannot be serialized to requested format");
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] to serialize to BSON, top-level type must be object, but is string");
}
SECTION("array")
{
json j = std::vector<int> {1, 2, 3, 4, 5, 6, 7};
REQUIRE_THROWS_AS(json::to_bson(j), json::type_error&);
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] JSON value of type 2 cannot be serialized to requested format");
CHECK_THROWS_WITH(json::to_bson(j), "[json.exception.type_error.317] to serialize to BSON, top-level type must be object, but is array");
}
}
@ -1126,3 +1126,92 @@ TEST_CASE("BSON numerical data")
}
}
}
TEST_CASE("BSON roundtrips", "[hide]")
{
SECTION("reference files")
{
for (std::string filename :
{
"test/data/json.org/1.json",
"test/data/json.org/2.json",
"test/data/json.org/3.json",
"test/data/json.org/4.json",
"test/data/json.org/5.json"
})
{
CAPTURE(filename);
SECTION(filename + ": std::vector<uint8_t>")
{
// parse JSON file
std::ifstream f_json(filename);
json j1 = json::parse(f_json);
// parse BSON file
std::ifstream f_bson(filename + ".bson", std::ios::binary);
std::vector<uint8_t> packed(
(std::istreambuf_iterator<char>(f_bson)),
std::istreambuf_iterator<char>());
json j2;
CHECK_NOTHROW(j2 = json::from_bson(packed));
// compare parsed JSON values
CHECK(j1 == j2);
}
SECTION(filename + ": std::ifstream")
{
// parse JSON file
std::ifstream f_json(filename);
json j1 = json::parse(f_json);
// parse BSON file
std::ifstream f_bson(filename + ".bson", std::ios::binary);
json j2;
CHECK_NOTHROW(j2 = json::from_bson(f_bson));
// compare parsed JSON values
CHECK(j1 == j2);
}
SECTION(filename + ": uint8_t* and size")
{
// parse JSON file
std::ifstream f_json(filename);
json j1 = json::parse(f_json);
// parse BSON file
std::ifstream f_bson(filename + ".bson", std::ios::binary);
std::vector<uint8_t> packed(
(std::istreambuf_iterator<char>(f_bson)),
std::istreambuf_iterator<char>());
json j2;
CHECK_NOTHROW(j2 = json::from_bson({packed.data(), packed.size()}));
// compare parsed JSON values
CHECK(j1 == j2);
}
SECTION(filename + ": output to output adapters")
{
// parse JSON file
std::ifstream f_json(filename);
json j1 = json::parse(f_json);
// parse BSON file
std::ifstream f_bson(filename + ".bson", std::ios::binary);
std::vector<uint8_t> packed(
(std::istreambuf_iterator<char>(f_bson)),
std::istreambuf_iterator<char>());
SECTION(filename + ": output adapters: std::vector<uint8_t>")
{
std::vector<uint8_t> vec;
json::to_bson(j1, vec);
CHECK(vec == packed);
}
}
}
}
}