Merge branch 'develop' into feature/fuzzer

This commit is contained in:
Niels Lohmann 2016-12-28 12:11:51 +01:00
commit 55be721615
25 changed files with 215 additions and 30 deletions

View file

@ -57,6 +57,7 @@ doxygen: create_output create_links
$(SED) -i 's@< ObjectType, ArrayType, StringType, BooleanType, NumberIntegerType, NumberFloatType, AllocatorType >@@g' html/*.html
$(SED) -i 's@< ObjectType, ArrayType, StringType, BooleanType, NumberIntegerType, NumberUnsignedType, NumberFloatType, AllocatorType >@@g' html/*.html
$(SED) -i 's@< ObjectType, ArrayType, StringType, BooleanType, NumberIntegerType, NumberUnsignedType, NumberFloatType, AllocatorType >@@g' html/*.html
$(SED) -i 's@&lt;&#160;ObjectType,&#160;ArrayType,&#160;StringType,&#160;BooleanType,&#160;NumberIntegerType,&#160;NumberUnsignedType,&#160;NumberFloatType,&#160;AllocatorType&#160;&gt;@@g' html/*.html
upload: clean doxygen check_output
cd html ; ../scripts/git-update-ghpages nlohmann/json

View file

@ -3793,7 +3793,7 @@ class basic_json
container `c`, the expression `c.front()` is equivalent to `*c.begin()`.
@return In case of a structured type (array or object), a reference to the
first element is returned. In cast of number, string, or boolean values, a
first element is returned. In case of number, string, or boolean values, a
reference to the value is returned.
@complexity Constant.
@ -3836,7 +3836,7 @@ class basic_json
@endcode
@return In case of a structured type (array or object), a reference to the
last element is returned. In cast of number, string, or boolean values, a
last element is returned. In case of number, string, or boolean values, a
reference to the value is returned.
@complexity Constant.
@ -4187,10 +4187,14 @@ class basic_json
element is not found or the JSON value is not an object, end() is
returned.
@note This method always returns @ref end() when executed on a JSON type
that is not an object.
@param[in] key key value of the element to search for
@return Iterator to an element with key equivalent to @a key. If no such
element is found, past-the-end (see end()) iterator is returned.
element is found or the JSON value is not an object, past-the-end (see
@ref end()) iterator is returned.
@complexity Logarithmic in the size of the JSON object.
@ -4233,6 +4237,9 @@ class basic_json
default `std::map` type, the return value will always be `0` (@a key was
not found) or `1` (@a key was found).
@note This method always returns `0` when executed on a JSON type that is
not an object.
@param[in] key key value of the element to count
@return Number of elements with key @a key. If the JSON value is not an
@ -4792,9 +4799,6 @@ class basic_json
object | `{}`
array | `[]`
@note Floating-point numbers are set to `0.0` which will be serialized to
`0`. The vale type remains @ref number_float_t.
@complexity Linear in the size of the JSON value.
@liveexample{The example below shows the effect of `clear()` to different
@ -6831,6 +6835,44 @@ class basic_json
}
}
/*
@brief checks if given lengths do not exceed the size of a given vector
To secure the access to the byte vector during CBOR/MessagePack
deserialization, bytes are copied from the vector into buffers. This
function checks if the number of bytes to copy (@a len) does not exceed the
size @s size of the vector. Additionally, an @a offset is given from where
to start reading the bytes.
This function checks whether reading the bytes is safe; that is, offset is a
valid index in the vector, offset+len
@param[in] size size of the byte vector
@param[in] len number of bytes to read
@param[in] offset offset where to start reading
vec: x x x x x X X X X X
^ ^ ^
0 offset len
@throws out_of_range if `len > v.size()`
*/
static void check_length(const size_t size, const size_t len, const size_t offset)
{
// simple case: requested length is greater than the vector's length
if (len > size or offset > size)
{
throw std::out_of_range("len out of range");
}
// second case: adding offset would result in overflow
if ((size > (std::numeric_limits<size_t>::max() - offset)))
{
throw std::out_of_range("len+offset out of range");
}
}
/*!
@brief checks if a given length does not exceed the size of a given vector
@ -6903,7 +6945,7 @@ class basic_json
const size_t len = v[current_idx] & 0x1f;
const size_t offset = current_idx + 1;
idx += len; // skip content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
}
@ -7007,7 +7049,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
const size_t offset = current_idx + 2;
idx += len + 1; // skip size byte + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7016,7 +7058,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
const size_t offset = current_idx + 3;
idx += len + 2; // skip 2 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7025,7 +7067,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
const size_t offset = current_idx + 5;
idx += len + 4; // skip 4 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7244,7 +7286,7 @@ class basic_json
const auto len = static_cast<size_t>(v[current_idx] - 0x60);
const size_t offset = current_idx + 1;
idx += len; // skip content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7253,7 +7295,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
const size_t offset = current_idx + 2;
idx += len + 1; // skip size byte + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7262,7 +7304,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
const size_t offset = current_idx + 3;
idx += len + 2; // skip 2 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7271,7 +7313,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
const size_t offset = current_idx + 5;
idx += len + 4; // skip 4 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7280,7 +7322,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint64_t>(v, current_idx));
const size_t offset = current_idx + 9;
idx += len + 8; // skip 8 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7621,6 +7663,11 @@ class basic_json
*/
static basic_json from_msgpack(const std::vector<uint8_t>& v)
{
if (v.empty())
{
throw std::invalid_argument("empty vector");
}
size_t i = 0;
return from_msgpack_internal(v, i);
}
@ -7678,6 +7725,11 @@ class basic_json
*/
static basic_json from_cbor(const std::vector<uint8_t>& v)
{
if (v.empty())
{
throw std::invalid_argument("empty vector");
}
size_t i = 0;
return from_cbor_internal(v, i);
}

View file

@ -3793,7 +3793,7 @@ class basic_json
container `c`, the expression `c.front()` is equivalent to `*c.begin()`.
@return In case of a structured type (array or object), a reference to the
first element is returned. In cast of number, string, or boolean values, a
first element is returned. In case of number, string, or boolean values, a
reference to the value is returned.
@complexity Constant.
@ -3836,7 +3836,7 @@ class basic_json
@endcode
@return In case of a structured type (array or object), a reference to the
last element is returned. In cast of number, string, or boolean values, a
last element is returned. In case of number, string, or boolean values, a
reference to the value is returned.
@complexity Constant.
@ -4187,10 +4187,14 @@ class basic_json
element is not found or the JSON value is not an object, end() is
returned.
@note This method always returns @ref end() when executed on a JSON type
that is not an object.
@param[in] key key value of the element to search for
@return Iterator to an element with key equivalent to @a key. If no such
element is found, past-the-end (see end()) iterator is returned.
element is found or the JSON value is not an object, past-the-end (see
@ref end()) iterator is returned.
@complexity Logarithmic in the size of the JSON object.
@ -4233,6 +4237,9 @@ class basic_json
default `std::map` type, the return value will always be `0` (@a key was
not found) or `1` (@a key was found).
@note This method always returns `0` when executed on a JSON type that is
not an object.
@param[in] key key value of the element to count
@return Number of elements with key @a key. If the JSON value is not an
@ -4792,9 +4799,6 @@ class basic_json
object | `{}`
array | `[]`
@note Floating-point numbers are set to `0.0` which will be serialized to
`0`. The vale type remains @ref number_float_t.
@complexity Linear in the size of the JSON value.
@liveexample{The example below shows the effect of `clear()` to different
@ -6831,6 +6835,44 @@ class basic_json
}
}
/*
@brief checks if given lengths do not exceed the size of a given vector
To secure the access to the byte vector during CBOR/MessagePack
deserialization, bytes are copied from the vector into buffers. This
function checks if the number of bytes to copy (@a len) does not exceed the
size @s size of the vector. Additionally, an @a offset is given from where
to start reading the bytes.
This function checks whether reading the bytes is safe; that is, offset is a
valid index in the vector, offset+len
@param[in] size size of the byte vector
@param[in] len number of bytes to read
@param[in] offset offset where to start reading
vec: x x x x x X X X X X
^ ^ ^
0 offset len
@throws out_of_range if `len > v.size()`
*/
static void check_length(const size_t size, const size_t len, const size_t offset)
{
// simple case: requested length is greater than the vector's length
if (len > size or offset > size)
{
throw std::out_of_range("len out of range");
}
// second case: adding offset would result in overflow
if ((size > (std::numeric_limits<size_t>::max() - offset)))
{
throw std::out_of_range("len+offset out of range");
}
}
/*!
@brief checks if a given length does not exceed the size of a given vector
@ -6903,7 +6945,7 @@ class basic_json
const size_t len = v[current_idx] & 0x1f;
const size_t offset = current_idx + 1;
idx += len; // skip content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
}
@ -7007,7 +7049,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
const size_t offset = current_idx + 2;
idx += len + 1; // skip size byte + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7016,7 +7058,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
const size_t offset = current_idx + 3;
idx += len + 2; // skip 2 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7025,7 +7067,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
const size_t offset = current_idx + 5;
idx += len + 4; // skip 4 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7244,7 +7286,7 @@ class basic_json
const auto len = static_cast<size_t>(v[current_idx] - 0x60);
const size_t offset = current_idx + 1;
idx += len; // skip content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7253,7 +7295,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
const size_t offset = current_idx + 2;
idx += len + 1; // skip size byte + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7262,7 +7304,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
const size_t offset = current_idx + 3;
idx += len + 2; // skip 2 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7271,7 +7313,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
const size_t offset = current_idx + 5;
idx += len + 4; // skip 4 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7280,7 +7322,7 @@ class basic_json
const auto len = static_cast<size_t>(get_from_vector<uint64_t>(v, current_idx));
const size_t offset = current_idx + 9;
idx += len + 8; // skip 8 size bytes + content bytes
check_length(v, len + offset);
check_length(v.size(), len, offset);
return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
}
@ -7621,6 +7663,11 @@ class basic_json
*/
static basic_json from_msgpack(const std::vector<uint8_t>& v)
{
if (v.empty())
{
throw std::invalid_argument("empty vector");
}
size_t i = 0;
return from_msgpack_internal(v, i);
}
@ -7678,6 +7725,11 @@ class basic_json
*/
static basic_json from_cbor(const std::vector<uint8_t>& v)
{
if (v.empty())
{
throw std::invalid_argument("empty vector");
}
size_t i = 0;
return from_cbor_internal(v, i);
}

View file

@ -0,0 +1 @@
<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

View file

@ -0,0 +1 @@
”{˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙˙úúúúúúúúúúúúetú

View file

@ -1186,6 +1186,84 @@ TEST_CASE("single CBOR roundtrip")
}
}
TEST_CASE("CBOR regressions")
{
SECTION("fuzz test results")
{
/*
The following test cases were found during a two-day session with
AFL-Fuzz. As a result, empty byte vectors and excessive lengths are
detected.
*/
for (std::string filename :
{
"test/data/cbor_regression/id:000000,sig:06,src:000223+000677,op:splice,rep:2",
"test/data/cbor_regression/id:000000,sig:06,src:000787,op:havoc,rep:8",
"test/data/cbor_regression/id:000000,sig:06,src:000833,op:havoc,rep:32",
"test/data/cbor_regression/id:000000,sig:06,src:000838,op:havoc,rep:64",
"test/data/cbor_regression/id:000000,sig:06,src:000846+001064,op:splice,rep:16",
"test/data/cbor_regression/id:000000,sig:06,src:000848,op:flip1,pos:0",
"test/data/cbor_regression/id:000000,sig:06,src:001435,op:havoc,rep:32",
"test/data/cbor_regression/id:000000,sig:06,src:001436,op:havoc,rep:4v",
"test/data/cbor_regression/id:000001,sig:06,src:000864+000903,op:splice,rep:2",
"test/data/cbor_regression/id:000001,sig:06,src:001310+001138,op:splice,rep:32",
"test/data/cbor_regression/id:000001,sig:06,src:001330+000569,op:splice,rep:64",
"test/data/cbor_regression/id:000001,sig:06,src:001413,op:havoc,rep:32",
"test/data/cbor_regression/id:000001,sig:06,src:001447,op:havoc,rep:4",
"test/data/cbor_regression/id:000001,sig:06,src:001465+000325,op:splice,rep:4",
"test/data/cbor_regression/id:000002,sig:06,src:000539,op:havoc,rep:8",
"test/data/cbor_regression/id:000002,sig:06,src:001301,op:havoc,rep:16",
"test/data/cbor_regression/id:000002,sig:06,src:001317+000850,op:splice,rep:8",
"test/data/cbor_regression/id:000002,sig:06,src:001382,op:havoc,rep:128",
"test/data/cbor_regression/id:000002,sig:06,src:001413+001036,op:splice,rep:4",
"test/data/cbor_regression/id:000003,sig:06,src:000846+000155,op:splice,rep:16",
"test/data/cbor_regression/id:000004,sig:06,src:001445,op:havoc,rep:128"
})
{
CAPTURE(filename);
try
{
// parse CBOR file
std::ifstream f_cbor(filename, std::ios::binary);
std::vector<uint8_t> vec1(
(std::istreambuf_iterator<char>(f_cbor)),
std::istreambuf_iterator<char>());
json j1 = json::from_cbor(vec1);
try
{
// step 2: round trip
std::vector<uint8_t> vec2 = json::to_cbor(j1);
// parse serialization
json j2 = json::from_cbor(vec2);
// deserializations must match
CHECK(j1 == j2);
}
catch (const std::invalid_argument&)
{
// parsing a CBOR serialization must not fail
CHECK(false);
}
}
catch (const std::invalid_argument&)
{
// parse errors are ok, because input may be random bytes
}
catch (const std::out_of_range&)
{
// parse errors are ok, because input may be random bytes
}
catch (const std::domain_error&)
{
// parse errors are ok, because input may be random bytes
}
}
}
}
TEST_CASE("CBOR roundtrips", "[hide]")
{
SECTION("input from flynn")