Merge branch 'develop' into feature/fuzzer

doc/Makefile

@@ -57,6 +57,7 @@ doxygen: create_output create_links
 	$(SED) -i 's@< ObjectType, ArrayType, StringType, BooleanType, NumberIntegerType, NumberFloatType, AllocatorType >@@g' html/*.html
 	$(SED) -i 's@< ObjectType, ArrayType, StringType, BooleanType, NumberIntegerType, NumberUnsignedType, NumberFloatType, AllocatorType >@@g' html/*.html
 	$(SED) -i 's@< ObjectType, ArrayType, StringType, BooleanType, NumberIntegerType, NumberUnsignedType, NumberFloatType, AllocatorType >@@g' html/*.html
+	$(SED) -i 's@&lt;&#160;ObjectType,&#160;ArrayType,&#160;StringType,&#160;BooleanType,&#160;NumberIntegerType,&#160;NumberUnsignedType,&#160;NumberFloatType,&#160;AllocatorType&#160;&gt;@@g' html/*.html
 
 upload: clean doxygen check_output
 	cd html ; ../scripts/git-update-ghpages nlohmann/json

src/json.hpp

@@ -3793,7 +3793,7 @@ class basic_json
     container `c`, the expression `c.front()` is equivalent to `*c.begin()`.
 
     @return In case of a structured type (array or object), a reference to the
-    first element is returned. In cast of number, string, or boolean values, a
+    first element is returned. In case of number, string, or boolean values, a
     reference to the value is returned.
 
     @complexity Constant.
@@ -3836,7 +3836,7 @@ class basic_json
     @endcode
 
     @return In case of a structured type (array or object), a reference to the
-    last element is returned. In cast of number, string, or boolean values, a
+    last element is returned. In case of number, string, or boolean values, a
     reference to the value is returned.
 
     @complexity Constant.
@@ -4187,10 +4187,14 @@ class basic_json
     element is not found or the JSON value is not an object, end() is
     returned.
 
+    @note This method always returns @ref end() when executed on a JSON type
+          that is not an object.
+
     @param[in] key key value of the element to search for
 
     @return Iterator to an element with key equivalent to @a key. If no such
-    element is found, past-the-end (see end()) iterator is returned.
+    element is found or the JSON value is not an object, past-the-end (see
+    @ref end()) iterator is returned.
 
     @complexity Logarithmic in the size of the JSON object.
 
@@ -4233,6 +4237,9 @@ class basic_json
     default `std::map` type, the return value will always be `0` (@a key was
     not found) or `1` (@a key was found).
 
+    @note This method always returns `0` when executed on a JSON type that is
+          not an object.
+
     @param[in] key key value of the element to count
 
     @return Number of elements with key @a key. If the JSON value is not an
@@ -4792,9 +4799,6 @@ class basic_json
     object      | `{}`
     array       | `[]`
 
-    @note Floating-point numbers are set to `0.0` which will be serialized to
-    `0`. The vale type remains @ref number_float_t.
-
     @complexity Linear in the size of the JSON value.
 
     @liveexample{The example below shows the effect of `clear()` to different
@@ -6831,6 +6835,44 @@ class basic_json
         }
     }
 
+
+    /*
+    @brief checks if given lengths do not exceed the size of a given vector
+
+    To secure the access to the byte vector during CBOR/MessagePack
+    deserialization, bytes are copied from the vector into buffers. This
+    function checks if the number of bytes to copy (@a len) does not exceed the
+    size @s size of the vector. Additionally, an @a offset is given from where
+    to start reading the bytes.
+
+    This function checks whether reading the bytes is safe; that is, offset is a
+    valid index in the vector, offset+len
+
+    @param[in] size    size of the byte vector
+    @param[in] len     number of bytes to read
+    @param[in] offset  offset where to start reading
+
+    vec:  x x x x x X X X X X
+          ^         ^         ^
+          0         offset    len
+
+    @throws out_of_range if `len > v.size()`
+    */
+    static void check_length(const size_t size, const size_t len, const size_t offset)
+    {
+        // simple case: requested length is greater than the vector's length
+        if (len > size or offset > size)
+        {
+            throw std::out_of_range("len out of range");
+        }
+
+        // second case: adding offset would result in overflow
+        if ((size > (std::numeric_limits<size_t>::max() - offset)))
+        {
+            throw std::out_of_range("len+offset out of range");
+        }
+    }
+
     /*!
     @brief checks if a given length does not exceed the size of a given vector
 
@@ -6903,7 +6945,7 @@ class basic_json
                 const size_t len = v[current_idx] & 0x1f;
                 const size_t offset = current_idx + 1;
                 idx += len; // skip content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
         }
@@ -7007,7 +7049,7 @@ class basic_json
                     const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
                     const size_t offset = current_idx + 2;
                     idx += len + 1; // skip size byte + content bytes
-                    check_length(v, len + offset);
+                    check_length(v.size(), len, offset);
                     return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
                 }
 
@@ -7016,7 +7058,7 @@ class basic_json
                     const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
                     const size_t offset = current_idx + 3;
                     idx += len + 2; // skip 2 size bytes + content bytes
-                    check_length(v, len + offset);
+                    check_length(v.size(), len, offset);
                     return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
                 }
 
@@ -7025,7 +7067,7 @@ class basic_json
                     const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
                     const size_t offset = current_idx + 5;
                     idx += len + 4; // skip 4 size bytes + content bytes
-                    check_length(v, len + offset);
+                    check_length(v.size(), len, offset);
                     return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
                 }
 
@@ -7244,7 +7286,7 @@ class basic_json
                 const auto len = static_cast<size_t>(v[current_idx] - 0x60);
                 const size_t offset = current_idx + 1;
                 idx += len; // skip content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7253,7 +7295,7 @@ class basic_json
                 const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
                 const size_t offset = current_idx + 2;
                 idx += len + 1; // skip size byte + content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7262,7 +7304,7 @@ class basic_json
                 const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
                 const size_t offset = current_idx + 3;
                 idx += len + 2; // skip 2 size bytes + content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7271,7 +7313,7 @@ class basic_json
                 const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
                 const size_t offset = current_idx + 5;
                 idx += len + 4; // skip 4 size bytes + content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7280,7 +7322,7 @@ class basic_json
                 const auto len = static_cast<size_t>(get_from_vector<uint64_t>(v, current_idx));
                 const size_t offset = current_idx + 9;
                 idx += len + 8; // skip 8 size bytes + content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7621,6 +7663,11 @@ class basic_json
     */
     static basic_json from_msgpack(const std::vector<uint8_t>& v)
     {
+        if (v.empty())
+        {
+            throw std::invalid_argument("empty vector");
+        }
+
         size_t i = 0;
         return from_msgpack_internal(v, i);
     }
@@ -7678,6 +7725,11 @@ class basic_json
     */
     static basic_json from_cbor(const std::vector<uint8_t>& v)
     {
+        if (v.empty())
+        {
+            throw std::invalid_argument("empty vector");
+        }
+
         size_t i = 0;
         return from_cbor_internal(v, i);
     }

src/json.hpp.re2c

@@ -3793,7 +3793,7 @@ class basic_json
     container `c`, the expression `c.front()` is equivalent to `*c.begin()`.
 
     @return In case of a structured type (array or object), a reference to the
-    first element is returned. In cast of number, string, or boolean values, a
+    first element is returned. In case of number, string, or boolean values, a
     reference to the value is returned.
 
     @complexity Constant.
@@ -3836,7 +3836,7 @@ class basic_json
     @endcode
 
     @return In case of a structured type (array or object), a reference to the
-    last element is returned. In cast of number, string, or boolean values, a
+    last element is returned. In case of number, string, or boolean values, a
     reference to the value is returned.
 
     @complexity Constant.
@@ -4187,10 +4187,14 @@ class basic_json
     element is not found or the JSON value is not an object, end() is
     returned.
 
+    @note This method always returns @ref end() when executed on a JSON type
+          that is not an object.
+
     @param[in] key key value of the element to search for
 
     @return Iterator to an element with key equivalent to @a key. If no such
-    element is found, past-the-end (see end()) iterator is returned.
+    element is found or the JSON value is not an object, past-the-end (see
+    @ref end()) iterator is returned.
 
     @complexity Logarithmic in the size of the JSON object.
 
@@ -4233,6 +4237,9 @@ class basic_json
     default `std::map` type, the return value will always be `0` (@a key was
     not found) or `1` (@a key was found).
 
+    @note This method always returns `0` when executed on a JSON type that is
+          not an object.
+
     @param[in] key key value of the element to count
 
     @return Number of elements with key @a key. If the JSON value is not an
@@ -4792,9 +4799,6 @@ class basic_json
     object      | `{}`
     array       | `[]`
 
-    @note Floating-point numbers are set to `0.0` which will be serialized to
-    `0`. The vale type remains @ref number_float_t.
-
     @complexity Linear in the size of the JSON value.
 
     @liveexample{The example below shows the effect of `clear()` to different
@@ -6831,6 +6835,44 @@ class basic_json
         }
     }
 
+
+    /*
+    @brief checks if given lengths do not exceed the size of a given vector
+
+    To secure the access to the byte vector during CBOR/MessagePack
+    deserialization, bytes are copied from the vector into buffers. This
+    function checks if the number of bytes to copy (@a len) does not exceed the
+    size @s size of the vector. Additionally, an @a offset is given from where
+    to start reading the bytes.
+
+    This function checks whether reading the bytes is safe; that is, offset is a
+    valid index in the vector, offset+len
+
+    @param[in] size    size of the byte vector
+    @param[in] len     number of bytes to read
+    @param[in] offset  offset where to start reading
+
+    vec:  x x x x x X X X X X
+          ^         ^         ^
+          0         offset    len
+
+    @throws out_of_range if `len > v.size()`
+    */
+    static void check_length(const size_t size, const size_t len, const size_t offset)
+    {
+        // simple case: requested length is greater than the vector's length
+        if (len > size or offset > size)
+        {
+            throw std::out_of_range("len out of range");
+        }
+
+        // second case: adding offset would result in overflow
+        if ((size > (std::numeric_limits<size_t>::max() - offset)))
+        {
+            throw std::out_of_range("len+offset out of range");
+        }
+    }
+
     /*!
     @brief checks if a given length does not exceed the size of a given vector
 
@@ -6903,7 +6945,7 @@ class basic_json
                 const size_t len = v[current_idx] & 0x1f;
                 const size_t offset = current_idx + 1;
                 idx += len; // skip content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
         }
@@ -7007,7 +7049,7 @@ class basic_json
                     const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
                     const size_t offset = current_idx + 2;
                     idx += len + 1; // skip size byte + content bytes
-                    check_length(v, len + offset);
+                    check_length(v.size(), len, offset);
                     return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
                 }
 
@@ -7016,7 +7058,7 @@ class basic_json
                     const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
                     const size_t offset = current_idx + 3;
                     idx += len + 2; // skip 2 size bytes + content bytes
-                    check_length(v, len + offset);
+                    check_length(v.size(), len, offset);
                     return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
                 }
 
@@ -7025,7 +7067,7 @@ class basic_json
                     const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
                     const size_t offset = current_idx + 5;
                     idx += len + 4; // skip 4 size bytes + content bytes
-                    check_length(v, len + offset);
+                    check_length(v.size(), len, offset);
                     return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
                 }
 
@@ -7244,7 +7286,7 @@ class basic_json
                 const auto len = static_cast<size_t>(v[current_idx] - 0x60);
                 const size_t offset = current_idx + 1;
                 idx += len; // skip content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7253,7 +7295,7 @@ class basic_json
                 const auto len = static_cast<size_t>(get_from_vector<uint8_t>(v, current_idx));
                 const size_t offset = current_idx + 2;
                 idx += len + 1; // skip size byte + content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7262,7 +7304,7 @@ class basic_json
                 const auto len = static_cast<size_t>(get_from_vector<uint16_t>(v, current_idx));
                 const size_t offset = current_idx + 3;
                 idx += len + 2; // skip 2 size bytes + content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7271,7 +7313,7 @@ class basic_json
                 const auto len = static_cast<size_t>(get_from_vector<uint32_t>(v, current_idx));
                 const size_t offset = current_idx + 5;
                 idx += len + 4; // skip 4 size bytes + content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7280,7 +7322,7 @@ class basic_json
                 const auto len = static_cast<size_t>(get_from_vector<uint64_t>(v, current_idx));
                 const size_t offset = current_idx + 9;
                 idx += len + 8; // skip 8 size bytes + content bytes
-                check_length(v, len + offset);
+                check_length(v.size(), len, offset);
                 return std::string(reinterpret_cast<const char*>(v.data()) + offset, len);
             }
 
@@ -7621,6 +7663,11 @@ class basic_json
     */
     static basic_json from_msgpack(const std::vector<uint8_t>& v)
     {
+        if (v.empty())
+        {
+            throw std::invalid_argument("empty vector");
+        }
+
         size_t i = 0;
         return from_msgpack_internal(v, i);
     }
@@ -7678,6 +7725,11 @@ class basic_json
     */
     static basic_json from_cbor(const std::vector<uint8_t>& v)
     {
+        if (v.empty())
+        {
+            throw std::invalid_argument("empty vector");
+        }
+
         size_t i = 0;
         return from_cbor_internal(v, i);
     }

test/data/cbor_regression/id:000000,sig:06,src:000223+000677,op:splice,rep:2

@@ -0,0 +1 @@
+亄<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

test/data/cbor_regression/id:000001,sig:06,src:001310+001138,op:splice,rep:32

@@ -0,0 +1 @@
+”{˙˙˙˙˙˙˙˙˙’˙˙˙˙˙˙˙˙úúúúúúúúúúúúetú

test/src/unit-cbor.cpp

@@ -1186,6 +1186,84 @@ TEST_CASE("single CBOR roundtrip")
     }
 }
 
+TEST_CASE("CBOR regressions")
+{
+    SECTION("fuzz test results")
+    {
+        /*
+        The following test cases were found during a two-day session with
+        AFL-Fuzz. As a result, empty byte vectors and excessive lengths are
+        detected.
+        */
+        for (std::string filename :
+                {
+                    "test/data/cbor_regression/id:000000,sig:06,src:000223+000677,op:splice,rep:2",
+                    "test/data/cbor_regression/id:000000,sig:06,src:000787,op:havoc,rep:8",
+                    "test/data/cbor_regression/id:000000,sig:06,src:000833,op:havoc,rep:32",
+                    "test/data/cbor_regression/id:000000,sig:06,src:000838,op:havoc,rep:64",
+                    "test/data/cbor_regression/id:000000,sig:06,src:000846+001064,op:splice,rep:16",
+                    "test/data/cbor_regression/id:000000,sig:06,src:000848,op:flip1,pos:0",
+                    "test/data/cbor_regression/id:000000,sig:06,src:001435,op:havoc,rep:32",
+                    "test/data/cbor_regression/id:000000,sig:06,src:001436,op:havoc,rep:4v",
+                    "test/data/cbor_regression/id:000001,sig:06,src:000864+000903,op:splice,rep:2",
+                    "test/data/cbor_regression/id:000001,sig:06,src:001310+001138,op:splice,rep:32",
+                    "test/data/cbor_regression/id:000001,sig:06,src:001330+000569,op:splice,rep:64",
+                    "test/data/cbor_regression/id:000001,sig:06,src:001413,op:havoc,rep:32",
+                    "test/data/cbor_regression/id:000001,sig:06,src:001447,op:havoc,rep:4",
+                    "test/data/cbor_regression/id:000001,sig:06,src:001465+000325,op:splice,rep:4",
+                    "test/data/cbor_regression/id:000002,sig:06,src:000539,op:havoc,rep:8",
+                    "test/data/cbor_regression/id:000002,sig:06,src:001301,op:havoc,rep:16",
+                    "test/data/cbor_regression/id:000002,sig:06,src:001317+000850,op:splice,rep:8",
+                    "test/data/cbor_regression/id:000002,sig:06,src:001382,op:havoc,rep:128",
+                    "test/data/cbor_regression/id:000002,sig:06,src:001413+001036,op:splice,rep:4",
+                    "test/data/cbor_regression/id:000003,sig:06,src:000846+000155,op:splice,rep:16",
+                    "test/data/cbor_regression/id:000004,sig:06,src:001445,op:havoc,rep:128"
+                })
+        {
+            CAPTURE(filename);
+
+            try
+            {
+                // parse CBOR file
+                std::ifstream f_cbor(filename, std::ios::binary);
+                std::vector<uint8_t> vec1(
+                    (std::istreambuf_iterator<char>(f_cbor)),
+                    std::istreambuf_iterator<char>());
+                json j1 = json::from_cbor(vec1);
+
+                try
+                {
+                    // step 2: round trip
+                    std::vector<uint8_t> vec2 = json::to_cbor(j1);
+
+                    // parse serialization
+                    json j2 = json::from_cbor(vec2);
+
+                    // deserializations must match
+                    CHECK(j1 == j2);
+                }
+                catch (const std::invalid_argument&)
+                {
+                    // parsing a CBOR serialization must not fail
+                    CHECK(false);
+                }
+            }
+            catch (const std::invalid_argument&)
+            {
+                // parse errors are ok, because input may be random bytes
+            }
+            catch (const std::out_of_range&)
+            {
+                // parse errors are ok, because input may be random bytes
+            }
+            catch (const std::domain_error&)
+            {
+                // parse errors are ok, because input may be random bytes
+            }
+        }
+    }
+}
+
 TEST_CASE("CBOR roundtrips", "[hide]")
 {
     SECTION("input from flynn")