j3d1/json
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

json_pointer::array_index: Use unsigned values for the array index when parsing

Browse source 
The current code uses std::stoi to convert the input string to an int
array_index. This limits the maximum addressable array size to ~2GB on
most platforms.

But all callers immediately convert the result of array_index to
BasicJsonType::size_type.

So let's parse it as unsigned long long, which allows us to have as
big arrays as available memory. And also makes the call sites nicer to
read.

One complication arises on platforms where size_type is smaller than
unsigned long long. We need to bail out on these if the parsed array
index does not fit into size_type.
This commit is contained in:
Thomas Braun 2020-06-20 15:36:28 +02:00
parent f0e73163f2
commit ecbb2756fd
4 changed files with 69 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
include/nlohmann/detail/json_pointer.hpp
View file

@ -3,6 +3,7 @@
#include <algorithm> // all_of #include <algorithm> // all_of
#include <cassert> // assert #include <cassert> // assert
#include <cctype> // isdigit #include <cctype> // isdigit
#include <limits> // max
#include <numeric> // accumulate #include <numeric> // accumulate
#include <string> // string #include <string> // string
#include <utility> // move #include <utility> // move
@ -328,9 +329,12 @@ class json_pointer
@throw parse_error.106 if an array index begins with '0' @throw parse_error.106 if an array index begins with '0'
@throw parse_error.109 if an array index begins not with a digit @throw parse_error.109 if an array index begins not with a digit
@throw out_of_range.404 if string @a s could not be converted to an integer @throw out_of_range.404 if string @a s could not be converted to an integer
@throw out_of_range.410 if an array index exceeds size_type
*/ */
static int array_index(const std::string& s) static typename BasicJsonType::size_type array_index(const std::string& s)
{ {
using size_type = typename BasicJsonType::size_type;
// error condition (cf. RFC 6901, Sect. 4) // error condition (cf. RFC 6901, Sect. 4)
if (JSON_HEDLEY_UNLIKELY(s.size() > 1 and s[0] == '0')) if (JSON_HEDLEY_UNLIKELY(s.size() > 1 and s[0] == '0'))
{ {
@ -346,10 +350,10 @@ class json_pointer
} }
std::size_t processed_chars = 0; std::size_t processed_chars = 0;
int res = 0; unsigned long long res = 0;
JSON_TRY JSON_TRY
{ {
res = std::stoi(s, &processed_chars); res = std::stoull(s, &processed_chars);
} }
JSON_CATCH(std::out_of_range&) JSON_CATCH(std::out_of_range&)
{ {
@ -362,7 +366,14 @@ class json_pointer
JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'")); JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'"));
} }
return res; // only triggered on special platforms (like 32bit), see also
// https://github.com/nlohmann/json/pull/2203
if (res >= static_cast<unsigned long long>((std::numeric_limits<size_type>::max)()))
{
JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type")); // LCOV_EXCL_LINE
}
return static_cast<size_type>(res);
} }
json_pointer top() const json_pointer top() const
@ -421,7 +432,7 @@ class json_pointer
case detail::value_t::array: case detail::value_t::array:
{ {
// create an entry in the array // create an entry in the array
result = &result->operator[](static_cast<size_type>(array_index(reference_token))); result = &result->operator[](array_index(reference_token));
break; break;
} }
@ -499,8 +510,7 @@ class json_pointer
else else
{ {
// convert array index to number; unchecked access // convert array index to number; unchecked access
ptr = &ptr->operator[]( ptr = &ptr->operator[](array_index(reference_token));
static_cast<size_type>(array_index(reference_token)));
} }
break; break;
} }
@ -544,7 +554,7 @@ class json_pointer
} }
// note: at performs range check // note: at performs range check
ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); ptr = &ptr->at(array_index(reference_token));
break; break;
} }
@ -594,8 +604,7 @@ class json_pointer
} }
// use unchecked array access // use unchecked array access
ptr = &ptr->operator[]( ptr = &ptr->operator[](array_index(reference_token));
static_cast<size_type>(array_index(reference_token)));
break; break;
} }
@ -638,7 +647,7 @@ class json_pointer
} }
// note: at performs range check // note: at performs range check
ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); ptr = &ptr->at(array_index(reference_token));
break; break;
} }
@ -702,7 +711,7 @@ class json_pointer
} }
} }
const auto idx = static_cast<size_type>(array_index(reference_token)); const auto idx = array_index(reference_token);
if (idx >= ptr->size()) if (idx >= ptr->size())
{ {
// index out of range // index out of range

4
include/nlohmann/json.hpp
View file

@ -8179,7 +8179,7 @@ class basic_json
else else
{ {
const auto idx = json_pointer::array_index(last_path); const auto idx = json_pointer::array_index(last_path);
if (JSON_HEDLEY_UNLIKELY(static_cast<size_type>(idx) > parent.size())) if (JSON_HEDLEY_UNLIKELY(idx > parent.size()))
{ {
// avoid undefined behavior // avoid undefined behavior
JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
@ -8222,7 +8222,7 @@ class basic_json
else if (parent.is_array()) else if (parent.is_array())
{ {
// note erase performs range check // note erase performs range check
parent.erase(static_cast<size_type>(json_pointer::array_index(last_path))); parent.erase(json_pointer::array_index(last_path));
} }
}; };

37
single_include/nlohmann/json.hpp
View file

@ -11032,6 +11032,7 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
#include <algorithm> // all_of #include <algorithm> // all_of
#include <cassert> // assert #include <cassert> // assert
#include <cctype> // isdigit #include <cctype> // isdigit
#include <limits> // max
#include <numeric> // accumulate #include <numeric> // accumulate
#include <string> // string #include <string> // string
#include <utility> // move #include <utility> // move
@ -11360,9 +11361,12 @@ class json_pointer
@throw parse_error.106 if an array index begins with '0' @throw parse_error.106 if an array index begins with '0'
@throw parse_error.109 if an array index begins not with a digit @throw parse_error.109 if an array index begins not with a digit
@throw out_of_range.404 if string @a s could not be converted to an integer @throw out_of_range.404 if string @a s could not be converted to an integer
@throw out_of_range.410 if an array index exceeds size_type
*/ */
static int array_index(const std::string& s) static typename BasicJsonType::size_type array_index(const std::string& s)
{ {
using size_type = typename BasicJsonType::size_type;
// error condition (cf. RFC 6901, Sect. 4) // error condition (cf. RFC 6901, Sect. 4)
if (JSON_HEDLEY_UNLIKELY(s.size() > 1 and s[0] == '0')) if (JSON_HEDLEY_UNLIKELY(s.size() > 1 and s[0] == '0'))
{ {
@ -11378,10 +11382,10 @@ class json_pointer
} }
std::size_t processed_chars = 0; std::size_t processed_chars = 0;
int res = 0; unsigned long long res = 0;
JSON_TRY JSON_TRY
{ {
res = std::stoi(s, &processed_chars); res = std::stoull(s, &processed_chars);
} }
JSON_CATCH(std::out_of_range&) JSON_CATCH(std::out_of_range&)
{ {
@ -11394,7 +11398,14 @@ class json_pointer
JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'")); JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'"));
} }
return res; // only triggered on special platforms (like 32bit), see also
// https://github.com/nlohmann/json/pull/2203
if (res >= static_cast<unsigned long long>((std::numeric_limits<size_type>::max)()))
{
JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type")); // LCOV_EXCL_LINE
}
return static_cast<size_type>(res);
} }
json_pointer top() const json_pointer top() const
@ -11453,7 +11464,7 @@ class json_pointer
case detail::value_t::array: case detail::value_t::array:
{ {
// create an entry in the array // create an entry in the array
result = &result->operator[](static_cast<size_type>(array_index(reference_token))); result = &result->operator[](array_index(reference_token));
break; break;
} }
@ -11531,8 +11542,7 @@ class json_pointer
else else
{ {
// convert array index to number; unchecked access // convert array index to number; unchecked access
ptr = &ptr->operator[]( ptr = &ptr->operator[](array_index(reference_token));
static_cast<size_type>(array_index(reference_token)));
} }
break; break;
} }
@ -11576,7 +11586,7 @@ class json_pointer
} }
// note: at performs range check // note: at performs range check
ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); ptr = &ptr->at(array_index(reference_token));
break; break;
} }
@ -11626,8 +11636,7 @@ class json_pointer
} }
// use unchecked array access // use unchecked array access
ptr = &ptr->operator[]( ptr = &ptr->operator[](array_index(reference_token));
static_cast<size_type>(array_index(reference_token)));
break; break;
} }
@ -11670,7 +11679,7 @@ class json_pointer
} }
// note: at performs range check // note: at performs range check
ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); ptr = &ptr->at(array_index(reference_token));
break; break;
} }
@ -11734,7 +11743,7 @@ class json_pointer
} }
} }
const auto idx = static_cast<size_type>(array_index(reference_token)); const auto idx = array_index(reference_token);
if (idx >= ptr->size()) if (idx >= ptr->size())
{ {
// index out of range // index out of range
@ -23971,7 +23980,7 @@ class basic_json
else else
{ {
const auto idx = json_pointer::array_index(last_path); const auto idx = json_pointer::array_index(last_path);
if (JSON_HEDLEY_UNLIKELY(static_cast<size_type>(idx) > parent.size())) if (JSON_HEDLEY_UNLIKELY(idx > parent.size()))
{ {
// avoid undefined behavior // avoid undefined behavior
JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
@ -24014,7 +24023,7 @@ class basic_json
else if (parent.is_array()) else if (parent.is_array())
{ {
// note erase performs range check // note erase performs range check
parent.erase(static_cast<size_type>(json_pointer::array_index(last_path))); parent.erase(json_pointer::array_index(last_path));
} }
}; };

29
test/src/unit-json_pointer.cpp
View file

@ -348,12 +348,29 @@ TEST_CASE("JSON pointers")
CHECK_THROWS_WITH(j_const["/1+1"_json_pointer] == 1, CHECK_THROWS_WITH(j_const["/1+1"_json_pointer] == 1,
"[json.exception.out_of_range.404] unresolved reference token '1+1'"); "[json.exception.out_of_range.404] unresolved reference token '1+1'");
CHECK_THROWS_AS(j["/111111111111111111111111"_json_pointer] = 1, json::out_of_range&); {
CHECK_THROWS_WITH(j["/111111111111111111111111"_json_pointer] = 1, auto too_large_index = std::to_string((std::numeric_limits<unsigned long long>::max)()) + "1";
"[json.exception.out_of_range.404] unresolved reference token '111111111111111111111111'"); json::json_pointer jp(std::string("/") + too_large_index);
CHECK_THROWS_AS(j_const["/111111111111111111111111"_json_pointer] == 1, json::out_of_range&); std::string throw_msg = std::string("[json.exception.out_of_range.404] unresolved reference token '") + too_large_index + "'";
CHECK_THROWS_WITH(j_const["/111111111111111111111111"_json_pointer] == 1,
"[json.exception.out_of_range.404] unresolved reference token '111111111111111111111111'"); CHECK_THROWS_AS(j[jp] = 1, json::out_of_range&);
CHECK_THROWS_WITH(j[jp] = 1, throw_msg.c_str());
CHECK_THROWS_AS(j_const[jp] == 1, json::out_of_range&);
CHECK_THROWS_WITH(j_const[jp] == 1, throw_msg.c_str());
}
if (sizeof(typename json::size_type) < sizeof(unsigned long long))
{
auto size_type_max_uul = static_cast<unsigned long long>((std::numeric_limits<json::size_type>::max)());
auto too_large_index = std::to_string(size_type_max_uul);
json::json_pointer jp(std::string("/") + too_large_index);
std::string throw_msg = std::string("[json.exception.out_of_range.410] array index ") + too_large_index + " exceeds size_type";
CHECK_THROWS_AS(j[jp] = 1, json::out_of_range&);
CHECK_THROWS_WITH(j[jp] = 1, throw_msg.c_str());
CHECK_THROWS_AS(j_const[jp] == 1, json::out_of_range&);
CHECK_THROWS_WITH(j_const[jp] == 1, throw_msg.c_str());
}
CHECK_THROWS_AS(j.at("/one"_json_pointer) = 1, json::parse_error&); CHECK_THROWS_AS(j.at("/one"_json_pointer) = 1, json::parse_error&);
CHECK_THROWS_WITH(j.at("/one"_json_pointer) = 1, CHECK_THROWS_WITH(j.at("/one"_json_pointer) = 1,