Merge pull request #2125 from nlohmann/binary_type

Clean up implementation of binary type
This commit is contained in:
Niels Lohmann 2020-05-20 18:58:29 +02:00 committed by GitHub
commit a82c80e9af
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
75 changed files with 1425 additions and 780 deletions

View file

@ -228,9 +228,9 @@ template <typename BasicJsonType, typename ConstructibleArrayType,
is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value and
not is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value and
not is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value and
not std::is_same<ConstructibleArrayType, typename BasicJsonType::binary_t>::value and
not is_basic_json<ConstructibleArrayType>::value,
int > = 0 >
auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr)
-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}),
j.template get<typename ConstructibleArrayType::value_type>(),
@ -245,6 +245,17 @@ void())
from_json_array_impl(j, arr, priority_tag<3> {});
}
template <typename BasicJsonType>
void from_json(const BasicJsonType& j, typename BasicJsonType::binary_t& bin)
{
if (JSON_HEDLEY_UNLIKELY(not j.is_binary()))
{
JSON_THROW(type_error::create(302, "type must be binary, but is " + std::string(j.type_name())));
}
bin = *j.template get_ptr<const typename BasicJsonType::binary_t*>();
}
template<typename BasicJsonType, typename ConstructibleObjectType,
enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0>
void from_json(const BasicJsonType& j, ConstructibleObjectType& obj)

View file

@ -74,7 +74,7 @@ struct external_constructor<value_t::binary>
static void construct(BasicJsonType& j, const typename BasicJsonType::binary_t& b)
{
j.m_type = value_t::binary;
typename BasicJsonType::internal_binary_t value{b};
typename BasicJsonType::binary_t value{b};
j.m_value = value;
j.assert_invariant();
}
@ -83,7 +83,7 @@ struct external_constructor<value_t::binary>
static void construct(BasicJsonType& j, typename BasicJsonType::binary_t&& b)
{
j.m_type = value_t::binary;
typename BasicJsonType::internal_binary_t value{std::move(b)};
typename BasicJsonType::binary_t value{std::move(b)};
j.m_value = value;
j.assert_invariant();
}
@ -278,9 +278,9 @@ void to_json(BasicJsonType& j, const std::vector<bool>& e)
template <typename BasicJsonType, typename CompatibleArrayType,
enable_if_t<is_compatible_array_type<BasicJsonType,
CompatibleArrayType>::value and
not is_compatible_object_type<
BasicJsonType, CompatibleArrayType>::value and
not is_compatible_object_type<BasicJsonType, CompatibleArrayType>::value and
not is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value and
not std::is_same<typename BasicJsonType::binary_t, CompatibleArrayType>::value and
not is_basic_json<CompatibleArrayType>::value,
int> = 0>
void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
@ -288,6 +288,12 @@ void to_json(BasicJsonType& j, const CompatibleArrayType& arr)
external_constructor<value_t::array>::construct(j, arr);
}
template <typename BasicJsonType>
void to_json(BasicJsonType& j, const typename BasicJsonType::binary_t& bin)
{
external_constructor<value_t::binary>::construct(j, bin);
}
template<typename BasicJsonType, typename T,
enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0>
void to_json(BasicJsonType& j, const std::valarray<T>& arr)

View file

@ -52,7 +52,7 @@ class binary_reader
using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
using number_float_t = typename BasicJsonType::number_float_t;
using string_t = typename BasicJsonType::string_t;
using internal_binary_t = typename BasicJsonType::internal_binary_t;
using binary_t = typename BasicJsonType::binary_t;
using json_sax_t = SAX;
public:
@ -219,7 +219,7 @@ class binary_reader
@return `true` if the byte array was successfully parsed
*/
template<typename NumberType>
bool get_bson_binary(const NumberType len, internal_binary_t& result)
bool get_bson_binary(const NumberType len, binary_t& result)
{
if (JSON_HEDLEY_UNLIKELY(len < 0))
{
@ -227,8 +227,10 @@ class binary_reader
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "byte array length cannot be negative, is " + std::to_string(len), "binary")));
}
result.has_subtype = true; // All BSON binary values have a subtype
get_number<std::uint8_t>(input_format_t::bson, result.subtype);
// All BSON binary values have a subtype
std::uint8_t subtype;
get_number<std::uint8_t>(input_format_t::bson, subtype);
result.set_subtype(subtype);
return get_binary(input_format_t::bson, len, result);
}
@ -274,7 +276,7 @@ class binary_reader
case 0x05: // binary
{
std::int32_t len;
internal_binary_t value;
binary_t value;
return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_binary(len, value) and sax->binary(value);
}
@ -530,7 +532,7 @@ class binary_reader
case 0x5B: // Binary data (eight-byte uint64_t for n follow)
case 0x5F: // Binary data (indefinite length)
{
internal_binary_t b;
binary_t b;
return get_cbor_binary(b) and sax->binary(b);
}
@ -860,7 +862,7 @@ class binary_reader
@return whether byte array creation completed
*/
bool get_cbor_binary(internal_binary_t& result)
bool get_cbor_binary(binary_t& result)
{
if (JSON_HEDLEY_UNLIKELY(not unexpect_eof(input_format_t::cbor, "binary")))
{
@ -901,32 +903,36 @@ class binary_reader
case 0x58: // Binary data (one-byte uint8_t for n follows)
{
std::uint8_t len;
return get_number(input_format_t::cbor, len) and get_binary(input_format_t::cbor, len, result);
return get_number(input_format_t::cbor, len) and
get_binary(input_format_t::cbor, len, result);
}
case 0x59: // Binary data (two-byte uint16_t for n follow)
{
std::uint16_t len;
return get_number(input_format_t::cbor, len) and get_binary(input_format_t::cbor, len, result);
return get_number(input_format_t::cbor, len) and
get_binary(input_format_t::cbor, len, result);
}
case 0x5A: // Binary data (four-byte uint32_t for n follow)
{
std::uint32_t len;
return get_number(input_format_t::cbor, len) and get_binary(input_format_t::cbor, len, result);
return get_number(input_format_t::cbor, len) and
get_binary(input_format_t::cbor, len, result);
}
case 0x5B: // Binary data (eight-byte uint64_t for n follow)
{
std::uint64_t len;
return get_number(input_format_t::cbor, len) and get_binary(input_format_t::cbor, len, result);
return get_number(input_format_t::cbor, len) and
get_binary(input_format_t::cbor, len, result);
}
case 0x5F: // Binary data (indefinite length)
{
while (get() != 0xFF)
{
internal_binary_t chunk;
binary_t chunk;
if (not get_cbor_binary(chunk))
{
return false;
@ -1276,7 +1282,7 @@ class binary_reader
case 0xD7: // fixext 8
case 0xD8: // fixext 16
{
internal_binary_t b;
binary_t b;
return get_msgpack_binary(b) and sax->binary(b);
}
@ -1499,83 +1505,106 @@ class binary_reader
@return whether byte array creation completed
*/
bool get_msgpack_binary(internal_binary_t& result)
bool get_msgpack_binary(binary_t& result)
{
// helper function to set the subtype
auto assign_and_return_true = [&result](std::int8_t subtype)
{
result.set_subtype(static_cast<std::uint8_t>(subtype));
return true;
};
switch (current)
{
case 0xC4: // bin 8
{
std::uint8_t len;
return get_number(input_format_t::msgpack, len) and get_binary(input_format_t::msgpack, len, result);
return get_number(input_format_t::msgpack, len) and
get_binary(input_format_t::msgpack, len, result);
}
case 0xC5: // bin 16
{
std::uint16_t len;
return get_number(input_format_t::msgpack, len) and get_binary(input_format_t::msgpack, len, result);
return get_number(input_format_t::msgpack, len) and
get_binary(input_format_t::msgpack, len, result);
}
case 0xC6: // bin 32
{
std::uint32_t len;
return get_number(input_format_t::msgpack, len) and get_binary(input_format_t::msgpack, len, result);
return get_number(input_format_t::msgpack, len) and
get_binary(input_format_t::msgpack, len, result);
}
case 0xC7: // ext 8
{
std::uint8_t len;
result.has_subtype = true;
std::int8_t subtype;
return get_number(input_format_t::msgpack, len) and
get_number(input_format_t::msgpack, result.subtype) and
get_binary(input_format_t::msgpack, len, result);
get_number(input_format_t::msgpack, subtype) and
get_binary(input_format_t::msgpack, len, result) and
assign_and_return_true(subtype);
}
case 0xC8: // ext 16
{
std::uint16_t len;
result.has_subtype = true;
std::int8_t subtype;
return get_number(input_format_t::msgpack, len) and
get_number(input_format_t::msgpack, result.subtype) and
get_binary(input_format_t::msgpack, len, result);
get_number(input_format_t::msgpack, subtype) and
get_binary(input_format_t::msgpack, len, result) and
assign_and_return_true(subtype);
}
case 0xC9: // ext 32
{
std::uint32_t len;
result.has_subtype = true;
std::int8_t subtype;
return get_number(input_format_t::msgpack, len) and
get_number(input_format_t::msgpack, result.subtype) and
get_binary(input_format_t::msgpack, len, result);
get_number(input_format_t::msgpack, subtype) and
get_binary(input_format_t::msgpack, len, result) and
assign_and_return_true(subtype);
}
case 0xD4: // fixext 1
{
result.has_subtype = true;
return get_number(input_format_t::msgpack, result.subtype) and get_binary(input_format_t::msgpack, 1, result);
std::int8_t subtype;
return get_number(input_format_t::msgpack, subtype) and
get_binary(input_format_t::msgpack, 1, result) and
assign_and_return_true(subtype);
}
case 0xD5: // fixext 2
{
result.has_subtype = true;
return get_number(input_format_t::msgpack, result.subtype) and get_binary(input_format_t::msgpack, 2, result);
std::int8_t subtype;
return get_number(input_format_t::msgpack, subtype) and
get_binary(input_format_t::msgpack, 2, result) and
assign_and_return_true(subtype);
}
case 0xD6: // fixext 4
{
result.has_subtype = true;
return get_number(input_format_t::msgpack, result.subtype) and get_binary(input_format_t::msgpack, 4, result);
std::int8_t subtype;
return get_number(input_format_t::msgpack, subtype) and
get_binary(input_format_t::msgpack, 4, result) and
assign_and_return_true(subtype);
}
case 0xD7: // fixext 8
{
result.has_subtype = true;
return get_number(input_format_t::msgpack, result.subtype) and get_binary(input_format_t::msgpack, 8, result);
std::int8_t subtype;
return get_number(input_format_t::msgpack, subtype) and
get_binary(input_format_t::msgpack, 8, result) and
assign_and_return_true(subtype);
}
case 0xD8: // fixext 16
{
result.has_subtype = true;
return get_number(input_format_t::msgpack, result.subtype) and get_binary(input_format_t::msgpack, 16, result);
std::int8_t subtype;
return get_number(input_format_t::msgpack, subtype) and
get_binary(input_format_t::msgpack, 16, result) and
assign_and_return_true(subtype);
}
default: // LCOV_EXCL_LINE
@ -2194,7 +2223,7 @@ class binary_reader
template<typename NumberType>
bool get_binary(const input_format_t format,
const NumberType len,
internal_binary_t& result)
binary_t& result)
{
bool success = true;
std::generate_n(std::back_inserter(result), len, [this, &success, &format]()

View file

@ -23,13 +23,9 @@ input.
template<typename BasicJsonType>
struct json_sax
{
/// type for (signed) integers
using number_integer_t = typename BasicJsonType::number_integer_t;
/// type for unsigned integers
using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
/// type for floating-point numbers
using number_float_t = typename BasicJsonType::number_float_t;
/// type for strings
using string_t = typename BasicJsonType::string_t;
using binary_t = typename BasicJsonType::binary_t;
@ -214,7 +210,7 @@ class json_sax_dom_parser
bool binary(binary_t& val)
{
handle_value(BasicJsonType::binary_array(std::move(val)));
handle_value(std::move(val));
return true;
}
@ -404,7 +400,7 @@ class json_sax_dom_callback_parser
bool binary(binary_t& val)
{
handle_value(BasicJsonType::binary_array(val));
handle_value(std::move(val));
return true;
}

View file

@ -19,7 +19,7 @@ template<typename BasicJsonType> struct internal_iterator
/// iterator for JSON arrays
typename BasicJsonType::array_t::iterator array_iterator {};
/// iterator for JSON binary arrays
typename BasicJsonType::binary_t::iterator binary_iterator {};
typename BasicJsonType::binary_t::container_type::iterator binary_iterator {};
/// generic iterator for all other types
primitive_iterator_t primitive_iterator {};
};

View file

@ -27,7 +27,7 @@ template<typename BasicJsonType, typename CharType>
class binary_writer
{
using string_t = typename BasicJsonType::string_t;
using internal_binary_t = typename BasicJsonType::internal_binary_t;
using binary_t = typename BasicJsonType::binary_t;
public:
/*!
@ -578,7 +578,7 @@ class binary_writer
{
// step 0: determine if the binary type has a set subtype to
// determine whether or not to use the ext or fixext types
const bool use_ext = j.m_value.binary->has_subtype;
const bool use_ext = j.m_value.binary->has_subtype();
// step 1: write control byte and the byte string length
const auto N = j.m_value.binary->size();
@ -658,7 +658,7 @@ class binary_writer
// step 1.5: if this is an ext type, write the subtype
if (use_ext)
{
write_number(j.m_value.binary->subtype);
write_number(static_cast<std::int8_t>(j.m_value.binary->subtype()));
}
// step 2: write the byte string
@ -1080,7 +1080,7 @@ class binary_writer
/*!
@return The size of the BSON-encoded binary array @a value
*/
static std::size_t calc_bson_binary_size(const typename BasicJsonType::internal_binary_t& value)
static std::size_t calc_bson_binary_size(const typename BasicJsonType::binary_t& value)
{
return sizeof(std::int32_t) + value.size() + 1ul;
}
@ -1108,17 +1108,12 @@ class binary_writer
@brief Writes a BSON element with key @a name and binary value @a value
*/
void write_bson_binary(const string_t& name,
const internal_binary_t& value)
const binary_t& value)
{
write_bson_entry_header(name, 0x05);
write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size()));
std::uint8_t subtype = 0x00; // Generic Binary Subtype
if (value.has_subtype)
{
subtype = value.subtype;
}
write_number(subtype);
write_number(value.has_subtype() ? value.subtype() : std::uint8_t(0x00));
oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());
}

View file

@ -45,7 +45,7 @@ class serializer
using number_float_t = typename BasicJsonType::number_float_t;
using number_integer_t = typename BasicJsonType::number_integer_t;
using number_unsigned_t = typename BasicJsonType::number_unsigned_t;
using binary_t = typename BasicJsonType::binary_t;
using binary_char_t = typename BasicJsonType::binary_t::value_type;
static constexpr std::uint8_t UTF8_ACCEPT = 0;
static constexpr std::uint8_t UTF8_REJECT = 1;
@ -84,19 +84,22 @@ class serializer
- strings and object keys are escaped using `escape_string()`
- integer numbers are converted implicitly via `operator<<`
- floating-point numbers are converted to a string using `"%g"` format
- if specified to, binary values are output using the syntax `b[]`, otherwise an exception is thrown
- binary values are serialized as objects containing the subtype and the
byte array
@param[in] val value to serialize
@param[in] pretty_print whether the output shall be pretty-printed
@param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters
in the output are escaped with `\uXXXX` sequences, and the result consists
of ASCII characters only.
@param[in] indent_step the indent level
@param[in] current_indent the current indent level (only used internally)
@param[in] serialize_binary whether the output shall include non-standard binary output
*/
void dump(const BasicJsonType& val, const bool pretty_print,
void dump(const BasicJsonType& val,
const bool pretty_print,
const bool ensure_ascii,
const unsigned int indent_step,
const unsigned int current_indent = 0,
const bool serialize_binary = false)
const unsigned int current_indent = 0)
{
switch (val.m_type)
{
@ -127,7 +130,7 @@ class serializer
o->write_character('\"');
dump_escaped(i->first, ensure_ascii);
o->write_characters("\": ", 3);
dump(i->second, true, ensure_ascii, indent_step, new_indent, serialize_binary);
dump(i->second, true, ensure_ascii, indent_step, new_indent);
o->write_characters(",\n", 2);
}
@ -138,7 +141,7 @@ class serializer
o->write_character('\"');
dump_escaped(i->first, ensure_ascii);
o->write_characters("\": ", 3);
dump(i->second, true, ensure_ascii, indent_step, new_indent, serialize_binary);
dump(i->second, true, ensure_ascii, indent_step, new_indent);
o->write_character('\n');
o->write_characters(indent_string.c_str(), current_indent);
@ -155,7 +158,7 @@ class serializer
o->write_character('\"');
dump_escaped(i->first, ensure_ascii);
o->write_characters("\":", 2);
dump(i->second, false, ensure_ascii, indent_step, current_indent, serialize_binary);
dump(i->second, false, ensure_ascii, indent_step, current_indent);
o->write_character(',');
}
@ -165,7 +168,7 @@ class serializer
o->write_character('\"');
dump_escaped(i->first, ensure_ascii);
o->write_characters("\":", 2);
dump(i->second, false, ensure_ascii, indent_step, current_indent, serialize_binary);
dump(i->second, false, ensure_ascii, indent_step, current_indent);
o->write_character('}');
}
@ -197,14 +200,14 @@ class serializer
i != val.m_value.array->cend() - 1; ++i)
{
o->write_characters(indent_string.c_str(), new_indent);
dump(*i, true, ensure_ascii, indent_step, new_indent, serialize_binary);
dump(*i, true, ensure_ascii, indent_step, new_indent);
o->write_characters(",\n", 2);
}
// last element
assert(not val.m_value.array->empty());
o->write_characters(indent_string.c_str(), new_indent);
dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent, serialize_binary);
dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent);
o->write_character('\n');
o->write_characters(indent_string.c_str(), current_indent);
@ -218,13 +221,13 @@ class serializer
for (auto i = val.m_value.array->cbegin();
i != val.m_value.array->cend() - 1; ++i)
{
dump(*i, false, ensure_ascii, indent_step, current_indent, serialize_binary);
dump(*i, false, ensure_ascii, indent_step, current_indent);
o->write_character(',');
}
// last element
assert(not val.m_value.array->empty());
dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent, serialize_binary);
dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent);
o->write_character(']');
}
@ -242,27 +245,73 @@ class serializer
case value_t::binary:
{
if (not serialize_binary)
if (pretty_print)
{
JSON_THROW(type_error::create(317, "cannot serialize binary data to text JSON"));
}
o->write_characters("{\n", 2);
if (val.m_value.binary->empty())
{
o->write_characters("b[]", 3);
// variable to hold indentation for recursive calls
const auto new_indent = current_indent + indent_step;
if (JSON_HEDLEY_UNLIKELY(indent_string.size() < new_indent))
{
indent_string.resize(indent_string.size() * 2, ' ');
}
o->write_characters(indent_string.c_str(), new_indent);
o->write_characters("\"bytes\": [", 10);
if (not val.m_value.binary->empty())
{
for (auto i = val.m_value.binary->cbegin();
i != val.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
o->write_characters(", ", 2);
}
dump_integer(val.m_value.binary->back());
}
o->write_characters("],\n", 3);
o->write_characters(indent_string.c_str(), new_indent);
o->write_characters("\"subtype\": ", 11);
if (val.m_value.binary->has_subtype())
{
dump_integer(val.m_value.binary->subtype());
}
else
{
o->write_characters("null", 4);
}
o->write_character('\n');
o->write_characters(indent_string.c_str(), current_indent);
o->write_character('}');
}
else
{
o->write_characters("b[", 2);
for (auto i = val.m_value.binary->cbegin();
i != val.m_value.binary->cend() - 1; ++i)
o->write_characters("{\"bytes\":[", 10);
if (not val.m_value.binary->empty())
{
dump_integer(*i);
o->write_character(',');
for (auto i = val.m_value.binary->cbegin();
i != val.m_value.binary->cend() - 1; ++i)
{
dump_integer(*i);
o->write_character(',');
}
dump_integer(val.m_value.binary->back());
}
dump_integer(val.m_value.binary->back());
o->write_character(']');
o->write_characters("],\"subtype\":", 12);
if (val.m_value.binary->has_subtype())
{
dump_integer(val.m_value.binary->subtype());
o->write_character('}');
}
else
{
o->write_characters("null}", 5);
}
}
return;
}
@ -624,7 +673,7 @@ class serializer
template<typename NumberType, detail::enable_if_t<
std::is_same<NumberType, number_unsigned_t>::value or
std::is_same<NumberType, number_integer_t>::value or
std::is_same<NumberType, typename binary_t::value_type>::value,
std::is_same<NumberType, binary_char_t>::value,
int> = 0>
void dump_integer(NumberType x)
{