j3d1/json
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

removed roundtripping of floats (#230)

Browse source
This commit is contained in:
Niels 2016-06-19 17:13:13 +02:00
parent 4e31a0e852
commit ad7a419a03
3 changed files with 43 additions and 349 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

186
src/json.hpp
View file

@ -711,73 +711,6 @@ class basic_json
private: private:
/*!
@brief a type to hold JSON type information
This bitfield type holds information about JSON types. It is internally
used to hold the basic JSON type enumeration, as well as additional
information in the case of values that have been parsed from a string
including whether of not it was created directly or parsed, and in the
case of floating point numbers the number of significant figures in the
original representaiton and if it was in exponential form, if a '+' was
included in the exponent and the capitilization of the exponent marker.
The sole purpose of this information is to permit accurate round trips.
@since version 2.0.0
*/
union type_data_t
{
struct
{
/// the type of the value (@ref value_t)
uint16_t type : 4;
/// whether the number was parsed from a string
uint16_t parsed : 1;
/// whether parsed number contained an exponent ('e'/'E')
uint16_t has_exp : 1;
/// whether parsed number contained a plus in the exponent
uint16_t exp_plus : 1;
/// whether parsed number's exponent was capitalized ('E')
uint16_t exp_cap : 1;
/// the number of figures for a parsed number
uint16_t precision : 8;
} bits;
uint16_t data;
/// return the type as value_t
operator value_t() const
{
return static_cast<value_t>(bits.type);
}
/// test type for equality (ignore other fields)
bool operator==(const value_t& rhs) const
{
return static_cast<value_t>(bits.type) == rhs;
}
/// assignment
type_data_t& operator=(value_t rhs)
{
bits.type = static_cast<uint16_t>(rhs) & 15; // avoid overflow
return *this;
}
/// construct from value_t
type_data_t(value_t t) noexcept
{
*reinterpret_cast<uint16_t*>(this) = 0;
bits.type = static_cast<uint16_t>(t) & 15; // avoid overflow
}
/// default constructor
type_data_t() noexcept
{
data = 0;
bits.type = reinterpret_cast<uint16_t>(value_t::null);
}
};
/// helper for exception-safe object creation /// helper for exception-safe object creation
template<typename T, typename... Args> template<typename T, typename... Args>
static T* create(Args&& ... args) static T* create(Args&& ... args)
@ -6183,79 +6116,23 @@ class basic_json
case value_t::number_float: case value_t::number_float:
{ {
// check if number was parsed from a string if (m_value.number_float == 0)
if (m_type.bits.parsed)
{ {
// check if parsed number had an exponent given // special case for zero to get "0.0"/"-0.0"
if (m_type.bits.has_exp) o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0");
{
// buffer size: precision (2^8-1 = 255) + other ('-.e-xxx' = 7) + null (1)
char buf[263];
int len;
// handle capitalization of the exponent
if (m_type.bits.exp_cap)
{
len = snprintf(buf, sizeof(buf), "%.*E",
m_type.bits.precision, m_value.number_float) + 1;
}
else
{
len = snprintf(buf, sizeof(buf), "%.*e",
m_type.bits.precision, m_value.number_float) + 1;
}
// remove '+' sign from the exponent if necessary
if (not m_type.bits.exp_plus)
{
if (len > static_cast<int>(sizeof(buf)))
{
len = sizeof(buf);
}
for (int i = 0; i < len; i++)
{
if (buf[i] == '+')
{
for (; i + 1 < len; i++)
{
buf[i] = buf[i + 1];
}
}
}
}
o << buf;
}
else
{
// no exponent - output as a decimal
std::stringstream ss;
ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems
ss << std::setprecision(m_type.bits.precision)
<< std::fixed << m_value.number_float;
o << ss.str();
}
} }
else else
{ {
if (m_value.number_float == 0) // Otherwise 6, 15 or 16 digits of precision allows
{ // round-trip IEEE 754 string->float->string,
// special case for zero to get "0.0"/"-0.0" // string->double->string or string->long
o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); // double->string; to be safe, we read this value from
} // std::numeric_limits<number_float_t>::digits10
else std::stringstream ss;
{ ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems
// Otherwise 6, 15 or 16 digits of precision allows ss << std::setprecision(std::numeric_limits<double>::digits10)
// round-trip IEEE 754 string->float->string, << m_value.number_float;
// string->double->string or string->long o << ss.str();
// double->string; to be safe, we read this value from
// std::numeric_limits<number_float_t>::digits10
std::stringstream ss;
ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems
ss << std::setprecision(std::numeric_limits<double>::digits10)
<< m_value.number_float;
o << ss.str();
}
} }
return; return;
} }
@ -6280,7 +6157,7 @@ class basic_json
////////////////////// //////////////////////
/// the type of the current element /// the type of the current element
type_data_t m_type = value_t::null; value_t m_type = value_t::null;
/// the value of the current element /// the value of the current element
json_value m_value = {}; json_value m_value = {};
@ -8457,18 +8334,12 @@ basic_json_parser_63:
number_integer_t or @ref number_unsigned_t then it sets the result number_integer_t or @ref number_unsigned_t then it sets the result
parameter accordingly. parameter accordingly.
The 'floating point representation' includes the number of significant
figures after the radix point, whether the number is in exponential or
decimal form, the capitalization of the exponent marker, and if the
optional '+' is present in the exponent. This information is necessary
to perform accurate round trips of floating point numbers.
If the number is a floating point number the number is then parsed If the number is a floating point number the number is then parsed
using @a std:strtod (or @a std:strtof or @a std::strtold). using @a std:strtod (or @a std:strtof or @a std::strtold).
@param[out] result @ref basic_json object to receive the number, or @param[out] result @ref basic_json object to receive the number, or
NAN if the conversion read past the current token. The latter case NAN if the conversion read past the current token. The latter case
needs to be treated by the caller function. needs to be treated by the caller function.
*/ */
void get_number(basic_json& result) const void get_number(basic_json& result) const
{ {
@ -8476,15 +8347,6 @@ basic_json_parser_63:
const lexer::lexer_char_t* curptr = m_start; const lexer::lexer_char_t* curptr = m_start;
// remember this number was parsed (for later serialization)
result.m_type.bits.parsed = true;
// 'found_radix_point' will be set to 0xFF upon finding a radix
// point and later used to mask in/out the precision depending
// whether a radix is found i.e. 'precision &= found_radix_point'
uint8_t found_radix_point = 0;
uint8_t precision = 0;
// accumulate the integer conversion result (unsigned for now) // accumulate the integer conversion result (unsigned for now)
number_unsigned_t value = 0; number_unsigned_t value = 0;
@ -8517,22 +8379,11 @@ basic_json_parser_63:
{ {
// don't count '.' but change to float // don't count '.' but change to float
type = value_t::number_float; type = value_t::number_float;
// reset precision count
precision = 0;
found_radix_point = 0xFF;
continue; continue;
} }
// assume exponent (if not then will fail parse): change to // assume exponent (if not then will fail parse): change to
// float, stop counting and record exponent details // float, stop counting and record exponent details
type = value_t::number_float; type = value_t::number_float;
result.m_type.bits.has_exp = true;
// exponent capitalization
result.m_type.bits.exp_cap = (*curptr == 'E');
// exponent '+' sign
result.m_type.bits.exp_plus = (*(++curptr) == '+');
break; break;
} }
@ -8554,13 +8405,8 @@ basic_json_parser_63:
value = temp; value = temp;
} }
} }
++precision;
} }
// If no radix point was found then precision would now be set to
// the number of digits, which is wrong - clear it.
result.m_type.bits.precision = precision & found_radix_point;
// save the value (if not a float) // save the value (if not a float)
if (type == value_t::number_unsigned) if (type == value_t::number_unsigned)
{ {

186
src/json.hpp.re2c
View file

@ -711,73 +711,6 @@ class basic_json
private: private:
/*!
@brief a type to hold JSON type information
This bitfield type holds information about JSON types. It is internally
used to hold the basic JSON type enumeration, as well as additional
information in the case of values that have been parsed from a string
including whether of not it was created directly or parsed, and in the
case of floating point numbers the number of significant figures in the
original representaiton and if it was in exponential form, if a '+' was
included in the exponent and the capitilization of the exponent marker.
The sole purpose of this information is to permit accurate round trips.
@since version 2.0.0
*/
union type_data_t
{
struct
{
/// the type of the value (@ref value_t)
uint16_t type : 4;
/// whether the number was parsed from a string
uint16_t parsed : 1;
/// whether parsed number contained an exponent ('e'/'E')
uint16_t has_exp : 1;
/// whether parsed number contained a plus in the exponent
uint16_t exp_plus : 1;
/// whether parsed number's exponent was capitalized ('E')
uint16_t exp_cap : 1;
/// the number of figures for a parsed number
uint16_t precision : 8;
} bits;
uint16_t data;
/// return the type as value_t
operator value_t() const
{
return static_cast<value_t>(bits.type);
}
/// test type for equality (ignore other fields)
bool operator==(const value_t& rhs) const
{
return static_cast<value_t>(bits.type) == rhs;
}
/// assignment
type_data_t& operator=(value_t rhs)
{
bits.type = static_cast<uint16_t>(rhs) & 15; // avoid overflow
return *this;
}
/// construct from value_t
type_data_t(value_t t) noexcept
{
*reinterpret_cast<uint16_t*>(this) = 0;
bits.type = static_cast<uint16_t>(t) & 15; // avoid overflow
}
/// default constructor
type_data_t() noexcept
{
data = 0;
bits.type = reinterpret_cast<uint16_t>(value_t::null);
}
};
/// helper for exception-safe object creation /// helper for exception-safe object creation
template<typename T, typename... Args> template<typename T, typename... Args>
static T* create(Args&& ... args) static T* create(Args&& ... args)
@ -6183,79 +6116,23 @@ class basic_json
case value_t::number_float: case value_t::number_float:
{ {
// check if number was parsed from a string if (m_value.number_float == 0)
if (m_type.bits.parsed)
{ {
// check if parsed number had an exponent given // special case for zero to get "0.0"/"-0.0"
if (m_type.bits.has_exp) o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0");
{
// buffer size: precision (2^8-1 = 255) + other ('-.e-xxx' = 7) + null (1)
char buf[263];
int len;
// handle capitalization of the exponent
if (m_type.bits.exp_cap)
{
len = snprintf(buf, sizeof(buf), "%.*E",
m_type.bits.precision, m_value.number_float) + 1;
}
else
{
len = snprintf(buf, sizeof(buf), "%.*e",
m_type.bits.precision, m_value.number_float) + 1;
}
// remove '+' sign from the exponent if necessary
if (not m_type.bits.exp_plus)
{
if (len > static_cast<int>(sizeof(buf)))
{
len = sizeof(buf);
}
for (int i = 0; i < len; i++)
{
if (buf[i] == '+')
{
for (; i + 1 < len; i++)
{
buf[i] = buf[i + 1];
}
}
}
}
o << buf;
}
else
{
// no exponent - output as a decimal
std::stringstream ss;
ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems
ss << std::setprecision(m_type.bits.precision)
<< std::fixed << m_value.number_float;
o << ss.str();
}
} }
else else
{ {
if (m_value.number_float == 0) // Otherwise 6, 15 or 16 digits of precision allows
{ // round-trip IEEE 754 string->float->string,
// special case for zero to get "0.0"/"-0.0" // string->double->string or string->long
o << (std::signbit(m_value.number_float) ? "-0.0" : "0.0"); // double->string; to be safe, we read this value from
} // std::numeric_limits<number_float_t>::digits10
else std::stringstream ss;
{ ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems
// Otherwise 6, 15 or 16 digits of precision allows ss << std::setprecision(std::numeric_limits<double>::digits10)
// round-trip IEEE 754 string->float->string, << m_value.number_float;
// string->double->string or string->long o << ss.str();
// double->string; to be safe, we read this value from
// std::numeric_limits<number_float_t>::digits10
std::stringstream ss;
ss.imbue(std::locale(std::locale(), new DecimalSeparator)); // fix locale problems
ss << std::setprecision(std::numeric_limits<double>::digits10)
<< m_value.number_float;
o << ss.str();
}
} }
return; return;
} }
@ -6280,7 +6157,7 @@ class basic_json
////////////////////// //////////////////////
/// the type of the current element /// the type of the current element
type_data_t m_type = value_t::null; value_t m_type = value_t::null;
/// the value of the current element /// the value of the current element
json_value m_value = {}; json_value m_value = {};
@ -7767,18 +7644,12 @@ class basic_json
number_integer_t or @ref number_unsigned_t then it sets the result number_integer_t or @ref number_unsigned_t then it sets the result
parameter accordingly. parameter accordingly.
The 'floating point representation' includes the number of significant
figures after the radix point, whether the number is in exponential or
decimal form, the capitalization of the exponent marker, and if the
optional '+' is present in the exponent. This information is necessary
to perform accurate round trips of floating point numbers.
If the number is a floating point number the number is then parsed If the number is a floating point number the number is then parsed
using @a std:strtod (or @a std:strtof or @a std::strtold). using @a std:strtod (or @a std:strtof or @a std::strtold).
@param[out] result @ref basic_json object to receive the number, or @param[out] result @ref basic_json object to receive the number, or
NAN if the conversion read past the current token. The latter case NAN if the conversion read past the current token. The latter case
needs to be treated by the caller function. needs to be treated by the caller function.
*/ */
void get_number(basic_json& result) const void get_number(basic_json& result) const
{ {
@ -7786,15 +7657,6 @@ class basic_json
const lexer::lexer_char_t* curptr = m_start; const lexer::lexer_char_t* curptr = m_start;
// remember this number was parsed (for later serialization)
result.m_type.bits.parsed = true;
// 'found_radix_point' will be set to 0xFF upon finding a radix
// point and later used to mask in/out the precision depending
// whether a radix is found i.e. 'precision &= found_radix_point'
uint8_t found_radix_point = 0;
uint8_t precision = 0;
// accumulate the integer conversion result (unsigned for now) // accumulate the integer conversion result (unsigned for now)
number_unsigned_t value = 0; number_unsigned_t value = 0;
@ -7827,22 +7689,11 @@ class basic_json
{ {
// don't count '.' but change to float // don't count '.' but change to float
type = value_t::number_float; type = value_t::number_float;
// reset precision count
precision = 0;
found_radix_point = 0xFF;
continue; continue;
} }
// assume exponent (if not then will fail parse): change to // assume exponent (if not then will fail parse): change to
// float, stop counting and record exponent details // float, stop counting and record exponent details
type = value_t::number_float; type = value_t::number_float;
result.m_type.bits.has_exp = true;
// exponent capitalization
result.m_type.bits.exp_cap = (*curptr == 'E');
// exponent '+' sign
result.m_type.bits.exp_plus = (*(++curptr) == '+');
break; break;
} }
@ -7864,13 +7715,8 @@ class basic_json
value = temp; value = temp;
} }
} }
++precision;
} }
// If no radix point was found then precision would now be set to
// the number of digits, which is wrong - clear it.
result.m_type.bits.precision = precision & found_radix_point;
// save the value (if not a float) // save the value (if not a float)
if (type == value_t::number_unsigned) if (type == value_t::number_unsigned)
{ {

20
test/src/unit.cpp
View file

@ -11900,14 +11900,14 @@ TEST_CASE("compliance tests from nativejson-benchmark")
"test/data/json_roundtrip/roundtrip21.json", "test/data/json_roundtrip/roundtrip21.json",
"test/data/json_roundtrip/roundtrip22.json", "test/data/json_roundtrip/roundtrip22.json",
"test/data/json_roundtrip/roundtrip23.json", "test/data/json_roundtrip/roundtrip23.json",
"test/data/json_roundtrip/roundtrip24.json", //"test/data/json_roundtrip/roundtrip24.json", // roundtrip error
"test/data/json_roundtrip/roundtrip25.json", //"test/data/json_roundtrip/roundtrip25.json", // roundtrip error
"test/data/json_roundtrip/roundtrip26.json", //"test/data/json_roundtrip/roundtrip26.json", // roundtrip error
"test/data/json_roundtrip/roundtrip27.json", //"test/data/json_roundtrip/roundtrip27.json", // roundtrip error
"test/data/json_roundtrip/roundtrip28.json", //"test/data/json_roundtrip/roundtrip28.json", // roundtrip error
"test/data/json_roundtrip/roundtrip29.json", "test/data/json_roundtrip/roundtrip29.json",
"test/data/json_roundtrip/roundtrip30.json", //"test/data/json_roundtrip/roundtrip30.json", // roundtrip error
"test/data/json_roundtrip/roundtrip31.json", //"test/data/json_roundtrip/roundtrip31.json", // roundtrip error
"test/data/json_roundtrip/roundtrip32.json" "test/data/json_roundtrip/roundtrip32.json"
}) })
{ {
@ -14050,8 +14050,10 @@ TEST_CASE("regression tests")
//CHECK(j2b.dump() == "23.42"); //CHECK(j2b.dump() == "23.42");
CHECK(j3a.dump() == "10000"); CHECK(j3a.dump() == "10000");
CHECK(j3b.dump() == "1E04"); CHECK(j3b.dump() == "10000");
CHECK(j3c.dump() == "1e04"); CHECK(j3c.dump() == "10000");
//CHECK(j3b.dump() == "1E04"); // roundtrip error
//CHECK(j3c.dump() == "1e04"); // roundtrip error
} }
SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator") SECTION("issue #233 - Can't use basic_json::iterator as a base iterator for std::move_iterator")