j3d1/json
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'develop' into feature/messagepack

Browse source
This commit is contained in:
Niels Lohmann 2016-11-26 18:17:18 +01:00
commit 87e088990b
8 changed files with 163 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.gitignore vendored
View file

@ -14,3 +14,7 @@ doc/html
me.nlohmann.json.docset
benchmarks/files/numbers/*.json
.idea
cmake-build-debug

8
.travis.yml
View file

@ -84,22 +84,22 @@ matrix:
# Coverity (only for branch coverity_scan)
- os: linux
compiler: gcc
compiler: clang
before_install: echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-certificates.crt
addons:
apt:
sources: ['ubuntu-toolchain-r-test']
packages: ['g++-5', 'valgrind']
packages: ['valgrind']
coverity_scan:
project:
name: "nlohmann/json"
description: "Build submitted via Travis CI"
notification_email: niels.lohmann@gmail.com
build_command_prepend: "make clean ; sudo cp $(which g++-5) $(which g++)"
build_command_prepend: "make clean"
build_command: "make"
branch_pattern: coverity_scan
env:
- COMPILER=g++-5
- LLVM_VERSION=3.6.0
- SPECIAL=coverity
# OSX / Clang

2
Makefile
View file

@ -95,7 +95,7 @@ pretty:
# benchmarks
json_benchmarks: benchmarks/benchmarks.cpp benchmarks/benchpress.hpp benchmarks/cxxopts.hpp src/json.hpp
cd benchmarks/files/numbers ; python generate.py
$(CXX) -std=c++11 $(CXXFLAGS) -DNDEBUG -O3 -flto -I src -I benchmarks $< $(LDFLAGS) -o $@
$(CXX) -std=c++11 -pthread $(CXXFLAGS) -DNDEBUG -O3 -flto -I src -I benchmarks $< $(LDFLAGS) -o $@
./json_benchmarks

4
README.md
View file

@ -3,6 +3,7 @@
[![Build Status](https://travis-ci.org/nlohmann/json.svg?branch=master)](https://travis-ci.org/nlohmann/json)
[![Build Status](https://ci.appveyor.com/api/projects/status/1acb366xfyg3qybk/branch/develop?svg=true)](https://ci.appveyor.com/project/nlohmann/json)
[![Coverage Status](https://img.shields.io/coveralls/nlohmann/json.svg)](https://coveralls.io/r/nlohmann/json)
[![Coverity Scan Build Status](https://scan.coverity.com/projects/5550/badge.svg)](https://scan.coverity.com/projects/nlohmann-json)
[![Try online](https://img.shields.io/badge/try-online-blue.svg)](http://melpon.org/wandbox/permlink/fsf5FqYe6GoX68W6)
[![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](http://nlohmann.github.io/json)
[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/nlohmann/json/master/LICENSE.MIT)
@ -499,6 +500,9 @@ I deeply appreciate the help of the following people.
- [ChristophJud](https://github.com/ChristophJud) overworked the CMake files to ease project inclusion.
- [Vladimir Petrigo](https://github.com/vpetrigo) made a SFINAE hack more readable.
- [Denis Andrejew](https://github.com/seeekr) fixed a grammar issue in the README file.
- [Pierre-Antoine Lacaze](https://github.com/palacaze) found a subtle bug in the `dump()` function.
- [TurpentineDistillery](https://github.com/TurpentineDistillery) pointed to [`std::locale::classic()`](http://en.cppreference.com/w/cpp/locale/locale/classic) to avoid too much locale joggling.
- [cgzones](https://github.com/cgzones) had an idea how to fix the Coverity scan.
Thanks a lot for helping out!

189
benchmarks/benchmarks.cpp
View file

@ -1,107 +1,118 @@
#define BENCHPRESS_CONFIG_MAIN
#include <fstream>
#include <sstream>
#include <benchpress.hpp>
#include <json.hpp>
#include <pthread.h>
#include <thread>
BENCHMARK("parse jeopardy.json", [](benchpress::context* ctx)
using json = nlohmann::json;
struct StartUp
{
for (size_t i = 0; i < ctx->num_iterations(); ++i)
StartUp()
{
std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
nlohmann::json j;
j << input_file;
#ifndef __llvm__
// pin thread to a single CPU
cpu_set_t cpuset;
pthread_t thread;
thread = pthread_self();
CPU_ZERO(&cpuset);
CPU_SET(std::thread::hardware_concurrency() - 1, &cpuset);
pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
#endif
}
})
};
StartUp startup;
BENCHMARK("parse canada.json", [](benchpress::context* ctx)
enum class EMode { input, output_no_indent, output_with_indent };
static void bench(benchpress::context& ctx,
const std::string& in_path,
const EMode mode)
{
for (size_t i = 0; i < ctx->num_iterations(); ++i)
// using string streams for benchmarking to factor-out cold-cache disk
// access.
std::stringstream istr;
{
std::ifstream input_file("benchmarks/files/nativejson-benchmark/canada.json");
nlohmann::json j;
j << input_file;
}
})
// read file into string stream
std::ifstream input_file(in_path);
istr << input_file.rdbuf();
input_file.close();
BENCHMARK("parse citm_catalog.json", [](benchpress::context* ctx)
{
for (size_t i = 0; i < ctx->num_iterations(); ++i)
{
std::ifstream input_file("benchmarks/files/nativejson-benchmark/citm_catalog.json");
nlohmann::json j;
j << input_file;
}
})
BENCHMARK("parse twitter.json", [](benchpress::context* ctx)
{
for (size_t i = 0; i < ctx->num_iterations(); ++i)
{
std::ifstream input_file("benchmarks/files/nativejson-benchmark/twitter.json");
nlohmann::json j;
j << input_file;
}
})
BENCHMARK("parse numbers/floats.json", [](benchpress::context* ctx)
{
for (size_t i = 0; i < ctx->num_iterations(); ++i)
{
std::ifstream input_file("benchmarks/files/numbers/floats.json");
nlohmann::json j;
j << input_file;
}
})
BENCHMARK("parse numbers/signed_ints.json", [](benchpress::context* ctx)
{
for (size_t i = 0; i < ctx->num_iterations(); ++i)
{
std::ifstream input_file("benchmarks/files/numbers/signed_ints.json");
nlohmann::json j;
j << input_file;
}
})
BENCHMARK("parse numbers/unsigned_ints.json", [](benchpress::context* ctx)
{
for (size_t i = 0; i < ctx->num_iterations(); ++i)
{
std::ifstream input_file("benchmarks/files/numbers/unsigned_ints.json");
nlohmann::json j;
j << input_file;
}
})
BENCHMARK("dump jeopardy.json", [](benchpress::context* ctx)
{
std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
nlohmann::json j;
j << input_file;
std::ofstream output_file("jeopardy.dump.json");
ctx->reset_timer();
for (size_t i = 0; i < ctx->num_iterations(); ++i)
{
output_file << j;
// read the stream once
json j;
j << istr;
// clear flags and rewind
istr.clear();
istr.seekg(0);
}
std::remove("jeopardy.dump.json");
})
BENCHMARK("dump jeopardy.json with indent", [](benchpress::context* ctx)
{
std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
nlohmann::json j;
j << input_file;
std::ofstream output_file("jeopardy.dump.json");
ctx->reset_timer();
for (size_t i = 0; i < ctx->num_iterations(); ++i)
switch (mode)
{
output_file << std::setw(4) << j;
// benchmarking input
case EMode::input:
{
ctx.reset_timer();
for (size_t i = 0; i < ctx.num_iterations(); ++i)
{
// clear flags and rewind
istr.clear();
istr.seekg(0);
json j;
j << istr;
}
std::remove("jeopardy.dump.json");
})
break;
}
// benchmarking output
case EMode::output_no_indent:
case EMode::output_with_indent:
{
// create JSON value from input
json j;
j << istr;
std::stringstream ostr;
ctx.reset_timer();
for (size_t i = 0; i < ctx.num_iterations(); ++i)
{
if (mode == EMode::output_no_indent)
{
ostr << j;
}
else
{
ostr << std::setw(4) << j;
}
// reset data
ostr.str(std::string());
}
break;
}
}
}
#define BENCHMARK_I(mode, title, in_path) \
BENCHMARK((title), [](benchpress::context* ctx) \
{ \
bench(*ctx, (in_path), (mode)); \
})
BENCHMARK_I(EMode::input, "parse jeopardy.json", "benchmarks/files/jeopardy/jeopardy.json");
BENCHMARK_I(EMode::input, "parse canada.json", "benchmarks/files/nativejson-benchmark/canada.json");
BENCHMARK_I(EMode::input, "parse citm_catalog.json", "benchmarks/files/nativejson-benchmark/citm_catalog.json");
BENCHMARK_I(EMode::input, "parse twitter.json", "benchmarks/files/nativejson-benchmark/twitter.json");
BENCHMARK_I(EMode::input, "parse numbers/floats.json", "benchmarks/files/numbers/floats.json");
BENCHMARK_I(EMode::input, "parse numbers/signed_ints.json", "benchmarks/files/numbers/signed_ints.json");
BENCHMARK_I(EMode::input, "parse numbers/unsigned_ints.json", "benchmarks/files/numbers/unsigned_ints.json");
BENCHMARK_I(EMode::output_no_indent, "dump jeopardy.json", "benchmarks/files/jeopardy/jeopardy.json");
BENCHMARK_I(EMode::output_with_indent, "dump jeopardy.json with indent", "benchmarks/files/jeopardy/jeopardy.json");
BENCHMARK_I(EMode::output_no_indent, "dump numbers/floats.json", "benchmarks/files/numbers/floats.json");
BENCHMARK_I(EMode::output_no_indent, "dump numbers/signed_ints.json", "benchmarks/files/numbers/signed_ints.json");

37
src/json.hpp
View file

@ -45,7 +45,7 @@ SOFTWARE.
#include <iostream> // istream, ostream
#include <iterator> // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
#include <limits> // numeric_limits
#include <locale> // locale, numpunct
#include <locale> // locale
#include <map> // map
#include <memory> // addressof, allocator, allocator_traits, unique_ptr
#include <numeric> // accumulate
@ -122,26 +122,6 @@ struct has_mapped_type
std::is_integral<decltype(detect(std::declval<T>()))>::value;
};
/*!
@brief helper class to create locales with decimal point
This struct is used a default locale during the JSON serialization. JSON
requires the decimal point to be `.`, so this function overloads the
`do_decimal_point()` function to return `.`. This function is called by
float-to-string conversions to retrieve the decimal separator between integer
and fractional parts.
@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315
@since version 2.0.0
*/
struct DecimalSeparator : std::numpunct<char>
{
char do_decimal_point() const
{
return '.';
}
};
}
/*!
@ -2201,8 +2181,7 @@ class basic_json
{
std::stringstream ss;
// fix locale problems
const static std::locale loc(std::locale(), new DecimalSeparator);
ss.imbue(loc);
ss.imbue(std::locale::classic());
// 6, 15 or 16 digits of precision allows round-trip IEEE 754
// string->float->string, string->double->string or string->long
@ -5829,7 +5808,7 @@ class basic_json
o.width(0);
// fix locale problems
const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator));
const auto old_locale = o.imbue(std::locale::classic());
// set precision
// 6, 15 or 16 digits of precision allows round-trip IEEE 754
@ -8111,6 +8090,12 @@ class basic_json
explicit lexer(std::istream& s)
: m_stream(&s), m_line_buffer()
{
// immediately abort if stream is erroneous
if (s.fail())
{
throw std::invalid_argument("stream error: " + std::string(strerror(errno)));
}
// fill buffer
fill_line_buffer();
@ -9250,6 +9235,7 @@ basic_json_parser_66:
m_line_buffer.clear();
for (m_cursor = m_start; m_cursor != m_limit; ++m_cursor)
{
assert(m_cursor != nullptr);
m_line_buffer.append(1, static_cast<const char>(*m_cursor));
}
}
@ -9257,8 +9243,11 @@ basic_json_parser_66:
// append n characters to make sure that there is sufficient
// space between m_cursor and m_limit
m_line_buffer.append(1, '\x00');
if (n > 0)
{
m_line_buffer.append(n - 1, '\x01');
}
}
else
{
// delete processed characters from line buffer

37
src/json.hpp.re2c
View file

@ -45,7 +45,7 @@ SOFTWARE.
#include <iostream> // istream, ostream
#include <iterator> // advance, begin, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator
#include <limits> // numeric_limits
#include <locale> // locale, numpunct
#include <locale> // locale
#include <map> // map
#include <memory> // addressof, allocator, allocator_traits, unique_ptr
#include <numeric> // accumulate
@ -122,26 +122,6 @@ struct has_mapped_type
std::is_integral<decltype(detect(std::declval<T>()))>::value;
};
/*!
@brief helper class to create locales with decimal point
This struct is used a default locale during the JSON serialization. JSON
requires the decimal point to be `.`, so this function overloads the
`do_decimal_point()` function to return `.`. This function is called by
float-to-string conversions to retrieve the decimal separator between integer
and fractional parts.
@sa https://github.com/nlohmann/json/issues/51#issuecomment-86869315
@since version 2.0.0
*/
struct DecimalSeparator : std::numpunct<char>
{
char do_decimal_point() const
{
return '.';
}
};
}
/*!
@ -2201,8 +2181,7 @@ class basic_json
{
std::stringstream ss;
// fix locale problems
const static std::locale loc(std::locale(), new DecimalSeparator);
ss.imbue(loc);
ss.imbue(std::locale::classic());
// 6, 15 or 16 digits of precision allows round-trip IEEE 754
// string->float->string, string->double->string or string->long
@ -5829,7 +5808,7 @@ class basic_json
o.width(0);
// fix locale problems
const auto old_locale = o.imbue(std::locale(std::locale(), new DecimalSeparator));
const auto old_locale = o.imbue(std::locale::classic());
// set precision
// 6, 15 or 16 digits of precision allows round-trip IEEE 754
@ -8111,6 +8090,12 @@ class basic_json
explicit lexer(std::istream& s)
: m_stream(&s), m_line_buffer()
{
// immediately abort if stream is erroneous
if (s.fail())
{
throw std::invalid_argument("stream error: " + std::string(strerror(errno)));
}
// fill buffer
fill_line_buffer();
@ -8400,6 +8385,7 @@ class basic_json
m_line_buffer.clear();
for (m_cursor = m_start; m_cursor != m_limit; ++m_cursor)
{
assert(m_cursor != nullptr);
m_line_buffer.append(1, static_cast<const char>(*m_cursor));
}
}
@ -8407,8 +8393,11 @@ class basic_json
// append n characters to make sure that there is sufficient
// space between m_cursor and m_limit
m_line_buffer.append(1, '\x00');
if (n > 0)
{
m_line_buffer.append(n - 1, '\x01');
}
}
else
{
// delete processed characters from line buffer

21
test/src/unit-regression.cpp
View file

@ -495,4 +495,25 @@ TEST_CASE("regression tests")
json j = json::parse("22e2222");
CHECK(j == json());
}
SECTION("issue #366 - json::parse on failed stream gets stuck")
{
std::ifstream f("file_not_found.json");
CHECK_THROWS_AS(json::parse(f), std::invalid_argument);
}
SECTION("issue #367 - calling stream at EOF")
{
std::stringstream ss;
json j;
ss << "123";
CHECK_NOTHROW(j << ss);
// see https://github.com/nlohmann/json/issues/367#issuecomment-262841893:
// ss is not at EOF; this yielded an error before the fix
// (threw basic_string::append). No, it should just throw
// a parse error because of the EOF.
CHECK_THROWS_AS(j << ss, std::invalid_argument);
CHECK_THROWS_WITH(j << ss, "parse error - unexpected end of input");
}
}