➕ using Google Benchmark #921
This commit is contained in:
Niels Lohmann
parent
6402077ac2
commit
a8f711a2f1
18 changed files with 125 additions and 2086 deletions
|
|
@ -1,132 +1,106 @@
|
|||
#define BENCHPRESS_CONFIG_MAIN
|
||||
|
||||
#include "benchmark/benchmark.h"
|
||||
#include "json.hpp"
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <benchpress.hpp>
|
||||
#include <json.hpp>
|
||||
#include <pthread.h>
|
||||
#include <thread>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
struct StartUp
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// parse JSON from file
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static void ParseFile(benchmark::State& state, const char* filename)
|
||||
{
|
||||
StartUp()
|
||||
while (state.KeepRunning())
|
||||
{
|
||||
#ifndef __llvm__
|
||||
// pin thread to a single CPU
|
||||
cpu_set_t cpuset;
|
||||
pthread_t thread;
|
||||
thread = pthread_self();
|
||||
CPU_ZERO(&cpuset);
|
||||
CPU_SET(std::thread::hardware_concurrency() - 1, &cpuset);
|
||||
pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
|
||||
#endif
|
||||
state.PauseTiming();
|
||||
auto* f = new std::ifstream(filename);
|
||||
auto* j = new json();
|
||||
state.ResumeTiming();
|
||||
|
||||
*j = json::parse(*f);
|
||||
|
||||
state.PauseTiming();
|
||||
delete f;
|
||||
delete j;
|
||||
state.ResumeTiming();
|
||||
}
|
||||
};
|
||||
StartUp startup;
|
||||
|
||||
enum class EMode { input, output_no_indent, output_with_indent };
|
||||
|
||||
static void bench(benchpress::context& ctx,
|
||||
const std::string& in_path,
|
||||
const EMode mode)
|
||||
{
|
||||
// using string streams for benchmarking to factor-out cold-cache disk
|
||||
// access.
|
||||
#if defined( FROMFILE )
|
||||
std::ifstream istr;
|
||||
{
|
||||
istr.open( in_path, std::ifstream::in );
|
||||
|
||||
// read the stream once
|
||||
json j;
|
||||
istr >> j;
|
||||
// clear flags and rewind
|
||||
istr.clear();
|
||||
istr.seekg(0);
|
||||
}
|
||||
#else
|
||||
std::stringstream istr;
|
||||
{
|
||||
// read file into string stream
|
||||
std::ifstream input_file(in_path);
|
||||
istr << input_file.rdbuf();
|
||||
input_file.close();
|
||||
|
||||
// read the stream once
|
||||
json j;
|
||||
istr >> j;
|
||||
// clear flags and rewind
|
||||
istr.clear();
|
||||
istr.seekg(0);
|
||||
}
|
||||
#endif
|
||||
|
||||
switch (mode)
|
||||
{
|
||||
// benchmarking input
|
||||
case EMode::input:
|
||||
{
|
||||
ctx.reset_timer();
|
||||
|
||||
for (size_t i = 0; i < ctx.num_iterations(); ++i)
|
||||
{
|
||||
// clear flags and rewind
|
||||
istr.clear();
|
||||
istr.seekg(0);
|
||||
json j;
|
||||
istr >> j;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
// benchmarking output
|
||||
case EMode::output_no_indent:
|
||||
case EMode::output_with_indent:
|
||||
{
|
||||
// create JSON value from input
|
||||
json j;
|
||||
istr >> j;
|
||||
std::stringstream ostr;
|
||||
|
||||
ctx.reset_timer();
|
||||
for (size_t i = 0; i < ctx.num_iterations(); ++i)
|
||||
{
|
||||
if (mode == EMode::output_no_indent)
|
||||
{
|
||||
ostr << j;
|
||||
}
|
||||
else
|
||||
{
|
||||
ostr << std::setw(4) << j;
|
||||
}
|
||||
|
||||
// reset data
|
||||
ostr.str(std::string());
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::ifstream file(filename, std::ios::binary | std::ios::ate);
|
||||
state.SetBytesProcessed(state.iterations() * file.tellg());
|
||||
}
|
||||
BENCHMARK_CAPTURE(ParseFile, jeopardy, "data/jeopardy/jeopardy.json");
|
||||
BENCHMARK_CAPTURE(ParseFile, canada, "data/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(ParseFile, citm_catalog, "data/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(ParseFile, twitter, "data/nativejson-benchmark/twitter.json");
|
||||
BENCHMARK_CAPTURE(ParseFile, floats, "data/numbers/floats.json");
|
||||
BENCHMARK_CAPTURE(ParseFile, signed_ints, "data/numbers/signed_ints.json");
|
||||
BENCHMARK_CAPTURE(ParseFile, unsigned_ints, "data/numbers/unsigned_ints.json");
|
||||
|
||||
#define BENCHMARK_I(mode, title, in_path) \
|
||||
BENCHMARK((title), [](benchpress::context* ctx) \
|
||||
{ \
|
||||
bench(*ctx, (in_path), (mode)); \
|
||||
})
|
||||
|
||||
BENCHMARK_I(EMode::input, "parse jeopardy.json", "files/jeopardy/jeopardy.json");
|
||||
BENCHMARK_I(EMode::input, "parse canada.json", "files/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_I(EMode::input, "parse citm_catalog.json", "files/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_I(EMode::input, "parse twitter.json", "files/nativejson-benchmark/twitter.json");
|
||||
BENCHMARK_I(EMode::input, "parse numbers/floats.json", "files/numbers/floats.json");
|
||||
BENCHMARK_I(EMode::input, "parse numbers/signed_ints.json", "files/numbers/signed_ints.json");
|
||||
BENCHMARK_I(EMode::input, "parse numbers/unsigned_ints.json", "files/numbers/unsigned_ints.json");
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// parse JSON from string
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
BENCHMARK_I(EMode::output_no_indent, "dump jeopardy.json", "files/jeopardy/jeopardy.json");
|
||||
BENCHMARK_I(EMode::output_with_indent, "dump jeopardy.json with indent", "files/jeopardy/jeopardy.json");
|
||||
BENCHMARK_I(EMode::output_no_indent, "dump numbers/floats.json", "files/numbers/floats.json");
|
||||
BENCHMARK_I(EMode::output_no_indent, "dump numbers/signed_ints.json", "files/numbers/signed_ints.json");
|
||||
static void ParseString(benchmark::State& state, const char* filename)
|
||||
{
|
||||
std::ifstream f(filename);
|
||||
std::string str((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
|
||||
|
||||
while (state.KeepRunning())
|
||||
{
|
||||
state.PauseTiming();
|
||||
auto* j = new json();
|
||||
state.ResumeTiming();
|
||||
|
||||
*j = json::parse(str);
|
||||
|
||||
state.PauseTiming();
|
||||
delete j;
|
||||
state.ResumeTiming();
|
||||
}
|
||||
|
||||
state.SetBytesProcessed(state.iterations() * str.size());
|
||||
}
|
||||
BENCHMARK_CAPTURE(ParseString, jeopardy, "data/jeopardy/jeopardy.json");
|
||||
BENCHMARK_CAPTURE(ParseString, canada, "data/nativejson-benchmark/canada.json");
|
||||
BENCHMARK_CAPTURE(ParseString, citm_catalog, "data/nativejson-benchmark/citm_catalog.json");
|
||||
BENCHMARK_CAPTURE(ParseString, twitter, "data/nativejson-benchmark/twitter.json");
|
||||
BENCHMARK_CAPTURE(ParseString, floats, "data/numbers/floats.json");
|
||||
BENCHMARK_CAPTURE(ParseString, signed_ints, "data/numbers/signed_ints.json");
|
||||
BENCHMARK_CAPTURE(ParseString, unsigned_ints, "data/numbers/unsigned_ints.json");
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
// serialize JSON
|
||||
//////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
static void Dump(benchmark::State& state, const char* filename, int indent)
|
||||
{
|
||||
std::ifstream f(filename);
|
||||
std::string str((std::istreambuf_iterator<char>(f)), std::istreambuf_iterator<char>());
|
||||
json j = json::parse(str);
|
||||
|
||||
while (state.KeepRunning())
|
||||
{
|
||||
j.dump(indent);
|
||||
}
|
||||
|
||||
state.SetBytesProcessed(state.iterations() * j.dump(indent).size());
|
||||
}
|
||||
BENCHMARK_CAPTURE(Dump, jeopardy / -, "data/jeopardy/jeopardy.json", -1);
|
||||
BENCHMARK_CAPTURE(Dump, jeopardy / 4, "data/jeopardy/jeopardy.json", 4);
|
||||
BENCHMARK_CAPTURE(Dump, canada / -, "data/nativejson-benchmark/canada.json", -1);
|
||||
BENCHMARK_CAPTURE(Dump, canada / 4, "data/nativejson-benchmark/canada.json", 4);
|
||||
BENCHMARK_CAPTURE(Dump, citm_catalog / -, "data/nativejson-benchmark/citm_catalog.json", -1);
|
||||
BENCHMARK_CAPTURE(Dump, citm_catalog / 4, "data/nativejson-benchmark/citm_catalog.json", 4);
|
||||
BENCHMARK_CAPTURE(Dump, twitter / -, "data/nativejson-benchmark/twitter.json", -1);
|
||||
BENCHMARK_CAPTURE(Dump, twitter / 4, "data/nativejson-benchmark/twitter.json", 4);
|
||||
BENCHMARK_CAPTURE(Dump, floats / -, "data/numbers/floats.json", -1);
|
||||
BENCHMARK_CAPTURE(Dump, floats / 4, "data/numbers/floats.json", 4);
|
||||
BENCHMARK_CAPTURE(Dump, signed_ints / -, "data/numbers/signed_ints.json", -1);
|
||||
BENCHMARK_CAPTURE(Dump, signed_ints / 4, "data/numbers/signed_ints.json", 4);
|
||||
BENCHMARK_CAPTURE(Dump, unsigned_ints / -, "data/numbers/unsigned_ints.json", -1);
|
||||
BENCHMARK_CAPTURE(Dump, unsigned_ints / 4, "data/numbers/unsigned_ints.json", 4);
|
||||
|
||||
|
||||
BENCHMARK_MAIN();
|
||||
|
|
|
|||
|
|
@ -1,158 +0,0 @@
|
|||
//
|
||||
// benchmarks_simple.cpp -- a less complex version of benchmarks.cpp, that better reflects actual performance
|
||||
//
|
||||
// For some reason, the complexity of benchmarks.cpp doesn't allow
|
||||
// the compiler to optimize code using json.hpp effectively. The
|
||||
// exact same tests, with the use of benchpress and cxxopts produces
|
||||
// much faster code, at least under g++.
|
||||
//
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <chrono>
|
||||
#include <list>
|
||||
#include <tuple>
|
||||
|
||||
#include <json.hpp>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
enum class EMode { input, output, indent };
|
||||
|
||||
static double bench(const EMode mode, size_t iters, const std::string& in_path )
|
||||
{
|
||||
// using string streams for benchmarking to factor-out cold-cache disk
|
||||
// access. Define FROMFILE to use file I/O instead.
|
||||
#if defined( FROMFILE )
|
||||
std::ifstream istr;
|
||||
{
|
||||
istr.open( in_path, std::ifstream::in );
|
||||
|
||||
// read the stream once
|
||||
json j;
|
||||
istr >> j;
|
||||
// clear flags and rewind
|
||||
istr.clear();
|
||||
istr.seekg(0);
|
||||
}
|
||||
#else
|
||||
std::stringstream istr;
|
||||
{
|
||||
// read file into string stream
|
||||
std::ifstream input_file(in_path);
|
||||
istr << input_file.rdbuf();
|
||||
input_file.close();
|
||||
|
||||
// read the stream once
|
||||
json j;
|
||||
istr >> j;
|
||||
// clear flags and rewind
|
||||
istr.clear();
|
||||
istr.seekg(0);
|
||||
}
|
||||
#endif
|
||||
double tps = 0;
|
||||
switch (mode)
|
||||
{
|
||||
// benchmarking input
|
||||
case EMode::input:
|
||||
{
|
||||
auto start = std::chrono::system_clock::now();
|
||||
for (size_t i = 0; i < iters; ++i)
|
||||
{
|
||||
// clear flags and rewind
|
||||
istr.clear();
|
||||
istr.seekg(0);
|
||||
json j;
|
||||
istr >> j;
|
||||
}
|
||||
auto ended = std::chrono::system_clock::now();
|
||||
tps = 1.0 / std::chrono::duration<double>( ended - start ).count();
|
||||
break;
|
||||
}
|
||||
|
||||
// benchmarking output
|
||||
case EMode::output:
|
||||
case EMode::indent:
|
||||
{
|
||||
// create JSON value from input
|
||||
json j;
|
||||
istr >> j;
|
||||
std::stringstream ostr;
|
||||
|
||||
auto start = std::chrono::system_clock::now();
|
||||
for (size_t i = 0; i < iters; ++i)
|
||||
{
|
||||
if (mode == EMode::indent)
|
||||
{
|
||||
ostr << j;
|
||||
}
|
||||
else
|
||||
{
|
||||
ostr << std::setw(4) << j;
|
||||
}
|
||||
|
||||
// reset data
|
||||
ostr.str(std::string());
|
||||
}
|
||||
auto ended = std::chrono::system_clock::now();
|
||||
tps = 1.0 / std::chrono::duration<double>( ended - start ).count();
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
return tps;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
struct average {
|
||||
T _sum { 0 };
|
||||
size_t _count { 0 };
|
||||
T operator+=( const T &val_ ) { _sum += val_; +_count++; return val_; }
|
||||
operator T() { return _sum / _count; }
|
||||
};
|
||||
|
||||
// Execute each test approximately enough times to get near 1
|
||||
// transaction per second, and compute the average; a single aggregate
|
||||
// number that gives a performance metric representing both parsing
|
||||
// and output.
|
||||
|
||||
int main( int, char ** )
|
||||
{
|
||||
std::list<std::tuple<std::string, EMode, size_t, std::string>> tests {
|
||||
{ "parse jeopardy.json", EMode::input, 2, "files/jeopardy/jeopardy.json" },
|
||||
{ "parse canada.json", EMode::input, 30, "files/nativejson-benchmark/canada.json" },
|
||||
{ "parse citm_catalog.json", EMode::input, 120, "files/nativejson-benchmark/citm_catalog.json" },
|
||||
{ "parse twitter.json", EMode::input, 225, "files/nativejson-benchmark/twitter.json" },
|
||||
{ "parse floats.json", EMode::input, 5, "files/numbers/floats.json" },
|
||||
{ "parse signed_ints.json", EMode::input, 6, "files/numbers/signed_ints.json" },
|
||||
{ "parse unsigned_ints.json", EMode::input, 6, "files/numbers/unsigned_ints.json" },
|
||||
{ "dump jeopardy.json", EMode::output, 5, "files/jeopardy/jeopardy.json" },
|
||||
{ "dump jeopardy.json w/ind.", EMode::indent, 5, "files/jeopardy/jeopardy.json" },
|
||||
{ "dump floats.json", EMode::output, 2, "files/numbers/floats.json" },
|
||||
{ "dump signed_ints.json", EMode::output, 20, "files/numbers/signed_ints.json" },
|
||||
};
|
||||
|
||||
average<double> avg;
|
||||
for ( auto t : tests ) {
|
||||
std::string name, path;
|
||||
EMode mode;
|
||||
size_t iters;
|
||||
std::tie(name, mode, iters, path) = t;
|
||||
auto tps = bench( mode, iters, path );
|
||||
avg += tps;
|
||||
std::cout
|
||||
<< std::left
|
||||
<< std::setw( 30 ) << name
|
||||
<< std::right
|
||||
<< " x " << std::setw( 3 ) << iters
|
||||
<< std::left
|
||||
<< " == " << std::setw( 10 ) << tps
|
||||
<< std::right
|
||||
<< " TPS, " << std::setw( 8 ) << std::round( tps * 1e6 / iters )
|
||||
<< " ms/op"
|
||||
<< std::endl;
|
||||
}
|
||||
std::cout << std::setw( 40 ) << "" << std::string( 10, '-' ) << std::endl;
|
||||
std::cout << std::setw( 40 ) << "" << std::setw( 10 ) << std::left << avg << " TPS Average" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue