From e3450cac720462ecd6e5b9ae707cbe5023c1e3cc Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 24 Nov 2016 18:24:26 +0100 Subject: [PATCH] :chart_with_upwards_trend: removing more variance of the benchmarks --- benchmarks/benchmarks.cpp | 203 ++++++++++++++++---------------------- 1 file changed, 85 insertions(+), 118 deletions(-) diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp index 251481ea..efb26cf2 100644 --- a/benchmarks/benchmarks.cpp +++ b/benchmarks/benchmarks.cpp @@ -1,11 +1,14 @@ #define BENCHPRESS_CONFIG_MAIN #include +#include #include #include #include #include +using json = nlohmann::json; + struct StartUp { StartUp() @@ -23,129 +26,93 @@ struct StartUp }; StartUp startup; -BENCHMARK("parse jeopardy.json", [](benchpress::context* ctx) -{ - for (size_t i = 0; i < ctx->num_iterations(); ++i) - { - ctx->stop_timer(); - std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json"); - nlohmann::json j; - ctx->start_timer(); - j << input_file; - ctx->stop_timer(); - } -}) +enum class EMode { input, output_no_indent, output_with_indent }; -BENCHMARK("parse canada.json", [](benchpress::context* ctx) +static void bench(benchpress::context& ctx, + const std::string& in_path, + const EMode mode) { - for (size_t i = 0; i < ctx->num_iterations(); ++i) + // using string streams for benchmarking to factor-out cold-cache disk + // access. + std::stringstream istr; { - ctx->stop_timer(); - std::ifstream input_file("benchmarks/files/nativejson-benchmark/canada.json"); - nlohmann::json j; - ctx->start_timer(); - j << input_file; - ctx->stop_timer(); - } -}) + // read file into string stream + std::ifstream input_file(in_path); + istr << input_file.rdbuf(); + input_file.close(); -BENCHMARK("parse citm_catalog.json", [](benchpress::context* ctx) -{ - for (size_t i = 0; i < ctx->num_iterations(); ++i) - { - ctx->stop_timer(); - std::ifstream input_file("benchmarks/files/nativejson-benchmark/citm_catalog.json"); - nlohmann::json j; - ctx->start_timer(); - j << input_file; - ctx->stop_timer(); - } -}) - -BENCHMARK("parse twitter.json", [](benchpress::context* ctx) -{ - for (size_t i = 0; i < ctx->num_iterations(); ++i) - { - ctx->stop_timer(); - std::ifstream input_file("benchmarks/files/nativejson-benchmark/twitter.json"); - nlohmann::json j; - ctx->start_timer(); - j << input_file; - ctx->stop_timer(); - } -}) - -BENCHMARK("parse numbers/floats.json", [](benchpress::context* ctx) -{ - for (size_t i = 0; i < ctx->num_iterations(); ++i) - { - ctx->stop_timer(); - std::ifstream input_file("benchmarks/files/numbers/floats.json"); - nlohmann::json j; - ctx->start_timer(); - j << input_file; - ctx->stop_timer(); - } -}) - -BENCHMARK("parse numbers/signed_ints.json", [](benchpress::context* ctx) -{ - for (size_t i = 0; i < ctx->num_iterations(); ++i) - { - ctx->stop_timer(); - std::ifstream input_file("benchmarks/files/numbers/signed_ints.json"); - nlohmann::json j; - ctx->start_timer(); - j << input_file; - ctx->stop_timer(); - } -}) - -BENCHMARK("parse numbers/unsigned_ints.json", [](benchpress::context* ctx) -{ - for (size_t i = 0; i < ctx->num_iterations(); ++i) - { - ctx->stop_timer(); - std::ifstream input_file("benchmarks/files/numbers/unsigned_ints.json"); - nlohmann::json j; - ctx->start_timer(); - j << input_file; - ctx->stop_timer(); - } -}) - -BENCHMARK("dump jeopardy.json", [](benchpress::context* ctx) -{ - std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json"); - nlohmann::json j; - j << input_file; - std::ofstream output_file("jeopardy.dump.json"); - - ctx->reset_timer(); - for (size_t i = 0; i < ctx->num_iterations(); ++i) - { - ctx->start_timer(); - output_file << j; - ctx->stop_timer(); + // read the stream once + json j; + j << istr; + // clear flags and rewind + istr.clear(); + istr.seekg(0); } - std::remove("jeopardy.dump.json"); -}) - -BENCHMARK("dump jeopardy.json with indent", [](benchpress::context* ctx) -{ - std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json"); - nlohmann::json j; - j << input_file; - std::ofstream output_file("jeopardy.dump.json"); - - ctx->reset_timer(); - for (size_t i = 0; i < ctx->num_iterations(); ++i) + switch (mode) { - ctx->start_timer(); - output_file << std::setw(4) << j; - ctx->stop_timer(); - } + // benchmarking input + case EMode::input: + { + ctx.reset_timer(); - std::remove("jeopardy.dump.json"); -}) + for (size_t i = 0; i < ctx.num_iterations(); ++i) + { + // clear flags and rewind + istr.clear(); + istr.seekg(0); + json j; + j << istr; + } + + break; + } + + // benchmarking output + case EMode::output_no_indent: + case EMode::output_with_indent: + { + // create JSON value from input + json j; + j << istr; + std::stringstream ostr; + + ctx.reset_timer(); + for (size_t i = 0; i < ctx.num_iterations(); ++i) + { + if (mode == EMode::output_no_indent) + { + ostr << j; + } + else + { + ostr << std::setw(4) << j; + } + + // reset data + ostr.str(std::string()); + } + + break; + } + } +} + +#define BENCHMARK_I(mode, title, in_path) \ + BENCHMARK((title), [](benchpress::context* ctx) \ + { \ + bench(*ctx, (in_path), (mode)); \ + }) + +BENCHMARK_I(EMode::input, "parse jeopardy.json", "benchmarks/files/jeopardy/jeopardy.json"); +BENCHMARK_I(EMode::input, "parse canada.json", "benchmarks/files/nativejson-benchmark/canada.json"); +BENCHMARK_I(EMode::input, "parse citm_catalog.json", "benchmarks/files/nativejson-benchmark/citm_catalog.json"); +BENCHMARK_I(EMode::input, "parse twitter.json", "benchmarks/files/nativejson-benchmark/twitter.json"); +BENCHMARK_I(EMode::input, "parse numbers/floats.json", "benchmarks/files/numbers/floats.json"); +BENCHMARK_I(EMode::input, "parse numbers/signed_ints.json", "benchmarks/files/numbers/signed_ints.json"); +BENCHMARK_I(EMode::input, "parse numbers/unsigned_ints.json", "benchmarks/files/numbers/unsigned_ints.json"); + +BENCHMARK_I(EMode::output_no_indent, "dump jeopardy.json", "benchmarks/files/jeopardy/jeopardy.json"); +BENCHMARK_I(EMode::output_with_indent, "dump jeopardy.json with indent", "benchmarks/files/jeopardy/jeopardy.json"); +BENCHMARK_I(EMode::output_no_indent, "dump numbers/floats.json", "benchmarks/files/numbers/floats.json"); +BENCHMARK_I(EMode::output_no_indent, "dump numbers/signed_ints.json", "benchmarks/files/numbers/signed_ints.json");