From c3a610d32aaf3dd1022af1d4c952a9b53ca9480d Mon Sep 17 00:00:00 2001
From: Niels Lohmann <niels.lohmann@gmail.com>
Date: Thu, 24 Nov 2016 17:45:01 +0100
Subject: [PATCH 1/2] :wrench: ignoring Clion files

---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index c7e847c4..bd8c8699 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,7 @@ doc/html
 me.nlohmann.json.docset
 
 benchmarks/files/numbers/*.json
+
+.idea
+cmake-build-debug
+

From e3450cac720462ecd6e5b9ae707cbe5023c1e3cc Mon Sep 17 00:00:00 2001
From: Niels Lohmann <niels.lohmann@gmail.com>
Date: Thu, 24 Nov 2016 18:24:26 +0100
Subject: [PATCH 2/2] :chart_with_upwards_trend: removing more variance of the
 benchmarks

---
 benchmarks/benchmarks.cpp | 203 ++++++++++++++++----------------------
 1 file changed, 85 insertions(+), 118 deletions(-)

diff --git a/benchmarks/benchmarks.cpp b/benchmarks/benchmarks.cpp
index 251481ea..efb26cf2 100644
--- a/benchmarks/benchmarks.cpp
+++ b/benchmarks/benchmarks.cpp
@@ -1,11 +1,14 @@
 #define BENCHPRESS_CONFIG_MAIN
 
 #include <fstream>
+#include <sstream>
 #include <benchpress.hpp>
 #include <json.hpp>
 #include <pthread.h>
 #include <thread>
 
+using json = nlohmann::json;
+
 struct StartUp
 {
     StartUp()
@@ -23,129 +26,93 @@ struct StartUp
 };
 StartUp startup;
 
-BENCHMARK("parse jeopardy.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        ctx->stop_timer();
-        std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
-        nlohmann::json j;
-        ctx->start_timer();
-        j << input_file;
-        ctx->stop_timer();
-    }
-})
+enum class EMode { input, output_no_indent, output_with_indent };
 
-BENCHMARK("parse canada.json", [](benchpress::context* ctx)
+static void bench(benchpress::context& ctx,
+                  const std::string& in_path,
+                  const EMode mode)
 {
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    // using string streams for benchmarking to factor-out cold-cache disk
+    // access.
+    std::stringstream istr;
     {
-        ctx->stop_timer();
-        std::ifstream input_file("benchmarks/files/nativejson-benchmark/canada.json");
-        nlohmann::json j;
-        ctx->start_timer();
-        j << input_file;
-        ctx->stop_timer();
-    }
-})
+        // read file into string stream
+        std::ifstream input_file(in_path);
+        istr << input_file.rdbuf();
+        input_file.close();
 
-BENCHMARK("parse citm_catalog.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        ctx->stop_timer();
-        std::ifstream input_file("benchmarks/files/nativejson-benchmark/citm_catalog.json");
-        nlohmann::json j;
-        ctx->start_timer();
-        j << input_file;
-        ctx->stop_timer();
-    }
-})
-
-BENCHMARK("parse twitter.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        ctx->stop_timer();
-        std::ifstream input_file("benchmarks/files/nativejson-benchmark/twitter.json");
-        nlohmann::json j;
-        ctx->start_timer();
-        j << input_file;
-        ctx->stop_timer();
-    }
-})
-
-BENCHMARK("parse numbers/floats.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        ctx->stop_timer();
-        std::ifstream input_file("benchmarks/files/numbers/floats.json");
-        nlohmann::json j;
-        ctx->start_timer();
-        j << input_file;
-        ctx->stop_timer();
-    }
-})
-
-BENCHMARK("parse numbers/signed_ints.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        ctx->stop_timer();
-        std::ifstream input_file("benchmarks/files/numbers/signed_ints.json");
-        nlohmann::json j;
-        ctx->start_timer();
-        j << input_file;
-        ctx->stop_timer();
-    }
-})
-
-BENCHMARK("parse numbers/unsigned_ints.json", [](benchpress::context* ctx)
-{
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        ctx->stop_timer();
-        std::ifstream input_file("benchmarks/files/numbers/unsigned_ints.json");
-        nlohmann::json j;
-        ctx->start_timer();
-        j << input_file;
-        ctx->stop_timer();
-    }
-})
-
-BENCHMARK("dump jeopardy.json", [](benchpress::context* ctx)
-{
-    std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
-    nlohmann::json j;
-    j << input_file;
-    std::ofstream output_file("jeopardy.dump.json");
-
-    ctx->reset_timer();
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
-    {
-        ctx->start_timer();
-        output_file << j;
-        ctx->stop_timer();
+        // read the stream once
+        json j;
+        j << istr;
+        // clear flags and rewind
+        istr.clear();
+        istr.seekg(0);
     }
 
-    std::remove("jeopardy.dump.json");
-})
-
-BENCHMARK("dump jeopardy.json with indent", [](benchpress::context* ctx)
-{
-    std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
-    nlohmann::json j;
-    j << input_file;
-    std::ofstream output_file("jeopardy.dump.json");
-
-    ctx->reset_timer();
-    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    switch (mode)
     {
-        ctx->start_timer();
-        output_file << std::setw(4) << j;
-        ctx->stop_timer();
-    }
+        // benchmarking input
+        case EMode::input:
+        {
+            ctx.reset_timer();
 
-    std::remove("jeopardy.dump.json");
-})
+            for (size_t i = 0; i < ctx.num_iterations(); ++i)
+            {
+				// clear flags and rewind
+                istr.clear();
+                istr.seekg(0);
+                json j;
+                j << istr;
+            }
+
+            break;
+        }
+
+        // benchmarking output
+        case EMode::output_no_indent:
+        case EMode::output_with_indent:
+        {
+            // create JSON value from input
+            json j;
+            j << istr;
+            std::stringstream ostr;
+
+            ctx.reset_timer();
+            for (size_t i = 0; i < ctx.num_iterations(); ++i)
+            {
+                if (mode == EMode::output_no_indent)
+                {
+                    ostr << j;
+                }
+                else
+                {
+                    ostr << std::setw(4) << j;
+                }
+
+                // reset data
+                ostr.str(std::string());
+            }
+
+            break;
+        }
+    }
+}
+
+#define BENCHMARK_I(mode, title, in_path)           \
+    BENCHMARK((title), [](benchpress::context* ctx) \
+    {                                               \
+        bench(*ctx, (in_path), (mode));             \
+    })
+
+BENCHMARK_I(EMode::input, "parse jeopardy.json",              "benchmarks/files/jeopardy/jeopardy.json");
+BENCHMARK_I(EMode::input, "parse canada.json",                "benchmarks/files/nativejson-benchmark/canada.json");
+BENCHMARK_I(EMode::input, "parse citm_catalog.json",          "benchmarks/files/nativejson-benchmark/citm_catalog.json");
+BENCHMARK_I(EMode::input, "parse twitter.json",               "benchmarks/files/nativejson-benchmark/twitter.json");
+BENCHMARK_I(EMode::input, "parse numbers/floats.json",        "benchmarks/files/numbers/floats.json");
+BENCHMARK_I(EMode::input, "parse numbers/signed_ints.json",   "benchmarks/files/numbers/signed_ints.json");
+BENCHMARK_I(EMode::input, "parse numbers/unsigned_ints.json", "benchmarks/files/numbers/unsigned_ints.json");
+
+BENCHMARK_I(EMode::output_no_indent,   "dump jeopardy.json",             "benchmarks/files/jeopardy/jeopardy.json");
+BENCHMARK_I(EMode::output_with_indent, "dump jeopardy.json with indent", "benchmarks/files/jeopardy/jeopardy.json");
+BENCHMARK_I(EMode::output_no_indent,   "dump numbers/floats.json",       "benchmarks/files/numbers/floats.json");
+BENCHMARK_I(EMode::output_no_indent,   "dump numbers/signed_ints.json",  "benchmarks/files/numbers/signed_ints.json");