added Benchpress benchmarks

.gitignore

@@ -2,3 +2,5 @@ json_unit
 html
 
 benchmark
+
+json_benchmarks

Makefile

@@ -9,7 +9,7 @@ all: json_unit
 
 # clean up
 clean:
-	rm -f json_unit
+	rm -f json_unit json_benchmarks
 
 # build unit tests
 json_unit: test/unit.cpp src/json.hpp test/catch.hpp
@@ -30,4 +30,9 @@ pretty:
 	   --indent-col1-comments --pad-oper --pad-header --align-pointer=type \
 	   --align-reference=type --add-brackets --convert-tabs --close-templates \
 	   --lineend=linux --preserve-date --suffix=none \
-	   src/json.hpp src/json.hpp.re2c test/unit.cpp
+	   src/json.hpp src/json.hpp.re2c test/unit.cpp benchmarks/benchmarks.cpp
+
+# benchmarks
+json_benchmarks: benchmarks/benchmarks.cpp benchmarks/benchpress.hpp benchmarks/cxxopts.hpp
+	$(CXX) -std=c++11 $(CXXFLAGS) -O3 -flto -I src -I benchmarks $< $(LDFLAGS) -o $@
+	./json_benchmarks

benchmarks/benchmarks.cpp

@@ -0,0 +1,71 @@
+#define BENCHPRESS_CONFIG_MAIN
+
+#include <fstream>
+#include <benchpress.hpp>
+#include <json.hpp>
+
+BENCHMARK("parse jeopardy.json", [](benchpress::context* ctx)
+{
+    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    {
+        std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
+        nlohmann::json j;
+        j << input_file;
+    }
+})
+
+BENCHMARK("parse canada.json", [](benchpress::context* ctx)
+{
+    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    {
+        std::ifstream input_file("benchmarks/files/nativejson-benchmark/canada.json");
+        nlohmann::json j;
+        j << input_file;
+    }
+})
+
+BENCHMARK("parse citm_catalog.json", [](benchpress::context* ctx)
+{
+    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    {
+        std::ifstream input_file("benchmarks/files/nativejson-benchmark/citm_catalog.json");
+        nlohmann::json j;
+        j << input_file;
+    }
+})
+
+BENCHMARK("parse twitter.json", [](benchpress::context* ctx)
+{
+    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    {
+        std::ifstream input_file("benchmarks/files/nativejson-benchmark/twitter.json");
+        nlohmann::json j;
+        j << input_file;
+    }
+})
+
+BENCHMARK("dump jeopardy.json", [](benchpress::context* ctx)
+{
+    std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
+    nlohmann::json j;
+    j << input_file;
+
+    ctx->reset_timer();
+    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    {
+        j.dump();
+    }
+})
+
+BENCHMARK("dump jeopardy.json with indent", [](benchpress::context* ctx)
+{
+    std::ifstream input_file("benchmarks/files/jeopardy/jeopardy.json");
+    nlohmann::json j;
+    j << input_file;
+
+    ctx->reset_timer();
+    for (size_t i = 0; i < ctx->num_iterations(); ++i)
+    {
+        j.dump(4);
+    }
+})

benchmarks/benchpress.hpp

@@ -0,0 +1,401 @@
+/*
+* Copyright (C) 2015 Christopher Gilbert.
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy
+* of this software and associated documentation files (the "Software"), to deal
+* in the Software without restriction, including without limitation the rights
+* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+* copies of the Software, and to permit persons to whom the Software is
+* furnished to do so, subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in all
+* copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+* SOFTWARE.
+*/
+#ifndef BENCHPRESS_HPP
+#define BENCHPRESS_HPP
+
+#include <algorithm>   // max, min
+#include <atomic>      // atomic_intmax_t
+#include <chrono>      // high_resolution_timer, duration
+#include <functional>  // function
+#include <iomanip>     // setw
+#include <iostream>    // cout
+#include <regex>       // regex, regex_match
+#include <sstream>     // stringstream
+#include <string>      // string
+#include <thread>      // thread
+#include <vector>      // vector
+
+namespace benchpress {
+
+/*
+ * The options class encapsulates all options for running benchmarks.
+ *
+ * When including benchpress, a main function can be emitted which includes a command-line parser for building an
+ * options object. However from time-to-time it may be necessary for the developer to have to build their own main
+ * stub and construct the options object manually.
+ *
+ * options opts;
+ * opts
+ *     .bench(".*")
+ *     .benchtime(1)
+ *     .cpu(4);
+ */
+class options {
+    std::string d_bench;
+    size_t      d_benchtime;
+    size_t      d_cpu;
+public:
+    options()
+        : d_bench(".*")
+        , d_benchtime(1)
+        , d_cpu(std::thread::hardware_concurrency())
+    {}
+    options& bench(const std::string& bench) {
+        d_bench = bench;
+        return *this;
+    }
+    options& benchtime(size_t benchtime) {
+        d_benchtime = benchtime;
+        return *this;
+    }
+    options& cpu(size_t cpu) {
+        d_cpu = cpu;
+        return *this;
+    }
+    std::string get_bench() const {
+        return d_bench;
+    }
+    size_t get_benchtime() const {
+        return d_benchtime;
+    }
+    size_t get_cpu() const {
+        return d_cpu;
+    }
+};
+
+class context;
+
+/*
+ * The benchmark_info class is used to store a function name / pointer pair.
+ *
+ * benchmark_info bi("example", [](benchpress::context* b) {
+ *     // benchmark function
+ * });
+ */
+class benchmark_info {
+    std::string                   d_name;
+    std::function<void(context*)> d_func;
+
+public:
+    benchmark_info(std::string name, std::function<void(context*)> func)
+        : d_name(name)
+        , d_func(func)
+    {}
+
+    std::string                   get_name() const { return d_name; }
+    std::function<void(context*)> get_func() const { return d_func; }
+};
+
+/*
+ * The registration class is responsible for providing a single global point of reference for registering
+ * benchmark functions.
+ *
+ * registration::get_ptr()->register_benchmark(info);
+ */
+class registration {
+    static registration*        d_this;
+    std::vector<benchmark_info> d_benchmarks;
+
+public:
+    static registration* get_ptr() {
+        if (nullptr == d_this) {
+            d_this = new registration();
+        }
+        return d_this;
+    }
+
+    void register_benchmark(benchmark_info& info) {
+        d_benchmarks.push_back(info);
+    }
+
+    std::vector<benchmark_info> get_benchmarks() { return d_benchmarks; }
+};
+
+/*
+ * The auto_register class is a helper used to register benchmarks.
+ */
+class auto_register {
+public:
+    auto_register(const std::string& name, std::function<void(context*)> func) {
+        benchmark_info info(name, func);
+        registration::get_ptr()->register_benchmark(info);
+    }
+};
+
+#define CONCAT(x, y) x ## y
+#define CONCAT2(x, y) CONCAT(x, y)
+
+// The BENCHMARK macro is a helper for creating benchmark functions and automatically registering them with the
+// registration class.
+#define BENCHMARK(x, f) benchpress::auto_register CONCAT2(register_, __LINE__)((x), (f));
+
+// This macro will prevent the compiler from removing a redundant code path which has no side-effects.
+#define DISABLE_REDUNDANT_CODE_OPT() { asm(""); }
+
+/*
+ * The result class is responsible for producing a printable string representation of a benchmark run.
+ */
+class result {
+    size_t                   d_num_iterations;
+    std::chrono::nanoseconds d_duration;
+    size_t                   d_num_bytes;
+
+public:
+    result(size_t num_iterations, std::chrono::nanoseconds duration, size_t num_bytes)
+        : d_num_iterations(num_iterations)
+        , d_duration(duration)
+        , d_num_bytes(num_bytes)
+    {}
+
+    size_t get_ns_per_op() const {
+        if (d_num_iterations <= 0) {
+            return 0;
+        }
+        return d_duration.count() / d_num_iterations;
+    }
+
+    double get_mb_per_s() const {
+        if (d_num_iterations <= 0 || d_duration.count() <= 0 || d_num_bytes <= 0) {
+            return 0;
+        }
+        return ((double(d_num_bytes) * double(d_num_iterations) / double(1e6)) /
+                double(std::chrono::duration_cast<std::chrono::seconds>(d_duration).count()));
+    }
+
+    std::string to_string() const {
+        std::stringstream tmp;
+        tmp << std::setw(12) << std::right << d_num_iterations;
+        size_t npo = get_ns_per_op();
+        tmp << std::setw(12) << std::right << npo << std::setw(0) << " ns/op";
+        double mbs = get_mb_per_s();
+        if (mbs > 0.0) {
+            tmp << std::setw(12) << std::right << mbs << std::setw(0) << " MB/s";
+        }
+        return std::string(tmp.str());
+    }
+};
+
+/*
+ * The parallel_context class is responsible for providing a thread-safe context for parallel benchmark code.
+ */
+class parallel_context {
+    std::atomic_intmax_t d_num_iterations;
+public:
+    parallel_context(size_t num_iterations)
+        : d_num_iterations(num_iterations)
+    {}
+
+    bool next() {
+        return (d_num_iterations.fetch_sub(1) > 0);
+    }
+};
+
+/*
+ * The context class is responsible for providing an interface for capturing benchmark metrics to benchmark functions.
+ */
+class context {
+    bool                                           d_timer_on;
+    std::chrono::high_resolution_clock::time_point d_start;
+    std::chrono::nanoseconds                       d_duration;
+    std::chrono::seconds                           d_benchtime;
+    size_t                                         d_num_iterations;
+    size_t                                         d_num_threads;
+    size_t                                         d_num_bytes;
+    benchmark_info                                 d_benchmark;
+
+public:
+    context(const benchmark_info& info, const options& opts)
+        : d_timer_on(false)
+        , d_start()
+        , d_duration()
+        , d_benchtime(std::chrono::seconds(opts.get_benchtime()))
+        , d_num_iterations(1)
+        , d_num_threads(opts.get_cpu())
+        , d_num_bytes(0)
+        , d_benchmark(info)
+    {}
+
+    size_t num_iterations() const { return d_num_iterations; }
+
+    void set_num_threads(size_t n) { d_num_threads = n; }
+    size_t num_threads() const { return d_num_threads; }
+
+    void start_timer() {
+        if (!d_timer_on) {
+            d_start = std::chrono::high_resolution_clock::now();
+            d_timer_on = true;
+        }
+    }
+    void stop_timer() {
+        if (d_timer_on) {
+            d_duration += std::chrono::high_resolution_clock::now() - d_start;
+            d_timer_on = false;
+        }
+    }
+    void reset_timer() {
+        if (d_timer_on) {
+            d_start = std::chrono::high_resolution_clock::now();
+        }
+        d_duration = std::chrono::nanoseconds::zero();
+    }
+
+    void set_bytes(int64_t bytes) { d_num_bytes = bytes; }
+
+    size_t get_ns_per_op() {
+        if (d_num_iterations <= 0) {
+            return 0;
+        }
+        return d_duration.count() / d_num_iterations;
+    }
+
+    void run_n(size_t n) {
+        d_num_iterations = n;
+        reset_timer();
+        start_timer();
+        d_benchmark.get_func()(this);
+        stop_timer();
+    }
+
+    void run_parallel(std::function<void(parallel_context*)> f) {
+        parallel_context pc(d_num_iterations);
+        std::vector<std::thread> threads;
+        for (size_t i = 0; i < d_num_threads; ++i) {
+            threads.push_back(std::thread([&pc,&f]() -> void {
+                f(&pc);
+            }));
+        }
+        for(auto& thread : threads){
+            thread.join();
+        }
+    }
+
+    result run() {
+        size_t n = 1;
+        run_n(n);
+        while (d_duration < d_benchtime && n < 1e9) {
+            size_t last = n;
+            if (get_ns_per_op() == 0) {
+                n = 1e9;
+            } else {
+                n = d_duration.count() / get_ns_per_op();
+            }
+            n = std::max(std::min(n+n/2, 100*last), last+1);
+            n = round_up(n);
+            run_n(n);
+        }
+        return result(n, d_duration, d_num_bytes);
+    }
+
+private:
+    template<typename T>
+    T round_down_10(T n) {
+        int tens = 0;
+        while (n > 10) {
+            n /= 10;
+            tens++;
+        }
+        int result = 1;
+        for (int i = 0; i < tens; ++i) {
+            result *= 10;
+        }
+        return result;
+    }
+
+    template<typename T>
+    T round_up(T n) {
+        T base = round_down_10(n);
+        if (n < (2 * base)) {
+            return 2 * base;
+        }
+        if (n < (5 * base)) {
+            return 5 * base;
+        }
+        return 10 * base;
+    }
+};
+
+/*
+ * The run_benchmarks function will run the registered benchmarks.
+ */
+void run_benchmarks(const options& opts) {
+    std::regex match_r(opts.get_bench());
+    auto benchmarks = registration::get_ptr()->get_benchmarks();
+    for (auto& info : benchmarks) {
+        if (std::regex_match(info.get_name(), match_r)) {
+            context c(info, opts);
+            auto r = c.run();
+            std::cout << std::setw(35) << std::left << info.get_name() << r.to_string() << std::endl;
+        }
+    }
+}
+
+} // namespace benchpress
+
+/*
+ * If BENCHPRESS_CONFIG_MAIN is defined when the file is included then a main function will be emitted which provides a
+ * command-line parser and then executes run_benchmarks.
+ */
+#ifdef BENCHPRESS_CONFIG_MAIN
+#include "cxxopts.hpp"
+benchpress::registration* benchpress::registration::d_this;
+int main(int argc, char** argv) {
+    std::chrono::high_resolution_clock::time_point bp_start = std::chrono::high_resolution_clock::now();
+    benchpress::options bench_opts;
+    try {
+        cxxopts::Options cmd_opts(argv[0], " - command line options");
+        cmd_opts.add_options()
+            ("bench", "run benchmarks matching the regular expression", cxxopts::value<std::string>()
+                ->default_value(".*"))
+            ("benchtime", "run enough iterations of each benchmark to take t seconds", cxxopts::value<size_t>()
+                ->default_value("1"))
+            ("cpu", "specify the number of threads to use for parallel benchmarks", cxxopts::value<size_t>()
+                ->default_value(std::to_string(std::thread::hardware_concurrency())))
+            ("help", "print help")
+        ;
+        cmd_opts.parse(argc, argv);
+        if (cmd_opts.count("help")) {
+            std::cout << cmd_opts.help({""}) << std::endl;
+            exit(0);
+        }
+        if (cmd_opts.count("bench")) {
+            bench_opts.bench(cmd_opts["bench"].as<std::string>());
+        }
+        if (cmd_opts.count("benchtime")) {
+            bench_opts.benchtime(cmd_opts["benchtime"].as<size_t>());
+        }
+        if (cmd_opts.count("cpu")) {
+            bench_opts.cpu(cmd_opts["cpu"].as<size_t>());
+        }
+    } catch (const cxxopts::OptionException& e) {
+        std::cout << "error parsing options: " << e.what() << std::endl;
+        exit(1);
+    }
+    benchpress::run_benchmarks(bench_opts);
+    float duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+            std::chrono::high_resolution_clock::now() - bp_start
+    ).count() / 1000.f;
+    std::cout << argv[0] << " " << duration << "s" << std::endl;
+    return 0;
+}
+#endif
+
+#endif // BENCHPRESS_HPP