From 62126278a605f8ec097f2420dfa717d83912cc70 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 25 Oct 2018 13:01:18 +0200 Subject: [PATCH] :hammer: added fix for arrays --- README.md | 16 +++++++-- .../nlohmann/detail/output/binary_writer.hpp | 7 ++-- single_include/nlohmann/json.hpp | 7 ++-- test/data/json.org/1.json.bson | Bin 393 -> 395 bytes test/data/json.org/2.json.bson | Bin 216 -> 219 bytes test/data/json.org/3.json.bson | Bin 406 -> 406 bytes test/data/json.org/4.json.bson | Bin 2786 -> 2791 bytes test/data/json.org/5.json.bson | Bin 730 -> 764 bytes test/src/unit-bson.cpp | 32 +++++++++++------- 9 files changed, 43 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index eebf86df..0572199f 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ - [JSON Merge Patch](#json-merge-patch) - [Implicit conversions](#implicit-conversions) - [Conversions to/from arbitrary types](#arbitrary-types-conversions) - - [Binary formats (CBOR, MessagePack, and UBJSON)](#binary-formats-cbor-messagepack-and-ubjson) + - [Binary formats (CBOR, BSON, MessagePack, and UBJSON)](#binary-formats-bson-cbor-messagepack-and-ubjson) - [Supported compilers](#supported-compilers) - [License](#license) - [Contact](#contact) @@ -874,14 +874,22 @@ struct bad_serializer }; ``` -### Binary formats (CBOR, MessagePack, and UBJSON) +### Binary formats (CBOR, BSON, MessagePack, and UBJSON -Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [CBOR](http://cbor.io) (Concise Binary Object Representation), [MessagePack](http://msgpack.org), and [UBJSON](http://ubjson.org) (Universal Binary JSON Specification) to efficiently encode JSON values to byte vectors and to decode such vectors. +Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [BSON](http://bsonspec.org) (Binary JSON), [CBOR](http://cbor.io) (Concise Binary Object Representation), [MessagePack](http://msgpack.org), and [UBJSON](http://ubjson.org) (Universal Binary JSON Specification) to efficiently encode JSON values to byte vectors and to decode such vectors. ```cpp // create a JSON value json j = R"({"compact": true, "schema": 0})"_json; +// serialize to BSON +std::vector v_bson = json::to_bson(j); + +// 0x1B, 0x00, 0x00, 0x00, 0x08, 0x63, 0x6F, 0x6D, 0x70, 0x61, 0x63, 0x74, 0x00, 0x01, 0x10, 0x73, 0x63, 0x68, 0x65, 0x6D, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + +// roundtrip +json j_from_bson = json::from_bson(v_bson); + // serialize to CBOR std::vector v_cbor = json::to_cbor(j); @@ -1138,6 +1146,8 @@ I deeply appreciate the help of the following people. - [Henry Schreiner](https://github.com/henryiii) added support for GCC 4.8. - [knilch](https://github.com/knilch0r) made sure the test suite does not stall when run in the wrong directory. - [Antonio Borondo](https://github.com/antonioborondo) fixed an MSVC 2017 warning. +- [efp](https://github.com/efp) added line and column information to parse errors. +- [julian-becker](https://github.com/julian-becker) added BSON support. Thanks a lot for helping out! Please [let me know](mailto:mail@nlohmann.me) if I forgot someone. diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 64a02aac..27833ab6 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -875,10 +875,11 @@ class binary_writer static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) { std::size_t embedded_document_size = 0ul; + std::size_t array_index = 0ul; for (const auto& el : value) { - embedded_document_size += calc_bson_element_size("", el); + embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el); } return sizeof(std::int32_t) + embedded_document_size + 1ul; @@ -893,9 +894,11 @@ class binary_writer write_bson_entry_header(name, 0x04); // array write_number(static_cast(calc_bson_array_size(value))); + std::size_t array_index = 0ul; + for (const auto& el : value) { - write_bson_element("", el); + write_bson_element(std::to_string(array_index++), el); } oa->write_character(static_cast(0x00)); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index cdd4e680..c9060781 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -9155,10 +9155,11 @@ class binary_writer static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) { std::size_t embedded_document_size = 0ul; + std::size_t array_index = 0ul; for (const auto& el : value) { - embedded_document_size += calc_bson_element_size("", el); + embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el); } return sizeof(std::int32_t) + embedded_document_size + 1ul; @@ -9173,9 +9174,11 @@ class binary_writer write_bson_entry_header(name, 0x04); // array write_number(static_cast(calc_bson_array_size(value))); + std::size_t array_index = 0ul; + for (const auto& el : value) { - write_bson_element("", el); + write_bson_element(std::to_string(array_index++), el); } oa->write_character(static_cast(0x00)); diff --git a/test/data/json.org/1.json.bson b/test/data/json.org/1.json.bson index e14e9b92304d8a37a803af2e6c870496679aefd6..a2466af8367caa6a96e7881eabd1657453354e81 100644 GIT binary patch delta 196 zcmeBV?q(L~W@KPsPS43NE>0|}WT;`BoXIF>$?OgiaLFuV2n0$qm1LIWq%truFfar& zK%{&!i%S?xKw7~nUGqwSnw1$DCfhK|Pv&Ga6>v;0%FnCJWncwqa`*L_XeZ4BH8nMj zp^;(YyigYwu-@R*RL7j+dWMp7qPS43NE>0|}WT*o1+(A5-%rXXlIL{}uxP-w7&UMWz0V-BxWSDqb z+Lg&MxhOxcGM9msfq@~|-PZ?ZLTVaAJy3`RY+i6`s$))ZK7%Asgo%L#NP<*CSP{NH j6F-GcE?^X!*k(F0QClx3UG;*SV9FPV8K(#6# diff --git a/test/data/json.org/2.json.bson b/test/data/json.org/2.json.bson index 0f4356e16daadc3fd1e5d602ac5b01a9637b0747..743822c57eda0cce5a56fc1a738786be582d0016 100644 GIT binary patch literal 219 zcmcc3z`($qo0?b3Z~@3-%1mKk1=4AmIjIaxWr;bZsbIbvh|gS*Ur<`WupB7Q0@9IL zlA6mf1Ek-8!2qNgqMilF^h+&gV9L)+&dE&9X5a@3I2WZRmZSoOUGkGPG=W+S8FUa@ pKvwz}q~;;?a6v_2%8eNG5X#wrOy`{ZVxSXYc5uUlpc;T&1^{E^FsuLo literal 216 zcmcb?z`($qo0?b3a2Cj7%1mKk1=4AmIjIcH1^ES~1q@4od=`+3%#zexhN&R+3Qmu!qnj6S*&dD!MMN!TM69v*R`#?sxft=33008zvFbx0z diff --git a/test/data/json.org/3.json.bson b/test/data/json.org/3.json.bson index deb7c5391fc76590baf686212ee5a4407c404dc6..2c43e3518d29c74c0d79c3b3d32e04655251c97e 100644 GIT binary patch literal 406 zcmZ`!u};G<5Ir0L14v|N@xTBb+H^#Si3NmEhYG=fgq--|SU7eRUnu1d_yPWczhh$J zOgm5sw)gJ6d++Xj3m{uqZIIxNvCvo!1~LE$J}F*@1>7?gWXT~MGMuZnb11KauT?oX zMWL5>s|?{)i?Gk8R$2eJ-p{##8u;Xbyg_Rk`oWOdRvCnc6d(Hv4j5deQJl;NKkmGN zRp2%Yaqv@KpVf8bW50NkKmQ^^Ict22o?w@?iUIcg9N`PW2Tz1nq+pNHg|pR@yu=6l0*h>pbqDp%;anZkJO@6 z1}rwP0abu>8BUI5w3@^yr3yDO1t`P7T%MVil3&gc0b~ngq-LgPlrVf@WSHnLK2b}J OPoO+Ar6dC+00IE2Z#VV; diff --git a/test/data/json.org/4.json.bson b/test/data/json.org/4.json.bson index 31812125a1c55004e0cc4ca87c8e67ce553d59ba..aceba2d8ab6f741a77fac37908bc4fabf4612ce9 100644 GIT binary patch delta 570 zcmY*WJ#Q015S_cd5Bo%jm0%o2QBXdlASY)#9|$C*fJt0{ECIGDNQ2h7_2uw=akmbp zfiC@O3OYbYRQv`CC}p}Jdfbj_LIaqT_>R)RO~uCN$`F6*7w|y8ieL2^ ze6LU9MXijMwlbC*r0hof7Jzy`jd4?(CE0mTD+_-?++7rni9&}3u}?WmI|<$GgCl+~ zi|-TSuqfV4i`S*GZ7<=ET8ug%{o%acz(3kLy85~CZ1^_iw9TW0fr4k%WqhmE@I0Bv zJ@pd4B0aoD0(_^g;tO>O3#5c~b;)iQ3N=?NM+*)-w{+QId8@0|uug((`;a zs=|R;kz+QXrk)6NqR3D2Wq#wols_ndPx1@+EZ4$c<_?nlV{GMao{sw0!>uUfY{)^D zd@|=sncshkf!wASx^R%?WnIRCbp{_;(^u5ps7Hs#QXtn0Ri3F<%d(YI*Y~>6%#5Gh V?Lp#YiNkSIh5HUZwJJmce*tQkrPu%f delta 519 zcmY+A&ubGw9K~m6H?uz|ZD?5xLL(`Owv|n_1gaELf5d~LRGNxNy^PIR*X)n5JKDs9 zh-X2=I|zc{e-Qryz4o98T0{^py+{#nq8^=X+JlGr@#gX6z4?51cij0MhY)>J-12y* zL$;MLQjy#bgd|^W<&ZX0yC3<|>+pn!WP;%bt>6niiY+=mx*^hrpUB>d;4R|dpj(WmU8ZV}QS)ca3r{hwW2H)c_1$MDHGg)vsZc5V@CtmwA5DVBJea;~Qs3A)yk=FrHxG?m2)Q3*hMLBJO^uvwg?`i=vcx;HfV#=pzW>~;;(=Lo zl{`WJ=lem!XS;+~)*5>DGFGhRu^^7y7bC?&imoIY{#cJuwMy7EYbtsYf9=Wfy|lfO lb36;4o5iTJm|uPr|Jqo{MvFQ^1+HZ+;Ml2M<@&~)+*Mr|fTv&qa%K+1fw9+M%Hp~d8Q cCQTVbO9o~m0~F03sW}Be%|?ck=P}s;0KuUt)&Kwi delta 205 zcmeyvdW)6&783&lb8c#0DZ}}R+=`NGKwJhP1_lPE%oGMzAnjj}ng zF3_HM#?%F expected = { - 0x41, 0x00, 0x00, 0x00, // size (little endian) + 0x49, 0x00, 0x00, 0x00, // size (little endian) 0x04, /// entry: embedded document 'e', 'n', 't', 'r', 'y', '\x00', - 0x35, 0x00, 0x00, 0x00, // size (little endian) - 0x10, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x02, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x03, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x05, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x06, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x07, 0x00, 0x00, 0x00, - 0x10, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x3D, 0x00, 0x00, 0x00, // size (little endian) + 0x10, '0', 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, '1', 0x00, 0x02, 0x00, 0x00, 0x00, + 0x10, '2', 0x00, 0x03, 0x00, 0x00, 0x00, + 0x10, '3', 0x00, 0x04, 0x00, 0x00, 0x00, + 0x10, '4', 0x00, 0x05, 0x00, 0x00, 0x00, + 0x10, '5', 0x00, 0x06, 0x00, 0x00, 0x00, + 0x10, '6', 0x00, 0x07, 0x00, 0x00, 0x00, + 0x10, '7', 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, // end marker (embedded document) 0x00 // end marker @@ -552,6 +552,7 @@ TEST_CASE("BSON") CHECK(parsed == expected); auto dumped = json::to_bson(parsed); CHECK(dumped == input); + CHECK(json::from_bson(dumped) == expected); } SECTION("Example 2") @@ -561,7 +562,7 @@ TEST_CASE("BSON") json expected = {{"BSON", {"awesome", 5.05, 1986}}}; CHECK(parsed == expected); auto dumped = json::to_bson(parsed); - //CHECK(dumped == input); // see https://github.com/nlohmann/json/pull/1254#issuecomment-432831216 + CHECK(dumped == input); CHECK(json::from_bson(dumped) == expected); } } @@ -1225,7 +1226,14 @@ TEST_CASE("BSON roundtrips", "[hide]") { std::vector vec; json::to_bson(j1, vec); - CHECK(vec == packed); + + if (vec != packed) + { + // the exact serializations may differ due to the order of + // object keys; in these cases, just compare whether both + // serializations create the same JSON value + CHECK(json::from_bson(vec) == json::from_bson(packed)); + } } } }