From 869035a6c9d05e6498bbfaa398b9a68c79831725 Mon Sep 17 00:00:00 2001 From: Niels Date: Sat, 9 May 2015 13:56:51 +0200 Subject: [PATCH] added compliance test cases --- src/json.hpp | 50 ++++++++++++++++----- src/json.hpp.re2c | 29 +++++++++--- test/json_tests/fail1.json | 1 + test/json_tests/fail10.json | 1 + test/json_tests/fail11.json | 1 + test/json_tests/fail12.json | 1 + test/json_tests/fail13.json | 1 + test/json_tests/fail14.json | 1 + test/json_tests/fail15.json | 1 + test/json_tests/fail16.json | 1 + test/json_tests/fail17.json | 1 + test/json_tests/fail18.json | 1 + test/json_tests/fail19.json | 1 + test/json_tests/fail2.json | 1 + test/json_tests/fail20.json | 1 + test/json_tests/fail21.json | 1 + test/json_tests/fail22.json | 1 + test/json_tests/fail23.json | 1 + test/json_tests/fail24.json | 1 + test/json_tests/fail25.json | 1 + test/json_tests/fail26.json | 1 + test/json_tests/fail27.json | 2 + test/json_tests/fail28.json | 2 + test/json_tests/fail29.json | 1 + test/json_tests/fail3.json | 1 + test/json_tests/fail30.json | 1 + test/json_tests/fail31.json | 1 + test/json_tests/fail32.json | 1 + test/json_tests/fail33.json | 1 + test/json_tests/fail4.json | 1 + test/json_tests/fail5.json | 1 + test/json_tests/fail6.json | 1 + test/json_tests/fail7.json | 1 + test/json_tests/fail8.json | 1 + test/json_tests/fail9.json | 1 + test/json_tests/pass1.json | 58 ++++++++++++++++++++++++ test/json_tests/pass2.json | 1 + test/json_tests/pass3.json | 6 +++ test/unit.cpp | 88 ++++++++++++++++++++++++++++++++----- 39 files changed, 240 insertions(+), 27 deletions(-) create mode 100644 test/json_tests/fail1.json create mode 100644 test/json_tests/fail10.json create mode 100644 test/json_tests/fail11.json create mode 100644 test/json_tests/fail12.json create mode 100644 test/json_tests/fail13.json create mode 100644 test/json_tests/fail14.json create mode 100644 test/json_tests/fail15.json create mode 100644 test/json_tests/fail16.json create mode 100644 test/json_tests/fail17.json create mode 100644 test/json_tests/fail18.json create mode 100644 test/json_tests/fail19.json create mode 100644 test/json_tests/fail2.json create mode 100644 test/json_tests/fail20.json create mode 100644 test/json_tests/fail21.json create mode 100644 test/json_tests/fail22.json create mode 100644 test/json_tests/fail23.json create mode 100644 test/json_tests/fail24.json create mode 100644 test/json_tests/fail25.json create mode 100644 test/json_tests/fail26.json create mode 100644 test/json_tests/fail27.json create mode 100644 test/json_tests/fail28.json create mode 100644 test/json_tests/fail29.json create mode 100644 test/json_tests/fail3.json create mode 100644 test/json_tests/fail30.json create mode 100644 test/json_tests/fail31.json create mode 100644 test/json_tests/fail32.json create mode 100644 test/json_tests/fail33.json create mode 100644 test/json_tests/fail4.json create mode 100644 test/json_tests/fail5.json create mode 100644 test/json_tests/fail6.json create mode 100644 test/json_tests/fail7.json create mode 100644 test/json_tests/fail8.json create mode 100644 test/json_tests/fail9.json create mode 100644 test/json_tests/pass1.json create mode 100644 test/json_tests/pass2.json create mode 100644 test/json_tests/pass3.json diff --git a/src/json.hpp b/src/json.hpp index fc0562e8..dfd7bb70 100644 --- a/src/json.hpp +++ b/src/json.hpp @@ -3631,7 +3631,7 @@ class basic_json static const unsigned char yybm[] = { 0, 64, 64, 64, 64, 64, 64, 64, - 64, 96, 96, 64, 64, 96, 64, 64, + 64, 32, 32, 64, 64, 32, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 96, 64, 0, 64, 64, 64, 64, 64, @@ -3914,11 +3914,26 @@ basic_json_parser_25: basic_json_parser_26: yyaccept = 0; yych = *(m_marker = ++m_cursor); - if (yych <= 0x00) + if (yych <= '\n') { + if (yych <= 0x00) + { + goto basic_json_parser_19; + } + if (yych <= 0x08) + { + goto basic_json_parser_31; + } goto basic_json_parser_19; } - goto basic_json_parser_31; + else + { + if (yych == '\r') + { + goto basic_json_parser_19; + } + goto basic_json_parser_31; + } basic_json_parser_27: ++m_cursor; { @@ -3939,7 +3954,7 @@ basic_json_parser_31: { goto basic_json_parser_30; } - if (yych <= 0x00) + if (yych <= '\r') { goto basic_json_parser_32; } @@ -4395,7 +4410,7 @@ basic_json_parser_59: m_buffer.erase(0, static_cast(offset_start)); std::string line; std::getline(*m_stream, line); - m_buffer += line; + m_buffer += "\n" + line; // add line with newline symbol m_content = reinterpret_cast(m_buffer.c_str()); m_start = m_content; @@ -4641,6 +4656,9 @@ basic_json_parser_59: return result; } + // no comma is expected here + unexpect(lexer::token_type::value_separator); + // otherwise: parse key-value pairs do { @@ -4707,6 +4725,9 @@ basic_json_parser_59: return result; } + // no comma is expected here + unexpect(lexer::token_type::value_separator); + // otherwise: parse values do { @@ -4796,11 +4817,8 @@ basic_json_parser_59: default: { - std::string error_msg = "parse error - unexpected \'"; - error_msg += m_lexer.get_token(); - error_msg += "\' ("; - error_msg += lexer::token_type_name(last_token) + ")"; - throw std::invalid_argument(error_msg); + // the last token was unexpected + unexpect(last_token); } } @@ -4830,6 +4848,18 @@ basic_json_parser_59: } } + inline void unexpect(typename lexer::token_type t) const + { + if (t == last_token) + { + std::string error_msg = "parse error - unexpected \'"; + error_msg += m_lexer.get_token(); + error_msg += "\' ("; + error_msg += lexer::token_type_name(last_token) + ")"; + throw std::invalid_argument(error_msg); + } + } + private: /// levels of recursion int depth = 0; diff --git a/src/json.hpp.re2c b/src/json.hpp.re2c index 1ed2af2f..51622f5a 100644 --- a/src/json.hpp.re2c +++ b/src/json.hpp.re2c @@ -3669,7 +3669,7 @@ class basic_json // string quotation_mark = [\"]; escape = [\\]; - unescaped = [^\"\\\000]; + unescaped = [^\"\\\000\t\n\r]; single_escaped = [\"\\/bfnrt]; unicode_escaped = [u][0-9a-fA-F]{4}; escaped = escape (single_escaped | unicode_escaped); @@ -3701,7 +3701,7 @@ class basic_json m_buffer.erase(0, static_cast(offset_start)); std::string line; std::getline(*m_stream, line); - m_buffer += line; + m_buffer += "\n" + line; // add line with newline symbol m_content = reinterpret_cast(m_buffer.c_str()); m_start = m_content; @@ -3947,6 +3947,9 @@ class basic_json return result; } + // no comma is expected here + unexpect(lexer::token_type::value_separator); + // otherwise: parse key-value pairs do { @@ -4013,6 +4016,9 @@ class basic_json return result; } + // no comma is expected here + unexpect(lexer::token_type::value_separator); + // otherwise: parse values do { @@ -4102,11 +4108,8 @@ class basic_json default: { - std::string error_msg = "parse error - unexpected \'"; - error_msg += m_lexer.get_token(); - error_msg += "\' ("; - error_msg += lexer::token_type_name(last_token) + ")"; - throw std::invalid_argument(error_msg); + // the last token was unexpected + unexpect(last_token); } } @@ -4136,6 +4139,18 @@ class basic_json } } + inline void unexpect(typename lexer::token_type t) const + { + if (t == last_token) + { + std::string error_msg = "parse error - unexpected \'"; + error_msg += m_lexer.get_token(); + error_msg += "\' ("; + error_msg += lexer::token_type_name(last_token) + ")"; + throw std::invalid_argument(error_msg); + } + } + private: /// levels of recursion int depth = 0; diff --git a/test/json_tests/fail1.json b/test/json_tests/fail1.json new file mode 100644 index 00000000..6216b865 --- /dev/null +++ b/test/json_tests/fail1.json @@ -0,0 +1 @@ +"A JSON payload should be an object or array, not a string." \ No newline at end of file diff --git a/test/json_tests/fail10.json b/test/json_tests/fail10.json new file mode 100644 index 00000000..5d8c0047 --- /dev/null +++ b/test/json_tests/fail10.json @@ -0,0 +1 @@ +{"Extra value after close": true} "misplaced quoted value" \ No newline at end of file diff --git a/test/json_tests/fail11.json b/test/json_tests/fail11.json new file mode 100644 index 00000000..76eb95b4 --- /dev/null +++ b/test/json_tests/fail11.json @@ -0,0 +1 @@ +{"Illegal expression": 1 + 2} \ No newline at end of file diff --git a/test/json_tests/fail12.json b/test/json_tests/fail12.json new file mode 100644 index 00000000..77580a45 --- /dev/null +++ b/test/json_tests/fail12.json @@ -0,0 +1 @@ +{"Illegal invocation": alert()} \ No newline at end of file diff --git a/test/json_tests/fail13.json b/test/json_tests/fail13.json new file mode 100644 index 00000000..379406b5 --- /dev/null +++ b/test/json_tests/fail13.json @@ -0,0 +1 @@ +{"Numbers cannot have leading zeroes": 013} \ No newline at end of file diff --git a/test/json_tests/fail14.json b/test/json_tests/fail14.json new file mode 100644 index 00000000..0ed366b3 --- /dev/null +++ b/test/json_tests/fail14.json @@ -0,0 +1 @@ +{"Numbers cannot be hex": 0x14} \ No newline at end of file diff --git a/test/json_tests/fail15.json b/test/json_tests/fail15.json new file mode 100644 index 00000000..fc8376b6 --- /dev/null +++ b/test/json_tests/fail15.json @@ -0,0 +1 @@ +["Illegal backslash escape: \x15"] \ No newline at end of file diff --git a/test/json_tests/fail16.json b/test/json_tests/fail16.json new file mode 100644 index 00000000..3fe21d4b --- /dev/null +++ b/test/json_tests/fail16.json @@ -0,0 +1 @@ +[\naked] \ No newline at end of file diff --git a/test/json_tests/fail17.json b/test/json_tests/fail17.json new file mode 100644 index 00000000..62b9214a --- /dev/null +++ b/test/json_tests/fail17.json @@ -0,0 +1 @@ +["Illegal backslash escape: \017"] \ No newline at end of file diff --git a/test/json_tests/fail18.json b/test/json_tests/fail18.json new file mode 100644 index 00000000..edac9271 --- /dev/null +++ b/test/json_tests/fail18.json @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]] \ No newline at end of file diff --git a/test/json_tests/fail19.json b/test/json_tests/fail19.json new file mode 100644 index 00000000..3b9c46fa --- /dev/null +++ b/test/json_tests/fail19.json @@ -0,0 +1 @@ +{"Missing colon" null} \ No newline at end of file diff --git a/test/json_tests/fail2.json b/test/json_tests/fail2.json new file mode 100644 index 00000000..6b7c11e5 --- /dev/null +++ b/test/json_tests/fail2.json @@ -0,0 +1 @@ +["Unclosed array" \ No newline at end of file diff --git a/test/json_tests/fail20.json b/test/json_tests/fail20.json new file mode 100644 index 00000000..27c1af3e --- /dev/null +++ b/test/json_tests/fail20.json @@ -0,0 +1 @@ +{"Double colon":: null} \ No newline at end of file diff --git a/test/json_tests/fail21.json b/test/json_tests/fail21.json new file mode 100644 index 00000000..62474573 --- /dev/null +++ b/test/json_tests/fail21.json @@ -0,0 +1 @@ +{"Comma instead of colon", null} \ No newline at end of file diff --git a/test/json_tests/fail22.json b/test/json_tests/fail22.json new file mode 100644 index 00000000..a7752581 --- /dev/null +++ b/test/json_tests/fail22.json @@ -0,0 +1 @@ +["Colon instead of comma": false] \ No newline at end of file diff --git a/test/json_tests/fail23.json b/test/json_tests/fail23.json new file mode 100644 index 00000000..494add1c --- /dev/null +++ b/test/json_tests/fail23.json @@ -0,0 +1 @@ +["Bad value", truth] \ No newline at end of file diff --git a/test/json_tests/fail24.json b/test/json_tests/fail24.json new file mode 100644 index 00000000..caff239b --- /dev/null +++ b/test/json_tests/fail24.json @@ -0,0 +1 @@ +['single quote'] \ No newline at end of file diff --git a/test/json_tests/fail25.json b/test/json_tests/fail25.json new file mode 100644 index 00000000..8b7ad23e --- /dev/null +++ b/test/json_tests/fail25.json @@ -0,0 +1 @@ +[" tab character in string "] \ No newline at end of file diff --git a/test/json_tests/fail26.json b/test/json_tests/fail26.json new file mode 100644 index 00000000..845d26a6 --- /dev/null +++ b/test/json_tests/fail26.json @@ -0,0 +1 @@ +["tab\ character\ in\ string\ "] \ No newline at end of file diff --git a/test/json_tests/fail27.json b/test/json_tests/fail27.json new file mode 100644 index 00000000..6b01a2ca --- /dev/null +++ b/test/json_tests/fail27.json @@ -0,0 +1,2 @@ +["line +break"] \ No newline at end of file diff --git a/test/json_tests/fail28.json b/test/json_tests/fail28.json new file mode 100644 index 00000000..621a0101 --- /dev/null +++ b/test/json_tests/fail28.json @@ -0,0 +1,2 @@ +["line\ +break"] \ No newline at end of file diff --git a/test/json_tests/fail29.json b/test/json_tests/fail29.json new file mode 100644 index 00000000..47ec421b --- /dev/null +++ b/test/json_tests/fail29.json @@ -0,0 +1 @@ +[0e] \ No newline at end of file diff --git a/test/json_tests/fail3.json b/test/json_tests/fail3.json new file mode 100644 index 00000000..168c81eb --- /dev/null +++ b/test/json_tests/fail3.json @@ -0,0 +1 @@ +{unquoted_key: "keys must be quoted"} \ No newline at end of file diff --git a/test/json_tests/fail30.json b/test/json_tests/fail30.json new file mode 100644 index 00000000..8ab0bc4b --- /dev/null +++ b/test/json_tests/fail30.json @@ -0,0 +1 @@ +[0e+] \ No newline at end of file diff --git a/test/json_tests/fail31.json b/test/json_tests/fail31.json new file mode 100644 index 00000000..1cce602b --- /dev/null +++ b/test/json_tests/fail31.json @@ -0,0 +1 @@ +[0e+-1] \ No newline at end of file diff --git a/test/json_tests/fail32.json b/test/json_tests/fail32.json new file mode 100644 index 00000000..45cba739 --- /dev/null +++ b/test/json_tests/fail32.json @@ -0,0 +1 @@ +{"Comma instead if closing brace": true, \ No newline at end of file diff --git a/test/json_tests/fail33.json b/test/json_tests/fail33.json new file mode 100644 index 00000000..ca5eb19d --- /dev/null +++ b/test/json_tests/fail33.json @@ -0,0 +1 @@ +["mismatch"} \ No newline at end of file diff --git a/test/json_tests/fail4.json b/test/json_tests/fail4.json new file mode 100644 index 00000000..9de168bf --- /dev/null +++ b/test/json_tests/fail4.json @@ -0,0 +1 @@ +["extra comma",] \ No newline at end of file diff --git a/test/json_tests/fail5.json b/test/json_tests/fail5.json new file mode 100644 index 00000000..ddf3ce3d --- /dev/null +++ b/test/json_tests/fail5.json @@ -0,0 +1 @@ +["double extra comma",,] \ No newline at end of file diff --git a/test/json_tests/fail6.json b/test/json_tests/fail6.json new file mode 100644 index 00000000..ed91580e --- /dev/null +++ b/test/json_tests/fail6.json @@ -0,0 +1 @@ +[ , "<-- missing value"] \ No newline at end of file diff --git a/test/json_tests/fail7.json b/test/json_tests/fail7.json new file mode 100644 index 00000000..8a96af3e --- /dev/null +++ b/test/json_tests/fail7.json @@ -0,0 +1 @@ +["Comma after the close"], \ No newline at end of file diff --git a/test/json_tests/fail8.json b/test/json_tests/fail8.json new file mode 100644 index 00000000..b28479c6 --- /dev/null +++ b/test/json_tests/fail8.json @@ -0,0 +1 @@ +["Extra close"]] \ No newline at end of file diff --git a/test/json_tests/fail9.json b/test/json_tests/fail9.json new file mode 100644 index 00000000..5815574f --- /dev/null +++ b/test/json_tests/fail9.json @@ -0,0 +1 @@ +{"Extra comma": true,} \ No newline at end of file diff --git a/test/json_tests/pass1.json b/test/json_tests/pass1.json new file mode 100644 index 00000000..70e26854 --- /dev/null +++ b/test/json_tests/pass1.json @@ -0,0 +1,58 @@ +[ + "JSON Test Pattern pass1", + {"object with 1 member":["array with 1 element"]}, + {}, + [], + -42, + true, + false, + null, + { + "integer": 1234567890, + "real": -9876.543210, + "e": 0.123456789e-12, + "E": 1.234567890E+34, + "": 23456789012E66, + "zero": 0, + "one": 1, + "space": " ", + "quote": "\"", + "backslash": "\\", + "controls": "\b\f\n\r\t", + "slash": "/ & \/", + "alpha": "abcdefghijklmnopqrstuvwyz", + "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", + "digit": "0123456789", + "0123456789": "digit", + "special": "`1~!@#$%^&*()_+-={':[,]}|;.?", + "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", + "true": true, + "false": false, + "null": null, + "array":[ ], + "object":{ }, + "address": "50 St. James Street", + "url": "http://www.JSON.org/", + "comment": "// /* */": " ", + " s p a c e d " :[1,2 , 3 + +, + +4 , 5 , 6 ,7 ],"compact":[1,2,3,4,5,6,7], + "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", + "quotes": "" \u0022 %22 0x22 034 "", + "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" +: "A key can be any string" + }, + 0.5 ,98.6 +, +99.44 +, + +1066, +1e1, +0.1e1, +1e-1, +1e00,2e+00,2e-00 +,"rosebud"] \ No newline at end of file diff --git a/test/json_tests/pass2.json b/test/json_tests/pass2.json new file mode 100644 index 00000000..d3c63c7a --- /dev/null +++ b/test/json_tests/pass2.json @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] \ No newline at end of file diff --git a/test/json_tests/pass3.json b/test/json_tests/pass3.json new file mode 100644 index 00000000..4528d51f --- /dev/null +++ b/test/json_tests/pass3.json @@ -0,0 +1,6 @@ +{ + "JSON Test Pattern pass3": { + "The outermost value": "must be an object or array.", + "In this test": "It is an object." + } +} diff --git a/test/unit.cpp b/test/unit.cpp index 236e1f93..d8f5d2d2 100644 --- a/test/unit.cpp +++ b/test/unit.cpp @@ -7398,16 +7398,17 @@ TEST_CASE("parser class") // exotic test cases for full coverage { - { - std::stringstream ss; - ss << "\"\\u000\n1\""; - CHECK(json::parser(ss).parse().get() == "\x01"); - } - { - std::stringstream ss; - ss << "\"\\u00\n01\""; - CHECK(json::parser(ss).parse().get() == "\x01"); - } + // that one got illegal + //{ + // std::stringstream ss; + // ss << "\"\\u000\n1\""; + // CHECK(json::parser(ss).parse().get() == "\x01"); + //} + //{ + // std::stringstream ss; + // ss << "\"\\u00\n01\""; + // CHECK(json::parser(ss).parse().get() == "\x01"); + //} } CHECK(json::parser("\"\\u0001\"").parse().get() == "\x01"); @@ -8439,6 +8440,73 @@ TEST_CASE("concepts") } } +TEST_CASE("JSON compliance") +{ + // test cases are from http://json.org/JSON_checker/ + + SECTION("expected failures") + { + for (auto filename : + { + //"test/json_tests/fail1.json", + "test/json_tests/fail2.json", + "test/json_tests/fail3.json", + "test/json_tests/fail4.json", + "test/json_tests/fail5.json", + "test/json_tests/fail6.json", + "test/json_tests/fail7.json", + "test/json_tests/fail8.json", + "test/json_tests/fail9.json", + "test/json_tests/fail10.json", + "test/json_tests/fail11.json", + "test/json_tests/fail12.json", + "test/json_tests/fail13.json", + "test/json_tests/fail14.json", + "test/json_tests/fail15.json", + "test/json_tests/fail16.json", + "test/json_tests/fail17.json", + //"test/json_tests/fail18.json", + "test/json_tests/fail19.json", + "test/json_tests/fail20.json", + "test/json_tests/fail21.json", + "test/json_tests/fail22.json", + "test/json_tests/fail23.json", + "test/json_tests/fail24.json", + "test/json_tests/fail25.json", + "test/json_tests/fail26.json", + "test/json_tests/fail27.json", + "test/json_tests/fail28.json", + "test/json_tests/fail29.json", + "test/json_tests/fail30.json", + "test/json_tests/fail31.json", + "test/json_tests/fail32.json", + "test/json_tests/fail33.json" + }) + { + CAPTURE(filename); + json j; + std::ifstream f(filename); + CHECK_THROWS_AS(j << f, std::invalid_argument); + } + } + + SECTION("expected passes") + { + for (auto filename : + { + "test/json_tests/pass1.json", + "test/json_tests/pass2.json", + "test/json_tests/pass3.json" + }) + { + CAPTURE(filename); + json j; + std::ifstream f(filename); + CHECK_NOTHROW(j << f); + } + } +} + TEST_CASE("regression tests") { SECTION("issue #60 - Double quotation mark is not parsed correctly")