j3d1/json
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

🚧 manual lexer

Browse source 
This commit removed the re2c lexer and replaced it by a manual version.
Its integration is not yet complete: number parsing does not respect
locales or overflows. Furthermore, parsing does not need to end with
EOF. Therefore, a lot of test cases fail. The idea is to push this
branch forward so we can conduct performance comparisons. So far, a
nice side effect are better diagnosis messages in case of parse errors.
This commit is contained in:
Niels Lohmann 2017-03-24 19:49:02 +01:00
parent 54db53c230
commit 40160f482a
No known key found for this signature in database
GPG key ID: 7F3CEA63AE251B69
9 changed files with 851 additions and 14699 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
Makefile
View file

@ -1,9 +1,5 @@
.PHONY: pretty clean ChangeLog.md .PHONY: pretty clean ChangeLog.md
# used programs
RE2C := $(shell command -v re2c 2> /dev/null)
SED = sed
# main target # main target
all: all:
$(MAKE) -C test $(MAKE) -C test
@ -183,13 +179,6 @@ clang_sanitize: clean
# maintainer targets # maintainer targets
########################################################################## ##########################################################################
# create scanner with re2c
re2c: src/json.hpp.re2c
ifndef RE2C
$(error "re2c is not available, please install re2c")
endif
$(RE2C) -W --utf-8 --encoding-policy fail --bit-vectors --nested-ifs --no-debug-info $< | $(SED) '1d' > src/json.hpp
# pretty printer # pretty printer
pretty: pretty:
astyle --style=allman --indent=spaces=4 --indent-modifiers \ astyle --style=allman --indent=spaces=4 --indent-modifiers \
@ -197,7 +186,7 @@ pretty:
--indent-col1-comments --pad-oper --pad-header --align-pointer=type \ --indent-col1-comments --pad-oper --pad-header --align-pointer=type \
--align-reference=type --add-brackets --convert-tabs --close-templates \ --align-reference=type --add-brackets --convert-tabs --close-templates \
--lineend=linux --preserve-date --suffix=none --formatted \ --lineend=linux --preserve-date --suffix=none --formatted \
src/json.hpp src/json.hpp.re2c test/src/*.cpp \ src/json.hpp test/src/*.cpp \
benchmarks/benchmarks.cpp doc/examples/*.cpp benchmarks/benchmarks.cpp doc/examples/*.cpp

7
errors.txt Normal file
View file

@ -0,0 +1,7 @@
- test/test-class_parser
- 617 failed
- test/test-regression
- 11 failed
- test/test-testsuites
- 43 failed

2409
src/json.hpp
View file

File diff suppressed because it is too large Load diff

12863
src/json.hpp.re2c
View file

File diff suppressed because it is too large Load diff

94
test/src/unit-class_lexer.cpp
View file

@ -38,79 +38,50 @@ TEST_CASE("lexer class")
{ {
SECTION("structural characters") SECTION("structural characters")
{ {
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("["), CHECK((json::lexer("[", 1).scan() == json::lexer::token_type::begin_array));
1).scan() == json::lexer::token_type::begin_array)); CHECK((json::lexer("]", 1).scan() == json::lexer::token_type::end_array));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("]"), CHECK((json::lexer("{", 1).scan() == json::lexer::token_type::begin_object));
1).scan() == json::lexer::token_type::end_array)); CHECK((json::lexer("}", 1).scan() == json::lexer::token_type::end_object));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("{"), CHECK((json::lexer(",", 1).scan() == json::lexer::token_type::value_separator));
1).scan() == json::lexer::token_type::begin_object)); CHECK((json::lexer(":", 1).scan() == json::lexer::token_type::name_separator));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("}"),
1).scan() == json::lexer::token_type::end_object));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>(","),
1).scan() == json::lexer::token_type::value_separator));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>(":"),
1).scan() == json::lexer::token_type::name_separator));
} }
SECTION("literal names") SECTION("literal names")
{ {
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("null"), CHECK((json::lexer("null", 4).scan() == json::lexer::token_type::literal_null));
4).scan() == json::lexer::token_type::literal_null)); CHECK((json::lexer("true", 4).scan() == json::lexer::token_type::literal_true));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("true"), CHECK((json::lexer("false", 5).scan() == json::lexer::token_type::literal_false));
4).scan() == json::lexer::token_type::literal_true));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("false"),
5).scan() == json::lexer::token_type::literal_false));
} }
SECTION("numbers") SECTION("numbers")
{ {
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("0"), CHECK((json::lexer("0", 1).scan() == json::lexer::token_type::value_unsigned));
1).scan() == json::lexer::token_type::value_unsigned)); CHECK((json::lexer("1", 1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("1"), CHECK((json::lexer("2", 1).scan() == json::lexer::token_type::value_unsigned));
1).scan() == json::lexer::token_type::value_unsigned)); CHECK((json::lexer("3", 1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("2"), CHECK((json::lexer("4", 1).scan() == json::lexer::token_type::value_unsigned));
1).scan() == json::lexer::token_type::value_unsigned)); CHECK((json::lexer("5", 1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("3"), CHECK((json::lexer("6", 1).scan() == json::lexer::token_type::value_unsigned));
1).scan() == json::lexer::token_type::value_unsigned)); CHECK((json::lexer("7", 1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("4"), CHECK((json::lexer("8", 1).scan() == json::lexer::token_type::value_unsigned));
1).scan() == json::lexer::token_type::value_unsigned)); CHECK((json::lexer("9", 1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("5"),
1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("6"),
1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("7"),
1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("8"),
1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("9"),
1).scan() == json::lexer::token_type::value_unsigned));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("-0"), CHECK((json::lexer("-0", 2).scan() == json::lexer::token_type::value_integer));
2).scan() == json::lexer::token_type::value_integer)); CHECK((json::lexer("-1", 2).scan() == json::lexer::token_type::value_integer));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("-1"),
2).scan() == json::lexer::token_type::value_integer));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("1.1"), CHECK((json::lexer("1.1", 3).scan() == json::lexer::token_type::value_float));
3).scan() == json::lexer::token_type::value_float)); CHECK((json::lexer("-1.1", 4).scan() == json::lexer::token_type::value_float));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("-1.1"), CHECK((json::lexer("1E10", 4).scan() == json::lexer::token_type::value_float));
4).scan() == json::lexer::token_type::value_float));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("1E10"),
4).scan() == json::lexer::token_type::value_float));
} }
SECTION("whitespace") SECTION("whitespace")
{ {
// result is end_of_input, because not token is following // result is end_of_input, because not token is following
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>(" "), CHECK((json::lexer(" ", 1).scan() == json::lexer::token_type::end_of_input));
1).scan() == json::lexer::token_type::end_of_input)); CHECK((json::lexer("\t", 1).scan() == json::lexer::token_type::end_of_input));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("\t"), CHECK((json::lexer("\n", 1).scan() == json::lexer::token_type::end_of_input));
1).scan() == json::lexer::token_type::end_of_input)); CHECK((json::lexer("\r", 1).scan() == json::lexer::token_type::end_of_input));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("\n"), CHECK((json::lexer(" \t\n\r\n\t ", 7).scan() == json::lexer::token_type::end_of_input));
1).scan() == json::lexer::token_type::end_of_input));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>("\r"),
1).scan() == json::lexer::token_type::end_of_input));
CHECK((json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>(" \t\n\r\n\t "),
7).scan() == json::lexer::token_type::end_of_input));
} }
} }
@ -141,8 +112,7 @@ TEST_CASE("lexer class")
// create string from the ASCII code // create string from the ASCII code
const auto s = std::string(1, static_cast<char>(c)); const auto s = std::string(1, static_cast<char>(c));
// store scan() result // store scan() result
const auto res = json::lexer(reinterpret_cast<const json::lexer::lexer_char_t*>(s.c_str()), const auto res = json::lexer(s.c_str(), 1).scan();
1).scan();
switch (c) switch (c)
{ {
@ -188,12 +158,14 @@ TEST_CASE("lexer class")
} }
} }
/* NOTE: to_unicode function has been removed
SECTION("to_unicode") SECTION("to_unicode")
{ {
// lexer to call to_unicode on // lexer to call to_unicode on
json::lexer dummy_lexer(reinterpret_cast<const json::lexer::lexer_char_t*>(""), 0); json::lexer dummy_lexer("", 0);
CHECK(dummy_lexer.to_unicode(0x1F4A9) == "💩"); CHECK(dummy_lexer.to_unicode(0x1F4A9) == "💩");
CHECK_THROWS_AS(dummy_lexer.to_unicode(0x200000), json::parse_error); CHECK_THROWS_AS(dummy_lexer.to_unicode(0x200000), json::parse_error);
CHECK_THROWS_WITH(dummy_lexer.to_unicode(0x200000), "[json.exception.parse_error.103] parse error: code points above 0x10FFFF are invalid"); CHECK_THROWS_WITH(dummy_lexer.to_unicode(0x200000), "[json.exception.parse_error.103] parse error: code points above 0x10FFFF are invalid");
} }
*/
} }

146
test/src/unit-class_parser.cpp
View file

@ -91,18 +91,18 @@ TEST_CASE("parser class")
// error: tab in string // error: tab in string
CHECK_THROWS_AS(json::parser("\"\t\"").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("\"\t\"").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("\"\t\"").parse(), CHECK_THROWS_WITH(json::parser("\"\t\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
// error: newline in string // error: newline in string
CHECK_THROWS_AS(json::parser("\"\n\"").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("\"\n\"").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("\"\r\"").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("\"\r\"").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("\"\n\"").parse(), CHECK_THROWS_WITH(json::parser("\"\n\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\r\"").parse(), CHECK_THROWS_WITH(json::parser("\"\r\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
// error: backspace in string // error: backspace in string
CHECK_THROWS_AS(json::parser("\"\b\"").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("\"\b\"").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("\"\b\"").parse(), CHECK_THROWS_WITH(json::parser("\"\b\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
// improve code coverage // improve code coverage
CHECK_THROWS_AS(json::parser("\uFF01").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("\uFF01").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("[-4:1,]").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("[-4:1,]").parse(), json::parse_error);
@ -306,39 +306,39 @@ TEST_CASE("parser class")
CHECK_THROWS_AS(json::parser("+0").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("+0").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("01").parse(), CHECK_THROWS_WITH(json::parser("01").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected '01'"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected '01'");
CHECK_THROWS_WITH(json::parser("-01").parse(), CHECK_THROWS_WITH(json::parser("-01").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected '-01'"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected '-01'");
CHECK_THROWS_WITH(json::parser("--1").parse(), CHECK_THROWS_WITH(json::parser("--1").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '-'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("1.").parse(), CHECK_THROWS_WITH(json::parser("1.").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected '.'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("1E").parse(), CHECK_THROWS_WITH(json::parser("1E").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected 'E'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("1E-").parse(), CHECK_THROWS_WITH(json::parser("1E-").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected 'E'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("1.E1").parse(), CHECK_THROWS_WITH(json::parser("1.E1").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected '.'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("-1E").parse(), CHECK_THROWS_WITH(json::parser("-1E").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected 'E'; expected end of input"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0E#").parse(), CHECK_THROWS_WITH(json::parser("-0E#").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected 'E'; expected end of input"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0E-#").parse(), CHECK_THROWS_WITH(json::parser("-0E-#").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected 'E'; expected end of input"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0#").parse(), CHECK_THROWS_WITH(json::parser("-0#").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected '#'; expected end of input"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected '#'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0.0:").parse(), CHECK_THROWS_WITH(json::parser("-0.0:").parse(),
"[json.exception.parse_error.101] parse error at 5: parse error - unexpected ':'; expected end of input"); "[json.exception.parse_error.101] parse error at 5: syntax error - unexpected ':'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0.0Z").parse(), CHECK_THROWS_WITH(json::parser("-0.0Z").parse(),
"[json.exception.parse_error.101] parse error at 5: parse error - unexpected 'Z'; expected end of input"); "[json.exception.parse_error.101] parse error at 5: syntax error - unexpected 'Z'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0E123:").parse(), CHECK_THROWS_WITH(json::parser("-0E123:").parse(),
"[json.exception.parse_error.101] parse error at 7: parse error - unexpected ':'; expected end of input"); "[json.exception.parse_error.101] parse error at 7: syntax error - unexpected ':'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0e0-:").parse(), CHECK_THROWS_WITH(json::parser("-0e0-:").parse(),
"[json.exception.parse_error.101] parse error at 5: parse error - unexpected '-'; expected end of input"); "[json.exception.parse_error.101] parse error at 5: syntax error - unexpected '-'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0e-:").parse(), CHECK_THROWS_WITH(json::parser("-0e-:").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected 'e'; expected end of input"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected 'e'; expected end of input");
CHECK_THROWS_WITH(json::parser("-0f").parse(), CHECK_THROWS_WITH(json::parser("-0f").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected 'f'; expected end of input"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected 'f'; expected end of input");
} }
} }
} }
@ -361,66 +361,66 @@ TEST_CASE("parser class")
CHECK_THROWS_AS(json::parser("1E/").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("1E/").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("1E:").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("1E:").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("0.").parse(), CHECK_THROWS_WITH(json::parser("0.").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected '.'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("-").parse(), CHECK_THROWS_WITH(json::parser("-").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '-'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("--").parse(), CHECK_THROWS_WITH(json::parser("--").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '-'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("-0.").parse(), CHECK_THROWS_WITH(json::parser("-0.").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected '.'; expected end of input"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("-.").parse(), CHECK_THROWS_WITH(json::parser("-.").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '-'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("-:").parse(), CHECK_THROWS_WITH(json::parser("-:").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '-'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '-'");
CHECK_THROWS_WITH(json::parser("0.:").parse(), CHECK_THROWS_WITH(json::parser("0.:").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected '.'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected '.'; expected end of input");
CHECK_THROWS_WITH(json::parser("e.").parse(), CHECK_THROWS_WITH(json::parser("e.").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected 'e'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 'e'");
CHECK_THROWS_WITH(json::parser("1e.").parse(), CHECK_THROWS_WITH(json::parser("1e.").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected 'e'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected 'e'; expected end of input");
CHECK_THROWS_WITH(json::parser("1e/").parse(), CHECK_THROWS_WITH(json::parser("1e/").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected 'e'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected 'e'; expected end of input");
CHECK_THROWS_WITH(json::parser("1e:").parse(), CHECK_THROWS_WITH(json::parser("1e:").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected 'e'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected 'e'; expected end of input");
CHECK_THROWS_WITH(json::parser("1E.").parse(), CHECK_THROWS_WITH(json::parser("1E.").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected 'E'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("1E/").parse(), CHECK_THROWS_WITH(json::parser("1E/").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected 'E'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected 'E'; expected end of input");
CHECK_THROWS_WITH(json::parser("1E:").parse(), CHECK_THROWS_WITH(json::parser("1E:").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected 'E'; expected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected 'E'; expected end of input");
// unexpected end of null // unexpected end of null
CHECK_THROWS_AS(json::parser("n").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("n").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("nu").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("nu").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("nul").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("nul").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("n").parse(), "[json.exception.parse_error.101] parse error at 1: parse error - unexpected 'n'"); CHECK_THROWS_WITH(json::parser("n").parse(), "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 'n'");
CHECK_THROWS_WITH(json::parser("nu").parse(), CHECK_THROWS_WITH(json::parser("nu").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected 'n'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 'n'");
CHECK_THROWS_WITH(json::parser("nul").parse(), CHECK_THROWS_WITH(json::parser("nul").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected 'n'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 'n'");
// unexpected end of true // unexpected end of true
CHECK_THROWS_AS(json::parser("t").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("t").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("tr").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("tr").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("tru").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("tru").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("t").parse(), "[json.exception.parse_error.101] parse error at 1: parse error - unexpected 't'"); CHECK_THROWS_WITH(json::parser("t").parse(), "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 't'");
CHECK_THROWS_WITH(json::parser("tr").parse(), CHECK_THROWS_WITH(json::parser("tr").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected 't'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 't'");
CHECK_THROWS_WITH(json::parser("tru").parse(), CHECK_THROWS_WITH(json::parser("tru").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected 't'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 't'");
// unexpected end of false // unexpected end of false
CHECK_THROWS_AS(json::parser("f").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("f").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("fa").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("fa").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("fal").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("fal").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("fals").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("fals").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("f").parse(), "[json.exception.parse_error.101] parse error at 1: parse error - unexpected 'f'"); CHECK_THROWS_WITH(json::parser("f").parse(), "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 'f'");
CHECK_THROWS_WITH(json::parser("fa").parse(), CHECK_THROWS_WITH(json::parser("fa").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected 'f'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 'f'");
CHECK_THROWS_WITH(json::parser("fal").parse(), CHECK_THROWS_WITH(json::parser("fal").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected 'f'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 'f'");
CHECK_THROWS_WITH(json::parser("fals").parse(), CHECK_THROWS_WITH(json::parser("fals").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected 'f'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected 'f'");
// missing/unexpected end of array // missing/unexpected end of array
CHECK_THROWS_AS(json::parser("[").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("[").parse(), json::parse_error);
@ -429,15 +429,15 @@ TEST_CASE("parser class")
CHECK_THROWS_AS(json::parser("[1,]").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("[1,]").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("]").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("]").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("[").parse(), CHECK_THROWS_WITH(json::parser("[").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected end of input"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected end of input");
CHECK_THROWS_WITH(json::parser("[1").parse(), CHECK_THROWS_WITH(json::parser("[1").parse(),
"[json.exception.parse_error.101] parse error at 3: parse error - unexpected end of input; expected ']'"); "[json.exception.parse_error.101] parse error at 3: syntax error - unexpected end of input; expected ']'");
CHECK_THROWS_WITH(json::parser("[1,").parse(), CHECK_THROWS_WITH(json::parser("[1,").parse(),
"[json.exception.parse_error.101] parse error at 4: parse error - unexpected end of input"); "[json.exception.parse_error.101] parse error at 4: syntax error - unexpected end of input");
CHECK_THROWS_WITH(json::parser("[1,]").parse(), CHECK_THROWS_WITH(json::parser("[1,]").parse(),
"[json.exception.parse_error.101] parse error at 4: parse error - unexpected ']'"); "[json.exception.parse_error.101] parse error at 4: syntax error - unexpected ']'");
CHECK_THROWS_WITH(json::parser("]").parse(), CHECK_THROWS_WITH(json::parser("]").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected ']'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected ']'");
// missing/unexpected end of object // missing/unexpected end of object
CHECK_THROWS_AS(json::parser("{").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("{").parse(), json::parse_error);
@ -447,17 +447,17 @@ TEST_CASE("parser class")
CHECK_THROWS_AS(json::parser("{\"foo\":1,}").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("{\"foo\":1,}").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("}").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("}").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("{").parse(), CHECK_THROWS_WITH(json::parser("{").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected end of input; expected string literal"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected end of input; expected string literal");
CHECK_THROWS_WITH(json::parser("{\"foo\"").parse(), CHECK_THROWS_WITH(json::parser("{\"foo\"").parse(),
"[json.exception.parse_error.101] parse error at 7: parse error - unexpected end of input; expected ':'"); "[json.exception.parse_error.101] parse error at 7: syntax error - unexpected end of input; expected ':'");
CHECK_THROWS_WITH(json::parser("{\"foo\":").parse(), CHECK_THROWS_WITH(json::parser("{\"foo\":").parse(),
"[json.exception.parse_error.101] parse error at 8: parse error - unexpected end of input"); "[json.exception.parse_error.101] parse error at 8: syntax error - unexpected end of input");
CHECK_THROWS_WITH(json::parser("{\"foo\":}").parse(), CHECK_THROWS_WITH(json::parser("{\"foo\":}").parse(),
"[json.exception.parse_error.101] parse error at 8: parse error - unexpected '}'"); "[json.exception.parse_error.101] parse error at 8: syntax error - unexpected '}'");
CHECK_THROWS_WITH(json::parser("{\"foo\":1,}").parse(), CHECK_THROWS_WITH(json::parser("{\"foo\":1,}").parse(),
"[json.exception.parse_error.101] parse error at 10: parse error - unexpected '}'; expected string literal"); "[json.exception.parse_error.101] parse error at 10: syntax error - unexpected '}'; expected string literal");
CHECK_THROWS_WITH(json::parser("}").parse(), CHECK_THROWS_WITH(json::parser("}").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '}'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '}'");
// missing/unexpected end of string // missing/unexpected end of string
CHECK_THROWS_AS(json::parser("\"").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("\"").parse(), json::parse_error);
@ -471,25 +471,25 @@ TEST_CASE("parser class")
CHECK_THROWS_AS(json::parser("\"\\u01").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("\"\\u01").parse(), json::parse_error);
CHECK_THROWS_AS(json::parser("\"\\u012").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("\"\\u012").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("\"").parse(), CHECK_THROWS_WITH(json::parser("\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\\"").parse(), CHECK_THROWS_WITH(json::parser("\"\\\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\u\"").parse(), CHECK_THROWS_WITH(json::parser("\"\\u\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\u0\"").parse(), CHECK_THROWS_WITH(json::parser("\"\\u0\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\u01\"").parse(), CHECK_THROWS_WITH(json::parser("\"\\u01\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\u012\"").parse(), CHECK_THROWS_WITH(json::parser("\"\\u012\"").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\u").parse(), CHECK_THROWS_WITH(json::parser("\"\\u").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\u0").parse(), CHECK_THROWS_WITH(json::parser("\"\\u0").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\u01").parse(), CHECK_THROWS_WITH(json::parser("\"\\u01").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser("\"\\u012").parse(), CHECK_THROWS_WITH(json::parser("\"\\u012").parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
// invalid escapes // invalid escapes
for (int c = 1; c < 128; ++c) for (int c = 1; c < 128; ++c)
@ -523,7 +523,7 @@ TEST_CASE("parser class")
{ {
CHECK_THROWS_AS(json::parser(s.c_str()).parse(), json::parse_error); CHECK_THROWS_AS(json::parser(s.c_str()).parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser(s.c_str()).parse(), CHECK_THROWS_WITH(json::parser(s.c_str()).parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
break; break;
} }
} }
@ -594,13 +594,13 @@ TEST_CASE("parser class")
CHECK_THROWS_AS(json::parser(s4.c_str()).parse(), json::parse_error); CHECK_THROWS_AS(json::parser(s4.c_str()).parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser(s1.c_str()).parse(), CHECK_THROWS_WITH(json::parser(s1.c_str()).parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser(s2.c_str()).parse(), CHECK_THROWS_WITH(json::parser(s2.c_str()).parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser(s3.c_str()).parse(), CHECK_THROWS_WITH(json::parser(s3.c_str()).parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
CHECK_THROWS_WITH(json::parser(s4.c_str()).parse(), CHECK_THROWS_WITH(json::parser(s4.c_str()).parse(),
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected '\"'"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected '\"'");
} }
} }
} }
@ -626,11 +626,11 @@ TEST_CASE("parser class")
// test case to make sure no comma preceeds the first key // test case to make sure no comma preceeds the first key
CHECK_THROWS_AS(json::parser("{,\"key\": false}").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("{,\"key\": false}").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("{,\"key\": false}").parse(), CHECK_THROWS_WITH(json::parser("{,\"key\": false}").parse(),
"[json.exception.parse_error.101] parse error at 2: parse error - unexpected ','"); "[json.exception.parse_error.101] parse error at 2: syntax error - unexpected ','");
// test case to make sure an object is properly closed // test case to make sure an object is properly closed
CHECK_THROWS_AS(json::parser("[{\"key\": false true]").parse(), json::parse_error); CHECK_THROWS_AS(json::parser("[{\"key\": false true]").parse(), json::parse_error);
CHECK_THROWS_WITH(json::parser("[{\"key\": false true]").parse(), CHECK_THROWS_WITH(json::parser("[{\"key\": false true]").parse(),
"[json.exception.parse_error.101] parse error at 19: parse error - unexpected true literal; expected '}'"); "[json.exception.parse_error.101] parse error at 19: syntax error - unexpected true literal; expected '}'");
// test case to make sure the callback is properly evaluated after reading a key // test case to make sure the callback is properly evaluated after reading a key
{ {

10
test/src/unit-deserialization.cpp
View file

@ -92,7 +92,7 @@ TEST_CASE("deserialization")
ss2 << "[\"foo\",1,2,3,false,{\"one\":1}"; ss2 << "[\"foo\",1,2,3,false,{\"one\":1}";
CHECK_THROWS_AS(json::parse(ss1), json::parse_error); CHECK_THROWS_AS(json::parse(ss1), json::parse_error);
CHECK_THROWS_WITH(json::parse(ss2), CHECK_THROWS_WITH(json::parse(ss2),
"[json.exception.parse_error.101] parse error at 30: parse error - unexpected end of input; expected ']'"); "[json.exception.parse_error.101] parse error at 29: syntax error - unexpected end of input; expected ']'");
} }
SECTION("string") SECTION("string")
@ -100,7 +100,7 @@ TEST_CASE("deserialization")
json::string_t s = "[\"foo\",1,2,3,false,{\"one\":1}"; json::string_t s = "[\"foo\",1,2,3,false,{\"one\":1}";
CHECK_THROWS_AS(json::parse(s), json::parse_error); CHECK_THROWS_AS(json::parse(s), json::parse_error);
CHECK_THROWS_WITH(json::parse(s), CHECK_THROWS_WITH(json::parse(s),
"[json.exception.parse_error.101] parse error at 29: parse error - unexpected end of input; expected ']'"); "[json.exception.parse_error.101] parse error at 29: syntax error - unexpected end of input; expected ']'");
} }
SECTION("operator<<") SECTION("operator<<")
@ -111,7 +111,7 @@ TEST_CASE("deserialization")
json j; json j;
CHECK_THROWS_AS(j << ss1, json::parse_error); CHECK_THROWS_AS(j << ss1, json::parse_error);
CHECK_THROWS_WITH(j << ss2, CHECK_THROWS_WITH(j << ss2,
"[json.exception.parse_error.101] parse error at 30: parse error - unexpected end of input; expected ']'"); "[json.exception.parse_error.101] parse error at 29: syntax error - unexpected end of input; expected ']'");
} }
SECTION("operator>>") SECTION("operator>>")
@ -122,14 +122,14 @@ TEST_CASE("deserialization")
json j; json j;
CHECK_THROWS_AS(ss1 >> j, json::parse_error); CHECK_THROWS_AS(ss1 >> j, json::parse_error);
CHECK_THROWS_WITH(ss2 >> j, CHECK_THROWS_WITH(ss2 >> j,
"[json.exception.parse_error.101] parse error at 30: parse error - unexpected end of input; expected ']'"); "[json.exception.parse_error.101] parse error at 29: syntax error - unexpected end of input; expected ']'");
} }
SECTION("user-defined string literal") SECTION("user-defined string literal")
{ {
CHECK_THROWS_AS("[\"foo\",1,2,3,false,{\"one\":1}"_json, json::parse_error); CHECK_THROWS_AS("[\"foo\",1,2,3,false,{\"one\":1}"_json, json::parse_error);
CHECK_THROWS_WITH("[\"foo\",1,2,3,false,{\"one\":1}"_json, CHECK_THROWS_WITH("[\"foo\",1,2,3,false,{\"one\":1}"_json,
"[json.exception.parse_error.101] parse error at 29: parse error - unexpected end of input; expected ']'"); "[json.exception.parse_error.101] parse error at 29: syntax error - unexpected end of input; expected ']'");
} }
} }

4
test/src/unit-regression.cpp
View file

@ -594,7 +594,7 @@ TEST_CASE("regression tests")
// a parse error because of the EOF. // a parse error because of the EOF.
CHECK_THROWS_AS(j << ss, json::parse_error); CHECK_THROWS_AS(j << ss, json::parse_error);
CHECK_THROWS_WITH(j << ss, CHECK_THROWS_WITH(j << ss,
"[json.exception.parse_error.101] parse error at 1: parse error - unexpected end of input"); "[json.exception.parse_error.101] parse error at 1: syntax error - unexpected end of input");
} }
SECTION("issue #389 - Integer-overflow (OSS-Fuzz issue 267)") SECTION("issue #389 - Integer-overflow (OSS-Fuzz issue 267)")
@ -911,6 +911,7 @@ TEST_CASE("regression tests")
CHECK(j["bool_vector"].dump() == "[false,true,false,false]"); CHECK(j["bool_vector"].dump() == "[false,true,false,false]");
} }
/* NOTE: m_line_buffer is not used any more
SECTION("issue #495 - fill_line_buffer incorrectly tests m_stream for eof but not fail or bad bits") SECTION("issue #495 - fill_line_buffer incorrectly tests m_stream for eof but not fail or bad bits")
{ {
SECTION("setting failbit") SECTION("setting failbit")
@ -943,6 +944,7 @@ TEST_CASE("regression tests")
CHECK_THROWS_WITH(l.fill_line_buffer(), "[json.exception.parse_error.111] parse error: bad input stream"); CHECK_THROWS_WITH(l.fill_line_buffer(), "[json.exception.parse_error.111] parse error: bad input stream");
} }
} }
*/
SECTION("issue #504 - assertion error (OSS-Fuzz 856)") SECTION("issue #504 - assertion error (OSS-Fuzz 856)")
{ {

4
test/src/unit-unicode.cpp
View file

@ -36,10 +36,11 @@ using nlohmann::json;
TEST_CASE("Unicode", "[hide]") TEST_CASE("Unicode", "[hide]")
{ {
/* NOTE: to_unicode is not used any more
SECTION("full enumeration of Unicode code points") SECTION("full enumeration of Unicode code points")
{ {
// lexer to call to_unicode on // lexer to call to_unicode on
json::lexer dummy_lexer(reinterpret_cast<const json::lexer::lexer_char_t*>(""), 0); json::lexer dummy_lexer("", 0);
// create an escaped string from a code point // create an escaped string from a code point
const auto codepoint_to_unicode = [](std::size_t cp) const auto codepoint_to_unicode = [](std::size_t cp)
@ -118,6 +119,7 @@ TEST_CASE("Unicode", "[hide]")
CHECK(j3 == j4); CHECK(j3 == j4);
} }
} }
*/
SECTION("read all unicode characters") SECTION("read all unicode characters")
{ {