Merge branch 'feature/jsontestsuite' into develop

Makefile

@@ -75,7 +75,7 @@ clang_sanitize: clean
 
 # create scanner with re2c
 re2c: src/json.hpp.re2c
-	$(RE2C) -W --bit-vectors --nested-ifs --no-debug-info $< | $(SED) '1d' > src/json.hpp
+	$(RE2C) -W --utf-8 --encoding-policy fail --bit-vectors --nested-ifs --no-debug-info $< | $(SED) '1d' > src/json.hpp
 
 # pretty printer
 pretty:

README.md

@@ -507,6 +507,11 @@ Thanks a lot for helping out!
 
 - The code contains numerous debug **assertions** which can be switched off by defining the preprocessor macro `NDEBUG`, see the [documentation of `assert`](http://en.cppreference.com/w/cpp/error/assert). In particular, note [`operator[]`](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a2e26bd0b0168abb61f67ad5bcd5b9fa1.html#a2e26bd0b0168abb61f67ad5bcd5b9fa1) implements **unchecked access** for const objects: If the given key is not present, the behavior is undefined (think of a dereferenced null pointer) and yields an [assertion failure](https://github.com/nlohmann/json/issues/289) if assertions are switched on. If you are not sure whether an element in an object exists, use checked access with the [`at()` function](https://nlohmann.github.io/json/classnlohmann_1_1basic__json_a674de1ee73e6bf4843fc5dc1351fb726.html#a674de1ee73e6bf4843fc5dc1351fb726).
 - As the exact type of a number is not defined in the [JSON specification](http://rfc7159.net/rfc7159), this library tries to choose the best fitting C++ number type automatically. As a result, the type `double` may be used to store numbers which may yield [**floating-point exceptions**](https://github.com/nlohmann/json/issues/181) in certain rare situations if floating-point exceptions have been unmasked in the calling code. These exceptions are not caused by the library and need to be fixed in the calling code, such as by re-masking the exceptions prior to calling library functions.
+- The library supports **Unicode input** as follows:
+  - Only **UTF-8** encoded input is supported which is the default encoding for JSON according to [RFC 7159](http://rfc7159.net/rfc7159#rfc.section.8.1).
+  - Other encodings such as Latin-1, UTF-16, or UTF-32 are not supported and will yield parse errors.
+  - [Unicode noncharacters](http://www.unicode.org/faq/private_use.html#nonchar1) will not be replaced by the library.
+  - Invalid surrogates (e.g., incomplete pairs such as `\uDEAD`) will yield parse errors.
 
 
 ## Execute unit tests
@@ -517,7 +522,7 @@ To compile and run the tests, you need to execute
 $ make check
 
 ===============================================================================
-All tests passed (8905169 assertions in 35 test cases)
+All tests passed (8905479 assertions in 36 test cases)
 ```
 
 Alternatively, you can use [CMake](https://cmake.org) and run

src/json.hpp.re2c

@@ -7620,6 +7620,14 @@ class basic_json
         {
             // fill buffer
             fill_line_buffer();
+
+            // skip UTF-8 byte-order mark
+            if (m_line_buffer.size() >= 3 and m_line_buffer.substr(0, 3) == "\xEF\xBB\xBF")
+            {
+                m_line_buffer[0] = ' ';
+                m_line_buffer[1] = ' ';
+                m_line_buffer[2] = ' ';
+            }
         }
 
         // switch off unwanted functions (due to pointer members)
@@ -7792,8 +7800,8 @@ class basic_json
                     re2c:define:YYCURSOR     = m_cursor;
                     re2c:define:YYLIMIT      = m_limit;
                     re2c:define:YYMARKER     = m_marker;
-                    re2c:define:YYFILL    = "fill_line_buffer()";
+                    re2c:define:YYFILL       = "fill_line_buffer(@@); // LCOV_EXCL_LINE";
-                    re2c:yyfill:parameter = 0;
+                    re2c:define:YYFILL:naked = 1;
                     re2c:indent:string       = "    ";
                     re2c:indent:top          = 1;
                     re2c:labelprefix         = "basic_json_parser_";
@@ -7802,10 +7810,6 @@ class basic_json
                     ws = [ \t\n\r]+;
                     ws   { continue; }
 
-                    // ignore byte-order-mark
-                    bom = "\xEF\xBB\xBF";
-                    bom   { continue; }
-
                     // structural characters
                     "[" { last_token_type = token_type::begin_array; break; }
                     "]" { last_token_type = token_type::end_array; break; }
@@ -7845,10 +7849,10 @@ class basic_json
                     string          { last_token_type = token_type::value_string; break; }
 
                     // end of file
-                    "\000"         { last_token_type = token_type::end_of_input; break; }
+                    "\x00"         { last_token_type = token_type::end_of_input; break; }
 
                     // anything else is an error
-                    .              { last_token_type = token_type::parse_error; break; }
+                    *              { last_token_type = token_type::parse_error; break; }
                 */
             }
 
@@ -7883,7 +7887,7 @@ class basic_json
             m_start
             m_content
         */
-        void fill_line_buffer()
+        void fill_line_buffer(size_t n = 0)
         {
             // number of processed characters (p)
             const auto offset_start = m_start - m_content;
@@ -7894,6 +7898,9 @@ class basic_json
 
             // no stream is used or end of file is reached
             if (m_stream == nullptr or m_stream->eof())
+            {
+                // skip this part if we are already using the line buffer
+                if (m_start != reinterpret_cast<const lexer_char_t*>(m_line_buffer.data()))
                 {
                     // copy unprocessed characters to line buffer
                     m_line_buffer.clear();
@@ -7901,11 +7908,12 @@ class basic_json
                     {
                         m_line_buffer.append(1, static_cast<const char>(*m_cursor));
                     }
+                }
 
-                // append 5 characters (size of longest keyword "false") to
+                // append n characters to make sure that there is sufficient
-                // make sure that there is sufficient space between m_cursor
+                // space between m_cursor and m_limit
-                // and m_limit
+                m_line_buffer.append(1, '\x00');
-                m_line_buffer.append(5, '\0');
+                m_line_buffer.append(n - 1, '\x01');
             }
             else
             {
@@ -7913,7 +7921,7 @@ class basic_json
                 m_line_buffer.erase(0, static_cast<size_t>(offset_start));
                 // read next line from input stream
                 std::string line;
-                std::getline(*m_stream, line);
+                std::getline(*m_stream, line, '\n');
                 // add line with newline symbol to the line buffer
                 m_line_buffer += line + "\n";
             }
@@ -8075,6 +8083,11 @@ class basic_json
                                 // skip the next 10 characters (xxxx\uyyyy)
                                 i += 10;
                             }
+                            else if (codepoint >= 0xDC00 and codepoint <= 0xDFFF)
+                            {
+                                // we found a lone low surrogate
+                                throw std::invalid_argument("missing high surrogate");
+                            }
                             else
                             {
                                 // add unicode character(s)

test/data/nst_json_testsuite/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Nicolas Seriot
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

test/data/nst_json_testsuite/test_parsing/i_number_neg_int_huge_exp.json

@@ -0,0 +1 @@
+[-1e+9999]

test/data/nst_json_testsuite/test_parsing/i_number_pos_double_huge_exp.json

@@ -0,0 +1 @@
+[1.5e+9999]

test/data/nst_json_testsuite/test_parsing/i_object_key_lone_2nd_surrogate.json

@@ -0,0 +1 @@
+{"\uDFAA":0}

test/data/nst_json_testsuite/test_parsing/i_string_1st_surrogate_but_2nd_missing.json

@@ -0,0 +1 @@
+["\uDADA"]

test/data/nst_json_testsuite/test_parsing/i_string_1st_valid_surrogate_2nd_invalid.json

@@ -0,0 +1 @@
+["\uD888\u1234"]

test/data/nst_json_testsuite/test_parsing/i_string_UTF-16_invalid_lonely_surrogate.json

@@ -0,0 +1 @@
+["\ud800"]

test/data/nst_json_testsuite/test_parsing/i_string_UTF-16_invalid_surrogate.json

@@ -0,0 +1 @@
+["\ud800abc"]

test/data/nst_json_testsuite/test_parsing/i_string_UTF-8_invalid_sequence.json

@@ -0,0 +1 @@
+["譌･ﾑ淫"]

test/data/nst_json_testsuite/test_parsing/i_string_incomplete_surrogate_and_escape_valid.json

@@ -0,0 +1 @@
+["\uD800\n"]

test/data/nst_json_testsuite/test_parsing/i_string_incomplete_surrogate_pair.json

@@ -0,0 +1 @@
+["\uDd1ea"]

test/data/nst_json_testsuite/test_parsing/i_string_incomplete_surrogates_escape_valid.json

@@ -0,0 +1 @@
+["\uD800\uD800\n"]

test/data/nst_json_testsuite/test_parsing/i_string_inverted_surrogates_U+1D11E.json

@@ -0,0 +1 @@
+["\uDd1e\uD834"]

test/data/nst_json_testsuite/test_parsing/i_string_lone_second_surrogate.json

@@ -0,0 +1 @@
+["\uDFAA"]

test/data/nst_json_testsuite/test_parsing/i_string_not_in_unicode_range.json

@@ -0,0 +1 @@
+["<22><><EFBFBD><EFBFBD>"]

test/data/nst_json_testsuite/test_parsing/i_string_truncated-utf-8.json

@@ -0,0 +1 @@
+["<22><>"]

test/data/nst_json_testsuite/test_parsing/i_string_unicode_U+10FFFE_nonchar.json

@@ -0,0 +1 @@
+["\uDBFF\uDFFE"]

test/data/nst_json_testsuite/test_parsing/i_string_unicode_U+1FFFE_nonchar.json

@@ -0,0 +1 @@
+["\uD83F\uDFFE"]

test/data/nst_json_testsuite/test_parsing/i_string_unicode_U+FDD0_nonchar.json

@@ -0,0 +1 @@
+["\uFDD0"]

test/data/nst_json_testsuite/test_parsing/i_string_unicode_U+FFFE_nonchar.json

@@ -0,0 +1 @@
+["\uFFFE"]

test/data/nst_json_testsuite/test_parsing/i_structure_500_nested_arrays.json

@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]

test/data/nst_json_testsuite/test_parsing/i_structure_UTF-8_BOM_empty_object.json

@@ -0,0 +1 @@
+{}

test/data/nst_json_testsuite/test_parsing/n_array_1_true_without_comma.json

@@ -0,0 +1 @@
+[1 true]

test/data/nst_json_testsuite/test_parsing/n_array_a_invalid_utf8.json

@@ -0,0 +1 @@
+[a蘊

test/data/nst_json_testsuite/test_parsing/n_array_colon_instead_of_comma.json

@@ -0,0 +1 @@
+["": 1]

test/data/nst_json_testsuite/test_parsing/n_array_comma_after_close.json

@@ -0,0 +1 @@
+[""],

test/data/nst_json_testsuite/test_parsing/n_array_comma_and_number.json

@@ -0,0 +1 @@
+[,1]

test/data/nst_json_testsuite/test_parsing/n_array_double_comma.json

@@ -0,0 +1 @@
+[1,,2]

test/data/nst_json_testsuite/test_parsing/n_array_double_extra_comma.json

@@ -0,0 +1 @@
+["x",,]

test/data/nst_json_testsuite/test_parsing/n_array_extra_close.json

@@ -0,0 +1 @@
+["x"]]

test/data/nst_json_testsuite/test_parsing/n_array_extra_comma.json

@@ -0,0 +1 @@
+["",]

test/data/nst_json_testsuite/test_parsing/n_array_incomplete.json

@@ -0,0 +1 @@
+["x"

test/data/nst_json_testsuite/test_parsing/n_array_incomplete_invalid_value.json

@@ -0,0 +1 @@
+[x

test/data/nst_json_testsuite/test_parsing/n_array_inner_array_no_comma.json

@@ -0,0 +1 @@
+[3[4]]

test/data/nst_json_testsuite/test_parsing/n_array_invalid_utf8.json

@@ -0,0 +1 @@
+[<EFBFBD>]

test/data/nst_json_testsuite/test_parsing/n_array_items_separated_by_semicolon.json

@@ -0,0 +1 @@
+[1:2]

test/data/nst_json_testsuite/test_parsing/n_array_just_comma.json

@@ -0,0 +1 @@
+[,]

test/data/nst_json_testsuite/test_parsing/n_array_just_minus.json

@@ -0,0 +1 @@
+[-]

test/data/nst_json_testsuite/test_parsing/n_array_missing_value.json

@@ -0,0 +1 @@
+[   , ""]

test/data/nst_json_testsuite/test_parsing/n_array_newlines_unclosed.json

@@ -0,0 +1,3 @@
+["a",
+4
+,1,

test/data/nst_json_testsuite/test_parsing/n_array_number_and_comma.json

@@ -0,0 +1 @@
+[1,]

test/data/nst_json_testsuite/test_parsing/n_array_number_and_several_commas.json

@@ -0,0 +1 @@
+[1,,]

test/data/nst_json_testsuite/test_parsing/n_array_spaces_vertical_tab_formfeed.json

@@ -0,0 +1 @@
+["a"\f]

test/data/nst_json_testsuite/test_parsing/n_array_star_inside.json

@@ -0,0 +1 @@
+[*]

test/data/nst_json_testsuite/test_parsing/n_array_unclosed.json

@@ -0,0 +1 @@
+[""

test/data/nst_json_testsuite/test_parsing/n_array_unclosed_trailing_comma.json

@@ -0,0 +1 @@
+[1,

test/data/nst_json_testsuite/test_parsing/n_array_unclosed_with_new_lines.json

@@ -0,0 +1,3 @@
+[1,
+1
+,1

test/data/nst_json_testsuite/test_parsing/n_array_unclosed_with_object_inside.json

@@ -0,0 +1 @@
+[{}

test/data/nst_json_testsuite/test_parsing/n_incomplete_false.json

@@ -0,0 +1 @@
+[fals]

test/data/nst_json_testsuite/test_parsing/n_incomplete_null.json

@@ -0,0 +1 @@
+[nul]

test/data/nst_json_testsuite/test_parsing/n_incomplete_true.json

@@ -0,0 +1 @@
+[tru]

test/data/nst_json_testsuite/test_parsing/n_number_++.json

@@ -0,0 +1 @@
+[++1234]

test/data/nst_json_testsuite/test_parsing/n_number_+1.json

@@ -0,0 +1 @@
+[+1]

test/data/nst_json_testsuite/test_parsing/n_number_+Inf.json

@@ -0,0 +1 @@
+[+Inf]

test/data/nst_json_testsuite/test_parsing/n_number_-01.json

@@ -0,0 +1 @@
+[-01]

test/data/nst_json_testsuite/test_parsing/n_number_-1.0..json

@@ -0,0 +1 @@
+[-1.0.]

test/data/nst_json_testsuite/test_parsing/n_number_-2..json

@@ -0,0 +1 @@
+[-2.]

test/data/nst_json_testsuite/test_parsing/n_number_-NaN.json

@@ -0,0 +1 @@
+[-NaN]

test/data/nst_json_testsuite/test_parsing/n_number_.-1.json

@@ -0,0 +1 @@
+[.-1]

test/data/nst_json_testsuite/test_parsing/n_number_.2e-3.json

@@ -0,0 +1 @@
+[.2e-3]

test/data/nst_json_testsuite/test_parsing/n_number_0.1.2.json

@@ -0,0 +1 @@
+[0.1.2]

test/data/nst_json_testsuite/test_parsing/n_number_0.3e+.json

@@ -0,0 +1 @@
+[0.3e+]

test/data/nst_json_testsuite/test_parsing/n_number_0.3e.json

@@ -0,0 +1 @@
+[0.3e]

test/data/nst_json_testsuite/test_parsing/n_number_0.e1.json

@@ -0,0 +1 @@
+[0.e1]

test/data/nst_json_testsuite/test_parsing/n_number_0_capital_E+.json

@@ -0,0 +1 @@
+[0E+]

test/data/nst_json_testsuite/test_parsing/n_number_0_capital_E.json

@@ -0,0 +1 @@
+[0E]

test/data/nst_json_testsuite/test_parsing/n_number_0e+.json

@@ -0,0 +1 @@
+[0e+]

test/data/nst_json_testsuite/test_parsing/n_number_0e.json

@@ -0,0 +1 @@
+[0e]

test/data/nst_json_testsuite/test_parsing/n_number_1.0e+.json

@@ -0,0 +1 @@
+[1.0e+]

test/data/nst_json_testsuite/test_parsing/n_number_1.0e-.json

@@ -0,0 +1 @@
+[1.0e-]

test/data/nst_json_testsuite/test_parsing/n_number_1.0e.json

@@ -0,0 +1 @@
+[1.0e]

test/data/nst_json_testsuite/test_parsing/n_number_1_000.json

@@ -0,0 +1 @@
+[1 000.0]

test/data/nst_json_testsuite/test_parsing/n_number_1eE2.json

@@ -0,0 +1 @@
+[1eE2]

test/data/nst_json_testsuite/test_parsing/n_number_2.e+3.json

@@ -0,0 +1 @@
+[2.e+3]

test/data/nst_json_testsuite/test_parsing/n_number_2.e-3.json

@@ -0,0 +1 @@
+[2.e-3]

test/data/nst_json_testsuite/test_parsing/n_number_2.e3.json

@@ -0,0 +1 @@
+[2.e3]

test/data/nst_json_testsuite/test_parsing/n_number_9.e+.json

@@ -0,0 +1 @@
+[9.e+]

test/data/nst_json_testsuite/test_parsing/n_number_Inf.json

@@ -0,0 +1 @@
+[Inf]

test/data/nst_json_testsuite/test_parsing/n_number_NaN.json

@@ -0,0 +1 @@
+[NaN]

test/data/nst_json_testsuite/test_parsing/n_number_U+FF11_fullwidth_digit_one.json

@@ -0,0 +1 @@
+[１]

test/data/nst_json_testsuite/test_parsing/n_number_expression.json

@@ -0,0 +1 @@
+[1+2]

test/data/nst_json_testsuite/test_parsing/n_number_hex_1_digit.json

@@ -0,0 +1 @@
+[0x1]

test/data/nst_json_testsuite/test_parsing/n_number_hex_2_digits.json

@@ -0,0 +1 @@
+[0x42]

test/data/nst_json_testsuite/test_parsing/n_number_infinity.json

@@ -0,0 +1 @@
+[Infinity]

test/data/nst_json_testsuite/test_parsing/n_number_invalid+-.json

@@ -0,0 +1 @@
+[0e+-1]

test/data/nst_json_testsuite/test_parsing/n_number_invalid-negative-real.json

@@ -0,0 +1 @@
+[-123.123foo]

test/data/nst_json_testsuite/test_parsing/n_number_invalid-utf-8-in-bigger-int.json

@@ -0,0 +1 @@
+[123å]

test/data/nst_json_testsuite/test_parsing/n_number_invalid-utf-8-in-exponent.json

@@ -0,0 +1 @@
+[1e1å]

test/data/nst_json_testsuite/test_parsing/n_number_invalid-utf-8-in-int.json

@@ -0,0 +1 @@
+[0å]

test/data/nst_json_testsuite/test_parsing/n_number_minus_infinity.json

@@ -0,0 +1 @@
+[-Infinity]

test/data/nst_json_testsuite/test_parsing/n_number_minus_sign_with_trailing_garbage.json

@@ -0,0 +1 @@
+[-foo]

test/data/nst_json_testsuite/test_parsing/n_number_minus_space_1.json

@@ -0,0 +1 @@
+[- 1]

test/data/nst_json_testsuite/test_parsing/n_number_neg_int_starting_with_zero.json

@@ -0,0 +1 @@
+[-012]

test/data/nst_json_testsuite/test_parsing/n_number_neg_real_without_int_part.json

@@ -0,0 +1 @@
+[-.123]

test/data/nst_json_testsuite/test_parsing/n_number_neg_with_garbage_at_end.json

@@ -0,0 +1 @@
+[-1x]

test/data/nst_json_testsuite/test_parsing/n_number_real_garbage_after_e.json

@@ -0,0 +1 @@
+[1ea]