j3d1/json
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

🔨 rename yytext to token_buffer (fixes #933)

Browse source
This commit is contained in:
Niels Lohmann 2018-01-29 21:56:13 +01:00
parent b3bd3b726b
commit 8049442c2a
No known key found for this signature in database
GPG key ID: 7F3CEA63AE251B69
2 changed files with 38 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
develop/detail/input/lexer.hpp
View file

@ -211,9 +211,10 @@ class lexer
@brief scan a string literal @brief scan a string literal
This function scans a string according to Sect. 7 of RFC 7159. While This function scans a string according to Sect. 7 of RFC 7159. While
scanning, bytes are escaped and copied into buffer yytext. Then the function scanning, bytes are escaped and copied into buffer token_buffer. Then the
returns successfully, yytext is *not* null-terminated (as it may contain \0 function returns successfully, token_buffer is *not* null-terminated (as it
bytes), and yytext.size() is the number of bytes in the string. may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.
@return token_type::value_string if string could be successfully scanned, @return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise token_type::parse_error otherwise
@ -223,7 +224,7 @@ class lexer
*/ */
token_type scan_string() token_type scan_string()
{ {
// reset yytext (ignore opening quote) // reset token_buffer (ignore opening quote)
reset(); reset();
// we entered the function by reading an open quote // we entered the function by reading an open quote
@ -695,7 +696,7 @@ class lexer
contains cycles, but any cycle can be left when EOF is read. Therefore, contains cycles, but any cycle can be left when EOF is read. Therefore,
the function is guaranteed to terminate. the function is guaranteed to terminate.
During scanning, the read bytes are stored in yytext. This string is During scanning, the read bytes are stored in token_buffer. This string is
then converted to a signed integer, an unsigned integer, or a then converted to a signed integer, an unsigned integer, or a
floating-point number. floating-point number.
@ -709,7 +710,7 @@ class lexer
*/ */
token_type scan_number() token_type scan_number()
{ {
// reset yytext to store the number's bytes // reset token_buffer to store the number's bytes
reset(); reset();
// the type of the parsed number; initially set to unsigned; will be // the type of the parsed number; initially set to unsigned; will be
@ -993,10 +994,10 @@ scan_number_done:
// try to parse integers first and fall back to floats // try to parse integers first and fall back to floats
if (number_type == token_type::value_unsigned) if (number_type == token_type::value_unsigned)
{ {
const auto x = std::strtoull(yytext.data(), &endptr, 10); const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
// we checked the number format before // we checked the number format before
assert(endptr == yytext.data() + yytext.size()); assert(endptr == token_buffer.data() + token_buffer.size());
if (errno == 0) if (errno == 0)
{ {
@ -1009,10 +1010,10 @@ scan_number_done:
} }
else if (number_type == token_type::value_integer) else if (number_type == token_type::value_integer)
{ {
const auto x = std::strtoll(yytext.data(), &endptr, 10); const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
// we checked the number format before // we checked the number format before
assert(endptr == yytext.data() + yytext.size()); assert(endptr == token_buffer.data() + token_buffer.size());
if (errno == 0) if (errno == 0)
{ {
@ -1026,10 +1027,10 @@ scan_number_done:
// this code is reached if we parse a floating-point number or if an // this code is reached if we parse a floating-point number or if an
// integer conversion above failed // integer conversion above failed
strtof(value_float, yytext.data(), &endptr); strtof(value_float, token_buffer.data(), &endptr);
// we checked the number format before // we checked the number format before
assert(endptr == yytext.data() + yytext.size()); assert(endptr == token_buffer.data() + token_buffer.size());
return token_type::value_float; return token_type::value_float;
} }
@ -1058,10 +1059,10 @@ scan_number_done:
// input management // input management
///////////////////// /////////////////////
/// reset yytext; current character is beginning of token /// reset token_buffer; current character is beginning of token
void reset() noexcept void reset() noexcept
{ {
yytext.clear(); token_buffer.clear();
token_string.clear(); token_string.clear();
token_string.push_back(std::char_traits<char>::to_char_type(current)); token_string.push_back(std::char_traits<char>::to_char_type(current));
} }
@ -1099,10 +1100,10 @@ scan_number_done:
} }
} }
/// add a character to yytext /// add a character to token_buffer
void add(int c) void add(int c)
{ {
yytext.push_back(std::char_traits<char>::to_char_type(c)); token_buffer.push_back(std::char_traits<char>::to_char_type(c));
} }
public: public:
@ -1131,7 +1132,7 @@ scan_number_done:
/// return current string value (implicitly resets the token; useful only once) /// return current string value (implicitly resets the token; useful only once)
std::string move_string() std::string move_string()
{ {
return std::move(yytext); return std::move(token_buffer);
} }
///////////////////// /////////////////////
@ -1259,7 +1260,7 @@ scan_number_done:
std::vector<char> token_string {}; std::vector<char> token_string {};
/// buffer for variable-length tokens (numbers, strings) /// buffer for variable-length tokens (numbers, strings)
std::string yytext {}; std::string token_buffer {};
/// a description of occurred lexer errors /// a description of occurred lexer errors
const char* error_message = ""; const char* error_message = "";

37
src/json.hpp
View file

@ -2025,9 +2025,10 @@ class lexer
@brief scan a string literal @brief scan a string literal
This function scans a string according to Sect. 7 of RFC 7159. While This function scans a string according to Sect. 7 of RFC 7159. While
scanning, bytes are escaped and copied into buffer yytext. Then the function scanning, bytes are escaped and copied into buffer token_buffer. Then the
returns successfully, yytext is *not* null-terminated (as it may contain \0 function returns successfully, token_buffer is *not* null-terminated (as it
bytes), and yytext.size() is the number of bytes in the string. may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.
@return token_type::value_string if string could be successfully scanned, @return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise token_type::parse_error otherwise
@ -2037,7 +2038,7 @@ class lexer
*/ */
token_type scan_string() token_type scan_string()
{ {
// reset yytext (ignore opening quote) // reset token_buffer (ignore opening quote)
reset(); reset();
// we entered the function by reading an open quote // we entered the function by reading an open quote
@ -2509,7 +2510,7 @@ class lexer
contains cycles, but any cycle can be left when EOF is read. Therefore, contains cycles, but any cycle can be left when EOF is read. Therefore,
the function is guaranteed to terminate. the function is guaranteed to terminate.
During scanning, the read bytes are stored in yytext. This string is During scanning, the read bytes are stored in token_buffer. This string is
then converted to a signed integer, an unsigned integer, or a then converted to a signed integer, an unsigned integer, or a
floating-point number. floating-point number.
@ -2523,7 +2524,7 @@ class lexer
*/ */
token_type scan_number() token_type scan_number()
{ {
// reset yytext to store the number's bytes // reset token_buffer to store the number's bytes
reset(); reset();
// the type of the parsed number; initially set to unsigned; will be // the type of the parsed number; initially set to unsigned; will be
@ -2807,10 +2808,10 @@ scan_number_done:
// try to parse integers first and fall back to floats // try to parse integers first and fall back to floats
if (number_type == token_type::value_unsigned) if (number_type == token_type::value_unsigned)
{ {
const auto x = std::strtoull(yytext.data(), &endptr, 10); const auto x = std::strtoull(token_buffer.data(), &endptr, 10);
// we checked the number format before // we checked the number format before
assert(endptr == yytext.data() + yytext.size()); assert(endptr == token_buffer.data() + token_buffer.size());
if (errno == 0) if (errno == 0)
{ {
@ -2823,10 +2824,10 @@ scan_number_done:
} }
else if (number_type == token_type::value_integer) else if (number_type == token_type::value_integer)
{ {
const auto x = std::strtoll(yytext.data(), &endptr, 10); const auto x = std::strtoll(token_buffer.data(), &endptr, 10);
// we checked the number format before // we checked the number format before
assert(endptr == yytext.data() + yytext.size()); assert(endptr == token_buffer.data() + token_buffer.size());
if (errno == 0) if (errno == 0)
{ {
@ -2840,10 +2841,10 @@ scan_number_done:
// this code is reached if we parse a floating-point number or if an // this code is reached if we parse a floating-point number or if an
// integer conversion above failed // integer conversion above failed
strtof(value_float, yytext.data(), &endptr); strtof(value_float, token_buffer.data(), &endptr);
// we checked the number format before // we checked the number format before
assert(endptr == yytext.data() + yytext.size()); assert(endptr == token_buffer.data() + token_buffer.size());
return token_type::value_float; return token_type::value_float;
} }
@ -2872,10 +2873,10 @@ scan_number_done:
// input management // input management
///////////////////// /////////////////////
/// reset yytext; current character is beginning of token /// reset token_buffer; current character is beginning of token
void reset() noexcept void reset() noexcept
{ {
yytext.clear(); token_buffer.clear();
token_string.clear(); token_string.clear();
token_string.push_back(std::char_traits<char>::to_char_type(current)); token_string.push_back(std::char_traits<char>::to_char_type(current));
} }
@ -2913,10 +2914,10 @@ scan_number_done:
} }
} }
/// add a character to yytext /// add a character to token_buffer
void add(int c) void add(int c)
{ {
yytext.push_back(std::char_traits<char>::to_char_type(c)); token_buffer.push_back(std::char_traits<char>::to_char_type(c));
} }
public: public:
@ -2945,7 +2946,7 @@ scan_number_done:
/// return current string value (implicitly resets the token; useful only once) /// return current string value (implicitly resets the token; useful only once)
std::string move_string() std::string move_string()
{ {
return std::move(yytext); return std::move(token_buffer);
} }
///////////////////// /////////////////////
@ -3073,7 +3074,7 @@ scan_number_done:
std::vector<char> token_string {}; std::vector<char> token_string {};
/// buffer for variable-length tokens (numbers, strings) /// buffer for variable-length tokens (numbers, strings)
std::string yytext {}; std::string token_buffer {};
/// a description of occurred lexer errors /// a description of occurred lexer errors
const char* error_message = ""; const char* error_message = "";