generic-library/rapidjson
1
0
Fork 0
mirror of https://github.com/Tencent/rapidjson.git synced 2025-03-09 03:03:38 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #584 from ludocode/trailing-commas

Browse Source 
Added optional support for trailing commas
This commit is contained in:
Milo Yip 2016-03-21 09:33:59 +08:00
commit ec32200507
3 changed files with 196 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/dom.md
View File

@ -116,6 +116,7 @@ Parse flags | Meaning
`kParseStopWhenDoneFlag` | After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate `kParseErrorDocumentRootNotSingular` error. Using this flag for parsing multiple JSONs in the same stream.
`kParseFullPrecisionFlag` | Parse number in full precision (slower). If this flag is not set, the normal precision (faster) is used. Normal precision has maximum 3 [ULP](http://en.wikipedia.org/wiki/Unit_in_the_last_place) error.
`kParseCommentsFlag` | Allow one-line `// ...` and multi-line `/* ... */` comments (relaxed JSON syntax).
`kParseTrailingCommasFlag` | Allow trailing commas at the end of objects and arrays (relaxed JSON syntax).
By using a non-type template parameter, instead of a function parameter, C++ compiler can generate code which is optimized for specified combinations, improving speed, and reducing code size (if only using a single specialization). The downside is the flags needed to be determined in compile-time.

36
include/rapidjson/reader.h
View File

@ -149,6 +149,7 @@ enum ParseFlag {
kParseFullPrecisionFlag = 16, //!< Parse number in full precision (but slower).
kParseCommentsFlag = 32, //!< Allow one-line (//) and multi-line (/**/) comments.
kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings.
kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays.
kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS
};
@ -636,6 +637,15 @@ private:
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell());
break;
}
if (parseFlags & kParseTrailingCommasFlag) {
if (is.Peek() == '}') {
if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount)))
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
is.Take();
return;
}
}
}
}
@ -676,6 +686,15 @@ private:
}
else
RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell());
if (parseFlags & kParseTrailingCommasFlag) {
if (is.Peek() == ']') {
if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount)))
RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
is.Take();
return;
}
}
}
}
@ -1522,7 +1541,7 @@ private:
IterativeParsingErrorState, // Left bracket
IterativeParsingErrorState, // Right bracket
IterativeParsingErrorState, // Left curly bracket
IterativeParsingErrorState, // Right curly bracket
IterativeParsingObjectFinishState, // Right curly bracket
IterativeParsingErrorState, // Comma
IterativeParsingErrorState, // Colon
IterativeParsingMemberKeyState, // String
@ -1568,7 +1587,7 @@ private:
// ElementDelimiter
{
IterativeParsingArrayInitialState, // Left bracket(push Element state)
IterativeParsingErrorState, // Right bracket
IterativeParsingArrayFinishState, // Right bracket
IterativeParsingObjectInitialState, // Left curly bracket(push Element state)
IterativeParsingErrorState, // Right curly bracket
IterativeParsingErrorState, // Comma
@ -1670,6 +1689,11 @@ private:
case IterativeParsingObjectFinishState:
{
// Transit from delimiter is only allowed when trailing commas are enabled
if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell());
return IterativeParsingErrorState;
}
// Get member count.
SizeType c = *stack_.template Pop<SizeType>(1);
// If the object is not empty, count the last member.
@ -1695,6 +1719,11 @@ private:
case IterativeParsingArrayFinishState:
{
// Transit from delimiter is only allowed when trailing commas are enabled
if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell());
return IterativeParsingErrorState;
}
// Get element count.
SizeType c = *stack_.template Pop<SizeType>(1);
// If the array is not empty, count the last element.
@ -1754,6 +1783,9 @@ private:
case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return;
case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return;
case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return;
case IterativeParsingKeyValueDelimiterState:
case IterativeParsingArrayInitialState:
case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return;
case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return;
default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); return;
}

161
test/unittest/readertest.cpp
View File

@ -778,6 +778,10 @@ TEST(Reader, ParseArray_Error) {
TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1}", 2);
TEST_ARRAY_ERROR(kParseErrorArrayMissCommaOrSquareBracket, "[1 2]", 3);
// Array cannot have a trailing comma (without kParseTrailingCommasFlag);
// a value must follow a comma
TEST_ARRAY_ERROR(kParseErrorValueInvalid, "[1,]", 3);
#undef TEST_ARRAY_ERROR
}
@ -978,6 +982,10 @@ TEST(Reader, ParseObject_Error) {
// Must be a comma or '}' after an object member
TEST_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, "{\"a\":1]", 6);
// Object cannot have a trailing comma (without kParseTrailingCommasFlag);
// an object member name must follow a comma
TEST_ERROR(kParseErrorObjectMissName, "{\"a\":1,}", 7);
// This tests that MemoryStream is checking the length in Peek().
{
MemoryStream ms("{\"a\"", 1);
@ -1119,6 +1127,16 @@ TEST(Reader, IterativeParsing_ErrorHandling) {
TESTERRORHANDLING("{\"a\": 1", kParseErrorObjectMissCommaOrCurlyBracket, 7u);
TESTERRORHANDLING("[1 2 3]", kParseErrorArrayMissCommaOrSquareBracket, 3u);
TESTERRORHANDLING("{\"a: 1", kParseErrorStringMissQuotationMark, 6u);
TESTERRORHANDLING("{\"a\":}", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("{\"a\":]", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("[1,2,}", kParseErrorValueInvalid, 5u);
TESTERRORHANDLING("[}]", kParseErrorValueInvalid, 1u);
TESTERRORHANDLING("[,]", kParseErrorValueInvalid, 1u);
TESTERRORHANDLING("[1,,]", kParseErrorValueInvalid, 3u);
// Trailing commas are not allowed without kParseTrailingCommasFlag
TESTERRORHANDLING("{\"a\": 1,}", kParseErrorObjectMissName, 8u);
TESTERRORHANDLING("[1,2,3,]", kParseErrorValueInvalid, 7u);
// Any JSON value can be a valid root element in RFC7159.
TESTERRORHANDLING("\"ab", kParseErrorStringMissQuotationMark, 3u);
@ -1552,6 +1570,149 @@ TEST(Reader, NumbersAsStrings) {
}
}
template <unsigned extraFlags>
void TestTrailingCommas() {
{
StringStream s("[1,2,3,]");
ParseArrayHandler<3> h;
Reader reader;
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(5u, h.step_);
}
{
const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false,"
"\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3],}";
StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}
{
// whitespace around trailing commas
const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false,"
"\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3\n,\n]\n,\n} ";
StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}
{
// comments around trailing commas
const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null,"
"\"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3/*test*/,/*test*/]/*test*/,/*test*/}";
StringStream s(json);
ParseObjectHandler h;
Reader reader;
EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag|kParseCommentsFlag>(s, h));
EXPECT_EQ(20u, h.step_);
}
}
TEST(Reader, TrailingCommas) {
TestTrailingCommas<kParseNoFlags>();
}
TEST(Reader, TrailingCommasIterative) {
TestTrailingCommas<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestMultipleTrailingCommaErrors() {
// only a single trailing comma is allowed.
{
StringStream s("[1,2,3,,]");
ParseArrayHandler<3> h;
Reader reader;
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorValueInvalid, r.Code());
EXPECT_EQ(7u, r.Offset());
}
{
const char* json = "{ \"hello\" : \"world\", \"t\" : true , \"f\" : false,"
"\"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3,],,}";
StringStream s(json);
ParseObjectHandler h;
Reader reader;
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorObjectMissName, r.Code());
EXPECT_EQ(95u, r.Offset());
}
}
TEST(Reader, MultipleTrailingCommaErrors) {
TestMultipleTrailingCommaErrors<kParseNoFlags>();
}
TEST(Reader, MultipleTrailingCommaErrorsIterative) {
TestMultipleTrailingCommaErrors<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestEmptyExceptForCommaErrors() {
// not allowed even with trailing commas enabled; the
// trailing comma must follow a value.
{
StringStream s("[,]");
ParseArrayHandler<3> h;
Reader reader;
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorValueInvalid, r.Code());
EXPECT_EQ(1u, r.Offset());
}
{
StringStream s("{,}");
ParseObjectHandler h;
Reader reader;
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorObjectMissName, r.Code());
EXPECT_EQ(1u, r.Offset());
}
}
TEST(Reader, EmptyExceptForCommaErrors) {
TestEmptyExceptForCommaErrors<kParseNoFlags>();
}
TEST(Reader, EmptyExceptForCommaErrorsIterative) {
TestEmptyExceptForCommaErrors<kParseIterativeFlag>();
}
template <unsigned extraFlags>
void TestTrailingCommaHandlerTermination() {
{
HandlerTerminateAtEndArray h;
Reader reader;
StringStream s("[1,2,3,]");
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorTermination, r.Code());
EXPECT_EQ(7u, r.Offset());
}
{
HandlerTerminateAtEndObject h;
Reader reader;
StringStream s("{\"t\": true, \"f\": false,}");
ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorTermination, r.Code());
EXPECT_EQ(23u, r.Offset());
}
}
TEST(Reader, TrailingCommaHandlerTermination) {
TestTrailingCommaHandlerTermination<kParseNoFlags>();
}
TEST(Reader, TrailingCommaHandlerTerminationIterative) {
TestTrailingCommaHandlerTermination<kParseIterativeFlag>();
}
#ifdef __GNUC__
RAPIDJSON_DIAG_POP
#endif