Merge remote-tracking branch 'pocoproject@github/develop' into develop

Foundation/include/Poco/JSONString.h

@@ -26,16 +26,11 @@
 namespace Poco {
 
 
-std::string Foundation_API toJSON(char c);
-	/// Utility function for escaping JSON characters.
-
-
 void Foundation_API toJSON(const std::string& value, std::ostream& out, bool wrap = true);
 	/// Formats string value into the supplied output stream by
 	/// escaping control characters.
 	/// If wrap is true, the resulting string is enclosed in double quotes
 
-
 std::string Foundation_API toJSON(const std::string& value, bool wrap = true);
 	/// Formats string value by escaping control characters.
 	/// If wrap is true, the resulting string is enclosed in double quotes

Foundation/include/Poco/UTF8String.h

@@ -58,6 +58,20 @@ struct Foundation_API UTF8
 	static void removeBOM(std::string& str);
 		/// Remove the UTF-8 Byte Order Mark sequence (0xEF, 0xBB, 0xBF)
 		/// from the beginning of the string, if it's there.
+
+	static std::string escape(const std::string& s);
+		/// Escapes a string. Special characters like tab, backslash, ... are
+		/// escaped. Unicode characters are escaped to \uxxxx.
+
+	static std::string escape(const std::string::const_iterator& begin, const std::string::const_iterator& end);
+		/// Escapes a string. Special characters like tab, backslash, ... are
+		/// escaped. Unicode characters are escaped to \uxxxx.
+
+	static std::string unescape(const std::string& s);
+		/// Creates an UTF8 string from a string that contains escaped characters.
+
+	static std::string unescape(const std::string::const_iterator& begin, const std::string::const_iterator& end);
+		/// Creates an UTF8 string from a string that contains escaped characters.
 };
 
 

Foundation/src/JSONString.cpp

@@ -13,39 +13,16 @@
 // SPDX-License-Identifier:	BSL-1.0
 //
 
-
 #include "Poco/JSONString.h"
+#include "Poco/UTF8String.h"
 #include <ostream>
 
-
 namespace Poco {
 
-
-std::string toJSON(char c)
-{
-	switch (c)
-	{
-	case '\\': return "\\\\";
-	case '"': return "\\\"";
-	case '/': return "\\/";
-	case '\b': return "\\b";
-	case '\f': return "\\f";
-	case '\n': return "\\n";
-	case '\r': return "\\r";
-	case '\t': return "\\t";
-	default: return std::string(1, c);
-	}
-}
-
-
 void toJSON(const std::string& value, std::ostream& out, bool wrap)
 {
 	if (wrap) out << '"';
-	for (std::string::const_iterator it = value.begin(),
+	out << UTF8::escape(value.begin(), value.end());
-		end = value.end(); it != end; ++it)
-	{
-		out << toJSON(*it);
-	}
 	if (wrap) out << '"';
 }
 
@@ -54,11 +31,7 @@ std::string toJSON(const std::string& value, bool wrap)
 {
 	std::string ret;
 	if (wrap) ret.append(1, '"');
-	for (std::string::const_iterator it = value.begin(),
+	ret.append(UTF8::escape(value.begin(), value.end()));
-		end = value.end(); it != end; ++it)
-	{
-		ret.append(toJSON(*it));
-	}
 	if (wrap) ret.append(1, '"');
 	return ret;
 }

Foundation/src/UTF8String.cpp

@@ -19,6 +19,8 @@
 #include "Poco/TextIterator.h"
 #include "Poco/TextConverter.h"
 #include "Poco/UTF8Encoding.h"
+#include "Poco/NumberFormatter.h"
+#include "Poco/Ascii.h"
 #include <algorithm>
 
 
@@ -171,5 +173,202 @@ void UTF8::removeBOM(std::string& str)
 	}
 }
 
+std::string UTF8::escape(const std::string &s)
+{
+	return escape(s.begin(), s.end());
+}
+
+std::string UTF8::escape(const std::string::const_iterator& begin, const std::string::const_iterator& end)
+{
+	static Poco::UInt32 offsetsFromUTF8[6] = {
+		0x00000000UL, 0x00003080UL, 0x000E2080UL,
+		0x03C82080UL, 0xFA082080UL, 0x82082080UL
+	};
+
+	std::string result;
+
+	std::string::const_iterator it = begin;
+
+	while(it != end)
+	{
+		Poco::UInt32 ch = 0;
+		unsigned int sz = 0;
+
+		do
+		{
+			ch <<= 6;
+			ch += (unsigned char)*it++;
+			sz++;
+		}
+		while (it != end && (*it & 0xC0) == 0x80 && sz < 6);
+		ch -= offsetsFromUTF8[sz-1];
+
+		if (ch == '\n') result += "\\n";
+		else if (ch == '\t') result += "\\t";
+		else if (ch == '\r') result += "\\r";
+		else if (ch == '\b') result += "\\b";
+		else if (ch == '\f') result += "\\f";
+		else if (ch == '\v') result += "\\v";
+		else if (ch == '\a') result += "\\a";
+		else if (ch == '\\') result +=  "\\\\";
+		else if (ch == '\"') result +=  "\\\"";
+		else if (ch == '/') result +=  "\\/";
+		else if (ch == '\0') result += "\\u0000";
+		else if (ch < 32 || ch == 0x7f)
+		{
+			result += "\\u";
+			NumberFormatter::appendHex(result, (unsigned short) ch, 4);
+		}
+		else if (ch > 0xFFFF)
+		{
+			ch -= 0x10000;
+			result += "\\u";
+			NumberFormatter::appendHex(result, (unsigned short) (( ch >> 10 ) & 0x03ff ) + 0xd800, 4);
+			result += "\\u";
+			NumberFormatter::appendHex(result, (unsigned short) (ch & 0x03ff ) + 0xdc00, 4);
+		}
+		else if (ch >= 0x80 && ch <= 0xFFFF)
+		{
+			result += "\\u";
+			NumberFormatter::appendHex(result, (unsigned short) ch, 4);
+		}
+		else
+		{
+			result += (char) ch;
+		}
+	}
+	return result;
+}
+
+std::string UTF8::unescape(const std::string &s)
+{
+	return unescape(s.begin(), s.end());
+}
+
+std::string UTF8::unescape(const std::string::const_iterator& begin, const std::string::const_iterator& end)
+{
+	std::string result;
+
+	std::string::const_iterator it = begin;
+
+	while (it != end)
+	{
+		Poco::UInt32 ch = (Poco::UInt32) *it++;
+
+		if (ch == '\\')
+		{
+			if ( it == end )
+			{
+				//Invalid sequence!
+			}
+
+			if (*it == 'n')
+			{
+				ch = '\n';
+				it++;
+			}
+			else if (*it == 't')
+			{
+				ch = '\t';
+				it++;
+			}
+			else if (*it == 'r')
+			{
+				ch = '\r';
+				it++;
+			}
+			else if (*it == 'b')
+			{
+				ch = '\b';
+				it++;
+			}
+			else if (*it == 'f')
+			{
+				ch = '\f';
+				it++;
+			}
+			else if (*it == 'v')
+			{
+				ch = '\v';
+				it++;
+			}
+			else if (*it == 'a')
+			{
+				ch = '\a';
+				it++;
+			}
+			else if (*it == 'u')
+			{
+				char digs[5];
+				memset(digs, 0, 5);
+				unsigned int dno = 0;
+
+				it++;
+
+				while (it != end && Ascii::isHexDigit(*it) && dno < 4) digs[dno++] = *it++;
+				if (dno > 0)
+				{
+					ch = strtol(digs, NULL, 16);
+				}
+
+				if( ch >= 0xD800 && ch <= 0xDBFF )
+				{
+					if ( it == end || *it != '\\' )
+					{
+						//Invalid sequence!
+					}
+					else
+					{
+						it++;
+						if ( it == end || *it != 'u' )
+						{
+							//Invalid sequence!
+						}
+						else
+						{
+							it++;
+						}
+					}
+
+					// UTF-16 surrogate pair. Go fetch other half
+					memset(digs, 0, 5);
+					dno = 0;
+					while (it != end && Ascii::isHexDigit(*it) && dno < 4) digs[dno++] = *it++;
+					if (dno > 0)
+					{
+						Poco::UInt32 temp = strtol(digs, NULL, 16);
+						if( temp >= 0xDC00 && temp <= 0xDFFF )
+						{
+							ch = ( ( ( ch - 0xD800 ) << 10 ) | ( temp - 0xDC00 ) ) + 0x10000;
+						}
+					}
+				}
+			}
+			else if (*it == 'U')
+			{
+				char digs[9];
+				memset(digs, 0, 9);
+				unsigned int dno = 0;
+
+				it++;
+				while (it != end && Ascii::isHexDigit(*it) && dno < 8)
+				{
+					digs[dno++] = *it++;
+				}
+				if (dno > 0)
+				{
+					ch = strtol(digs, NULL, 16);
+				}
+			}
+		}
+
+		unsigned char utf8[4];
+		UTF8Encoding encoding;
+		int sz = encoding.convert(ch, utf8, 4);
+		result.append((char*) utf8, sz);
+	}
+
+	return result;
+}
 
 } // namespace Poco

Foundation/src/VarHolder.cpp

@@ -16,7 +16,7 @@
 
 #include "Poco/Dynamic/VarHolder.h"
 #include "Poco/Dynamic/Var.h"
-#include "Poco/JSONString.h"
+#include "Poco/UTF8String.h"
 
 
 namespace Poco {
@@ -38,12 +38,7 @@ namespace Impl {
 
 void escape(std::string& target, const std::string& source)
 {
-	std::string::const_iterator it(source.begin());
+	target = UTF8::escape(source.begin(), source.end());
-	std::string::const_iterator end(source.end());
-	for (; it != end; ++it)
-	{
-		target.append(Poco::toJSON(*it));
-	}
 }
 
 
@@ -60,9 +55,11 @@ bool isJSONString(const Var& any)
 
 void appendJSONString(std::string& val, const Var& any)
 {
-	val += '"';
+	std::string json(val);
-	escape(val, any.convert<std::string>());
+	val.append(1, '"');
-	val += '"';
+	escape(json, any.convert<std::string>());
+	val.append(json);
+	val.append(1, '"');
 }
 
 

Foundation/testsuite/src/StringTest.cpp

@@ -1086,15 +1086,15 @@ void StringTest::benchmarkFloatToStr()
 
 void StringTest::testJSONString()
 {
-	assert (toJSON('\\') == "\\\\");
+	assert (toJSON("\\", false) == "\\\\");
-	assert (toJSON('"') == "\\\"");
+	assert (toJSON("\"", false) == "\\\"");
-	assert (toJSON('/') == "\\/");
+	assert (toJSON("/", false) == "\\/");
-	assert (toJSON('\b') == "\\b");
+	assert (toJSON("\b", false) == "\\b");
-	assert (toJSON('\f') == "\\f");
+	assert (toJSON("\f", false) == "\\f");
-	assert (toJSON('\n') == "\\n");
+	assert (toJSON("\n", false) == "\\n");
-	assert (toJSON('\r') == "\\r");
+	assert (toJSON("\r", false) == "\\r");
-	assert (toJSON('\t') == "\\t");
+	assert (toJSON("\t", false) == "\\t");
-	assert (toJSON('a') == "a");
+	assert (toJSON("a", false) == "a");
 
 	// ??? on MSVC, the assert macro expansion
 	// fails to compile when this string is inline ???

JSON/testsuite/src/JSONTest.cpp

@@ -1782,6 +1782,38 @@ void JSONTest::testSmallBuffer()
 	parser.parse(jsonStr);
 }
 
+void JSONTest::testEscape0()
+{
+	Poco::JSON::Object::Ptr json = new Poco::JSON::Object();
+
+	std::string nullString("B\0b", 3);
+	json->set("name", nullString);
+
+	std::stringstream ss;
+	json->stringify(ss);
+
+	assert(ss.str().compare("{\"name\":\"B\\u0000b\"}") == 0);
+}
+
+void JSONTest::testEscapeUnicode()
+{
+	Poco::JSON::Object::Ptr json = new Poco::JSON::Object();
+	std::string chinese("{ \"name\" : \"\\u4e2d\" }");
+	Poco::JSON::Parser parser(new Poco::JSON::ParseHandler());
+	Var result = parser.parse(chinese);
+
+	assert(result.type() == typeid(Object::Ptr));
+
+	Object::Ptr object = result.extract<Object::Ptr>();
+
+	std::stringstream ss;
+	object->stringify(ss);
+
+	//assert(ss.str().compare("{\"name\":\"B\\u0000b\"}") == 0);
+
+	std::cout << ss.str() << std::endl;
+}
+
 
 std::string JSONTest::getTestFilesPath(const std::string& type)
 {
@@ -1798,6 +1830,7 @@ std::string JSONTest::getTestFilesPath(const std::string& type)
 	ostr.str("");
 	ostr << "/JSON/testsuite/data/" << type << '/';
 	validDir = Poco::Environment::get("POCO_BASE") + ostr.str();
+	std::cout << validDir << std::endl;
 	pathPattern = validDir;
 
 	if (Poco::File(pathPattern).exists())
@@ -1855,6 +1888,8 @@ CppUnit::Test* JSONTest::suite()
 	CppUnit_addTest(pSuite, JSONTest, testTemplate);
 	CppUnit_addTest(pSuite, JSONTest, testUnicode);
 	CppUnit_addTest(pSuite, JSONTest, testSmallBuffer);
+	CppUnit_addTest(pSuite, JSONTest, testEscape0);
+	CppUnit_addTest(pSuite, JSONTest, testEscapeUnicode);
 
 	return pSuite;
 }

JSON/testsuite/src/JSONTest.h

@@ -77,7 +77,8 @@ public:
 	void testUnicode();
 	void testInvalidUnicodeJanssonFiles();
 	void testSmallBuffer();
-
+	void testEscape0();
+	void testEscapeUnicode();
 	void setUp();
 	void tearDown();