added support for C++14/17 number literals

CppParser/include/Poco/CppParser/CppToken.h

@@ -33,7 +33,7 @@ class CppParser_API CppToken: public Poco::Token
 public:
 	CppToken();
 	~CppToken();
-	
+
 protected:
 	void syntaxError(const std::string& expected, const std::string& actual);
 };
@@ -66,7 +66,7 @@ public:
 		OP_BITAND_ASSIGN,   // &=
 		OP_AND,             // &&
 		OP_BITOR,           // |
-		OP_BITOR_ASSIGN,    // |= 
+		OP_BITOR_ASSIGN,    // |=
 		OP_OR,              // ||
 		OP_XOR,             // ^
 		OP_XOR_ASSIGN,      // ^=
@@ -92,17 +92,17 @@ public:
 		OP_SEMICOLON,       // ;
 		OP_QUESTION         // ?
 	};
-	
+
 	OperatorToken();
 	~OperatorToken();
 	Poco::Token::Class tokenClass() const;
 	bool start(char c, std::istream& istr);
 	void finish(std::istream& istr);
 	int asInteger() const;
-	
+
 private:
 	typedef std::map<std::string, int> OpMap;
-	
+
 	OpMap _opMap;
 };
 
@@ -197,17 +197,17 @@ public:
 		KW_XOR,
 		KW_XOR_EQ
 	};
-	
+
 	IdentifierToken();
 	~IdentifierToken();
 	Poco::Token::Class tokenClass() const;
 	bool start(char c, std::istream& istr);
 	void finish(std::istream& istr);
 	int asInteger() const;
-	
+
 private:
 	typedef std::map<std::string, int> KWMap;
-	
+
 	KWMap _kwMap;
 };
 
@@ -244,9 +244,16 @@ public:
 	Poco::Token::Class tokenClass() const;
 	bool start(char c, std::istream& istr);
 	void finish(std::istream& istr);
+
 	int asInteger() const;
 	double asFloat() const;
-	
+
+protected:
+	void finishHex(std::istream& istr, int next);
+	void finishBin(std::istream& istr, int next);
+	void finishExp(std::istream& istr, int next);
+	void finishSuffix(std::istream& istr, int next);
+
 private:
 	bool _isFloat;
 };

CppParser/src/CppToken.cpp

@@ -96,7 +96,7 @@ OperatorToken::OperatorToken()
 	_opMap[":"] = i++;
 	_opMap["::"] = i++;
 	_opMap[";"] = i++;
-	_opMap["?"] = i++;	 
+	_opMap["?"] = i++;
 }
 
 
@@ -345,7 +345,7 @@ void IdentifierToken::finish(std::istream& istr)
 	while ((next >= 'A' && next <= 'Z') ||
 		   (next >= 'a' && next <= 'z') ||
 		   (next >= '0' && next <= '9') ||
-		   (next == '_' || next == '$')) 
+		   (next == '_' || next == '$'))
 	{
 		_value += (char) istr.get();
 		next = istr.peek();
@@ -388,7 +388,7 @@ bool StringLiteralToken::start(char c, std::istream& istr)
 
 void StringLiteralToken::finish(std::istream& istr)
 {
-	int next = istr.peek();	
+	int next = istr.peek();
 	while (next != -1 && next != '"' && next != '\n' && next != '\r')
 	{
 		if (next == '\\') _value += (char) istr.get();
@@ -447,7 +447,7 @@ bool CharLiteralToken::start(char c, std::istream& istr)
 
 void CharLiteralToken::finish(std::istream& istr)
 {
-	int next = istr.peek();	
+	int next = istr.peek();
 	while (next != -1 && next != '\'' && next != '\n' && next != '\r')
 	{
 		if (next == '\\') _value += (char) istr.get();
@@ -513,23 +513,18 @@ void NumberLiteralToken::finish(std::istream& istr)
 	_isFloat = false;
 	if (_value[0] != '.') // starts with digit
 	{
-		if (next == 'x')
+		if (_value[0] == '0')
 		{
-			_value += (char) istr.get();
+			if (next == 'x' || next == 'X')
-			next = istr.peek();
-			while (std::isxdigit(next)) 
-			{ 
-				_value += (char) istr.get(); 
-				next = istr.peek(); 
-			}
-			while (next == 'L' || next == 'l' || next == 'U' || next == 'u')
 			{
-				_value += (char) istr.get(); 
+				return finishHex(istr, next);
-				next = istr.peek();
+			}
+			else if (next == 'b' || next == 'B')
+			{
+				return finishBin(istr, next);
 			}
-			return;
 		}
-		while (next >= '0' && next <= '9')
+		while ((next >= '0' && next <= '9') || next == '\'')
 		{
 			_value += (char) istr.get();
 			next = istr.peek();
@@ -563,37 +558,91 @@ void NumberLiteralToken::finish(std::istream& istr)
 	if (next == 'e' || next == 'E')
 	{
 		_isFloat = true;
+		finishExp(istr, next);
+	}
+	finishSuffix(istr, next);
+}
+
+
+void NumberLiteralToken::finishHex(std::istream& istr, int next)
+{
+	_value += (char) istr.get();
+	next = istr.peek();
+	while (std::isxdigit(next) || next == '\'')
+	{
 		_value += (char) istr.get();
 		next = istr.peek();
-		if (next == '+' || next == '-')
+	}
+	if (next == '.')
+	{
+		_isFloat = true;
+		_value += (char) istr.get();
+		next = istr.peek();
+		while (std::isxdigit(next) || next == '\'')
 		{
 			_value += (char) istr.get();
 			next = istr.peek();
 		}
-		if (next >= '0' && next <= '9')
+	}
+	if (next == 'p' || next == 'P')
+	{
+		finishExp(istr, next);
+	}
+	finishSuffix(istr, next);
+}
+
+
+void NumberLiteralToken::finishBin(std::istream& istr, int next)
+{
+	_value += (char) istr.get();
+	next = istr.peek();
+	while (next == '0' || next == '1' || next == '\'')
+	{
+		_value += (char) istr.get();
+		next = istr.peek();
+	}
+	finishSuffix(istr, next);
+}
+
+
+void NumberLiteralToken::finishExp(std::istream& istr, int next)
+{
+	_isFloat = true;
+	_value += (char) istr.get();
+	next = istr.peek();
+	if (next == '+' || next == '-')
+	{
+		_value += (char) istr.get();
+		next = istr.peek();
+	}
+	if (next >= '0' && next <= '9')
+	{
+		while (next >= '0' && next <= '9')
 		{
-			while (next >= '0' && next <= '9')
+			_value += (char) istr.get();
-			{
+			next = istr.peek();
-				_value += (char) istr.get();
-				next = istr.peek();
-			}
-		}
-		else
-		{
-			std::string s(1, (char) next);
-			syntaxError("digit", s);
 		}
 	}
+	else
+	{
+		std::string s(1, (char) next);
+		syntaxError("digit", s);
+	}
+}
+
+
+void NumberLiteralToken::finishSuffix(std::istream& istr, int next)
+{
 	if (_isFloat)
 	{
 		if (next == 'L' || next == 'l' || next == 'F' || next == 'f')
-			_value += (char) istr.get(); 
+			_value += (char) istr.get();
 	}
 	else
 	{
 		while (next == 'L' || next == 'l' || next == 'U' || next == 'u')
 		{
-			_value += (char) istr.get(); 
+			_value += (char) istr.get();
 			next = istr.peek();
 		}
 	}
@@ -603,7 +652,6 @@ void NumberLiteralToken::finish(std::istream& istr)
 int NumberLiteralToken::asInteger() const
 {
 	return static_cast<int>(std::strtol(_value.c_str(), 0, 0));
-	
 }
 
 
@@ -656,7 +704,7 @@ void CommentToken::finish(std::istream& istr)
 		{
 			next = istr.get();
 			_value += (char) next;
-			if (next == '*' && istr.peek() == '/') 
+			if (next == '*' && istr.peek() == '/')
 			{
 				_value += (char) istr.get();
 				break;
@@ -704,7 +752,7 @@ void PreprocessorToken::finish(std::istream& istr)
 	int next = istr.peek();
 	while (next != -1 && next != '\r' && next != '\n')
 	{
-		if (next == '\\') 
+		if (next == '\\')
 		{
 			istr.get();
 			int p = istr.peek();

CppParser/testsuite/src/CppParserTest.cpp

@@ -13,6 +13,8 @@
 #include "CppUnit/TestSuite.h"
 #include "Poco/CppParser/Utility.h"
 #include "Poco/CppParser/Symbol.h"
+#include "Poco/CppParser/CppToken.h"
+#include <sstream>
 #include <iostream>
 
 
@@ -33,6 +35,7 @@ std::string options("/I \"C:\\Program Files\\Microsoft Visual Studio 8\\VC\\INCL
 				"/C, /P, /TP");
 std::string path("C:\\Program Files\\Microsoft Visual Studio 8\\Common7\\IDE;C:\\Program Files\\Microsoft Visual Studio 8\\VC\\BIN;C:\\Program Files\\Microsoft Visual Studio 8\\Common7\\Tools;;C:\\Program Files\\Microsoft Visual Studio 8\\Common7\\Tools\\bin");
 
+
 CppParserTest::CppParserTest(const std::string& name): CppUnit::TestCase(name)
 {
 }
@@ -52,15 +55,15 @@ void CppParserTest::testParseDir()
 	inc.push_back("./*.h");
 	std::vector<std::string> exc;
 	Utility::parseDir(inc, exc, st, linker, options, path);
-	
+
 	NameSpace::SymbolTable::const_iterator it = st.begin();
 	NameSpace::SymbolTable::const_iterator itEnd = st.end();
-	for (it; it != itEnd; ++it)
+	for (; it != itEnd; ++it)
 	{
 		std::cout << it->first << ": ";
 		Symbol* pSym = it->second;
 		std::cout << pSym->name() << ", " << pSym->getDocumentation() << "\n";
-		
+
 	}
 }
 
@@ -70,15 +73,15 @@ void CppParserTest::testExtractName()
 	std::string decl("int _var");
 	std::string name = Symbol::extractName(decl);
 	assertTrue (name == "_var");
-	
+
 	decl = "void func(int arg1, int arg2)";
 	name = Symbol::extractName(decl);
 	assertTrue (name == "func");
-	
+
 	decl = "const std::vector<NS::MyType>* var";
 	name = Symbol::extractName(decl);
 	assertTrue (name == "var");
-	
+
 	decl = "const std::vector<NS::MyType>* func(int arg) = 0";
 	name = Symbol::extractName(decl);
 	assertTrue (name == "func");
@@ -94,7 +97,7 @@ void CppParserTest::testExtractName()
 	decl = "template <typename A, typename B> B func(A a, B b)";
 	name = Symbol::extractName(decl);
 	assertTrue (name == "func");
-	
+
 	decl = "template <typename A, typename B> class Class";
 	name = Symbol::extractName(decl);
 	assertTrue (name == "Class");
@@ -113,6 +116,70 @@ void CppParserTest::testExtractName()
 }
 
 
+void CppParserTest::testNumberLiterals()
+{
+	testNumberLiteral("123");
+	testNumberLiteral("0123");
+	testNumberLiteral("0x123");
+	testNumberLiteral("0XABC");
+	testNumberLiteral("0b010101");
+	testNumberLiteral("0B101010");
+
+	testNumberLiteral("123L");
+	testNumberLiteral("0123L");
+	testNumberLiteral("0x123L");
+	testNumberLiteral("0XABCL");
+	testNumberLiteral("0b010101L");
+	testNumberLiteral("0B101010L");
+
+	testNumberLiteral("123LL");
+	testNumberLiteral("0123LL");
+	testNumberLiteral("0x123LL");
+	testNumberLiteral("0XABCLL");
+	testNumberLiteral("0b010101LL");
+	testNumberLiteral("0B101010LL");
+
+	testNumberLiteral("123U");
+	testNumberLiteral("0123U");
+	testNumberLiteral("0x123U");
+	testNumberLiteral("0XABCU");
+	testNumberLiteral("0b010101U");
+	testNumberLiteral("0B101010U");
+
+	testNumberLiteral("123UL");
+	testNumberLiteral("0123UL");
+	testNumberLiteral("0x123UL");
+	testNumberLiteral("0XABCUL");
+	testNumberLiteral("0b010101UL");
+	testNumberLiteral("0B101010UL");
+
+	testNumberLiteral("1.23");
+	testNumberLiteral(".123");
+	testNumberLiteral("1.23e+3");
+	testNumberLiteral("1.23E-3");
+
+	testNumberLiteral("0x1fp1");
+	testNumberLiteral("0x1f.e3p+12");
+	testNumberLiteral("0x1f.e3p-12");
+
+	testNumberLiteral("123'456");
+	testNumberLiteral("1'234'567");
+	testNumberLiteral("0x1234'5678");
+	testNumberLiteral("0xFFFF'FFFFULL");
+}
+
+
+void CppParserTest::testNumberLiteral(const std::string& literal)
+{
+	NumberLiteralToken tok;
+
+	std::istringstream istr(literal);
+	assertTrue (tok.start(istr.get(), istr));
+	tok.finish(istr);
+	assertEquals (tok.tokenString(), literal);
+}
+
+
 void CppParserTest::setUp()
 {
 }
@@ -129,6 +196,7 @@ CppUnit::Test* CppParserTest::suite()
 
 	CppUnit_addTest(pSuite, CppParserTest, testParseDir);
 	CppUnit_addTest(pSuite, CppParserTest, testExtractName);
+	CppUnit_addTest(pSuite, CppParserTest, testNumberLiterals);
 
 	return pSuite;
 }

CppParser/testsuite/src/CppParserTest.h

@@ -26,12 +26,16 @@ public:
 
 	void testParseDir();
 	void testExtractName();
+	void testNumberLiterals();
 
 	void setUp();
 	void tearDown();
 
 	static CppUnit::Test* suite();
 
+protected:
+	void testNumberLiteral(const std::string& literal);
+
 private:
 };