From d6f97fe3466bacfac65e410ff68c0576eb65c077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCnter=20Obiltschnig?= Date: Sun, 30 Dec 2018 21:00:17 +0100 Subject: [PATCH] added support for C++14/17 number literals --- CppParser/include/Poco/CppParser/CppToken.h | 25 +++-- CppParser/src/CppToken.cpp | 116 ++++++++++++++------ CppParser/testsuite/src/CppParserTest.cpp | 82 ++++++++++++-- CppParser/testsuite/src/CppParserTest.h | 4 + 4 files changed, 177 insertions(+), 50 deletions(-) diff --git a/CppParser/include/Poco/CppParser/CppToken.h b/CppParser/include/Poco/CppParser/CppToken.h index e53fb9903..cd0de36e0 100644 --- a/CppParser/include/Poco/CppParser/CppToken.h +++ b/CppParser/include/Poco/CppParser/CppToken.h @@ -33,7 +33,7 @@ class CppParser_API CppToken: public Poco::Token public: CppToken(); ~CppToken(); - + protected: void syntaxError(const std::string& expected, const std::string& actual); }; @@ -66,7 +66,7 @@ public: OP_BITAND_ASSIGN, // &= OP_AND, // && OP_BITOR, // | - OP_BITOR_ASSIGN, // |= + OP_BITOR_ASSIGN, // |= OP_OR, // || OP_XOR, // ^ OP_XOR_ASSIGN, // ^= @@ -92,17 +92,17 @@ public: OP_SEMICOLON, // ; OP_QUESTION // ? }; - + OperatorToken(); ~OperatorToken(); Poco::Token::Class tokenClass() const; bool start(char c, std::istream& istr); void finish(std::istream& istr); int asInteger() const; - + private: typedef std::map OpMap; - + OpMap _opMap; }; @@ -197,17 +197,17 @@ public: KW_XOR, KW_XOR_EQ }; - + IdentifierToken(); ~IdentifierToken(); Poco::Token::Class tokenClass() const; bool start(char c, std::istream& istr); void finish(std::istream& istr); int asInteger() const; - + private: typedef std::map KWMap; - + KWMap _kwMap; }; @@ -244,9 +244,16 @@ public: Poco::Token::Class tokenClass() const; bool start(char c, std::istream& istr); void finish(std::istream& istr); + int asInteger() const; double asFloat() const; - + +protected: + void finishHex(std::istream& istr, int next); + void finishBin(std::istream& istr, int next); + void finishExp(std::istream& istr, int next); + void finishSuffix(std::istream& istr, int next); + private: bool _isFloat; }; diff --git a/CppParser/src/CppToken.cpp b/CppParser/src/CppToken.cpp index 81f9cca0b..e930e33c2 100644 --- a/CppParser/src/CppToken.cpp +++ b/CppParser/src/CppToken.cpp @@ -96,7 +96,7 @@ OperatorToken::OperatorToken() _opMap[":"] = i++; _opMap["::"] = i++; _opMap[";"] = i++; - _opMap["?"] = i++; + _opMap["?"] = i++; } @@ -345,7 +345,7 @@ void IdentifierToken::finish(std::istream& istr) while ((next >= 'A' && next <= 'Z') || (next >= 'a' && next <= 'z') || (next >= '0' && next <= '9') || - (next == '_' || next == '$')) + (next == '_' || next == '$')) { _value += (char) istr.get(); next = istr.peek(); @@ -388,7 +388,7 @@ bool StringLiteralToken::start(char c, std::istream& istr) void StringLiteralToken::finish(std::istream& istr) { - int next = istr.peek(); + int next = istr.peek(); while (next != -1 && next != '"' && next != '\n' && next != '\r') { if (next == '\\') _value += (char) istr.get(); @@ -447,7 +447,7 @@ bool CharLiteralToken::start(char c, std::istream& istr) void CharLiteralToken::finish(std::istream& istr) { - int next = istr.peek(); + int next = istr.peek(); while (next != -1 && next != '\'' && next != '\n' && next != '\r') { if (next == '\\') _value += (char) istr.get(); @@ -513,23 +513,18 @@ void NumberLiteralToken::finish(std::istream& istr) _isFloat = false; if (_value[0] != '.') // starts with digit { - if (next == 'x') + if (_value[0] == '0') { - _value += (char) istr.get(); - next = istr.peek(); - while (std::isxdigit(next)) - { - _value += (char) istr.get(); - next = istr.peek(); - } - while (next == 'L' || next == 'l' || next == 'U' || next == 'u') + if (next == 'x' || next == 'X') { - _value += (char) istr.get(); - next = istr.peek(); + return finishHex(istr, next); + } + else if (next == 'b' || next == 'B') + { + return finishBin(istr, next); } - return; } - while (next >= '0' && next <= '9') + while ((next >= '0' && next <= '9') || next == '\'') { _value += (char) istr.get(); next = istr.peek(); @@ -563,37 +558,91 @@ void NumberLiteralToken::finish(std::istream& istr) if (next == 'e' || next == 'E') { _isFloat = true; + finishExp(istr, next); + } + finishSuffix(istr, next); +} + + +void NumberLiteralToken::finishHex(std::istream& istr, int next) +{ + _value += (char) istr.get(); + next = istr.peek(); + while (std::isxdigit(next) || next == '\'') + { _value += (char) istr.get(); next = istr.peek(); - if (next == '+' || next == '-') + } + if (next == '.') + { + _isFloat = true; + _value += (char) istr.get(); + next = istr.peek(); + while (std::isxdigit(next) || next == '\'') { _value += (char) istr.get(); next = istr.peek(); } - if (next >= '0' && next <= '9') + } + if (next == 'p' || next == 'P') + { + finishExp(istr, next); + } + finishSuffix(istr, next); +} + + +void NumberLiteralToken::finishBin(std::istream& istr, int next) +{ + _value += (char) istr.get(); + next = istr.peek(); + while (next == '0' || next == '1' || next == '\'') + { + _value += (char) istr.get(); + next = istr.peek(); + } + finishSuffix(istr, next); +} + + +void NumberLiteralToken::finishExp(std::istream& istr, int next) +{ + _isFloat = true; + _value += (char) istr.get(); + next = istr.peek(); + if (next == '+' || next == '-') + { + _value += (char) istr.get(); + next = istr.peek(); + } + if (next >= '0' && next <= '9') + { + while (next >= '0' && next <= '9') { - while (next >= '0' && next <= '9') - { - _value += (char) istr.get(); - next = istr.peek(); - } - } - else - { - std::string s(1, (char) next); - syntaxError("digit", s); + _value += (char) istr.get(); + next = istr.peek(); } } + else + { + std::string s(1, (char) next); + syntaxError("digit", s); + } +} + + +void NumberLiteralToken::finishSuffix(std::istream& istr, int next) +{ if (_isFloat) { if (next == 'L' || next == 'l' || next == 'F' || next == 'f') - _value += (char) istr.get(); + _value += (char) istr.get(); } else { while (next == 'L' || next == 'l' || next == 'U' || next == 'u') { - _value += (char) istr.get(); + _value += (char) istr.get(); next = istr.peek(); } } @@ -603,7 +652,6 @@ void NumberLiteralToken::finish(std::istream& istr) int NumberLiteralToken::asInteger() const { return static_cast(std::strtol(_value.c_str(), 0, 0)); - } @@ -656,7 +704,7 @@ void CommentToken::finish(std::istream& istr) { next = istr.get(); _value += (char) next; - if (next == '*' && istr.peek() == '/') + if (next == '*' && istr.peek() == '/') { _value += (char) istr.get(); break; @@ -704,7 +752,7 @@ void PreprocessorToken::finish(std::istream& istr) int next = istr.peek(); while (next != -1 && next != '\r' && next != '\n') { - if (next == '\\') + if (next == '\\') { istr.get(); int p = istr.peek(); diff --git a/CppParser/testsuite/src/CppParserTest.cpp b/CppParser/testsuite/src/CppParserTest.cpp index 2c6bb13cd..7a38b7166 100644 --- a/CppParser/testsuite/src/CppParserTest.cpp +++ b/CppParser/testsuite/src/CppParserTest.cpp @@ -13,6 +13,8 @@ #include "CppUnit/TestSuite.h" #include "Poco/CppParser/Utility.h" #include "Poco/CppParser/Symbol.h" +#include "Poco/CppParser/CppToken.h" +#include #include @@ -33,6 +35,7 @@ std::string options("/I \"C:\\Program Files\\Microsoft Visual Studio 8\\VC\\INCL "/C, /P, /TP"); std::string path("C:\\Program Files\\Microsoft Visual Studio 8\\Common7\\IDE;C:\\Program Files\\Microsoft Visual Studio 8\\VC\\BIN;C:\\Program Files\\Microsoft Visual Studio 8\\Common7\\Tools;;C:\\Program Files\\Microsoft Visual Studio 8\\Common7\\Tools\\bin"); + CppParserTest::CppParserTest(const std::string& name): CppUnit::TestCase(name) { } @@ -52,15 +55,15 @@ void CppParserTest::testParseDir() inc.push_back("./*.h"); std::vector exc; Utility::parseDir(inc, exc, st, linker, options, path); - + NameSpace::SymbolTable::const_iterator it = st.begin(); NameSpace::SymbolTable::const_iterator itEnd = st.end(); - for (it; it != itEnd; ++it) + for (; it != itEnd; ++it) { std::cout << it->first << ": "; Symbol* pSym = it->second; std::cout << pSym->name() << ", " << pSym->getDocumentation() << "\n"; - + } } @@ -70,15 +73,15 @@ void CppParserTest::testExtractName() std::string decl("int _var"); std::string name = Symbol::extractName(decl); assertTrue (name == "_var"); - + decl = "void func(int arg1, int arg2)"; name = Symbol::extractName(decl); assertTrue (name == "func"); - + decl = "const std::vector* var"; name = Symbol::extractName(decl); assertTrue (name == "var"); - + decl = "const std::vector* func(int arg) = 0"; name = Symbol::extractName(decl); assertTrue (name == "func"); @@ -94,7 +97,7 @@ void CppParserTest::testExtractName() decl = "template B func(A a, B b)"; name = Symbol::extractName(decl); assertTrue (name == "func"); - + decl = "template class Class"; name = Symbol::extractName(decl); assertTrue (name == "Class"); @@ -113,6 +116,70 @@ void CppParserTest::testExtractName() } +void CppParserTest::testNumberLiterals() +{ + testNumberLiteral("123"); + testNumberLiteral("0123"); + testNumberLiteral("0x123"); + testNumberLiteral("0XABC"); + testNumberLiteral("0b010101"); + testNumberLiteral("0B101010"); + + testNumberLiteral("123L"); + testNumberLiteral("0123L"); + testNumberLiteral("0x123L"); + testNumberLiteral("0XABCL"); + testNumberLiteral("0b010101L"); + testNumberLiteral("0B101010L"); + + testNumberLiteral("123LL"); + testNumberLiteral("0123LL"); + testNumberLiteral("0x123LL"); + testNumberLiteral("0XABCLL"); + testNumberLiteral("0b010101LL"); + testNumberLiteral("0B101010LL"); + + testNumberLiteral("123U"); + testNumberLiteral("0123U"); + testNumberLiteral("0x123U"); + testNumberLiteral("0XABCU"); + testNumberLiteral("0b010101U"); + testNumberLiteral("0B101010U"); + + testNumberLiteral("123UL"); + testNumberLiteral("0123UL"); + testNumberLiteral("0x123UL"); + testNumberLiteral("0XABCUL"); + testNumberLiteral("0b010101UL"); + testNumberLiteral("0B101010UL"); + + testNumberLiteral("1.23"); + testNumberLiteral(".123"); + testNumberLiteral("1.23e+3"); + testNumberLiteral("1.23E-3"); + + testNumberLiteral("0x1fp1"); + testNumberLiteral("0x1f.e3p+12"); + testNumberLiteral("0x1f.e3p-12"); + + testNumberLiteral("123'456"); + testNumberLiteral("1'234'567"); + testNumberLiteral("0x1234'5678"); + testNumberLiteral("0xFFFF'FFFFULL"); +} + + +void CppParserTest::testNumberLiteral(const std::string& literal) +{ + NumberLiteralToken tok; + + std::istringstream istr(literal); + assertTrue (tok.start(istr.get(), istr)); + tok.finish(istr); + assertEquals (tok.tokenString(), literal); +} + + void CppParserTest::setUp() { } @@ -129,6 +196,7 @@ CppUnit::Test* CppParserTest::suite() CppUnit_addTest(pSuite, CppParserTest, testParseDir); CppUnit_addTest(pSuite, CppParserTest, testExtractName); + CppUnit_addTest(pSuite, CppParserTest, testNumberLiterals); return pSuite; } diff --git a/CppParser/testsuite/src/CppParserTest.h b/CppParser/testsuite/src/CppParserTest.h index f388f582d..0c3b523c3 100644 --- a/CppParser/testsuite/src/CppParserTest.h +++ b/CppParser/testsuite/src/CppParserTest.h @@ -26,12 +26,16 @@ public: void testParseDir(); void testExtractName(); + void testNumberLiterals(); void setUp(); void tearDown(); static CppUnit::Test* suite(); +protected: + void testNumberLiteral(const std::string& literal); + private: };