From beedf13d01696bfd05a6ecbc825b4270646ddb49 Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Sat, 3 Oct 2015 16:38:41 -0600 Subject: [PATCH 1/5] Make binary literals sized like other integer types --- include/chaiscript/dispatchkit/bootstrap.hpp | 2 + .../chaiscript/language/chaiscript_parser.hpp | 126 +++++++----------- unittests/integer_literal_test.cpp | 3 +- unittests/number_suffixes.chai | 4 +- 4 files changed, 55 insertions(+), 80 deletions(-) diff --git a/include/chaiscript/dispatchkit/bootstrap.hpp b/include/chaiscript/dispatchkit/bootstrap.hpp index 0cd7bab..ef5a94b 100644 --- a/include/chaiscript/dispatchkit/bootstrap.hpp +++ b/include/chaiscript/dispatchkit/bootstrap.hpp @@ -509,6 +509,8 @@ namespace chaiscript bootstrap_pod_type("long", m); bootstrap_pod_type("unsigned_int", m); bootstrap_pod_type("unsigned_long", m); + bootstrap_pod_type("long_long", m); + bootstrap_pod_type("unsigned_long_long", m); bootstrap_pod_type("size_t", m); bootstrap_pod_type("char", m); bootstrap_pod_type("wchar_t", m); diff --git a/include/chaiscript/language/chaiscript_parser.hpp b/include/chaiscript/language/chaiscript_parser.hpp index ddc2111..a05129a 100644 --- a/include/chaiscript/language/chaiscript_parser.hpp +++ b/include/chaiscript/language/chaiscript_parser.hpp @@ -621,8 +621,7 @@ namespace chaiscript - template - static Boxed_Value buildInt(const IntType &t_type, const std::string &t_val) + static Boxed_Value buildInt(const int base, const std::string &t_val, const bool prefixed) { bool unsigned_ = false; bool long_ = false; @@ -649,52 +648,57 @@ namespace chaiscript } } - std::stringstream ss(t_val.substr(0, i)); - ss >> t_type; - std::stringstream testu(t_val.substr(0, i)); - uint64_t u; - testu >> t_type >> u; - - bool unsignedrequired = false; - - if ((u >> (sizeof(int) * 8)) > 0) - { - //requires something bigger than int - long_ = true; - } + const auto val = prefixed?std::string(t_val.begin()+2,t_val.end()):t_val; static_assert(sizeof(long) == sizeof(uint64_t) || sizeof(long) * 2 == sizeof(uint64_t), "Unexpected sizing of integer types"); + bool unsignedrequired = false; - if ((sizeof(long) < sizeof(uint64_t)) - && (u >> ((sizeof(uint64_t) - sizeof(long)) * 8)) > 0) - { - //requires something bigger than long - longlong_ = true; - } + try { + auto u = std::stoll(val,nullptr,base); - - const size_t size = [&]()->size_t{ - if (longlong_) + if ((u >> (sizeof(int) * 8)) > 0) { - return sizeof(int64_t) * 8; - } else if (long_) { - return sizeof(long) * 8; - } else { - return sizeof(int) * 8; + //requires something bigger than int + long_ = true; } - }(); - if ( (u >> (size - 1)) > 0) - { + if ((sizeof(long) < sizeof(uint64_t)) + && (u >> ((sizeof(uint64_t) - sizeof(long)) * 8)) > 0) + { + //requires something bigger than long + longlong_ = true; + } + + const size_t size = [&]()->size_t{ + if (longlong_) + { + return sizeof(int64_t) * 8; + } else if (long_) { + return sizeof(long) * 8; + } else { + return sizeof(int) * 8; + } + }(); + + if ( (u >> (size - 1)) > 0) + { + unsignedrequired = true; + } + } catch (const std::out_of_range &) { + // it cannot fit in a signed long long... + std::cout << "forcing long long for '" << val << "'\n"; unsignedrequired = true; + + long_ = sizeof(unsigned long long) == sizeof(unsigned long); + longlong_ = sizeof(unsigned long long) != sizeof(unsigned long); } if (unsignedrequired && !unsigned_) { - if (t_type == &std::hex || t_type == &std::oct) + if (base != 10) { - // with hex and octal we are happy to just make it unsigned + // with bin, hex and oct we are happy to just make it unsigned unsigned_ = true; } else { // with decimal we must bump it up to the next size @@ -711,32 +715,20 @@ namespace chaiscript { if (longlong_) { - uint64_t val; - ss >> val; - return const_var(val); + return const_var(stoull(val,nullptr,base)); } else if (long_) { - unsigned long val; - ss >> val; - return const_var(val); + return const_var(stoul(val,nullptr,base)); } else { - unsigned int val; - ss >> val; - return const_var(val); + return const_var(static_cast(stoul(val,nullptr,base))); } } else { if (longlong_) { - int64_t val; - ss >> val; - return const_var(val); + return const_var(stoll(val,nullptr,base)); } else if (long_) { - long val; - ss >> val; - return const_var(val); + return const_var(stol(val,nullptr,base)); } else { - int val; - ss >> val; - return const_var(val); + return const_var(stoi(val,nullptr,base)); } } } @@ -758,35 +750,15 @@ namespace chaiscript if (m_position.has_more() && char_in_alphabet(*m_position, detail::float_alphabet) ) { if (Hex_()) { auto match = Position::str(start, m_position); - auto bv = buildInt(std::hex, match); + auto bv = buildInt(16, match, true); m_match_stack.emplace_back(make_node(std::move(match), start.line, start.col, std::move(bv))); return true; } if (Binary_()) { auto match = Position::str(start, m_position); - int64_t temp_int = 0; - size_t pos = 0; - const auto end = match.length(); - - while ((pos < end) && (pos < (2 + sizeof(int) * 8))) { - temp_int <<= 1; - if (match[pos] == '1') { - temp_int += 1; - } - ++pos; - } - - Boxed_Value i = [&]()->Boxed_Value{ - if (match.length() <= sizeof(int) * 8) - { - return const_var(static_cast(temp_int)); - } else { - return const_var(temp_int); - } - }(); - - m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(i))); + auto bv = buildInt(2, match, true); + m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); return true; } if (Float_()) { @@ -799,11 +771,11 @@ namespace chaiscript IntSuffix_(); auto match = Position::str(start, m_position); if (!match.empty() && (match[0] == '0')) { - auto bv = buildInt(std::oct, match); + auto bv = buildInt(8, match, false); m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); } else if (!match.empty()) { - auto bv = buildInt(std::dec, match); + auto bv = buildInt(10, match, false); m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); } else { return false; diff --git a/unittests/integer_literal_test.cpp b/unittests/integer_literal_test.cpp index ecaff8a..9a02c20 100644 --- a/unittests/integer_literal_test.cpp +++ b/unittests/integer_literal_test.cpp @@ -5,9 +5,10 @@ template bool test_literal(T val, const std::string &str) { + std::cout << "Comparing : " << val; chaiscript::ChaiScript chai; T val2 = chai.eval(str); - std::cout << "Comparing : " << val << " " << val2 << '\n'; + std::cout << " " << val2 << '\n'; return val == val2; } diff --git a/unittests/number_suffixes.chai b/unittests/number_suffixes.chai index 99ac03f..b8fd1f5 100644 --- a/unittests/number_suffixes.chai +++ b/unittests/number_suffixes.chai @@ -2,8 +2,8 @@ assert_equal(true, int_type.bare_equal(1.get_type_info())) assert_equal(true, unsigned_int_type.bare_equal(1u.get_type_info())) assert_equal(true, unsigned_long_type.bare_equal(1lu.get_type_info())) assert_equal(true, long_type.bare_equal(1l.get_type_info())) -assert_equal(true, int64_t_type.bare_equal(1ll.get_type_info())) -assert_equal(true, uint64_t_type.bare_equal(1ull.get_type_info())) +assert_equal(true, long_long_type.bare_equal(1ll.get_type_info())) +assert_equal(true, unsigned_long_long_type.bare_equal(1ull.get_type_info())) assert_equal(true, double_type.bare_equal(1.6.get_type_info())) assert_equal(true, float_type.bare_equal(1.6f.get_type_info())) From e221ceaa4c3fd8254ff730ec96e9010105936471 Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Sat, 3 Oct 2015 17:11:03 -0600 Subject: [PATCH 2/5] Greatly simplify integer sizing code --- .../chaiscript/language/chaiscript_parser.hpp | 87 ++++--------------- 1 file changed, 17 insertions(+), 70 deletions(-) diff --git a/include/chaiscript/language/chaiscript_parser.hpp b/include/chaiscript/language/chaiscript_parser.hpp index a05129a..c995f7d 100644 --- a/include/chaiscript/language/chaiscript_parser.hpp +++ b/include/chaiscript/language/chaiscript_parser.hpp @@ -648,87 +648,34 @@ namespace chaiscript } } - const auto val = prefixed?std::string(t_val.begin()+2,t_val.end()):t_val; - static_assert(sizeof(long) == sizeof(uint64_t) || sizeof(long) * 2 == sizeof(uint64_t), "Unexpected sizing of integer types"); - bool unsignedrequired = false; +// static_assert(sizeof(long) == sizeof(uint64_t) || sizeof(long) * 2 == sizeof(uint64_t), "Unexpected sizing of integer types"); try { auto u = std::stoll(val,nullptr,base); - if ((u >> (sizeof(int) * 8)) > 0) - { - //requires something bigger than int - long_ = true; + if (!unsigned_ && !long_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { + return const_var(static_cast(u)); + } else if ((unsigned_ || base != 10) && !long_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { + return const_var(static_cast(u)); + } else if (!unsigned_ && !longlong_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { + return const_var(static_cast(u)); + } else if ((unsigned_ || base != 10) && !longlong_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { + return const_var(static_cast(u)); + } else if (!unsigned_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { + return const_var(static_cast(u)); + } else { + return const_var(static_cast(u)); } - if ((sizeof(long) < sizeof(uint64_t)) - && (u >> ((sizeof(uint64_t) - sizeof(long)) * 8)) > 0) - { - //requires something bigger than long - longlong_ = true; - } - - const size_t size = [&]()->size_t{ - if (longlong_) - { - return sizeof(int64_t) * 8; - } else if (long_) { - return sizeof(long) * 8; - } else { - return sizeof(int) * 8; - } - }(); - - if ( (u >> (size - 1)) > 0) - { - unsignedrequired = true; - } } catch (const std::out_of_range &) { - // it cannot fit in a signed long long... - std::cout << "forcing long long for '" << val << "'\n"; - unsignedrequired = true; + auto u = std::stoull(val,nullptr,base); - long_ = sizeof(unsigned long long) == sizeof(unsigned long); - longlong_ = sizeof(unsigned long long) != sizeof(unsigned long); - } - - if (unsignedrequired && !unsigned_) - { - if (base != 10) - { - // with bin, hex and oct we are happy to just make it unsigned - unsigned_ = true; + if (u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { + return const_var(static_cast(u)); } else { - // with decimal we must bump it up to the next size - if (long_) - { - longlong_ = true; - } else if (!long_ && !longlong_) { - long_ = true; - } - } - } - - if (unsigned_) - { - if (longlong_) - { - return const_var(stoull(val,nullptr,base)); - } else if (long_) { - return const_var(stoul(val,nullptr,base)); - } else { - return const_var(static_cast(stoul(val,nullptr,base))); - } - } else { - if (longlong_) - { - return const_var(stoll(val,nullptr,base)); - } else if (long_) { - return const_var(stol(val,nullptr,base)); - } else { - return const_var(stoi(val,nullptr,base)); + return const_var(static_cast(u)); } } } From d2cf12f94849c453927fbb840451158ace649207 Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Sat, 3 Oct 2015 21:01:52 -0600 Subject: [PATCH 3/5] Add tests for binary literals --- .../chaiscript/language/chaiscript_parser.hpp | 12 +++++-- unittests/integer_literal_test.cpp | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/include/chaiscript/language/chaiscript_parser.hpp b/include/chaiscript/language/chaiscript_parser.hpp index c995f7d..f582ab1 100644 --- a/include/chaiscript/language/chaiscript_parser.hpp +++ b/include/chaiscript/language/chaiscript_parser.hpp @@ -650,11 +650,14 @@ namespace chaiscript const auto val = prefixed?std::string(t_val.begin()+2,t_val.end()):t_val; -// static_assert(sizeof(long) == sizeof(uint64_t) || sizeof(long) * 2 == sizeof(uint64_t), "Unexpected sizing of integer types"); - try { auto u = std::stoll(val,nullptr,base); +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsign-compare" +#endif + if (!unsigned_ && !long_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { return const_var(static_cast(u)); } else if ((unsigned_ || base != 10) && !long_ && u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { @@ -669,6 +672,11 @@ namespace chaiscript return const_var(static_cast(u)); } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + + } catch (const std::out_of_range &) { auto u = std::stoull(val,nullptr,base); diff --git a/unittests/integer_literal_test.cpp b/unittests/integer_literal_test.cpp index 9a02c20..58e954a 100644 --- a/unittests/integer_literal_test.cpp +++ b/unittests/integer_literal_test.cpp @@ -75,6 +75,39 @@ int main() && TEST_LITERAL(177777777777777777) && TEST_LITERAL(1777777777777777777) + && test_literal(0xF, "0b1111") + && test_literal(0xFF, "0b11111111") + && test_literal(0xFFF, "0b111111111111") + && test_literal(0xFFFF, "0b1111111111111111") + && test_literal(0xFFFFF, "0b11111111111111111111") + && test_literal(0xFFFFFF, "0b111111111111111111111111") + && test_literal(0xFFFFFFF, "0b1111111111111111111111111111") + && test_literal(0xFFFFFFFF, "0b11111111111111111111111111111111") + && test_literal(0xFFFFFFFFF, "0b111111111111111111111111111111111111") + && test_literal(0xFFFFFFFFFF, "0b1111111111111111111111111111111111111111") + && test_literal(0xFFFFFFFFFFF, "0b11111111111111111111111111111111111111111111") + && test_literal(0xFFFFFFFFFFFF, "0b111111111111111111111111111111111111111111111111") + && test_literal(0xFFFFFFFFFFFFF, "0b1111111111111111111111111111111111111111111111111111") + && test_literal(0xFFFFFFFFFFFFFF, "0b11111111111111111111111111111111111111111111111111111111") + && test_literal(0xFFFFFFFFFFFFFFF, "0b111111111111111111111111111111111111111111111111111111111111") + && test_literal(0xFFFFFFFFFFFFFFFF, "0b1111111111111111111111111111111111111111111111111111111111111111") + + && test_literal(0x7, "0b111") + && test_literal(0x7F, "0b1111111") + && test_literal(0x7FF, "0b11111111111") + && test_literal(0x7FFF, "0b111111111111111") + && test_literal(0x7FFFF, "0b1111111111111111111") + && test_literal(0x7FFFFF, "0b11111111111111111111111") + && test_literal(0x7FFFFFF, "0b111111111111111111111111111") + && test_literal(0x7FFFFFFF, "0b1111111111111111111111111111111") + && test_literal(0x7FFFFFFFF, "0b11111111111111111111111111111111111") + && test_literal(0x7FFFFFFFFF, "0b111111111111111111111111111111111111111") + && test_literal(0x7FFFFFFFFFF, "0b1111111111111111111111111111111111111111111") + && test_literal(0x7FFFFFFFFFFF, "0b11111111111111111111111111111111111111111111111") + && test_literal(0x7FFFFFFFFFFFF, "0b111111111111111111111111111111111111111111111111111") + && test_literal(0x7FFFFFFFFFFFFF, "0b1111111111111111111111111111111111111111111111111111111") + && test_literal(0x7FFFFFFFFFFFFFF, "0b11111111111111111111111111111111111111111111111111111111111") + && test_literal(0x7FFFFFFFFFFFFFFF, "0b111111111111111111111111111111111111111111111111111111111111111") ) { return EXIT_SUCCESS; From 14b3870efb4eb25a49d5af34636de6ab3060cb40 Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Sun, 4 Oct 2015 08:53:22 -0600 Subject: [PATCH 4/5] Fix integer overflow and bad numeric parses --- .../chaiscript/language/chaiscript_parser.hpp | 80 +++++++++++-------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/include/chaiscript/language/chaiscript_parser.hpp b/include/chaiscript/language/chaiscript_parser.hpp index f582ab1..57387d3 100644 --- a/include/chaiscript/language/chaiscript_parser.hpp +++ b/include/chaiscript/language/chaiscript_parser.hpp @@ -676,14 +676,19 @@ namespace chaiscript #pragma GCC diagnostic pop #endif - } catch (const std::out_of_range &) { - auto u = std::stoull(val,nullptr,base); + // too big to be signed + try { + auto u = std::stoull(val,nullptr,base); - if (u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { - return const_var(static_cast(u)); - } else { - return const_var(static_cast(u)); + if (u >= std::numeric_limits::min() && u <= std::numeric_limits::max()) { + return const_var(static_cast(u)); + } else { + return const_var(static_cast(u)); + } + } catch (const std::out_of_range &) { + // it's just simply too big + return const_var(std::numeric_limits::max()); } } } @@ -703,39 +708,44 @@ namespace chaiscript } else { const auto start = m_position; if (m_position.has_more() && char_in_alphabet(*m_position, detail::float_alphabet) ) { - if (Hex_()) { - auto match = Position::str(start, m_position); - auto bv = buildInt(16, match, true); - m_match_stack.emplace_back(make_node(std::move(match), start.line, start.col, std::move(bv))); - return true; - } + try { + if (Hex_()) { + auto match = Position::str(start, m_position); + auto bv = buildInt(16, match, true); + m_match_stack.emplace_back(make_node(std::move(match), start.line, start.col, std::move(bv))); + return true; + } - if (Binary_()) { - auto match = Position::str(start, m_position); - auto bv = buildInt(2, match, true); - m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); - return true; - } - if (Float_()) { - auto match = Position::str(start, m_position); - auto bv = buildFloat(match); - m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); - return true; - } - else { - IntSuffix_(); - auto match = Position::str(start, m_position); - if (!match.empty() && (match[0] == '0')) { - auto bv = buildInt(8, match, false); + if (Binary_()) { + auto match = Position::str(start, m_position); + auto bv = buildInt(2, match, true); m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); + return true; } - else if (!match.empty()) { - auto bv = buildInt(10, match, false); - m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); - } else { - return false; + if (Float_()) { + auto match = Position::str(start, m_position); + auto bv = buildFloat(match); + m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); + return true; } - return true; + else { + IntSuffix_(); + auto match = Position::str(start, m_position); + if (!match.empty() && (match[0] == '0')) { + auto bv = buildInt(8, match, false); + m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); + } + else if (!match.empty()) { + auto bv = buildInt(10, match, false); + m_match_stack.push_back(make_node(std::move(match), start.line, start.col, std::move(bv))); + } else { + return false; + } + return true; + } + } catch (const std::invalid_argument &) { + // error parsing number passed in to buildFloat/buildInt + return false; } } else { From 1add4c4b0f05a355158f2b2e0bfc36329bc9c8e8 Mon Sep 17 00:00:00 2001 From: Jason Turner Date: Sun, 4 Oct 2015 14:32:23 -0600 Subject: [PATCH 5/5] Fix issues with integer parsing on MSVC See #212 --- .../chaiscript/language/chaiscript_parser.hpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/include/chaiscript/language/chaiscript_parser.hpp b/include/chaiscript/language/chaiscript_parser.hpp index 57387d3..11a2d04 100644 --- a/include/chaiscript/language/chaiscript_parser.hpp +++ b/include/chaiscript/language/chaiscript_parser.hpp @@ -17,9 +17,19 @@ #include + #include "../dispatchkit/boxed_value.hpp" #include "chaiscript_common.hpp" + +#if defined(CHAISCRIPT_MSVC) && defined(max) && defined(min) +#pragma push_macro("max") // Why Microsoft? why? This is worse than bad +#undef max +#pragma push_macro("min") +#undef min +#endif + + namespace chaiscript { /// \brief Classes and functions used during the parsing process. @@ -912,6 +922,8 @@ namespace chaiscript { } + Char_Parser &operator=(const Char_Parser &) = delete; + ~Char_Parser(){ if (is_octal) { process_octal(); @@ -2393,5 +2405,12 @@ namespace chaiscript } } + +#ifdef CHAISCRIPT_MSVC +#pragma pop_macro("min") +#pragma pop_macro("max") +#endif + + #endif /* CHAISCRIPT_PARSER_HPP_ */