diff --git a/langkit/CMakeLists.txt b/langkit/CMakeLists.txt index 799f2f7..d622f94 100644 --- a/langkit/CMakeLists.txt +++ b/langkit/CMakeLists.txt @@ -11,10 +11,10 @@ find_package( Boost 1.36.0 COMPONENTS regex unit_test_framework) if(Boost_FOUND) include_directories(${Boost_INCLUDE_DIRS}) - add_executable(langkit_test main.cpp langkit_lexer.cpp langkit_parser.cpp) + add_executable(langkit_test main.cpp langkit_parser.cpp) target_link_libraries(langkit_test ${Boost_LIBRARIES}) - add_library(langkit SHARED langkit_lexer.cpp langkit_parser.cpp) + add_library(langkit SHARED langkit_parser.cpp) target_link_libraries(langkit ${Boost_LIBRARIES}) add_executable(langkit_unittest unittest.cpp) diff --git a/langkit/langkit_lexer.cpp b/langkit/langkit_lexer.cpp index 20400b7..e555dff 100644 --- a/langkit/langkit_lexer.cpp +++ b/langkit/langkit_lexer.cpp @@ -4,136 +4,3 @@ #include #include "langkit_lexer.hpp" -Lexer Lexer::operator<<(const Pattern &p) { - lex_patterns.push_back(p); - return *this; -} - -std::vector Lexer::lex(const std::string &input, const char *filename) { - std::vector::iterator iter, end, iter2, end2; - std::vector retval; - bool found; - std::string::const_iterator input_iter = input.begin(), input_end = input.end(); - - int current_col = 0; - int current_line = 0; - boost::match_results what; - - while (input_iter != input_end) { - found = false; - - if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) { - std::string comment_start(what[0]); - input_iter += comment_start.size(); - - bool found_eol = false; - - while ((!found_eol) && (input_iter != input_end)) { - boost::match_results eol_delim; - if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { - std::string comment_end(eol_delim[0]); - input_iter += comment_end.size(); - ++current_line; - current_col = 0; - found_eol = true; - break; - } - if ((!found_eol) && (input_iter != input_end)) { - ++input_iter; - } - } - } - else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) { - std::string comment_start(what[0]); - input_iter += comment_start.size(); - - bool found_eoc = false; - - while ((!found_eoc) && (input_iter != input_end)) { - boost::match_results eol_delim; - if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { - std::string comment_end(eol_delim[0]); - input_iter += comment_end.size(); - ++current_line; - current_col = 0; - } - boost::match_results eoc_delim; - if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) { - std::string comment_end(eoc_delim[0]); - input_iter += comment_end.size(); - current_col += comment_end.size(); - found_eoc = true; - break; - } - if ((!found_eoc) && (input_iter != input_end)) { - ++input_iter; - ++current_col; - } - } - - if (!found_eoc) { - std::cout << "Incomplete comment block! Add exceptions!" << std::endl; - return retval; - } - } - else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) { - std::string whitespace(what[0]); - input_iter += whitespace.size(); - current_col += whitespace.size(); - found = true; - } - else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) { - const std::string cr(what[0]); - - boost::match_results if_delim; - if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) { - TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename)); - t->start.column = current_col; - t->start.line = current_line; - current_col += t->text.size(); - t->end.column = current_col; - t->end.line = current_line; - retval.push_back(t); - } - - input_iter += cr.size(); - ++current_line; - current_col = 0; - found = true; - } - else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) { - TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename)); - t->start.column = current_col; - t->start.line = current_line; - current_col += t->text.size(); - t->end.column = current_col; - t->end.line = current_line; - retval.push_back(t); - input_iter += t->text.size(); - found = true; - } - else { - for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { - if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) { - TokenPtr t(new Token(what[0], iter->identifier, filename)); - t->start.column = current_col; - t->start.line = current_line; - current_col += t->text.size(); - t->end.column = current_col; - t->end.line = current_line; - retval.push_back(t); - input_iter += t->text.size(); - found = true; - break; - } - } - - if (!found) { - const std::string err(input_iter, input_end); - std::cout << "Unknown string at: " << err << std::endl; - return retval; - } - } - } - return retval; -} diff --git a/langkit/langkit_lexer.hpp b/langkit/langkit_lexer.hpp index 7d8d7bc..c5629f0 100644 --- a/langkit/langkit_lexer.hpp +++ b/langkit/langkit_lexer.hpp @@ -48,8 +48,139 @@ struct Lexer { Pattern multiline_comment_end_pattern; Pattern singleline_comment_pattern; - Lexer operator<<(const Pattern &p); - std::vector lex(const std::string &input, const char *fname); + Lexer Lexer::operator<<(const Pattern &p) { + lex_patterns.push_back(p); + return *this; + } + + std::vector Lexer::lex(const std::string &input, const char *filename) { + std::vector::iterator iter, end, iter2, end2; + std::vector retval; + bool found; + std::string::const_iterator input_iter = input.begin(), input_end = input.end(); + + int current_col = 0; + int current_line = 0; + boost::match_results what; + + while (input_iter != input_end) { + found = false; + + if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) { + std::string comment_start(what[0]); + input_iter += comment_start.size(); + + bool found_eol = false; + + while ((!found_eol) && (input_iter != input_end)) { + boost::match_results eol_delim; + if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { + std::string comment_end(eol_delim[0]); + input_iter += comment_end.size(); + ++current_line; + current_col = 0; + found_eol = true; + break; + } + if ((!found_eol) && (input_iter != input_end)) { + ++input_iter; + } + } + } + else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) { + std::string comment_start(what[0]); + input_iter += comment_start.size(); + + bool found_eoc = false; + + while ((!found_eoc) && (input_iter != input_end)) { + boost::match_results eol_delim; + if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) { + std::string comment_end(eol_delim[0]); + input_iter += comment_end.size(); + ++current_line; + current_col = 0; + } + boost::match_results eoc_delim; + if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) { + std::string comment_end(eoc_delim[0]); + input_iter += comment_end.size(); + current_col += comment_end.size(); + found_eoc = true; + break; + } + if ((!found_eoc) && (input_iter != input_end)) { + ++input_iter; + ++current_col; + } + } + + if (!found_eoc) { + std::cout << "Incomplete comment block! Add exceptions!" << std::endl; + return retval; + } + } + else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) { + std::string whitespace(what[0]); + input_iter += whitespace.size(); + current_col += whitespace.size(); + found = true; + } + else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) { + const std::string cr(what[0]); + + boost::match_results if_delim; + if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) { + TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename)); + t->start.column = current_col; + t->start.line = current_line; + current_col += t->text.size(); + t->end.column = current_col; + t->end.line = current_line; + retval.push_back(t); + } + + input_iter += cr.size(); + ++current_line; + current_col = 0; + found = true; + } + else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) { + TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename)); + t->start.column = current_col; + t->start.line = current_line; + current_col += t->text.size(); + t->end.column = current_col; + t->end.line = current_line; + retval.push_back(t); + input_iter += t->text.size(); + found = true; + } + else { + for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { + if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) { + TokenPtr t(new Token(what[0], iter->identifier, filename)); + t->start.column = current_col; + t->start.line = current_line; + current_col += t->text.size(); + t->end.column = current_col; + t->end.line = current_line; + retval.push_back(t); + input_iter += t->text.size(); + found = true; + break; + } + } + + if (!found) { + const std::string err(input_iter, input_end); + std::cout << "Unknown string at: " << err << std::endl; + return retval; + } + } + } + return retval; + } void set_skip(const Pattern &p) { skip_pattern = p; diff --git a/wesley/main.cpp b/wesley/main.cpp index 0d28c6a..25213c1 100644 --- a/wesley/main.cpp +++ b/wesley/main.cpp @@ -199,6 +199,7 @@ Rule build_parser_rules() { Rule source_elem; Rule source_elems; Rule statement_list; + Rule paren_block; Rule rule = *(Ign(Id(TokenType::Semicolon))) >> source_elems >> *(Ign(Id(TokenType::Semicolon))); @@ -216,6 +217,7 @@ Rule build_parser_rules() { block; params = Id(TokenType::Identifier) >> *(Ign(Str(",")) >> Id(TokenType::Identifier)); block = *(Ign(Id(TokenType::Semicolon))) >> Ign(Id(TokenType::Curly_Open)) >> *(Ign(Id(TokenType::Semicolon))) >> ~statement_list >> *(Ign(Id(TokenType::Semicolon))) >> Ign(Id(TokenType::Curly_Close)); + equation = *(((vardecl | arraycall | Id(TokenType::Identifier)) >> Str("=")) | ((vardecl | arraycall | Id(TokenType::Identifier)) >> Str("+=")) | ((vardecl | arraycall | Id(TokenType::Identifier)) >> Str("-=")) | @@ -227,18 +229,21 @@ Rule build_parser_rules() { expression = term >> *((Str("+") >> term) | (Str("-") >> term)); term = factor >> *((Str("*") >> factor) | (Str("/") >> factor)); factor = methodcall | arraycall | value | negate | prefix | (Ign(Str("+")) >> value); + value = vardecl | arrayinit | block | paren_block | return_statement | break_statement | + funcall | Id(TokenType::Identifier) | Id(TokenType::Real_Number) | Id(TokenType::Integer) | Id(TokenType::Quoted_String) | + Id(TokenType::Single_Quoted_String) ; + funcall = Id(TokenType::Identifier) >> Ign(Id(TokenType::Parens_Open)) >> ~(boolean >> *(Ign(Str("," )) >> boolean)) >> Ign(Id(TokenType::Parens_Close)); methodcall = value >> +(Ign(Str(".")) >> funcall); negate = Ign(Str("-")) >> boolean; prefix = (Str("++") >> (boolean | arraycall)) | (Str("--") >> (boolean | arraycall)); arraycall = value >> +((Ign(Id(TokenType::Square_Open)) >> boolean >> Ign(Id(TokenType::Square_Close)))); - value = vardecl | arrayinit | block | (Ign(Id(TokenType::Parens_Open)) >> equation >> Ign(Id(TokenType::Parens_Close))) | return_statement | break_statement | - funcall | Id(TokenType::Identifier) | Id(TokenType::Real_Number) | Id(TokenType::Integer) | Id(TokenType::Quoted_String) | - Id(TokenType::Single_Quoted_String) ; + arrayinit = Ign(Id(TokenType::Square_Open)) >> ~(boolean >> *(Ign(Str(",")) >> boolean)) >> Ign(Id(TokenType::Square_Close)); vardecl = Ign(Str("var")) >> Id(TokenType::Identifier); return_statement = Ign(Str("return")) >> ~boolean; break_statement = Wrap(Ign(Str("break"))); + paren_block = (Ign(Id(TokenType::Parens_Open)) >> equation >> Ign(Id(TokenType::Parens_Close))); return rule; }