diff --git a/langkit/langkit_lexer.cpp b/langkit/langkit_lexer.cpp index f694a2a..c0394eb 100644 --- a/langkit/langkit_lexer.cpp +++ b/langkit/langkit_lexer.cpp @@ -11,19 +11,24 @@ Lexer Lexer::operator<<(const Pattern &p) { std::vector Lexer::lex(const std::string &input) { std::vector::iterator iter, end; - //std::string::const_iterator str_end = input.end(); std::vector retval; bool found; std::string::const_iterator input_iter = input.begin(); + int current_col = 0; + int current_line = 0; + while (input_iter != input.end()) { found = false; for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { boost::match_results what; if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { Token t(what[0], iter->identifier); - t.start.column = input_iter - input.begin(); - t.end.column = t.start.column + t.text.size(); + t.start.column = current_col; + t.start.line = current_line; + current_col += t.text.size(); + t.end.column = current_col; + t.end.line = current_line; retval.push_back(t); input_iter += t.text.size(); found = true; @@ -36,15 +41,30 @@ std::vector Lexer::lex(const std::string &input) { if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { std::string whitespace(what[0]); input_iter += whitespace.size(); + current_col += whitespace.size(); found = true; break; } } if (!found) { - const std::string err(input_iter, input.end()); - std::cout << "Unknown string at: " << err << std::endl; - return retval; + for (iter = line_sep_patterns.begin(), end = line_sep_patterns.end(); iter != end; ++iter) { + boost::match_results what; + if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { + std::string cr(what[0]); + input_iter += cr.size(); + found = true; + ++current_line; + current_col = 0; + break; + } + } + + if (!found) { + const std::string err(input_iter, input.end()); + std::cout << "Unknown string at: " << err << std::endl; + return retval; + } } } } @@ -54,3 +74,11 @@ std::vector Lexer::lex(const std::string &input) { void Lexer::set_skip(const Pattern &p) { skip_patterns.push_back(p); } + +void Lexer::set_line_sep(const Pattern &p) { + line_sep_patterns.push_back(p); +} + +void Lexer::set_command_sep(const Pattern &p) { + command_sep_patterns.push_back(p); +} diff --git a/langkit/langkit_lexer.hpp b/langkit/langkit_lexer.hpp index 49e97d2..479ccd7 100644 --- a/langkit/langkit_lexer.hpp +++ b/langkit/langkit_lexer.hpp @@ -8,14 +8,14 @@ #include struct File_Position { - int row; + int line; int column; char *filename; - File_Position(int file_row, int file_column, char *fname) - : row(file_row), column(file_column), filename(fname) { } + File_Position(int file_line, int file_column, char *fname) + : line(file_line), column(file_column), filename(fname) { } - File_Position() : row(0), column(0), filename(NULL) { } + File_Position() : line(0), column(0), filename(NULL) { } }; struct Pattern { @@ -36,11 +36,15 @@ struct Token { struct Lexer { std::vector lex_patterns; std::vector skip_patterns; + std::vector command_sep_patterns; + std::vector line_sep_patterns; Lexer operator<<(const Pattern &p); std::vector lex(const std::string &input); void set_skip(const Pattern &p); + void set_line_sep(const Pattern &p); + void set_command_sep(const Pattern &p); }; diff --git a/langkit/main.cpp b/langkit/main.cpp index c5f012c..4af9c65 100644 --- a/langkit/main.cpp +++ b/langkit/main.cpp @@ -3,6 +3,7 @@ #include #include +#include #include "langkit_lexer.hpp" //#include "lexer.hpp" @@ -10,23 +11,47 @@ //#include "eval.hpp" class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, - Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String }; }; + Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon }; }; void debug_print(std::vector &tokens) { for (unsigned int i = 0; i < tokens.size(); ++i) { - std::cout << "Token: " << tokens[i].text << "(" << tokens[i].identifier << ") @ " << tokens[i].start.column - << " to " << tokens[i].end.column << std::endl; + std::cout << "Token: " << tokens[i].text << "(" << tokens[i].identifier << ") @ (" << tokens[i].start.column + << ", " << tokens[i].start.line << ") to (" << tokens[i].end.column << ", " << tokens[i].end.line << ") " << std::endl; } } +std::string load_file(const char *filename) { + std::ifstream infile (filename, std::ios::in | std::ios::ate); + + if (!infile.is_open()) { + std::cerr << "Can not open " << filename << std::endl; + exit(0); + } + + std::streampos size = infile.tellg(); + infile.seekg(0, std::ios::beg); + + std::cout << "Allocating: " << size << " bytes" << std::endl; + + std::vector v(size); + infile.read(&v[0], size); + + std::string ret_val (v.empty() ? std::string() : std::string (v.begin(), v.end()).c_str()); + + return ret_val; +} + int main(int argc, char *argv[]) { std::string input; Lexer lexer; - lexer.set_skip(Pattern("\\s+", TokenType::Whitespace)); + lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace)); + lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return)); + lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon)); + lexer << Pattern("[A-Za-z]+", TokenType::Identifier); lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number); - lexer << Pattern("[!@#$%^&*\\-+=/:]+", TokenType::Operator); + lexer << Pattern("[!@#$%^&*\\-+=/<>]+", TokenType::Operator); lexer << Pattern("\\(", TokenType::Parens_Open); lexer << Pattern("\\)", TokenType::Parens_Close); lexer << Pattern("\\[", TokenType::Square_Open); @@ -34,57 +59,21 @@ int main(int argc, char *argv[]) { lexer << Pattern("\\{", TokenType::Curly_Open); lexer << Pattern("\\}", TokenType::Curly_Close); lexer << Pattern(",", TokenType::Comma); - lexer << Pattern("[!@#$%^&*\\-+=/<>]+", TokenType::Operator); lexer << Pattern("\"(?:[^\"\\\\]|\\\\.)*\"", TokenType::Quoted_String); lexer << Pattern("'(?:[^'\\\\]|\\\\.)*'", TokenType::Single_Quoted_String); - std::cout << "Expression> "; - std::getline(std::cin, input); - while (input != "quit") { - std::vector tokens = lexer.lex(input); + if (argc < 2) { + std::cout << "Expression> "; + std::getline(std::cin, input); + while (input != "quit") { + std::vector tokens = lexer.lex(input); + debug_print(tokens); + std::cout << "Expression> "; + std::getline(std::cin, input); + } + } + else { + std::vector tokens = lexer.lex(load_file(argv[1])); debug_print(tokens); - std::cout << "Expression> "; - std::getline(std::cin, input); } } - -/* -int main(int argc, char *argv[]) { - std::string input; - std::map symbols; - - std::cout << "Expression> "; - std::getline(std::cin, input); - while (input != "quit") { - if (input == "vars") { - //debug_print(symbols); - } - else { - try { - NodePtr lex_nodes = lex(input); - //clean_whitespace(lex_nodes); - //NodePtr parse_nodes = parse(lex_nodes); - //Result result = eval(parse_nodes, symbols); - - //debug_print(lex_nodes); - //std::cout << std::endl; - //debug_print(parse_nodes, ""); - //if (result.type == ResultType::NUMBER) { - // std::cout << result.value << std::endl; - //} - } - catch (LexerError &le) { - std::cerr << "Lexer error: " << le.reason << std::endl; - } - //catch (ParserError &pe) { - // std::cerr << "Parser error: " << pe.reason << std::endl; - //} - //catch (EvalError &ee) { - // std::cerr << "Eval error: " << ee.reason << std::endl; - //} - } - std::cout << "Expression> "; - std::getline(std::cin, input); - } -} -*/ diff --git a/langkit/tests/multiline_test.gt b/langkit/tests/multiline_test.gt new file mode 100644 index 0000000..1bbef88 --- /dev/null +++ b/langkit/tests/multiline_test.gt @@ -0,0 +1,5 @@ +3+4 +7 + 9 + + +5 + 7