Added line counting to lexer.

langkit/langkit_lexer.cpp

@@ -11,19 +11,24 @@ Lexer Lexer::operator<<(const Pattern &p) {
 
 std::vector<Token> Lexer::lex(const std::string &input) {
     std::vector<Pattern>::iterator iter, end;
-    //std::string::const_iterator str_end = input.end();
     std::vector<Token> retval;
     bool found;
     std::string::const_iterator input_iter = input.begin();
 
+    int current_col = 0;
+    int current_line = 0;
+
     while (input_iter != input.end()) {
         found = false;
         for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) {
             boost::match_results<std::string::const_iterator> what;
             if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
                 Token t(what[0], iter->identifier);
-                t.start.column = input_iter - input.begin();
+                t.start.column = current_col;
-                t.end.column = t.start.column + t.text.size();
+                t.start.line = current_line;
+                current_col += t.text.size();
+                t.end.column = current_col;
+                t.end.line = current_line;
                 retval.push_back(t);
                 input_iter += t.text.size();
                 found = true;
@@ -36,15 +41,30 @@ std::vector<Token> Lexer::lex(const std::string &input) {
                 if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
                     std::string whitespace(what[0]);
                     input_iter += whitespace.size();
+                    current_col += whitespace.size();
                     found = true;
                     break;
                 }
             }
 
             if (!found) {
-                const std::string err(input_iter, input.end());
+                for (iter = line_sep_patterns.begin(), end = line_sep_patterns.end(); iter != end; ++iter) {
-                std::cout << "Unknown string at: " << err << std::endl;
+                    boost::match_results<std::string::const_iterator> what;
-                return retval;
+                    if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
+                        std::string cr(what[0]);
+                        input_iter += cr.size();
+                        found = true;
+                        ++current_line;
+                        current_col = 0;
+                        break;
+                    }
+                }
+
+                if (!found) {
+                    const std::string err(input_iter, input.end());
+                    std::cout << "Unknown string at: " << err << std::endl;
+                    return retval;
+                }
             }
         }
     }
@@ -54,3 +74,11 @@ std::vector<Token> Lexer::lex(const std::string &input) {
 void Lexer::set_skip(const Pattern &p) {
     skip_patterns.push_back(p);
 }
+
+void Lexer::set_line_sep(const Pattern &p) {
+    line_sep_patterns.push_back(p);
+}
+
+void Lexer::set_command_sep(const Pattern &p) {
+    command_sep_patterns.push_back(p);
+}

langkit/langkit_lexer.hpp

@@ -8,14 +8,14 @@
 #include <boost/regex.hpp>
 
 struct File_Position {
-    int row;
+    int line;
     int column;
     char *filename;
 
-    File_Position(int file_row, int file_column, char *fname)
+    File_Position(int file_line, int file_column, char *fname)
-        : row(file_row), column(file_column), filename(fname) { }
+        : line(file_line), column(file_column), filename(fname) { }
 
-    File_Position() : row(0), column(0), filename(NULL) { }
+    File_Position() : line(0), column(0), filename(NULL) { }
 };
 
 struct Pattern {
@@ -36,11 +36,15 @@ struct Token {
 struct Lexer {
     std::vector<Pattern> lex_patterns;
     std::vector<Pattern> skip_patterns;
+    std::vector<Pattern> command_sep_patterns;
+    std::vector<Pattern> line_sep_patterns;
 
     Lexer operator<<(const Pattern &p);
     std::vector<Token> lex(const std::string &input);
 
     void set_skip(const Pattern &p);
+    void set_line_sep(const Pattern &p);
+    void set_command_sep(const Pattern &p);
 };
 
 

langkit/main.cpp

@@ -3,6 +3,7 @@
 
 #include <iostream>
 #include <map>
+#include <fstream>
 
 #include "langkit_lexer.hpp"
 //#include "lexer.hpp"
@@ -10,23 +11,47 @@
 //#include "eval.hpp"
 
 class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close,
-    Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String }; };
+    Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon }; };
 
 void debug_print(std::vector<Token> &tokens) {
     for (unsigned int i = 0; i < tokens.size(); ++i) {
-        std::cout << "Token: " << tokens[i].text << "(" << tokens[i].identifier << ") @ " << tokens[i].start.column
+        std::cout << "Token: " << tokens[i].text << "(" << tokens[i].identifier << ") @ (" << tokens[i].start.column
-            << " to " << tokens[i].end.column << std::endl;
+            << ", " << tokens[i].start.line << ") to (" << tokens[i].end.column << ", " << tokens[i].end.line << ") " << std::endl;
     }
 }
 
+std::string load_file(const char *filename) {
+    std::ifstream infile (filename, std::ios::in | std::ios::ate);
+
+    if (!infile.is_open()) {
+        std::cerr << "Can not open " << filename << std::endl;
+        exit(0);
+    }
+
+    std::streampos size = infile.tellg();
+    infile.seekg(0, std::ios::beg);
+
+    std::cout << "Allocating: " << size << " bytes" << std::endl;
+
+    std::vector<char> v(size);
+    infile.read(&v[0], size);
+
+    std::string ret_val (v.empty() ? std::string() : std::string (v.begin(), v.end()).c_str());
+
+    return ret_val;
+}
+
 int main(int argc, char *argv[]) {
     std::string input;
 
     Lexer lexer;
-    lexer.set_skip(Pattern("\\s+", TokenType::Whitespace));
+    lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace));
+    lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return));
+    lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon));
+
     lexer << Pattern("[A-Za-z]+", TokenType::Identifier);
     lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number);
-    lexer << Pattern("[!@#$%^&*\\-+=/:]+", TokenType::Operator);
+    lexer << Pattern("[!@#$%^&*\\-+=/<>]+", TokenType::Operator);
     lexer << Pattern("\\(", TokenType::Parens_Open);
     lexer << Pattern("\\)", TokenType::Parens_Close);
     lexer << Pattern("\\[", TokenType::Square_Open);
@@ -34,57 +59,21 @@ int main(int argc, char *argv[]) {
     lexer << Pattern("\\{", TokenType::Curly_Open);
     lexer << Pattern("\\}", TokenType::Curly_Close);
     lexer << Pattern(",", TokenType::Comma);
-    lexer << Pattern("[!@#$%^&*\\-+=/<>]+", TokenType::Operator);
     lexer << Pattern("\"(?:[^\"\\\\]|\\\\.)*\"", TokenType::Quoted_String);
     lexer << Pattern("'(?:[^'\\\\]|\\\\.)*'", TokenType::Single_Quoted_String);
 
-    std::cout << "Expression> ";
+    if (argc < 2) {
-    std::getline(std::cin, input);
+        std::cout << "Expression> ";
-    while (input != "quit") {
+        std::getline(std::cin, input);
-        std::vector<Token> tokens = lexer.lex(input);
+        while (input != "quit") {
+            std::vector<Token> tokens = lexer.lex(input);
+            debug_print(tokens);
+            std::cout << "Expression> ";
+            std::getline(std::cin, input);
+        }
+    }
+    else {
+        std::vector<Token> tokens = lexer.lex(load_file(argv[1]));
         debug_print(tokens);
-        std::cout << "Expression> ";
-        std::getline(std::cin, input);
     }
 }
-
-/*
-int main(int argc, char *argv[]) {
-    std::string input;
-    std::map<std::string, NodePtr> symbols;
-
-    std::cout << "Expression> ";
-    std::getline(std::cin, input);
-    while (input != "quit") {
-        if (input == "vars") {
-            //debug_print(symbols);
-        }
-        else {
-            try {
-                NodePtr lex_nodes = lex(input);
-                //clean_whitespace(lex_nodes);
-                //NodePtr parse_nodes = parse(lex_nodes);
-                //Result result = eval(parse_nodes, symbols);
-
-                //debug_print(lex_nodes);
-                //std::cout << std::endl;
-                //debug_print(parse_nodes, "");
-                //if (result.type == ResultType::NUMBER) {
-                //    std::cout << result.value << std::endl;
-                //}
-            }
-            catch (LexerError &le) {
-                std::cerr << "Lexer error: " << le.reason << std::endl;
-            }
-            //catch (ParserError &pe) {
-            //    std::cerr << "Parser error: " << pe.reason << std::endl;
-            //}
-            //catch (EvalError &ee) {
-            //    std::cerr << "Eval error: " << ee.reason << std::endl;
-            //}
-        }
-        std::cout << "Expression> ";
-        std::getline(std::cin, input);
-    }
-}
-*/

langkit/tests/multiline_test.gt

@@ -0,0 +1,5 @@
+3+4
+7 +		9
+
+
+5 + 7