Starting to stand up the parser using std::pair

langkit/langkit_lexer.hpp

@@ -30,6 +30,8 @@ struct Token {
     int identifier;
     File_Position start, end;
 
+    std::vector<Token> children;
+
     Token(const std::string &token_text, int id) : text(token_text), identifier(id) { }
 };
 

langkit/langkit_parser.cpp

@@ -1,49 +1,52 @@
 // This file is distributed under the BSD License.
 // See LICENSE.TXT for details.
 
+#include <utility>
+
 #include "langkit_lexer.hpp"
 #include "langkit_parser.hpp"
 
-Token_Iterator String_Rule(Token_Iterator iter, Token_Iterator end, const std::string &val) {
+std::pair<Token_Iterator, Token> String_Rule(Token_Iterator iter, Token_Iterator end, const std::string &val) {
     if (iter != end) {
         if (iter->text == val) {
-            return ++iter;
+            return std::pair<Token_Iterator, Token>(++iter, *iter);
         }
     }
 
-    return iter;
+    return std::pair<Token_Iterator, Token>(iter, *iter);
 }
 
-Token_Iterator Type_Rule(Token_Iterator iter, Token_Iterator end, const int val) {
+std::pair<Token_Iterator, Token> Type_Rule(Token_Iterator iter, Token_Iterator end, const int val) {
     if (iter != end) {
         if (iter->identifier == val) {
-            return ++iter;
+            return std::pair<Token_Iterator, Token>(++iter, *iter);
         }
     }
 
-    return iter;
+    return std::pair<Token_Iterator, Token>(iter, *iter);
 }
 
-Token_Iterator Or_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs) {
+std::pair<Token_Iterator, Token> Or_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs) {
     Token_Iterator new_iter;
 
     if (iter != end) {
-        new_iter = lhs.rule(iter, end);
+        new_iter = lhs.rule(iter, end).first;
 
         if (new_iter != iter) {
-            return new_iter;
+            return std::pair<Token_Iterator, Token>(new_iter, *iter);
         }
         else {
-            new_iter = rhs.rule(iter, end);
+            new_iter = rhs.rule(iter, end).first;
             if (new_iter != iter) {
-                return new_iter;
+                return std::pair<Token_Iterator, Token>(new_iter, *iter);
             }
         }
     }
-    return iter;
+    return std::pair<Token_Iterator, Token>(iter, *iter);
 }
 
-Token_Iterator And_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs) {
+/*
+std::pair<Token_Iterator, Token> And_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs) {
     Token_Iterator lhs_iter, rhs_iter;
 
     if (iter != end) {
@@ -52,10 +55,15 @@ Token_Iterator And_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs
         if (lhs_iter != iter) {
             rhs_iter = rhs.rule(iter, end);
             if (rhs_iter != iter) {
-                return rhs_iter;
+                return std::pair<Token_Iterator, Token>(rhs_iter, *iter);
             }
         }
     }
 
-    return iter;
+    return std::pair<Token_Iterator, Token>(iter, *iter);
+}
+*/
+
+std::pair<Token_Iterator, Token> Rule::operator()(Token_Iterator iter, Token_Iterator end) {
+    return this->rule(iter, end);
 }

langkit/langkit_parser.hpp

@@ -11,7 +11,18 @@
 typedef std::vector<Token>::iterator Token_Iterator;
 
 struct Rule {
-    boost::function<Token_Iterator(Token_Iterator iter, Token_Iterator end)> rule;
+    int identifier;
+    boost::function<std::pair<Token_Iterator, Token>(Token_Iterator iter, Token_Iterator end)> rule;
+    std::pair<Token_Iterator, Token> operator()(Token_Iterator iter, Token_Iterator end);
+
+    Rule() : identifier(-1) {}
+    Rule(int id) : identifier(id) {}
 };
 
+std::pair<Token_Iterator, Token> String_Rule(Token_Iterator iter, Token_Iterator end, const std::string &val);
+std::pair<Token_Iterator, Token> Type_Rule(Token_Iterator iter, Token_Iterator end, const int val);
+std::pair<Token_Iterator, Token> Or_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs);
+
+
+
 #endif /* LANGKIT_PARSER_HPP_ */

langkit/main.cpp

@@ -1,22 +1,25 @@
 // This file is distributed under the BSD License.
 // See LICENSE.TXT for details.
 
+#include <boost/bind.hpp>
+
 #include <iostream>
 #include <map>
 #include <fstream>
 
 #include "langkit_lexer.hpp"
-//#include "lexer.hpp"
-//#include "parser.hpp"
-//#include "eval.hpp"
+#include "langkit_parser.hpp"
 
 class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close,
     Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon }; };
 
+void debug_print(Token &token) {
+    std::cout << "Token: " << token.text << "(" << token.identifier << ") @ (" << token.start.column
+        << ", " << token.start.line << ") to (" << token.end.column << ", " << token.end.line << ") " << std::endl;
+}
 void debug_print(std::vector<Token> &tokens) {
     for (unsigned int i = 0; i < tokens.size(); ++i) {
-        std::cout << "Token: " << tokens[i].text << "(" << tokens[i].identifier << ") @ (" << tokens[i].start.column
-            << ", " << tokens[i].start.line << ") to (" << tokens[i].end.column << ", " << tokens[i].end.line << ") " << std::endl;
+        debug_print(tokens[i]);
     }
 }
 
@@ -39,6 +42,23 @@ std::string load_file(const char *filename) {
     return ret_val;
 }
 
+void parse(std::vector<Token> &tokens) {
+    Rule rule;
+    rule.rule = boost::bind(String_Rule, _1, _2, "def");
+
+    Token_Iterator iter = tokens.begin(), end = tokens.end();
+
+
+    std::pair<Token_Iterator, Token> results = rule(iter, end);
+
+    if (results.first != iter) {
+        std::cout << "Parse successful: " << std::endl;
+        debug_print(results.second);
+    }
+
+}
+
+
 int main(int argc, char *argv[]) {
     std::string input;
 
@@ -66,6 +86,8 @@ int main(int argc, char *argv[]) {
         while (input != "quit") {
             std::vector<Token> tokens = lexer.lex(input);
             debug_print(tokens);
+            parse(tokens);
+
             std::cout << "Expression> ";
             std::getline(std::cin, input);
         }