Lots of small bugfixes re:parser. pimpl now works with ast keep and new_id

langkit/langkit_parser.cpp

@@ -9,7 +9,7 @@
 #include "langkit_lexer.hpp"
 #include "langkit_parser.hpp"
 
-std::pair<Token_Iterator, bool> String_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, const std::string &val, bool keep) {
+std::pair<Token_Iterator, bool> String_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, const std::string &val) {
     if (*iter != *end) {
         if ((*iter)->text == val) {
             if (keep) {
@@ -22,7 +22,7 @@ std::pair<Token_Iterator, bool> String_Rule(Token_Iterator iter, Token_Iterator
     return std::pair<Token_Iterator, bool>(iter, false);
 }
 
-std::pair<Token_Iterator, bool> Type_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, const int val, bool keep) {
+std::pair<Token_Iterator, bool> Type_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, const int val) {
     if (*iter != *end) {
         if ((*iter)->identifier == val) {
             if (keep) {
@@ -35,7 +35,7 @@ std::pair<Token_Iterator, bool> Type_Rule(Token_Iterator iter, Token_Iterator en
     return std::pair<Token_Iterator, bool>(iter, false);
 }
 
-std::pair<Token_Iterator, bool> Or_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, Rule lhs, Rule rhs) {
+std::pair<Token_Iterator, bool> Or_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, Rule lhs, Rule rhs) {
     Token_Iterator new_iter;
     unsigned int prev_size = parent->children.size();
 
@@ -61,7 +61,7 @@ std::pair<Token_Iterator, bool> Or_Rule(Token_Iterator iter, Token_Iterator end,
     return std::pair<Token_Iterator, bool>(iter, false);
 }
 
-std::pair<Token_Iterator, bool> And_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, Rule lhs, Rule rhs) {
+std::pair<Token_Iterator, bool> And_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, Rule lhs, Rule rhs) {
     Token_Iterator lhs_iter, rhs_iter;
     unsigned int prev_size = parent->children.size();
 
@@ -84,22 +84,23 @@ std::pair<Token_Iterator, bool> And_Rule(Token_Iterator iter, Token_Iterator end
     return std::pair<Token_Iterator, bool>(iter, false);
 }
 
-std::pair<Token_Iterator, bool> Rule::operator()(Token_Iterator iter, Token_Iterator end, TokenPtr parent) {
-    return impl->rule(iter, end, parent);
-}
-
 Rule Str(const std::string &text, bool keep) {
-    return Rule(boost::bind(String_Rule, _1, _2, _3, text, keep));
+    return Rule(boost::bind(String_Rule, _1, _2, _3, _4, _5, text), keep);
 }
 Rule Id(int id, bool keep) {
-    return Rule(boost::bind(Type_Rule, _1, _2, _3, id, keep));
+    return Rule(boost::bind(Type_Rule, _1, _2, _3, _4, _5, id), keep);
 }
 
 Rule Str(const std::string &text) {
-    return Rule(boost::bind(String_Rule, _1, _2, _3, text, false));
+    return Rule(boost::bind(String_Rule, _1, _2, _3, _4, _5, text));
 }
 
 Rule Id(int id) {
-    return Rule(boost::bind(Type_Rule, _1, _2, _3, id, false));
+    return Rule(boost::bind(Type_Rule, _1, _2, _3, _4, _5, id));
 }
 
+Rule Ign(Rule rule) {
+    rule.impl->keep = false;
+
+    return rule;
+}

langkit/langkit_parser.hpp

@@ -9,60 +9,64 @@
 #include "langkit_lexer.hpp"
 
 typedef std::vector<TokenPtr>::iterator Token_Iterator;
+typedef boost::function<std::pair<Token_Iterator, bool>(Token_Iterator, Token_Iterator, TokenPtr, bool, int)> RuleFun;
 typedef std::tr1::shared_ptr<struct RuleImpl> RuleImplPtr;
-typedef boost::function<std::pair<Token_Iterator, bool>(Token_Iterator iter, Token_Iterator end, TokenPtr parent)> RuleFun;
 
 struct RuleImpl {
-    int identifier;
     RuleFun rule;
+    bool keep;
+    int new_id;
 
-    RuleImpl() : identifier(-1) {}
+    RuleImpl() : keep(true), new_id(-1) { }
-    RuleImpl(int id) : identifier(id) {}
+
-    RuleImpl(RuleFun fun) : rule(fun) {}
+    std::pair<Token_Iterator, bool> operator()(Token_Iterator iter, Token_Iterator end, TokenPtr parent) {
+        return rule(iter, end, parent, keep, new_id);
+    }
 };
 
 std::pair<Token_Iterator, bool> String_Rule
-    (Token_Iterator iter, Token_Iterator end, TokenPtr parent, const std::string &val, bool keep);
+    (Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, const std::string &val);
 
 std::pair<Token_Iterator, bool> Type_Rule
-    (Token_Iterator iter, Token_Iterator end, TokenPtr parent, const int val, bool keep);
+    (Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, const int val);
 
 std::pair<Token_Iterator, bool> Or_Rule
-    (Token_Iterator iter, Token_Iterator end, TokenPtr parent, struct Rule lhs, struct Rule rhs);
+    (Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, struct Rule lhs, struct Rule rhs);
 
 std::pair<Token_Iterator, bool> And_Rule
-    (Token_Iterator iter, Token_Iterator end, TokenPtr parent, struct Rule lhs, struct Rule rhs);
+    (Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, struct Rule lhs, struct Rule rhs);
 
 struct Rule {
-    Rule() : impl(new RuleImpl(-1)) {}
+    RuleImplPtr impl;
-    Rule(int id) : impl(new RuleImpl(id)) {}
-    Rule(RuleFun fun) : impl(new RuleImpl(fun)) {}
 
-    std::pair<Token_Iterator, bool> operator()(Token_Iterator iter, Token_Iterator end, TokenPtr parent);
+    Rule() : impl(new RuleImpl()) { }
+    Rule(int id) : impl(new RuleImpl()) { impl->new_id = id; }
+    Rule(RuleFun fun) : impl(new RuleImpl()) { impl->rule = fun; }
+    Rule(RuleFun fun, bool keep) : impl(new RuleImpl()) { impl->rule = fun; impl->keep = keep; }
+
+    std::pair<Token_Iterator, bool> operator()(Token_Iterator iter, Token_Iterator end, TokenPtr parent) {
+        return (*impl)(iter, end, parent);
+    }
 
     Rule &operator=(const Rule &rule) {
-        impl->identifier = rule.get_impl()->identifier;
+        *impl = *(rule.impl);
-        impl->rule = rule.get_impl()->rule;
 
         return *this;
     }
 
     Rule operator|(const Rule &rhs) {
-        return Rule(boost::bind(Or_Rule, _1, _2, _3, *this, rhs));
+        return Rule(boost::bind(Or_Rule, _1, _2, _3, _4, _5, *this, rhs));
     }
 
-    Rule operator&(const Rule &rhs) {
+    Rule operator<<(const Rule &rhs) {
-        return Rule(boost::bind(And_Rule, _1, _2, _3, *this, rhs));
+        return Rule(boost::bind(And_Rule, _1, _2, _3, _4, _5, *this, rhs));
     }
 
-    void set_rule(RuleFun fun) {
+    //const RuleImplPtr get_impl() const { return impl; }
-        impl->rule = fun;
-    }
 
-    const RuleImplPtr get_impl() const { return impl; }
+//private:
+    //RuleImplPtr impl;
 
-private:
-    RuleImplPtr impl;
 };
 
 
@@ -72,4 +76,6 @@ Rule Id(int id, bool keep);
 Rule Str(const std::string &text);
 Rule Id(int id);
 
+Rule Ign(Rule rule);
+
 #endif /* LANGKIT_PARSER_HPP_ */

langkit/main.cpp

@@ -48,15 +48,15 @@ std::string load_file(const char *filename) {
 }
 
 void parse(std::vector<TokenPtr> &tokens) {
-    /*
+
     Rule lhs;
     Rule rhs;
-    Rule rule = lhs & rhs;
+    Rule rule = lhs << rhs;
     lhs = Str("def", true);
     rhs = Id(TokenType::Identifier, true);
-    */
 
-    Rule rule = Str("def") & Id(TokenType::Identifier, true);
+
+    //Rule rule = Str("def", false) << Id(TokenType::Identifier, true);
 
     Token_Iterator iter = tokens.begin(), end = tokens.end();
     TokenPtr parent(new Token("Root", 0, "test"));