generic-library/ChaiScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Renamed 'Nop' to 'Epsilon' to match spirit. Debugged non-moving parsers. Added 'Wrap' rule that will allow a Rule to wrap a single other Rule and get a tree for the node.

Browse Source
This commit is contained in:
Jonathan Turner
2009-06-10 14:10:15 +00:00
parent c4e3328bb1
commit e7a0bf70fd
4 changed files with 92 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
langkit/langkit_parser.cpp
View File

@@ -55,8 +55,12 @@ std::pair<Token_Iterator, bool> Or_Rule(Token_Iterator iter, Token_Iterator end,
if (new_id != -1) { if (new_id != -1) {
parent->filename = (*iter)->filename; parent->filename = (*iter)->filename;
parent->start = (*iter)->start; parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end; if (result.first == iter) {
parent->end = (*iter)->start;
}
else {
parent->end = (*(result.first - 1))->end;
}
prev_parent->children.push_back(parent); prev_parent->children.push_back(parent);
} }
return std::pair<Token_Iterator, bool>(result.first, true); return std::pair<Token_Iterator, bool>(result.first, true);
@@ -73,8 +77,12 @@ std::pair<Token_Iterator, bool> Or_Rule(Token_Iterator iter, Token_Iterator end,
parent->filename = (*iter)->filename; parent->filename = (*iter)->filename;
parent->start = (*iter)->start; parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end; if (result.first == iter) {
parent->end = (*iter)->start;
}
else {
parent->end = (*(result.first - 1))->end;
}
prev_parent->children.push_back(parent); prev_parent->children.push_back(parent);
} }
@@ -112,8 +120,12 @@ std::pair<Token_Iterator, bool> And_Rule(Token_Iterator iter, Token_Iterator end
parent->filename = (*iter)->filename; parent->filename = (*iter)->filename;
parent->start = (*iter)->start; parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end; if (result.first == iter) {
parent->end = (*iter)->start;
}
else {
parent->end = (*(result.first - 1))->end;
}
prev_parent->children.push_back(parent); prev_parent->children.push_back(parent);
} }
@@ -152,7 +164,12 @@ std::pair<Token_Iterator, bool> Kleene_Rule
parent->filename = (*iter)->filename; parent->filename = (*iter)->filename;
parent->start = (*iter)->start; parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end; if (result.first == iter) {
parent->end = (*iter)->start;
}
else {
parent->end = (*(result.first - 1))->end;
}
prev_parent->children.push_back(parent); prev_parent->children.push_back(parent);
} }
@@ -192,8 +209,12 @@ std::pair<Token_Iterator, bool> Plus_Rule
parent->filename = (*iter)->filename; parent->filename = (*iter)->filename;
parent->start = (*iter)->start; parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end; if (result.first == iter) {
parent->end = (*iter)->start;
}
else {
parent->end = (*(result.first - 1))->end;
}
prev_parent->children.push_back(parent); prev_parent->children.push_back(parent);
} }
@@ -232,8 +253,12 @@ std::pair<Token_Iterator, bool> Optional_Rule
parent->filename = (*iter)->filename; parent->filename = (*iter)->filename;
parent->start = (*iter)->start; parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end; if (result.first == iter) {
parent->end = (*iter)->start;
}
else {
parent->end = (*(result.first - 1))->end;
}
prev_parent->children.push_back(parent); prev_parent->children.push_back(parent);
} }
@@ -244,7 +269,7 @@ std::pair<Token_Iterator, bool> Optional_Rule
} }
} }
std::pair<Token_Iterator, bool> Nop_Rule std::pair<Token_Iterator, bool> Epsilon_Rule
(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, Rule rule) { (Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, Rule rule) {
TokenPtr prev_parent = parent; TokenPtr prev_parent = parent;
@@ -255,8 +280,7 @@ std::pair<Token_Iterator, bool> Nop_Rule
} }
std::pair<Token_Iterator, bool> result; std::pair<Token_Iterator, bool> result;
result.second = true; if ((new_iter != end)) {
if ((new_iter != end) && (result.second == true)) {
result = rule(new_iter, end, parent); result = rule(new_iter, end, parent);
new_iter = result.first; new_iter = result.first;
} }
@@ -264,12 +288,49 @@ std::pair<Token_Iterator, bool> Nop_Rule
if (new_id != -1) { if (new_id != -1) {
parent->filename = (*iter)->filename; parent->filename = (*iter)->filename;
parent->start = (*iter)->start; parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end; if (result.first == iter) {
parent->end = (*iter)->start;
}
else {
parent->end = (*(result.first - 1))->end;
}
prev_parent->children.push_back(parent); prev_parent->children.push_back(parent);
} }
return std::pair<Token_Iterator, bool>(iter, true); return std::pair<Token_Iterator, bool>(iter, result.second);
}
std::pair<Token_Iterator, bool> Wrap_Rule
(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, Rule rule) {
TokenPtr prev_parent = parent;
Token_Iterator new_iter = iter;
if (new_id != -1) {
parent = TokenPtr(new Token("", new_id, parent->filename));
}
std::pair<Token_Iterator, bool> result;
if ((new_iter != end)) {
result = rule(new_iter, end, parent);
new_iter = result.first;
}
if (new_id != -1) {
parent->filename = (*iter)->filename;
parent->start = (*iter)->start;
if (result.first == iter) {
parent->end = (*iter)->start;
}
else {
parent->end = (*(result.first - 1))->end;
}
prev_parent->children.push_back(parent);
}
return std::pair<Token_Iterator, bool>(result.first, result.second);
} }
Rule Str(const std::string &text, bool keep) { Rule Str(const std::string &text, bool keep) {
@@ -294,6 +355,10 @@ Rule Ign(Rule rule) {
return rule; return rule;
} }
Rule Nop(Rule rule) { Rule Epsilon(Rule rule) {
return Rule(boost::bind(Nop_Rule, _1, _2, _3, _4, _5, rule)); return Rule(boost::bind(Epsilon_Rule, _1, _2, _3, _4, _5, rule));
}
Rule Wrap(Rule rule) {
return Rule(boost::bind(Wrap_Rule, _1, _2, _3, _4, _5, rule));
} }

8
langkit/langkit_parser.hpp
View File

@@ -50,7 +50,10 @@ std::pair<Token_Iterator, bool> Plus_Rule
std::pair<Token_Iterator, bool> Optional_Rule std::pair<Token_Iterator, bool> Optional_Rule
(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, struct Rule rule); (Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, struct Rule rule);
std::pair<Token_Iterator, bool> Nop_Rule std::pair<Token_Iterator, bool> Epsilon_Rule
(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, struct Rule rule);
std::pair<Token_Iterator, bool> Wrap_Rule
(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, struct Rule rule); (Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, struct Rule rule);
struct Rule { struct Rule {
@@ -102,6 +105,7 @@ Rule Str(const std::string &text);
Rule Id(int id); Rule Id(int id);
Rule Ign(Rule rule); Rule Ign(Rule rule);
Rule Nop(Rule rule); Rule Epsilon(Rule rule);
Rule Wrap(Rule rule);
#endif /* LANGKIT_PARSER_HPP_ */ #endif /* LANGKIT_PARSER_HPP_ */

2
langkit/main.cpp
View File

@@ -96,7 +96,7 @@ void parse(std::vector<TokenPtr> &tokens, const char *filename) {
return_statement = Ign(Str("return")) >> expression; return_statement = Ign(Str("return")) >> expression;
*/ */
Rule rule(TokenType::Equation); Rule rule(TokenType::Equation);
rule = Str("break"); rule = Wrap(Str("break"));
std::cout << "Check: " << rule.impl->new_id << std::endl; std::cout << "Check: " << rule.impl->new_id << std::endl;

8
wesley/main.cpp
View File

@@ -226,13 +226,13 @@ Rule build_parser_rules() {
negate = Ign(Str("-")) >> boolean; negate = Ign(Str("-")) >> boolean;
prefix = (Str("++") >> (boolean | arraycall)) | (Str("--") >> (boolean | arraycall)); prefix = (Str("++") >> (boolean | arraycall)) | (Str("--") >> (boolean | arraycall));
arraycall = value >> +((Ign(Id(TokenType::Square_Open)) >> boolean >> Ign(Id(TokenType::Square_Close)))); arraycall = value >> +((Ign(Id(TokenType::Square_Open)) >> boolean >> Ign(Id(TokenType::Square_Close))));
value = vardecl | arrayinit | block | (Ign(Id(TokenType::Parens_Open)) >> boolean >> Ign(Id(TokenType::Parens_Close))) | return_statement | value = vardecl | arrayinit | block | (Ign(Id(TokenType::Parens_Open)) >> boolean >> Ign(Id(TokenType::Parens_Close))) | return_statement | break_statement |
funcall | Id(TokenType::Identifier) | Id(TokenType::Real_Number) | Id(TokenType::Integer) | Id(TokenType::Quoted_String) | funcall | Id(TokenType::Identifier) | Id(TokenType::Real_Number) | Id(TokenType::Integer) | Id(TokenType::Quoted_String) |
Id(TokenType::Single_Quoted_String) ; Id(TokenType::Single_Quoted_String) ;
arrayinit = Ign(Id(TokenType::Square_Open)) >> ~(boolean >> *(Ign(Str(",")) >> boolean)) >> Ign(Id(TokenType::Square_Close)); arrayinit = Ign(Id(TokenType::Square_Open)) >> ~(boolean >> *(Ign(Str(",")) >> boolean)) >> Ign(Id(TokenType::Square_Close));
vardecl = Ign(Str("var")) >> Id(TokenType::Identifier); vardecl = Ign(Str("var")) >> Id(TokenType::Identifier);
return_statement = Ign(Str("return")) >> ~boolean; return_statement = Ign(Str("return")) >> ~boolean;
//break_statement = Ign(Str("break")) >> ~Ign(Id(TokenType::Semicolon)); break_statement = Wrap(Ign(Str("break")));
return rule; return rule;
} }
@@ -287,10 +287,6 @@ Boxed_Value eval_token(BoxedCPP_System &ss, TokenPtr node) {
else if (node->text == "false") { else if (node->text == "false") {
retval = Boxed_Value(false); retval = Boxed_Value(false);
} }
else if (node->text == "break") {
//todo: this is a WORKAROUND for parser combinator limitations
throw BreakLoop(node);
}
else { else {
try { try {
retval = ss.get_object(node->text); retval = ss.get_object(node->text);