generic-library/ChaiScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added comment lexing. Removed previous optimization. Flattened lex grammar.

Browse Source
This commit is contained in:
Jonathan Turner 2009-06-01 13:40:24 +00:00
parent f692834fa8
commit 307e557e5b
3 changed files with 127 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
langkit/langkit_lexer.cpp
View File

@ -10,53 +10,112 @@ Lexer Lexer::operator<<(const Pattern &p) {
} }
std::vector<TokenPtr> Lexer::lex(const std::string &input, const char *filename) { std::vector<TokenPtr> Lexer::lex(const std::string &input, const char *filename) {
std::vector<Pattern>::iterator iter, end, iter2; std::vector<Pattern>::iterator iter, end, iter2, end2;
std::vector<TokenPtr> retval; std::vector<TokenPtr> retval;
bool found; bool found;
std::string::const_iterator input_iter = input.begin(); std::string::const_iterator input_iter = input.begin(), input_end = input.end();
int current_col = 0; int current_col = 0;
int current_line = 0; int current_line = 0;
boost::match_results<std::string::const_iterator> what;
std::string master_lex_pattern; while (input_iter != input_end) {
unsigned int i = 0;
for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) {
if (i > 0) {
master_lex_pattern += "|";
}
master_lex_pattern += "(" + iter->regex_string + ")";
++i;
}
boost::regex lex_regex(master_lex_pattern);
while (input_iter != input.end()) {
found = false; found = false;
/* if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) {
for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { std::string comment_start(what[0]);
boost::match_results<std::string::const_iterator> what; input_iter += comment_start.size();
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
TokenPtr t(new Token(what[0], iter->identifier, filename)); bool found_eol = false;
while ((!found_eol) && (input_iter != input_end)) {
boost::match_results<std::string::const_iterator> eol_delim;
if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) {
std::string comment_end(eol_delim[0]);
input_iter += comment_end.size();
++current_line;
current_col = 0;
found_eol = true;
break;
}
if ((!found_eol) && (input_iter != input_end)) {
++input_iter;
}
}
}
else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) {
std::string comment_start(what[0]);
input_iter += comment_start.size();
bool found_eoc = false;
while ((!found_eoc) && (input_iter != input_end)) {
boost::match_results<std::string::const_iterator> eol_delim;
if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) {
std::string comment_end(eol_delim[0]);
input_iter += comment_end.size();
++current_line;
current_col = 0;
break;
}
boost::match_results<std::string::const_iterator> eoc_delim;
if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) {
std::string comment_end(eoc_delim[0]);
input_iter += comment_end.size();
current_col += comment_end.size();
found_eoc = true;
break;
}
if ((!found_eoc) && (input_iter != input_end)) {
++input_iter;
}
}
if (!found_eoc) {
std::cout << "Incomplete comment block! Add exceptions!" << std::endl;
return retval;
}
}
else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) {
std::string whitespace(what[0]);
input_iter += whitespace.size();
current_col += whitespace.size();
found = true;
}
else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) {
const std::string cr(what[0]);
boost::match_results<std::string::const_iterator> if_delim;
if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) {
TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename));
t->start.column = current_col; t->start.column = current_col;
t->start.line = current_line; t->start.line = current_line;
current_col += t->text.size(); current_col += t->text.size();
t->end.column = current_col; t->end.column = current_col;
t->end.line = current_line; t->end.line = current_line;
retval.push_back(t); retval.push_back(t);
input_iter += t->text.size();
found = true;
break;
} }
}
*/
boost::match_results<std::string::const_iterator> what; input_iter += cr.size();
if (regex_search(input_iter, input.end(), what, lex_regex, boost::match_continuous)) { ++current_line;
for (i = 0; i < lex_patterns.size(); ++i) { current_col = 0;
if (!(std::string(what[i+1])).empty()) { found = true;
TokenPtr t(new Token(what[i+1], lex_patterns[i].identifier, filename)); }
else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) {
TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename));
t->start.column = current_col;
t->start.line = current_line;
current_col += t->text.size();
t->end.column = current_col;
t->end.line = current_line;
retval.push_back(t);
input_iter += t->text.size();
found = true;
}
else {
for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) {
if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) {
TokenPtr t(new Token(what[0], iter->identifier, filename));
t->start.column = current_col; t->start.column = current_col;
t->start.line = current_line; t->start.line = current_line;
current_col += t->text.size(); current_col += t->text.size();
@ -68,87 +127,13 @@ std::vector<TokenPtr> Lexer::lex(const std::string &input, const char *filename)
break; break;
} }
} }
}
if (!found) {
for (iter = skip_patterns.begin(), end = skip_patterns.end(); iter != end; ++iter) {
boost::match_results<std::string::const_iterator> what;
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
std::string whitespace(what[0]);
input_iter += whitespace.size();
current_col += whitespace.size();
found = true;
break;
}
}
if (!found) { if (!found) {
for (iter = line_sep_patterns.begin(), end = line_sep_patterns.end(); iter != end; ++iter) { const std::string err(input_iter, input_end);
boost::match_results<std::string::const_iterator> what; std::cout << "Unknown string at: " << err << std::endl;
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { return retval;
const std::string cr(what[0]);
for (iter2 = command_sep_patterns.begin(), end = command_sep_patterns.end(); iter2 != end; ++iter2) {
boost::match_results<std::string::const_iterator> if_delim;
if (regex_search(cr.begin(), cr.end(), if_delim, iter2->regex, boost::match_continuous)) {
TokenPtr t(new Token(if_delim[0], iter2->identifier, filename));
t->start.column = current_col;
t->start.line = current_line;
current_col += t->text.size();
t->end.column = current_col;
t->end.line = current_line;
retval.push_back(t);
break;
}
}
input_iter += cr.size();
found = true;
++current_line;
current_col = 0;
break;
}
}
if (!found) {
for (iter = command_sep_patterns.begin(), end = command_sep_patterns.end(); iter != end; ++iter) {
boost::match_results<std::string::const_iterator> what;
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
TokenPtr t(new Token(what[0], iter->identifier, filename));
t->start.column = current_col;
t->start.line = current_line;
current_col += t->text.size();
t->end.column = current_col;
t->end.line = current_line;
retval.push_back(t);
input_iter += t->text.size();
found = true;
break;
}
}
if (!found) {
const std::string err(input_iter, input.end());
std::cout << "Unknown string at: " << err << std::endl;
return retval;
}
}
} }
} }
} }
return retval; return retval;
} }
void Lexer::set_skip(const Pattern &p) {
skip_patterns.push_back(p);
}
void Lexer::set_line_sep(const Pattern &p) {
line_sep_patterns.push_back(p);
}
void Lexer::set_command_sep(const Pattern &p) {
command_sep_patterns.push_back(p);
}

32
langkit/langkit_lexer.hpp
View File

@ -20,10 +20,10 @@ struct File_Position {
struct Pattern { struct Pattern {
boost::regex regex; boost::regex regex;
std::string regex_string;
int identifier; int identifier;
Pattern(const std::string &regexp, int id) : regex(regexp), regex_string(regexp), identifier(id) { } Pattern() { }
Pattern(const std::string &regexp, int id) : regex(regexp), identifier(id) { }
}; };
typedef std::tr1::shared_ptr<struct Token> TokenPtr; typedef std::tr1::shared_ptr<struct Token> TokenPtr;
@ -41,16 +41,32 @@ struct Token {
struct Lexer { struct Lexer {
std::vector<Pattern> lex_patterns; std::vector<Pattern> lex_patterns;
std::vector<Pattern> skip_patterns; Pattern skip_pattern;
std::vector<Pattern> command_sep_patterns; Pattern command_sep_pattern;
std::vector<Pattern> line_sep_patterns; Pattern line_sep_pattern;
Pattern multiline_comment_start_pattern;
Pattern multiline_comment_end_pattern;
Pattern singleline_comment_pattern;
Lexer operator<<(const Pattern &p); Lexer operator<<(const Pattern &p);
std::vector<TokenPtr> lex(const std::string &input, const char *fname); std::vector<TokenPtr> lex(const std::string &input, const char *fname);
void set_skip(const Pattern &p); void set_skip(const Pattern &p) {
void set_line_sep(const Pattern &p); skip_pattern = p;
void set_command_sep(const Pattern &p); }
void set_line_sep(const Pattern &p) {
line_sep_pattern = p;
}
void set_command_sep(const Pattern &p) {
command_sep_pattern = p;
}
void set_multiline_comment(const Pattern &start, const Pattern &end) {
multiline_comment_start_pattern = start;
multiline_comment_end_pattern = end;
}
void set_singleline_comment(const Pattern &p) {
singleline_comment_pattern = p;
}
}; };

17
langkit/main.cpp
View File

@ -12,11 +12,11 @@
class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close,
Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon, Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon,
Function_Def, Scoped_Block, Statement, Equation, Return, Add}; }; Function_Def, Scoped_Block, Statement, Equation, Return, Add, Comment}; };
void debug_print(TokenPtr token, std::string prepend) { void debug_print(TokenPtr token, std::string prepend) {
std::cout << prepend << "Token: " << token->text << "(" << token->identifier << ") @ " << token->filename << ": (" << token->start.column std::cout << prepend << "Token: " << token->text << "(" << token->identifier << ") @ " << token->filename << ": (" << token->start.line
<< ", " << token->start.line << ") to (" << token->end.column << ", " << token->end.line << ") " << std::endl; << ", " << token->start.column << ") to (" << token->end.line << ", " << token->end.column << ") " << std::endl;
for (unsigned int i = 0; i < token->children.size(); ++i) { for (unsigned int i = 0; i < token->children.size(); ++i) {
debug_print(token->children[i], prepend + " "); debug_print(token->children[i], prepend + " ");
@ -111,13 +111,14 @@ void parse(std::vector<TokenPtr> &tokens) {
std::pair<Token_Iterator, bool> results = rule(iter, end, parent); std::pair<Token_Iterator, bool> results = rule(iter, end, parent);
/*
while (results.second) { while (results.second) {
results = rule(results.first + 1, end, parent); results = rule(results.first + 1, end, parent);
//debug_print(parent, ""); //debug_print(parent, "");
} }
*/
/*
if (results.second) { if (results.second) {
std::cout << "Parse successful: " << std::endl; std::cout << "Parse successful: " << std::endl;
debug_print(parent, ""); debug_print(parent, "");
@ -126,7 +127,7 @@ void parse(std::vector<TokenPtr> &tokens) {
std::cout << "Parse failed: " << std::endl; std::cout << "Parse failed: " << std::endl;
debug_print(parent, ""); debug_print(parent, "");
} }
*/
} }
@ -137,10 +138,12 @@ int main(int argc, char *argv[]) {
lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace)); lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace));
lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return)); lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return));
lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon)); lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon));
lexer.set_multiline_comment(Pattern("/\\*", TokenType::Comment), Pattern("\\*/", TokenType::Comment));
lexer.set_singleline_comment(Pattern("//", TokenType::Comment));
lexer << Pattern("[A-Za-z]+", TokenType::Identifier); lexer << Pattern("[A-Za-z]+", TokenType::Identifier);
lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number); lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number);
lexer << Pattern("[!@#$%^&*\\-+=/<>]+", TokenType::Operator); lexer << Pattern("[!@#$%^&*\\-+=<>/]+", TokenType::Operator);
lexer << Pattern("\\(", TokenType::Parens_Open); lexer << Pattern("\\(", TokenType::Parens_Open);
lexer << Pattern("\\)", TokenType::Parens_Close); lexer << Pattern("\\)", TokenType::Parens_Close);
lexer << Pattern("\\[", TokenType::Square_Open); lexer << Pattern("\\[", TokenType::Square_Open);