Added line counting to lexer.

This commit is contained in:
Jonathan Turner
2009-05-26 16:59:29 +00:00
parent 304198b9bb
commit 5424b6be41
4 changed files with 89 additions and 63 deletions

View File

@@ -11,19 +11,24 @@ Lexer Lexer::operator<<(const Pattern &p) {
std::vector<Token> Lexer::lex(const std::string &input) {
std::vector<Pattern>::iterator iter, end;
//std::string::const_iterator str_end = input.end();
std::vector<Token> retval;
bool found;
std::string::const_iterator input_iter = input.begin();
int current_col = 0;
int current_line = 0;
while (input_iter != input.end()) {
found = false;
for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) {
boost::match_results<std::string::const_iterator> what;
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
Token t(what[0], iter->identifier);
t.start.column = input_iter - input.begin();
t.end.column = t.start.column + t.text.size();
t.start.column = current_col;
t.start.line = current_line;
current_col += t.text.size();
t.end.column = current_col;
t.end.line = current_line;
retval.push_back(t);
input_iter += t.text.size();
found = true;
@@ -36,15 +41,30 @@ std::vector<Token> Lexer::lex(const std::string &input) {
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
std::string whitespace(what[0]);
input_iter += whitespace.size();
current_col += whitespace.size();
found = true;
break;
}
}
if (!found) {
const std::string err(input_iter, input.end());
std::cout << "Unknown string at: " << err << std::endl;
return retval;
for (iter = line_sep_patterns.begin(), end = line_sep_patterns.end(); iter != end; ++iter) {
boost::match_results<std::string::const_iterator> what;
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
std::string cr(what[0]);
input_iter += cr.size();
found = true;
++current_line;
current_col = 0;
break;
}
}
if (!found) {
const std::string err(input_iter, input.end());
std::cout << "Unknown string at: " << err << std::endl;
return retval;
}
}
}
}
@@ -54,3 +74,11 @@ std::vector<Token> Lexer::lex(const std::string &input) {
void Lexer::set_skip(const Pattern &p) {
skip_patterns.push_back(p);
}
void Lexer::set_line_sep(const Pattern &p) {
line_sep_patterns.push_back(p);
}
void Lexer::set_command_sep(const Pattern &p) {
command_sep_patterns.push_back(p);
}