207 lines
8.3 KiB
C++
207 lines
8.3 KiB
C++
// This file is distributed under the BSD License.
|
|
// See LICENSE.TXT for details.
|
|
|
|
#ifndef LANGKIT_LEXER_HPP_
|
|
#define LANGKIT_LEXER_HPP_
|
|
|
|
#include <boost/regex.hpp>
|
|
#include <tr1/memory>
|
|
#include <string>
|
|
|
|
namespace langkit
|
|
{
|
|
struct File_Position {
|
|
int line;
|
|
int column;
|
|
|
|
File_Position(int file_line, int file_column)
|
|
: line(file_line), column(file_column) { }
|
|
|
|
File_Position() : line(0), column(0) { }
|
|
};
|
|
|
|
struct Pattern {
|
|
boost::regex regex;
|
|
int identifier;
|
|
|
|
Pattern() { }
|
|
Pattern(const std::string ®exp, int id) : regex(regexp), identifier(id) { }
|
|
};
|
|
|
|
typedef std::tr1::shared_ptr<struct Token> TokenPtr;
|
|
|
|
struct Token {
|
|
std::string text;
|
|
int identifier;
|
|
const char *filename;
|
|
File_Position start, end;
|
|
|
|
std::vector<TokenPtr> children;
|
|
|
|
Token(const std::string &token_text, int id, const char *fname) : text(token_text), identifier(id), filename(fname) { }
|
|
};
|
|
|
|
struct Lexer {
|
|
std::vector<Pattern> lex_patterns;
|
|
Pattern skip_pattern;
|
|
Pattern command_sep_pattern;
|
|
Pattern line_sep_pattern;
|
|
Pattern multiline_comment_start_pattern;
|
|
Pattern multiline_comment_end_pattern;
|
|
Pattern singleline_comment_pattern;
|
|
|
|
Lexer operator<<(const Pattern &p) {
|
|
lex_patterns.push_back(p);
|
|
return *this;
|
|
}
|
|
|
|
std::vector<TokenPtr> lex(const std::string &input, const char *filename) {
|
|
std::vector<Pattern>::iterator iter, end, iter2, end2;
|
|
std::vector<TokenPtr> retval;
|
|
bool found;
|
|
std::string::const_iterator input_iter = input.begin(), input_end = input.end();
|
|
|
|
int current_col = 0;
|
|
int current_line = 0;
|
|
boost::match_results<std::string::const_iterator> what;
|
|
|
|
while (input_iter != input_end) {
|
|
found = false;
|
|
|
|
if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) {
|
|
std::string comment_start(what[0]);
|
|
input_iter += comment_start.size();
|
|
|
|
bool found_eol = false;
|
|
|
|
while ((!found_eol) && (input_iter != input_end)) {
|
|
boost::match_results<std::string::const_iterator> eol_delim;
|
|
if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) {
|
|
std::string comment_end(eol_delim[0]);
|
|
input_iter += comment_end.size();
|
|
++current_line;
|
|
current_col = 0;
|
|
found_eol = true;
|
|
break;
|
|
}
|
|
if ((!found_eol) && (input_iter != input_end)) {
|
|
++input_iter;
|
|
}
|
|
}
|
|
}
|
|
else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) {
|
|
std::string comment_start(what[0]);
|
|
input_iter += comment_start.size();
|
|
|
|
bool found_eoc = false;
|
|
|
|
while ((!found_eoc) && (input_iter != input_end)) {
|
|
boost::match_results<std::string::const_iterator> eol_delim;
|
|
if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) {
|
|
std::string comment_end(eol_delim[0]);
|
|
input_iter += comment_end.size();
|
|
++current_line;
|
|
current_col = 0;
|
|
}
|
|
boost::match_results<std::string::const_iterator> eoc_delim;
|
|
if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) {
|
|
std::string comment_end(eoc_delim[0]);
|
|
input_iter += comment_end.size();
|
|
current_col += comment_end.size();
|
|
found_eoc = true;
|
|
break;
|
|
}
|
|
if ((!found_eoc) && (input_iter != input_end)) {
|
|
++input_iter;
|
|
++current_col;
|
|
}
|
|
}
|
|
|
|
if (!found_eoc) {
|
|
std::cout << "Incomplete comment block! Add exceptions!" << std::endl;
|
|
return retval;
|
|
}
|
|
}
|
|
else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) {
|
|
std::string whitespace(what[0]);
|
|
input_iter += whitespace.size();
|
|
current_col += whitespace.size();
|
|
found = true;
|
|
}
|
|
else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) {
|
|
const std::string cr(what[0]);
|
|
|
|
boost::match_results<std::string::const_iterator> if_delim;
|
|
if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) {
|
|
TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename));
|
|
t->start.column = current_col;
|
|
t->start.line = current_line;
|
|
current_col += t->text.size();
|
|
t->end.column = current_col;
|
|
t->end.line = current_line;
|
|
retval.push_back(t);
|
|
}
|
|
|
|
input_iter += cr.size();
|
|
++current_line;
|
|
current_col = 0;
|
|
found = true;
|
|
}
|
|
else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) {
|
|
TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename));
|
|
t->start.column = current_col;
|
|
t->start.line = current_line;
|
|
current_col += t->text.size();
|
|
t->end.column = current_col;
|
|
t->end.line = current_line;
|
|
retval.push_back(t);
|
|
input_iter += t->text.size();
|
|
found = true;
|
|
}
|
|
else {
|
|
for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) {
|
|
if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) {
|
|
TokenPtr t(new Token(what[0], iter->identifier, filename));
|
|
t->start.column = current_col;
|
|
t->start.line = current_line;
|
|
current_col += t->text.size();
|
|
t->end.column = current_col;
|
|
t->end.line = current_line;
|
|
retval.push_back(t);
|
|
input_iter += t->text.size();
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!found) {
|
|
const std::string err(input_iter, input_end);
|
|
std::cout << "Unknown string at: " << err << std::endl;
|
|
return retval;
|
|
}
|
|
}
|
|
}
|
|
return retval;
|
|
}
|
|
|
|
void set_skip(const Pattern &p) {
|
|
skip_pattern = p;
|
|
}
|
|
void set_line_sep(const Pattern &p) {
|
|
line_sep_pattern = p;
|
|
}
|
|
void set_command_sep(const Pattern &p) {
|
|
command_sep_pattern = p;
|
|
}
|
|
void set_multiline_comment(const Pattern &start, const Pattern &end) {
|
|
multiline_comment_start_pattern = start;
|
|
multiline_comment_end_pattern = end;
|
|
}
|
|
void set_singleline_comment(const Pattern &p) {
|
|
singleline_comment_pattern = p;
|
|
}
|
|
};
|
|
|
|
}
|
|
#endif /* LANGKIT_LEXER_HPP_ */
|