generic-library/ChaiScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add langkit namespace

Browse Source
This commit is contained in:
Jason Turner
2009-06-13 22:19:30 +00:00
parent ce52cae45c
commit 416242286a
5 changed files with 509 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

314
langkit/langkit_lexer.hpp
View File

@@ -8,197 +8,199 @@
#include <tr1/memory>
#include <string>
struct File_Position {
int line;
int column;
namespace langkit
{
struct File_Position {
int line;
int column;
File_Position(int file_line, int file_column)
: line(file_line), column(file_column) { }
File_Position(int file_line, int file_column)
: line(file_line), column(file_column) { }
File_Position() : line(0), column(0) { }
};
File_Position() : line(0), column(0) { }
};
struct Pattern {
boost::regex regex;
int identifier;
struct Pattern {
boost::regex regex;
int identifier;
Pattern() { }
Pattern(const std::string &regexp, int id) : regex(regexp), identifier(id) { }
};
Pattern() { }
Pattern(const std::string &regexp, int id) : regex(regexp), identifier(id) { }
};
typedef std::tr1::shared_ptr<struct Token> TokenPtr;
typedef std::tr1::shared_ptr<struct Token> TokenPtr;
struct Token {
std::string text;
int identifier;
const char *filename;
File_Position start, end;
struct Token {
std::string text;
int identifier;
const char *filename;
File_Position start, end;
std::vector<TokenPtr> children;
std::vector<TokenPtr> children;
Token(const std::string &token_text, int id, const char *fname) : text(token_text), identifier(id), filename(fname) { }
};
Token(const std::string &token_text, int id, const char *fname) : text(token_text), identifier(id), filename(fname) { }
};
struct Lexer {
std::vector<Pattern> lex_patterns;
Pattern skip_pattern;
Pattern command_sep_pattern;
Pattern line_sep_pattern;
Pattern multiline_comment_start_pattern;
Pattern multiline_comment_end_pattern;
Pattern singleline_comment_pattern;
struct Lexer {
std::vector<Pattern> lex_patterns;
Pattern skip_pattern;
Pattern command_sep_pattern;
Pattern line_sep_pattern;
Pattern multiline_comment_start_pattern;
Pattern multiline_comment_end_pattern;
Pattern singleline_comment_pattern;
Lexer operator<<(const Pattern &p) {
lex_patterns.push_back(p);
return *this;
}
Lexer operator<<(const Pattern &p) {
lex_patterns.push_back(p);
return *this;
}
std::vector<TokenPtr> lex(const std::string &input, const char *filename) {
std::vector<Pattern>::iterator iter, end, iter2, end2;
std::vector<TokenPtr> retval;
bool found;
std::string::const_iterator input_iter = input.begin(), input_end = input.end();
std::vector<TokenPtr> lex(const std::string &input, const char *filename) {
std::vector<Pattern>::iterator iter, end, iter2, end2;
std::vector<TokenPtr> retval;
bool found;
std::string::const_iterator input_iter = input.begin(), input_end = input.end();
int current_col = 0;
int current_line = 0;
boost::match_results<std::string::const_iterator> what;
int current_col = 0;
int current_line = 0;
boost::match_results<std::string::const_iterator> what;
while (input_iter != input_end) {
found = false;
while (input_iter != input_end) {
found = false;
if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) {
std::string comment_start(what[0]);
input_iter += comment_start.size();
if (regex_search(input_iter, input_end, what, singleline_comment_pattern.regex, boost::match_continuous)) {
std::string comment_start(what[0]);
input_iter += comment_start.size();
bool found_eol = false;
bool found_eol = false;
while ((!found_eol) && (input_iter != input_end)) {
boost::match_results<std::string::const_iterator> eol_delim;
if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) {
std::string comment_end(eol_delim[0]);
input_iter += comment_end.size();
++current_line;
current_col = 0;
found_eol = true;
break;
}
if ((!found_eol) && (input_iter != input_end)) {
++input_iter;
while ((!found_eol) && (input_iter != input_end)) {
boost::match_results<std::string::const_iterator> eol_delim;
if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) {
std::string comment_end(eol_delim[0]);
input_iter += comment_end.size();
++current_line;
current_col = 0;
found_eol = true;
break;
}
if ((!found_eol) && (input_iter != input_end)) {
++input_iter;
}
}
}
}
else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) {
std::string comment_start(what[0]);
input_iter += comment_start.size();
else if (regex_search(input_iter, input_end, what, multiline_comment_start_pattern.regex, boost::match_continuous)) {
std::string comment_start(what[0]);
input_iter += comment_start.size();
bool found_eoc = false;
bool found_eoc = false;
while ((!found_eoc) && (input_iter != input_end)) {
boost::match_results<std::string::const_iterator> eol_delim;
if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) {
std::string comment_end(eol_delim[0]);
input_iter += comment_end.size();
++current_line;
current_col = 0;
while ((!found_eoc) && (input_iter != input_end)) {
boost::match_results<std::string::const_iterator> eol_delim;
if (regex_search(input_iter, input_end, eol_delim, line_sep_pattern.regex, boost::match_continuous)) {
std::string comment_end(eol_delim[0]);
input_iter += comment_end.size();
++current_line;
current_col = 0;
}
boost::match_results<std::string::const_iterator> eoc_delim;
if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) {
std::string comment_end(eoc_delim[0]);
input_iter += comment_end.size();
current_col += comment_end.size();
found_eoc = true;
break;
}
if ((!found_eoc) && (input_iter != input_end)) {
++input_iter;
++current_col;
}
}
boost::match_results<std::string::const_iterator> eoc_delim;
if (regex_search(input_iter, input_end, eoc_delim, multiline_comment_end_pattern.regex, boost::match_continuous)) {
std::string comment_end(eoc_delim[0]);
input_iter += comment_end.size();
current_col += comment_end.size();
found_eoc = true;
break;
}
if ((!found_eoc) && (input_iter != input_end)) {
++input_iter;
++current_col;
if (!found_eoc) {
std::cout << "Incomplete comment block! Add exceptions!" << std::endl;
return retval;
}
}
if (!found_eoc) {
std::cout << "Incomplete comment block! Add exceptions!" << std::endl;
return retval;
else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) {
std::string whitespace(what[0]);
input_iter += whitespace.size();
current_col += whitespace.size();
found = true;
}
}
else if (regex_search(input_iter, input_end, what, skip_pattern.regex, boost::match_continuous)) {
std::string whitespace(what[0]);
input_iter += whitespace.size();
current_col += whitespace.size();
found = true;
}
else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) {
const std::string cr(what[0]);
else if (regex_search(input_iter, input_end, what, line_sep_pattern.regex, boost::match_continuous)) {
const std::string cr(what[0]);
boost::match_results<std::string::const_iterator> if_delim;
if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) {
TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename));
t->start.column = current_col;
t->start.line = current_line;
current_col += t->text.size();
t->end.column = current_col;
t->end.line = current_line;
retval.push_back(t);
}
input_iter += cr.size();
++current_line;
current_col = 0;
found = true;
}
else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) {
TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename));
t->start.column = current_col;
t->start.line = current_line;
current_col += t->text.size();
t->end.column = current_col;
t->end.line = current_line;
retval.push_back(t);
input_iter += t->text.size();
found = true;
}
else {
for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) {
if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) {
TokenPtr t(new Token(what[0], iter->identifier, filename));
boost::match_results<std::string::const_iterator> if_delim;
if (regex_search(cr.begin(), cr.end(), if_delim, command_sep_pattern.regex, boost::match_continuous)) {
TokenPtr t(new Token(if_delim[0], command_sep_pattern.identifier, filename));
t->start.column = current_col;
t->start.line = current_line;
current_col += t->text.size();
t->end.column = current_col;
t->end.line = current_line;
retval.push_back(t);
input_iter += t->text.size();
found = true;
break;
}
input_iter += cr.size();
++current_line;
current_col = 0;
found = true;
}
else if (regex_search(input_iter, input_end, what, command_sep_pattern.regex, boost::match_continuous)) {
TokenPtr t(new Token(what[0], command_sep_pattern.identifier, filename));
t->start.column = current_col;
t->start.line = current_line;
current_col += t->text.size();
t->end.column = current_col;
t->end.line = current_line;
retval.push_back(t);
input_iter += t->text.size();
found = true;
}
else {
for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) {
if (regex_search(input_iter, input_end, what, iter->regex, boost::match_continuous)) {
TokenPtr t(new Token(what[0], iter->identifier, filename));
t->start.column = current_col;
t->start.line = current_line;
current_col += t->text.size();
t->end.column = current_col;
t->end.line = current_line;
retval.push_back(t);
input_iter += t->text.size();
found = true;
break;
}
}
if (!found) {
const std::string err(input_iter, input_end);
std::cout << "Unknown string at: " << err << std::endl;
return retval;
}
}
if (!found) {
const std::string err(input_iter, input_end);
std::cout << "Unknown string at: " << err << std::endl;
return retval;
}
}
return retval;
}
return retval;
}
void set_skip(const Pattern &p) {
skip_pattern = p;
}
void set_line_sep(const Pattern &p) {
line_sep_pattern = p;
}
void set_command_sep(const Pattern &p) {
command_sep_pattern = p;
}
void set_multiline_comment(const Pattern &start, const Pattern &end) {
multiline_comment_start_pattern = start;
multiline_comment_end_pattern = end;
}
void set_singleline_comment(const Pattern &p) {
singleline_comment_pattern = p;
}
};
void set_skip(const Pattern &p) {
skip_pattern = p;
}
void set_line_sep(const Pattern &p) {
line_sep_pattern = p;
}
void set_command_sep(const Pattern &p) {
command_sep_pattern = p;
}
void set_multiline_comment(const Pattern &start, const Pattern &end) {
multiline_comment_start_pattern = start;
multiline_comment_end_pattern = end;
}
void set_singleline_comment(const Pattern &p) {
singleline_comment_pattern = p;
}
};
}
#endif /* LANGKIT_LEXER_HPP_ */