generic-library/ChaiScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added comment lexing. Removed previous optimization. Flattened lex grammar.

Browse Source
This commit is contained in:
Jonathan Turner
2009-06-01 13:40:24 +00:00
parent f692834fa8
commit 307e557e5b
3 changed files with 127 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
langkit/langkit_lexer.hpp
View File

@@ -20,10 +20,10 @@ struct File_Position {
struct Pattern {
boost::regex regex;
std::string regex_string;
int identifier;
Pattern(const std::string &regexp, int id) : regex(regexp), regex_string(regexp), identifier(id) { }
Pattern() { }
Pattern(const std::string &regexp, int id) : regex(regexp), identifier(id) { }
};
typedef std::tr1::shared_ptr<struct Token> TokenPtr;
@@ -41,16 +41,32 @@ struct Token {
struct Lexer {
std::vector<Pattern> lex_patterns;
std::vector<Pattern> skip_patterns;
std::vector<Pattern> command_sep_patterns;
std::vector<Pattern> line_sep_patterns;
Pattern skip_pattern;
Pattern command_sep_pattern;
Pattern line_sep_pattern;
Pattern multiline_comment_start_pattern;
Pattern multiline_comment_end_pattern;
Pattern singleline_comment_pattern;
Lexer operator<<(const Pattern &p);
std::vector<TokenPtr> lex(const std::string &input, const char *fname);
void set_skip(const Pattern &p);
void set_line_sep(const Pattern &p);
void set_command_sep(const Pattern &p);
void set_skip(const Pattern &p) {
skip_pattern = p;
}
void set_line_sep(const Pattern &p) {
line_sep_pattern = p;
}
void set_command_sep(const Pattern &p) {
command_sep_pattern = p;
}
void set_multiline_comment(const Pattern &start, const Pattern &end) {
multiline_comment_start_pattern = start;
multiline_comment_end_pattern = end;
}
void set_singleline_comment(const Pattern &p) {
singleline_comment_pattern = p;
}
};