generic-library/ChaiScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added line counting to lexer.

Browse Source
This commit is contained in:
Jonathan Turner
2009-05-26 16:59:29 +00:00
parent 304198b9bb
commit 5424b6be41
4 changed files with 89 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
langkit/langkit_lexer.cpp
View File

@@ -11,19 +11,24 @@ Lexer Lexer::operator<<(const Pattern &p) {
std::vector<Token> Lexer::lex(const std::string &input) {
std::vector<Pattern>::iterator iter, end;
//std::string::const_iterator str_end = input.end();
std::vector<Token> retval;
bool found;
std::string::const_iterator input_iter = input.begin();
int current_col = 0;
int current_line = 0;
while (input_iter != input.end()) {
found = false;
for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) {
boost::match_results<std::string::const_iterator> what;
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
Token t(what[0], iter->identifier);
t.start.column = input_iter - input.begin();
t.end.column = t.start.column + t.text.size();
t.start.column = current_col;
t.start.line = current_line;
current_col += t.text.size();
t.end.column = current_col;
t.end.line = current_line;
retval.push_back(t);
input_iter += t.text.size();
found = true;
@@ -36,15 +41,30 @@ std::vector<Token> Lexer::lex(const std::string &input) {
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
std::string whitespace(what[0]);
input_iter += whitespace.size();
current_col += whitespace.size();
found = true;
break;
}
}
if (!found) {
const std::string err(input_iter, input.end());
std::cout << "Unknown string at: " << err << std::endl;
return retval;
for (iter = line_sep_patterns.begin(), end = line_sep_patterns.end(); iter != end; ++iter) {
boost::match_results<std::string::const_iterator> what;
if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) {
std::string cr(what[0]);
input_iter += cr.size();
found = true;
++current_line;
current_col = 0;
break;
}
}
if (!found) {
const std::string err(input_iter, input.end());
std::cout << "Unknown string at: " << err << std::endl;
return retval;
}
}
}
}
@@ -54,3 +74,11 @@ std::vector<Token> Lexer::lex(const std::string &input) {
void Lexer::set_skip(const Pattern &p) {
skip_patterns.push_back(p);
}
void Lexer::set_line_sep(const Pattern &p) {
line_sep_patterns.push_back(p);
}
void Lexer::set_command_sep(const Pattern &p) {
command_sep_patterns.push_back(p);
}

12
langkit/langkit_lexer.hpp
View File

@@ -8,14 +8,14 @@
#include <boost/regex.hpp>
struct File_Position {
int row;
int line;
int column;
char *filename;
File_Position(int file_row, int file_column, char *fname)
: row(file_row), column(file_column), filename(fname) { }
File_Position(int file_line, int file_column, char *fname)
: line(file_line), column(file_column), filename(fname) { }
File_Position() : row(0), column(0), filename(NULL) { }
File_Position() : line(0), column(0), filename(NULL) { }
};
struct Pattern {
@@ -36,11 +36,15 @@ struct Token {
struct Lexer {
std::vector<Pattern> lex_patterns;
std::vector<Pattern> skip_patterns;
std::vector<Pattern> command_sep_patterns;
std::vector<Pattern> line_sep_patterns;
Lexer operator<<(const Pattern &p);
std::vector<Token> lex(const std::string &input);
void set_skip(const Pattern &p);
void set_line_sep(const Pattern &p);
void set_command_sep(const Pattern &p);
};

95
langkit/main.cpp
View File

@@ -3,6 +3,7 @@
#include <iostream>
#include <map>
#include <fstream>
#include "langkit_lexer.hpp"
//#include "lexer.hpp"
@@ -10,23 +11,47 @@
//#include "eval.hpp"
class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close,
Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String }; };
Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon }; };
void debug_print(std::vector<Token> &tokens) {
for (unsigned int i = 0; i < tokens.size(); ++i) {
std::cout << "Token: " << tokens[i].text << "(" << tokens[i].identifier << ") @ " << tokens[i].start.column
<< " to " << tokens[i].end.column << std::endl;
std::cout << "Token: " << tokens[i].text << "(" << tokens[i].identifier << ") @ (" << tokens[i].start.column
<< ", " << tokens[i].start.line << ") to (" << tokens[i].end.column << ", " << tokens[i].end.line << ") " << std::endl;
}
}
std::string load_file(const char *filename) {
std::ifstream infile (filename, std::ios::in | std::ios::ate);
if (!infile.is_open()) {
std::cerr << "Can not open " << filename << std::endl;
exit(0);
}
std::streampos size = infile.tellg();
infile.seekg(0, std::ios::beg);
std::cout << "Allocating: " << size << " bytes" << std::endl;
std::vector<char> v(size);
infile.read(&v[0], size);
std::string ret_val (v.empty() ? std::string() : std::string (v.begin(), v.end()).c_str());
return ret_val;
}
int main(int argc, char *argv[]) {
std::string input;
Lexer lexer;
lexer.set_skip(Pattern("\\s+", TokenType::Whitespace));
lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace));
lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return));
lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon));
lexer << Pattern("[A-Za-z]+", TokenType::Identifier);
lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number);
lexer << Pattern("[!@#$%^&*\\-+=/:]+", TokenType::Operator);
lexer << Pattern("[!@#$%^&*\\-+=/<>]+", TokenType::Operator);
lexer << Pattern("\\(", TokenType::Parens_Open);
lexer << Pattern("\\)", TokenType::Parens_Close);
lexer << Pattern("\\[", TokenType::Square_Open);
@@ -34,57 +59,21 @@ int main(int argc, char *argv[]) {
lexer << Pattern("\\{", TokenType::Curly_Open);
lexer << Pattern("\\}", TokenType::Curly_Close);
lexer << Pattern(",", TokenType::Comma);
lexer << Pattern("[!@#$%^&*\\-+=/<>]+", TokenType::Operator);
lexer << Pattern("\"(?:[^\"\\\\]|\\\\.)*\"", TokenType::Quoted_String);
lexer << Pattern("'(?:[^'\\\\]|\\\\.)*'", TokenType::Single_Quoted_String);
std::cout << "Expression> ";
std::getline(std::cin, input);
while (input != "quit") {
std::vector<Token> tokens = lexer.lex(input);
if (argc < 2) {
std::cout << "Expression> ";
std::getline(std::cin, input);
while (input != "quit") {
std::vector<Token> tokens = lexer.lex(input);
debug_print(tokens);
std::cout << "Expression> ";
std::getline(std::cin, input);
}
}
else {
std::vector<Token> tokens = lexer.lex(load_file(argv[1]));
debug_print(tokens);
std::cout << "Expression> ";
std::getline(std::cin, input);
}
}
/*
int main(int argc, char *argv[]) {
std::string input;
std::map<std::string, NodePtr> symbols;
std::cout << "Expression> ";
std::getline(std::cin, input);
while (input != "quit") {
if (input == "vars") {
//debug_print(symbols);
}
else {
try {
NodePtr lex_nodes = lex(input);
//clean_whitespace(lex_nodes);
//NodePtr parse_nodes = parse(lex_nodes);
//Result result = eval(parse_nodes, symbols);
//debug_print(lex_nodes);
//std::cout << std::endl;
//debug_print(parse_nodes, "");
//if (result.type == ResultType::NUMBER) {
// std::cout << result.value << std::endl;
//}
}
catch (LexerError &le) {
std::cerr << "Lexer error: " << le.reason << std::endl;
}
//catch (ParserError &pe) {
// std::cerr << "Parser error: " << pe.reason << std::endl;
//}
//catch (EvalError &ee) {
// std::cerr << "Eval error: " << ee.reason << std::endl;
//}
}
std::cout << "Expression> ";
std::getline(std::cin, input);
}
}
*/

5
langkit/tests/multiline_test.gt Normal file
View File

@@ -0,0 +1,5 @@
3+4
7 + 9
5 + 7