From 332fdf0c8e1395a9ee55216dcc37a1a433db9ac7 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Wed, 27 May 2009 15:07:29 +0000 Subject: [PATCH] Switch Token to smart pointer --- langkit/langkit_lexer.cpp | 18 +++++++++--------- langkit/langkit_lexer.hpp | 9 ++++++--- langkit/langkit_parser.cpp | 26 +++++++++++++------------- langkit/langkit_parser.hpp | 13 +++++++------ langkit/main.cpp | 17 ++++++++--------- 5 files changed, 43 insertions(+), 40 deletions(-) diff --git a/langkit/langkit_lexer.cpp b/langkit/langkit_lexer.cpp index 0550f88..630ad25 100644 --- a/langkit/langkit_lexer.cpp +++ b/langkit/langkit_lexer.cpp @@ -9,9 +9,9 @@ Lexer Lexer::operator<<(const Pattern &p) { return *this; } -std::vector Lexer::lex(const std::string &input, const char *filename) { +std::vector Lexer::lex(const std::string &input, const char *filename) { std::vector::iterator iter, end; - std::vector retval; + std::vector retval; bool found; std::string::const_iterator input_iter = input.begin(); @@ -23,14 +23,14 @@ std::vector Lexer::lex(const std::string &input, const char *filename) { for (iter = lex_patterns.begin(), end = lex_patterns.end(); iter != end; ++iter) { boost::match_results what; if (regex_search(input_iter, input.end(), what, iter->regex, boost::match_continuous)) { - Token t(what[0], iter->identifier, filename); - t.start.column = current_col; - t.start.line = current_line; - current_col += t.text.size(); - t.end.column = current_col; - t.end.line = current_line; + TokenPtr t(new Token(what[0], iter->identifier, filename)); + t->start.column = current_col; + t->start.line = current_line; + current_col += t->text.size(); + t->end.column = current_col; + t->end.line = current_line; retval.push_back(t); - input_iter += t.text.size(); + input_iter += t->text.size(); found = true; break; } diff --git a/langkit/langkit_lexer.hpp b/langkit/langkit_lexer.hpp index 72c5251..13c1f0f 100644 --- a/langkit/langkit_lexer.hpp +++ b/langkit/langkit_lexer.hpp @@ -4,8 +4,9 @@ #ifndef LANGKIT_LEXER_HPP_ #define LANGKIT_LEXER_HPP_ -#include #include +#include +#include struct File_Position { int line; @@ -24,13 +25,15 @@ struct Pattern { Pattern(const std::string ®exp, int id) : regex(regexp), identifier(id) { } }; +typedef std::tr1::shared_ptr TokenPtr; + struct Token { std::string text; int identifier; const char *filename; File_Position start, end; - std::vector children; + std::vector children; Token(const std::string &token_text, int id, const char *fname) : text(token_text), identifier(id), filename(fname) { } }; @@ -42,7 +45,7 @@ struct Lexer { std::vector line_sep_patterns; Lexer operator<<(const Pattern &p); - std::vector lex(const std::string &input, const char *fname); + std::vector lex(const std::string &input, const char *fname); void set_skip(const Pattern &p); void set_line_sep(const Pattern &p); diff --git a/langkit/langkit_parser.cpp b/langkit/langkit_parser.cpp index 1ab9563..5ddffa4 100644 --- a/langkit/langkit_parser.cpp +++ b/langkit/langkit_parser.cpp @@ -6,43 +6,43 @@ #include "langkit_lexer.hpp" #include "langkit_parser.hpp" -std::pair String_Rule(Token_Iterator iter, Token_Iterator end, const std::string &val) { +std::pair String_Rule(Token_Iterator iter, Token_Iterator end, const std::string &val) { if (iter != end) { - if (iter->text == val) { - return std::pair(++iter, *iter); + if ((*iter)->text == val) { + return std::pair(++iter, *iter); } } - return std::pair(iter, *iter); + return std::pair(iter, *iter); } -std::pair Type_Rule(Token_Iterator iter, Token_Iterator end, const int val) { +std::pair Type_Rule(Token_Iterator iter, Token_Iterator end, const int val) { if (iter != end) { - if (iter->identifier == val) { - return std::pair(++iter, *iter); + if ((*iter)->identifier == val) { + return std::pair(++iter, *iter); } } - return std::pair(iter, *iter); + return std::pair(iter, *iter); } -std::pair Or_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs) { +std::pair Or_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs) { Token_Iterator new_iter; if (iter != end) { new_iter = lhs.rule(iter, end).first; if (new_iter != iter) { - return std::pair(new_iter, *iter); + return std::pair(new_iter, *iter); } else { new_iter = rhs.rule(iter, end).first; if (new_iter != iter) { - return std::pair(new_iter, *iter); + return std::pair(new_iter, *iter); } } } - return std::pair(iter, *iter); + return std::pair(iter, *iter); } /* @@ -64,6 +64,6 @@ std::pair And_Rule(Token_Iterator iter, Token_Iterator en } */ -std::pair Rule::operator()(Token_Iterator iter, Token_Iterator end) { +std::pair Rule::operator()(Token_Iterator iter, Token_Iterator end) { return this->rule(iter, end); } diff --git a/langkit/langkit_parser.hpp b/langkit/langkit_parser.hpp index 298e68e..9539638 100644 --- a/langkit/langkit_parser.hpp +++ b/langkit/langkit_parser.hpp @@ -8,20 +8,21 @@ #include "langkit_lexer.hpp" -typedef std::vector::iterator Token_Iterator; + +typedef std::vector::iterator Token_Iterator; struct Rule { int identifier; - boost::function(Token_Iterator iter, Token_Iterator end)> rule; - std::pair operator()(Token_Iterator iter, Token_Iterator end); + boost::function(Token_Iterator iter, Token_Iterator end)> rule; + std::pair operator()(Token_Iterator iter, Token_Iterator end); Rule() : identifier(-1) {} Rule(int id) : identifier(id) {} }; -std::pair String_Rule(Token_Iterator iter, Token_Iterator end, const std::string &val); -std::pair Type_Rule(Token_Iterator iter, Token_Iterator end, const int val); -std::pair Or_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs); +std::pair String_Rule(Token_Iterator iter, Token_Iterator end, const std::string &val); +std::pair Type_Rule(Token_Iterator iter, Token_Iterator end, const int val); +std::pair Or_Rule(Token_Iterator iter, Token_Iterator end, const Rule &lhs, const Rule &rhs); diff --git a/langkit/main.cpp b/langkit/main.cpp index 67d9be1..e12dd62 100644 --- a/langkit/main.cpp +++ b/langkit/main.cpp @@ -13,11 +13,11 @@ class TokenType { public: enum Type { Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon }; }; -void debug_print(Token &token) { - std::cout << "Token: " << token.text << "(" << token.identifier << ") @ " << token.filename << ": (" << token.start.column - << ", " << token.start.line << ") to (" << token.end.column << ", " << token.end.line << ") " << std::endl; +void debug_print(TokenPtr token) { + std::cout << "Token: " << token->text << "(" << token->identifier << ") @ " << token->filename << ": (" << token->start.column + << ", " << token->start.line << ") to (" << token->end.column << ", " << token->end.line << ") " << std::endl; } -void debug_print(std::vector &tokens) { +void debug_print(std::vector &tokens) { for (unsigned int i = 0; i < tokens.size(); ++i) { debug_print(tokens[i]); } @@ -42,14 +42,13 @@ std::string load_file(const char *filename) { return ret_val; } -void parse(std::vector &tokens) { +void parse(std::vector &tokens) { Rule rule; rule.rule = boost::bind(String_Rule, _1, _2, "def"); Token_Iterator iter = tokens.begin(), end = tokens.end(); - - std::pair results = rule(iter, end); + std::pair results = rule(iter, end); if (results.first != iter) { std::cout << "Parse successful: " << std::endl; @@ -84,7 +83,7 @@ int main(int argc, char *argv[]) { std::cout << "Expression> "; std::getline(std::cin, input); while (input != "quit") { - std::vector tokens = lexer.lex(input, "INPUT"); + std::vector tokens = lexer.lex(input, "INPUT"); debug_print(tokens); parse(tokens); @@ -93,7 +92,7 @@ int main(int argc, char *argv[]) { } } else { - std::vector tokens = lexer.lex(load_file(argv[1]), argv[1]); + std::vector tokens = lexer.lex(load_file(argv[1]), argv[1]); debug_print(tokens); } }