From d984f973f4ac705f28f2d28cf6f9ae0df3430abf Mon Sep 17 00:00:00 2001 From: Jonathan Turner Date: Tue, 2 Jun 2009 23:29:04 +0000 Subject: [PATCH] Added 'wesley' project. Added eval. Fixed some of the parsers to do correct delegation --- CMakeLists.txt | 8 +- boxedcpp/proxy_functions.hpp | 8 +- langkit/CMakeLists.txt | 2 + langkit/langkit_parser.cpp | 104 ++++++++------ langkit/main.cpp | 46 +++++-- wesley/CMakeLists.txt | 21 +++ wesley/main.cpp | 257 +++++++++++++++++++++++++++++++++++ 7 files changed, 386 insertions(+), 60 deletions(-) create mode 100644 wesley/CMakeLists.txt create mode 100644 wesley/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b0773c..501f464 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,5 +8,9 @@ SET (CMAKE_BUILD_TYPE gdb) SET (CMAKE_C_FLAGS_GDB " -Wall -ggdb") SET (CMAKE_CXX_FLAGS_GDB " -Wall -ggdb") -add_subdirectory(langkit bin/langkit) -add_subdirectory(boxedcpp bin/boxedcpp) +include_directories(boxedcpp) + +add_subdirectory(langkit bin) +add_subdirectory(boxedcpp bin) +add_subdirectory(wesley bin) + \ No newline at end of file diff --git a/boxedcpp/proxy_functions.hpp b/boxedcpp/proxy_functions.hpp index 9684ef9..cb2d6cf 100644 --- a/boxedcpp/proxy_functions.hpp +++ b/boxedcpp/proxy_functions.hpp @@ -1,7 +1,7 @@ #include #define gettypeinfo(z,n,text) ti.push_back(Get_Type_Info()()); -#define casthelper(z,n,text) ,Cast_Helper()(params[n]) +#define casthelper(z,n,text) ,Cast_Helper()(params[n]) #ifndef BOOST_PP_IS_ITERATING @@ -168,13 +168,13 @@ Boxed_Value dispatch(const std::vector std::vector build_param_type_list(const boost::function &f) { @@ -187,7 +187,7 @@ std::vector build_param_type_list(const boost::function -Boxed_Value call_func(const boost::function &f, +Boxed_Value call_func(const boost::function &f, const std::vector ¶ms) { if (params.size() != n) diff --git a/langkit/CMakeLists.txt b/langkit/CMakeLists.txt index 551d2b7..799f2f7 100644 --- a/langkit/CMakeLists.txt +++ b/langkit/CMakeLists.txt @@ -14,6 +14,8 @@ if(Boost_FOUND) add_executable(langkit_test main.cpp langkit_lexer.cpp langkit_parser.cpp) target_link_libraries(langkit_test ${Boost_LIBRARIES}) + add_library(langkit SHARED langkit_lexer.cpp langkit_parser.cpp) + target_link_libraries(langkit ${Boost_LIBRARIES}) add_executable(langkit_unittest unittest.cpp) target_link_libraries(langkit_unittest ${Boost_LIBRARIES}) diff --git a/langkit/langkit_parser.cpp b/langkit/langkit_parser.cpp index 0cbfb11..6c51b03 100644 --- a/langkit/langkit_parser.cpp +++ b/langkit/langkit_parser.cpp @@ -37,26 +37,34 @@ std::pair Type_Rule(Token_Iterator iter, Token_Iterator en std::pair Or_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, Rule lhs, Rule rhs) { Token_Iterator new_iter; - unsigned int prev_size = parent->children.size(); + unsigned int prev_size; TokenPtr prev_parent = parent; if (new_id != -1) { parent = TokenPtr(new Token("", new_id, parent->filename)); } + prev_size = parent->children.size(); + if (iter != end) { std::pair result = lhs(iter, end, parent); if (result.second) { + if (new_id != -1) { + parent->filename = (*iter)->filename; + parent->start = (*iter)->start; + parent->end = (*(result.first - 1))->end; - parent->filename = (*iter)->filename; - parent->start = (*iter)->start; - parent->end = (*(result.first - 1))->end; - - + prev_parent->children.push_back(parent); + } return std::pair(result.first, true); } else { + if (parent->children.size() != prev_size) { + //Clear out the partial matches + parent->children.erase(parent->children.begin() + prev_size, parent->children.end()); + } + result = rhs(iter, end, parent); if (result.second) { if (new_id != -1) { @@ -95,7 +103,7 @@ std::pair And_Rule(Token_Iterator iter, Token_Iterator end if (iter != end) { std::pair result = lhs(iter, end, parent); - if ((result.second) && (result.first != end)) { + if (result.second) { result = rhs(result.first, end, parent); if (result.second) { if (new_id != -1) { @@ -127,25 +135,30 @@ std::pair Kleene_Rule std::pair result; Token_Iterator new_iter = iter; - if (new_id != -1) { - parent = TokenPtr(new Token("", new_id, parent->filename)); + if (iter != end) { + if (new_id != -1) { + parent = TokenPtr(new Token("", new_id, parent->filename)); + } + + result.second = true; + while (result.second == true) { + result = rule(new_iter, end, parent); + new_iter = result.first; + } + + if (new_id != -1) { + + parent->filename = (*iter)->filename; + parent->start = (*iter)->start; + parent->end = (*(result.first - 1))->end; + + prev_parent->children.push_back(parent); + } + return std::pair(result.first, true); } - - result.second = true; - while ((new_iter != end) && (result.second == true)) { - result = rule(new_iter, end, parent); - new_iter = result.first; + else { + return std::pair(iter, true); } - - if (new_id != -1) { - - parent->filename = (*iter)->filename; - parent->start = (*iter)->start; - parent->end = (*(result.first - 1))->end; - - prev_parent->children.push_back(parent); - } - return std::pair(result.first, true); } std::pair Plus_Rule @@ -201,27 +214,32 @@ std::pair Optional_Rule TokenPtr prev_parent = parent; Token_Iterator new_iter = iter; - if (new_id != -1) { - parent = TokenPtr(new Token("", new_id, parent->filename)); + if (iter != end) { + if (new_id != -1) { + parent = TokenPtr(new Token("", new_id, parent->filename)); + } + + std::pair result; + result.second = true; + if ((new_iter != end) && (result.second == true)) { + result = rule(new_iter, end, parent); + new_iter = result.first; + } + + if (new_id != -1) { + + parent->filename = (*iter)->filename; + parent->start = (*iter)->start; + parent->end = (*(result.first - 1))->end; + + + prev_parent->children.push_back(parent); + } + return std::pair(result.first, true); } - - std::pair result; - result.second = true; - if ((new_iter != end) && (result.second == true)) { - result = rule(new_iter, end, parent); - new_iter = result.first; + else { + return std::pair(iter, true); } - - if (new_id != -1) { - - parent->filename = (*iter)->filename; - parent->start = (*iter)->start; - parent->end = (*(result.first - 1))->end; - - - prev_parent->children.push_back(parent); - } - return std::pair(result.first, true); } std::pair Nop_Rule diff --git a/langkit/main.cpp b/langkit/main.cpp index 75baf6f..a06fbd8 100644 --- a/langkit/main.cpp +++ b/langkit/main.cpp @@ -10,13 +10,24 @@ #include "langkit_lexer.hpp" #include "langkit_parser.hpp" -class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, - Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon, - Function_Def, Scoped_Block, Statement, Equation, Return, Add, Comment}; }; +class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, //6 + Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon, //15 + Function_Def, Scoped_Block, Statement, Equation, Return, Expression, Term, Factor, Add, Subtract, Multiply, Divide, Negate, Comment, //29 + Value, Fun_Call }; }; + +char *tokentype_to_string(int tokentype) { + char *token_types[] = {"File", "Whitespace", "Identifier", "Number", "Operator", "Parens_Open", "Parens_Close", //6 + "Square_Open", "Square_Close", "Curly_Open", "Curly_Close", "Comma", "Quoted_String", "Single_Quoted_String", "Carriage_Return", "Semicolon", //15 + "Function_Def", "Scoped_Block", "Statement", "Equation", "Return", "Expression", "Term", "Factor", "Add", "Subtract", "Multiply", "Divide", "Negate", "Comment", //29 + "Value", "Fun_Call" }; + + return token_types[tokentype]; +} void debug_print(TokenPtr token, std::string prepend) { - std::cout << prepend << "Token: " << token->text << "(" << token->identifier << ") @ " << token->filename << ": (" << token->start.line - << ", " << token->start.column << ") to (" << token->end.line << ", " << token->end.column << ") " << std::endl; + std::cout << prepend << "Token: " << token->text << "(" << tokentype_to_string(token->identifier) << ") @ " << token->filename + << ": (" << token->start.line << ", " << token->start.column << ") to (" + << token->end.line << ", " << token->end.column << ") " << std::endl; for (unsigned int i = 0; i < token->children.size(); ++i) { debug_print(token->children[i], prepend + " "); @@ -68,17 +79,30 @@ void parse(std::vector &tokens, const char *filename) { Rule params; Rule block(TokenType::Scoped_Block); - Rule rule(TokenType::Function_Def); + //Rule rule(TokenType::Function_Def); Rule statement(TokenType::Statement); Rule return_statement(TokenType::Return); - Rule add(TokenType::Add); + Rule expression(TokenType::Expression); + Rule term(TokenType::Term); + Rule factor(TokenType::Factor); + Rule negate(TokenType::Negate); + Rule funcall(TokenType::Fun_Call); + Rule value; - rule = Ign(Str("def")) >> Id(TokenType::Identifier) >> ~(Ign(Str("(")) >> ~params >> Ign(Str(")"))) >> block; + /* + Rule rule = Ign(Str("def")) >> Id(TokenType::Identifier) >> ~(Ign(Str("(")) >> ~params >> Ign(Str(")"))) >> block; params = Id(TokenType::Identifier) >> *(Ign(Str(",")) >> Id(TokenType::Identifier)); block = Ign(Str("{")) >> ~return_statement >> Ign(Str("}")); - return_statement = Ign(Str("return")) >> add; - add = Id(TokenType::Identifier) >> Ign(Str("+")) >> Id(TokenType::Identifier); + return_statement = Ign(Str("return")) >> expression; + */ + Rule rule = *(expression >> *Ign(Id(TokenType::Semicolon))); + expression = term >> *((Str("+") >> term) | (Str("-") >> term)); + term = factor >> *((Str("*") >> factor) | (Str("/") >> factor)); + factor = value | negate | (Ign(Str("+")) >> value); + funcall = Id(TokenType::Identifier) >> Ign(Id(TokenType::Parens_Open)) >> ~(expression >> *(Ign(Str("," )) >> expression)) >> Ign(Id(TokenType::Parens_Close)); + negate = Ign(Str("-")) >> factor; + value = funcall | Id(TokenType::Identifier) | Id(TokenType::Number) | Id(TokenType::Quoted_String) | Id(TokenType::Single_Quoted_String); /* Rule rule = Str("x") << Id(TokenType::Semicolon); @@ -139,7 +163,7 @@ int main(int argc, char *argv[]) { lexer.set_multiline_comment(Pattern("/\\*", TokenType::Comment), Pattern("\\*/", TokenType::Comment)); lexer.set_singleline_comment(Pattern("//", TokenType::Comment)); - lexer << Pattern("[A-Za-z]+", TokenType::Identifier); + lexer << Pattern("[A-Za-z_]+", TokenType::Identifier); lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number); lexer << Pattern("[!@#$%^&*\\-+=<>]+|/[!@#$%^&\\-+=<>]*", TokenType::Operator); lexer << Pattern("\\(", TokenType::Parens_Open); diff --git a/wesley/CMakeLists.txt b/wesley/CMakeLists.txt new file mode 100644 index 0000000..b32b61d --- /dev/null +++ b/wesley/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 2.6) +enable_testing() + +project(wesley) + +SET (CMAKE_BUILD_TYPE gdb) +SET (CMAKE_C_FLAGS_GDB " -Wall -ggdb") +SET (CMAKE_CXX_FLAGS_GDB " -Wall -ggdb") + +find_package(Boost 1.36.0 COMPONENTS regex unit_test_framework) +if(Boost_FOUND) + include_directories(${Boost_INCLUDE_DIRS}) + include_directories(../langkit ../boxedcpp) + + add_executable(wesley_test main.cpp) + + target_link_libraries(wesley_test ${Boost_LIBRARIES} langkit) + +endif() + + diff --git a/wesley/main.cpp b/wesley/main.cpp new file mode 100644 index 0000000..3db86b2 --- /dev/null +++ b/wesley/main.cpp @@ -0,0 +1,257 @@ +#include +#include +#include + +#include +#include +#include + +#include "boxedcpp.hpp" +#include "bootstrap.hpp" +#include "bootstrap_stl.hpp" + +#include "langkit_lexer.hpp" +#include "langkit_parser.hpp" + +class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, + Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon, + Function_Def, Scoped_Block, Statement, Equation, Return, Expression, Term, Factor, Negate, Comment, + Value, Fun_Call }; }; + +char *tokentype_to_string(int tokentype) { + char *token_types[] = {"File", "Whitespace", "Identifier", "Number", "Operator", "Parens_Open", "Parens_Close", + "Square_Open", "Square_Close", "Curly_Open", "Curly_Close", "Comma", "Quoted_String", "Single_Quoted_String", "Carriage_Return", "Semicolon", + "Function_Def", "Scoped_Block", "Statement", "Equation", "Return", "Expression", "Term", "Factor", "Negate", "Comment", + "Value", "Fun_Call" }; + + return token_types[tokentype]; +} + +void debug_print(TokenPtr token, std::string prepend) { + std::cout << prepend << "Token: " << token->text << "(" << tokentype_to_string(token->identifier) << ") @ " << token->filename + << ": (" << token->start.line << ", " << token->start.column << ") to (" + << token->end.line << ", " << token->end.column << ") " << std::endl; + + for (unsigned int i = 0; i < token->children.size(); ++i) { + debug_print(token->children[i], prepend + " "); + } +} + +void debug_print(std::vector &tokens) { + for (unsigned int i = 0; i < tokens.size(); ++i) { + debug_print(tokens[i], ""); + } +} + +//A function that prints any string passed to it +void print_string(const std::string &s) +{ + std::cout << s << std::endl; +} + +void print_double(const double &d) +{ + std::cout << d << std::endl; +} + +std::string load_file(const char *filename) { + std::ifstream infile (filename, std::ios::in | std::ios::ate); + + if (!infile.is_open()) { + std::cerr << "Can not open " << filename << std::endl; + exit(0); + } + + std::streampos size = infile.tellg(); + infile.seekg(0, std::ios::beg); + + std::vector v(size); + infile.read(&v[0], size); + + std::string ret_val (v.empty() ? std::string() : std::string (v.begin(), v.end()).c_str()); + + return ret_val; +} + +Boxed_Value eval_token(TokenPtr node, BoxedCPP_System ss) { + Boxed_Value retval; + unsigned int i; + + switch (node->identifier) { + case (TokenType::Value) : + case (TokenType::File) : + for (i = 0; i < node->children.size(); ++i) { + eval_token(node->children[i], ss); + } + break; + case (TokenType::Identifier) : + break; + case (TokenType::Number) : + retval = Boxed_Value(double(atof(node->text.c_str()))); + break; + case (TokenType::Quoted_String) : + retval = Boxed_Value(node->text); + break; + case (TokenType::Single_Quoted_String) : + retval = Boxed_Value(node->text); + break; + case (TokenType::Factor) : + case (TokenType::Expression) : + case (TokenType::Term) : { + retval = eval_token(node->children[0], ss); + if (node->children.size() > 1) { + for (i = 1; i < node->children.size(); i += 2) { + Param_List_Builder plb; + plb << retval; + plb << eval_token(node->children[i + 1], ss); + + retval = dispatch(ss.get_function(node->children[i]->text), plb); + } + } + } + break; + case (TokenType::Fun_Call) : { + Param_List_Builder plb; + for (i = 1; i < node->children.size(); ++i) { + plb << eval_token(node->children[i], ss); + } + retval = dispatch(ss.get_function(node->children[0]->text), plb); + } + break; + case (TokenType::Scoped_Block) : + case (TokenType::Statement) : + case (TokenType::Negate) : + case (TokenType::Return) : + case (TokenType::Carriage_Return) : + case (TokenType::Semicolon) : + case (TokenType::Equation) : + case (TokenType::Function_Def) : + case (TokenType::Comment) : + case (TokenType::Operator) : + case (TokenType::Whitespace) : + case (TokenType::Parens_Open) : + case (TokenType::Parens_Close) : + case (TokenType::Square_Open) : + case (TokenType::Square_Close) : + case (TokenType::Curly_Open) : + case (TokenType::Curly_Close) : + case (TokenType::Comma) : + break; + } + + return retval; +} + +void eval(TokenPtr parent) { + BoxedCPP_System ss; + bootstrap(ss); + bootstrap_vector >(ss); + //dump_system(ss); + + //Register a new function, this one with typing for us, so we don't have to ubox anything + //right here + ss.register_function(boost::function(&print_string), "print"); + ss.register_function(boost::function(&print_double), "print"); + + /* + Param_List_Builder plb; + plb << Boxed_Value(double(2.5)); + Boxed_Value val = dispatch(ss.get_function("to_string"), plb); + + Param_List_Builder plb2; + plb2 << val; + dispatch(ss.get_function("print"), plb2); + */ + + Boxed_Value value = eval_token(parent, ss); +} + +void parse(std::vector &tokens, const char *filename) { + + + Rule params; + Rule block(TokenType::Scoped_Block); + Rule statement(TokenType::Statement); + Rule return_statement(TokenType::Return); + Rule expression(TokenType::Expression); + Rule term(TokenType::Term); + Rule factor(TokenType::Factor); + Rule negate(TokenType::Negate); + Rule funcall(TokenType::Fun_Call); + Rule value; + + Rule rule = *(expression >> *Ign(Id(TokenType::Semicolon))); + expression = term >> *((Str("+") >> term) | (Str("-") >> term)); + term = factor >> *((Str("*") >> factor) | (Str("/") >> factor)); + factor = value | negate | (Ign(Str("+")) >> value); + funcall = Id(TokenType::Identifier) >> Ign(Id(TokenType::Parens_Open)) >> ~(expression >> *(Ign(Str("," )) >> expression)) >> Ign(Id(TokenType::Parens_Close)); + negate = Ign(Str("-")) >> factor; + + value = funcall | Id(TokenType::Identifier) | Id(TokenType::Number) | Id(TokenType::Quoted_String) | Id(TokenType::Single_Quoted_String); + + Token_Iterator iter = tokens.begin(), end = tokens.end(); + TokenPtr parent(new Token("Root", TokenType::File, filename)); + + std::pair results = rule(iter, end, parent); + + if (results.second) { + //std::cout << "Parse successful: " << std::endl; + //debug_print(parent, ""); + eval(parent); + } + else { + std::cout << "Parse failed: " << std::endl; + debug_print(parent, ""); + } +} + + +int main(int argc, char *argv[]) { + std::string input; + + Lexer lexer; + lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace)); + lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return)); + lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon)); + lexer.set_multiline_comment(Pattern("/\\*", TokenType::Comment), Pattern("\\*/", TokenType::Comment)); + lexer.set_singleline_comment(Pattern("//", TokenType::Comment)); + + lexer << Pattern("[A-Za-z_]+", TokenType::Identifier); + lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number); + lexer << Pattern("[!@#$%^&*\\-+=<>]+|/[!@#$%^&\\-+=<>]*", TokenType::Operator); + lexer << Pattern("\\(", TokenType::Parens_Open); + lexer << Pattern("\\)", TokenType::Parens_Close); + lexer << Pattern("\\[", TokenType::Square_Open); + lexer << Pattern("\\]", TokenType::Square_Close); + lexer << Pattern("\\{", TokenType::Curly_Open); + lexer << Pattern("\\}", TokenType::Curly_Close); + lexer << Pattern(",", TokenType::Comma); + lexer << Pattern("\"(?:[^\"\\\\]|\\\\.)*\"", TokenType::Quoted_String); + lexer << Pattern("'(?:[^'\\\\]|\\\\.)*'", TokenType::Single_Quoted_String); + + if (argc < 2) { + std::cout << "eval> "; + std::getline(std::cin, input); + while (input != "quit") { + std::vector tokens = lexer.lex(input, "INPUT"); + + for (unsigned int i = 0; i < tokens.size(); ++i) { + if ((tokens[i]->identifier == TokenType::Quoted_String) || (tokens[i]->identifier == TokenType::Single_Quoted_String)) { + tokens[i]->text = tokens[i]->text.substr(1, tokens[i]->text.size()-2); + } + } + + //debug_print(tokens); + parse(tokens, "INPUT"); + + std::cout << "eval> "; + std::getline(std::cin, input); + } + } + else { + std::vector tokens = lexer.lex(load_file(argv[1]), argv[1]); + debug_print(tokens); + parse(tokens, argv[1]); + } +} +