Added 'wesley' project. Added eval. Fixed some of the parsers to do correct delegation

CMakeLists.txt

@@ -8,5 +8,9 @@ SET (CMAKE_BUILD_TYPE gdb)
 SET (CMAKE_C_FLAGS_GDB " -Wall -ggdb")
 SET (CMAKE_CXX_FLAGS_GDB " -Wall -ggdb")
 
-add_subdirectory(langkit bin/langkit)
+include_directories(boxedcpp)
-add_subdirectory(boxedcpp bin/boxedcpp)
+
+add_subdirectory(langkit bin)
+add_subdirectory(boxedcpp bin)
+add_subdirectory(wesley bin)
+ 

boxedcpp/proxy_functions.hpp

@@ -1,7 +1,7 @@
 #include <boost/preprocessor.hpp>
 
 #define gettypeinfo(z,n,text)  ti.push_back(Get_Type_Info<Param ## n>()());
-#define casthelper(z,n,text) ,Cast_Helper<Param ## n>()(params[n]) 
+#define casthelper(z,n,text) ,Cast_Helper<Param ## n>()(params[n])
 
 
 #ifndef  BOOST_PP_IS_ITERATING
@@ -168,13 +168,13 @@ Boxed_Value dispatch(const std::vector<std::pair<const std::string, boost::share
     }
   }
 
-  throw std::runtime_error("No mathcing function to dispatch to");
+  throw std::runtime_error("No matching function to dispatch to");
 }
 
 # endif
 #else
 # define n BOOST_PP_ITERATION()
- 
+
 template<typename Ret, BOOST_PP_ENUM_PARAMS(n, typename Param) >
 std::vector<Type_Info> build_param_type_list(const boost::function<Ret (BOOST_PP_ENUM_PARAMS(n, Param))> &f)
 {
@@ -187,7 +187,7 @@ std::vector<Type_Info> build_param_type_list(const boost::function<Ret (BOOST_PP
 }
 
 template<typename Ret, BOOST_PP_ENUM_PARAMS(n, typename Param)>
-Boxed_Value call_func(const boost::function<Ret (BOOST_PP_ENUM_PARAMS(n, Param))> &f, 
+Boxed_Value call_func(const boost::function<Ret (BOOST_PP_ENUM_PARAMS(n, Param))> &f,
     const std::vector<Boxed_Value> &params)
 {
   if (params.size() != n)

langkit/CMakeLists.txt

@@ -14,6 +14,8 @@ if(Boost_FOUND)
    add_executable(langkit_test main.cpp langkit_lexer.cpp langkit_parser.cpp)
    target_link_libraries(langkit_test ${Boost_LIBRARIES})
 
+   add_library(langkit SHARED langkit_lexer.cpp langkit_parser.cpp)
+   target_link_libraries(langkit ${Boost_LIBRARIES})
 
    add_executable(langkit_unittest unittest.cpp)
    target_link_libraries(langkit_unittest ${Boost_LIBRARIES})

langkit/langkit_parser.cpp

@@ -37,26 +37,34 @@ std::pair<Token_Iterator, bool> Type_Rule(Token_Iterator iter, Token_Iterator en
 
 std::pair<Token_Iterator, bool> Or_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, Rule lhs, Rule rhs) {
     Token_Iterator new_iter;
-    unsigned int prev_size = parent->children.size();
+    unsigned int prev_size;
     TokenPtr prev_parent = parent;
 
     if (new_id != -1) {
         parent = TokenPtr(new Token("", new_id, parent->filename));
     }
 
+    prev_size = parent->children.size();
+
     if (iter != end) {
         std::pair<Token_Iterator, bool> result = lhs(iter, end, parent);
 
         if (result.second) {
+            if (new_id != -1) {
+                parent->filename = (*iter)->filename;
+                parent->start = (*iter)->start;
+                parent->end = (*(result.first - 1))->end;
 
-            parent->filename = (*iter)->filename;
+                prev_parent->children.push_back(parent);
-            parent->start = (*iter)->start;
+            }
-            parent->end = (*(result.first - 1))->end;
-
-
             return std::pair<Token_Iterator, bool>(result.first, true);
         }
         else {
+            if (parent->children.size() != prev_size) {
+                //Clear out the partial matches
+                parent->children.erase(parent->children.begin() + prev_size, parent->children.end());
+            }
+
             result = rhs(iter, end, parent);
             if (result.second) {
                 if (new_id != -1) {
@@ -95,7 +103,7 @@ std::pair<Token_Iterator, bool> And_Rule(Token_Iterator iter, Token_Iterator end
     if (iter != end) {
         std::pair<Token_Iterator, bool> result = lhs(iter, end, parent);
 
-        if ((result.second) && (result.first != end)) {
+        if (result.second) {
             result = rhs(result.first, end, parent);
             if (result.second) {
                 if (new_id != -1) {
@@ -127,25 +135,30 @@ std::pair<Token_Iterator, bool> Kleene_Rule
     std::pair<Token_Iterator, bool> result;
     Token_Iterator new_iter = iter;
 
-    if (new_id != -1) {
+    if (iter != end) {
-        parent = TokenPtr(new Token("", new_id, parent->filename));
+        if (new_id != -1) {
+            parent = TokenPtr(new Token("", new_id, parent->filename));
+        }
+
+        result.second = true;
+        while (result.second == true) {
+            result = rule(new_iter, end, parent);
+            new_iter = result.first;
+        }
+
+        if (new_id != -1) {
+
+            parent->filename = (*iter)->filename;
+            parent->start = (*iter)->start;
+            parent->end = (*(result.first - 1))->end;
+
+            prev_parent->children.push_back(parent);
+        }
+        return std::pair<Token_Iterator, bool>(result.first, true);
     }
-
+    else {
-    result.second = true;
+        return std::pair<Token_Iterator, bool>(iter, true);
-    while ((new_iter != end) && (result.second == true)) {
-        result = rule(new_iter, end, parent);
-        new_iter = result.first;
     }
-
-    if (new_id != -1) {
-
-        parent->filename = (*iter)->filename;
-        parent->start = (*iter)->start;
-        parent->end = (*(result.first - 1))->end;
-
-        prev_parent->children.push_back(parent);
-    }
-    return std::pair<Token_Iterator, bool>(result.first, true);
 }
 
 std::pair<Token_Iterator, bool> Plus_Rule
@@ -201,27 +214,32 @@ std::pair<Token_Iterator, bool> Optional_Rule
     TokenPtr prev_parent = parent;
     Token_Iterator new_iter = iter;
 
-    if (new_id != -1) {
+    if (iter != end) {
-        parent = TokenPtr(new Token("", new_id, parent->filename));
+        if (new_id != -1) {
+            parent = TokenPtr(new Token("", new_id, parent->filename));
+        }
+
+        std::pair<Token_Iterator, bool> result;
+        result.second = true;
+        if ((new_iter != end) && (result.second == true)) {
+            result = rule(new_iter, end, parent);
+            new_iter = result.first;
+        }
+
+        if (new_id != -1) {
+
+            parent->filename = (*iter)->filename;
+            parent->start = (*iter)->start;
+            parent->end = (*(result.first - 1))->end;
+
+
+            prev_parent->children.push_back(parent);
+        }
+        return std::pair<Token_Iterator, bool>(result.first, true);
     }
-
+    else {
-    std::pair<Token_Iterator, bool> result;
+        return std::pair<Token_Iterator, bool>(iter, true);
-    result.second = true;
-    if ((new_iter != end) && (result.second == true)) {
-        result = rule(new_iter, end, parent);
-        new_iter = result.first;
     }
-
-    if (new_id != -1) {
-
-        parent->filename = (*iter)->filename;
-        parent->start = (*iter)->start;
-        parent->end = (*(result.first - 1))->end;
-
-
-        prev_parent->children.push_back(parent);
-    }
-    return std::pair<Token_Iterator, bool>(result.first, true);
 }
 
 std::pair<Token_Iterator, bool> Nop_Rule

langkit/main.cpp

@@ -10,13 +10,24 @@
 #include "langkit_lexer.hpp"
 #include "langkit_parser.hpp"
 
-class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close,
+class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, //6
-    Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon,
+    Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon, //15
-    Function_Def, Scoped_Block, Statement, Equation, Return, Add, Comment}; };
+    Function_Def, Scoped_Block, Statement, Equation, Return, Expression, Term, Factor, Add, Subtract, Multiply, Divide, Negate, Comment, //29
+    Value, Fun_Call }; };
+
+char *tokentype_to_string(int tokentype) {
+    char *token_types[] = {"File", "Whitespace", "Identifier", "Number", "Operator", "Parens_Open", "Parens_Close", //6
+        "Square_Open", "Square_Close", "Curly_Open", "Curly_Close", "Comma", "Quoted_String", "Single_Quoted_String", "Carriage_Return", "Semicolon", //15
+        "Function_Def", "Scoped_Block", "Statement", "Equation", "Return", "Expression", "Term", "Factor", "Add", "Subtract", "Multiply", "Divide", "Negate", "Comment", //29
+        "Value", "Fun_Call" };
+
+    return token_types[tokentype];
+}
 
 void debug_print(TokenPtr token, std::string prepend) {
-    std::cout << prepend << "Token: " << token->text << "(" << token->identifier << ") @ " << token->filename << ": ("  << token->start.line
+    std::cout << prepend << "Token: " << token->text << "(" << tokentype_to_string(token->identifier) << ") @ " << token->filename
-        << ", " << token->start.column << ") to (" << token->end.line << ", " << token->end.column << ") " << std::endl;
+        << ": ("  << token->start.line << ", " << token->start.column << ") to ("
+        << token->end.line << ", " << token->end.column << ") " << std::endl;
 
     for (unsigned int i = 0; i < token->children.size(); ++i) {
         debug_print(token->children[i], prepend + "  ");
@@ -68,17 +79,30 @@ void parse(std::vector<TokenPtr> &tokens, const char *filename) {
 
     Rule params;
     Rule block(TokenType::Scoped_Block);
-    Rule rule(TokenType::Function_Def);
+    //Rule rule(TokenType::Function_Def);
     Rule statement(TokenType::Statement);
     Rule return_statement(TokenType::Return);
-    Rule add(TokenType::Add);
+    Rule expression(TokenType::Expression);
+    Rule term(TokenType::Term);
+    Rule factor(TokenType::Factor);
+    Rule negate(TokenType::Negate);
+    Rule funcall(TokenType::Fun_Call);
+    Rule value;
 
-    rule = Ign(Str("def")) >> Id(TokenType::Identifier) >> ~(Ign(Str("(")) >> ~params >> Ign(Str(")"))) >> block;
+    /*
+    Rule rule = Ign(Str("def")) >> Id(TokenType::Identifier) >> ~(Ign(Str("(")) >> ~params >> Ign(Str(")"))) >> block;
     params = Id(TokenType::Identifier) >> *(Ign(Str(",")) >> Id(TokenType::Identifier));
     block = Ign(Str("{")) >> ~return_statement >> Ign(Str("}"));
-    return_statement = Ign(Str("return")) >> add;
+    return_statement = Ign(Str("return")) >> expression;
-    add = Id(TokenType::Identifier) >> Ign(Str("+")) >> Id(TokenType::Identifier);
+    */
+    Rule rule = *(expression >> *Ign(Id(TokenType::Semicolon)));
+    expression = term >> *((Str("+") >> term) | (Str("-") >> term));
+    term = factor >> *((Str("*") >> factor) | (Str("/") >> factor));
+    factor = value | negate | (Ign(Str("+")) >> value);
+    funcall = Id(TokenType::Identifier) >> Ign(Id(TokenType::Parens_Open)) >> ~(expression >> *(Ign(Str("," )) >> expression)) >> Ign(Id(TokenType::Parens_Close));
+    negate = Ign(Str("-")) >> factor;
 
+    value = funcall | Id(TokenType::Identifier) | Id(TokenType::Number) | Id(TokenType::Quoted_String) | Id(TokenType::Single_Quoted_String);
 
     /*
     Rule rule = Str("x") << Id(TokenType::Semicolon);
@@ -139,7 +163,7 @@ int main(int argc, char *argv[]) {
     lexer.set_multiline_comment(Pattern("/\\*", TokenType::Comment), Pattern("\\*/", TokenType::Comment));
     lexer.set_singleline_comment(Pattern("//", TokenType::Comment));
 
-    lexer << Pattern("[A-Za-z]+", TokenType::Identifier);
+    lexer << Pattern("[A-Za-z_]+", TokenType::Identifier);
     lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number);
     lexer << Pattern("[!@#$%^&*\\-+=<>]+|/[!@#$%^&\\-+=<>]*", TokenType::Operator);
     lexer << Pattern("\\(", TokenType::Parens_Open);

wesley/CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 2.6)
+enable_testing()
+
+project(wesley)
+
+SET (CMAKE_BUILD_TYPE gdb)
+SET (CMAKE_C_FLAGS_GDB " -Wall -ggdb")
+SET (CMAKE_CXX_FLAGS_GDB " -Wall -ggdb")
+
+find_package(Boost 1.36.0 COMPONENTS regex unit_test_framework)
+if(Boost_FOUND)
+   include_directories(${Boost_INCLUDE_DIRS})
+   include_directories(../langkit ../boxedcpp)
+   
+   add_executable(wesley_test main.cpp)
+
+   target_link_libraries(wesley_test ${Boost_LIBRARIES} langkit)
+
+endif()
+
+

wesley/main.cpp

@@ -0,0 +1,257 @@
+#include <boost/bind.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/function.hpp>
+
+#include <iostream>
+#include <map>
+#include <fstream>
+
+#include "boxedcpp.hpp"
+#include "bootstrap.hpp"
+#include "bootstrap_stl.hpp"
+
+#include "langkit_lexer.hpp"
+#include "langkit_parser.hpp"
+
+class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close,
+    Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon,
+    Function_Def, Scoped_Block, Statement, Equation, Return, Expression, Term, Factor, Negate, Comment,
+    Value, Fun_Call }; };
+
+char *tokentype_to_string(int tokentype) {
+    char *token_types[] = {"File", "Whitespace", "Identifier", "Number", "Operator", "Parens_Open", "Parens_Close",
+        "Square_Open", "Square_Close", "Curly_Open", "Curly_Close", "Comma", "Quoted_String", "Single_Quoted_String", "Carriage_Return", "Semicolon",
+        "Function_Def", "Scoped_Block", "Statement", "Equation", "Return", "Expression", "Term", "Factor", "Negate", "Comment",
+        "Value", "Fun_Call" };
+
+    return token_types[tokentype];
+}
+
+void debug_print(TokenPtr token, std::string prepend) {
+    std::cout << prepend << "Token: " << token->text << "(" << tokentype_to_string(token->identifier) << ") @ " << token->filename
+        << ": ("  << token->start.line << ", " << token->start.column << ") to ("
+        << token->end.line << ", " << token->end.column << ") " << std::endl;
+
+    for (unsigned int i = 0; i < token->children.size(); ++i) {
+        debug_print(token->children[i], prepend + "  ");
+    }
+}
+
+void debug_print(std::vector<TokenPtr> &tokens) {
+    for (unsigned int i = 0; i < tokens.size(); ++i) {
+        debug_print(tokens[i], "");
+    }
+}
+
+//A function that prints any string passed to it
+void print_string(const std::string &s)
+{
+  std::cout << s << std::endl;
+}
+
+void print_double(const double &d)
+{
+  std::cout << d << std::endl;
+}
+
+std::string load_file(const char *filename) {
+    std::ifstream infile (filename, std::ios::in | std::ios::ate);
+
+    if (!infile.is_open()) {
+        std::cerr << "Can not open " << filename << std::endl;
+        exit(0);
+    }
+
+    std::streampos size = infile.tellg();
+    infile.seekg(0, std::ios::beg);
+
+    std::vector<char> v(size);
+    infile.read(&v[0], size);
+
+    std::string ret_val (v.empty() ? std::string() : std::string (v.begin(), v.end()).c_str());
+
+    return ret_val;
+}
+
+Boxed_Value eval_token(TokenPtr node, BoxedCPP_System ss) {
+    Boxed_Value retval;
+    unsigned int i;
+
+    switch (node->identifier) {
+        case (TokenType::Value) :
+        case (TokenType::File) :
+            for (i = 0; i < node->children.size(); ++i) {
+                eval_token(node->children[i], ss);
+            }
+        break;
+        case (TokenType::Identifier) :
+        break;
+        case (TokenType::Number) :
+            retval = Boxed_Value(double(atof(node->text.c_str())));
+        break;
+        case (TokenType::Quoted_String) :
+            retval = Boxed_Value(node->text);
+        break;
+        case (TokenType::Single_Quoted_String) :
+            retval = Boxed_Value(node->text);
+        break;
+        case (TokenType::Factor) :
+        case (TokenType::Expression) :
+        case (TokenType::Term) : {
+            retval = eval_token(node->children[0], ss);
+            if (node->children.size() > 1) {
+                for (i = 1; i < node->children.size(); i += 2) {
+                    Param_List_Builder plb;
+                    plb << retval;
+                    plb << eval_token(node->children[i + 1], ss);
+
+                    retval = dispatch(ss.get_function(node->children[i]->text), plb);
+                }
+            }
+        }
+        break;
+        case (TokenType::Fun_Call) : {
+            Param_List_Builder plb;
+            for (i = 1; i < node->children.size(); ++i) {
+                plb << eval_token(node->children[i], ss);
+            }
+            retval = dispatch(ss.get_function(node->children[0]->text), plb);
+        }
+        break;
+        case (TokenType::Scoped_Block) :
+        case (TokenType::Statement) :
+        case (TokenType::Negate) :
+        case (TokenType::Return) :
+        case (TokenType::Carriage_Return) :
+        case (TokenType::Semicolon) :
+        case (TokenType::Equation) :
+        case (TokenType::Function_Def) :
+        case (TokenType::Comment) :
+        case (TokenType::Operator) :
+        case (TokenType::Whitespace) :
+        case (TokenType::Parens_Open) :
+        case (TokenType::Parens_Close) :
+        case (TokenType::Square_Open) :
+        case (TokenType::Square_Close) :
+        case (TokenType::Curly_Open) :
+        case (TokenType::Curly_Close) :
+        case (TokenType::Comma) :
+        break;
+    }
+
+    return retval;
+}
+
+void eval(TokenPtr parent) {
+    BoxedCPP_System ss;
+    bootstrap(ss);
+    bootstrap_vector<std::vector<int> >(ss);
+    //dump_system(ss);
+
+    //Register a new function, this one with typing for us, so we don't have to ubox anything
+    //right here
+    ss.register_function(boost::function<void (const std::string &)>(&print_string), "print");
+    ss.register_function(boost::function<void (const double &)>(&print_double), "print");
+
+    /*
+    Param_List_Builder plb;
+    plb << Boxed_Value(double(2.5));
+    Boxed_Value val = dispatch(ss.get_function("to_string"), plb);
+
+    Param_List_Builder plb2;
+    plb2 << val;
+    dispatch(ss.get_function("print"), plb2);
+    */
+
+    Boxed_Value value = eval_token(parent, ss);
+}
+
+void parse(std::vector<TokenPtr> &tokens, const char *filename) {
+
+
+    Rule params;
+    Rule block(TokenType::Scoped_Block);
+    Rule statement(TokenType::Statement);
+    Rule return_statement(TokenType::Return);
+    Rule expression(TokenType::Expression);
+    Rule term(TokenType::Term);
+    Rule factor(TokenType::Factor);
+    Rule negate(TokenType::Negate);
+    Rule funcall(TokenType::Fun_Call);
+    Rule value;
+
+    Rule rule = *(expression >> *Ign(Id(TokenType::Semicolon)));
+    expression = term >> *((Str("+") >> term) | (Str("-") >> term));
+    term = factor >> *((Str("*") >> factor) | (Str("/") >> factor));
+    factor = value | negate | (Ign(Str("+")) >> value);
+    funcall = Id(TokenType::Identifier) >> Ign(Id(TokenType::Parens_Open)) >> ~(expression >> *(Ign(Str("," )) >> expression)) >> Ign(Id(TokenType::Parens_Close));
+    negate = Ign(Str("-")) >> factor;
+
+    value = funcall | Id(TokenType::Identifier) | Id(TokenType::Number) | Id(TokenType::Quoted_String) | Id(TokenType::Single_Quoted_String);
+
+    Token_Iterator iter = tokens.begin(), end = tokens.end();
+    TokenPtr parent(new Token("Root", TokenType::File, filename));
+
+    std::pair<Token_Iterator, bool> results = rule(iter, end, parent);
+
+    if (results.second) {
+        //std::cout << "Parse successful: " << std::endl;
+        //debug_print(parent, "");
+        eval(parent);
+    }
+    else {
+        std::cout << "Parse failed: " << std::endl;
+        debug_print(parent, "");
+    }
+}
+
+
+int main(int argc, char *argv[]) {
+    std::string input;
+
+    Lexer lexer;
+    lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace));
+    lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return));
+    lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon));
+    lexer.set_multiline_comment(Pattern("/\\*", TokenType::Comment), Pattern("\\*/", TokenType::Comment));
+    lexer.set_singleline_comment(Pattern("//", TokenType::Comment));
+
+    lexer << Pattern("[A-Za-z_]+", TokenType::Identifier);
+    lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number);
+    lexer << Pattern("[!@#$%^&*\\-+=<>]+|/[!@#$%^&\\-+=<>]*", TokenType::Operator);
+    lexer << Pattern("\\(", TokenType::Parens_Open);
+    lexer << Pattern("\\)", TokenType::Parens_Close);
+    lexer << Pattern("\\[", TokenType::Square_Open);
+    lexer << Pattern("\\]", TokenType::Square_Close);
+    lexer << Pattern("\\{", TokenType::Curly_Open);
+    lexer << Pattern("\\}", TokenType::Curly_Close);
+    lexer << Pattern(",", TokenType::Comma);
+    lexer << Pattern("\"(?:[^\"\\\\]|\\\\.)*\"", TokenType::Quoted_String);
+    lexer << Pattern("'(?:[^'\\\\]|\\\\.)*'", TokenType::Single_Quoted_String);
+
+    if (argc < 2) {
+        std::cout << "eval> ";
+        std::getline(std::cin, input);
+        while (input != "quit") {
+            std::vector<TokenPtr> tokens = lexer.lex(input, "INPUT");
+
+            for (unsigned int i = 0; i < tokens.size(); ++i) {
+                if ((tokens[i]->identifier == TokenType::Quoted_String) || (tokens[i]->identifier == TokenType::Single_Quoted_String)) {
+                    tokens[i]->text = tokens[i]->text.substr(1, tokens[i]->text.size()-2);
+                }
+            }
+
+            //debug_print(tokens);
+            parse(tokens, "INPUT");
+
+            std::cout << "eval> ";
+            std::getline(std::cin, input);
+        }
+    }
+    else {
+        std::vector<TokenPtr> tokens = lexer.lex(load_file(argv[1]), argv[1]);
+        debug_print(tokens);
+        parse(tokens, argv[1]);
+    }
+}
+