generic-library/ChaiScript
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added 'wesley' project. Added eval. Fixed some of the parsers to do correct delegation

Browse Source
This commit is contained in:
Jonathan Turner
2009-06-02 23:29:04 +00:00
parent fc3b9de2d2
commit d984f973f4
7 changed files with 386 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
CMakeLists.txt
View File

@@ -8,5 +8,9 @@ SET (CMAKE_BUILD_TYPE gdb)
SET (CMAKE_C_FLAGS_GDB " -Wall -ggdb")
SET (CMAKE_CXX_FLAGS_GDB " -Wall -ggdb")
add_subdirectory(langkit bin/langkit)
add_subdirectory(boxedcpp bin/boxedcpp)
include_directories(boxedcpp)
add_subdirectory(langkit bin)
add_subdirectory(boxedcpp bin)
add_subdirectory(wesley bin)

8
boxedcpp/proxy_functions.hpp
View File

@@ -1,7 +1,7 @@
#include <boost/preprocessor.hpp>
#define gettypeinfo(z,n,text) ti.push_back(Get_Type_Info<Param ## n>()());
#define casthelper(z,n,text) ,Cast_Helper<Param ## n>()(params[n])
#define casthelper(z,n,text) ,Cast_Helper<Param ## n>()(params[n])
#ifndef BOOST_PP_IS_ITERATING
@@ -168,13 +168,13 @@ Boxed_Value dispatch(const std::vector<std::pair<const std::string, boost::share
}
}
throw std::runtime_error("No mathcing function to dispatch to");
throw std::runtime_error("No matching function to dispatch to");
}
# endif
#else
# define n BOOST_PP_ITERATION()
template<typename Ret, BOOST_PP_ENUM_PARAMS(n, typename Param) >
std::vector<Type_Info> build_param_type_list(const boost::function<Ret (BOOST_PP_ENUM_PARAMS(n, Param))> &f)
{
@@ -187,7 +187,7 @@ std::vector<Type_Info> build_param_type_list(const boost::function<Ret (BOOST_PP
}
template<typename Ret, BOOST_PP_ENUM_PARAMS(n, typename Param)>
Boxed_Value call_func(const boost::function<Ret (BOOST_PP_ENUM_PARAMS(n, Param))> &f,
Boxed_Value call_func(const boost::function<Ret (BOOST_PP_ENUM_PARAMS(n, Param))> &f,
const std::vector<Boxed_Value> &params)
{
if (params.size() != n)

2
langkit/CMakeLists.txt
View File

@@ -14,6 +14,8 @@ if(Boost_FOUND)
add_executable(langkit_test main.cpp langkit_lexer.cpp langkit_parser.cpp)
target_link_libraries(langkit_test ${Boost_LIBRARIES})
add_library(langkit SHARED langkit_lexer.cpp langkit_parser.cpp)
target_link_libraries(langkit ${Boost_LIBRARIES})
add_executable(langkit_unittest unittest.cpp)
target_link_libraries(langkit_unittest ${Boost_LIBRARIES})

104
langkit/langkit_parser.cpp
View File

@@ -37,26 +37,34 @@ std::pair<Token_Iterator, bool> Type_Rule(Token_Iterator iter, Token_Iterator en
std::pair<Token_Iterator, bool> Or_Rule(Token_Iterator iter, Token_Iterator end, TokenPtr parent, bool keep, int new_id, Rule lhs, Rule rhs) {
Token_Iterator new_iter;
unsigned int prev_size = parent->children.size();
unsigned int prev_size;
TokenPtr prev_parent = parent;
if (new_id != -1) {
parent = TokenPtr(new Token("", new_id, parent->filename));
}
prev_size = parent->children.size();
if (iter != end) {
std::pair<Token_Iterator, bool> result = lhs(iter, end, parent);
if (result.second) {
if (new_id != -1) {
parent->filename = (*iter)->filename;
parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end;
parent->filename = (*iter)->filename;
parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end;
prev_parent->children.push_back(parent);
}
return std::pair<Token_Iterator, bool>(result.first, true);
}
else {
if (parent->children.size() != prev_size) {
//Clear out the partial matches
parent->children.erase(parent->children.begin() + prev_size, parent->children.end());
}
result = rhs(iter, end, parent);
if (result.second) {
if (new_id != -1) {
@@ -95,7 +103,7 @@ std::pair<Token_Iterator, bool> And_Rule(Token_Iterator iter, Token_Iterator end
if (iter != end) {
std::pair<Token_Iterator, bool> result = lhs(iter, end, parent);
if ((result.second) && (result.first != end)) {
if (result.second) {
result = rhs(result.first, end, parent);
if (result.second) {
if (new_id != -1) {
@@ -127,25 +135,30 @@ std::pair<Token_Iterator, bool> Kleene_Rule
std::pair<Token_Iterator, bool> result;
Token_Iterator new_iter = iter;
if (new_id != -1) {
parent = TokenPtr(new Token("", new_id, parent->filename));
if (iter != end) {
if (new_id != -1) {
parent = TokenPtr(new Token("", new_id, parent->filename));
}
result.second = true;
while (result.second == true) {
result = rule(new_iter, end, parent);
new_iter = result.first;
}
if (new_id != -1) {
parent->filename = (*iter)->filename;
parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end;
prev_parent->children.push_back(parent);
}
return std::pair<Token_Iterator, bool>(result.first, true);
}
result.second = true;
while ((new_iter != end) && (result.second == true)) {
result = rule(new_iter, end, parent);
new_iter = result.first;
else {
return std::pair<Token_Iterator, bool>(iter, true);
}
if (new_id != -1) {
parent->filename = (*iter)->filename;
parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end;
prev_parent->children.push_back(parent);
}
return std::pair<Token_Iterator, bool>(result.first, true);
}
std::pair<Token_Iterator, bool> Plus_Rule
@@ -201,27 +214,32 @@ std::pair<Token_Iterator, bool> Optional_Rule
TokenPtr prev_parent = parent;
Token_Iterator new_iter = iter;
if (new_id != -1) {
parent = TokenPtr(new Token("", new_id, parent->filename));
if (iter != end) {
if (new_id != -1) {
parent = TokenPtr(new Token("", new_id, parent->filename));
}
std::pair<Token_Iterator, bool> result;
result.second = true;
if ((new_iter != end) && (result.second == true)) {
result = rule(new_iter, end, parent);
new_iter = result.first;
}
if (new_id != -1) {
parent->filename = (*iter)->filename;
parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end;
prev_parent->children.push_back(parent);
}
return std::pair<Token_Iterator, bool>(result.first, true);
}
std::pair<Token_Iterator, bool> result;
result.second = true;
if ((new_iter != end) && (result.second == true)) {
result = rule(new_iter, end, parent);
new_iter = result.first;
else {
return std::pair<Token_Iterator, bool>(iter, true);
}
if (new_id != -1) {
parent->filename = (*iter)->filename;
parent->start = (*iter)->start;
parent->end = (*(result.first - 1))->end;
prev_parent->children.push_back(parent);
}
return std::pair<Token_Iterator, bool>(result.first, true);
}
std::pair<Token_Iterator, bool> Nop_Rule

46
langkit/main.cpp
View File

@@ -10,13 +10,24 @@
#include "langkit_lexer.hpp"
#include "langkit_parser.hpp"
class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close,
Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon,
Function_Def, Scoped_Block, Statement, Equation, Return, Add, Comment}; };
class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close, //6
Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon, //15
Function_Def, Scoped_Block, Statement, Equation, Return, Expression, Term, Factor, Add, Subtract, Multiply, Divide, Negate, Comment, //29
Value, Fun_Call }; };
char *tokentype_to_string(int tokentype) {
char *token_types[] = {"File", "Whitespace", "Identifier", "Number", "Operator", "Parens_Open", "Parens_Close", //6
"Square_Open", "Square_Close", "Curly_Open", "Curly_Close", "Comma", "Quoted_String", "Single_Quoted_String", "Carriage_Return", "Semicolon", //15
"Function_Def", "Scoped_Block", "Statement", "Equation", "Return", "Expression", "Term", "Factor", "Add", "Subtract", "Multiply", "Divide", "Negate", "Comment", //29
"Value", "Fun_Call" };
return token_types[tokentype];
}
void debug_print(TokenPtr token, std::string prepend) {
std::cout << prepend << "Token: " << token->text << "(" << token->identifier << ") @ " << token->filename << ": (" << token->start.line
<< ", " << token->start.column << ") to (" << token->end.line << ", " << token->end.column << ") " << std::endl;
std::cout << prepend << "Token: " << token->text << "(" << tokentype_to_string(token->identifier) << ") @ " << token->filename
<< ": (" << token->start.line << ", " << token->start.column << ") to ("
<< token->end.line << ", " << token->end.column << ") " << std::endl;
for (unsigned int i = 0; i < token->children.size(); ++i) {
debug_print(token->children[i], prepend + " ");
@@ -68,17 +79,30 @@ void parse(std::vector<TokenPtr> &tokens, const char *filename) {
Rule params;
Rule block(TokenType::Scoped_Block);
Rule rule(TokenType::Function_Def);
//Rule rule(TokenType::Function_Def);
Rule statement(TokenType::Statement);
Rule return_statement(TokenType::Return);
Rule add(TokenType::Add);
Rule expression(TokenType::Expression);
Rule term(TokenType::Term);
Rule factor(TokenType::Factor);
Rule negate(TokenType::Negate);
Rule funcall(TokenType::Fun_Call);
Rule value;
rule = Ign(Str("def")) >> Id(TokenType::Identifier) >> ~(Ign(Str("(")) >> ~params >> Ign(Str(")"))) >> block;
/*
Rule rule = Ign(Str("def")) >> Id(TokenType::Identifier) >> ~(Ign(Str("(")) >> ~params >> Ign(Str(")"))) >> block;
params = Id(TokenType::Identifier) >> *(Ign(Str(",")) >> Id(TokenType::Identifier));
block = Ign(Str("{")) >> ~return_statement >> Ign(Str("}"));
return_statement = Ign(Str("return")) >> add;
add = Id(TokenType::Identifier) >> Ign(Str("+")) >> Id(TokenType::Identifier);
return_statement = Ign(Str("return")) >> expression;
*/
Rule rule = *(expression >> *Ign(Id(TokenType::Semicolon)));
expression = term >> *((Str("+") >> term) | (Str("-") >> term));
term = factor >> *((Str("*") >> factor) | (Str("/") >> factor));
factor = value | negate | (Ign(Str("+")) >> value);
funcall = Id(TokenType::Identifier) >> Ign(Id(TokenType::Parens_Open)) >> ~(expression >> *(Ign(Str("," )) >> expression)) >> Ign(Id(TokenType::Parens_Close));
negate = Ign(Str("-")) >> factor;
value = funcall | Id(TokenType::Identifier) | Id(TokenType::Number) | Id(TokenType::Quoted_String) | Id(TokenType::Single_Quoted_String);
/*
Rule rule = Str("x") << Id(TokenType::Semicolon);
@@ -139,7 +163,7 @@ int main(int argc, char *argv[]) {
lexer.set_multiline_comment(Pattern("/\\*", TokenType::Comment), Pattern("\\*/", TokenType::Comment));
lexer.set_singleline_comment(Pattern("//", TokenType::Comment));
lexer << Pattern("[A-Za-z]+", TokenType::Identifier);
lexer << Pattern("[A-Za-z_]+", TokenType::Identifier);
lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number);
lexer << Pattern("[!@#$%^&*\\-+=<>]+|/[!@#$%^&\\-+=<>]*", TokenType::Operator);
lexer << Pattern("\\(", TokenType::Parens_Open);

21
wesley/CMakeLists.txt Normal file
View File

@@ -0,0 +1,21 @@
cmake_minimum_required(VERSION 2.6)
enable_testing()
project(wesley)
SET (CMAKE_BUILD_TYPE gdb)
SET (CMAKE_C_FLAGS_GDB " -Wall -ggdb")
SET (CMAKE_CXX_FLAGS_GDB " -Wall -ggdb")
find_package(Boost 1.36.0 COMPONENTS regex unit_test_framework)
if(Boost_FOUND)
include_directories(${Boost_INCLUDE_DIRS})
include_directories(../langkit ../boxedcpp)
add_executable(wesley_test main.cpp)
target_link_libraries(wesley_test ${Boost_LIBRARIES} langkit)
endif()

257
wesley/main.cpp Normal file
View File

@@ -0,0 +1,257 @@
#include <boost/bind.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/function.hpp>
#include <iostream>
#include <map>
#include <fstream>
#include "boxedcpp.hpp"
#include "bootstrap.hpp"
#include "bootstrap_stl.hpp"
#include "langkit_lexer.hpp"
#include "langkit_parser.hpp"
class TokenType { public: enum Type { File, Whitespace, Identifier, Number, Operator, Parens_Open, Parens_Close,
Square_Open, Square_Close, Curly_Open, Curly_Close, Comma, Quoted_String, Single_Quoted_String, Carriage_Return, Semicolon,
Function_Def, Scoped_Block, Statement, Equation, Return, Expression, Term, Factor, Negate, Comment,
Value, Fun_Call }; };
char *tokentype_to_string(int tokentype) {
char *token_types[] = {"File", "Whitespace", "Identifier", "Number", "Operator", "Parens_Open", "Parens_Close",
"Square_Open", "Square_Close", "Curly_Open", "Curly_Close", "Comma", "Quoted_String", "Single_Quoted_String", "Carriage_Return", "Semicolon",
"Function_Def", "Scoped_Block", "Statement", "Equation", "Return", "Expression", "Term", "Factor", "Negate", "Comment",
"Value", "Fun_Call" };
return token_types[tokentype];
}
void debug_print(TokenPtr token, std::string prepend) {
std::cout << prepend << "Token: " << token->text << "(" << tokentype_to_string(token->identifier) << ") @ " << token->filename
<< ": (" << token->start.line << ", " << token->start.column << ") to ("
<< token->end.line << ", " << token->end.column << ") " << std::endl;
for (unsigned int i = 0; i < token->children.size(); ++i) {
debug_print(token->children[i], prepend + " ");
}
}
void debug_print(std::vector<TokenPtr> &tokens) {
for (unsigned int i = 0; i < tokens.size(); ++i) {
debug_print(tokens[i], "");
}
}
//A function that prints any string passed to it
void print_string(const std::string &s)
{
std::cout << s << std::endl;
}
void print_double(const double &d)
{
std::cout << d << std::endl;
}
std::string load_file(const char *filename) {
std::ifstream infile (filename, std::ios::in | std::ios::ate);
if (!infile.is_open()) {
std::cerr << "Can not open " << filename << std::endl;
exit(0);
}
std::streampos size = infile.tellg();
infile.seekg(0, std::ios::beg);
std::vector<char> v(size);
infile.read(&v[0], size);
std::string ret_val (v.empty() ? std::string() : std::string (v.begin(), v.end()).c_str());
return ret_val;
}
Boxed_Value eval_token(TokenPtr node, BoxedCPP_System ss) {
Boxed_Value retval;
unsigned int i;
switch (node->identifier) {
case (TokenType::Value) :
case (TokenType::File) :
for (i = 0; i < node->children.size(); ++i) {
eval_token(node->children[i], ss);
}
break;
case (TokenType::Identifier) :
break;
case (TokenType::Number) :
retval = Boxed_Value(double(atof(node->text.c_str())));
break;
case (TokenType::Quoted_String) :
retval = Boxed_Value(node->text);
break;
case (TokenType::Single_Quoted_String) :
retval = Boxed_Value(node->text);
break;
case (TokenType::Factor) :
case (TokenType::Expression) :
case (TokenType::Term) : {
retval = eval_token(node->children[0], ss);
if (node->children.size() > 1) {
for (i = 1; i < node->children.size(); i += 2) {
Param_List_Builder plb;
plb << retval;
plb << eval_token(node->children[i + 1], ss);
retval = dispatch(ss.get_function(node->children[i]->text), plb);
}
}
}
break;
case (TokenType::Fun_Call) : {
Param_List_Builder plb;
for (i = 1; i < node->children.size(); ++i) {
plb << eval_token(node->children[i], ss);
}
retval = dispatch(ss.get_function(node->children[0]->text), plb);
}
break;
case (TokenType::Scoped_Block) :
case (TokenType::Statement) :
case (TokenType::Negate) :
case (TokenType::Return) :
case (TokenType::Carriage_Return) :
case (TokenType::Semicolon) :
case (TokenType::Equation) :
case (TokenType::Function_Def) :
case (TokenType::Comment) :
case (TokenType::Operator) :
case (TokenType::Whitespace) :
case (TokenType::Parens_Open) :
case (TokenType::Parens_Close) :
case (TokenType::Square_Open) :
case (TokenType::Square_Close) :
case (TokenType::Curly_Open) :
case (TokenType::Curly_Close) :
case (TokenType::Comma) :
break;
}
return retval;
}
void eval(TokenPtr parent) {
BoxedCPP_System ss;
bootstrap(ss);
bootstrap_vector<std::vector<int> >(ss);
//dump_system(ss);
//Register a new function, this one with typing for us, so we don't have to ubox anything
//right here
ss.register_function(boost::function<void (const std::string &)>(&print_string), "print");
ss.register_function(boost::function<void (const double &)>(&print_double), "print");
/*
Param_List_Builder plb;
plb << Boxed_Value(double(2.5));
Boxed_Value val = dispatch(ss.get_function("to_string"), plb);
Param_List_Builder plb2;
plb2 << val;
dispatch(ss.get_function("print"), plb2);
*/
Boxed_Value value = eval_token(parent, ss);
}
void parse(std::vector<TokenPtr> &tokens, const char *filename) {
Rule params;
Rule block(TokenType::Scoped_Block);
Rule statement(TokenType::Statement);
Rule return_statement(TokenType::Return);
Rule expression(TokenType::Expression);
Rule term(TokenType::Term);
Rule factor(TokenType::Factor);
Rule negate(TokenType::Negate);
Rule funcall(TokenType::Fun_Call);
Rule value;
Rule rule = *(expression >> *Ign(Id(TokenType::Semicolon)));
expression = term >> *((Str("+") >> term) | (Str("-") >> term));
term = factor >> *((Str("*") >> factor) | (Str("/") >> factor));
factor = value | negate | (Ign(Str("+")) >> value);
funcall = Id(TokenType::Identifier) >> Ign(Id(TokenType::Parens_Open)) >> ~(expression >> *(Ign(Str("," )) >> expression)) >> Ign(Id(TokenType::Parens_Close));
negate = Ign(Str("-")) >> factor;
value = funcall | Id(TokenType::Identifier) | Id(TokenType::Number) | Id(TokenType::Quoted_String) | Id(TokenType::Single_Quoted_String);
Token_Iterator iter = tokens.begin(), end = tokens.end();
TokenPtr parent(new Token("Root", TokenType::File, filename));
std::pair<Token_Iterator, bool> results = rule(iter, end, parent);
if (results.second) {
//std::cout << "Parse successful: " << std::endl;
//debug_print(parent, "");
eval(parent);
}
else {
std::cout << "Parse failed: " << std::endl;
debug_print(parent, "");
}
}
int main(int argc, char *argv[]) {
std::string input;
Lexer lexer;
lexer.set_skip(Pattern("[ \\t]+", TokenType::Whitespace));
lexer.set_line_sep(Pattern("\\n|\\r\\n", TokenType::Carriage_Return));
lexer.set_command_sep(Pattern(";|\\r\\n|\\n", TokenType::Semicolon));
lexer.set_multiline_comment(Pattern("/\\*", TokenType::Comment), Pattern("\\*/", TokenType::Comment));
lexer.set_singleline_comment(Pattern("//", TokenType::Comment));
lexer << Pattern("[A-Za-z_]+", TokenType::Identifier);
lexer << Pattern("[0-9]+(\\.[0-9]+)?", TokenType::Number);
lexer << Pattern("[!@#$%^&*\\-+=<>]+|/[!@#$%^&\\-+=<>]*", TokenType::Operator);
lexer << Pattern("\\(", TokenType::Parens_Open);
lexer << Pattern("\\)", TokenType::Parens_Close);
lexer << Pattern("\\[", TokenType::Square_Open);
lexer << Pattern("\\]", TokenType::Square_Close);
lexer << Pattern("\\{", TokenType::Curly_Open);
lexer << Pattern("\\}", TokenType::Curly_Close);
lexer << Pattern(",", TokenType::Comma);
lexer << Pattern("\"(?:[^\"\\\\]|\\\\.)*\"", TokenType::Quoted_String);
lexer << Pattern("'(?:[^'\\\\]|\\\\.)*'", TokenType::Single_Quoted_String);
if (argc < 2) {
std::cout << "eval> ";
std::getline(std::cin, input);
while (input != "quit") {
std::vector<TokenPtr> tokens = lexer.lex(input, "INPUT");
for (unsigned int i = 0; i < tokens.size(); ++i) {
if ((tokens[i]->identifier == TokenType::Quoted_String) || (tokens[i]->identifier == TokenType::Single_Quoted_String)) {
tokens[i]->text = tokens[i]->text.substr(1, tokens[i]->text.size()-2);
}
}
//debug_print(tokens);
parse(tokens, "INPUT");
std::cout << "eval> ";
std::getline(std::cin, input);
}
}
else {
std::vector<TokenPtr> tokens = lexer.lex(load_file(argv[1]), argv[1]);
debug_print(tokens);
parse(tokens, argv[1]);
}
}