Merge pull request #241 from ChaiScript/add_performance_tests

Add performance tests

.decent_ci-Linux.yaml

@@ -2,7 +2,8 @@ compilers:
   - name: "clang"
     version: "3.5"
     skip_packaging: true
-    cmake_extra_flags: -DUSE_LIBCXX:BOOL=OFF -DBUILD_SAMPLES:BOOL=ON -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON
+    cmake_extra_flags: -DUSE_LIBCXX:BOOL=OFF -DBUILD_SAMPLES:BOOL=ON -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DRUN_PERFORMANCE_TESTS:BOOL=ON
+    collect_performance_results: true
   - name: "clang"
     build_tag: "LibC++"
     version: "3.5"
@@ -20,15 +21,16 @@ compilers:
     cmake_extra_flags: -DUSE_LIBCXX:BOOL=OFF -DBUILD_SAMPLES:BOOL=ON -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_THREAD_SANITIZER:BOOL=ON
   - name: "gcc"
     version: "4.8"
-    cmake_extra_flags: -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DBUILD_TESTING:BOOL=ON
+    cmake_extra_flags: -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DBUILD_TESTING:BOOL=ON -DRUN_PERFORMANCE_TESTS:BOOL=ON
+    collect_performance_results: true
   - name: "gcc"
     version: "4.6"
     skip_packaging: true
-    cmake_extra_flags: -DBUILD_SAMPLES:BOOL=ON -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON
+    cmake_extra_flags: -DBUILD_SAMPLES:BOOL=ON -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DRUN_PERFORMANCE_TESTS:BOOL=ON
+    collect_performance_results: true
   - name: cppcheck
     compiler_extra_flags: --enable=all -I include --inline-suppr -Umax  --suppress="*:cmake*" --suppress="*:unittests/catch.hpp" --force
   - name: custom_check
     commands:
        - ./contrib/check_for_tabs.rb
-    
 

CMakeLists.txt

@@ -20,6 +20,7 @@ option(BUILD_MODULES "Build Extra Modules (stl)" TRUE)
 option(BUILD_SAMPLES "Build Samples Folder" FALSE)
 option(RUN_FUZZY_TESTS "Run tests generated by AFL" FALSE)
 option(USE_STD_MAKE_SHARED "Use std::make_shared instead of chaiscript::make_shared" FALSE)
+option(RUN_PERFORMANCE_TESTS "Run Performance Tests" FALSE)
 
 mark_as_advanced(USE_STD_MAKE_SHARED)
 
@@ -122,16 +123,12 @@ configure_file(Doxyfile.in ${CMAKE_BINARY_DIR}/Doxyfile)
 include(CTest)
 include(CPack)
 
-include(cmake/CheckCXX11Features.cmake)
-
 if(NOT MINGW)
   find_library(READLINE_LIBRARY NAMES readline PATH /usr/lib /usr/local/lib /opt/local/lib)
 endif()
 
-if(HAS_CXX11_VARIADIC_TEMPLATES)
-  message(STATUS "Variadic Template support detected")
-else()
-  message(SEND_ERROR "The selected compiler does not support the C++11 feature Variadic Templates.")
+if (UNIX AND NOT APPLE)
+  find_program(VALGRIND NAMES valgrind PATH /usr/bin /usr/local/bin)
 endif()
 
 enable_testing()
@@ -283,6 +280,9 @@ endif()
 file(GLOB UNIT_TESTS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}/unittests/ ${CMAKE_CURRENT_SOURCE_DIR}/unittests/*.chai ${CMAKE_CURRENT_SOURCE_DIR}/unittests/3.x/*.chai)
 list(SORT UNIT_TESTS)
 
+file(GLOB PERFORMANCE_TESTS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}/performance_tests/ ${CMAKE_CURRENT_SOURCE_DIR}/performance_tests/*.chai)
+list(SORT PERFORMANCE_TESTS)
+
 
 if (RUN_FUZZY_TESTS)
 
@@ -347,8 +347,8 @@ if(BUILD_TESTING)
           string(REGEX MATCHALL "TEST_CASE\\([ ]*\"[^\"]+\"" found_tests ${contents})
           foreach(hit ${found_tests})
             string(REGEX REPLACE "TEST_CASE\\([ ]*(\"[^\"]+\").*" "\\1" test_name ${hit})
-            add_test(${test_name} "${executable}" ${test_name})
-            set_tests_properties(${test_name} PROPERTIES TIMEOUT 660 ENVIRONMENT "PATH=${NEWPATH}")
+            add_test(compiled.${test_name} "${executable}" ${test_name})
+            set_tests_properties(compiled.${test_name} PROPERTIES TIMEOUT 660 ENVIRONMENT "PATH=${NEWPATH}")
           endforeach()
         endif()
       endif()
@@ -382,12 +382,24 @@ if(BUILD_TESTING)
   )
 
 
+  set(TESTS "")
+
   foreach(filename ${UNIT_TESTS})
-    message(STATUS "Adding test ${filename}")
-    add_test(${filename} chai ${CMAKE_CURRENT_SOURCE_DIR}/unittests/unit_test.inc ${CMAKE_CURRENT_SOURCE_DIR}/unittests/${filename})
+    message(STATUS "Adding unit test ${filename}")
+    add_test(unit.${filename} chai ${CMAKE_CURRENT_SOURCE_DIR}/unittests/unit_test.inc ${CMAKE_CURRENT_SOURCE_DIR}/unittests/${filename})
+    list(APPEND TESTS unit.${filename})
   endforeach()
 
-  set_property(TEST ${UNIT_TESTS}
+  if (RUN_PERFORMANCE_TESTS)
+    foreach(filename ${PERFORMANCE_TESTS})
+      message(STATUS "Adding performance test ${filename}")
+
+      add_test(NAME performance.${filename} COMMAND ${VALGRIND} --tool=callgrind --callgrind-out-file=callgrind.performance.${filename} $<TARGET_FILE:chai> ${CMAKE_CURRENT_SOURCE_DIR}/performance_tests/${filename})
+      list(APPEND TESTS performance.${filename})
+    endforeach()
+  endif()
+
+  set_property(TEST ${TESTS}
     PROPERTY ENVIRONMENT
     "CHAI_USE_PATH=${CMAKE_CURRENT_SOURCE_DIR}/unittests/"
     "CHAI_MODULE_PATH=${CMAKE_CURRENT_BINARY_DIR}/"

cmake/CheckCXX11Features.cmake

@@ -1,103 +0,0 @@
-# Checks for C++11 features
-#  CXX11_FEATURE_LIST - a list containing all supported features
-#  HAS_CXX11_AUTO               - auto keyword
-#  HAS_CXX11_NULLPTR            - nullptr
-#  HAS_CXX11_LAMBDA             - lambdas
-#  HAS_CXX11_STATIC_ASSERT      - static_assert()
-#  HAS_CXX11_RVALUE_REFERENCES  - rvalue references
-#  HAS_CXX11_DECLTYPE           - decltype keyword
-#  HAS_CXX11_CSTDINT_H          - cstdint header
-#  HAS_CXX11_LONG_LONG          - long long signed & unsigned types
-#  HAS_CXX11_VARIADIC_TEMPLATES - variadic templates
-#  HAS_CXX11_CONSTEXPR          - constexpr keyword
-#  HAS_CXX11_SIZEOF_MEMBER      - sizeof() non-static members
-#  HAS_CXX11_FUNC               - __func__ preprocessor constant
-#
-# Original script by Rolf Eike Beer
-# Modifications by Andreas Weis
-#
-CMAKE_MINIMUM_REQUIRED(VERSION 2.8.3)
-
-SET(CHECK_CXX11_OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-IF(CMAKE_COMPILER_IS_GNUCXX)
-	SET(CMAKE_CXX_FLAGS "-std=c++0x")
-endif()
-
-MACRO(CXX11_CHECK_FEATURE FEATURE_NAME FEATURE_NUMBER RESULT_VAR)
-	IF (NOT DEFINED ${RESULT_VAR})
-		SET(_bindir "${CMAKE_CURRENT_BINARY_DIR}/cxx11/cxx11_${FEATURE_NAME}")
-
-		IF (${FEATURE_NUMBER})
-			SET(_SRCFILE_BASE ${CMAKE_CURRENT_LIST_DIR}/c++11-test-${FEATURE_NAME}-N${FEATURE_NUMBER})
-			SET(_LOG_NAME "\"${FEATURE_NAME}\" (N${FEATURE_NUMBER})")
-		ELSE (${FEATURE_NUMBER})
-			SET(_SRCFILE_BASE ${CMAKE_CURRENT_LIST_DIR}/c++11-test-${FEATURE_NAME})
-			SET(_LOG_NAME "\"${FEATURE_NAME}\"")
-		ENDIF (${FEATURE_NUMBER})
-		MESSAGE(STATUS "Checking C++11 support for ${_LOG_NAME}")
-
-		SET(_SRCFILE "${_SRCFILE_BASE}.cpp")
-		SET(_SRCFILE_FAIL "${_SRCFILE_BASE}_fail.cpp")
-		SET(_SRCFILE_FAIL_COMPILE "${_SRCFILE_BASE}_fail_compile.cpp")
-
-		IF (CROSS_COMPILING)
-			try_compile(${RESULT_VAR} "${_bindir}" "${_SRCFILE}")
-			IF (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL})
-				try_compile(${RESULT_VAR} "${_bindir}_fail" "${_SRCFILE_FAIL}")
-			ENDIF (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL})
-		ELSE (CROSS_COMPILING)
-			try_run(_RUN_RESULT_VAR _COMPILE_RESULT_VAR
-					"${_bindir}" "${_SRCFILE}")
-			IF (_COMPILE_RESULT_VAR AND NOT _RUN_RESULT_VAR)
-				SET(${RESULT_VAR} TRUE)
-			ELSE (_COMPILE_RESULT_VAR AND NOT _RUN_RESULT_VAR)
-				SET(${RESULT_VAR} FALSE)
-			ENDIF (_COMPILE_RESULT_VAR AND NOT _RUN_RESULT_VAR)
-			IF (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL})
-				try_run(_RUN_RESULT_VAR _COMPILE_RESULT_VAR
-						"${_bindir}_fail" "${_SRCFILE_FAIL}")
-				IF (_COMPILE_RESULT_VAR AND _RUN_RESULT_VAR)
-					SET(${RESULT_VAR} TRUE)
-				ELSE (_COMPILE_RESULT_VAR AND _RUN_RESULT_VAR)
-					SET(${RESULT_VAR} FALSE)
-				ENDIF (_COMPILE_RESULT_VAR AND _RUN_RESULT_VAR)
-			ENDIF (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL})
-		ENDIF (CROSS_COMPILING)
-		IF (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL_COMPILE})
-			try_compile(_TMP_RESULT "${_bindir}_fail_compile" "${_SRCFILE_FAIL_COMPILE}")
-			IF (_TMP_RESULT)
-				SET(${RESULT_VAR} FALSE)
-			ELSE (_TMP_RESULT)
-				SET(${RESULT_VAR} TRUE)
-			ENDIF (_TMP_RESULT)
-		ENDIF (${RESULT_VAR} AND EXISTS ${_SRCFILE_FAIL_COMPILE})
-
-		IF (${RESULT_VAR})
-			MESSAGE(STATUS "Checking C++11 support for ${_LOG_NAME} -- works")
-			LIST(APPEND CXX11_FEATURE_LIST ${RESULT_VAR})
-		ELSE (${RESULT_VAR})
-			MESSAGE(STATUS "Checking C++11 support for ${_LOG_NAME} -- not supported")
-		ENDIF (${RESULT_VAR})
-		SET(${RESULT_VAR} ${${RESULT_VAR}} CACHE INTERNAL "C++11 support for ${_LOG_NAME}")
-	ENDIF (NOT DEFINED ${RESULT_VAR})
-ENDMACRO(CXX11_CHECK_FEATURE)
-
-CXX11_CHECK_FEATURE("auto"               2546 HAS_CXX11_AUTO)
-CXX11_CHECK_FEATURE("nullptr"            2431 HAS_CXX11_NULLPTR)
-CXX11_CHECK_FEATURE("lambda"             2927 HAS_CXX11_LAMBDA)
-CXX11_CHECK_FEATURE("static_assert"      1720 HAS_CXX11_STATIC_ASSERT)
-CXX11_CHECK_FEATURE("rvalue_references"  2118 HAS_CXX11_RVALUE_REFERENCES)
-CXX11_CHECK_FEATURE("decltype"           2343 HAS_CXX11_DECLTYPE)
-CXX11_CHECK_FEATURE("cstdint"            ""   HAS_CXX11_CSTDINT_H)
-CXX11_CHECK_FEATURE("long_long"          1811 HAS_CXX11_LONG_LONG)
-CXX11_CHECK_FEATURE("variadic_templates" 2555 HAS_CXX11_VARIADIC_TEMPLATES)
-CXX11_CHECK_FEATURE("constexpr"          2235 HAS_CXX11_CONSTEXPR)
-CXX11_CHECK_FEATURE("sizeof_member"      2253 HAS_CXX11_SIZEOF_MEMBER)
-CXX11_CHECK_FEATURE("__func__"           2340 HAS_CXX11_FUNC)
-
-SET(CXX11_FEATURE_LIST ${CXX11_FEATURE_LIST} CACHE STRING "C++11 feature support list")
-MARK_AS_ADVANCED(FORCE CXX11_FEATURE_LIST)
-
-SET(CMAKE_CXX_FLAGS ${CHECK_CXX11_OLD_CMAKE_CXX_FLAGS})
-UNSET(CHECK_CXX11_OLD_CMAKE_CXX_FLAGS)
-

cmake/c++11-test-__func__-N2340.cpp

@@ -1,8 +0,0 @@
-#include <cstring>
-
-int main()
-{
-	if (!__func__) { return 1; }
-	if(std::strlen(__func__) <= 0) { return 1; }
-	return 0;
-}

cmake/c++11-test-auto-N2546.cpp

@@ -1,12 +0,0 @@
-
-int main()
-{
-	auto i = 5;
-	auto f = 3.14159f;
-	auto d = 3.14159;
-	bool ret = (
-		(sizeof(f) < sizeof(d)) &&
-		(sizeof(i) == sizeof(int))
-	);
-	return ret ? 0 : 1;
-}

cmake/c++11-test-constexpr-N2235.cpp

@@ -1,19 +0,0 @@
-constexpr int square(int x)
-{
-	return x*x;
-}
-
-constexpr int the_answer()
-{
-	return 42;
-}
-
-int main()
-{
-	int test_arr[square(3)];
-	bool ret = (
-		(square(the_answer()) == 1764) &&
-		(sizeof(test_arr)/sizeof(test_arr[0]) == 9)
-	);
-	return ret ? 0 : 1;
-}

cmake/c++11-test-cstdint.cpp

@@ -1,10 +0,0 @@
-#include <cstdint>
-int main()
-{
-	bool test = 
-		(sizeof(int8_t) == 1) &&
-		(sizeof(int16_t) == 2) &&
-		(sizeof(int32_t) == 4) &&
-		(sizeof(int64_t) == 8);
-	return test ? 0 : 1;
-}

cmake/c++11-test-decltype-N2343.cpp

@@ -1,11 +0,0 @@
-
-bool check_size(int i)
-{
-	return sizeof(int) == sizeof(decltype(i));
-}
-
-int main()
-{
-	bool ret = check_size(42);
-	return ret ? 0 : 1;
-}

cmake/c++11-test-lambda-N2927.cpp

@@ -1,5 +0,0 @@
-int main()
-{
-	int ret = 0;
-	return ([&ret]() -> int { return ret; })();
-}

cmake/c++11-test-long_long-N1811.cpp

@@ -1,7 +0,0 @@
-int main(void)
-{
-	long long l;
-	unsigned long long ul;
-
-	return ((sizeof(l) >= 8) && (sizeof(ul) >= 8)) ? 0 : 1;
-}

cmake/c++11-test-nullptr-N2431.cpp

@@ -1,5 +0,0 @@
-int main()
-{
-	int* test = nullptr;
-	return test ? 1 : 0;
-}

cmake/c++11-test-nullptr-N2431_fail_compile.cpp

@@ -1,5 +0,0 @@
-int main()
-{
-	int i = nullptr;
-	return 1;
-}

cmake/c++11-test-rvalue_references-N2118.cpp

@@ -1,15 +0,0 @@
-int foo(int& lvalue)
-{
-	return 123;
-}
-
-int foo(int&& rvalue)
-{
-	return 321;
-}
-
-int main()
-{
-	int i = 42;
-	return ((foo(i) == 123) && (foo(42) == 321)) ? 0 : 1;
-}

cmake/c++11-test-sizeof_member-N2253.cpp

@@ -1,14 +0,0 @@
-struct foo {
-	char bar;
-	int baz;
-};
-
-int main(void)
-{
-	bool ret = (
-		(sizeof(foo::bar) == 1) &&
-		(sizeof(foo::baz) >= sizeof(foo::bar)) &&
-		(sizeof(foo) >= sizeof(foo::bar)+sizeof(foo::baz))
-	);
-	return ret ? 0 : 1;
-}

cmake/c++11-test-static_assert-N1720.cpp

@@ -1,5 +0,0 @@
-int main()
-{
-	static_assert(0 < 1, "your ordering of integers is screwed");
-	return 0;
-}

cmake/c++11-test-static_assert-N1720_fail_compile.cpp

@@ -1,5 +0,0 @@
-int main()
-{
-	static_assert(1 < 0, "this should fail");
-	return 0;
-}

cmake/c++11-test-variadic_templates-N2555.cpp

@@ -1,23 +0,0 @@
-int Accumulate()
-{
-	return 0;
-}
-
-template<typename T, typename... Ts>
-int Accumulate(T v, Ts... vs)
-{
-	return v + Accumulate(vs...);
-}
-
-template<int... Is>
-int CountElements()
-{
-	return sizeof...(Is);
-}
-
-int main()
-{
-	int acc = Accumulate(1, 2, 3, 4, -5);
-	int count = CountElements<1,2,3,4,5>();
-	return ((acc == 5) && (count == 5)) ? 0 : 1;
-}

include/chaiscript/dispatchkit/type_info.hpp

@@ -83,12 +83,12 @@ namespace chaiscript
           && (*m_bare_type_info) == ti;
       }
 
-      CHAISCRIPT_CONSTEXPR bool is_const() const CHAISCRIPT_NOEXCEPT { return static_cast<bool>(m_flags & (1 << is_const_flag)); }
-      CHAISCRIPT_CONSTEXPR bool is_reference() const CHAISCRIPT_NOEXCEPT { return static_cast<bool>(m_flags & (1 << is_reference_flag)); }
-      CHAISCRIPT_CONSTEXPR bool is_void() const CHAISCRIPT_NOEXCEPT { return static_cast<bool>(m_flags & (1 << is_void_flag)); }
-      CHAISCRIPT_CONSTEXPR bool is_arithmetic() const CHAISCRIPT_NOEXCEPT { return static_cast<bool>(m_flags & (1 << is_arithmetic_flag)); }
-      CHAISCRIPT_CONSTEXPR bool is_undef() const CHAISCRIPT_NOEXCEPT { return static_cast<bool>(m_flags & (1 << is_undef_flag)); }
-      CHAISCRIPT_CONSTEXPR bool is_pointer() const CHAISCRIPT_NOEXCEPT { return static_cast<bool>(m_flags & (1 << is_pointer_flag)); }
+      CHAISCRIPT_CONSTEXPR bool is_const() const CHAISCRIPT_NOEXCEPT { return (m_flags & (1 << is_const_flag)) != 0; }
+      CHAISCRIPT_CONSTEXPR bool is_reference() const CHAISCRIPT_NOEXCEPT { return (m_flags & (1 << is_reference_flag)) != 0; }
+      CHAISCRIPT_CONSTEXPR bool is_void() const CHAISCRIPT_NOEXCEPT { return (m_flags & (1 << is_void_flag)) != 0; }
+      CHAISCRIPT_CONSTEXPR bool is_arithmetic() const CHAISCRIPT_NOEXCEPT { return (m_flags & (1 << is_arithmetic_flag)) != 0; }
+      CHAISCRIPT_CONSTEXPR bool is_undef() const CHAISCRIPT_NOEXCEPT { return (m_flags & (1 << is_undef_flag)) != 0; }
+      CHAISCRIPT_CONSTEXPR bool is_pointer() const CHAISCRIPT_NOEXCEPT { return (m_flags & (1 << is_pointer_flag)) != 0; }
 
       std::string name() const
       {