From a3c8e86c0ba77529cf2f03a642fc45e57adbe599 Mon Sep 17 00:00:00 2001 From: Hans Johnson Date: Sat, 12 Jan 2019 12:32:15 -0600 Subject: [PATCH] ENH: Refactor and enhance the CI testing infrastructure 1) Improve travis build script for use outside travis. Allow the script used for CI builds to also be used locally in a similar manner to the CI use of the scrips 2) Add ctest compatible testing and CDASH support Report testing and building results to https://my.cdash.org/index.php?project=jsoncpp NOTE: The new ctest infrastructure is not yet robust on winodws Do no yet enable the new features for running test with ctest on windows platform. The previous behaviors are maintainted, but enhance test reporting from windows is not yet supported. 3) Add a cmake coverage testing option Ensure that cmake builds on linux are tested. Ensure that code coverage is reported. 4) Move conditional environment checking into the matrix Avoid multiple places where conditional logic is used to change compiler behavior. As more test environments are created fromt the travis.yml matrix, all settings should be obvious from that one location. 5) Tests with known regressions from the jsonchecker are suppressed Tests that are known to pass with jsoncpp more lenient syntax enforcement are exluded from tests in test/runjsontests.py --- .travis.yml | 51 ++++++++-- .travis_scripts/cmake_builder.sh | 130 ++++++++++++++++++++++++ .travis_scripts/meson_builder.sh | 81 +++++++++++++++ .travis_scripts/travis.install.linux.sh | 4 - .travis_scripts/travis.sh | 34 ------- CMakeLists.txt | 8 +- CTestConfig.cmake | 15 +++ README.md | 6 +- appveyor.yml | 6 +- src/jsontestrunner/CMakeLists.txt | 13 +++ src/test_lib_json/CMakeLists.txt | 8 +- test/runjsontests.py | 40 +++++++- 12 files changed, 341 insertions(+), 55 deletions(-) create mode 100755 .travis_scripts/cmake_builder.sh create mode 100755 .travis_scripts/meson_builder.sh delete mode 100755 .travis_scripts/travis.sh create mode 100644 CTestConfig.cmake diff --git a/.travis.yml b/.travis.yml index 393bd5c..fe71962 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,19 +26,52 @@ matrix: allow_failures: - os: osx include: - - os: osx + - name: Mac clang meson static release testing + os: osx osx_image: xcode9.4 compiler: clang - env: LIB_TYPE=static BUILD_TYPE=release - - os: linux + env: + CXX="clang++-3.5" + CC="clang-3.5" + LIB_TYPE=static + BUILD_TYPE=release + script: ./.travis_scripts/meson_builder.sh + - name: trusty clang meson static release testing + os: linux dist: trusty compiler: clang - env: LIB_TYPE=static BUILD_TYPE=release + env: + CXX="clang++-3.5" + CC="clang-3.5" + LIB_TYPE=static + BUILD_TYPE=release + # before_install and install steps only needed for linux meson builds + before_install: + - source ./.travis_scripts/travis.before_install.${TRAVIS_OS_NAME}.sh + install: + - source ./.travis_scripts/travis.install.${TRAVIS_OS_NAME}.sh + script: ./.travis_scripts/meson_builder.sh + - name: xenial gcc cmake coverage + os: linux + dist: xenial + compiler: gcc + env: + CXX=g++ + CC=gcc + DO_Coverage=ON + BUILD_TOOL="Unix Makefiles" + BUILD_TYPE=Debug + LIB_TYPE=shared + DESTDIR=/tmp/cmake_json_cpp + script: ./.travis_scripts/cmake_builder.sh +# Valgrind has too many false positives from the python wrapping. Need a good suppression file +# - name: xenial gcc cmake coverage +# os: linux +# dist: xenial +# compiler: gcc +# env: DO_MemCheck=ON CXX=/usr/bin/g++ BUILD_TOOL="Unix Makefiles" BUILD_TYPE=Debug LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp +# script: ./.travis_scripts/cmake_builder.sh notifications: email: false -before_install: -- source ./.travis_scripts/travis.before_install.${TRAVIS_OS_NAME}.sh -install: -- source ./.travis_scripts/travis.install.${TRAVIS_OS_NAME}.sh -script: ./.travis_scripts/travis.sh + diff --git a/.travis_scripts/cmake_builder.sh b/.travis_scripts/cmake_builder.sh new file mode 100755 index 0000000..c011878 --- /dev/null +++ b/.travis_scripts/cmake_builder.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env sh +# This script can be used on the command line directly to configure several +# different build environments. +# This is called by `.travis.yml` via Travis CI. +# Travis supplies $TRAVIS_OS_NAME. +# http://docs.travis-ci.com/user/multi-os/ +# Our .travis.yml also defines: + +# - BUILD_TYPE=Release/Debug +# - LIB_TYPE=static/shared +# +# Optional environmental variables +# - DESTDIR <- used for setting the install prefix +# - BUILD_TOOL=["Unix Makefile"|"Ninja"] +# - BUILDNAME <-- how to identify this build on the dashboard +# - DO_MemCheck <- if set, try to use valgrind +# - DO_Coverage <- if set, try to do dashboard coverage testing +# + +env_set=1 +if ${BUILD_TYPE+false}; then + echo "BUILD_TYPE not set in environment." + env_set=0 +fi +if ${LIB_TYPE+false}; then + echo "LIB_TYPE not set in environment." + env_set=0 +fi +if ${CXX+false}; then + echo "CXX not set in environment." + env_set=0 +fi + + +if [ ${env_set} -eq 0 ]; then + echo "USAGE: CXX=$(which clang++) BUILD_TYPE=[Release|Debug] LIB_TYPE=[static|shared] $0" + echo "" + echo "Examples:" + echo " CXX=$(which clang++) BUILD_TYPE=Release LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp $0" + echo " CXX=$(which clang++) BUILD_TYPE=Debug LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp $0" + echo " CXX=$(which clang++) BUILD_TYPE=Release LIB_TYPE=static DESTDIR=/tmp/cmake_json_cpp $0" + echo " CXX=$(which clang++) BUILD_TYPE=Debug LIB_TYPE=static DESTDIR=/tmp/cmake_json_cpp $0" + + echo " CXX=$(which g++) BUILD_TYPE=Release LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp $0" + echo " CXX=$(which g++) BUILD_TYPE=Debug LIB_TYPE=shared DESTDIR=/tmp/cmake_json_cpp $0" + echo " CXX=$(which g++) BUILD_TYPE=Release LIB_TYPE=static DESTDIR=/tmp/cmake_json_cpp $0" + echo " CXX=$(which g++) BUILD_TYPE=Debug LIB_TYPE=static DESTDIR=/tmp/cmake_json_cpp $0" + + exit -1 +fi + +if ${DESTDIR+false}; then + DESTDIR="/usr/local" +fi + +# -e: fail on error +# -v: show commands +# -x: show expanded commands +set -vex + +env | sort + +which cmake +cmake --version + +echo ${CXX} +${CXX} --version +_COMPILER_NAME=`basename ${CXX}` +if [ "${BUILD_TYPE}" == "shared" ]; then + _CMAKE_BUILD_SHARED_LIBS=ON +else + _CMAKE_BUILD_SHARED_LIBS=OFF +fi + +CTEST_TESTING_OPTION="-D ExperimentalTest" +# - DO_MemCheck <- if set, try to use valgrind +if ! ${DO_MemCheck+false}; then + valgrind --version + CTEST_TESTING_OPTION="-D ExperimentalMemCheck" +else +# - DO_Coverage <- if set, try to do dashboard coverage testing + if ! ${DO_Coverage+false}; then + export CXXFLAGS="-fprofile-arcs -ftest-coverage" + export LDFLAGS="-fprofile-arcs -ftest-coverage" + CTEST_TESTING_OPTION="-D ExperimentalTest -D ExperimentalCoverage" + #gcov --version + fi +fi + +# Ninja = Generates build.ninja files. +if ${BUILD_TOOL+false}; then + BUILD_TOOL="Ninja" + export _BUILD_EXE=ninja + which ninja + ninja --version +else +# Unix Makefiles = Generates standard UNIX makefiles. + export _BUILD_EXE=make +fi + +_BUILD_DIR_NAME="build-cmake_${BUILD_TYPE}_${LIB_TYPE}_${_COMPILER_NAME}_${_BUILD_EXE}" +mkdir -p ${_BUILD_DIR_NAME} +cd "${_BUILD_DIR_NAME}" + if ${BUILDNAME+false}; then + _HOSTNAME=`hostname -s` + BUILDNAME="${_HOSTNAME}_${BUILD_TYPE}_${LIB_TYPE}_${_COMPILER_NAME}_${_BUILD_EXE}" + fi + cmake \ + -G "${BUILD_TOOL}" \ + -DBUILDNAME:STRING="${BUILDNAME}" \ + -DCMAKE_CXX_COMPILER:PATH=${CXX} \ + -DCMAKE_BUILD_TYPE:STRING=${BUILD_TYPE} \ + -DBUILD_SHARED_LIBS:BOOL=${_CMAKE_BUILD_SHARED_LIBS} \ + -DCMAKE_INSTALL_PREFIX:PATH=${DESTDIR} \ + ../ + + ctest -C ${BUILD_TYPE} -D ExperimentalStart -D ExperimentalConfigure -D ExperimentalBuild ${CTEST_TESTING_OPTION} -D ExperimentalSubmit + # Final step is to verify that installation succeeds + cmake --build . --config ${BUILD_TYPE} --target install + + if [ "${DESTDIR}" != "/usr/local" ]; then + ${_BUILD_EXE} install + fi +cd - + +if ${CLEANUP+false}; then + echo "Skipping cleanup: build directory will persist." +else + rm -r "${_BUILD_DIR_NAME}" +fi diff --git a/.travis_scripts/meson_builder.sh b/.travis_scripts/meson_builder.sh new file mode 100755 index 0000000..dbf03cb --- /dev/null +++ b/.travis_scripts/meson_builder.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env sh +# This script can be used on the command line directly to configure several +# different build environments. +# This is called by `.travis.yml` via Travis CI. +# Travis supplies $TRAVIS_OS_NAME. +# http://docs.travis-ci.com/user/multi-os/ +# Our .travis.yml also defines: + +# - BUILD_TYPE=release/debug +# - LIB_TYPE=static/shared + +env_set=1 +if ${BUILD_TYPE+false}; then + echo "BUILD_TYPE not set in environment." + env_set=0 +fi +if ${LIB_TYPE+false}; then + echo "LIB_TYPE not set in environment." + env_set=0 +fi +if ${CXX+false}; then + echo "CXX not set in environment." + env_set=0 +fi + + +if [ ${env_set} -eq 0 ]; then + echo "USAGE: CXX=$(which clang++) BUILD_TYPE=[release|debug] LIB_TYPE=[static|shared] $0" + echo "" + echo "Examples:" + echo " CXX=$(which clang++) BUILD_TYPE=release LIB_TYPE=shared DESTDIR=/tmp/meson_json_cpp $0" + echo " CXX=$(which clang++) BUILD_TYPE=debug LIB_TYPE=shared DESTDIR=/tmp/meson_json_cpp $0" + echo " CXX=$(which clang++) BUILD_TYPE=release LIB_TYPE=static DESTDIR=/tmp/meson_json_cpp $0" + echo " CXX=$(which clang++) BUILD_TYPE=debug LIB_TYPE=static DESTDIR=/tmp/meson_json_cpp $0" + + echo " CXX=$(which g++) BUILD_TYPE=release LIB_TYPE=shared DESTDIR=/tmp/meson_json_cpp $0" + echo " CXX=$(which g++) BUILD_TYPE=debug LIB_TYPE=shared DESTDIR=/tmp/meson_json_cpp $0" + echo " CXX=$(which g++) BUILD_TYPE=release LIB_TYPE=static DESTDIR=/tmp/meson_json_cpp $0" + echo " CXX=$(which g++) BUILD_TYPE=debug LIB_TYPE=static DESTDIR=/tmp/meson_json_cpp $0" + + exit -1 +fi + +if ${DESTDIR+false}; then + DESTDIR="/usr/local" +fi + +# -e: fail on error +# -v: show commands +# -x: show expanded commands +set -vex + + +env | sort + +which python3 +which meson +which ninja +echo ${CXX} +${CXX} --version +python3 --version +meson --version +ninja --version +_COMPILER_NAME=`basename ${CXX}` +_BUILD_DIR_NAME="build-${BUILD_TYPE}_${LIB_TYPE}_${_COMPILER_NAME}" +meson --buildtype ${BUILD_TYPE} --default-library ${LIB_TYPE} . "${_BUILD_DIR_NAME}" +ninja -v -j 2 -C "${_BUILD_DIR_NAME}" +#ninja -v -j 2 -C "${_BUILD_DIR_NAME}" test +cd "${_BUILD_DIR_NAME}" + meson test --no-rebuild --print-errorlogs + + if [ "${DESTDIR}" != "/usr/local" ]; then + ninja install + fi +cd - + +if ${CLEANUP+false}; then + echo "Skipping cleanup: build directory will persist." +else + rm -r "${_BUILD_DIR_NAME}" +fi diff --git a/.travis_scripts/travis.install.linux.sh b/.travis_scripts/travis.install.linux.sh index a049f76..84c3a61 100644 --- a/.travis_scripts/travis.install.linux.sh +++ b/.travis_scripts/travis.install.linux.sh @@ -4,10 +4,6 @@ wget https://github.com/ninja-build/ninja/releases/download/v1.7.2/ninja-linux.z unzip -q ninja-linux.zip -d build pip3 install meson -# /usr/bin/gcc is 4.6 always, but gcc-X.Y is available. -if [[ $CXX = g++ ]]; then export CXX="g++-4.9" CC="gcc-4.9"; fi -# /usr/bin/clang has a conflict with gcc, so use clang-X.Y. -if [[ $CXX = clang++ ]]; then export CXX="clang++-3.5" CC="clang-3.5"; fi echo ${PATH} ls /usr/local ls /usr/local/bin diff --git a/.travis_scripts/travis.sh b/.travis_scripts/travis.sh deleted file mode 100755 index 7691100..0000000 --- a/.travis_scripts/travis.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env sh -# This is called by `.travis.yml` via Travis CI. -# Travis supplies $TRAVIS_OS_NAME. -# http://docs.travis-ci.com/user/multi-os/ -# Our .travis.yml also defines: -# - SHARED_LIB=ON/OFF -# - STATIC_LIB=ON/OFF -# - CMAKE_PKG=ON/OFF -# - BUILD_TYPE=release/debug -# - VERBOSE_MAKE=false/true -# - VERBOSE (set or not) - -# -e: fail on error -# -v: show commands -# -x: show expanded commands -set -vex - -env | sort - -which python3 -which meson -which ninja -echo ${CXX} -${CXX} --version -python3 --version -meson --version -ninja --version -meson --buildtype ${BUILD_TYPE} --default-library ${LIB_TYPE} . build-${LIB_TYPE} -ninja -v -C build-${LIB_TYPE} -#ninja -v -C build-${LIB_TYPE} test -cd build-${LIB_TYPE} -meson test --no-rebuild --print-errorlogs -cd - -rm -r build-${LIB_TYPE} diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a6e8dc..145a77c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,8 +70,6 @@ set( JSONCPP_VERSION ${JSONCPP_VERSION_MAJOR}.${JSONCPP_VERSION_MINOR}.${JSONCPP message(STATUS "JsonCpp Version: ${JSONCPP_VERSION_MAJOR}.${JSONCPP_VERSION_MINOR}.${JSONCPP_VERSION_PATCH}") set( JSONCPP_SOVERSION 19 ) -enable_testing() - option(JSONCPP_WITH_TESTS "Compile and (for jsoncpp_check) run JsonCpp test executables" ON) option(JSONCPP_WITH_POST_BUILD_UNITTEST "Automatically run unit-tests as a post build step" ON) option(JSONCPP_WITH_WARNING_AS_ERROR "Force compilation to fail if a warning occurs" OFF) @@ -175,8 +173,14 @@ if(JSONCPP_WITH_CMAKE_PACKAGE) FILE jsoncppConfig.cmake) endif() +if(JSONCPP_WITH_TESTS) + enable_testing() + include(CTest) +endif() + # Build the different applications add_subdirectory( src ) #install the includes add_subdirectory( include ) + diff --git a/CTestConfig.cmake b/CTestConfig.cmake new file mode 100644 index 0000000..b8fc6d5 --- /dev/null +++ b/CTestConfig.cmake @@ -0,0 +1,15 @@ +## This file should be placed in the root directory of your project. +## Then modify the CMakeLists.txt file in the root directory of your +## project to incorporate the testing dashboard. +## +## # The following are required to submit to the CDash dashboard: +## ENABLE_TESTING() +## INCLUDE(CTest) + +set(CTEST_PROJECT_NAME "jsoncpp") +set(CTEST_NIGHTLY_START_TIME "01:23:45 UTC") + +set(CTEST_DROP_METHOD "https") +set(CTEST_DROP_SITE "my.cdash.org") +set(CTEST_DROP_LOCATION "/submit.php?project=jsoncpp") +set(CTEST_DROP_SITE_CDASH TRUE) diff --git a/README.md b/README.md index 3edee80..8f82594 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,11 @@ format to store user input files. ## Contributing to JsonCpp ### Building and testing with Meson/Ninja -Thanks to David Seifert (@SoapGentoo), we (the maintainers) now use [meson](http://mesonbuild.com/) and [ninja](https://ninja-build.org/) to build for debugging, as well as for continuous integration (see [`travis.sh`](travis.sh) ). Other systems may work, but minor things like version strings might break. +Thanks to David Seifert (@SoapGentoo), we (the maintainers) now use +[meson](http://mesonbuild.com/) and [ninja](https://ninja-build.org/) to build +for debugging, as well as for continuous integration (see +[`./travis_scripts/meson_builder.sh`](./travis_scripts/meson_builder.sh) ). Other systems may work, but minor +things like version strings might break. First, install both meson (which requires Python3) and ninja. If you wish to install to a directory other than /usr/local, set an environment variable called DESTDIR with the desired path: diff --git a/appveyor.yml b/appveyor.yml index a723fca..447a212 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,7 +10,11 @@ environment: build_script: - cmake --version - cd c:\projects\jsoncpp - - cmake -G "%CMAKE_GENERATOR%" -DCMAKE_INSTALL_PREFIX=%CD:\=/%/install -DBUILD_SHARED_LIBS=ON . + - cmake -G "%CMAKE_GENERATOR%" -DCMAKE_INSTALL_PREFIX:PATH=%CD:\=/%/install -DBUILD_SHARED_LIBS:BOOL=ON . + # Use ctest to make a dashboard build ctest -D Experimental(Start|Update|Configure|Build|Test|Coverage|MemCheck|Submit) + #NOTE Testing on window is not yet finished - ctest -C Release -D ExperimentalStart -D ExperimentalConfigure -D ExperimentalBuild -D ExperimentalTest -D ExperimentalSubmit + - ctest -C Release -D ExperimentalStart -D ExperimentalConfigure -D ExperimentalBuild -D ExperimentalSubmit + # Final step is to verify that installation succeeds - cmake --build . --config Release --target install deploy: diff --git a/src/jsontestrunner/CMakeLists.txt b/src/jsontestrunner/CMakeLists.txt index 9862399..4ca0909 100644 --- a/src/jsontestrunner/CMakeLists.txt +++ b/src/jsontestrunner/CMakeLists.txt @@ -15,9 +15,22 @@ if(PYTHONINTERP_FOUND) # Run end to end parser/writer tests set(TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../test) set(RUNJSONTESTS_PATH ${TEST_DIR}/runjsontests.py) + + # Run unit tests in post-build + # (default cmake workflow hides away the test result into a file, resulting in poor dev workflow?!?) add_custom_target(jsoncpp_readerwriter_tests "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $ "${TEST_DIR}/data" DEPENDS jsontestrunner_exe jsoncpp_test ) add_custom_target(jsoncpp_check DEPENDS jsoncpp_readerwriter_tests) + + ## Create tests for dashboard submission, allows easy review of CI results https://my.cdash.org/index.php?project=jsoncpp + add_test(NAME jsoncpp_readerwriter + COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" $ "${TEST_DIR}/data" + WORKING_DIRECTORY "${TEST_DIR}/data" + ) + add_test(NAME jsoncpp_readerwriter_json_checker + COMMAND "${PYTHON_EXECUTABLE}" -B "${RUNJSONTESTS_PATH}" --with-json-checker $ "${TEST_DIR}/data" + WORKING_DIRECTORY "${TEST_DIR}/data" + ) endif() diff --git a/src/test_lib_json/CMakeLists.txt b/src/test_lib_json/CMakeLists.txt index 7c7b6af..8a3970f 100644 --- a/src/test_lib_json/CMakeLists.txt +++ b/src/test_lib_json/CMakeLists.txt @@ -1,6 +1,6 @@ # vim: et ts=4 sts=4 sw=4 tw=0 -add_executable( jsoncpp_test +add_executable( jsoncpp_test jsontest.cpp jsontest.h main.cpp @@ -31,6 +31,10 @@ if(JSONCPP_WITH_POST_BUILD_UNITTEST) POST_BUILD COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) endif() + ## Create tests for dashboard submission, allows easy review of CI results https://my.cdash.org/index.php?project=jsoncpp + add_test(NAME jsoncpp_test + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ + ) endif() -set_target_properties(jsoncpp_test PROPERTIES OUTPUT_NAME jsoncpp_test) +set_target_properties(jsoncpp_test PROPERTIES OUTPUT_NAME jsoncpp_test) diff --git a/test/runjsontests.py b/test/runjsontests.py index 48cfe82..dfdeca3 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -55,7 +55,7 @@ def compareOutputs(expected, actual, message): """ % (message, diff_line, safeGetLine(expected,diff_line), safeGetLine(actual,diff_line)) - + def safeReadFile(path): try: return open(path, 'rt', encoding = 'utf-8').read() @@ -69,7 +69,43 @@ def runAllTests(jsontest_executable_path, input_dir = None, input_dir = os.path.join(os.getcwd(), 'data') tests = glob(os.path.join(input_dir, '*.json')) if with_json_checker: - test_jsonchecker = glob(os.path.join(input_dir, '../jsonchecker', '*.json')) + all_test_jsonchecker = glob(os.path.join(input_dir, '../jsonchecker', '*.json')) + # These tests fail with strict json support, but pass with jsoncpp extra lieniency + """ + Failure details: + * Test ../jsonchecker/fail25.json + Parsing should have failed: + [" tab character in string "] + + * Test ../jsonchecker/fail13.json + Parsing should have failed: + {"Numbers cannot have leading zeroes": 013} + + * Test ../jsonchecker/fail18.json + Parsing should have failed: + [[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]] + + * Test ../jsonchecker/fail8.json + Parsing should have failed: + ["Extra close"]] + + * Test ../jsonchecker/fail7.json + Parsing should have failed: + ["Comma after the close"], + + * Test ../jsonchecker/fail10.json + Parsing should have failed: + {"Extra value after close": true} "misplaced quoted value" + + * Test ../jsonchecker/fail27.json + Parsing should have failed: + ["line + break"] + """ + known_differences_withjsonchecker = [ "fail25.json", "fail13.json", "fail18.json", "fail8.json", + "fail7.json", "fail10.json", "fail27.json" ] + test_jsonchecker = [ test for test in all_test_jsonchecker if os.path.basename(test) not in known_differences_withjsonchecker ] + else: test_jsonchecker = [] failed_tests = []