diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e3c95be4..32cef39da 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,6 @@ if(DEFINED CMAKE_BUILD_TYPE) endif() project(OpenCV CXX C) - include(cmake/OpenCVUtils.cmake) # ---------------------------------------------------------------------------- @@ -361,7 +360,6 @@ include(cmake/OpenCVFindLibsPerf.cmake) # ---------------------------------------------------------------------------- # Detect other 3rd-party libraries/tools # ---------------------------------------------------------------------------- - # --- LATEX for pdf documentation --- if(BUILD_DOCS) include(cmake/OpenCVFindLATEX.cmake) @@ -393,6 +391,9 @@ if(WITH_OPENCL) include(cmake/OpenCVDetectOpenCL.cmake) endif() +# --- Matlab/Octave --- +include(cmake/OpenCVFindMatlab.cmake) + # ---------------------------------------------------------------------------- # Solution folders: # ---------------------------------------------------------------------------- @@ -829,6 +830,15 @@ if(NOT ANDROID) endif() status(" Java tests:" BUILD_TESTS AND (CAN_BUILD_ANDROID_PROJECTS OR HAVE_opencv_java) THEN YES ELSE NO) +# ========================= matlab ========================= +status("") +status(" Matlab:") +status(" mex:" MATLAB_MEX_SCRIPT THEN "${MATLAB_MEX_SCRIPT}" ELSE NO) +if (MATLAB_FOUND) + get_property(MEX_WORKS GLOBAL PROPERTY MEX_WORKS) + status(" Compiler/generator:" MEX_WORKS THEN "Working" ELSE "Not working (bindings will not be generated)") +endif() + # ========================== documentation ========================== if(BUILD_DOCS) status("") diff --git a/cmake/OpenCVFindMatlab.cmake b/cmake/OpenCVFindMatlab.cmake new file mode 100644 index 000000000..80e9a9a4a --- /dev/null +++ b/cmake/OpenCVFindMatlab.cmake @@ -0,0 +1,212 @@ +# ----- Find Matlab/Octave ----- +# +# OpenCVFindMatlab.cmake attempts to locate the install path of Matlab in order +# to extract the mex headers, libraries and shell scripts. If found +# successfully, the following variables will be defined +# +# MATLAB_FOUND: true/false +# MATLAB_ROOT_DIR: Root of Matlab installation +# MATLAB_BIN: The main Matlab "executable" (shell script) +# MATLAB_MEX_SCRIPT: The mex script used to compile mex files +# MATLAB_BIN: The actual Matlab executable +# MATLAB_INCLUDE_DIR: Path to "mex.h" +# MATLAB_LIBRARY_DIR: Path to mex and matrix libraries +# MATLAB_LIBS: The Matlab libs, usually mx, mex, mat +# MATLAB_MEXEXT: The mex library extension. It will be one of: +# mexwin32, mexwin64, mexglx, mexa64, mexmac, +# mexmaci, mexmaci64, mexsol, mexs64 +# MATLAB_ARCH: The installation architecture. It is simply +# the MEXEXT with the preceding "mex" removed +# +# There doesn't appear to be an elegant way to detect all versions of Matlab +# across different platforms. If you know the matlab path and want to avoid +# the search, you can define the path to the Matlab root when invoking cmake: +# +# cmake -DMATLAB_ROOT_DIR='/PATH/TO/ROOT_DIR' .. + + + +# ----- set_library_presuffix ----- +# +# Matlab tends to use some non-standard prefixes and suffixes on its libraries. +# For example, libmx.dll on Windows (Windows does not add prefixes) and +# mkl.dylib on OS X (OS X uses "lib" prefixes). +# On some versions of Windows the .dll suffix also appears to not be checked. +# +# This function modifies the library prefixes and suffixes used by +# find_library when finding Matlab libraries. It does not affect scopes +# outside of this file. +function(set_libarch_prefix_suffix) + if (UNIX AND NOT APPLE) + set(CMAKE_FIND_LIBRARY_PREFIXES "lib" PARENT_SCOPE) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".so" ".a" PARENT_SCOPE) + elseif (APPLE) + set(CMAKE_FIND_LIBRARY_PREFIXES "lib" PARENT_SCOPE) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".dylib" ".a" PARENT_SCOPE) + elseif (WIN32) + set(CMAKE_FIND_LIBRARY_PREFIXES "lib" PARENT_SCOPE) + set(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll" PARENT_SCOPE) + endif() +endfunction() + + + +# ----- locate_matlab_root ----- +# +# Attempt to find the path to the Matlab installation. If successful, sets +# the absolute path in the variable MATLAB_ROOT_DIR +function(locate_matlab_root) + + # --- LINUX --- + if (UNIX AND NOT APPLE) + # possible root locations, in order of likelihood + set(SEARCH_DIRS_ /usr/local /opt/local /usr /opt) + foreach (DIR_ ${SEARCH_DIRS_}) + file(GLOB MATLAB_ROOT_DIR_ ${DIR_}/*matlab*) + if (MATLAB_ROOT_DIR_) + # sort in order from highest to lowest + list(SORT MATLAB_ROOT_DIR_) + list(REVERSE MATLAB_ROOT_DIR_) + list(GET MATLAB_ROOT_DIR_ 0 MATLAB_ROOT_DIR_) + break() + endif() + endforeach() + + # --- APPLE --- + elseif (APPLE) + # possible root locations, in order of likelihood + set(SEARCH_DIRS_ /Applications /usr/local /opt/local /usr /opt) + foreach (DIR_ ${SEARCH_DIRS_}) + file(GLOB MATLAB_ROOT_DIR_ ${DIR_}/*matlab*) + if (MATLAB_ROOT_DIR_) + # sort in order from highest to lowest + # normally it's in the format MATLAB_R[20XX][A/B] + list(SORT MATLAB_ROOT_DIR_) + list(REVERSE MATLAB_ROOT_DIR_) + list(GET MATLAB_ROOT_DIR_ 0 MATLAB_ROOT_DIR_) + break() + endif() + endforeach() + + # --- WINDOWS --- + elseif (WIN32) + # search the path to see if Matlab exists there + # fingers crossed it is, otherwise we have to start hunting through the registry :/ + string(REGEX REPLACE ".*[;=](.*[Mm][Aa][Tt][Ll][Aa][Bb][^;]*)\\\\bin.*" "\\1" MATLAB_ROOT_DIR_ "$ENV{PATH}") + + # registry-hacking + # determine the available Matlab versions + set(REG_EXTENSION_ "SOFTWARE\\Mathworks\\MATLAB") + set(REG_ROOTS_ "HKEY_LOCAL_MACHINE" "HKEY_CURRENT_USER") + foreach(REG_ROOT_ ${REG_ROOTS_}) + execute_process(COMMAND reg query "${REG_ROOT_}\\${REG_EXTENSION_}" OUTPUT_VARIABLE QUERY_RESPONSE_) + if (QUERY_RESPONSE_) + string(REGEX MATCHALL "[0-9]\\.[0-9]" VERSION_STRINGS_ ${QUERY_RESPONSE_}) + list(APPEND VERSIONS_ ${VERSION_STRINGS_}) + endif() + endforeach() + + # select the highest version + list(APPEND VERSIONS_ "0.0") + list(SORT VERSIONS_) + list(REVERSE VERSIONS_) + list(GET VERSIONS_ 0 VERSION_) + + # request the MATLABROOT from the registry + foreach(REG_ROOT_ ${REG_ROOTS_}) + get_filename_component(QUERY_RESPONSE_ [${REG_ROOT_}\\${REG_EXTENSION_}\\${VERSION_};MATLABROOT] ABSOLUTE) + if (NOT ${MATLAB_ROOT_DIR_} AND NOT ${QUERY_RESPONSE_} MATCHES "registry$") + set(MATLAB_ROOT_DIR_ ${QUERY_RESPONSE_}) + endif() + endforeach() + endif() + + # export the root into the parent scope + if (MATLAB_ROOT_DIR_) + set(MATLAB_ROOT_DIR ${MATLAB_ROOT_DIR_} PARENT_SCOPE) + endif() +endfunction() + + + +# ----- locate_matlab_components ----- +# +# Given a directory MATLAB_ROOT_DIR, attempt to find the Matlab components +# (include directory and libraries) under the root. If everything is found, +# sets the variable MATLAB_FOUND to TRUE +function(locate_matlab_components MATLAB_ROOT_DIR) + # get the mex extension + if (UNIX) + execute_process(COMMAND ${MATLAB_ROOT_DIR}/bin/mexext OUTPUT_VARIABLE MATLAB_MEXEXT_) + elseif (WIN32) + execute_process(COMMAND ${MATLAB_ROOT_DIR}/bin/mexext.bat OUTPUT_VARIABLE MATLAB_MEXEXT_) + endif() + if (NOT MATLAB_MEXEXT_) + return() + endif() + string(STRIP ${MATLAB_MEXEXT_} MATLAB_MEXEXT_) + + # map the mexext to an architecture extension + set(ARCHITECTURES_ "maci64" "maci" "glnxa64" "glnx64" "sol64" "sola64" "win32" "win64" ) + foreach(ARCHITECTURE_ ${ARCHITECTURES_}) + if(EXISTS ${MATLAB_ROOT_DIR}/bin/${ARCHITECTURE_}) + set(MATLAB_ARCH_ ${ARCHITECTURE_}) + break() + endif() + endforeach() + + # get the path to the libraries + set(MATLAB_LIBRARY_DIR_ ${MATLAB_ROOT_DIR}/bin/${MATLAB_ARCH_}) + + # get the libraries + set_libarch_prefix_suffix() + find_library(MATLAB_LIB_MX_ mx PATHS ${MATLAB_LIBRARY_DIR_} NO_DEFAULT_PATH) + find_library(MATLAB_LIB_MEX_ mex PATHS ${MATLAB_LIBRARY_DIR_} NO_DEFAULT_PATH) + find_library(MATLAB_LIB_MAT_ mat PATHS ${MATLAB_LIBRARY_DIR_} NO_DEFAULT_PATH) + set(MATLAB_LIBS_ ${MATLAB_LIB_MX_} ${MATLAB_LIB_MEX_} ${MATLAB_LIB_MAT_}) + + # get the include path + find_path(MATLAB_INCLUDE_DIR_ mex.h ${MATLAB_ROOT_DIR}/extern/include) + + # get the mex shell script + find_file(MATLAB_MEX_SCRIPT_ NAMES mex mex.bat PATHS ${MATLAB_ROOT_DIR}/bin NO_DEFAULT_PATH) + + # get the Matlab executable + find_file(MATLAB_BIN_ NAMES matlab matlab.exe PATHS ${MATLAB_ROOT_DIR}/bin NO_DEFAULT_PATH) + + # export into parent scope + if (MATLAB_MEX_SCRIPT_ AND MATLAB_LIBS_ AND MATLAB_INCLUDE_DIR_) + set(MATLAB_BIN ${MATLAB_BIN_} PARENT_SCOPE) + set(MATLAB_MEX_SCRIPT ${MATLAB_MEX_SCRIPT_} PARENT_SCOPE) + set(MATLAB_INCLUDE_DIR ${MATLAB_INCLUDE_DIR_} PARENT_SCOPE) + set(MATLAB_LIBS ${MATLAB_LIBS_} PARENT_SCOPE) + set(MATLAB_LIBRARY_DIR ${MATLAB_LIBRARY_DIR_} PARENT_SCOPE) + set(MATLAB_MEXEXT ${MATLAB_MEXEXT_} PARENT_SCOPE) + set(MATLAB_ARCH ${MATLAB_ARCH_} PARENT_SCOPE) + endif() + return() +endfunction() + + + +# ---------------------------------------------------------------------------- +# FIND MATLAB COMPONENTS +# ---------------------------------------------------------------------------- +if (NOT MATLAB_FOUND) + + # guilty until proven innocent + set(MATLAB_FOUND FALSE) + + # attempt to find the Matlab root folder + if (NOT MATLAB_ROOT_DIR) + locate_matlab_root() + endif() + + # given the matlab root folder, find the library locations + if (MATLAB_ROOT_DIR) + locate_matlab_components(${MATLAB_ROOT_DIR}) + endif() + find_package_handle_standard_args(Matlab DEFAULT_MSG MATLAB_MEX_SCRIPT MATLAB_INCLUDE_DIR + MATLAB_ROOT_DIR MATLAB_LIBS MATLAB_LIBRARY_DIR + MATLAB_MEXEXT MATLAB_ARCH MATLAB_BIN) +endif() diff --git a/modules/matlab/CMakeLists.txt b/modules/matlab/CMakeLists.txt new file mode 100644 index 000000000..b1119f9d2 --- /dev/null +++ b/modules/matlab/CMakeLists.txt @@ -0,0 +1,302 @@ +# ---------------------------------------------------------------------------- +# CMake file for Matlab/Octave support +# +# Matlab code generation and compilation is broken down into two distinct +# stages: configure time and build time. The idea is that we want to give +# the user reasonable guarantees that once they type 'make', wrapper +# generation is unlikely to fail. Therefore we run a series of tests at +# configure time to check the working status of the core components. +# +# Configure Time +# During configure time, the script attempts to ascertain whether the +# generator and mex compiler are working for a given architecture. +# Currently this involves: +# 1) Generating a simple CV_EXPORTS_W symbol and checking whether a file +# of the symbol name is generated +# 2) Compiling a simple mex gateway to check that Bridge.hpp and mex.h +# can be found, and that a file with the mexext is produced +# +# Build Time +# If the configure time tests pass, then we assume Matlab wrapper generation +# will not fail during build time. We simply glob all of the symbols in +# the OpenCV module headers, generate intermediate .cpp files, then compile +# them with mex. +# ---------------------------------------------------------------------------- + +# PREPEND +# Given a list of strings IN and a TOKEN, prepend the token to each string +# and append to OUT. This is used for passing command line "-I", "-L" and "-l" +# arguments to mex. e.g. +# prepend("-I" OUT /path/to/include/dir) --> -I/path/to/include/dir +macro(PREPEND TOKEN OUT IN) + foreach(VAR ${IN}) + list(APPEND ${OUT} "${TOKEN}${VAR}") + endforeach() +endmacro() + + +# WARN_MIXED_PRECISION +# Formats a warning message if the compiler and Matlab bitness is different +macro(WARN_MIXED_PRECISION COMPILER_BITNESS MATLAB_BITNESS) + set(MSG "Your compiler is ${COMPILER_BITNESS}-bit") + set(MSG "${MSG} but your version of Matlab is ${MATLAB_BITNESS}-bit.") + set(MSG "${MSG} To build Matlab bindings, please switch to a ${MATLAB_BITNESS}-bit compiler.") + message(WARNING ${MSG}) +endmacro() + +# ---------------------------------------------------------------------------- +# Architecture checks +# ---------------------------------------------------------------------------- +# make sure we're on a supported architecture with Matlab and python installed +if (IOS OR ANDROID OR NOT MATLAB_FOUND) + ocv_module_disable(matlab) + return() +elseif (NOT PYTHONLIBS_FOUND) + message(WARNING "A required dependency of the matlab module (PythonLibs) was not found. Disabling Matlab bindings...") + ocv_module_disable(matlab) + return() +endif() + + +# If the user built OpenCV as X-bit, but they have a Y-bit version of Matlab, +# attempting to link to OpenCV during binding generation will fail, since +# mixed precision pointers are not allowed. Disable the bindings. +math(EXPR ARCH "${CMAKE_SIZEOF_VOID_P} * 8") +if (${ARCH} EQUAL 32 AND ${MATLAB_ARCH} MATCHES "64") + warn_mixed_precision("32" "64") + ocv_module_disable(matlab) + return() +elseif (${ARCH} EQUAL 64 AND NOT ${MATLAB_ARCH} MATCHES "64") + warn_mixed_precision("64" "32") + ocv_module_disable(matlab) + return() +endif() + +# If it's MSVC, warn the user that bindings will only be built in Release mode. +# Debug mode seems to cause issues... +if (MSVC) + message(STATUS "Warning: Matlab bindings will only be built in Release configurations") +endif() + + +# ---------------------------------------------------------------------------- +# Configure time components +# ---------------------------------------------------------------------------- +set(the_description "The Matlab/Octave bindings") +ocv_add_module(matlab BINDINGS + OPTIONAL opencv_core + opencv_imgproc opencv_ml opencv_highgui + opencv_objdetect opencv_flann opencv_features2d + opencv_photo opencv_video opencv_videostab + opencv_calib opencv_calib3d + opencv_stitching opencv_superres + opencv_nonfree +) + +# get the commit information +execute_process(COMMAND git log -1 --pretty=%H OUTPUT_VARIABLE GIT_COMMIT ERROR_QUIET) +string(REGEX REPLACE "(\r?\n)+$" "" GIT_COMMIT "${GIT_COMMIT}") + +# set the path to the C++ header and doc parser +set(HDR_PARSER_PATH ${CMAKE_SOURCE_DIR}/modules/python/src2) +set(RST_PARSER_PATH ${CMAKE_SOURCE_DIR}/modules/java/generator) + +# set mex compiler options +prepend("-I" MEX_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/include) +prepend("-L" MEX_LIB_DIR ${LIBRARY_OUTPUT_PATH}/$(Configuration)) +set(MEX_OPTS "-largeArrayDims") + +if (BUILD_TESTS) + add_subdirectory(test) +endif() +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) + + +# intersection of available modules and optional dependencies +# 1. populate the command-line include directories (-I/path/to/module/header, ...) +# 2. populate the command-line link libraries (-lopencv_core, ...) for Debug and Release +set(MATLAB_DEPS ${OPENCV_MODULE_${the_module}_REQ_DEPS} ${OPENCV_MODULE_${the_module}_OPT_DEPS}) +foreach(opencv_module ${MATLAB_DEPS}) + if (HAVE_${opencv_module}) + string(REPLACE "opencv_" "" module ${opencv_module}) + list(APPEND opencv_modules ${module}) + list(APPEND ${the_module}_ACTUAL_DEPS ${opencv_module}) + prepend("-I" MEX_INCLUDE_DIRS "${OPENCV_MODULE_${opencv_module}_LOCATION}/include") + prepend("-l" MEX_LIBS ${opencv_module}${OPENCV_DLLVERSION}) + prepend("-l" MEX_DEBUG_LIBS ${opencv_module}${OPENCV_DLLVERSION}${OPENCV_DEBUG_POSTFIX}) + endif() +endforeach() + +# add extra headers by hand +list(APPEND opencv_extra_hdrs "core=${OPENCV_MODULE_opencv_core_LOCATION}/include/opencv2/core/base.hpp") +list(APPEND opencv_extra_hdrs "video=${OPENCV_MODULE_opencv_video_LOCATION}/include/opencv2/video/tracking.hpp") + +# pass the OPENCV_CXX_EXTRA_FLAGS through to the mex compiler +# remove the visibility modifiers, so the mex gateway is visible +# TODO: get mex working without warnings +string(REGEX REPLACE "[^\ ]*visibility[^\ ]*" "" MEX_CXXFLAGS "${OPENCV_EXTRA_FLAGS} ${OPENCV_EXTRA_CXX_FLAGS}") + +# Configure checks +# Check to see whether the generator and the mex compiler are working. +# The checks currently test: +# - whether the python generator can be found +# - whether the python generator correctly outputs a file for a definition +# - whether the mex compiler can find the required headers +# - whether the mex compiler can compile a trivial definition +if (NOT MEX_WORKS) + # attempt to generate a gateway for a function + message(STATUS "Trying to generate Matlab code") + execute_process( + COMMAND ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/generator/gen_matlab.py + --hdrparser ${HDR_PARSER_PATH} + --rstparser ${RST_PARSER_PATH} + --extra "test=${CMAKE_CURRENT_SOURCE_DIR}/test/test_generator.hpp" + --outdir ${CMAKE_BINARY_DIR}/junk + ERROR_VARIABLE GEN_ERROR + OUTPUT_QUIET + ) + + if (GEN_ERROR) + message(${GEN_ERROR}) + message(STATUS "Error generating Matlab code. Disabling Matlab bindings...") + ocv_module_disable(matlab) + return() + else() + message(STATUS "Trying to generate Matlab code - OK") + endif() + + # attempt to compile a gateway using mex + message(STATUS "Trying to compile mex file") + execute_process( + COMMAND ${MATLAB_MEX_SCRIPT} ${MEX_OPTS} "CXXFLAGS=\$CXXFLAGS ${MEX_CXX_FLAGS}" + ${MEX_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/test/test_compiler.cpp + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/junk + ERROR_VARIABLE MEX_ERROR + OUTPUT_QUIET + ) + + if (MEX_ERROR) + message(${MEX_ERROR}) + message(STATUS "Error compiling mex file. Disabling Matlab bindings...") + ocv_module_disable(matlab) + return() + else() + message(STATUS "Trying to compile mex file - OK") + endif() +endif() + +# if we make it here, mex works! +set_property(GLOBAL PROPERTY MEX_WORKS TRUE) +set(MEX_WORKS True CACHE BOOL ADVANCED) + + +# ---------------------------------------------------------------------------- +# Build time components +# ---------------------------------------------------------------------------- + +# proxies +# these proxies are used to trigger the add_custom_commands +# (which do the real work) only when they're outdated +set(GENERATE_PROXY ${CMAKE_CURRENT_BINARY_DIR}/generate.proxy) +set(COMPILE_PROXY ${CMAKE_CURRENT_BINARY_DIR}/compile.proxy) +# TODO: Remove following line before merging with master +file(REMOVE ${GENERATE_PROXY} ${COMPILE_PROXY}) + +# generate +# call the python executable to generate the Matlab gateways +add_custom_command( + OUTPUT ${GENERATE_PROXY} + COMMAND ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/generator/gen_matlab.py + --hdrparser ${HDR_PARSER_PATH} + --rstparser ${RST_PARSER_PATH} + --moduleroot ${CMAKE_SOURCE_DIR}/modules + --modules ${opencv_modules} + --extra ${opencv_extra_hdrs} + --outdir ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/generator/build_info.py + --os ${CMAKE_SYSTEM} + --arch ${ARCH} ${CMAKE_SYSTEM_PROCESSOR} + --compiler ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} + --mex_arch ${MATLAB_ARCH} + --mex_script ${MATLAB_MEX_SCRIPT} + --cxx_flags ${MEX_CXXFLAGS} + --opencv_version ${OPENCV_VERSION} + --commit ${GIT_COMMIT} + --modules ${opencv_modules} + --configuration "$(Configuration)" ${CMAKE_BUILD_TYPE} + --outdir ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/generator/cvmex.py + --opts="${MEX_OPTS}" + --include_dirs="${MEX_INCLUDE_DIRS}" + --lib_dir=${MEX_LIB_DIR} + --libs="${MEX_LIBS}" + --flags ${MEX_CXXFLAGS} + --outdir ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/test/help.m ${CMAKE_CURRENT_BINARY_DIR}/+cv + COMMAND ${CMAKE_COMMAND} -E touch ${GENERATE_PROXY} + COMMENT "Generating Matlab source files" +) + +# compile +# call the mex compiler to compile the gateways +# because we don't know the source files at configure-time, this +# has to be executed in a separate script in cmake's script processing mode +add_custom_command( + OUTPUT ${COMPILE_PROXY} + COMMAND ${CMAKE_COMMAND} -DMATLAB_MEX_SCRIPT=${MATLAB_MEX_SCRIPT} + -DMATLAB_MEXEXT=${MATLAB_MEXEXT} + -DMEX_OPTS=${MEX_OPTS} + -DMEX_CXXFLAGS=${MEX_CXX_FLAGS} + -DMEX_INCLUDE_DIRS="${MEX_INCLUDE_DIRS}" + -DMEX_LIB_DIR=${MEX_LIB_DIR} + -DCONFIGURATION="$(Configuration)" + -DMEX_LIBS="${MEX_LIBS}" + -DMEX_DEBUG_LIBS="${MEX_DEBUG_LIBS}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/compile.cmake + COMMAND ${CMAKE_COMMAND} -E touch ${COMPILE_PROXY} + COMMENT "Compiling Matlab source files. This could take a while..." +) + +# targets +# opencv_matlab_sources --> opencv_matlab +add_custom_target(${the_module}_sources ALL DEPENDS ${GENERATE_PROXY}) +add_custom_target(${the_module} ALL DEPENDS ${COMPILE_PROXY}) +add_dependencies(${the_module} ${the_module}_sources ${${the_module}_ACTUAL_DEPS}) + +if (ENABLE_SOLUTION_FOLDERS) + set_target_properties(${the_module} PROPERTIES FOLDER "modules") +endif() + + +# ---------------------------------------------------------------------------- +# Install time components +# ---------------------------------------------------------------------------- +# NOTE: Trailing slashes on the DIRECTORY paths are important! +# TODO: What needs to be done with rpath???? + +# install the +cv directory verbatim +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${OPENCV_INCLUDE_INSTALL_PATH}) +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/+cv/ DESTINATION matlab/+cv) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/cv.m DESTINATION matlab) + +# update the custom mex compiler to point to the install locations +string(REPLACE ";" "\\ " MEX_OPTS "${MEX_OPTS}") +string(REPLACE ";" "\\ " MEX_LIBS "${MEX_LIBS}") +string(REPLACE " " "\\ " MEX_CXXFLAGS ${MEX_CXXFLAGS}) +string(REPLACE ";" "\\ " MEX_INCLUDE_DIRS "${MEX_INCLUDE_DIRS}") +install(CODE + "execute_process( + COMMAND ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/generator/cvmex.py + --opts=${MEX_OPTS} + --include_dirs=-I${CMAKE_INSTALL_PREFIX}/${OPENCV_INCLUDE_INSTALL_PATH} + --lib_dir=-L${CMAKE_INSTALL_PREFIX}/${OPENCV_LIB_INSTALL_PATH} + --libs=${MEX_LIBS} + --flags=${MEX_CXXFLAGS} + --outdir ${CMAKE_INSTALL_PREFIX}/matlab + )" +) diff --git a/modules/matlab/LICENSE b/modules/matlab/LICENSE new file mode 100644 index 000000000..f75ed932b --- /dev/null +++ b/modules/matlab/LICENSE @@ -0,0 +1,42 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this +// license. If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote +// products derived from this software without specific prior written +// permission. +// +// This software is provided by the copyright holders and contributors "as is" +// and any express or implied warranties, including, but not limited to, the +// implied warranties of merchantability and fitness for a particular purpose +// are disclaimed. In no event shall the Intel Corporation or contributors be +// liable for any direct, indirect, incidental, special, exemplary, or +// consequential damages (including, but not limited to, procurement of +// substitute goods or services; loss of use, data, or profits; or business +// interruption) however caused and on any theory of liability, whether in +// contract, strict liability, or tort (including negligence or otherwise) +// arising in any way out of the use of this software, even if advised of the +// possibility of such damage. +// +//////////////////////////////////////////////////////////////////////////////// \ No newline at end of file diff --git a/modules/matlab/README.md b/modules/matlab/README.md new file mode 100644 index 000000000..0c8074818 --- /dev/null +++ b/modules/matlab/README.md @@ -0,0 +1,383 @@ +OpenCV Matlab Code Generator +============================ +This module contains a code generator to automatically produce Matlab mex wrappers for other modules within the OpenCV library. Once compiled and added to the Matlab path, this gives users the ability to call OpenCV methods natively from within Matlab. + + +Build +----- +The Matlab code generator is fully integrated into the OpenCV build system. If cmake finds a Matlab installation available on the host system while configuring OpenCV, it will attempt to generate Matlab wrappers for all OpenCV modules. If cmake is having trouble finding your Matlab installation, you can explicitly point it to the root by defining the `MATLAB_ROOT_DIR` variable. For example, on a Mac you could type: + + cmake -DMATLAB_ROOT_DIR=/Applications/MATLAB_R2013a.app .. + + +Install +------- +In order to use the bindings, you will need to add them to the Matlab path. The path to add is: + +1. `${CMAKE_BUILD_DIR}/modules/matlab` if you are working from the build tree, or +2. `${CMAKE_INSTALL_PREFIX}/matlab` if you have installed OpenCV + +In Matlab, simply run: + + addpath('/path/to/opencv/matlab/'); + + +Run +--- +Once you've added the bindings directory to the Matlab path, you can start using them straight away! OpenCV calls need to be prefixed with a 'cv' qualifier, to disambiguate them from Matlab methods of the same name. For example, to compute the dft of a matrix, you might do the following: + +```matlab +% load an image (Matlab) +I = imread('cameraman.tif'); + +% compute the DFT (OpenCV) +If = cv.dft(I, cv.DFT_COMPLEX_OUTPUT); +``` + +As you can see, both OpenCV methods and constants can be used with 'cv' qualification. You can also call: + + help cv.dft + +to get help on the purpose and call signature of a particular method, or + + help cv + +to get general help regarding the OpenCV bindings. If you ever run into issues with the bindings + + cv.buildInformation(); + +will produce a printout of diagnostic information pertaining to your particular build of OS, OpenCV and Matlab. It is useful to submit this information alongside a bug report to the OpenCV team. + +Writing your own mex files +-------------------------- +The Matlab bindings come with a set of utilities to help you quickly write your own mex files using OpenCV definitions. By doing so, you have all the speed and freedom of C++, with the power of OpenCV's math expressions and optimizations. + +The first thing you need to learn how to do is write a mex-file with Matlab constructs. Following is a brief example: + +```cpp +// include useful constructs +// this automatically includes opencv core.hpp and mex.h) +#include +using namespace cv; +using namespace matlab; +using namespace bridge; + +// define the mex gateway +void mexFunction(int nlhs, mxArray* plhs[], + int nrhs, const mxArray* prhs[]) { + + // claim the inputs into scoped management + MxArrayVector raw(prhs, prhs+nrhs); + + // add an argument parser to automatically handle basic options + ArgumentParser parser("my function"); + parser.addVariant(1, 1, "opt"); + MxArrayVector reordered = parser.parse(raw); + + // if we get here, we know the inputs are valid and reordered. Unpack... + BridgeVector inputs(reordered.begin(), reordered.end()); + Mat required = inputs[0].toMat(); + string optional = inputs[1].empty() ? "Default string" : inputs[1].toString(); + + try { + // Do stuff... + } catch(Exception& e) { + error(e.what()); + } catch(...) { + error("Uncaught exception occurred"); + } + + // allocate an output + Bridge out = required; + plhs[0] = out.toMxArray().releaseOwnership(); +} +``` + +There are a couple of important things going on in this example. Firstly, you need to include `` to enable the bridging capabilities. Once you've done this, you get some nice utilities for free. `MxArray` is a class that wraps Matlab's `mxArray*` class in an OOP-style interface. `ArgumentParser` is a class that handles default, optional and named arguments for you, along with multiple possible calling syntaxes. Finally, `Bridge` is a class that allows bidirectional conversions between OpenCV/std and Matlab types. + +Once you have written your file, it can be compiled with the provided mex utility: + + cv.mex('my_function.cpp'); + +This utility automatically links in all of the necessary OpenCV libraries to make your function work. + +NOTE: OpenCV uses exceptions throughout the codebase. It is a **very** good idea to wrap your code in exception handlers to avoid crashing Matlab in the event of an exception being thrown. + +------------------------------------------------------------------ + + +Developer +========= +The following sections contain information for developers seeking to use, understand or extend the Matlab bindings. The bindings are generated in python using a powerful templating engine called Jinja2. Because Matlab mex gateways have a common structure, they are well suited to templatization. There are separate templates for formatting C++ classes, Matlab classes, C++ functions, constants (enums) and documentation. + +The task of the generator is two-fold: + +1. To parse the OpenCV headers and build a semantic tree that can be fed to the template engine +2. To define type conversions between C++/OpenCV and Matlab types + +Once a source file has been generated for each OpenCV definition, and type conversions have been established, the mex compiler is invoked to produce the mex gateway (shared object) and link in the OpenCV libraries. + + +File layout +----------- +opencv/modules/matlab (this module) + +* `CMakeLists.txt` (main cmake configuration file) +* `README.md` (this file) +* `compile.cmake` (the cmake help script for compiling generated source code) +* `generator` (the folder containing generator code) + * `jinja2` (the binding templating engine) + * `filters.py` (template filters) + * `gen_matlab.py` (the binding generator control script) + * `parse_tree.py` (python class to refactor the hdr_parser.py output) + * `templates` (the raw templates for populating classes, constants, functions and docs) +* `include` (C++ headers for the bindings) + * `mxarray.hpp` (C++ OOP-style interface for Matlab mxArray* class) + * `bridge.hpp` (type conversions) + * `map.hpp` (hash map interface for instance storage and method lookup) +* `io` (FileStorage interface for .mat files) +* `test` (generator, compiler and binding test scripts) + + +Call Tree +--------- +The cmake call tree can be broken into 3 main components: + +1. configure time +2. build time +3. install time + +**Find Matlab (configure)** +The first thing to do is discover a Matlab installation on the host system. This is handled by the `OpenCVFindMatlab.cmake` in `opencv/cmake`. On Windows machines it searches the registry and path, while on *NIX machines it searches a set of canonical install paths. Once Matlab has been found, a number of variables are defined, such as the path to the mex compiler, the mex libraries, the mex include paths, the architectural extension, etc. + +**Test the generator (configure)** +Attempt to produce a source file for a simple definition. This tests whether python and pythonlibs are correctly invoked on the host. + +**Test the mex compiler (configure)** +Attempt to compile a simple definition using the mex compiler. A mex file is actually just a shared object with a special exported symbol `_mexFunction` which serves as the entry-point to the function. As such, the mex compiler is just a set of scripts configuring the system compiler. In most cases this is the same as the OpenCV compiler, but *could* be different. The test checks whether the mex and generator includes can be found, the system libraries can be linked and the passed compiler flags are compatible. + +If any of the configure time tests fail, the bindings will be disabled, but the main OpenCV configure will continue without error. The configuration summary will contain the block: + + Matlab + mex: /Applications/MATLAB_R2013a.app/bin/mex + compiler/generator: Not working (bindings will not be generated) + +**Generate the sources (build)** +Given a set of modules (the intersection of the OpenCV modules being built and the matlab module optional dependencies), the `CppHeaderParser()` from `opencv/modules/python/src2/hdr_parser.py` will parse the module headers and produce a set of definitions. + +The `ParseTree()` from `opencv/modules/matlab/generator/parse_tree.py` takes this set of definitions and refactors them into a semantic tree better suited to templatization. For example, a trivial definition from the header parser may look something like: + +```python +[fill, void, ['/S'], [cv::Mat&, mat, '', ['/I', '/O']]] +``` + +The equivalent refactored output will look like: + +```python + Function + name = 'fill' + rtype = 'void' + static = True + req = + Argument + name = 'mat' + type = 'cv::Mat' + ref = '&' + I = True + O = True + default = '' +``` + +The added semantics (Namespace, Class, Function, Argument, name, etc) make it easier for the templating engine to parse, slice and populate definitions. + +Once the definitions have been parsed, `gen_matlab.py` passes each definition to the template engine with the appropriate template (class, function, enum, doc) and the filled template gets written to the `${CMAKE_CURRENT_BUILD_DIR}/src` directory. + +The generator relies upon a proxy object called `generate.proxy` to determine when the sources are out of date and need to be re-generated. + +**Compile the sources (build)** +Once the sources have been generated, they are compiled by the mex compiler. The `compile.cmake` script in `opencv/modules/matlab/` takes responsibility for iterating over each source file in `${CMAKE_CURRENT_BUILD_DIR}/src` and compiling it with the passed includes and OpenCV libraries. + +The flags used to compile the main OpenCV libraries are also forwarded to the mex compiler. So if, for example, you compiled OpenCV with SSE support, the mex bindings will also use SSE. Likewise, if you compile OpenCV in debug mode, the bindings will link to the debug version of the libraries. + +Importantly, the mex compiler includes the `mxarray.hpp`, `bridge.hpp` and `map.hpp` files from the `opencv/modules/matlab/include` directory. `mxarray.hpp` defines a `MxArray` class which wraps Matlab's `mxArray*` type in a more friendly OOP-syle interface. `bridge.hpp` defines a `Bridge` class which is able to perform type conversions between Matlab types and std/OpenCV types. It can be extended with new definitions using the plugin interface described in that file. + +The compiler relies upon a proxy object called `compile.proxy` to determine when the generated sources are out of date and need to be re-compiled. + +**Install the files (install)** +At install time, the mex files are put into place at `${CMAKE_INSTALL_PREFIX}/matlab` and their linkages updated. + + +Jinja2 +------ +Jinja2 is a powerful templating engine, similar to python's builtin `string.Template` class but implementing the model-view-controller paradigm. For example, a trivial view could be populated as follows: + +**view.py** + +```html+django +{{ title }} + +``` + +**model.py** + +```python +class User(object): + __init__(self): + self.username = '' + self.url = '' + +def sanitize(text): + """Filter for escaping html tags to prevent code injection""" +``` + +**controller.py** + +```python +def populate(users): +# initialize jinja +jtemplate = jinja2.Environment(loader=FileSystemLoader()) + +# add the filters to the engine +jtemplate['sanitize'] = sanitize + +# get the view +template = jtemplate.get_template('view') + +# populate the template with a list of User objects +populated = template.render(title='all users', users=users) + +# write to file +with open('users.html', 'wb') as f: + f.write(populated) +``` + +Thus the style and layout of the view is kept separate from the content (model). This modularity improves readability and maintainability of both the view and content and (for my own sanity) has helped significantly in debugging errors. + +File Reference +-------------- +**gen_matlab.py** +gen_matlab has the following call signature: + + gen_matlab.py --hdrparser path/to/hdr_parser/dir + --rstparser path/to/rst_parser/dir + --moduleroot path/to/opencv/modules + --modules core imgproc highgui etc + --extra namespace=/additional/header/to/parse + --outdir /path/to/place/generated/src + +**build_info.py** +build_info has the following call signature: + + build_info.py --os operating_system_string + --arch bitness processor + --compiler id version + --mex_arch arch_string + --mex_script /path/to/mex/script + --cxx_flags -list -of -flags -to -passthrough + --opencv_version version_string + --commit commit_hash_if_using_git + --modules core imgproc highgui etc + --configuration Debug/Release + --outdir path/to/place/build/info + +**parse_tree.py** +To build a parse tree, first parse a set of headers, then invoke the parse tree to refactor the output: + +```python +# parse a set of definitions into a dictionary of namespaces +parser = CppHeaderParser() +ns['core'] = parser.parse('path/to/opencv/core.hpp') + +# refactor into a semantic tree +parse_tree = ParseTree() +parse_tree.build(ns) + +# iterate over the tree +for namespace in parse_tree.namespaces: + for clss in namespace.classes: + # do stuff + for method in namespace.methods: + # do stuff +``` + +**mxarray.hpp** +mxarray.hpp defines a class called `MxArray` which provides an OOP-style interface for Matlab's homogeneous `mxArray*` type. To create an `MxArray`, you can either inherit an existing array + +```cpp +MxArray mat(prhs[0]); +``` + +or create a new array + +```cpp +MxArray mat(5, 5, Matlab::Traits::ScalarType); +MxArray mat = MxArray::Matrix(5, 5); +``` + +The default constructor allocates a `0 x 0` array. Once you have encapculated an `mxArray*` you can access its properties through member functions: + +```cpp +mat.rows(); +mat.cols(); +mat.size(); +mat.channels(); +mat.isComplex(); +mat.isNumeric(); +mat.isLogical(); +mat.isClass(); +mat.className(); +mat.real(); +mat.imag(); +``` + +The MxArray object uses scoped memory management. If you wish to pass an MxArray back to Matlab (as a lhs pointer), you need to explicitly release ownership of the array so that it is not destroyed when it leaves scope: + +```cpp +plhs[0] = mat.releaseOwnership(); +``` + +mxarray.hpp also includes a number of helper utilities that make working in mex-world a little easier. One such utility is the `ArgumentParser`. `ArgumentParser` automatically handles required and optional arguments to a method, and even enables named arguments as used in many core Matlab functions. For example, if you had a function with the following signature: + +```cpp +void f(Mat first, Mat second, Mat mask=Mat(), int dtype=-1); +``` + +then you can create an `ArgumentParser` as follows: + +```cpp +ArgumentParser parser("f"); +parser.addVariant(2, 2, "mask", "dtype"); +MxArrayVector inputs = parser.parse(prhs, prhs+nrhs); +``` + +and that will make available the following calling syntaxes: + +```matlab +f(first, second); +f(first, second, mask); +f(first, second, mask, dtype); +f(first, second, 'dtype', dtype, 'mask', mask); % optional ordering does not matter +f(first, second, 'dtype', dtype); % only second optional argument provided +f(first, second, mask, 'dtype', dtype); % mixture of ordered and named +``` + +Further, the output of the `parser.parse()` method will always contain the total number of required and optional arguments that the method can take, with unspecified arguments given by empty matrices. Thus, to check if an optional argument has been given, you can do: + +```cpp +int dtype = inputs[3].empty() ? -1 : inputs[3].scalar(); +``` + +**bridge.hpp** +The bridge interface defines a `Bridge` class which provides type conversion between std/OpenCV and Matlab types. A type conversion must provide the following: + +```cpp +Bridge& operator=(const MyObject&); +MyObject toMyObject(); +operator MyObject(); +``` + +The binding generator will then automatically call the conversion operators (either explicitly or implicitly) if your `MyObject` class is encountered as an input or return from a parsed definition. diff --git a/modules/matlab/compile.cmake b/modules/matlab/compile.cmake new file mode 100644 index 000000000..ebc3fd9a6 --- /dev/null +++ b/modules/matlab/compile.cmake @@ -0,0 +1,46 @@ +# LISTIFY +# Given a string of space-delimited tokens, reparse as a string of +# semi-colon delimited tokens, which in CMake land is exactly equivalent +# to a list +macro(listify OUT_LIST IN_STRING) + string(REPLACE " " ";" ${OUT_LIST} ${IN_STRING}) +endmacro() + +# listify multiple-argument inputs +listify(MEX_INCLUDE_DIRS_LIST ${MEX_INCLUDE_DIRS}) +if (${CONFIGURATION} MATCHES "Debug") + listify(MEX_LIBS_LIST ${MEX_DEBUG_LIBS}) +else() + listify(MEX_LIBS_LIST ${MEX_LIBS}) +endif() + +# if it's MSVC building a Debug configuration, don't build bindings +if ("${CONFIGURATION}" MATCHES "Debug") + message(STATUS "Matlab bindings are only available in Release configurations. Skipping...") + return() +endif() + +# for each generated source file: +# 1. check if the file has already been compiled +# 2. attempt compile if required +# 3. if the compile fails, throw an error and cancel compilation +file(GLOB SOURCE_FILES "${CMAKE_CURRENT_BINARY_DIR}/src/*.cpp") +foreach(SOURCE_FILE ${SOURCE_FILES}) + # strip out the filename + get_filename_component(FILENAME ${SOURCE_FILE} NAME_WE) + # compile the source file using mex + execute_process(COMMAND echo ${FILENAME}) + if (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/+cv/${FILENAME}.${MATLAB_MEXEXT}) + execute_process( + COMMAND ${MATLAB_MEX_SCRIPT} ${MEX_OPTS} "CXXFLAGS=\$CXXFLAGS ${MEX_CXXFLAGS}" ${MEX_INCLUDE_DIRS_LIST} + ${MEX_LIB_DIR} ${MEX_LIBS_LIST} ${SOURCE_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/+cv + OUTPUT_QUIET + ERROR_VARIABLE FAILED + ) + endif() + # TODO: If a mex file fails to compile, should we error out? + if (FAILED) + message(FATAL_ERROR "Failed to compile ${FILENAME}: ${FAILED}") + endif() +endforeach() diff --git a/modules/matlab/generator/build_info.py b/modules/matlab/generator/build_info.py new file mode 100644 index 000000000..beb91bd10 --- /dev/null +++ b/modules/matlab/generator/build_info.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +def substitute(build, output_dir): + + # setup the template engine + template_dir = os.path.join(os.path.dirname(__file__), 'templates') + jtemplate = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True) + + # add the filters + jtemplate.filters['csv'] = csv + jtemplate.filters['stripExtraSpaces'] = stripExtraSpaces + + # load the template + template = jtemplate.get_template('template_build_info.m') + + # create the build directory + output_dir = output_dir+'/+cv' + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + # populate template + populated = template.render(build=build, time=time) + with open(os.path.join(output_dir, 'buildInformation.m'), 'wb') as f: + f.write(populated) + +if __name__ == "__main__": + """ + Usage: python build_info.py --os os_version_string + --arch [bitness processor] + --compiler [id version] + --mex_arch arch_string + --mex_script /path/to/mex/script + --cxx_flags [-list -of -flags -to -passthrough] + --opencv_version version_string + --commit commit_hash_if_using_git + --modules [core imgproc highgui etc] + --configuration Debug/Release + --outdir /path/to/write/build/info + + build_info.py generates a Matlab function that can be invoked with a call to + >> cv.buildInformation(); + + This function prints a summary of the user's OS, OpenCV and Matlab build + given the information passed to this module. build_info.py invokes Jinja2 + on the template_build_info.m template. + """ + + # parse the input options + import sys, re, os, time + from argparse import ArgumentParser + parser = ArgumentParser() + parser.add_argument('--os') + parser.add_argument('--arch', nargs=2) + parser.add_argument('--compiler', nargs='+') + parser.add_argument('--mex_arch') + parser.add_argument('--mex_script') + parser.add_argument('--mex_opts', default=['-largeArrayDims'], nargs='*') + parser.add_argument('--cxx_flags', default=[], nargs='*') + parser.add_argument('--opencv_version', default='', nargs='?') + parser.add_argument('--commit', default='Not in working git tree', nargs='?') + parser.add_argument('--modules', nargs='+') + parser.add_argument('--configuration') + parser.add_argument('--outdir') + build = parser.parse_args() + + from filters import * + from jinja2 import Environment, FileSystemLoader + + # populate the build info template + substitute(build, build.outdir) diff --git a/modules/matlab/generator/cvmex.py b/modules/matlab/generator/cvmex.py new file mode 100644 index 000000000..fc47100e1 --- /dev/null +++ b/modules/matlab/generator/cvmex.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +def substitute(cv, output_dir): + + # setup the template engine + template_dir = os.path.join(os.path.dirname(__file__), 'templates') + jtemplate = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True) + + # add the filters + jtemplate.filters['cellarray'] = cellarray + jtemplate.filters['split'] = split + jtemplate.filters['csv'] = csv + + # load the template + template = jtemplate.get_template('template_cvmex_base.m') + + # create the build directory + output_dir = output_dir+'/+cv' + if not os.path.isdir(output_dir): + os.mkdir(output_dir) + + # populate template + populated = template.render(cv=cv, time=time) + with open(os.path.join(output_dir, 'mex.m'), 'wb') as f: + f.write(populated) + +if __name__ == "__main__": + """ + Usage: python cvmex.py --opts [-list -of -opts] + --include_dirs [-list -of -opencv_include_directories] + --lib_dir opencv_lib_directory + --libs [-lopencv_core -lopencv_imgproc ...] + --flags [-Wall -opencv_build_flags ...] + --outdir /path/to/generated/output + + cvmex.py generates a custom mex compiler that automatically links OpenCV + libraries to built sources where appropriate. The calling syntax is the + same as the builtin mex compiler, with added cv qualification: + >> cv.mex(..., ...); + """ + + # parse the input options + import sys, re, os, time + from argparse import ArgumentParser + parser = ArgumentParser() + parser.add_argument('--opts') + parser.add_argument('--include_dirs') + parser.add_argument('--lib_dir') + parser.add_argument('--libs') + parser.add_argument('--flags') + parser.add_argument('--outdir') + cv = parser.parse_args() + + from filters import * + from jinja2 import Environment, FileSystemLoader + + # populate the mex base template + substitute(cv, cv.outdir) diff --git a/modules/matlab/generator/filters.py b/modules/matlab/generator/filters.py new file mode 100644 index 000000000..f2d59b2b0 --- /dev/null +++ b/modules/matlab/generator/filters.py @@ -0,0 +1,180 @@ +from textwrap import TextWrapper +from string import split, join +import re, os +# precompile a URL matching regular expression +urlexpr = re.compile(r"((https?):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+-=\\\.&]*)", re.MULTILINE|re.UNICODE) + +def inputs(args): + '''Keeps only the input arguments in a list of elements. + In OpenCV input arguments are all arguments with names + not beginning with 'dst' + ''' + try: + return [arg for arg in args['only'] if arg.I and not arg.O] + except: + return [arg for arg in args if arg.I] + +def ninputs(fun): + '''Counts the number of input arguments in the input list''' + return len(inputs(fun.req)) + len(inputs(fun.opt)) + +def outputs(args): + '''Determines whether any of the given arguments is an output + reference, and returns a list of only those elements. + In OpenCV, output references are preceeded by 'dst' + ''' + try: + return [arg for arg in args['only'] if arg.O and not arg.I] + except: + return [arg for arg in args if arg.O] + +def only(args): + '''Returns exclusively the arguments which are only inputs + or only outputs''' + d = {}; + d['only'] = args + return d + +def void(arg): + '''Is the input 'void' ''' + return arg == 'void' + +def flip(arg): + '''flip the sign of the input''' + return not arg + +def noutputs(fun): + '''Counts the number of output arguments in the input list''' + return int(not void(fun.rtp)) + len(outputs(fun.req)) + len(outputs(fun.opt)) + +def convertibleToInt(string): + '''Can the input string be evaluated to an integer?''' + salt = '1+' + try: + exec(salt+string) + return True + except: + return False + +def binaryToDecimal(string): + '''Attempt to convert the input string to floating point representation''' + try: + return str(eval(string)) + except: + return string + +def formatMatlabConstant(string, table): + ''' + Given a string representing a Constant, and a table of all Constants, + attempt to resolve the Constant into a valid Matlab expression + For example, the input + DEPENDENT_VALUE = 1 << FIXED_VALUE + needs to be converted to + DEPENDENT_VALUE = bitshift(1, cv.FIXED_VALUE); + ''' + # split the string into expressions + words = re.split('(\W+)', string) + # add a 'cv' prefix if an expression is also a key in the lookup table + words = ''.join([('cv.'+word if word in table else word) for word in words]) + # attempt to convert arithmetic expressions and binary/hex to decimal + words = binaryToDecimal(words) + # convert any remaining bitshifts to Matlab 'bitshift' methods + shift = re.sub('[\(\) ]', '', words).split('<<') + words = 'bitshift('+shift[0]+', '+shift[1]+')' if len(shift) == 2 else words + return words + +def matlabURL(string): + """This filter is used to construct a Matlab specific URL that calls the + system browser instead of the (insanely bad) builtin Matlab browser""" + return re.sub(urlexpr, '\\1', string) + +def capitalizeFirst(text): + '''Capitalize only the first character of the text string''' + return text[0].upper() + text[1:] + +def toUpperCamelCase(text): + '''variable_name --> VariableName''' + return ''.join([capitalizeFirst(word) for word in text.split('_')]) + +def toLowerCamelCase(text): + '''variable_name --> variableName''' + upper_camel = toUpperCamelCase(test) + return upper_camel[0].lower() + upper_camel[1:] + +def toUnderCase(text): + '''VariableName --> variable_name''' + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + +def stripTags(text): + ''' + strip or convert html tags from a text string + content --> content + --> '' + < --> < + > --> > + &le --> <= + &ge --> >= + ''' + upper = lambda pattern: pattern.group(1).upper() + text = re.sub('(.*?)', upper, text) + text = re.sub('<([^=\s].*?)>', '', text) + text = re.sub('<', '<', text) + text = re.sub('>', '>', text) + text = re.sub('&le', '<=', text) + text = re.sub('&ge', '>=', text) + return text + +def qualify(text, name): + '''Adds uppercase 'CV.' qualification to any occurrences of name in text''' + return re.sub(name.upper(), 'CV.'+name.upper(), text) + +def slugify(text): + '''A_Function_name --> a-function-name''' + return text.lower().replace('_', '-') + +def filename(fullpath): + '''Returns only the filename without an extension from a file path + eg. /path/to/file.txt --> file + ''' + return os.path.splitext(os.path.basename(fullpath))[0] + +def split(text, delimiter=' '): + '''Split a text string into a list using the specified delimiter''' + return text.split(delimiter) + +def csv(items, sep=', '): + '''format a list with a separator (comma if not specified)''' + return sep.join(item for item in items) + +def cellarray(items, escape='\''): + '''format a list of items as a matlab cell array''' + return '{' + ', '.join(escape+item+escape for item in items) + '}' + +def stripExtraSpaces(text): + '''Removes superfluous whitespace from a string, including the removal + of all leading and trailing whitespace''' + return ' '.join(text.split()) + +def comment(text, wrap=80, escape='% ', escape_first='', escape_last=''): + '''comment filter + Takes a string in text, and wraps it to wrap characters in length with + preceding comment escape sequence on each line. escape_first and + escape_last can be used for languages which define block comments. + Examples: + C++ inline comment comment(80, '// ') + C block comment: comment(80, ' * ', '/*', ' */') + Matlab comment: comment(80, '% ') + Matlab block comment: comment(80, '', '%{', '%}') + Python docstrings: comment(80, '', '\'\'\'', '\'\'\'') + ''' + + tw = TextWrapper(width=wrap-len(escape)) + if escape_first: + escape_first = escape_first+'\n' + if escape_last: + escape_last = '\n'+escape_last + escapn = '\n'+escape + lines = text.split('\n') + wlines = (tw.wrap(line) for line in lines) + return escape_first+escape+join((join(line, escapn) for line in wlines), escapn)+escape_last diff --git a/modules/matlab/generator/gen_matlab.py b/modules/matlab/generator/gen_matlab.py new file mode 100644 index 000000000..8acc6facb --- /dev/null +++ b/modules/matlab/generator/gen_matlab.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python + +class MatlabWrapperGenerator(object): + """ + MatlabWrapperGenerator is a class for generating Matlab mex sources from + a set of C++ headers. MatlabWrapperGenerator objects can be default + constructed. Given an instance, the gen() method performs the translation. + """ + + def gen(self, module_root, modules, extras, output_dir): + """ + Generate a set of Matlab mex source files by parsing exported symbols + in a set of C++ headers. The headers can be input in one (or both) of + two methods: + 1. specify module_root and modules + Given a path to the OpenCV module root and a list of module names, + the headers to parse are implicitly constructed. + 2. specifiy header locations explicitly in extras + Each element in the list of extras must be of the form: + 'namespace=/full/path/to/extra/header.hpp' where 'namespace' is + the namespace in which the definitions should be added. + The output_dir specifies the directory to write the generated sources + to. + """ + # parse each of the files and store in a dictionary + # as a separate "namespace" + parser = CppHeaderParser() + rst = rst_parser.RstParser(parser) + rst_parser.verbose = False + rst_parser.show_warnings = False + rst_parser.show_errors = False + rst_parser.show_critical_errors = False + + ns = dict((key, []) for key in modules) + doc = dict((key, []) for key in modules) + path_template = Template('${module}/include/opencv2/${module}.hpp') + + for module in modules: + # construct a header path from the module root and a path template + header = os.path.join(module_root, path_template.substitute(module=module)) + # parse the definitions + ns[module] = parser.parse(header) + # parse the documentation + rst.parse(module, os.path.join(module_root, module)) + doc[module] = rst.definitions + rst.definitions = {} + + for extra in extras: + module = extra.split("=")[0] + header = extra.split("=")[1] + ns[module] = ns[module] + parser.parse(header) if module in ns else parser.parse(header) + + # cleanify the parser output + parse_tree = ParseTree() + parse_tree.build(ns) + + # setup the template engine + template_dir = os.path.join(os.path.dirname(__file__), 'templates') + jtemplate = Environment(loader=FileSystemLoader(template_dir), trim_blocks=True, lstrip_blocks=True) + + # add the custom filters + jtemplate.filters['formatMatlabConstant'] = formatMatlabConstant + jtemplate.filters['convertibleToInt'] = convertibleToInt + jtemplate.filters['toUpperCamelCase'] = toUpperCamelCase + jtemplate.filters['toLowerCamelCase'] = toLowerCamelCase + jtemplate.filters['toUnderCase'] = toUnderCase + jtemplate.filters['matlabURL'] = matlabURL + jtemplate.filters['stripTags'] = stripTags + jtemplate.filters['filename'] = filename + jtemplate.filters['comment'] = comment + jtemplate.filters['inputs'] = inputs + jtemplate.filters['ninputs'] = ninputs + jtemplate.filters['outputs'] = outputs + jtemplate.filters['noutputs'] = noutputs + jtemplate.filters['qualify'] = qualify + jtemplate.filters['slugify'] = slugify + jtemplate.filters['only'] = only + jtemplate.filters['void'] = void + jtemplate.filters['not'] = flip + + # load the templates + tfunction = jtemplate.get_template('template_function_base.cpp') + tclassm = jtemplate.get_template('template_class_base.m') + tclassc = jtemplate.get_template('template_class_base.cpp') + tdoc = jtemplate.get_template('template_doc_base.m') + tconst = jtemplate.get_template('template_map_base.m') + + # create the build directory + output_source_dir = output_dir+'/src' + output_private_dir = output_source_dir+'/private' + output_class_dir = output_dir+'/+cv' + output_map_dir = output_dir+'/map' + if not os.path.isdir(output_source_dir): + os.mkdir(output_source_dir) + if not os.path.isdir(output_private_dir): + os.mkdir(output_private_dir) + if not os.path.isdir(output_class_dir): + os.mkdir(output_class_dir) + if not os.path.isdir(output_map_dir): + os.mkdir(output_map_dir) + + # populate templates + for namespace in parse_tree.namespaces: + # functions + for method in namespace.methods: + populated = tfunction.render(fun=method, time=time, includes=namespace.name) + with open(output_source_dir+'/'+method.name+'.cpp', 'wb') as f: + f.write(populated) + if namespace.name in doc and method.name in doc[namespace.name]: + populated = tdoc.render(fun=method, doc=doc[namespace.name][method.name], time=time) + with open(output_class_dir+'/'+method.name+'.m', 'wb') as f: + f.write(populated) + # classes + for clss in namespace.classes: + # cpp converter + populated = tclassc.render(clss=clss, time=time) + with open(output_private_dir+'/'+clss.name+'Bridge.cpp', 'wb') as f: + f.write(populated) + # matlab classdef + populated = tclassm.render(clss=clss, time=time) + with open(output_class_dir+'/'+clss.name+'.m', 'wb') as f: + f.write(populated) + + # create a global constants lookup table + const = dict(constants(todict(parse_tree.namespaces))) + populated = tconst.render(constants=const, time=time) + with open(output_dir+'/cv.m', 'wb') as f: + f.write(populated) + + +if __name__ == "__main__": + """ + Usage: python gen_matlab.py --hdrparser /path/to/hdr_parser/dir + --rstparser /path/to/rst_parser/dir + --moduleroot /path/to/opencv/modules + --modules [core imgproc objdetect etc] + --extra namespace=/path/to/extra/header.hpp + --outdir /path/to/output/generated/srcs + + gen_matlab.py is the main control script for generating matlab source + files from given set of headers. Internally, gen_matlab: + 1. constructs the headers to parse from the module root and list of modules + 2. parses the headers using CppHeaderParser + 3. refactors the definitions using ParseTree + 4. parses .rst docs using RstParser + 5. populates the templates for classes, function, enums and docs from the + definitions + + gen_matlab.py requires the following inputs: + --hdrparser the path to the header parser directory + (opencv/modules/python/src2) + --rstparser the path to the rst parser directory + (opencv/modules/java/generator) + --moduleroot (optional) path to the opencv directory containing the modules + --modules (optional - required if --moduleroot specified) the modules + to produce bindings for. The path to the include directories + as well as the namespaces are constructed from the modules + and the moduleroot + --extra extra headers explicitly defined to parse. This must be in + the format "namepsace=/path/to/extra/header.hpp". For example, + the core module requires the extra header: + "core=/opencv/modules/core/include/opencv2/core/core/base.hpp" + --outdir the output directory to put the generated matlab sources. In + the OpenCV build this is "${CMAKE_CURRENT_BUILD_DIR}/src" + """ + + # parse the input options + import sys, re, os, time + from argparse import ArgumentParser + parser = ArgumentParser() + parser.add_argument('--hdrparser') + parser.add_argument('--rstparser') + parser.add_argument('--moduleroot', default='', required=False) + parser.add_argument('--modules', nargs='*', default=[], required=False) + parser.add_argument('--extra', nargs='*', default=[], required=False) + parser.add_argument('--outdir') + args = parser.parse_args() + + # add the hdr_parser and rst_parser modules to the path + sys.path.append(args.hdrparser) + sys.path.append(args.rstparser) + + from string import Template + from hdr_parser import CppHeaderParser + import rst_parser + from parse_tree import ParseTree, todict, constants + from filters import * + from jinja2 import Environment, FileSystemLoader + + # create the generator + mwg = MatlabWrapperGenerator() + mwg.gen(args.moduleroot, args.modules, args.extra, args.outdir) diff --git a/modules/matlab/generator/jinja2/AUTHORS b/modules/matlab/generator/jinja2/AUTHORS new file mode 100644 index 000000000..943f625f8 --- /dev/null +++ b/modules/matlab/generator/jinja2/AUTHORS @@ -0,0 +1,33 @@ +Jinja is written and maintained by the Jinja Team and various +contributors: + +Lead Developer: + +- Armin Ronacher + +Developers: + +- Christoph Hack +- Georg Brandl + +Contributors: + +- Bryan McLemore +- Mickaël Guérin +- Cameron Knight +- Lawrence Journal-World. +- David Cramer + +Patches and suggestions: + +- Ronny Pfannschmidt +- Axel Böhm +- Alexey Melchakov +- Bryan McLemore +- Clovis Fabricio (nosklo) +- Cameron Knight +- Peter van Dijk (Habbie) +- Stefan Ebner +- Rene Leonhardt +- Thomas Waldmann +- Cory Benfield (Lukasa) diff --git a/modules/matlab/generator/jinja2/LICENSE b/modules/matlab/generator/jinja2/LICENSE new file mode 100644 index 000000000..31bf900e5 --- /dev/null +++ b/modules/matlab/generator/jinja2/LICENSE @@ -0,0 +1,31 @@ +Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. + +Some rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * The names of the contributors may not be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/modules/matlab/generator/jinja2/__init__.py b/modules/matlab/generator/jinja2/__init__.py new file mode 100644 index 000000000..fe9e8224d --- /dev/null +++ b/modules/matlab/generator/jinja2/__init__.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +""" + jinja2 + ~~~~~~ + + Jinja2 is a template engine written in pure Python. It provides a + Django inspired non-XML syntax but supports inline expressions and + an optional sandboxed environment. + + Nutshell + -------- + + Here a small example of a Jinja2 template:: + + {% extends 'base.html' %} + {% block title %}Memberlist{% endblock %} + {% block content %} + + {% endblock %} + + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +__docformat__ = 'restructuredtext en' +__version__ = '2.8-dev' + +# high level interface +from jinja2.environment import Environment, Template + +# loaders +from jinja2.loaders import BaseLoader, FileSystemLoader, \ + DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \ + ModuleLoader + +# bytecode caches +from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \ + MemcachedBytecodeCache + +# undefined types +from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined + +# exceptions +from jinja2.exceptions import TemplateError, UndefinedError, \ + TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \ + TemplateAssertionError + +# decorators and public utilities +from jinja2.filters import environmentfilter, contextfilter, \ + evalcontextfilter +from jinja2.utils import Markup, escape, clear_caches, \ + environmentfunction, evalcontextfunction, contextfunction, \ + is_undefined + +__all__ = [ + 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader', + 'DictLoader', 'FunctionLoader', 'PrefixLoader', + 'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache', + 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined', + 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound', + 'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError', + 'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape', + 'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined', + 'evalcontextfilter', 'evalcontextfunction' +] diff --git a/modules/matlab/generator/jinja2/_compat.py b/modules/matlab/generator/jinja2/_compat.py new file mode 100644 index 000000000..1326cbc66 --- /dev/null +++ b/modules/matlab/generator/jinja2/_compat.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +""" + jinja2._compat + ~~~~~~~~~~~~~~ + + Some py2/py3 compatibility support based on a stripped down + version of six so we don't have to depend on a specific version + of it. + + :copyright: Copyright 2013 by the Jinja team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import sys + +PY2 = sys.version_info[0] == 2 +PYPY = hasattr(sys, 'pypy_translation_info') +_identity = lambda x: x + + +if not PY2: + unichr = chr + range_type = range + text_type = str + string_types = (str,) + + iterkeys = lambda d: iter(d.keys()) + itervalues = lambda d: iter(d.values()) + iteritems = lambda d: iter(d.items()) + + import pickle + from io import BytesIO, StringIO + NativeStringIO = StringIO + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + ifilter = filter + imap = map + izip = zip + intern = sys.intern + + implements_iterator = _identity + implements_to_string = _identity + encode_filename = _identity + get_next = lambda x: x.__next__ + +else: + unichr = unichr + text_type = unicode + range_type = xrange + string_types = (str, unicode) + + iterkeys = lambda d: d.iterkeys() + itervalues = lambda d: d.itervalues() + iteritems = lambda d: d.iteritems() + + import cPickle as pickle + from cStringIO import StringIO as BytesIO, StringIO + NativeStringIO = BytesIO + + exec('def reraise(tp, value, tb=None):\n raise tp, value, tb') + + from itertools import imap, izip, ifilter + intern = intern + + def implements_iterator(cls): + cls.next = cls.__next__ + del cls.__next__ + return cls + + def implements_to_string(cls): + cls.__unicode__ = cls.__str__ + cls.__str__ = lambda x: x.__unicode__().encode('utf-8') + return cls + + get_next = lambda x: x.next + + def encode_filename(filename): + if isinstance(filename, unicode): + return filename.encode('utf-8') + return filename + + +def with_metaclass(meta, *bases): + # This requires a bit of explanation: the basic idea is to make a + # dummy metaclass for one level of class instanciation that replaces + # itself with the actual metaclass. Because of internal type checks + # we also need to make sure that we downgrade the custom metaclass + # for one level to something closer to type (that's why __call__ and + # __init__ comes back from type etc.). + # + # This has the advantage over six.with_metaclass in that it does not + # introduce dummy classes into the final MRO. + class metaclass(meta): + __call__ = type.__call__ + __init__ = type.__init__ + def __new__(cls, name, this_bases, d): + if this_bases is None: + return type.__new__(cls, name, (), d) + return meta(name, bases, d) + return metaclass('temporary_class', None, {}) + + +try: + from urllib.parse import quote_from_bytes as url_quote +except ImportError: + from urllib import quote as url_quote diff --git a/modules/matlab/generator/jinja2/bccache.py b/modules/matlab/generator/jinja2/bccache.py new file mode 100644 index 000000000..b534e2712 --- /dev/null +++ b/modules/matlab/generator/jinja2/bccache.py @@ -0,0 +1,311 @@ +# -*- coding: utf-8 -*- +""" + jinja2.bccache + ~~~~~~~~~~~~~~ + + This module implements the bytecode cache system Jinja is optionally + using. This is useful if you have very complex template situations and + the compiliation of all those templates slow down your application too + much. + + Situations where this is useful are often forking web applications that + are initialized on the first request. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD. +""" +from os import path, listdir +import sys +import marshal +import tempfile +import fnmatch +from hashlib import sha1 +from jinja2.utils import open_if_exists +from jinja2._compat import BytesIO, pickle, PY2 + + +# marshal works better on 3.x, one hack less required +if not PY2: + marshal_dump = marshal.dump + marshal_load = marshal.load +else: + + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + + +bc_version = 2 + +# magic version used to only change with new jinja versions. With 2.6 +# we change this to also take Python version changes into account. The +# reason for this is that Python tends to segfault if fed earlier bytecode +# versions because someone thought it would be a good idea to reuse opcodes +# or make Python incompatible with earlier versions. +bc_magic = 'j2'.encode('ascii') + \ + pickle.dumps(bc_version, 2) + \ + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1]) + + +class Bucket(object): + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment, key, checksum): + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self): + """Resets the bucket (unloads the bytecode).""" + self.code = None + + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + self.code = marshal_load(f) + + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError('can\'t write empty bucket') + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + marshal_dump(self.code, f) + + def bytecode_from_string(self, string): + """Load bytecode from a string.""" + self.load_bytecode(BytesIO(string)) + + def bytecode_to_string(self): + """Return the bytecode as string.""" + out = BytesIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache(object): + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with open(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with open(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja2. + """ + + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self): + """Clears the cache. This method is not used by Jinja2 but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode('utf-8')) + if filename is not None: + filename = '|' + filename + if isinstance(filename, unicode): + filename = filename.encode('utf-8') + hash.update(filename) + return hash.hexdigest() + + def get_source_checksum(self, source): + """Returns a checksum for the source.""" + return sha1(source.encode('utf-8')).hexdigest() + + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified the system temporary items folder is used. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__(self, directory=None, pattern='__jinja2_%s.cache'): + if directory is None: + directory = tempfile.gettempdir() + self.directory = directory + self.pattern = pattern + + def _get_cache_filename(self, bucket): + return path.join(self.directory, self.pattern % bucket.key) + + def load_bytecode(self, bucket): + f = open_if_exists(self._get_cache_filename(bucket), 'rb') + if f is not None: + try: + bucket.load_bytecode(f) + finally: + f.close() + + def dump_bytecode(self, bucket): + f = open(self._get_cache_filename(bucket), 'wb') + try: + bucket.write_bytecode(f) + finally: + f.close() + + def clear(self): + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + files = fnmatch.filter(listdir(self.directory), self.pattern % '*') + for filename in files: + try: + remove(path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `werkzeug `_.contrib.cache + - `python-memcached `_ + - `cmemcache `_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only unicode. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. + """ + + def __init__(self, client, prefix='jinja2/bytecode/', timeout=None, + ignore_memcache_errors=True): + self.client = client + self.prefix = prefix + self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors + + def load_bytecode(self, bucket): + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + code = None + if code is not None: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket): + args = (self.prefix + bucket.key, bucket.bytecode_to_string()) + if self.timeout is not None: + args += (self.timeout,) + try: + self.client.set(*args) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/modules/matlab/generator/jinja2/compiler.py b/modules/matlab/generator/jinja2/compiler.py new file mode 100644 index 000000000..ad3417883 --- /dev/null +++ b/modules/matlab/generator/jinja2/compiler.py @@ -0,0 +1,1647 @@ +# -*- coding: utf-8 -*- +""" + jinja2.compiler + ~~~~~~~~~~~~~~~ + + Compiles nodes into python code. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +from itertools import chain +from copy import deepcopy +from keyword import iskeyword as is_python_keyword +from jinja2 import nodes +from jinja2.nodes import EvalContext +from jinja2.visitor import NodeVisitor +from jinja2.exceptions import TemplateAssertionError +from jinja2.utils import Markup, concat, escape +from jinja2._compat import range_type, text_type, string_types, \ + iteritems, NativeStringIO, imap + + +operators = { + 'eq': '==', + 'ne': '!=', + 'gt': '>', + 'gteq': '>=', + 'lt': '<', + 'lteq': '<=', + 'in': 'in', + 'notin': 'not in' +} + +# what method to iterate over items do we want to use for dict iteration +# in generated code? on 2.x let's go with iteritems, on 3.x with items +if hasattr(dict, 'iteritems'): + dict_item_iter = 'iteritems' +else: + dict_item_iter = 'items' + + +# does if 0: dummy(x) get us x into the scope? +def unoptimize_before_dead_code(): + x = 42 + def f(): + if 0: dummy(x) + return f + +# The getattr is necessary for pypy which does not set this attribute if +# no closure is on the function +unoptimize_before_dead_code = bool( + getattr(unoptimize_before_dead_code(), '__closure__', None)) + + +def generate(node, environment, name, filename, stream=None, + defer_init=False): + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError('Can\'t compile non template nodes') + generator = CodeGenerator(environment, name, filename, stream, defer_init) + generator.visit(node) + if stream is None: + return generator.stream.getvalue() + + +def has_safe_repr(value): + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + if isinstance(value, (bool, int, float, complex, range_type, + Markup) + string_types): + return True + if isinstance(value, (tuple, list, set, frozenset)): + for item in value: + if not has_safe_repr(item): + return False + return True + elif isinstance(value, dict): + for key, value in iteritems(value): + if not has_safe_repr(key): + return False + if not has_safe_repr(value): + return False + return True + return False + + +def find_undeclared(nodes, names): + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class Identifiers(object): + """Tracks the status of identifiers in frames.""" + + def __init__(self): + # variables that are known to be declared (probably from outer + # frames or because they are special for the frame) + self.declared = set() + + # undeclared variables from outer scopes + self.outer_undeclared = set() + + # names that are accessed without being explicitly declared by + # this one or any of the outer scopes. Names can appear both in + # declared and undeclared. + self.undeclared = set() + + # names that are declared locally + self.declared_locally = set() + + # names that are declared by parameters + self.declared_parameter = set() + + def add_special(self, name): + """Register a special name like `loop`.""" + self.undeclared.discard(name) + self.declared.add(name) + + def is_declared(self, name): + """Check if a name is declared in this or an outer scope.""" + if name in self.declared_locally or name in self.declared_parameter: + return True + return name in self.declared + + def copy(self): + return deepcopy(self) + + +class Frame(object): + """Holds compile time information for us.""" + + def __init__(self, eval_ctx, parent=None): + self.eval_ctx = eval_ctx + self.identifiers = Identifiers() + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = parent and parent.require_output_check + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer = None + + # the name of the block we're in, otherwise None. + self.block = parent and parent.block or None + + # a set of actually assigned names + self.assigned_names = set() + + # the parent of this frame + self.parent = parent + + if parent is not None: + self.identifiers.declared.update( + parent.identifiers.declared | + parent.identifiers.declared_parameter | + parent.assigned_names + ) + self.identifiers.outer_undeclared.update( + parent.identifiers.undeclared - + self.identifiers.declared + ) + self.buffer = parent.buffer + + def copy(self): + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.identifiers = object.__new__(self.identifiers.__class__) + rv.identifiers.__dict__.update(self.identifiers.__dict__) + return rv + + def inspect(self, nodes): + """Walk the node and check for identifiers. If the scope is hard (eg: + enforce on a python level) overrides from outer scopes are tracked + differently. + """ + visitor = FrameIdentifierVisitor(self.identifiers) + for node in nodes: + visitor.visit(node) + + def find_shadowed(self, extra=()): + """Find all the shadowed names. extra is an iterable of variables + that may be defined with `add_special` which may occour scoped. + """ + i = self.identifiers + return (i.declared | i.outer_undeclared) & \ + (i.declared_locally | i.declared_parameter) | \ + set(x for x in extra if i.is_declared(x)) + + def inner(self): + """Return an inner frame.""" + return Frame(self.eval_ctx, self) + + def soft(self): + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + """ + rv = self.copy() + rv.rootlevel = False + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self): + self.filters = set() + self.tests = set() + + def visit_Filter(self, node): + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node): + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names): + self.names = set(names) + self.undeclared = set() + + def visit_Name(self, node): + if node.ctx == 'load' and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node): + """Stop visiting a blocks.""" + + +class FrameIdentifierVisitor(NodeVisitor): + """A visitor for `Frame.inspect`.""" + + def __init__(self, identifiers): + self.identifiers = identifiers + + def visit_Name(self, node): + """All assignments to names go through this function.""" + if node.ctx == 'store': + self.identifiers.declared_locally.add(node.name) + elif node.ctx == 'param': + self.identifiers.declared_parameter.add(node.name) + elif node.ctx == 'load' and not \ + self.identifiers.is_declared(node.name): + self.identifiers.undeclared.add(node.name) + + def visit_If(self, node): + self.visit(node.test) + real_identifiers = self.identifiers + + old_names = real_identifiers.declared_locally | \ + real_identifiers.declared_parameter + + def inner_visit(nodes): + if not nodes: + return set() + self.identifiers = real_identifiers.copy() + for subnode in nodes: + self.visit(subnode) + rv = self.identifiers.declared_locally - old_names + # we have to remember the undeclared variables of this branch + # because we will have to pull them. + real_identifiers.undeclared.update(self.identifiers.undeclared) + self.identifiers = real_identifiers + return rv + + body = inner_visit(node.body) + else_ = inner_visit(node.else_ or ()) + + # the differences between the two branches are also pulled as + # undeclared variables + real_identifiers.undeclared.update(body.symmetric_difference(else_) - + real_identifiers.declared) + + # remember those that are declared. + real_identifiers.declared_locally.update(body | else_) + + def visit_Macro(self, node): + self.identifiers.declared_locally.add(node.name) + + def visit_Import(self, node): + self.generic_visit(node) + self.identifiers.declared_locally.add(node.target) + + def visit_FromImport(self, node): + self.generic_visit(node) + for name in node.names: + if isinstance(name, tuple): + self.identifiers.declared_locally.add(name[1]) + else: + self.identifiers.declared_locally.add(name) + + def visit_Assign(self, node): + """Visit assignments in the correct order.""" + self.visit(node.node) + self.visit(node.target) + + def visit_For(self, node): + """Visiting stops at for blocks. However the block sequence + is visited as part of the outer scope. + """ + self.visit(node.iter) + + def visit_CallBlock(self, node): + self.visit(node.call) + + def visit_FilterBlock(self, node): + self.visit(node.filter) + + def visit_Scope(self, node): + """Stop visiting at scopes.""" + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + + def __init__(self, environment, name, filename, stream=None, + defer_init=False): + if stream is None: + stream = NativeStringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + self.created_block_context = False + self.defer_init = defer_init + + # aliases for imports + self.import_aliases = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests = {} + self.filters = {} + + # the debug information + self.debug_info = [] + self._write_debug_info = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # -- Various compilation helpers + + def fail(self, msg, lineno): + """Fail with a :exc:`TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self): + """Get a new unique identifier.""" + self._last_identifier += 1 + return 't_%d' % self._last_identifier + + def buffer(self, frame): + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline('%s = []' % frame.buffer) + + def return_buffer_contents(self, frame): + """Return the buffer contents of the frame.""" + if frame.eval_ctx.volatile: + self.writeline('if context.eval_ctx.autoescape:') + self.indent() + self.writeline('return Markup(concat(%s))' % frame.buffer) + self.outdent() + self.writeline('else:') + self.indent() + self.writeline('return concat(%s)' % frame.buffer) + self.outdent() + elif frame.eval_ctx.autoescape: + self.writeline('return Markup(concat(%s))' % frame.buffer) + else: + self.writeline('return concat(%s)' % frame.buffer) + + def indent(self): + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step=1): + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame, node=None): + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline('yield ', node) + else: + self.writeline('%s.append(' % frame.buffer, node) + + def end_write(self, frame): + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(')') + + def simple_write(self, s, frame, node=None): + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes, frame): + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically + unless the force_generator parameter is set to False. + """ + if frame.buffer is None: + self.writeline('if 0: yield None') + else: + self.writeline('pass') + try: + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x): + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write('\n' * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, + self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(' ' * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline(self, x, node=None, extra=0): + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node=None, extra=0): + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature(self, node, frame, extra_kwargs=None): + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occour. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = False + for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): + if is_python_keyword(kwarg): + kwarg_workaround = True + break + + for arg in node.args: + self.write(', ') + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(', ') + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write(', %s=%s' % (key, value)) + if node.dyn_args: + self.write(', *') + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(', **dict({') + else: + self.write(', **{') + for kwarg in node.kwargs: + self.write('%r: ' % kwarg.key) + self.visit(kwarg.value, frame) + self.write(', ') + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write('%r: %s, ' % (key, value)) + if node.dyn_kwargs is not None: + self.write('}, **') + self.visit(node.dyn_kwargs, frame) + self.write(')') + else: + self.write('}') + + elif node.dyn_kwargs is not None: + self.write(', **') + self.visit(node.dyn_kwargs, frame) + + def pull_locals(self, frame): + """Pull all the references identifiers into the local scope.""" + for name in frame.identifiers.undeclared: + self.writeline('l_%s = context.resolve(%r)' % (name, name)) + + def pull_dependencies(self, nodes): + """Pull all the dependencies.""" + visitor = DependencyFinderVisitor() + for node in nodes: + visitor.visit(node) + for dependency in 'filters', 'tests': + mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): + if name not in mapping: + mapping[name] = self.temporary_identifier() + self.writeline('%s = environment.%s[%r]' % + (mapping[name], dependency, name)) + + def unoptimize_scope(self, frame): + """Disable Python optimizations for the frame.""" + # XXX: this is not that nice but it has no real overhead. It + # mainly works because python finds the locals before dead code + # is removed. If that breaks we have to add a dummy function + # that just accepts the arguments and does nothing. + if frame.identifiers.declared: + self.writeline('%sdummy(%s)' % ( + unoptimize_before_dead_code and 'if 0: ' or '', + ', '.join('l_' + name for name in frame.identifiers.declared) + )) + + def push_scope(self, frame, extra_vars=()): + """This function returns all the shadowed variables in a dict + in the form name: alias and will write the required assignments + into the current scope. No indentation takes place. + + This also predefines locally declared variables from the loop + body because under some circumstances it may be the case that + + `extra_vars` is passed to `Frame.find_shadowed`. + """ + aliases = {} + for name in frame.find_shadowed(extra_vars): + aliases[name] = ident = self.temporary_identifier() + self.writeline('%s = l_%s' % (ident, name)) + to_declare = set() + for name in frame.identifiers.declared_locally: + if name not in aliases: + to_declare.add('l_' + name) + if to_declare: + self.writeline(' = '.join(to_declare) + ' = missing') + return aliases + + def pop_scope(self, aliases, frame): + """Restore all aliases and delete unused variables.""" + for name, alias in iteritems(aliases): + self.writeline('l_%s = %s' % (name, alias)) + to_delete = set() + for name in frame.identifiers.declared_locally: + if name not in aliases: + to_delete.add('l_' + name) + if to_delete: + # we cannot use the del statement here because enclosed + # scopes can trigger a SyntaxError: + # a = 42; b = lambda: a; del a + self.writeline(' = '.join(to_delete) + ' = missing') + + def function_scoping(self, node, frame, children=None, + find_special=True): + """In Jinja a few statements require the help of anonymous + functions. Those are currently macros and call blocks and in + the future also recursive loops. As there is currently + technical limitation that doesn't allow reading and writing a + variable in a scope where the initial value is coming from an + outer scope, this function tries to fall back with a common + error message. Additionally the frame passed is modified so + that the argumetns are collected and callers are looked up. + + This will return the modified frame. + """ + # we have to iterate twice over it, make sure that works + if children is None: + children = node.iter_child_nodes() + children = list(children) + func_frame = frame.inner() + func_frame.inspect(children) + + # variables that are undeclared (accessed before declaration) and + # declared locally *and* part of an outside scope raise a template + # assertion error. Reason: we can't generate reasonable code from + # it without aliasing all the variables. + # this could be fixed in Python 3 where we have the nonlocal + # keyword or if we switch to bytecode generation + overridden_closure_vars = ( + func_frame.identifiers.undeclared & + func_frame.identifiers.declared & + (func_frame.identifiers.declared_locally | + func_frame.identifiers.declared_parameter) + ) + if overridden_closure_vars: + self.fail('It\'s not possible to set and access variables ' + 'derived from an outer scope! (affects: %s)' % + ', '.join(sorted(overridden_closure_vars)), node.lineno) + + # remove variables from a closure from the frame's undeclared + # identifiers. + func_frame.identifiers.undeclared -= ( + func_frame.identifiers.undeclared & + func_frame.identifiers.declared + ) + + # no special variables for this scope, abort early + if not find_special: + return func_frame + + func_frame.accesses_kwargs = False + func_frame.accesses_varargs = False + func_frame.accesses_caller = False + func_frame.arguments = args = ['l_' + x.name for x in node.args] + + undeclared = find_undeclared(children, ('caller', 'kwargs', 'varargs')) + + if 'caller' in undeclared: + func_frame.accesses_caller = True + func_frame.identifiers.add_special('caller') + args.append('l_caller') + if 'kwargs' in undeclared: + func_frame.accesses_kwargs = True + func_frame.identifiers.add_special('kwargs') + args.append('l_kwargs') + if 'varargs' in undeclared: + func_frame.accesses_varargs = True + func_frame.identifiers.add_special('varargs') + args.append('l_varargs') + return func_frame + + def macro_body(self, node, frame, children=None): + """Dump the function def of a macro or call block.""" + frame = self.function_scoping(node, frame, children) + # macros are delayed, they never require output checks + frame.require_output_check = False + args = frame.arguments + # XXX: this is an ugly fix for the loop nesting bug + # (tests.test_old_bugs.test_loop_call_bug). This works around + # a identifier nesting problem we have in general. It's just more + # likely to happen in loops which is why we work around it. The + # real solution would be "nonlocal" all the identifiers that are + # leaking into a new python frame and might be used both unassigned + # and assigned. + if 'loop' in frame.identifiers.declared: + args = args + ['l_loop=l_loop'] + self.writeline('def macro(%s):' % ', '.join(args), node) + self.indent() + self.buffer(frame) + self.pull_locals(frame) + self.blockvisit(node.body, frame) + self.return_buffer_contents(frame) + self.outdent() + return frame + + def macro_def(self, node, frame): + """Dump the macro definition for the def created by macro_body.""" + arg_tuple = ', '.join(repr(x.name) for x in node.args) + name = getattr(node, 'name', None) + if len(node.args) == 1: + arg_tuple += ',' + self.write('Macro(environment, macro, %r, (%s), (' % + (name, arg_tuple)) + for arg in node.defaults: + self.visit(arg, frame) + self.write(', ') + self.write('), %r, %r, %r)' % ( + bool(frame.accesses_kwargs), + bool(frame.accesses_varargs), + bool(frame.accesses_caller) + )) + + def position(self, node): + """Return a human readable position for the node.""" + rv = 'line %d' % node.lineno + if self.name is not None: + rv += ' in ' + repr(self.name) + return rv + + # -- Statement Visitors + + def visit_Template(self, node, frame=None): + assert frame is None, 'no root frame allowed' + eval_ctx = EvalContext(self.environment, self.name) + + from jinja2.runtime import __all__ as exported + self.writeline('from __future__ import division') + self.writeline('from jinja2.runtime import ' + ', '.join(exported)) + if not unoptimize_before_dead_code: + self.writeline('dummy = lambda *x: None') + + # if we want a deferred initialization we cannot move the + # environment into a local name + envenv = not self.defer_init and ', environment=environment' or '' + + # do we have an extends tag at all? If not, we can save some + # overhead by just not processing any inheritance code. + have_extends = node.find(nodes.Extends) is not None + + # find all blocks + for block in node.find_all(nodes.Block): + if block.name in self.blocks: + self.fail('block %r defined twice' % block.name, block.lineno) + self.blocks[block.name] = block + + # find all imports and import them + for import_ in node.find_all(nodes.ImportedName): + if import_.importname not in self.import_aliases: + imp = import_.importname + self.import_aliases[imp] = alias = self.temporary_identifier() + if '.' in imp: + module, obj = imp.rsplit('.', 1) + self.writeline('from %s import %s as %s' % + (module, obj, alias)) + else: + self.writeline('import %s as %s' % (imp, alias)) + + # add the load name + self.writeline('name = %r' % self.name) + + # generate the root render function. + self.writeline('def root(context%s):' % envenv, extra=1) + + # process the root + frame = Frame(eval_ctx) + frame.inspect(node.body) + frame.toplevel = frame.rootlevel = True + frame.require_output_check = have_extends and not self.has_known_extends + self.indent() + if have_extends: + self.writeline('parent_template = None') + if 'self' in find_undeclared(node.body, ('self',)): + frame.identifiers.add_special('self') + self.writeline('l_self = TemplateReference(context)') + self.pull_locals(frame) + self.pull_dependencies(node.body) + self.blockvisit(node.body, frame) + self.outdent() + + # make sure that the parent root is called. + if have_extends: + if not self.has_known_extends: + self.indent() + self.writeline('if parent_template is not None:') + self.indent() + self.writeline('for event in parent_template.' + 'root_render_func(context):') + self.indent() + self.writeline('yield event') + self.outdent(2 + (not self.has_known_extends)) + + # at this point we now have the blocks collected and can visit them too. + for name, block in iteritems(self.blocks): + block_frame = Frame(eval_ctx) + block_frame.inspect(block.body) + block_frame.block = name + self.writeline('def block_%s(context%s):' % (name, envenv), + block, 1) + self.indent() + undeclared = find_undeclared(block.body, ('self', 'super')) + if 'self' in undeclared: + block_frame.identifiers.add_special('self') + self.writeline('l_self = TemplateReference(context)') + if 'super' in undeclared: + block_frame.identifiers.add_special('super') + self.writeline('l_super = context.super(%r, ' + 'block_%s)' % (name, name)) + self.pull_locals(block_frame) + self.pull_dependencies(block.body) + self.blockvisit(block.body, block_frame) + self.outdent() + + self.writeline('blocks = {%s}' % ', '.join('%r: block_%s' % (x, x) + for x in self.blocks), + extra=1) + + # add a function that returns the debug info + self.writeline('debug_info = %r' % '&'.join('%s=%s' % x for x + in self.debug_info)) + + def visit_Block(self, node, frame): + """Call a block and register it for the template.""" + level = 1 + if frame.toplevel: + # if we know that we are a child template, there is no need to + # check if we are one + if self.has_known_extends: + return + if self.extends_so_far > 0: + self.writeline('if parent_template is None:') + self.indent() + level += 1 + context = node.scoped and 'context.derived(locals())' or 'context' + self.writeline('for event in context.blocks[%r][0](%s):' % ( + node.name, context), node) + self.indent() + self.simple_write('event', frame) + self.outdent(level) + + def visit_Extends(self, node, frame): + """Calls the extender.""" + if not frame.toplevel: + self.fail('cannot use extend from a non top-level scope', + node.lineno) + + # if the number of extends statements in general is zero so + # far, we don't have to add a check if something extended + # the template before this one. + if self.extends_so_far > 0: + + # if we have a known extends we just add a template runtime + # error into the generated code. We could catch that at compile + # time too, but i welcome it not to confuse users by throwing the + # same error at different times just "because we can". + if not self.has_known_extends: + self.writeline('if parent_template is not None:') + self.indent() + self.writeline('raise TemplateRuntimeError(%r)' % + 'extended multiple times') + + # if we have a known extends already we don't need that code here + # as we know that the template execution will end here. + if self.has_known_extends: + raise CompilerExit() + else: + self.outdent() + + self.writeline('parent_template = environment.get_template(', node) + self.visit(node.template, frame) + self.write(', %r)' % self.name) + self.writeline('for name, parent_block in parent_template.' + 'blocks.%s():' % dict_item_iter) + self.indent() + self.writeline('context.blocks.setdefault(name, []).' + 'append(parent_block)') + self.outdent() + + # if this extends statement was in the root level we can take + # advantage of that information and simplify the generated code + # in the top level from this point onwards + if frame.rootlevel: + self.has_known_extends = True + + # and now we have one more + self.extends_so_far += 1 + + def visit_Include(self, node, frame): + """Handles includes.""" + if node.with_context: + self.unoptimize_scope(frame) + if node.ignore_missing: + self.writeline('try:') + self.indent() + + func_name = 'get_or_select_template' + if isinstance(node.template, nodes.Const): + if isinstance(node.template.value, string_types): + func_name = 'get_template' + elif isinstance(node.template.value, (tuple, list)): + func_name = 'select_template' + elif isinstance(node.template, (nodes.Tuple, nodes.List)): + func_name = 'select_template' + + self.writeline('template = environment.%s(' % func_name, node) + self.visit(node.template, frame) + self.write(', %r)' % self.name) + if node.ignore_missing: + self.outdent() + self.writeline('except TemplateNotFound:') + self.indent() + self.writeline('pass') + self.outdent() + self.writeline('else:') + self.indent() + + if node.with_context: + self.writeline('include_context = template.new_context(' + 'context.parent, True, locals())') + self.writeline('for name, context_blocks in context.' + 'blocks.%s():' % dict_item_iter) + self.indent() + self.writeline('include_context.blocks.setdefault(' + 'name, [])[0:0] = context_blocks') + self.outdent() + self.writeline('for event in template.root_render_func(' + 'include_context):') + else: + self.writeline('for event in template.module._body_stream:') + + self.indent() + self.simple_write('event', frame) + self.outdent() + + if node.ignore_missing: + self.outdent() + + def visit_Import(self, node, frame): + """Visit regular imports.""" + if node.with_context: + self.unoptimize_scope(frame) + self.writeline('l_%s = ' % node.target, node) + if frame.toplevel: + self.write('context.vars[%r] = ' % node.target) + self.write('environment.get_template(') + self.visit(node.template, frame) + self.write(', %r).' % self.name) + if node.with_context: + self.write('make_module(context.parent, True, locals())') + else: + self.write('module') + if frame.toplevel and not node.target.startswith('_'): + self.writeline('context.exported_vars.discard(%r)' % node.target) + frame.assigned_names.add(node.target) + + def visit_FromImport(self, node, frame): + """Visit named imports.""" + self.newline(node) + self.write('included_template = environment.get_template(') + self.visit(node.template, frame) + self.write(', %r).' % self.name) + if node.with_context: + self.write('make_module(context.parent, True)') + else: + self.write('module') + + var_names = [] + discarded_names = [] + for name in node.names: + if isinstance(name, tuple): + name, alias = name + else: + alias = name + self.writeline('l_%s = getattr(included_template, ' + '%r, missing)' % (alias, name)) + self.writeline('if l_%s is missing:' % alias) + self.indent() + self.writeline('l_%s = environment.undefined(%r %% ' + 'included_template.__name__, ' + 'name=%r)' % + (alias, 'the template %%r (imported on %s) does ' + 'not export the requested name %s' % ( + self.position(node), + repr(name) + ), name)) + self.outdent() + if frame.toplevel: + var_names.append(alias) + if not alias.startswith('_'): + discarded_names.append(alias) + frame.assigned_names.add(alias) + + if var_names: + if len(var_names) == 1: + name = var_names[0] + self.writeline('context.vars[%r] = l_%s' % (name, name)) + else: + self.writeline('context.vars.update({%s})' % ', '.join( + '%r: l_%s' % (name, name) for name in var_names + )) + if discarded_names: + if len(discarded_names) == 1: + self.writeline('context.exported_vars.discard(%r)' % + discarded_names[0]) + else: + self.writeline('context.exported_vars.difference_' + 'update((%s))' % ', '.join(imap(repr, discarded_names))) + + def visit_For(self, node, frame): + # when calculating the nodes for the inner frame we have to exclude + # the iterator contents from it + children = node.iter_child_nodes(exclude=('iter',)) + if node.recursive: + loop_frame = self.function_scoping(node, frame, children, + find_special=False) + else: + loop_frame = frame.inner() + loop_frame.inspect(children) + + # try to figure out if we have an extended loop. An extended loop + # is necessary if the loop is in recursive mode if the special loop + # variable is accessed in the body. + extended_loop = node.recursive or 'loop' in \ + find_undeclared(node.iter_child_nodes( + only=('body',)), ('loop',)) + + # if we don't have an recursive loop we have to find the shadowed + # variables at that point. Because loops can be nested but the loop + # variable is a special one we have to enforce aliasing for it. + if not node.recursive: + aliases = self.push_scope(loop_frame, ('loop',)) + + # otherwise we set up a buffer and add a function def + else: + self.writeline('def loop(reciter, loop_render_func, depth=0):', node) + self.indent() + self.buffer(loop_frame) + aliases = {} + + # make sure the loop variable is a special one and raise a template + # assertion error if a loop tries to write to loop + if extended_loop: + self.writeline('l_loop = missing') + loop_frame.identifiers.add_special('loop') + for name in node.find_all(nodes.Name): + if name.ctx == 'store' and name.name == 'loop': + self.fail('Can\'t assign to special loop variable ' + 'in for-loop target', name.lineno) + + self.pull_locals(loop_frame) + if node.else_: + iteration_indicator = self.temporary_identifier() + self.writeline('%s = 1' % iteration_indicator) + + # Create a fake parent loop if the else or test section of a + # loop is accessing the special loop variable and no parent loop + # exists. + if 'loop' not in aliases and 'loop' in find_undeclared( + node.iter_child_nodes(only=('else_', 'test')), ('loop',)): + self.writeline("l_loop = environment.undefined(%r, name='loop')" % + ("'loop' is undefined. the filter section of a loop as well " + "as the else block don't have access to the special 'loop'" + " variable of the current loop. Because there is no parent " + "loop it's undefined. Happened in loop on %s" % + self.position(node))) + + self.writeline('for ', node) + self.visit(node.target, loop_frame) + self.write(extended_loop and ', l_loop in LoopContext(' or ' in ') + + # if we have an extened loop and a node test, we filter in the + # "outer frame". + if extended_loop and node.test is not None: + self.write('(') + self.visit(node.target, loop_frame) + self.write(' for ') + self.visit(node.target, loop_frame) + self.write(' in ') + if node.recursive: + self.write('reciter') + else: + self.visit(node.iter, loop_frame) + self.write(' if (') + test_frame = loop_frame.copy() + self.visit(node.test, test_frame) + self.write('))') + + elif node.recursive: + self.write('reciter') + else: + self.visit(node.iter, loop_frame) + + if node.recursive: + self.write(', loop_render_func, depth):') + else: + self.write(extended_loop and '):' or ':') + + # tests in not extended loops become a continue + if not extended_loop and node.test is not None: + self.indent() + self.writeline('if not ') + self.visit(node.test, loop_frame) + self.write(':') + self.indent() + self.writeline('continue') + self.outdent(2) + + self.indent() + self.blockvisit(node.body, loop_frame) + if node.else_: + self.writeline('%s = 0' % iteration_indicator) + self.outdent() + + if node.else_: + self.writeline('if %s:' % iteration_indicator) + self.indent() + self.blockvisit(node.else_, loop_frame) + self.outdent() + + # reset the aliases if there are any. + if not node.recursive: + self.pop_scope(aliases, loop_frame) + + # if the node was recursive we have to return the buffer contents + # and start the iteration code + if node.recursive: + self.return_buffer_contents(loop_frame) + self.outdent() + self.start_write(frame, node) + self.write('loop(') + self.visit(node.iter, frame) + self.write(', loop)') + self.end_write(frame) + + def visit_If(self, node, frame): + if_frame = frame.soft() + self.writeline('if ', node) + self.visit(node.test, if_frame) + self.write(':') + self.indent() + self.blockvisit(node.body, if_frame) + self.outdent() + if node.else_: + self.writeline('else:') + self.indent() + self.blockvisit(node.else_, if_frame) + self.outdent() + + def visit_Macro(self, node, frame): + macro_frame = self.macro_body(node, frame) + self.newline() + if frame.toplevel: + if not node.name.startswith('_'): + self.write('context.exported_vars.add(%r)' % node.name) + self.writeline('context.vars[%r] = ' % node.name) + self.write('l_%s = ' % node.name) + self.macro_def(node, macro_frame) + frame.assigned_names.add(node.name) + + def visit_CallBlock(self, node, frame): + children = node.iter_child_nodes(exclude=('call',)) + call_frame = self.macro_body(node, frame, children) + self.writeline('caller = ') + self.macro_def(node, call_frame) + self.start_write(frame, node) + self.visit_Call(node.call, call_frame, forward_caller=True) + self.end_write(frame) + + def visit_FilterBlock(self, node, frame): + filter_frame = frame.inner() + filter_frame.inspect(node.iter_child_nodes()) + aliases = self.push_scope(filter_frame) + self.pull_locals(filter_frame) + self.buffer(filter_frame) + self.blockvisit(node.body, filter_frame) + self.start_write(frame, node) + self.visit_Filter(node.filter, filter_frame) + self.end_write(frame) + self.pop_scope(aliases, filter_frame) + + def visit_ExprStmt(self, node, frame): + self.newline(node) + self.visit(node.node, frame) + + def visit_Output(self, node, frame): + # if we have a known extends statement, we don't output anything + # if we are in a require_output_check section + if self.has_known_extends and frame.require_output_check: + return + + if self.environment.finalize: + finalize = lambda x: text_type(self.environment.finalize(x)) + else: + finalize = text_type + + # if we are inside a frame that requires output checking, we do so + outdent_later = False + if frame.require_output_check: + self.writeline('if parent_template is None:') + self.indent() + outdent_later = True + + # try to evaluate as many chunks as possible into a static + # string at compile time. + body = [] + for child in node.nodes: + try: + const = child.as_const(frame.eval_ctx) + except nodes.Impossible: + body.append(child) + continue + # the frame can't be volatile here, becaus otherwise the + # as_const() function would raise an Impossible exception + # at that point. + try: + if frame.eval_ctx.autoescape: + if hasattr(const, '__html__'): + const = const.__html__() + else: + const = escape(const) + const = finalize(const) + except Exception: + # if something goes wrong here we evaluate the node + # at runtime for easier debugging + body.append(child) + continue + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + # if we have less than 3 nodes or a buffer we yield or extend/append + if len(body) < 3 or frame.buffer is not None: + if frame.buffer is not None: + # for one item we append, for more we extend + if len(body) == 1: + self.writeline('%s.append(' % frame.buffer) + else: + self.writeline('%s.extend((' % frame.buffer) + self.indent() + for item in body: + if isinstance(item, list): + val = repr(concat(item)) + if frame.buffer is None: + self.writeline('yield ' + val) + else: + self.writeline(val + ', ') + else: + if frame.buffer is None: + self.writeline('yield ', item) + else: + self.newline(item) + close = 1 + if frame.eval_ctx.volatile: + self.write('(context.eval_ctx.autoescape and' + ' escape or to_string)(') + elif frame.eval_ctx.autoescape: + self.write('escape(') + else: + self.write('to_string(') + if self.environment.finalize is not None: + self.write('environment.finalize(') + close += 1 + self.visit(item, frame) + self.write(')' * close) + if frame.buffer is not None: + self.write(', ') + if frame.buffer is not None: + # close the open parentheses + self.outdent() + self.writeline(len(body) == 1 and ')' or '))') + + # otherwise we create a format string as this is faster in that case + else: + format = [] + arguments = [] + for item in body: + if isinstance(item, list): + format.append(concat(item).replace('%', '%%')) + else: + format.append('%s') + arguments.append(item) + self.writeline('yield ') + self.write(repr(concat(format)) + ' % (') + idx = -1 + self.indent() + for argument in arguments: + self.newline(argument) + close = 0 + if frame.eval_ctx.volatile: + self.write('(context.eval_ctx.autoescape and' + ' escape or to_string)(') + close += 1 + elif frame.eval_ctx.autoescape: + self.write('escape(') + close += 1 + if self.environment.finalize is not None: + self.write('environment.finalize(') + close += 1 + self.visit(argument, frame) + self.write(')' * close + ', ') + self.outdent() + self.writeline(')') + + if outdent_later: + self.outdent() + + def visit_Assign(self, node, frame): + self.newline(node) + # toplevel assignments however go into the local namespace and + # the current template's context. We create a copy of the frame + # here and add a set so that the Name visitor can add the assigned + # names here. + if frame.toplevel: + assignment_frame = frame.copy() + assignment_frame.toplevel_assignments = set() + else: + assignment_frame = frame + self.visit(node.target, assignment_frame) + self.write(' = ') + self.visit(node.node, frame) + + # make sure toplevel assignments are added to the context. + if frame.toplevel: + public_names = [x for x in assignment_frame.toplevel_assignments + if not x.startswith('_')] + if len(assignment_frame.toplevel_assignments) == 1: + name = next(iter(assignment_frame.toplevel_assignments)) + self.writeline('context.vars[%r] = l_%s' % (name, name)) + else: + self.writeline('context.vars.update({') + for idx, name in enumerate(assignment_frame.toplevel_assignments): + if idx: + self.write(', ') + self.write('%r: l_%s' % (name, name)) + self.write('})') + if public_names: + if len(public_names) == 1: + self.writeline('context.exported_vars.add(%r)' % + public_names[0]) + else: + self.writeline('context.exported_vars.update((%s))' % + ', '.join(imap(repr, public_names))) + + # -- Expression Visitors + + def visit_Name(self, node, frame): + if node.ctx == 'store' and frame.toplevel: + frame.toplevel_assignments.add(node.name) + self.write('l_' + node.name) + frame.assigned_names.add(node.name) + + def visit_Const(self, node, frame): + val = node.value + if isinstance(val, float): + self.write(str(val)) + else: + self.write(repr(val)) + + def visit_TemplateData(self, node, frame): + try: + self.write(repr(node.as_const(frame.eval_ctx))) + except nodes.Impossible: + self.write('(context.eval_ctx.autoescape and Markup or identity)(%r)' + % node.data) + + def visit_Tuple(self, node, frame): + self.write('(') + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(idx == 0 and ',)' or ')') + + def visit_List(self, node, frame): + self.write('[') + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item, frame) + self.write(']') + + def visit_Dict(self, node, frame): + self.write('{') + for idx, item in enumerate(node.items): + if idx: + self.write(', ') + self.visit(item.key, frame) + self.write(': ') + self.visit(item.value, frame) + self.write('}') + + def binop(operator, interceptable=True): + def visitor(self, node, frame): + if self.environment.sandboxed and \ + operator in self.environment.intercepted_binops: + self.write('environment.call_binop(context, %r, ' % operator) + self.visit(node.left, frame) + self.write(', ') + self.visit(node.right, frame) + else: + self.write('(') + self.visit(node.left, frame) + self.write(' %s ' % operator) + self.visit(node.right, frame) + self.write(')') + return visitor + + def uaop(operator, interceptable=True): + def visitor(self, node, frame): + if self.environment.sandboxed and \ + operator in self.environment.intercepted_unops: + self.write('environment.call_unop(context, %r, ' % operator) + self.visit(node.node, frame) + else: + self.write('(' + operator) + self.visit(node.node, frame) + self.write(')') + return visitor + + visit_Add = binop('+') + visit_Sub = binop('-') + visit_Mul = binop('*') + visit_Div = binop('/') + visit_FloorDiv = binop('//') + visit_Pow = binop('**') + visit_Mod = binop('%') + visit_And = binop('and', interceptable=False) + visit_Or = binop('or', interceptable=False) + visit_Pos = uaop('+') + visit_Neg = uaop('-') + visit_Not = uaop('not ', interceptable=False) + del binop, uaop + + def visit_Concat(self, node, frame): + if frame.eval_ctx.volatile: + func_name = '(context.eval_ctx.volatile and' \ + ' markup_join or unicode_join)' + elif frame.eval_ctx.autoescape: + func_name = 'markup_join' + else: + func_name = 'unicode_join' + self.write('%s((' % func_name) + for arg in node.nodes: + self.visit(arg, frame) + self.write(', ') + self.write('))') + + def visit_Compare(self, node, frame): + self.visit(node.expr, frame) + for op in node.ops: + self.visit(op, frame) + + def visit_Operand(self, node, frame): + self.write(' %s ' % operators[node.op]) + self.visit(node.expr, frame) + + def visit_Getattr(self, node, frame): + self.write('environment.getattr(') + self.visit(node.node, frame) + self.write(', %r)' % node.attr) + + def visit_Getitem(self, node, frame): + # slices bypass the environment getitem method. + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write('[') + self.visit(node.arg, frame) + self.write(']') + else: + self.write('environment.getitem(') + self.visit(node.node, frame) + self.write(', ') + self.visit(node.arg, frame) + self.write(')') + + def visit_Slice(self, node, frame): + if node.start is not None: + self.visit(node.start, frame) + self.write(':') + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(':') + self.visit(node.step, frame) + + def visit_Filter(self, node, frame): + self.write(self.filters[node.name] + '(') + func = self.environment.filters.get(node.name) + if func is None: + self.fail('no filter named %r' % node.name, node.lineno) + if getattr(func, 'contextfilter', False): + self.write('context, ') + elif getattr(func, 'evalcontextfilter', False): + self.write('context.eval_ctx, ') + elif getattr(func, 'environmentfilter', False): + self.write('environment, ') + + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write('(context.eval_ctx.autoescape and' + ' Markup(concat(%s)) or concat(%s))' % + (frame.buffer, frame.buffer)) + elif frame.eval_ctx.autoescape: + self.write('Markup(concat(%s))' % frame.buffer) + else: + self.write('concat(%s)' % frame.buffer) + self.signature(node, frame) + self.write(')') + + def visit_Test(self, node, frame): + self.write(self.tests[node.name] + '(') + if node.name not in self.environment.tests: + self.fail('no test named %r' % node.name, node.lineno) + self.visit(node.node, frame) + self.signature(node, frame) + self.write(')') + + def visit_CondExpr(self, node, frame): + def write_expr2(): + if node.expr2 is not None: + return self.visit(node.expr2, frame) + self.write('environment.undefined(%r)' % ('the inline if-' + 'expression on %s evaluated to false and ' + 'no else section was defined.' % self.position(node))) + + self.write('(') + self.visit(node.expr1, frame) + self.write(' if ') + self.visit(node.test, frame) + self.write(' else ') + write_expr2() + self.write(')') + + def visit_Call(self, node, frame, forward_caller=False): + if self.environment.sandboxed: + self.write('environment.call(context, ') + else: + self.write('context.call(') + self.visit(node.node, frame) + extra_kwargs = forward_caller and {'caller': 'caller'} or None + self.signature(node, frame, extra_kwargs) + self.write(')') + + def visit_Keyword(self, node, frame): + self.write(node.key + '=') + self.visit(node.value, frame) + + # -- Unused nodes for extensions + + def visit_MarkSafe(self, node, frame): + self.write('Markup(') + self.visit(node.expr, frame) + self.write(')') + + def visit_MarkSafeIfAutoescape(self, node, frame): + self.write('(context.eval_ctx.autoescape and Markup or identity)(') + self.visit(node.expr, frame) + self.write(')') + + def visit_EnvironmentAttribute(self, node, frame): + self.write('environment.' + node.name) + + def visit_ExtensionAttribute(self, node, frame): + self.write('environment.extensions[%r].%s' % (node.identifier, node.name)) + + def visit_ImportedName(self, node, frame): + self.write(self.import_aliases[node.importname]) + + def visit_InternalName(self, node, frame): + self.write(node.name) + + def visit_ContextReference(self, node, frame): + self.write('context') + + def visit_Continue(self, node, frame): + self.writeline('continue', node) + + def visit_Break(self, node, frame): + self.writeline('break', node) + + def visit_Scope(self, node, frame): + scope_frame = frame.inner() + scope_frame.inspect(node.iter_child_nodes()) + aliases = self.push_scope(scope_frame) + self.pull_locals(scope_frame) + self.blockvisit(node.body, scope_frame) + self.pop_scope(aliases, scope_frame) + + def visit_EvalContextModifier(self, node, frame): + for keyword in node.options: + self.writeline('context.eval_ctx.%s = ' % keyword.key) + self.visit(keyword.value, frame) + try: + val = keyword.value.as_const(frame.eval_ctx) + except nodes.Impossible: + frame.eval_ctx.volatile = True + else: + setattr(frame.eval_ctx, keyword.key, val) + + def visit_ScopedEvalContextModifier(self, node, frame): + old_ctx_name = self.temporary_identifier() + safed_ctx = frame.eval_ctx.save() + self.writeline('%s = context.eval_ctx.save()' % old_ctx_name) + self.visit_EvalContextModifier(node, frame) + for child in node.body: + self.visit(child, frame) + frame.eval_ctx.revert(safed_ctx) + self.writeline('context.eval_ctx.revert(%s)' % old_ctx_name) diff --git a/modules/matlab/generator/jinja2/debug.py b/modules/matlab/generator/jinja2/debug.py new file mode 100644 index 000000000..dc8a39825 --- /dev/null +++ b/modules/matlab/generator/jinja2/debug.py @@ -0,0 +1,337 @@ +# -*- coding: utf-8 -*- +""" + jinja2.debug + ~~~~~~~~~~~~ + + Implements the debug interface for Jinja. This module does some pretty + ugly stuff with the Python traceback system in order to achieve tracebacks + with correct line numbers, locals and contents. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import sys +import traceback +from types import TracebackType, CodeType +from jinja2.utils import missing, internal_code +from jinja2.exceptions import TemplateSyntaxError +from jinja2._compat import iteritems, reraise + +# on pypy we can take advantage of transparent proxies +try: + from __pypy__ import tproxy +except ImportError: + tproxy = None + + +# how does the raise helper look like? +try: + exec("raise TypeError, 'foo'") +except SyntaxError: + raise_helper = 'raise __jinja_exception__[1]' +except TypeError: + raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]' + + +class TracebackFrameProxy(object): + """Proxies a traceback frame.""" + + def __init__(self, tb): + self.tb = tb + self._tb_next = None + + @property + def tb_next(self): + return self._tb_next + + def set_next(self, next): + if tb_set_next is not None: + try: + tb_set_next(self.tb, next and next.tb or None) + except Exception: + # this function can fail due to all the hackery it does + # on various python implementations. We just catch errors + # down and ignore them if necessary. + pass + self._tb_next = next + + @property + def is_jinja_frame(self): + return '__jinja_template__' in self.tb.tb_frame.f_globals + + def __getattr__(self, name): + return getattr(self.tb, name) + + +def make_frame_proxy(frame): + proxy = TracebackFrameProxy(frame) + if tproxy is None: + return proxy + def operation_handler(operation, *args, **kwargs): + if operation in ('__getattribute__', '__getattr__'): + return getattr(proxy, args[0]) + elif operation == '__setattr__': + proxy.__setattr__(*args, **kwargs) + else: + return getattr(proxy, operation)(*args, **kwargs) + return tproxy(TracebackType, operation_handler) + + +class ProcessedTraceback(object): + """Holds a Jinja preprocessed traceback for printing or reraising.""" + + def __init__(self, exc_type, exc_value, frames): + assert frames, 'no frames for this traceback?' + self.exc_type = exc_type + self.exc_value = exc_value + self.frames = frames + + # newly concatenate the frames (which are proxies) + prev_tb = None + for tb in self.frames: + if prev_tb is not None: + prev_tb.set_next(tb) + prev_tb = tb + prev_tb.set_next(None) + + def render_as_text(self, limit=None): + """Return a string with the traceback.""" + lines = traceback.format_exception(self.exc_type, self.exc_value, + self.frames[0], limit=limit) + return ''.join(lines).rstrip() + + def render_as_html(self, full=False): + """Return a unicode string with the traceback as rendered HTML.""" + from jinja2.debugrenderer import render_traceback + return u'%s\n\n' % ( + render_traceback(self, full=full), + self.render_as_text().decode('utf-8', 'replace') + ) + + @property + def is_template_syntax_error(self): + """`True` if this is a template syntax error.""" + return isinstance(self.exc_value, TemplateSyntaxError) + + @property + def exc_info(self): + """Exception info tuple with a proxy around the frame objects.""" + return self.exc_type, self.exc_value, self.frames[0] + + @property + def standard_exc_info(self): + """Standard python exc_info for re-raising""" + tb = self.frames[0] + # the frame will be an actual traceback (or transparent proxy) if + # we are on pypy or a python implementation with support for tproxy + if type(tb) is not TracebackType: + tb = tb.tb + return self.exc_type, self.exc_value, tb + + +def make_traceback(exc_info, source_hint=None): + """Creates a processed traceback object from the exc_info.""" + exc_type, exc_value, tb = exc_info + if isinstance(exc_value, TemplateSyntaxError): + exc_info = translate_syntax_error(exc_value, source_hint) + initial_skip = 0 + else: + initial_skip = 1 + return translate_exception(exc_info, initial_skip) + + +def translate_syntax_error(error, source=None): + """Rewrites a syntax error to please traceback systems.""" + error.source = source + error.translated = True + exc_info = (error.__class__, error, None) + filename = error.filename + if filename is None: + filename = '' + return fake_exc_info(exc_info, filename, error.lineno) + + +def translate_exception(exc_info, initial_skip=0): + """If passed an exc_info it will automatically rewrite the exceptions + all the way down to the correct line numbers and frames. + """ + tb = exc_info[2] + frames = [] + + # skip some internal frames if wanted + for x in range(initial_skip): + if tb is not None: + tb = tb.tb_next + initial_tb = tb + + while tb is not None: + # skip frames decorated with @internalcode. These are internal + # calls we can't avoid and that are useless in template debugging + # output. + if tb.tb_frame.f_code in internal_code: + tb = tb.tb_next + continue + + # save a reference to the next frame if we override the current + # one with a faked one. + next = tb.tb_next + + # fake template exceptions + template = tb.tb_frame.f_globals.get('__jinja_template__') + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + tb = fake_exc_info(exc_info[:2] + (tb,), template.filename, + lineno)[2] + + frames.append(make_frame_proxy(tb)) + tb = next + + # if we don't have any exceptions in the frames left, we have to + # reraise it unchanged. + # XXX: can we backup here? when could this happen? + if not frames: + reraise(exc_info[0], exc_info[1], exc_info[2]) + + return ProcessedTraceback(exc_info[0], exc_info[1], frames) + + +def fake_exc_info(exc_info, filename, lineno): + """Helper for `translate_exception`.""" + exc_type, exc_value, tb = exc_info + + # figure the real context out + if tb is not None: + real_locals = tb.tb_frame.f_locals.copy() + ctx = real_locals.get('context') + if ctx: + locals = ctx.get_all() + else: + locals = {} + for name, value in iteritems(real_locals): + if name.startswith('l_') and value is not missing: + locals[name[2:]] = value + + # if there is a local called __jinja_exception__, we get + # rid of it to not break the debug functionality. + locals.pop('__jinja_exception__', None) + else: + locals = {} + + # assamble fake globals we need + globals = { + '__name__': filename, + '__file__': filename, + '__jinja_exception__': exc_info[:2], + + # we don't want to keep the reference to the template around + # to not cause circular dependencies, but we mark it as Jinja + # frame for the ProcessedTraceback + '__jinja_template__': None + } + + # and fake the exception + code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec') + + # if it's possible, change the name of the code. This won't work + # on some python environments such as google appengine + try: + if tb is None: + location = 'template' + else: + function = tb.tb_frame.f_code.co_name + if function == 'root': + location = 'top-level template code' + elif function.startswith('block_'): + location = 'block "%s"' % function[6:] + else: + location = 'template' + code = CodeType(0, code.co_nlocals, code.co_stacksize, + code.co_flags, code.co_code, code.co_consts, + code.co_names, code.co_varnames, filename, + location, code.co_firstlineno, + code.co_lnotab, (), ()) + except: + pass + + # execute the code and catch the new traceback + try: + exec(code, globals, locals) + except: + exc_info = sys.exc_info() + new_tb = exc_info[2].tb_next + + # return without this frame + return exc_info[:2] + (new_tb,) + + +def _init_ugly_crap(): + """This function implements a few ugly things so that we can patch the + traceback objects. The function returned allows resetting `tb_next` on + any python traceback object. Do not attempt to use this on non cpython + interpreters + """ + import ctypes + from types import TracebackType + + # figure out side of _Py_ssize_t + if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'): + _Py_ssize_t = ctypes.c_int64 + else: + _Py_ssize_t = ctypes.c_int + + # regular python + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + # python with trace + if hasattr(sys, 'getobjects'): + class _PyObject(ctypes.Structure): + pass + _PyObject._fields_ = [ + ('_ob_next', ctypes.POINTER(_PyObject)), + ('_ob_prev', ctypes.POINTER(_PyObject)), + ('ob_refcnt', _Py_ssize_t), + ('ob_type', ctypes.POINTER(_PyObject)) + ] + + class _Traceback(_PyObject): + pass + _Traceback._fields_ = [ + ('tb_next', ctypes.POINTER(_Traceback)), + ('tb_frame', ctypes.POINTER(_PyObject)), + ('tb_lasti', ctypes.c_int), + ('tb_lineno', ctypes.c_int) + ] + + def tb_set_next(tb, next): + """Set the tb_next attribute of a traceback object.""" + if not (isinstance(tb, TracebackType) and + (next is None or isinstance(next, TracebackType))): + raise TypeError('tb_set_next arguments must be traceback objects') + obj = _Traceback.from_address(id(tb)) + if tb.tb_next is not None: + old = _Traceback.from_address(id(tb.tb_next)) + old.ob_refcnt -= 1 + if next is None: + obj.tb_next = ctypes.POINTER(_Traceback)() + else: + next = _Traceback.from_address(id(next)) + next.ob_refcnt += 1 + obj.tb_next = ctypes.pointer(next) + + return tb_set_next + + +# try to get a tb_set_next implementation if we don't have transparent +# proxies. +tb_set_next = None +if tproxy is None: + try: + tb_set_next = _init_ugly_crap() + except: + pass + del _init_ugly_crap diff --git a/modules/matlab/generator/jinja2/defaults.py b/modules/matlab/generator/jinja2/defaults.py new file mode 100644 index 000000000..4ab9390b3 --- /dev/null +++ b/modules/matlab/generator/jinja2/defaults.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +""" + jinja2.defaults + ~~~~~~~~~~~~~~~ + + Jinja default filters and tags. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +from jinja2._compat import range_type +from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner + + +# defaults for the parser / lexer +BLOCK_START_STRING = '{%' +BLOCK_END_STRING = '%}' +VARIABLE_START_STRING = '{{' +VARIABLE_END_STRING = '}}' +COMMENT_START_STRING = '{#' +COMMENT_END_STRING = '#}' +LINE_STATEMENT_PREFIX = None +LINE_COMMENT_PREFIX = None +TRIM_BLOCKS = False +LSTRIP_BLOCKS = False +NEWLINE_SEQUENCE = '\n' +KEEP_TRAILING_NEWLINE = False + + +# default filters, tests and namespace +from jinja2.filters import FILTERS as DEFAULT_FILTERS +DEFAULT_NAMESPACE = { + 'range': range_type, + 'dict': lambda **kw: kw, + 'lipsum': generate_lorem_ipsum, + 'cycler': Cycler, + 'joiner': Joiner +} + + +# export all constants +__all__ = tuple(x for x in locals().keys() if x.isupper()) diff --git a/modules/matlab/generator/jinja2/environment.py b/modules/matlab/generator/jinja2/environment.py new file mode 100644 index 000000000..f25aaf413 --- /dev/null +++ b/modules/matlab/generator/jinja2/environment.py @@ -0,0 +1,1190 @@ +# -*- coding: utf-8 -*- +""" + jinja2.environment + ~~~~~~~~~~~~~~~~~~ + + Provides a class that holds runtime and parsing time options. + + :copyright: (c) 2010 by the Jinja Team. + :license: BSD, see LICENSE for more details. +""" +import os +import sys +from jinja2 import nodes +from jinja2.defaults import BLOCK_START_STRING, \ + BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ + COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ + LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ + DEFAULT_FILTERS, DEFAULT_NAMESPACE, \ + KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS +from jinja2.lexer import get_lexer, TokenStream +from jinja2.parser import Parser +from jinja2.nodes import EvalContext +from jinja2.optimizer import optimize +from jinja2.compiler import generate +from jinja2.runtime import Undefined, new_context +from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ + TemplatesNotFound, TemplateRuntimeError +from jinja2.utils import import_string, LRUCache, Markup, missing, \ + concat, consume, internalcode +from jinja2._compat import imap, ifilter, string_types, iteritems, \ + text_type, reraise, implements_iterator, implements_to_string, \ + get_next, encode_filename, PY2, PYPY +from functools import reduce + + +# for direct template usage we have up to ten living environments +_spontaneous_environments = LRUCache(10) + +# the function to create jinja traceback objects. This is dynamically +# imported on the first exception in the exception handler. +_make_traceback = None + + +def get_spontaneous_environment(*args): + """Return a new spontaneous environment. A spontaneous environment is an + unnamed and unaccessible (in theory) environment that is used for + templates generated from a string and not from the file system. + """ + try: + env = _spontaneous_environments.get(args) + except TypeError: + return Environment(*args) + if env is not None: + return env + _spontaneous_environments[args] = env = Environment(*args) + env.shared = True + return env + + +def create_cache(size): + """Return the cache class for the given size.""" + if size == 0: + return None + if size < 0: + return {} + return LRUCache(size) + + +def copy_cache(cache): + """Create an empty copy of the given cache.""" + if cache is None: + return None + elif type(cache) is dict: + return {} + return LRUCache(cache.capacity) + + +def load_extensions(environment, extensions): + """Load the extensions from the list and bind it to the environment. + Returns a dict of instantiated environments. + """ + result = {} + for extension in extensions: + if isinstance(extension, string_types): + extension = import_string(extension) + result[extension.identifier] = extension(environment) + return result + + +def _environment_sanity_check(environment): + """Perform a sanity check on the environment.""" + assert issubclass(environment.undefined, Undefined), 'undefined must ' \ + 'be a subclass of undefined because filters depend on it.' + assert environment.block_start_string != \ + environment.variable_start_string != \ + environment.comment_start_string, 'block, variable and comment ' \ + 'start strings must be different' + assert environment.newline_sequence in ('\r', '\r\n', '\n'), \ + 'newline_sequence set to unknown line ending string.' + return environment + + +class Environment(object): + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here the possible initialization parameters: + + `block_start_string` + The string marking the begin of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the begin of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the begin of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + based comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation `. + + `optimized` + should the optimizer be enabled? Default is `True`. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that can be used to process the result of a variable + expression before it is output. For example one can convert + `None` implicitly into an empty string here. + + `autoescape` + If set to true the XML/HTML autoescaping feature is enabled by + default. For more details about auto escaping see + :class:`~jinja2.utils.Markup`. As of Jinja 2.4 this can also + be a callable that is passed the template name and has to + return `True` or `False` depending on autoescape should be + enabled by default. + + .. versionchanged:: 2.4 + `autoescape` can now be a function + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``50`` which means + that if more than 50 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + `auto_reload` is set to `True` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. + sandboxed = False + + #: True if the environment is just an overlay + overlayed = False + + #: the environment this environment is linked to if it is an overlay + linked_to = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + #: these are currently EXPERIMENTAL undocumented features. + exception_handler = None + exception_formatter = None + + def __init__(self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=50, + auto_reload=True, + bytecode_cache=None): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneous environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix + self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks + self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline + + # runtime information + self.undefined = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # load extensions + self.extensions = load_extensions(self, extensions) + + _environment_sanity_check(self) + + def add_extension(self, extension): + """Adds an extension after the environment was created. + + .. versionadded:: 2.5 + """ + self.extensions.update(load_extensions(self, [extension])) + + def extend(self, **attributes): + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions ` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in iteritems(attributes): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay(self, block_start_string=missing, block_end_string=missing, + variable_start_string=missing, variable_end_string=missing, + comment_start_string=missing, comment_end_string=missing, + line_statement_prefix=missing, line_comment_prefix=missing, + trim_blocks=missing, lstrip_blocks=missing, + extensions=missing, optimized=missing, + undefined=missing, finalize=missing, autoescape=missing, + loader=missing, cache_size=missing, auto_reload=missing, + bytecode_cache=missing): + """Create a new overlay environment that shares all the data with the + current environment except of cache and the overridden attributes. + Extensions cannot be removed for an overlayed environment. An overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + """ + args = dict(locals()) + del args['self'], args['cache_size'], args['extensions'] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlayed = True + rv.linked_to = self + + for key, value in iteritems(args): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in iteritems(self.extensions): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(rv, extensions)) + + return _environment_sanity_check(rv) + + lexer = property(get_lexer, doc="The lexer for this environment.") + + def iter_extensions(self): + """Iterates over the extensions by priority.""" + return iter(sorted(self.extensions.values(), + key=lambda x: x.priority)) + + def getitem(self, obj, argument): + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (TypeError, LookupError): + if isinstance(argument, string_types): + try: + attr = str(argument) + except Exception: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a bytestring. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def call_filter(self, name, value, args=None, kwargs=None, + context=None, eval_ctx=None): + """Invokes a filter on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + raise TemplateRuntimeError('no filter named %r' % name) + args = list(args or ()) + if getattr(func, 'contextfilter', False): + if context is None: + raise TemplateRuntimeError('Attempted to invoke context ' + 'filter without context') + args.insert(0, context) + elif getattr(func, 'evalcontextfilter', False): + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, 'environmentfilter', False): + args.insert(0, self) + return func(value, *args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + raise TemplateRuntimeError('no test named %r' % name) + return func(value, *(args or ()), **(kwargs or {})) + + @internalcode + def parse(self, source, name=None, filename=None): + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja2 extensions ` + this gives you a good overview of the node tree generated. + """ + try: + return self._parse(source, name, filename) + except TemplateSyntaxError: + exc_info = sys.exc_info() + self.handle_exception(exc_info, source_hint=source) + + def _parse(self, source, name, filename): + """Internal parsing function used by `parse` and `compile`.""" + return Parser(self, source, name, encode_filename(filename)).parse() + + def lex(self, source, name=None, filename=None): + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development ` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = text_type(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError: + exc_info = sys.exc_info() + self.handle_exception(exc_info, source_hint=source) + + def preprocess(self, source, name=None, filename=None): + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce(lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), text_type(source)) + + def _tokenize(self, source, name, filename=None, state=None): + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.iter_extensions(): + stream = ext.filter_stream(stream) + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + return stream + + def _generate(self, source, name, filename, defer_init=False): + """Internal hook that can be overridden to hook a different generate + method in. + + .. versionadded:: 2.5 + """ + return generate(source, self, name, filename, defer_init=defer_init) + + def _compile(self, source, filename): + """Internal hook that can be overridden to hook a different compile + method in. + + .. versionadded:: 2.5 + """ + return compile(source, filename, 'exec') + + @internalcode + def compile(self, source, name=None, filename=None, raw=False, + defer_init=False): + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. + """ + source_hint = None + try: + if isinstance(source, string_types): + source_hint = source + source = self._parse(source, name, filename) + if self.optimized: + source = optimize(source, self) + source = self._generate(source, name, filename, + defer_init=defer_init) + if raw: + return source + if filename is None: + filename = '