Added simple batch build script for CMake.

devtools/agent_vmw7.json

@@ -0,0 +1,33 @@
+{
+    "cmake_variants" : [
+        {"name": "generator",
+         "generators": [
+            {"generator": [
+                "Visual Studio 7 .NET 2003",
+                "Visual Studio 9 2008",
+                "Visual Studio 9 2008 Win64",
+                "Visual Studio 10",
+                "Visual Studio 10 Win64",
+                "Visual Studio 11",
+                "Visual Studio 11 Win64"
+                ] 
+            },
+            {"generator": ["MinGW Makefiles"], 
+             "env_prepend": [{"path": "c:/wut/prg/MinGW/bin"}]
+            }
+         ] 
+        },
+        {"name": "shared_dll",
+         "variables": [
+            ["JSONCPP_LIB_BUILD_SHARED=true"],
+            ["JSONCPP_LIB_BUILD_SHARED=false"]
+          ]
+        },
+        {"name": "build_type",
+         "build_types": [
+            "debug", 
+            "release" 
+            ]
+        }
+    ]
+}

devtools/batchbuild.py

@@ -0,0 +1,280 @@
+import collections
+import itertools
+import json
+import os
+import os.path
+import re
+import shutil
+import string
+import subprocess
+import sys
+import cgi
+
+class BuildDesc:
+    def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None):
+        self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ]
+        self.variables = variables or []
+        self.build_type = build_type
+        self.generator = generator
+
+    def merged_with( self, build_desc ):
+        """Returns a new BuildDesc by merging field content.
+           Prefer build_desc fields to self fields for single valued field.
+        """
+        return BuildDesc( self.prepend_envs + build_desc.prepend_envs,
+                          self.variables + build_desc.variables,
+                          build_desc.build_type or self.build_type,
+                          build_desc.generator or self.generator )
+
+    def env( self ):
+        environ = os.environ.copy()
+        for values_by_name in self.prepend_envs:
+            for var, value in values_by_name.items():
+                var = var.upper()
+                if type(value) is unicode:
+                    value = value.encode( sys.getdefaultencoding() )
+                if var in environ:
+                    environ[var] = value + os.pathsep + environ[var]
+                else:
+                    environ[var] = value
+        return environ
+
+    def cmake_args( self ):
+        args = ["-D%s" % var for var in self.variables]
+        # skip build type for Visual Studio solution as it cause warning
+        if self.build_type and 'Visual' not in self.generator:
+            args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type )
+        if self.generator:
+            args.extend( ['-G', self.generator] )
+        return args
+
+    def __repr__( self ):
+        return "BuildDesc( %s, build_type=%s )" %  (" ".join( self.cmake_args()), self.build_type)
+
+class BuildData:
+    def __init__( self, desc, work_dir, source_dir ):
+        self.desc = desc
+        self.work_dir = work_dir
+        self.source_dir = source_dir
+        self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' )
+        self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' )
+        self.cmake_succeeded = False
+        self.build_succeeded = False
+
+    def execute_build(self):
+        print 'Build %s' % self.desc
+        self._make_new_work_dir( )
+        self.cmake_succeeded = self._generate_makefiles( )
+        if self.cmake_succeeded:
+            self.build_succeeded = self._build_using_makefiles( )
+        return self.build_succeeded
+
+    def _generate_makefiles(self):
+        print '  Generating makefiles: ',
+        cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )]
+        succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path )
+        print 'done' if succeeded else 'FAILED'
+        return succeeded
+
+    def _build_using_makefiles(self):
+        print '  Building:',
+        cmd = ['cmake', '--build', self.work_dir]
+        if self.desc.build_type:
+            cmd += ['--config', self.desc.build_type]
+        succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path )
+        print 'done' if succeeded else 'FAILED'
+        return succeeded
+
+    def _execute_build_subprocess(self, cmd, env, log_path):
+        process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir,
+                                    env=env )
+        stdout, _ = process.communicate( )
+        succeeded = (process.returncode == 0)
+        with open( log_path, 'wb' ) as flog:
+            log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode
+            flog.write( fix_eol( log ) )
+        return succeeded
+
+    def _make_new_work_dir(self):
+        if os.path.isdir( self.work_dir ):
+            print '  Removing work directory', self.work_dir
+            shutil.rmtree( self.work_dir, ignore_errors=True )
+        if not os.path.isdir( self.work_dir ):
+            os.makedirs( self.work_dir )
+
+def fix_eol( stdout ):
+    """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n).
+    """
+    return re.sub( '\r*\n', os.linesep, stdout )
+
+def load_build_variants_from_config( config_path ):
+    with open( config_path, 'rb' ) as fconfig:
+        data = json.load( fconfig )
+    variants = data[ 'cmake_variants' ]
+    build_descs_by_axis = collections.defaultdict( list )
+    for axis in variants:
+        axis_name = axis["name"]
+        build_descs = []
+        if "generators" in axis:
+            for generator_data in axis["generators"]:
+                for generator in generator_data["generator"]:
+                    build_desc = BuildDesc( generator=generator,
+                                            prepend_envs=generator_data.get("env_prepend") )
+                    build_descs.append( build_desc )
+        elif "variables" in axis:
+            for variables in axis["variables"]:
+                build_desc = BuildDesc( variables=variables )
+                build_descs.append( build_desc )
+        elif "build_types" in axis:
+            for build_type in axis["build_types"]:
+                build_desc = BuildDesc( build_type=build_type )
+                build_descs.append( build_desc )
+        build_descs_by_axis[axis_name].extend( build_descs )
+    return build_descs_by_axis
+
+def generate_build_variants( build_descs_by_axis ):
+    """Returns a list of BuildDesc generated for the partial BuildDesc for each axis."""
+    axis_names = build_descs_by_axis.keys()
+    build_descs = []
+    for axis_name, axis_build_descs in build_descs_by_axis.items():
+        if len(build_descs):
+            # for each existing build_desc and each axis build desc, create a new build_desc
+            new_build_descs = []
+            for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs):
+                new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) )
+            build_descs = new_build_descs
+        else:
+            build_descs = axis_build_descs
+    return build_descs
+
+HTML_TEMPLATE = string.Template('''<html>
+<head>
+    <title>$title</title>
+    <style type="text/css">
+    td.failed {background-color:#f08080;}
+    td.ok {background-color:#c0eec0;}
+    </style>
+</head>
+<body>
+<table border="1">
+<thead>
+    <tr>
+        <th>Variables</th>
+        $th_vars
+    </tr>
+    <tr>
+        <th>Build type</th>
+        $th_build_types
+    </tr>
+</thead>
+<tbody>
+$tr_builds
+</tbody>
+</table>
+</body></html>''')
+
+def generate_html_report( html_report_path, builds ):
+    report_dir = os.path.dirname( html_report_path )
+    # Vertical axis: generator
+    # Horizontal: variables, then build_type
+    builds_by_generator = collections.defaultdict( list )
+    variables = set()
+    build_types_by_variable = collections.defaultdict( set )
+    build_by_pos_key = {} # { (generator, var_key, build_type): build }
+    for build in builds:
+        builds_by_generator[build.desc.generator].append( build )
+        var_key = tuple(sorted(build.desc.variables))
+        variables.add( var_key )
+        build_types_by_variable[var_key].add( build.desc.build_type )
+        pos_key = (build.desc.generator, var_key, build.desc.build_type)
+        build_by_pos_key[pos_key] = build
+    variables = sorted( variables )
+    th_vars = []
+    th_build_types = []
+    for variable in variables:
+        build_types = sorted( build_types_by_variable[variable] )
+        nb_build_type = len(build_types_by_variable[variable])
+        th_vars.append( '<th colspan="%d">%s</th>' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) )
+        for build_type in build_types:
+            th_build_types.append( '<th>%s</th>' % cgi.escape(build_type) )
+    tr_builds = []
+    for generator in sorted( builds_by_generator ):
+        tds = [ '<td>%s</td>\n' % cgi.escape( generator ) ]
+        for variable in variables:
+            build_types = sorted( build_types_by_variable[variable] )
+            for build_type in build_types:
+                pos_key = (generator, variable, build_type)
+                build = build_by_pos_key.get(pos_key)
+                if build:
+                    cmake_status = 'ok' if build.cmake_succeeded else 'FAILED'
+                    build_status = 'ok' if build.build_succeeded else 'FAILED'
+                    cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir )
+                    build_log_url = os.path.relpath( build.build_log_path, report_dir )
+                    td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % (
+                        build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status)
+                    if build.cmake_succeeded:
+                        td += '<br><a href="%s" class="%s">Build: %s</a>' % (
+                            build_log_url, build_status.lower(), build_status)
+                    td += '</td>'
+                else:
+                    td = '<td></td>'
+                tds.append( td )
+        tr_builds.append( '<tr>%s</tr>' % '\n'.join( tds ) )
+    html = HTML_TEMPLATE.substitute(
+        title='Batch build report',
+        th_vars=' '.join(th_vars),
+        th_build_types=' '.join( th_build_types),
+        tr_builds='\n'.join( tr_builds ) )
+    with open( html_report_path, 'wt' ) as fhtml:
+        fhtml.write( html )
+    print 'HTML report generated in:', html_report_path
+
+def main():
+    usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...]
+Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run
+as described in CONFIG_JSON_PATH building in WORK_DIR.
+
+Example of call:
+python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json
+"""
+    from optparse import OptionParser
+    parser = OptionParser(usage=usage)
+    parser.allow_interspersed_args = True
+#    parser.add_option('-v', '--verbose', dest="verbose", action='store_true',
+#        help="""Be verbose.""")
+    parser.enable_interspersed_args()
+    options, args = parser.parse_args()
+    if len(args) < 3:
+        parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." )
+    work_dir = args[0]
+    source_dir = args[1].rstrip('/\\')
+    config_paths = args[2:]
+    for config_path in config_paths:
+        if not os.path.isfile( config_path ):
+            parser.error( "Can not read: %r" % config_path )
+
+    # generate build variants
+    build_descs = []
+    for config_path in config_paths:
+        build_descs_by_axis = load_build_variants_from_config( config_path )
+        build_descs.extend( generate_build_variants( build_descs_by_axis ) )
+    print 'Build variants (%d):' % len(build_descs)
+    # assign build directory for each variant
+    if not os.path.isdir( work_dir ):
+        os.makedirs( work_dir )
+    builds = []
+    with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap:
+        for index, build_desc in enumerate( build_descs ):
+            build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) )
+            builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) )
+            fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) )
+    for build in builds:
+        build.execute_build()
+    html_report_path = os.path.join( work_dir, 'batchbuild-report.html' )
+    generate_html_report( html_report_path, builds )
+    print 'Done'
+        
+
+if __name__ == '__main__':
+    main()
+