This commit is contained in:
Christopher Dunn 2014-11-19 23:34:15 -06:00
parent bd1e895287
commit 433876866d

View File

@ -1,280 +1,280 @@
import collections import collections
import itertools import itertools
import json import json
import os import os
import os.path import os.path
import re import re
import shutil import shutil
import string import string
import subprocess import subprocess
import sys import sys
import cgi import cgi
class BuildDesc: class BuildDesc:
def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None): def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None):
self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ] self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ]
self.variables = variables or [] self.variables = variables or []
self.build_type = build_type self.build_type = build_type
self.generator = generator self.generator = generator
def merged_with( self, build_desc ): def merged_with( self, build_desc ):
"""Returns a new BuildDesc by merging field content. """Returns a new BuildDesc by merging field content.
Prefer build_desc fields to self fields for single valued field. Prefer build_desc fields to self fields for single valued field.
""" """
return BuildDesc( self.prepend_envs + build_desc.prepend_envs, return BuildDesc( self.prepend_envs + build_desc.prepend_envs,
self.variables + build_desc.variables, self.variables + build_desc.variables,
build_desc.build_type or self.build_type, build_desc.build_type or self.build_type,
build_desc.generator or self.generator ) build_desc.generator or self.generator )
def env( self ): def env( self ):
environ = os.environ.copy() environ = os.environ.copy()
for values_by_name in self.prepend_envs: for values_by_name in self.prepend_envs:
for var, value in values_by_name.items(): for var, value in values_by_name.items():
var = var.upper() var = var.upper()
if type(value) is unicode: if type(value) is unicode:
value = value.encode( sys.getdefaultencoding() ) value = value.encode( sys.getdefaultencoding() )
if var in environ: if var in environ:
environ[var] = value + os.pathsep + environ[var] environ[var] = value + os.pathsep + environ[var]
else: else:
environ[var] = value environ[var] = value
return environ return environ
def cmake_args( self ): def cmake_args( self ):
args = ["-D%s" % var for var in self.variables] args = ["-D%s" % var for var in self.variables]
# skip build type for Visual Studio solution as it cause warning # skip build type for Visual Studio solution as it cause warning
if self.build_type and 'Visual' not in self.generator: if self.build_type and 'Visual' not in self.generator:
args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type ) args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type )
if self.generator: if self.generator:
args.extend( ['-G', self.generator] ) args.extend( ['-G', self.generator] )
return args return args
def __repr__( self ): def __repr__( self ):
return "BuildDesc( %s, build_type=%s )" % (" ".join( self.cmake_args()), self.build_type) return "BuildDesc( %s, build_type=%s )" % (" ".join( self.cmake_args()), self.build_type)
class BuildData: class BuildData:
def __init__( self, desc, work_dir, source_dir ): def __init__( self, desc, work_dir, source_dir ):
self.desc = desc self.desc = desc
self.work_dir = work_dir self.work_dir = work_dir
self.source_dir = source_dir self.source_dir = source_dir
self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' ) self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' )
self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' ) self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' )
self.cmake_succeeded = False self.cmake_succeeded = False
self.build_succeeded = False self.build_succeeded = False
def execute_build(self): def execute_build(self):
print 'Build %s' % self.desc print 'Build %s' % self.desc
self._make_new_work_dir( ) self._make_new_work_dir( )
self.cmake_succeeded = self._generate_makefiles( ) self.cmake_succeeded = self._generate_makefiles( )
if self.cmake_succeeded: if self.cmake_succeeded:
self.build_succeeded = self._build_using_makefiles( ) self.build_succeeded = self._build_using_makefiles( )
return self.build_succeeded return self.build_succeeded
def _generate_makefiles(self): def _generate_makefiles(self):
print ' Generating makefiles: ', print ' Generating makefiles: ',
cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )] cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )]
succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path ) succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path )
print 'done' if succeeded else 'FAILED' print 'done' if succeeded else 'FAILED'
return succeeded return succeeded
def _build_using_makefiles(self): def _build_using_makefiles(self):
print ' Building:', print ' Building:',
cmd = ['cmake', '--build', self.work_dir] cmd = ['cmake', '--build', self.work_dir]
if self.desc.build_type: if self.desc.build_type:
cmd += ['--config', self.desc.build_type] cmd += ['--config', self.desc.build_type]
succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path ) succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path )
print 'done' if succeeded else 'FAILED' print 'done' if succeeded else 'FAILED'
return succeeded return succeeded
def _execute_build_subprocess(self, cmd, env, log_path): def _execute_build_subprocess(self, cmd, env, log_path):
process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir, process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir,
env=env ) env=env )
stdout, _ = process.communicate( ) stdout, _ = process.communicate( )
succeeded = (process.returncode == 0) succeeded = (process.returncode == 0)
with open( log_path, 'wb' ) as flog: with open( log_path, 'wb' ) as flog:
log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode
flog.write( fix_eol( log ) ) flog.write( fix_eol( log ) )
return succeeded return succeeded
def _make_new_work_dir(self): def _make_new_work_dir(self):
if os.path.isdir( self.work_dir ): if os.path.isdir( self.work_dir ):
print ' Removing work directory', self.work_dir print ' Removing work directory', self.work_dir
shutil.rmtree( self.work_dir, ignore_errors=True ) shutil.rmtree( self.work_dir, ignore_errors=True )
if not os.path.isdir( self.work_dir ): if not os.path.isdir( self.work_dir ):
os.makedirs( self.work_dir ) os.makedirs( self.work_dir )
def fix_eol( stdout ): def fix_eol( stdout ):
"""Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n). """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n).
""" """
return re.sub( '\r*\n', os.linesep, stdout ) return re.sub( '\r*\n', os.linesep, stdout )
def load_build_variants_from_config( config_path ): def load_build_variants_from_config( config_path ):
with open( config_path, 'rb' ) as fconfig: with open( config_path, 'rb' ) as fconfig:
data = json.load( fconfig ) data = json.load( fconfig )
variants = data[ 'cmake_variants' ] variants = data[ 'cmake_variants' ]
build_descs_by_axis = collections.defaultdict( list ) build_descs_by_axis = collections.defaultdict( list )
for axis in variants: for axis in variants:
axis_name = axis["name"] axis_name = axis["name"]
build_descs = [] build_descs = []
if "generators" in axis: if "generators" in axis:
for generator_data in axis["generators"]: for generator_data in axis["generators"]:
for generator in generator_data["generator"]: for generator in generator_data["generator"]:
build_desc = BuildDesc( generator=generator, build_desc = BuildDesc( generator=generator,
prepend_envs=generator_data.get("env_prepend") ) prepend_envs=generator_data.get("env_prepend") )
build_descs.append( build_desc ) build_descs.append( build_desc )
elif "variables" in axis: elif "variables" in axis:
for variables in axis["variables"]: for variables in axis["variables"]:
build_desc = BuildDesc( variables=variables ) build_desc = BuildDesc( variables=variables )
build_descs.append( build_desc ) build_descs.append( build_desc )
elif "build_types" in axis: elif "build_types" in axis:
for build_type in axis["build_types"]: for build_type in axis["build_types"]:
build_desc = BuildDesc( build_type=build_type ) build_desc = BuildDesc( build_type=build_type )
build_descs.append( build_desc ) build_descs.append( build_desc )
build_descs_by_axis[axis_name].extend( build_descs ) build_descs_by_axis[axis_name].extend( build_descs )
return build_descs_by_axis return build_descs_by_axis
def generate_build_variants( build_descs_by_axis ): def generate_build_variants( build_descs_by_axis ):
"""Returns a list of BuildDesc generated for the partial BuildDesc for each axis.""" """Returns a list of BuildDesc generated for the partial BuildDesc for each axis."""
axis_names = build_descs_by_axis.keys() axis_names = build_descs_by_axis.keys()
build_descs = [] build_descs = []
for axis_name, axis_build_descs in build_descs_by_axis.items(): for axis_name, axis_build_descs in build_descs_by_axis.items():
if len(build_descs): if len(build_descs):
# for each existing build_desc and each axis build desc, create a new build_desc # for each existing build_desc and each axis build desc, create a new build_desc
new_build_descs = [] new_build_descs = []
for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs): for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs):
new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) ) new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) )
build_descs = new_build_descs build_descs = new_build_descs
else: else:
build_descs = axis_build_descs build_descs = axis_build_descs
return build_descs return build_descs
HTML_TEMPLATE = string.Template('''<html> HTML_TEMPLATE = string.Template('''<html>
<head> <head>
<title>$title</title> <title>$title</title>
<style type="text/css"> <style type="text/css">
td.failed {background-color:#f08080;} td.failed {background-color:#f08080;}
td.ok {background-color:#c0eec0;} td.ok {background-color:#c0eec0;}
</style> </style>
</head> </head>
<body> <body>
<table border="1"> <table border="1">
<thead> <thead>
<tr> <tr>
<th>Variables</th> <th>Variables</th>
$th_vars $th_vars
</tr> </tr>
<tr> <tr>
<th>Build type</th> <th>Build type</th>
$th_build_types $th_build_types
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
$tr_builds $tr_builds
</tbody> </tbody>
</table> </table>
</body></html>''') </body></html>''')
def generate_html_report( html_report_path, builds ): def generate_html_report( html_report_path, builds ):
report_dir = os.path.dirname( html_report_path ) report_dir = os.path.dirname( html_report_path )
# Vertical axis: generator # Vertical axis: generator
# Horizontal: variables, then build_type # Horizontal: variables, then build_type
builds_by_generator = collections.defaultdict( list ) builds_by_generator = collections.defaultdict( list )
variables = set() variables = set()
build_types_by_variable = collections.defaultdict( set ) build_types_by_variable = collections.defaultdict( set )
build_by_pos_key = {} # { (generator, var_key, build_type): build } build_by_pos_key = {} # { (generator, var_key, build_type): build }
for build in builds: for build in builds:
builds_by_generator[build.desc.generator].append( build ) builds_by_generator[build.desc.generator].append( build )
var_key = tuple(sorted(build.desc.variables)) var_key = tuple(sorted(build.desc.variables))
variables.add( var_key ) variables.add( var_key )
build_types_by_variable[var_key].add( build.desc.build_type ) build_types_by_variable[var_key].add( build.desc.build_type )
pos_key = (build.desc.generator, var_key, build.desc.build_type) pos_key = (build.desc.generator, var_key, build.desc.build_type)
build_by_pos_key[pos_key] = build build_by_pos_key[pos_key] = build
variables = sorted( variables ) variables = sorted( variables )
th_vars = [] th_vars = []
th_build_types = [] th_build_types = []
for variable in variables: for variable in variables:
build_types = sorted( build_types_by_variable[variable] ) build_types = sorted( build_types_by_variable[variable] )
nb_build_type = len(build_types_by_variable[variable]) nb_build_type = len(build_types_by_variable[variable])
th_vars.append( '<th colspan="%d">%s</th>' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) ) th_vars.append( '<th colspan="%d">%s</th>' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) )
for build_type in build_types: for build_type in build_types:
th_build_types.append( '<th>%s</th>' % cgi.escape(build_type) ) th_build_types.append( '<th>%s</th>' % cgi.escape(build_type) )
tr_builds = [] tr_builds = []
for generator in sorted( builds_by_generator ): for generator in sorted( builds_by_generator ):
tds = [ '<td>%s</td>\n' % cgi.escape( generator ) ] tds = [ '<td>%s</td>\n' % cgi.escape( generator ) ]
for variable in variables: for variable in variables:
build_types = sorted( build_types_by_variable[variable] ) build_types = sorted( build_types_by_variable[variable] )
for build_type in build_types: for build_type in build_types:
pos_key = (generator, variable, build_type) pos_key = (generator, variable, build_type)
build = build_by_pos_key.get(pos_key) build = build_by_pos_key.get(pos_key)
if build: if build:
cmake_status = 'ok' if build.cmake_succeeded else 'FAILED' cmake_status = 'ok' if build.cmake_succeeded else 'FAILED'
build_status = 'ok' if build.build_succeeded else 'FAILED' build_status = 'ok' if build.build_succeeded else 'FAILED'
cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir ) cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir )
build_log_url = os.path.relpath( build.build_log_path, report_dir ) build_log_url = os.path.relpath( build.build_log_path, report_dir )
td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % ( td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % (
build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status) build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status)
if build.cmake_succeeded: if build.cmake_succeeded:
td += '<br><a href="%s" class="%s">Build: %s</a>' % ( td += '<br><a href="%s" class="%s">Build: %s</a>' % (
build_log_url, build_status.lower(), build_status) build_log_url, build_status.lower(), build_status)
td += '</td>' td += '</td>'
else: else:
td = '<td></td>' td = '<td></td>'
tds.append( td ) tds.append( td )
tr_builds.append( '<tr>%s</tr>' % '\n'.join( tds ) ) tr_builds.append( '<tr>%s</tr>' % '\n'.join( tds ) )
html = HTML_TEMPLATE.substitute( html = HTML_TEMPLATE.substitute(
title='Batch build report', title='Batch build report',
th_vars=' '.join(th_vars), th_vars=' '.join(th_vars),
th_build_types=' '.join( th_build_types), th_build_types=' '.join( th_build_types),
tr_builds='\n'.join( tr_builds ) ) tr_builds='\n'.join( tr_builds ) )
with open( html_report_path, 'wt' ) as fhtml: with open( html_report_path, 'wt' ) as fhtml:
fhtml.write( html ) fhtml.write( html )
print 'HTML report generated in:', html_report_path print 'HTML report generated in:', html_report_path
def main(): def main():
usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...] usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...]
Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run
as described in CONFIG_JSON_PATH building in WORK_DIR. as described in CONFIG_JSON_PATH building in WORK_DIR.
Example of call: Example of call:
python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json
""" """
from optparse import OptionParser from optparse import OptionParser
parser = OptionParser(usage=usage) parser = OptionParser(usage=usage)
parser.allow_interspersed_args = True parser.allow_interspersed_args = True
# parser.add_option('-v', '--verbose', dest="verbose", action='store_true', # parser.add_option('-v', '--verbose', dest="verbose", action='store_true',
# help="""Be verbose.""") # help="""Be verbose.""")
parser.enable_interspersed_args() parser.enable_interspersed_args()
options, args = parser.parse_args() options, args = parser.parse_args()
if len(args) < 3: if len(args) < 3:
parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." ) parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." )
work_dir = args[0] work_dir = args[0]
source_dir = args[1].rstrip('/\\') source_dir = args[1].rstrip('/\\')
config_paths = args[2:] config_paths = args[2:]
for config_path in config_paths: for config_path in config_paths:
if not os.path.isfile( config_path ): if not os.path.isfile( config_path ):
parser.error( "Can not read: %r" % config_path ) parser.error( "Can not read: %r" % config_path )
# generate build variants # generate build variants
build_descs = [] build_descs = []
for config_path in config_paths: for config_path in config_paths:
build_descs_by_axis = load_build_variants_from_config( config_path ) build_descs_by_axis = load_build_variants_from_config( config_path )
build_descs.extend( generate_build_variants( build_descs_by_axis ) ) build_descs.extend( generate_build_variants( build_descs_by_axis ) )
print 'Build variants (%d):' % len(build_descs) print 'Build variants (%d):' % len(build_descs)
# assign build directory for each variant # assign build directory for each variant
if not os.path.isdir( work_dir ): if not os.path.isdir( work_dir ):
os.makedirs( work_dir ) os.makedirs( work_dir )
builds = [] builds = []
with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap: with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap:
for index, build_desc in enumerate( build_descs ): for index, build_desc in enumerate( build_descs ):
build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) ) build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) )
builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) ) builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) )
fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) ) fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) )
for build in builds: for build in builds:
build.execute_build() build.execute_build()
html_report_path = os.path.join( work_dir, 'batchbuild-report.html' ) html_report_path = os.path.join( work_dir, 'batchbuild-report.html' )
generate_html_report( html_report_path, builds ) generate_html_report( html_report_path, builds )
print 'Done' print 'Done'
if __name__ == '__main__': if __name__ == '__main__':
main() main()