diff --git a/devtools/batchbuild.py b/devtools/batchbuild.py index 5da74db..f0f8de6 100644 --- a/devtools/batchbuild.py +++ b/devtools/batchbuild.py @@ -1,280 +1,280 @@ -import collections -import itertools -import json -import os -import os.path -import re -import shutil -import string -import subprocess -import sys -import cgi - -class BuildDesc: - def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None): - self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ] - self.variables = variables or [] - self.build_type = build_type - self.generator = generator - - def merged_with( self, build_desc ): - """Returns a new BuildDesc by merging field content. - Prefer build_desc fields to self fields for single valued field. - """ - return BuildDesc( self.prepend_envs + build_desc.prepend_envs, - self.variables + build_desc.variables, - build_desc.build_type or self.build_type, - build_desc.generator or self.generator ) - - def env( self ): - environ = os.environ.copy() - for values_by_name in self.prepend_envs: - for var, value in values_by_name.items(): - var = var.upper() - if type(value) is unicode: - value = value.encode( sys.getdefaultencoding() ) - if var in environ: - environ[var] = value + os.pathsep + environ[var] - else: - environ[var] = value - return environ - - def cmake_args( self ): - args = ["-D%s" % var for var in self.variables] - # skip build type for Visual Studio solution as it cause warning - if self.build_type and 'Visual' not in self.generator: - args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type ) - if self.generator: - args.extend( ['-G', self.generator] ) - return args - - def __repr__( self ): - return "BuildDesc( %s, build_type=%s )" % (" ".join( self.cmake_args()), self.build_type) - -class BuildData: - def __init__( self, desc, work_dir, source_dir ): - self.desc = desc - self.work_dir = work_dir - self.source_dir = source_dir - self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' ) - self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' ) - self.cmake_succeeded = False - self.build_succeeded = False - - def execute_build(self): - print 'Build %s' % self.desc - self._make_new_work_dir( ) - self.cmake_succeeded = self._generate_makefiles( ) - if self.cmake_succeeded: - self.build_succeeded = self._build_using_makefiles( ) - return self.build_succeeded - - def _generate_makefiles(self): - print ' Generating makefiles: ', - cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )] - succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path ) - print 'done' if succeeded else 'FAILED' - return succeeded - - def _build_using_makefiles(self): - print ' Building:', - cmd = ['cmake', '--build', self.work_dir] - if self.desc.build_type: - cmd += ['--config', self.desc.build_type] - succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path ) - print 'done' if succeeded else 'FAILED' - return succeeded - - def _execute_build_subprocess(self, cmd, env, log_path): - process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir, - env=env ) - stdout, _ = process.communicate( ) - succeeded = (process.returncode == 0) - with open( log_path, 'wb' ) as flog: - log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode - flog.write( fix_eol( log ) ) - return succeeded - - def _make_new_work_dir(self): - if os.path.isdir( self.work_dir ): - print ' Removing work directory', self.work_dir - shutil.rmtree( self.work_dir, ignore_errors=True ) - if not os.path.isdir( self.work_dir ): - os.makedirs( self.work_dir ) - -def fix_eol( stdout ): - """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n). - """ - return re.sub( '\r*\n', os.linesep, stdout ) - -def load_build_variants_from_config( config_path ): - with open( config_path, 'rb' ) as fconfig: - data = json.load( fconfig ) - variants = data[ 'cmake_variants' ] - build_descs_by_axis = collections.defaultdict( list ) - for axis in variants: - axis_name = axis["name"] - build_descs = [] - if "generators" in axis: - for generator_data in axis["generators"]: - for generator in generator_data["generator"]: - build_desc = BuildDesc( generator=generator, - prepend_envs=generator_data.get("env_prepend") ) - build_descs.append( build_desc ) - elif "variables" in axis: - for variables in axis["variables"]: - build_desc = BuildDesc( variables=variables ) - build_descs.append( build_desc ) - elif "build_types" in axis: - for build_type in axis["build_types"]: - build_desc = BuildDesc( build_type=build_type ) - build_descs.append( build_desc ) - build_descs_by_axis[axis_name].extend( build_descs ) - return build_descs_by_axis - -def generate_build_variants( build_descs_by_axis ): - """Returns a list of BuildDesc generated for the partial BuildDesc for each axis.""" - axis_names = build_descs_by_axis.keys() - build_descs = [] - for axis_name, axis_build_descs in build_descs_by_axis.items(): - if len(build_descs): - # for each existing build_desc and each axis build desc, create a new build_desc - new_build_descs = [] - for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs): - new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) ) - build_descs = new_build_descs - else: - build_descs = axis_build_descs - return build_descs - -HTML_TEMPLATE = string.Template(''' - - $title - - - - - - - - $th_vars - - - - $th_build_types - - - -$tr_builds - -
Variables
Build type
-''') - -def generate_html_report( html_report_path, builds ): - report_dir = os.path.dirname( html_report_path ) - # Vertical axis: generator - # Horizontal: variables, then build_type - builds_by_generator = collections.defaultdict( list ) - variables = set() - build_types_by_variable = collections.defaultdict( set ) - build_by_pos_key = {} # { (generator, var_key, build_type): build } - for build in builds: - builds_by_generator[build.desc.generator].append( build ) - var_key = tuple(sorted(build.desc.variables)) - variables.add( var_key ) - build_types_by_variable[var_key].add( build.desc.build_type ) - pos_key = (build.desc.generator, var_key, build.desc.build_type) - build_by_pos_key[pos_key] = build - variables = sorted( variables ) - th_vars = [] - th_build_types = [] - for variable in variables: - build_types = sorted( build_types_by_variable[variable] ) - nb_build_type = len(build_types_by_variable[variable]) - th_vars.append( '%s' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) ) - for build_type in build_types: - th_build_types.append( '%s' % cgi.escape(build_type) ) - tr_builds = [] - for generator in sorted( builds_by_generator ): - tds = [ '%s\n' % cgi.escape( generator ) ] - for variable in variables: - build_types = sorted( build_types_by_variable[variable] ) - for build_type in build_types: - pos_key = (generator, variable, build_type) - build = build_by_pos_key.get(pos_key) - if build: - cmake_status = 'ok' if build.cmake_succeeded else 'FAILED' - build_status = 'ok' if build.build_succeeded else 'FAILED' - cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir ) - build_log_url = os.path.relpath( build.build_log_path, report_dir ) - td = 'CMake: %s' % ( - build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status) - if build.cmake_succeeded: - td += '
Build: %s' % ( - build_log_url, build_status.lower(), build_status) - td += '' - else: - td = '' - tds.append( td ) - tr_builds.append( '%s' % '\n'.join( tds ) ) - html = HTML_TEMPLATE.substitute( - title='Batch build report', - th_vars=' '.join(th_vars), - th_build_types=' '.join( th_build_types), - tr_builds='\n'.join( tr_builds ) ) - with open( html_report_path, 'wt' ) as fhtml: - fhtml.write( html ) - print 'HTML report generated in:', html_report_path - -def main(): - usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...] -Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run -as described in CONFIG_JSON_PATH building in WORK_DIR. - -Example of call: -python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json -""" - from optparse import OptionParser - parser = OptionParser(usage=usage) - parser.allow_interspersed_args = True -# parser.add_option('-v', '--verbose', dest="verbose", action='store_true', -# help="""Be verbose.""") - parser.enable_interspersed_args() - options, args = parser.parse_args() - if len(args) < 3: - parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." ) - work_dir = args[0] - source_dir = args[1].rstrip('/\\') - config_paths = args[2:] - for config_path in config_paths: - if not os.path.isfile( config_path ): - parser.error( "Can not read: %r" % config_path ) - - # generate build variants - build_descs = [] - for config_path in config_paths: - build_descs_by_axis = load_build_variants_from_config( config_path ) - build_descs.extend( generate_build_variants( build_descs_by_axis ) ) - print 'Build variants (%d):' % len(build_descs) - # assign build directory for each variant - if not os.path.isdir( work_dir ): - os.makedirs( work_dir ) - builds = [] - with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap: - for index, build_desc in enumerate( build_descs ): - build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) ) - builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) ) - fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) ) - for build in builds: - build.execute_build() - html_report_path = os.path.join( work_dir, 'batchbuild-report.html' ) - generate_html_report( html_report_path, builds ) - print 'Done' - - -if __name__ == '__main__': - main() - +import collections +import itertools +import json +import os +import os.path +import re +import shutil +import string +import subprocess +import sys +import cgi + +class BuildDesc: + def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None): + self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ] + self.variables = variables or [] + self.build_type = build_type + self.generator = generator + + def merged_with( self, build_desc ): + """Returns a new BuildDesc by merging field content. + Prefer build_desc fields to self fields for single valued field. + """ + return BuildDesc( self.prepend_envs + build_desc.prepend_envs, + self.variables + build_desc.variables, + build_desc.build_type or self.build_type, + build_desc.generator or self.generator ) + + def env( self ): + environ = os.environ.copy() + for values_by_name in self.prepend_envs: + for var, value in values_by_name.items(): + var = var.upper() + if type(value) is unicode: + value = value.encode( sys.getdefaultencoding() ) + if var in environ: + environ[var] = value + os.pathsep + environ[var] + else: + environ[var] = value + return environ + + def cmake_args( self ): + args = ["-D%s" % var for var in self.variables] + # skip build type for Visual Studio solution as it cause warning + if self.build_type and 'Visual' not in self.generator: + args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type ) + if self.generator: + args.extend( ['-G', self.generator] ) + return args + + def __repr__( self ): + return "BuildDesc( %s, build_type=%s )" % (" ".join( self.cmake_args()), self.build_type) + +class BuildData: + def __init__( self, desc, work_dir, source_dir ): + self.desc = desc + self.work_dir = work_dir + self.source_dir = source_dir + self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' ) + self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' ) + self.cmake_succeeded = False + self.build_succeeded = False + + def execute_build(self): + print 'Build %s' % self.desc + self._make_new_work_dir( ) + self.cmake_succeeded = self._generate_makefiles( ) + if self.cmake_succeeded: + self.build_succeeded = self._build_using_makefiles( ) + return self.build_succeeded + + def _generate_makefiles(self): + print ' Generating makefiles: ', + cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )] + succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path ) + print 'done' if succeeded else 'FAILED' + return succeeded + + def _build_using_makefiles(self): + print ' Building:', + cmd = ['cmake', '--build', self.work_dir] + if self.desc.build_type: + cmd += ['--config', self.desc.build_type] + succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path ) + print 'done' if succeeded else 'FAILED' + return succeeded + + def _execute_build_subprocess(self, cmd, env, log_path): + process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir, + env=env ) + stdout, _ = process.communicate( ) + succeeded = (process.returncode == 0) + with open( log_path, 'wb' ) as flog: + log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode + flog.write( fix_eol( log ) ) + return succeeded + + def _make_new_work_dir(self): + if os.path.isdir( self.work_dir ): + print ' Removing work directory', self.work_dir + shutil.rmtree( self.work_dir, ignore_errors=True ) + if not os.path.isdir( self.work_dir ): + os.makedirs( self.work_dir ) + +def fix_eol( stdout ): + """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n). + """ + return re.sub( '\r*\n', os.linesep, stdout ) + +def load_build_variants_from_config( config_path ): + with open( config_path, 'rb' ) as fconfig: + data = json.load( fconfig ) + variants = data[ 'cmake_variants' ] + build_descs_by_axis = collections.defaultdict( list ) + for axis in variants: + axis_name = axis["name"] + build_descs = [] + if "generators" in axis: + for generator_data in axis["generators"]: + for generator in generator_data["generator"]: + build_desc = BuildDesc( generator=generator, + prepend_envs=generator_data.get("env_prepend") ) + build_descs.append( build_desc ) + elif "variables" in axis: + for variables in axis["variables"]: + build_desc = BuildDesc( variables=variables ) + build_descs.append( build_desc ) + elif "build_types" in axis: + for build_type in axis["build_types"]: + build_desc = BuildDesc( build_type=build_type ) + build_descs.append( build_desc ) + build_descs_by_axis[axis_name].extend( build_descs ) + return build_descs_by_axis + +def generate_build_variants( build_descs_by_axis ): + """Returns a list of BuildDesc generated for the partial BuildDesc for each axis.""" + axis_names = build_descs_by_axis.keys() + build_descs = [] + for axis_name, axis_build_descs in build_descs_by_axis.items(): + if len(build_descs): + # for each existing build_desc and each axis build desc, create a new build_desc + new_build_descs = [] + for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs): + new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) ) + build_descs = new_build_descs + else: + build_descs = axis_build_descs + return build_descs + +HTML_TEMPLATE = string.Template(''' + + $title + + + + + + + + $th_vars + + + + $th_build_types + + + +$tr_builds + +
Variables
Build type
+''') + +def generate_html_report( html_report_path, builds ): + report_dir = os.path.dirname( html_report_path ) + # Vertical axis: generator + # Horizontal: variables, then build_type + builds_by_generator = collections.defaultdict( list ) + variables = set() + build_types_by_variable = collections.defaultdict( set ) + build_by_pos_key = {} # { (generator, var_key, build_type): build } + for build in builds: + builds_by_generator[build.desc.generator].append( build ) + var_key = tuple(sorted(build.desc.variables)) + variables.add( var_key ) + build_types_by_variable[var_key].add( build.desc.build_type ) + pos_key = (build.desc.generator, var_key, build.desc.build_type) + build_by_pos_key[pos_key] = build + variables = sorted( variables ) + th_vars = [] + th_build_types = [] + for variable in variables: + build_types = sorted( build_types_by_variable[variable] ) + nb_build_type = len(build_types_by_variable[variable]) + th_vars.append( '%s' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) ) + for build_type in build_types: + th_build_types.append( '%s' % cgi.escape(build_type) ) + tr_builds = [] + for generator in sorted( builds_by_generator ): + tds = [ '%s\n' % cgi.escape( generator ) ] + for variable in variables: + build_types = sorted( build_types_by_variable[variable] ) + for build_type in build_types: + pos_key = (generator, variable, build_type) + build = build_by_pos_key.get(pos_key) + if build: + cmake_status = 'ok' if build.cmake_succeeded else 'FAILED' + build_status = 'ok' if build.build_succeeded else 'FAILED' + cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir ) + build_log_url = os.path.relpath( build.build_log_path, report_dir ) + td = 'CMake: %s' % ( + build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status) + if build.cmake_succeeded: + td += '
Build: %s' % ( + build_log_url, build_status.lower(), build_status) + td += '' + else: + td = '' + tds.append( td ) + tr_builds.append( '%s' % '\n'.join( tds ) ) + html = HTML_TEMPLATE.substitute( + title='Batch build report', + th_vars=' '.join(th_vars), + th_build_types=' '.join( th_build_types), + tr_builds='\n'.join( tr_builds ) ) + with open( html_report_path, 'wt' ) as fhtml: + fhtml.write( html ) + print 'HTML report generated in:', html_report_path + +def main(): + usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...] +Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run +as described in CONFIG_JSON_PATH building in WORK_DIR. + +Example of call: +python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json +""" + from optparse import OptionParser + parser = OptionParser(usage=usage) + parser.allow_interspersed_args = True +# parser.add_option('-v', '--verbose', dest="verbose", action='store_true', +# help="""Be verbose.""") + parser.enable_interspersed_args() + options, args = parser.parse_args() + if len(args) < 3: + parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." ) + work_dir = args[0] + source_dir = args[1].rstrip('/\\') + config_paths = args[2:] + for config_path in config_paths: + if not os.path.isfile( config_path ): + parser.error( "Can not read: %r" % config_path ) + + # generate build variants + build_descs = [] + for config_path in config_paths: + build_descs_by_axis = load_build_variants_from_config( config_path ) + build_descs.extend( generate_build_variants( build_descs_by_axis ) ) + print 'Build variants (%d):' % len(build_descs) + # assign build directory for each variant + if not os.path.isdir( work_dir ): + os.makedirs( work_dir ) + builds = [] + with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap: + for index, build_desc in enumerate( build_descs ): + build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) ) + builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) ) + fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) ) + for build in builds: + build.execute_build() + html_report_path = os.path.join( work_dir, 'batchbuild-report.html' ) + generate_html_report( html_report_path, builds ) + print 'Done' + + +if __name__ == '__main__': + main() +