Merge pull request #139 from cdunn2001/some-python-changes

Some python changes.

* Better messaging.
* Make `doxybuild.py` work with python3.4

devtools/batchbuild.py

@@ -211,18 +211,15 @@ def generate_html_report( html_report_path, builds ):
                     build_status = 'ok' if build.build_succeeded else 'FAILED'
                     cmake_log_url = os.path.relpath(build.cmake_log_path, report_dir)
                     build_log_url = os.path.relpath(build.build_log_path, report_dir)
-                    td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % (
+                    td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % (                        build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status)
-                        build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status)
                     if build.cmake_succeeded:
-                        td += '<br><a href="%s" class="%s">Build: %s</a>' % (
+                        td += '<br><a href="%s" class="%s">Build: %s</a>' % (                            build_log_url, build_status.lower(), build_status)
-                            build_log_url, build_status.lower(), build_status)
                     td += '</td>'
                 else:
                     td = '<td></td>'
                 tds.append(td)
         tr_builds.append('<tr>%s</tr>' % '\n'.join(tds))
-    html = HTML_TEMPLATE.substitute(
+    html = HTML_TEMPLATE.substitute(        title='Batch build report',
-        title='Batch build report',
         th_vars=' '.join(th_vars),
         th_build_types=' '.join(th_build_types),
         tr_builds='\n'.join(tr_builds))

devtools/tarball.py

@@ -1,5 +1,5 @@
-import os.path
+from contextlib import closing
-import gzip
+import os
 import tarfile
 
 TARGZ_DEFAULT_COMPRESSION_LEVEL = 9
@@ -29,25 +29,19 @@ def make_tarball(tarball_path, sources, base_dir, prefix_dir=''):
                 path_in_tar = archive_name(path)
                 tar.add(path, path_in_tar)
     compression = TARGZ_DEFAULT_COMPRESSION_LEVEL
-    tar = tarfile.TarFile.gzopen( tarball_path, 'w', compresslevel=compression )
+    with closing(tarfile.TarFile.open(tarball_path, 'w:gz',
-    try:
+            compresslevel=compression)) as tar:
         for source in sources:
             source_path = source
             if os.path.isdir(source):
-                os.path.walk(source_path, visit, tar)
+                for dirpath, dirnames, filenames in os.walk(source_path):
+                    visit(tar, dirpath, filenames)
             else:
                 path_in_tar = archive_name(source_path)
                 tar.add(source_path, path_in_tar)      # filename, arcname
-    finally:
-        tar.close()
 
 def decompress(tarball_path, base_dir):
     """Decompress the gzipped tarball into directory base_dir.
     """
-    # !!! This class method is not documented in the online doc
+    with closing(tarfile.TarFile.open(tarball_path)) as tar:
-    # nor is bz2open!
-    tar = tarfile.TarFile.gzopen(tarball_path, mode='r')
-    try:
         tar.extractall(base_dir)
-    finally:
-        tar.close()

doxybuild.py

@@ -1,13 +1,28 @@
 """Script to generate doxygen documentation.
 """
 from __future__ import print_function
+from __future__ import unicode_literals
 from devtools import tarball
+from contextlib import contextmanager
+import subprocess
+import traceback
 import re
 import os
-import os.path
 import sys
 import shutil
 
+@contextmanager
+def cd(newdir):
+    """
+    http://stackoverflow.com/questions/431684/how-do-i-cd-in-python
+    """
+    prevdir = os.getcwd()
+    os.chdir(newdir)
+    try:
+        yield
+    finally:
+        os.chdir(prevdir)
+
 def find_program(*filenames):
     """find a program in folders path_lst, and sets env[var]
     @param filenames: a list of possible names of the program to search for
@@ -28,51 +43,54 @@ def do_subst_in_file(targetfile, sourcefile, dict):
     For example, if dict is {'%VERSION%': '1.2345', '%BASE%': 'MyProg'},
     then all instances of %VERSION% in the file will be replaced with 1.2345 etc.
     """
-    try:
+    with open(sourcefile, 'r') as f:
-        f = open(sourcefile, 'rb')
         contents = f.read()
-        f.close()
-    except:
-        print("Can't read source file %s"%sourcefile)
-        raise
     for (k,v) in list(dict.items()):
         v = v.replace('\\','\\\\') 
         contents = re.sub(k, v, contents)
-    try:
+    with open(targetfile, 'w') as f:
-        f = open(targetfile, 'wb')
         f.write(contents)
-        f.close()
+
+def getstatusoutput(cmd):
+    """cmd is a list.
+    """
+    try:
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        output, _ = process.communicate()
+        status = process.returncode
     except:
-        print("Can't write target file %s"%targetfile)
+        status = -1
-        raise
+        output = traceback.format_exc()
+    return status, output
+
+def run_cmd(cmd, silent=False):
+    """Raise exception on failure.
+    """
+    info = 'Running: %r in %r' %(' '.join(cmd), os.getcwd())
+    print(info)
+    sys.stdout.flush()
+    if silent:
+        status, output = getstatusoutput(cmd)
+    else:
+        status, output = os.system(' '.join(cmd)), ''
+    if status:
+        msg = 'Error while %s ...\n\terror=%d, output="""%s"""' %(info, status, output)
+        raise Exception(msg)
+
+def assert_is_exe(path):
+    if not path:
+        raise Exception('path is empty.')
+    if not os.path.isfile(path):
+        raise Exception('%r is not a file.' %path)
+    if not os.access(path, os.X_OK):
+        raise Exception('%r is not executable by this user.' %path)
 
 def run_doxygen(doxygen_path, config_file, working_dir, is_silent):
+    assert_is_exe(doxygen_path)
     config_file = os.path.abspath(config_file)
-    doxygen_path = doxygen_path
+    with cd(working_dir):
-    old_cwd = os.getcwd()
-    try:
-        os.chdir( working_dir )
         cmd = [doxygen_path, config_file]
-        print('Running:', ' '.join( cmd ))
+        run_cmd(cmd, is_silent)
-        try:
-            import subprocess
-        except:
-            if os.system( ' '.join( cmd ) ) != 0:
-                print('Documentation generation failed')
-                return False
-        else:
-            if is_silent:
-                process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
-            else:
-                process = subprocess.Popen( cmd )
-            stdout, _ = process.communicate()
-            if process.returncode:
-                print('Documentation generation failed:')
-                print(stdout)
-                return False
-        return True
-    finally:
-        os.chdir( old_cwd )
 
 def build_doc(options,  make_release=False):
     if make_release:
@@ -113,9 +131,9 @@ def build_doc( options,  make_release=False ):
         os.makedirs(output_dir)
 
     do_subst_in_file('doc/doxyfile', 'doc/doxyfile.in', subst_keys)
-    ok = run_doxygen( options.doxygen_path, 'doc/doxyfile', 'doc', is_silent=options.silent )
+    run_doxygen(options.doxygen_path, 'doc/doxyfile', 'doc', is_silent=options.silent)
     if not options.silent:
-        print(open(warning_log_path, 'rb').read())
+        print(open(warning_log_path, 'r').read())
     index_path = os.path.abspath(os.path.join('doc', subst_keys['%HTML_OUTPUT%'], 'index.html'))
     print('Generated documentation can be found in:')
     print(index_path)

scons-tools/srcdist.py

@@ -161,8 +161,7 @@ def generate(env):
    Add builders and construction variables for the
    SrcDist tool.
    """
-##   doxyfile_scanner = env.Scanner(
+##   doxyfile_scanner = env.Scanner(##      DoxySourceScan,
-##      DoxySourceScan,
 ##      "DoxySourceScan",
 ##      scan_check = DoxySourceScanCheck,
 ##)

test/runjsontests.py

@@ -75,8 +75,7 @@ def runAllTests( jsontest_executable_path, input_dir = None,
         print('TESTING:', input_path, end=' ')
         options = is_json_checker_test and '--json-checker' or ''
         options += ' --json-writer %s'%writerClass
-        cmd = '%s%s %s "%s"' % (
+        cmd = '%s%s %s "%s"' % (            valgrind_path, jsontest_executable_path, options,
-            valgrind_path, jsontest_executable_path, options,
             input_path)
         status, process_output = getStatusOutput(cmd)
         if is_json_checker_test:

test/rununittests.py

@@ -53,8 +53,7 @@ def runAllTests( exe_path, use_valgrind=False ):
         print()
         for name, result in failures:
             print(result)
-        print('%d/%d tests passed (%d failure(s))' % (
+        print('%d/%d tests passed (%d failure(s))' % (            pass_count, len(test_names), failed_count))
-            pass_count, len(test_names), failed_count))
         return 1
     else:
         print('All %d tests passed' % len(test_names))