2012-11-23 19:57:22 +01:00
#!/usr/bin/env python
2011-09-09 15:21:26 +02:00
import testlog_parser , sys , os , xml , glob , re
2011-09-06 17:30:28 +02:00
from table_formatter import *
from optparse import OptionParser
2012-01-16 08:04:26 +01:00
numeric_re = re . compile ( " ( \ d+) " )
cvtype_re = re . compile ( " (8U|8S|16U|16S|32S|32F|64F)C( \ d { 1,3}) " )
cvtypes = { ' 8U ' : 0 , ' 8S ' : 1 , ' 16U ' : 2 , ' 16S ' : 3 , ' 32S ' : 4 , ' 32F ' : 5 , ' 64F ' : 6 }
convert = lambda text : int ( text ) if text . isdigit ( ) else text
keyselector = lambda a : cvtype_re . sub ( lambda match : " " + str ( cvtypes . get ( match . group ( 1 ) , 7 ) + ( int ( match . group ( 2 ) ) - 1 ) * 8 ) + " " , a )
alphanum_keyselector = lambda key : [ convert ( c ) for c in numeric_re . split ( keyselector ( key ) ) ]
2011-11-22 14:57:44 +01:00
2011-10-18 14:31:08 +02:00
def getSetName ( tset , idx , columns , short = True ) :
2012-01-13 13:37:10 +01:00
if columns and len ( columns ) > idx :
2011-10-18 14:31:08 +02:00
prefix = columns [ idx ]
else :
prefix = None
if short and prefix :
return prefix
name = tset [ 0 ] . replace ( " .xml " , " " ) . replace ( " _ " , " \n " )
if prefix :
2011-12-16 05:42:34 +01:00
return prefix + " \n " + ( " - " * int ( len ( max ( prefix . split ( " \n " ) , key = len ) ) * 1.5 ) ) + " \n " + name
2011-10-18 14:31:08 +02:00
return name
2011-09-06 17:30:28 +02:00
if __name__ == " __main__ " :
if len ( sys . argv ) < 2 :
print >> sys . stderr , " Usage: \n " , os . path . basename ( sys . argv [ 0 ] ) , " <log_name1>.xml [<log_name2>.xml ...] "
exit ( 0 )
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
parser = OptionParser ( )
parser . add_option ( " -o " , " --output " , dest = " format " , help = " output results in text format (can be ' txt ' , ' html ' or ' auto ' - default) " , metavar = " FMT " , default = " auto " )
parser . add_option ( " -m " , " --metric " , dest = " metric " , help = " output metric " , metavar = " NAME " , default = " gmean " )
parser . add_option ( " -u " , " --units " , dest = " units " , help = " units for output values (s, ms (default), mks, ns or ticks) " , metavar = " UNITS " , default = " ms " )
parser . add_option ( " -f " , " --filter " , dest = " filter " , help = " regex to filter tests " , metavar = " REGEX " , default = None )
2011-10-18 14:31:08 +02:00
parser . add_option ( " " , " --module " , dest = " module " , default = None , metavar = " NAME " , help = " module prefix for test names " )
parser . add_option ( " " , " --columns " , dest = " columns " , default = None , metavar = " NAMES " , help = " comma-separated list of column aliases " )
2011-09-06 17:30:28 +02:00
parser . add_option ( " " , " --no-relatives " , action = " store_false " , dest = " calc_relatives " , default = True , help = " do not output relative values " )
2013-02-26 16:37:37 +01:00
parser . add_option ( " " , " --with-cycles-reduction " , action = " store_true " , dest = " calc_cr " , default = False , help = " output cycle reduction percentages " )
parser . add_option ( " " , " --with-score " , action = " store_true " , dest = " calc_score " , default = False , help = " output automatic classification of speedups " )
2011-09-06 17:30:28 +02:00
parser . add_option ( " " , " --show-all " , action = " store_true " , dest = " showall " , default = False , help = " also include empty and \" notrun \" lines " )
2011-09-09 15:21:26 +02:00
parser . add_option ( " " , " --match " , dest = " match " , default = None )
parser . add_option ( " " , " --match-replace " , dest = " match_replace " , default = " " )
2013-02-25 16:03:26 +01:00
parser . add_option ( " " , " --regressions-only " , dest = " regressionsOnly " , default = None , metavar = " X-FACTOR " , help = " show only tests with performance regressions not " )
2011-09-06 17:30:28 +02:00
( options , args ) = parser . parse_args ( )
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
options . generateHtml = detectHtmlOutputType ( options . format )
if options . metric not in metrix_table :
options . metric = " gmean "
2011-10-18 14:31:08 +02:00
if options . metric . endswith ( " % " ) or options . metric . endswith ( " $ " ) :
2011-09-06 17:30:28 +02:00
options . calc_relatives = False
2011-10-18 14:31:08 +02:00
options . calc_cr = False
if options . columns :
2011-12-16 05:42:34 +01:00
options . columns = [ s . strip ( ) . replace ( " \\ n " , " \n " ) for s in options . columns . split ( " , " ) ]
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
# expand wildcards and filter duplicates
2012-01-06 08:41:04 +01:00
files = [ ]
seen = set ( )
2011-09-06 17:30:28 +02:00
for arg in args :
if ( " * " in arg ) or ( " ? " in arg ) :
2012-01-06 08:41:04 +01:00
flist = [ os . path . abspath ( f ) for f in glob . glob ( arg ) ]
flist = sorted ( flist , key = lambda text : str ( text ) . replace ( " M " , " _ " ) )
files . extend ( [ x for x in flist if x not in seen and not seen . add ( x ) ] )
2011-09-06 17:30:28 +02:00
else :
2012-01-06 08:41:04 +01:00
fname = os . path . abspath ( arg )
2012-01-06 09:44:28 +01:00
if fname not in seen and not seen . add ( fname ) :
2012-01-06 08:41:04 +01:00
files . append ( fname )
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
# read all passed files
test_sets = [ ]
for arg in files :
try :
tests = testlog_parser . parseLogFile ( arg )
if options . filter :
expr = re . compile ( options . filter )
2011-09-09 15:21:26 +02:00
tests = [ t for t in tests if expr . search ( str ( t ) ) ]
if options . match :
tests = [ t for t in tests if t . get ( " status " ) != " notrun " ]
2011-09-06 17:30:28 +02:00
if tests :
test_sets . append ( ( os . path . basename ( arg ) , tests ) )
except IOError as err :
sys . stderr . write ( " IOError reading \" " + arg + " \" - " + str ( err ) + os . linesep )
except xml . parsers . expat . ExpatError as err :
sys . stderr . write ( " ExpatError reading \" " + arg + " \" - " + str ( err ) + os . linesep )
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
if not test_sets :
sys . stderr . write ( " Error: no test data found " + os . linesep )
quit ( )
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
# find matches
setsCount = len ( test_sets )
test_cases = { }
2012-10-17 01:18:30 +02:00
2011-09-09 15:21:26 +02:00
name_extractor = lambda name : str ( name )
if options . match :
reg = re . compile ( options . match )
name_extractor = lambda name : reg . sub ( options . match_replace , str ( name ) )
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
for i in range ( setsCount ) :
for case in test_sets [ i ] [ 1 ] :
2011-09-09 15:21:26 +02:00
name = name_extractor ( case )
2011-10-18 14:31:08 +02:00
if options . module :
name = options . module + " :: " + name
2011-09-06 17:30:28 +02:00
if name not in test_cases :
test_cases [ name ] = [ None ] * setsCount
test_cases [ name ] [ i ] = case
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
# build table
getter = metrix_table [ options . metric ] [ 1 ]
2013-02-26 16:37:37 +01:00
getter_score = metrix_table [ " score " ] [ 1 ]
2011-09-06 17:30:28 +02:00
if options . calc_relatives :
getter_p = metrix_table [ options . metric + " % " ] [ 1 ]
2011-10-18 14:31:08 +02:00
if options . calc_cr :
getter_cr = metrix_table [ options . metric + " $ " ] [ 1 ]
2011-09-06 17:30:28 +02:00
tbl = table ( metrix_table [ options . metric ] [ 0 ] )
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
# header
2011-10-12 16:41:36 +02:00
tbl . newColumn ( " name " , " Name of Test " , align = " left " , cssclass = " col_name " )
2011-09-06 17:30:28 +02:00
i = 0
for set in test_sets :
2011-10-18 14:31:08 +02:00
tbl . newColumn ( str ( i ) , getSetName ( set , i , options . columns , False ) , align = " center " )
2011-09-06 17:30:28 +02:00
i + = 1
2011-10-18 14:31:08 +02:00
metric_sets = test_sets [ 1 : ]
if options . calc_cr :
i = 1
for set in metric_sets :
tbl . newColumn ( str ( i ) + " $ " , getSetName ( set , i , options . columns ) + " \n vs \n " + getSetName ( test_sets [ 0 ] , 0 , options . columns ) + " \n (cycles reduction) " , align = " center " , cssclass = " col_cr " )
i + = 1
2011-09-06 17:30:28 +02:00
if options . calc_relatives :
i = 1
2011-10-18 14:31:08 +02:00
for set in metric_sets :
tbl . newColumn ( str ( i ) + " % " , getSetName ( set , i , options . columns ) + " \n vs \n " + getSetName ( test_sets [ 0 ] , 0 , options . columns ) + " \n (x-factor) " , align = " center " , cssclass = " col_rel " )
2011-09-06 17:30:28 +02:00
i + = 1
2013-02-26 16:37:37 +01:00
if options . calc_score :
i = 1
for set in metric_sets :
tbl . newColumn ( str ( i ) + " S " , getSetName ( set , i , options . columns ) + " \n vs \n " + getSetName ( test_sets [ 0 ] , 0 , options . columns ) + " \n (score) " , align = " center " , cssclass = " col_name " )
i + = 1
2012-10-17 01:18:30 +02:00
2011-09-06 17:30:28 +02:00
# rows
2012-01-09 08:45:19 +01:00
prevGroupName = None
2011-09-06 17:30:28 +02:00
needNewRow = True
2012-01-09 08:45:19 +01:00
lastRow = None
2011-11-22 14:57:44 +01:00
for name in sorted ( test_cases . iterkeys ( ) , key = alphanum_keyselector ) :
2011-09-06 17:30:28 +02:00
cases = test_cases [ name ]
if needNewRow :
2012-01-09 08:45:19 +01:00
lastRow = tbl . newRow ( )
2011-09-06 17:30:28 +02:00
if not options . showall :
needNewRow = False
tbl . newCell ( " name " , name )
2012-10-17 01:18:30 +02:00
2012-01-09 08:45:19 +01:00
groupName = next ( c for c in cases if c ) . shortName ( )
if groupName != prevGroupName :
prop = lastRow . props . get ( " cssclass " , " " )
if " firstingroup " not in prop :
lastRow . props [ " cssclass " ] = prop + " firstingroup "
prevGroupName = groupName
2011-09-06 17:30:28 +02:00
for i in range ( setsCount ) :
case = cases [ i ]
if case is None :
tbl . newCell ( str ( i ) , " - " )
if options . calc_relatives and i > 0 :
tbl . newCell ( str ( i ) + " % " , " - " )
2011-10-18 14:31:08 +02:00
if options . calc_cr and i > 0 :
tbl . newCell ( str ( i ) + " $ " , " - " )
2013-02-26 16:37:37 +01:00
if options . calc_score and i > 0 :
tbl . newCell ( str ( i ) + " $ " , " - " )
2011-09-06 17:30:28 +02:00
else :
status = case . get ( " status " )
if status != " run " :
tbl . newCell ( str ( i ) , status , color = " red " )
if status != " notrun " :
needNewRow = True
if options . calc_relatives and i > 0 :
tbl . newCell ( str ( i ) + " % " , " - " , color = " red " )
2011-10-18 14:31:08 +02:00
if options . calc_cr and i > 0 :
tbl . newCell ( str ( i ) + " $ " , " - " , color = " red " )
2013-02-26 16:37:37 +01:00
if options . calc_score and i > 0 :
tbl . newCell ( str ( i ) + " S " , " - " , color = " red " )
2011-09-06 17:30:28 +02:00
else :
val = getter ( case , cases [ 0 ] , options . units )
if options . calc_relatives and i > 0 and val :
valp = getter_p ( case , cases [ 0 ] , options . units )
else :
valp = None
2011-10-18 14:31:08 +02:00
if options . calc_cr and i > 0 and val :
valcr = getter_cr ( case , cases [ 0 ] , options . units )
else :
valcr = None
2013-02-26 16:37:37 +01:00
if options . calc_score and i > 0 and val :
val_score = getter_score ( case , cases [ 0 ] , options . units )
else :
val_score = None
2011-09-06 17:30:28 +02:00
if not valp or i == 0 :
color = None
elif valp > 1.05 :
color = " green "
2011-10-18 14:31:08 +02:00
elif valp < 0.95 :
color = " red "
2011-09-06 17:30:28 +02:00
else :
color = None
if val :
needNewRow = True
2011-10-18 14:31:08 +02:00
tbl . newCell ( str ( i ) , formatValue ( val , options . metric , options . units ) , val , color = color )
2011-09-06 17:30:28 +02:00
if options . calc_relatives and i > 0 :
2011-10-18 14:31:08 +02:00
tbl . newCell ( str ( i ) + " % " , formatValue ( valp , " % " ) , valp , color = color , bold = color )
if options . calc_cr and i > 0 :
tbl . newCell ( str ( i ) + " $ " , formatValue ( valcr , " $ " ) , valcr , color = color , bold = color )
2013-02-26 16:37:37 +01:00
if options . calc_score and i > 0 :
tbl . newCell ( str ( i ) + " S " , formatValue ( val_score , " S " ) , val_score , color = color , bold = color )
2011-09-06 17:30:28 +02:00
if not needNewRow :
tbl . trimLastRow ( )
2013-02-25 16:03:26 +01:00
if options . regressionsOnly :
for r in reversed ( range ( len ( tbl . rows ) ) ) :
delete = True
i = 1
for set in metric_sets :
val = tbl . rows [ r ] . cells [ len ( tbl . rows [ r ] . cells ) - i ] . value
if val is not None and val < float ( options . regressionsOnly ) :
delete = False
i + = 1
if ( delete ) :
tbl . rows . pop ( r )
2011-09-06 17:30:28 +02:00
# output table
if options . generateHtml :
2011-09-09 14:15:09 +02:00
if options . format == " moinwiki " :
tbl . htmlPrintTable ( sys . stdout , True )
else :
htmlPrintHeader ( sys . stdout , " Summary report for %s tests from %s test logs " % ( len ( test_cases ) , setsCount ) )
tbl . htmlPrintTable ( sys . stdout )
htmlPrintFooter ( sys . stdout )
2011-09-06 17:30:28 +02:00
else :
tbl . consolePrintTable ( sys . stdout )
2013-02-25 16:03:26 +01:00
if options . regressionsOnly :
sys . exit ( len ( tbl . rows ) )