diff --git a/lutinDebug.py b/lutinDebug.py index 955b531..9502f28 100644 --- a/lutinDebug.py +++ b/lutinDebug.py @@ -11,6 +11,7 @@ import os import thread import lutinMultiprocess import threading +import re debugLevel=3 debugColor=False @@ -49,58 +50,80 @@ def enable_color(): global color_cyan color_cyan = "\033[36m" -def verbose(input): +def verbose(input, force=False): global debugLock global debugLevel - if debugLevel >= 5: + if debugLevel >= 5 \ + or force == True: debugLock.acquire() print(color_blue + input + color_default) debugLock.release() -def debug(input): +def debug(input, force=False): global debugLock global debugLevel - if debugLevel >= 4: + if debugLevel >= 4 \ + or force == True: debugLock.acquire() print(color_green + input + color_default) debugLock.release() -def info(input): +def info(input, force=False): global debugLock global debugLevel - if debugLevel >= 3: + if debugLevel >= 3 \ + or force == True: debugLock.acquire() print(input + color_default) debugLock.release() -def warning(input): +def warning(input, force=False): global debugLock global debugLevel - if debugLevel >= 2: + if debugLevel >= 2 \ + or force == True: debugLock.acquire() print(color_purple + "[WARNING] " + input + color_default) debugLock.release() -def error(input, threadID=-1): +def error(input, threadID=-1, force=False, crash=True): global debugLock global debugLevel - if debugLevel >= 1: + if debugLevel >= 1 \ + or force == True: debugLock.acquire() print(color_red + "[ERROR] " + input + color_default) debugLock.release() - lutinMultiprocess.error_occured() - if threadID != -1: - thread.interrupt_main() - exit(-1) - #os_exit(-1) - #raise "error happend" + if crash==True: + lutinMultiprocess.error_occured() + if threadID != -1: + thread.interrupt_main() + exit(-1) + #os_exit(-1) + #raise "error happend" -def print_element(type, lib, dir, name): +def print_element(type, lib, dir, name, force=False): global debugLock global debugLevel - if debugLevel >= 3: + if debugLevel >= 3 \ + or force == True: debugLock.acquire() print(color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default) debugLock.release() - +def print_compilator(myString): + global debugColor + global debugLock + if debugColor == True: + myString = myString.replace('\\n', '\n') + myString = myString.replace('\\t', '\t') + myString = myString.replace('error:', color_red+'error:'+color_default) + myString = myString.replace('warning:', color_purple+'warning:'+color_default) + myString = myString.replace('note:', color_green+'note:'+color_default) + myString = re.sub(r'([/\w_-]+\.\w+):', r'-COLORIN-\1-COLOROUT-:', myString) + myString = myString.replace('-COLORIN-', color_yellow) + myString = myString.replace('-COLOROUT-', color_default) + + debugLock.acquire() + print(myString) + debugLock.release() \ No newline at end of file diff --git a/lutinEnv.py b/lutinEnv.py index f447070..c05c9fd 100644 --- a/lutinEnv.py +++ b/lutinEnv.py @@ -29,7 +29,7 @@ printPrettyMode=False def set_print_pretty_mode(val): global printPrettyMode - if val==True: + if val == True: printPrettyMode = True else: printPrettyMode = False @@ -38,9 +38,10 @@ def get_print_pretty_mode(): global printPrettyMode return printPrettyMode -def print_pretty(myString): +def print_pretty(myString, force=False): global printPrettyMode - if True==printPrettyMode: + if printPrettyMode == True \ + or force == True: if myString[len(myString)-1]==' ' : tmpcmdLine = myString[:len(myString)-1] else : diff --git a/lutinMultiprocess.py b/lutinMultiprocess.py index 345e4ce..09a626c 100644 --- a/lutinMultiprocess.py +++ b/lutinMultiprocess.py @@ -16,11 +16,21 @@ import os import subprocess import lutinTools import lutinEnv +import shlex queueLock = threading.Lock() workQueue = Queue.Queue() currentThreadWorking = 0 threads = [] +# To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation +currentIdExecution = 0 +errorExecution = { + "id":-1, + "cmd":"", + "return":0, + "err":"", + "out":"", +} exitFlag = False # resuest stop of the thread isinit = False # the thread are initialized @@ -40,27 +50,62 @@ def store_command(cmdLine, file): file2.close() -def run_command(cmdLine, storeCmdLine=""): - debug.debug(lutinEnv.print_pretty(cmdLine)) +def run_command(cmdLine, storeCmdLine="", buildId=-1, file=""): + global errorOccured + global exitFlag + global currentIdExecution + # prepare command line: + args = shlex.split(cmdLine) + #debug.verbose("cmd = " + str(args)) try: - retcode = subprocess.call(cmdLine, shell=True) - except OSError as e: - print >>sys.stderr, "Execution failed:", e - - if retcode != 0: - global errorOccured + # create the subprocess + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + debug.error("subprocess.CalledProcessError : TODO ...") + # launch the subprocess: + output, err = p.communicate() + # Check error : + if p.returncode == 0: + debug.debug(lutinEnv.print_pretty(cmdLine)) + queueLock.acquire() + # TODO : Print the output all the time .... ==> to show warnings ... + if buildId >= 0 and (output != "" or err != ""): + debug.warning("output in subprocess compiling: '" + file + "'") + if output != "": + debug.print_compilator(output) + if err != "": + debug.print_compilator(err) + queueLock.release() + else: errorOccured = True - global exitFlag exitFlag = True - if retcode == 2: - debug.error("can not compile file ... [keyboard interrrupt]") + # if No ID : Not in a multiprocess mode ==> just stop here + if buildId < 0: + debug.debug(lutinEnv.print_pretty(cmdLine), force=True) + debug.print_compilator(output) + debug.print_compilator(err) + if p.returncode == 2: + debug.error("can not compile file ... [keyboard interrrupt]") + else: + debug.error("can not compile file ... ret : " + str(p.returncode)) else: - debug.error("can not compile file ... ret : " + str(retcode)) + # in multiprocess interface + queueLock.acquire() + # if an other write an error before, check if the current process is started before ==> then is the first error + if errorExecution["id"] >= buildId: + # nothing to do ... + queueLock.release() + return; + errorExecution["id"] = buildId + errorExecution["cmd"] = cmdLine + errorExecution["return"] = p.returncode + errorExecution["err"] = err, + errorExecution["out"] = output, + queueLock.release() + # not write the command file... return - # write cmd line only after to prevent errors ... store_command(cmdLine, storeCmdLine) - @@ -76,7 +121,7 @@ class myThread(threading.Thread): global exitFlag global currentThreadWorking workingSet = False - while False==exitFlag: + while exitFlag == False: self.lock.acquire() if not self.queue.empty(): if workingSet==False: @@ -89,8 +134,8 @@ class myThread(threading.Thread): comment = data[2] cmdLine = data[1] cmdStoreFile = data[3] - debug.print_element( "[" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3]) - run_command(cmdLine, cmdStoreFile) + debug.print_element( "[" + str(data[4]) + "][" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3]) + run_command(cmdLine, cmdStoreFile, buildId=data[4], file=comment[3]) else: debug.warning("unknow request command : " + data[0]) else: @@ -146,21 +191,24 @@ def un_init(): def run_in_pool(cmdLine, comment, storeCmdLine=""): + global currentIdExecution if processorAvaillable <= 1: debug.print_element(comment[0], comment[1], comment[2], comment[3]) - run_command(cmdLine, storeCmdLine) + run_command(cmdLine, storeCmdLine, file=comment[3]) return # multithreaded mode init() # Fill the queue queueLock.acquire() debug.verbose("add : in pool cmdLine") - workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine]) + workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine, currentIdExecution]) + currentIdExecution +=1; queueLock.release() def pool_synchrosize(): global errorOccured + global errorExecution if processorAvaillable <= 1: #in this case : nothing to synchronise return @@ -179,7 +227,17 @@ def pool_synchrosize(): if False==errorOccured: debug.verbose("queue is empty") else: - debug.debug("Thread return with error ... ==> stop all the pool") un_init() - debug.error("Pool error occured ...") + debug.debug("Thread return with error ... ==> stop all the pool") + if errorExecution["id"] == -1: + debug.error("Pool error occured ... (No return information on Pool)") + return + debug.error("Error in an pool element : [" + str(errorExecution["id"]) + "]", crash=False) + debug.debug(lutinEnv.print_pretty(errorExecution["cmd"]), force=True) + debug.print_compilator(str(errorExecution["out"][0])) + debug.print_compilator(str(errorExecution["err"][0])) + if errorExecution["return"] == 2: + debug.error("can not compile file ... [keyboard interrrupt]") + else: + debug.error("can not compile file ... return value : " + str(errorExecution["return"]))