[DEV] separate multiprocess log and display it corectly with the compile line
This commit is contained in:
parent
7c72aa8b84
commit
dd03e46832
@ -11,6 +11,7 @@ import os
|
||||
import thread
|
||||
import lutinMultiprocess
|
||||
import threading
|
||||
import re
|
||||
|
||||
debugLevel=3
|
||||
debugColor=False
|
||||
@ -49,45 +50,51 @@ def enable_color():
|
||||
global color_cyan
|
||||
color_cyan = "\033[36m"
|
||||
|
||||
def verbose(input):
|
||||
def verbose(input, force=False):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 5:
|
||||
if debugLevel >= 5 \
|
||||
or force == True:
|
||||
debugLock.acquire()
|
||||
print(color_blue + input + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def debug(input):
|
||||
def debug(input, force=False):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 4:
|
||||
if debugLevel >= 4 \
|
||||
or force == True:
|
||||
debugLock.acquire()
|
||||
print(color_green + input + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def info(input):
|
||||
def info(input, force=False):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 3:
|
||||
if debugLevel >= 3 \
|
||||
or force == True:
|
||||
debugLock.acquire()
|
||||
print(input + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def warning(input):
|
||||
def warning(input, force=False):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 2:
|
||||
if debugLevel >= 2 \
|
||||
or force == True:
|
||||
debugLock.acquire()
|
||||
print(color_purple + "[WARNING] " + input + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def error(input, threadID=-1):
|
||||
def error(input, threadID=-1, force=False, crash=True):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 1:
|
||||
if debugLevel >= 1 \
|
||||
or force == True:
|
||||
debugLock.acquire()
|
||||
print(color_red + "[ERROR] " + input + color_default)
|
||||
debugLock.release()
|
||||
if crash==True:
|
||||
lutinMultiprocess.error_occured()
|
||||
if threadID != -1:
|
||||
thread.interrupt_main()
|
||||
@ -95,12 +102,28 @@ def error(input, threadID=-1):
|
||||
#os_exit(-1)
|
||||
#raise "error happend"
|
||||
|
||||
def print_element(type, lib, dir, name):
|
||||
def print_element(type, lib, dir, name, force=False):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 3:
|
||||
if debugLevel >= 3 \
|
||||
or force == True:
|
||||
debugLock.acquire()
|
||||
print(color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def print_compilator(myString):
|
||||
global debugColor
|
||||
global debugLock
|
||||
if debugColor == True:
|
||||
myString = myString.replace('\\n', '\n')
|
||||
myString = myString.replace('\\t', '\t')
|
||||
myString = myString.replace('error:', color_red+'error:'+color_default)
|
||||
myString = myString.replace('warning:', color_purple+'warning:'+color_default)
|
||||
myString = myString.replace('note:', color_green+'note:'+color_default)
|
||||
myString = re.sub(r'([/\w_-]+\.\w+):', r'-COLORIN-\1-COLOROUT-:', myString)
|
||||
myString = myString.replace('-COLORIN-', color_yellow)
|
||||
myString = myString.replace('-COLOROUT-', color_default)
|
||||
|
||||
debugLock.acquire()
|
||||
print(myString)
|
||||
debugLock.release()
|
||||
@ -29,7 +29,7 @@ printPrettyMode=False
|
||||
|
||||
def set_print_pretty_mode(val):
|
||||
global printPrettyMode
|
||||
if val==True:
|
||||
if val == True:
|
||||
printPrettyMode = True
|
||||
else:
|
||||
printPrettyMode = False
|
||||
@ -38,9 +38,10 @@ def get_print_pretty_mode():
|
||||
global printPrettyMode
|
||||
return printPrettyMode
|
||||
|
||||
def print_pretty(myString):
|
||||
def print_pretty(myString, force=False):
|
||||
global printPrettyMode
|
||||
if True==printPrettyMode:
|
||||
if printPrettyMode == True \
|
||||
or force == True:
|
||||
if myString[len(myString)-1]==' ' :
|
||||
tmpcmdLine = myString[:len(myString)-1]
|
||||
else :
|
||||
|
||||
@ -16,11 +16,21 @@ import os
|
||||
import subprocess
|
||||
import lutinTools
|
||||
import lutinEnv
|
||||
import shlex
|
||||
|
||||
queueLock = threading.Lock()
|
||||
workQueue = Queue.Queue()
|
||||
currentThreadWorking = 0
|
||||
threads = []
|
||||
# To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation
|
||||
currentIdExecution = 0
|
||||
errorExecution = {
|
||||
"id":-1,
|
||||
"cmd":"",
|
||||
"return":0,
|
||||
"err":"",
|
||||
"out":"",
|
||||
}
|
||||
|
||||
exitFlag = False # resuest stop of the thread
|
||||
isinit = False # the thread are initialized
|
||||
@ -40,30 +50,65 @@ def store_command(cmdLine, file):
|
||||
file2.close()
|
||||
|
||||
|
||||
def run_command(cmdLine, storeCmdLine=""):
|
||||
debug.debug(lutinEnv.print_pretty(cmdLine))
|
||||
try:
|
||||
retcode = subprocess.call(cmdLine, shell=True)
|
||||
except OSError as e:
|
||||
print >>sys.stderr, "Execution failed:", e
|
||||
|
||||
if retcode != 0:
|
||||
def run_command(cmdLine, storeCmdLine="", buildId=-1, file=""):
|
||||
global errorOccured
|
||||
errorOccured = True
|
||||
global exitFlag
|
||||
global currentIdExecution
|
||||
# prepare command line:
|
||||
args = shlex.split(cmdLine)
|
||||
#debug.verbose("cmd = " + str(args))
|
||||
try:
|
||||
# create the subprocess
|
||||
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
except subprocess.CalledProcessError as e:
|
||||
debug.error("subprocess.CalledProcessError : TODO ...")
|
||||
# launch the subprocess:
|
||||
output, err = p.communicate()
|
||||
# Check error :
|
||||
if p.returncode == 0:
|
||||
debug.debug(lutinEnv.print_pretty(cmdLine))
|
||||
queueLock.acquire()
|
||||
# TODO : Print the output all the time .... ==> to show warnings ...
|
||||
if buildId >= 0 and (output != "" or err != ""):
|
||||
debug.warning("output in subprocess compiling: '" + file + "'")
|
||||
if output != "":
|
||||
debug.print_compilator(output)
|
||||
if err != "":
|
||||
debug.print_compilator(err)
|
||||
queueLock.release()
|
||||
else:
|
||||
errorOccured = True
|
||||
exitFlag = True
|
||||
if retcode == 2:
|
||||
# if No ID : Not in a multiprocess mode ==> just stop here
|
||||
if buildId < 0:
|
||||
debug.debug(lutinEnv.print_pretty(cmdLine), force=True)
|
||||
debug.print_compilator(output)
|
||||
debug.print_compilator(err)
|
||||
if p.returncode == 2:
|
||||
debug.error("can not compile file ... [keyboard interrrupt]")
|
||||
else:
|
||||
debug.error("can not compile file ... ret : " + str(retcode))
|
||||
debug.error("can not compile file ... ret : " + str(p.returncode))
|
||||
else:
|
||||
# in multiprocess interface
|
||||
queueLock.acquire()
|
||||
# if an other write an error before, check if the current process is started before ==> then is the first error
|
||||
if errorExecution["id"] >= buildId:
|
||||
# nothing to do ...
|
||||
queueLock.release()
|
||||
return;
|
||||
errorExecution["id"] = buildId
|
||||
errorExecution["cmd"] = cmdLine
|
||||
errorExecution["return"] = p.returncode
|
||||
errorExecution["err"] = err,
|
||||
errorExecution["out"] = output,
|
||||
queueLock.release()
|
||||
# not write the command file...
|
||||
return
|
||||
|
||||
# write cmd line only after to prevent errors ...
|
||||
store_command(cmdLine, storeCmdLine)
|
||||
|
||||
|
||||
|
||||
|
||||
class myThread(threading.Thread):
|
||||
def __init__(self, threadID, lock, queue):
|
||||
threading.Thread.__init__(self)
|
||||
@ -76,7 +121,7 @@ class myThread(threading.Thread):
|
||||
global exitFlag
|
||||
global currentThreadWorking
|
||||
workingSet = False
|
||||
while False==exitFlag:
|
||||
while exitFlag == False:
|
||||
self.lock.acquire()
|
||||
if not self.queue.empty():
|
||||
if workingSet==False:
|
||||
@ -89,8 +134,8 @@ class myThread(threading.Thread):
|
||||
comment = data[2]
|
||||
cmdLine = data[1]
|
||||
cmdStoreFile = data[3]
|
||||
debug.print_element( "[" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3])
|
||||
run_command(cmdLine, cmdStoreFile)
|
||||
debug.print_element( "[" + str(data[4]) + "][" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3])
|
||||
run_command(cmdLine, cmdStoreFile, buildId=data[4], file=comment[3])
|
||||
else:
|
||||
debug.warning("unknow request command : " + data[0])
|
||||
else:
|
||||
@ -146,21 +191,24 @@ def un_init():
|
||||
|
||||
|
||||
def run_in_pool(cmdLine, comment, storeCmdLine=""):
|
||||
global currentIdExecution
|
||||
if processorAvaillable <= 1:
|
||||
debug.print_element(comment[0], comment[1], comment[2], comment[3])
|
||||
run_command(cmdLine, storeCmdLine)
|
||||
run_command(cmdLine, storeCmdLine, file=comment[3])
|
||||
return
|
||||
# multithreaded mode
|
||||
init()
|
||||
# Fill the queue
|
||||
queueLock.acquire()
|
||||
debug.verbose("add : in pool cmdLine")
|
||||
workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine])
|
||||
workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine, currentIdExecution])
|
||||
currentIdExecution +=1;
|
||||
queueLock.release()
|
||||
|
||||
|
||||
def pool_synchrosize():
|
||||
global errorOccured
|
||||
global errorExecution
|
||||
if processorAvaillable <= 1:
|
||||
#in this case : nothing to synchronise
|
||||
return
|
||||
@ -179,7 +227,17 @@ def pool_synchrosize():
|
||||
if False==errorOccured:
|
||||
debug.verbose("queue is empty")
|
||||
else:
|
||||
debug.debug("Thread return with error ... ==> stop all the pool")
|
||||
un_init()
|
||||
debug.error("Pool error occured ...")
|
||||
debug.debug("Thread return with error ... ==> stop all the pool")
|
||||
if errorExecution["id"] == -1:
|
||||
debug.error("Pool error occured ... (No return information on Pool)")
|
||||
return
|
||||
debug.error("Error in an pool element : [" + str(errorExecution["id"]) + "]", crash=False)
|
||||
debug.debug(lutinEnv.print_pretty(errorExecution["cmd"]), force=True)
|
||||
debug.print_compilator(str(errorExecution["out"][0]))
|
||||
debug.print_compilator(str(errorExecution["err"][0]))
|
||||
if errorExecution["return"] == 2:
|
||||
debug.error("can not compile file ... [keyboard interrrupt]")
|
||||
else:
|
||||
debug.error("can not compile file ... return value : " + str(errorExecution["return"]))
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user