[DEV] separate multiprocess log and display it corectly with the compile line

lutinDebug.py

@@ -11,6 +11,7 @@ import os
 import thread
 import lutinMultiprocess
 import threading
+import re
 
 debugLevel=3
 debugColor=False
@@ -49,45 +50,51 @@ def enable_color():
 	global color_cyan
 	color_cyan   = "\033[36m"
 
-def verbose(input):
+def verbose(input, force=False):
 	global debugLock
 	global debugLevel
-	if debugLevel >= 5:
+	if    debugLevel >= 5 \
+	   or force == True:
 		debugLock.acquire()
 		print(color_blue + input + color_default)
 		debugLock.release()
 
-def debug(input):
+def debug(input, force=False):
 	global debugLock
 	global debugLevel
-	if debugLevel >= 4:
+	if    debugLevel >= 4 \
+	   or force == True:
 		debugLock.acquire()
 		print(color_green + input + color_default)
 		debugLock.release()
 
-def info(input):
+def info(input, force=False):
 	global debugLock
 	global debugLevel
-	if debugLevel >= 3:
+	if    debugLevel >= 3 \
+	   or force == True:
 		debugLock.acquire()
 		print(input + color_default)
 		debugLock.release()
 
-def warning(input):
+def warning(input, force=False):
 	global debugLock
 	global debugLevel
-	if debugLevel >= 2:
+	if    debugLevel >= 2 \
+	   or force == True:
 		debugLock.acquire()
 		print(color_purple + "[WARNING] " + input + color_default)
 		debugLock.release()
 
-def error(input, threadID=-1):
+def error(input, threadID=-1, force=False, crash=True):
 	global debugLock
 	global debugLevel
-	if debugLevel >= 1:
+	if    debugLevel >= 1 \
+	   or force == True:
 		debugLock.acquire()
 		print(color_red + "[ERROR] " + input + color_default)
 		debugLock.release()
+	if crash==True:
 		lutinMultiprocess.error_occured()
 		if threadID != -1:
 			thread.interrupt_main()
@@ -95,12 +102,28 @@ def error(input, threadID=-1):
 		#os_exit(-1)
 		#raise "error happend"
 
-def print_element(type, lib, dir, name):
+def print_element(type, lib, dir, name, force=False):
 	global debugLock
 	global debugLevel
-	if debugLevel >= 3:
+	if    debugLevel >= 3 \
+	   or force == True:
 		debugLock.acquire()
 		print(color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default)
 		debugLock.release()
 
+def print_compilator(myString):
+	global debugColor
+	global debugLock
+	if debugColor == True:
+		myString = myString.replace('\\n', '\n')
+		myString = myString.replace('\\t', '\t')
+		myString = myString.replace('error:', color_red+'error:'+color_default)
+		myString = myString.replace('warning:', color_purple+'warning:'+color_default)
+		myString = myString.replace('note:', color_green+'note:'+color_default)
+		myString = re.sub(r'([/\w_-]+\.\w+):', r'-COLORIN-\1-COLOROUT-:', myString)
+		myString = myString.replace('-COLORIN-', color_yellow)
+		myString = myString.replace('-COLOROUT-', color_default)
 	
+	debugLock.acquire()
+	print(myString)
+	debugLock.release()

lutinEnv.py

@@ -29,7 +29,7 @@ printPrettyMode=False
 
 def set_print_pretty_mode(val):
 	global printPrettyMode
-	if val==True:
+	if val == True:
 		printPrettyMode = True
 	else:
 		printPrettyMode = False
@@ -38,9 +38,10 @@ def get_print_pretty_mode():
 	global printPrettyMode
 	return printPrettyMode
 
-def print_pretty(myString):
+def print_pretty(myString, force=False):
 	global printPrettyMode
-	if True==printPrettyMode:
+	if    printPrettyMode == True \
+	   or force == True:
 		if myString[len(myString)-1]==' ' : 
 			tmpcmdLine = myString[:len(myString)-1]
 		else :

lutinMultiprocess.py

@@ -16,11 +16,21 @@ import os
 import subprocess
 import lutinTools
 import lutinEnv
+import shlex
 
 queueLock = threading.Lock()
 workQueue = Queue.Queue()
 currentThreadWorking = 0
 threads = []
+# To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation
+currentIdExecution = 0
+errorExecution = {
+	"id":-1,
+	"cmd":"",
+	"return":0,
+	"err":"",
+	"out":"",
+}
 
 exitFlag = False # resuest stop of the thread
 isinit = False # the thread are initialized
@@ -40,30 +50,65 @@ def store_command(cmdLine, file):
 		file2.close()
 
 
-def run_command(cmdLine, storeCmdLine=""):
+def run_command(cmdLine, storeCmdLine="", buildId=-1, file=""):
-	debug.debug(lutinEnv.print_pretty(cmdLine))
-	try:
-		retcode = subprocess.call(cmdLine, shell=True)
-	except OSError as e:
-		print >>sys.stderr, "Execution failed:", e
-	
-	if retcode != 0:
 	global errorOccured
-		errorOccured = True
 	global exitFlag
+	global currentIdExecution
+	# prepare command line:
+	args = shlex.split(cmdLine)
+	#debug.verbose("cmd = " + str(args))
+	try:
+		# create the subprocess
+		p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+	except subprocess.CalledProcessError as e:
+		debug.error("subprocess.CalledProcessError : TODO ...")
+	# launch the subprocess:
+	output, err = p.communicate()
+	# Check error :
+	if p.returncode == 0:
+		debug.debug(lutinEnv.print_pretty(cmdLine))
+		queueLock.acquire()
+		# TODO : Print the output all the time .... ==> to show warnings ...
+		if buildId >= 0 and (output != "" or err != ""):
+			debug.warning("output in subprocess compiling: '" + file + "'")
+		if output != "":
+			debug.print_compilator(output)
+		if err != "":
+			debug.print_compilator(err)
+		queueLock.release()
+	else:
+		errorOccured = True
 		exitFlag = True
-		if retcode == 2:
+		# if No ID : Not in a multiprocess mode ==> just stop here
+		if buildId < 0:
+			debug.debug(lutinEnv.print_pretty(cmdLine), force=True)
+			debug.print_compilator(output)
+			debug.print_compilator(err)
+			if p.returncode == 2:
 				debug.error("can not compile file ... [keyboard interrrupt]")
 			else:
-			debug.error("can not compile file ... ret : " + str(retcode))
+				debug.error("can not compile file ... ret : " + str(p.returncode))
+		else:
+			# in multiprocess interface
+			queueLock.acquire()
+			# if an other write an error before, check if the current process is started before ==> then is the first error
+			if errorExecution["id"] >= buildId:
+				# nothing to do ...
+				queueLock.release()
+				return;
+			errorExecution["id"] = buildId
+			errorExecution["cmd"] = cmdLine
+			errorExecution["return"] = p.returncode
+			errorExecution["err"] = err,
+			errorExecution["out"] = output,
+			queueLock.release()
+		# not write the command file...
 		return
-	
 	# write cmd line only after to prevent errors ...
 	store_command(cmdLine, storeCmdLine)
 
 
 
-
 class myThread(threading.Thread):
 	def __init__(self, threadID, lock, queue):
 		threading.Thread.__init__(self)
@@ -76,7 +121,7 @@ class myThread(threading.Thread):
 		global exitFlag
 		global currentThreadWorking
 		workingSet = False
-		while False==exitFlag:
+		while exitFlag == False:
 			self.lock.acquire()
 			if not self.queue.empty():
 				if workingSet==False:
@@ -89,8 +134,8 @@ class myThread(threading.Thread):
 					comment = data[2]
 					cmdLine = data[1]
 					cmdStoreFile = data[3]
-					debug.print_element( "[" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3])
+					debug.print_element( "[" + str(data[4]) + "][" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3])
-					run_command(cmdLine, cmdStoreFile)
+					run_command(cmdLine, cmdStoreFile, buildId=data[4], file=comment[3])
 				else:
 					debug.warning("unknow request command : " + data[0])
 			else:
@@ -146,21 +191,24 @@ def un_init():
 
 
 def run_in_pool(cmdLine, comment, storeCmdLine=""):
+	global currentIdExecution
 	if processorAvaillable <= 1:
 		debug.print_element(comment[0], comment[1], comment[2], comment[3])
-		run_command(cmdLine, storeCmdLine)
+		run_command(cmdLine, storeCmdLine, file=comment[3])
 		return
 	# multithreaded mode
 	init()
 	# Fill the queue
 	queueLock.acquire()
 	debug.verbose("add : in pool cmdLine")
-	workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine])
+	workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine, currentIdExecution])
+	currentIdExecution +=1;
 	queueLock.release()
 	
 
 def pool_synchrosize():
 	global errorOccured
+	global errorExecution
 	if processorAvaillable <= 1:
 		#in this case : nothing to synchronise
 		return
@@ -179,7 +227,17 @@ def pool_synchrosize():
 	if False==errorOccured:
 		debug.verbose("queue is empty")
 	else:
-		debug.debug("Thread return with error ... ==> stop all the pool")
 		un_init()
-		debug.error("Pool error occured ...")
+		debug.debug("Thread return with error ... ==> stop all the pool")
+		if errorExecution["id"] == -1:
+			debug.error("Pool error occured ... (No return information on Pool)")
+			return
+		debug.error("Error in an pool element : [" + str(errorExecution["id"]) + "]", crash=False)
+		debug.debug(lutinEnv.print_pretty(errorExecution["cmd"]), force=True)
+		debug.print_compilator(str(errorExecution["out"][0]))
+		debug.print_compilator(str(errorExecution["err"][0]))
+		if errorExecution["return"] == 2:
+			debug.error("can not compile file ... [keyboard interrrupt]")
+		else:
+			debug.error("can not compile file ... return value : " + str(errorExecution["return"]))