[DEV] multiprocess availlable and stable (nearly as effitient as Make)

lutin.py

@@ -7,6 +7,7 @@ import fnmatch
 import lutinDebug as debug
 import lutinEnv
 import lutinModule
+import lutinMultiprocess
 
 """
 	Display the help of this makefile
@@ -30,6 +31,8 @@ def usage():
 	print "			Display makefile output in color"
 	print "		-f / --force"
 	print "			Force the rebuild without checking the dependency"
+	print "		-j= / --jobs"
+	print "			Specifies the number of jobs (commands) to run simultaneously."
 	print "	[properties] : keep in the sequency of the cible"
 	print "		-t=... / --target=..."
 	print "			(Android/Linux/MacOs/Windows) Select a target (by default the platform is the computer that compile this"
@@ -53,33 +56,55 @@ def usage():
 
 # preparse the argument to get the erbose element for debug mode
 def parseGenericArg(argument,active):
-	if argument == "-h" or argument == "--help":
+	if    argument == "-h" \
+	   or argument == "--help":
 		#display help
 		if active==False:
 			usage()
 		return True
+	elif    argument[:3] == "-j=" \
+	     or argument[:2] == "-j" \
+	     or argument[:7] == "--jobs=":
+		if active==True:
+			val = "1"
+			if argument[:3] == "-j=":
+				val = argument[3:]
+			elif argument[:2] == "-j":
+				if len(argument) == 2:
+					val = "1"
+				else:
+					val = argument[2:]
+			else:
+				val = argument[7:]
+			lutinMultiprocess.SetCoreNumber(int(val))
+		return True
+	elif    argument[:3] == "-v=" \
+	     or argument[:2] == "-v" \
+	     or argument[:10] == "--verbose=" \
+	     or argument[:9] == "--verbose":
+		if active==True:
+			val = "5"
+			if argument[:3] == "-v=":
+				val = argument[3:]
 			elif argument[:2] == "-v":
-		if active==True:
+				if len(argument) == 2:
-			if len(argument)==2:
+					val = "5"
-				debug.SetLevel(5)
 				else:
-				debug.SetLevel(int(argument[2:]))
+					val = argument[2:]
+			else:
+				if len(argument) == 9:
+					val = "5"
+				else:
+					val = argument[10:]
+			debug.SetLevel(int(val))
 		return True
-	elif argument[:9] == "--verbose":
+	elif    argument == "-c" \
-		if active==True:
+	     or argument == "--color":
-			if len(argument)==9:
-				debug.SetLevel(5)
-			else:
-				if argument[:10] == "--verbose=":
-					debug.SetLevel(int(argument[10:]))
-				else:
-					debug.SetLevel(int(argument[9:]))
-		return True
-	elif argument == "-c" or argument == "--color":
 		if active==True:
 			debug.EnableColor()
 		return True
-	elif argument == "-f" or argument == "--force":
+	elif    argument == "-f" \
+	     or argument == "--force":
 		if active==True:
 			lutinEnv.SetForceMode(True)
 		return True
@@ -169,6 +194,8 @@ def Start():
 		if target == None:
 			target = lutinTarget.TargetLoad(targetName, compilator, mode)
 		target.Build("all")
+	# stop all started threads 
+	lutinMultiprocess.UnInit()
 
 """
 	When the user use with make.py we initialise ourself

lutinDebug.py

@@ -2,6 +2,8 @@
 import os
 import thread
 import lutinMultiprocess
+import threading
+
 debugLevel=3
 debugColor=False
 
@@ -13,6 +15,9 @@ color_blue   = ""
 color_purple = ""
 color_cyan   = ""
 
+
+debugLock = threading.Lock()
+
 def SetLevel(id):
 	global debugLevel
 	debugLevel = id
@@ -37,24 +42,44 @@ def EnableColor():
 	color_cyan   = "\033[36m"
 
 def verbose(input):
+	global debugLock
+	global debugLevel
 	if debugLevel >= 5:
-		print color_blue + input + color_default
+		debugLock.acquire()
+		print(color_blue + input + color_default)
+		debugLock.release()
 
 def debug(input):
+	global debugLock
+	global debugLevel
 	if debugLevel >= 4:
-		print color_green + input + color_default
+		debugLock.acquire()
+		print(color_green + input + color_default)
+		debugLock.release()
 
 def info(input):
+	global debugLock
+	global debugLevel
 	if debugLevel >= 3:
-		print input + color_default
+		debugLock.acquire()
+		print(input + color_default)
+		debugLock.release()
 
 def warning(input):
+	global debugLock
+	global debugLevel
 	if debugLevel >= 2:
-		print color_purple + "[WARNING] " + input + color_default
+		debugLock.acquire()
+		print(color_purple + "[WARNING] " + input + color_default)
+		debugLock.release()
 
 def error(input):
+	global debugLock
+	global debugLevel
 	if debugLevel >= 1:
-		print color_red + "[ERROR] " + input + color_default
+		debugLock.acquire()
+		print(color_red + "[ERROR] " + input + color_default)
+		debugLock.release()
 	lutinMultiprocess.ErrorOccured()
 	thread.interrupt_main()
 	exit(-1)
@@ -62,5 +87,11 @@ def error(input):
 	#raise "error happend"
 
 def printElement(type, lib, dir, name):
+	global debugLock
+	global debugLevel
 	if debugLevel >= 3:
-		print color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default
+		debugLock.acquire()
+		print(color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default)
+		debugLock.release()
+
+

lutinModule.py

@@ -168,7 +168,7 @@ class module:
 		lutinTools.CreateDirectoryOfFile(file_dst)
 		comment = ["c++", self.name, "<==", file]
 		#process element
-		lutinMultiprocess.RunInPool(cmdLine, comment)
+		lutinMultiprocess.RunInPool(cmdLine, comment, file_cmd)
 		return file_dst
 	
 	###############################################################################
@@ -197,7 +197,7 @@ class module:
 		lutinTools.CreateDirectoryOfFile(file_dst)
 		comment = ["c", self.name, "<==", file]
 		# process element
-		lutinMultiprocess.RunInPool(cmdLine, comment)
+		lutinMultiprocess.RunInPool(cmdLine, comment, file_cmd)
 		return file_dst
 	
 	

lutinMultiprocess.py

@@ -5,17 +5,30 @@ import threading
 import time
 import Queue
 import os
+import subprocess
 
-
+def RunCommand(cmdLine, storeCmdLine=""):
-def RunCommand(cmdLine):
 	debug.debug(cmdLine)
-	ret = os.system(cmdLine)
+	retcode = -1
+	try:
+		retcode = subprocess.call(cmdLine, shell=True)
+	except OSError as e:
+		print >>sys.stderr, "Execution failed:", e
+	
+	
+	# write cmd line only after to prvent errors ...
+	if storeCmdLine!="":
+		file2 = open(storeCmdLine, "w")
+		file2.write(cmdLine)
+		file2.flush()
+		file2.close()
 	# TODO : Use "subprocess" instead ==> permit to pipline the renderings ...
-	if ret != 0:
+	
-		if ret == 2:
+	if retcode != 0:
+		if retcode == 2:
 			debug.error("can not compile file ... [keyboard interrrupt]")
 		else:
-			debug.error("can not compile file ... ret : " + str(ret))
+			debug.error("can not compile file ... ret : " + str(retcode))
 
 exitFlag = False
 
@@ -27,32 +40,40 @@ class myThread(threading.Thread):
 		self.queue = queue
 		self.lock = lock
 	def run(self):
-		print("Starting " + self.name)
+		debug.verbose("Starting " + self.name)
 		global exitFlag
-		global queueLock
+		global currentThreadWorking
-		global workQueue
+		workingSet = False
 		while False==exitFlag:
 			self.lock.acquire()
 			if not self.queue.empty():
+				if workingSet==False:
+					currentThreadWorking += 1
+					workingSet = True
 				data = self.queue.get()
 				self.lock.release()
-				print "%s processing %s" % (self.name, data[0])
+				debug.verbose(self.name + " processing '" + data[0] + "'")
 				if data[0]=="cmdLine":
 					comment = data[2]
 					cmdLine = data[1]
-					debug.printElement(comment[0], comment[1], comment[2], comment[3])
+					cmdStoreFile = data[3]
-					RunCommand(cmdLine)
+					debug.printElement( "[" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3])
+					RunCommand(cmdLine, cmdStoreFile)
 				else:
 					debug.warning("unknow request command : " + data[0])
 			else:
+				if workingSet==True:
+					currentThreadWorking -= 1
+					workingSet=False
 				# no element to parse, just wait ...
 				self.lock.release()
 				time.sleep(0.2)
 		# kill requested ...
-		print("Exiting " + self.name)
+		debug.verbose("Exiting " + self.name)
 
 queueLock = threading.Lock()
 workQueue = Queue.Queue()
+currentThreadWorking = 0
 threads = []
 
 isInit = False
@@ -63,7 +84,9 @@ def ErrorOccured():
 	exitFlag = True
 
 def SetCoreNumber(numberOfcore):
+	global processorAvaillable
 	processorAvaillable = numberOfcore
+	debug.debug(" set number of core for multi process compilation : " + str(processorAvaillable))
 	# nothing else to do
 
 def Init():
@@ -91,23 +114,23 @@ def UnInit():
 	if processorAvaillable > 1:
 		# Wait for all threads to complete
 		for tmp in threads:
-			print("join thread ... \n")
+			debug.verbose("join thread ...")
 			tmp.join()
-		print "Exiting Main Thread"
+		debug.verbose("Exiting ALL Threads")
 
 
 
-def RunInPool(cmdLine, comment):
+def RunInPool(cmdLine, comment, storeCmdLine=""):
 	if processorAvaillable <= 1:
 		debug.printElement(comment[0], comment[1], comment[2], comment[3])
-		RunCommand(cmdLine)
+		RunCommand(cmdLine, storeCmdLine)
 		return
 	# multithreaded mode
 	Init()
 	# Fill the queue
 	queueLock.acquire()
 	debug.verbose("add : in pool cmdLine")
-	workQueue.put(["cmdLine", cmdLine, comment])
+	workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine])
 	queueLock.release()
 	
 
@@ -121,6 +144,9 @@ def PoolSynchrosize():
 	while not workQueue.empty():
 		time.sleep(0.2)
 		pass
+	# Wait all thread have ended their current process
+	while currentThreadWorking != 0:
+		time.sleep(0.2)
+		pass
 	debug.verbose("queue is empty")
-	os.path.flush()