[DEV] multiprocess availlable and stable (nearly as effitient as Make)
This commit is contained in:
parent
6c1649f36f
commit
e0cebcb1a0
61
lutin.py
61
lutin.py
@ -7,6 +7,7 @@ import fnmatch
|
||||
import lutinDebug as debug
|
||||
import lutinEnv
|
||||
import lutinModule
|
||||
import lutinMultiprocess
|
||||
|
||||
"""
|
||||
Display the help of this makefile
|
||||
@ -30,6 +31,8 @@ def usage():
|
||||
print " Display makefile output in color"
|
||||
print " -f / --force"
|
||||
print " Force the rebuild without checking the dependency"
|
||||
print " -j= / --jobs"
|
||||
print " Specifies the number of jobs (commands) to run simultaneously."
|
||||
print " [properties] : keep in the sequency of the cible"
|
||||
print " -t=... / --target=..."
|
||||
print " (Android/Linux/MacOs/Windows) Select a target (by default the platform is the computer that compile this"
|
||||
@ -53,33 +56,55 @@ def usage():
|
||||
|
||||
# preparse the argument to get the erbose element for debug mode
|
||||
def parseGenericArg(argument,active):
|
||||
if argument == "-h" or argument == "--help":
|
||||
if argument == "-h" \
|
||||
or argument == "--help":
|
||||
#display help
|
||||
if active==False:
|
||||
usage()
|
||||
return True
|
||||
elif argument[:2] == "-v":
|
||||
elif argument[:3] == "-j=" \
|
||||
or argument[:2] == "-j" \
|
||||
or argument[:7] == "--jobs=":
|
||||
if active==True:
|
||||
if len(argument)==2:
|
||||
debug.SetLevel(5)
|
||||
else:
|
||||
debug.SetLevel(int(argument[2:]))
|
||||
return True
|
||||
elif argument[:9] == "--verbose":
|
||||
if active==True:
|
||||
if len(argument)==9:
|
||||
debug.SetLevel(5)
|
||||
else:
|
||||
if argument[:10] == "--verbose=":
|
||||
debug.SetLevel(int(argument[10:]))
|
||||
val = "1"
|
||||
if argument[:3] == "-j=":
|
||||
val = argument[3:]
|
||||
elif argument[:2] == "-j":
|
||||
if len(argument) == 2:
|
||||
val = "1"
|
||||
else:
|
||||
debug.SetLevel(int(argument[9:]))
|
||||
val = argument[2:]
|
||||
else:
|
||||
val = argument[7:]
|
||||
lutinMultiprocess.SetCoreNumber(int(val))
|
||||
return True
|
||||
elif argument == "-c" or argument == "--color":
|
||||
elif argument[:3] == "-v=" \
|
||||
or argument[:2] == "-v" \
|
||||
or argument[:10] == "--verbose=" \
|
||||
or argument[:9] == "--verbose":
|
||||
if active==True:
|
||||
val = "5"
|
||||
if argument[:3] == "-v=":
|
||||
val = argument[3:]
|
||||
elif argument[:2] == "-v":
|
||||
if len(argument) == 2:
|
||||
val = "5"
|
||||
else:
|
||||
val = argument[2:]
|
||||
else:
|
||||
if len(argument) == 9:
|
||||
val = "5"
|
||||
else:
|
||||
val = argument[10:]
|
||||
debug.SetLevel(int(val))
|
||||
return True
|
||||
elif argument == "-c" \
|
||||
or argument == "--color":
|
||||
if active==True:
|
||||
debug.EnableColor()
|
||||
return True
|
||||
elif argument == "-f" or argument == "--force":
|
||||
elif argument == "-f" \
|
||||
or argument == "--force":
|
||||
if active==True:
|
||||
lutinEnv.SetForceMode(True)
|
||||
return True
|
||||
@ -169,6 +194,8 @@ def Start():
|
||||
if target == None:
|
||||
target = lutinTarget.TargetLoad(targetName, compilator, mode)
|
||||
target.Build("all")
|
||||
# stop all started threads
|
||||
lutinMultiprocess.UnInit()
|
||||
|
||||
"""
|
||||
When the user use with make.py we initialise ourself
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
import os
|
||||
import thread
|
||||
import lutinMultiprocess
|
||||
import threading
|
||||
|
||||
debugLevel=3
|
||||
debugColor=False
|
||||
|
||||
@ -13,6 +15,9 @@ color_blue = ""
|
||||
color_purple = ""
|
||||
color_cyan = ""
|
||||
|
||||
|
||||
debugLock = threading.Lock()
|
||||
|
||||
def SetLevel(id):
|
||||
global debugLevel
|
||||
debugLevel = id
|
||||
@ -37,24 +42,44 @@ def EnableColor():
|
||||
color_cyan = "\033[36m"
|
||||
|
||||
def verbose(input):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 5:
|
||||
print color_blue + input + color_default
|
||||
debugLock.acquire()
|
||||
print(color_blue + input + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def debug(input):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 4:
|
||||
print color_green + input + color_default
|
||||
debugLock.acquire()
|
||||
print(color_green + input + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def info(input):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 3:
|
||||
print input + color_default
|
||||
debugLock.acquire()
|
||||
print(input + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def warning(input):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 2:
|
||||
print color_purple + "[WARNING] " + input + color_default
|
||||
debugLock.acquire()
|
||||
print(color_purple + "[WARNING] " + input + color_default)
|
||||
debugLock.release()
|
||||
|
||||
def error(input):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 1:
|
||||
print color_red + "[ERROR] " + input + color_default
|
||||
debugLock.acquire()
|
||||
print(color_red + "[ERROR] " + input + color_default)
|
||||
debugLock.release()
|
||||
lutinMultiprocess.ErrorOccured()
|
||||
thread.interrupt_main()
|
||||
exit(-1)
|
||||
@ -62,5 +87,11 @@ def error(input):
|
||||
#raise "error happend"
|
||||
|
||||
def printElement(type, lib, dir, name):
|
||||
global debugLock
|
||||
global debugLevel
|
||||
if debugLevel >= 3:
|
||||
print color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default
|
||||
debugLock.acquire()
|
||||
print(color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default)
|
||||
debugLock.release()
|
||||
|
||||
|
||||
|
||||
@ -168,7 +168,7 @@ class module:
|
||||
lutinTools.CreateDirectoryOfFile(file_dst)
|
||||
comment = ["c++", self.name, "<==", file]
|
||||
#process element
|
||||
lutinMultiprocess.RunInPool(cmdLine, comment)
|
||||
lutinMultiprocess.RunInPool(cmdLine, comment, file_cmd)
|
||||
return file_dst
|
||||
|
||||
###############################################################################
|
||||
@ -197,7 +197,7 @@ class module:
|
||||
lutinTools.CreateDirectoryOfFile(file_dst)
|
||||
comment = ["c", self.name, "<==", file]
|
||||
# process element
|
||||
lutinMultiprocess.RunInPool(cmdLine, comment)
|
||||
lutinMultiprocess.RunInPool(cmdLine, comment, file_cmd)
|
||||
return file_dst
|
||||
|
||||
|
||||
|
||||
@ -5,17 +5,30 @@ import threading
|
||||
import time
|
||||
import Queue
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
|
||||
def RunCommand(cmdLine):
|
||||
def RunCommand(cmdLine, storeCmdLine=""):
|
||||
debug.debug(cmdLine)
|
||||
ret = os.system(cmdLine)
|
||||
retcode = -1
|
||||
try:
|
||||
retcode = subprocess.call(cmdLine, shell=True)
|
||||
except OSError as e:
|
||||
print >>sys.stderr, "Execution failed:", e
|
||||
|
||||
|
||||
# write cmd line only after to prvent errors ...
|
||||
if storeCmdLine!="":
|
||||
file2 = open(storeCmdLine, "w")
|
||||
file2.write(cmdLine)
|
||||
file2.flush()
|
||||
file2.close()
|
||||
# TODO : Use "subprocess" instead ==> permit to pipline the renderings ...
|
||||
if ret != 0:
|
||||
if ret == 2:
|
||||
|
||||
if retcode != 0:
|
||||
if retcode == 2:
|
||||
debug.error("can not compile file ... [keyboard interrrupt]")
|
||||
else:
|
||||
debug.error("can not compile file ... ret : " + str(ret))
|
||||
debug.error("can not compile file ... ret : " + str(retcode))
|
||||
|
||||
exitFlag = False
|
||||
|
||||
@ -27,32 +40,40 @@ class myThread(threading.Thread):
|
||||
self.queue = queue
|
||||
self.lock = lock
|
||||
def run(self):
|
||||
print("Starting " + self.name)
|
||||
debug.verbose("Starting " + self.name)
|
||||
global exitFlag
|
||||
global queueLock
|
||||
global workQueue
|
||||
global currentThreadWorking
|
||||
workingSet = False
|
||||
while False==exitFlag:
|
||||
self.lock.acquire()
|
||||
if not self.queue.empty():
|
||||
if workingSet==False:
|
||||
currentThreadWorking += 1
|
||||
workingSet = True
|
||||
data = self.queue.get()
|
||||
self.lock.release()
|
||||
print "%s processing %s" % (self.name, data[0])
|
||||
debug.verbose(self.name + " processing '" + data[0] + "'")
|
||||
if data[0]=="cmdLine":
|
||||
comment = data[2]
|
||||
cmdLine = data[1]
|
||||
debug.printElement(comment[0], comment[1], comment[2], comment[3])
|
||||
RunCommand(cmdLine)
|
||||
cmdStoreFile = data[3]
|
||||
debug.printElement( "[" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3])
|
||||
RunCommand(cmdLine, cmdStoreFile)
|
||||
else:
|
||||
debug.warning("unknow request command : " + data[0])
|
||||
else:
|
||||
if workingSet==True:
|
||||
currentThreadWorking -= 1
|
||||
workingSet=False
|
||||
# no element to parse, just wait ...
|
||||
self.lock.release()
|
||||
time.sleep(0.2)
|
||||
# kill requested ...
|
||||
print("Exiting " + self.name)
|
||||
debug.verbose("Exiting " + self.name)
|
||||
|
||||
queueLock = threading.Lock()
|
||||
workQueue = Queue.Queue()
|
||||
currentThreadWorking = 0
|
||||
threads = []
|
||||
|
||||
isInit = False
|
||||
@ -63,7 +84,9 @@ def ErrorOccured():
|
||||
exitFlag = True
|
||||
|
||||
def SetCoreNumber(numberOfcore):
|
||||
global processorAvaillable
|
||||
processorAvaillable = numberOfcore
|
||||
debug.debug(" set number of core for multi process compilation : " + str(processorAvaillable))
|
||||
# nothing else to do
|
||||
|
||||
def Init():
|
||||
@ -91,23 +114,23 @@ def UnInit():
|
||||
if processorAvaillable > 1:
|
||||
# Wait for all threads to complete
|
||||
for tmp in threads:
|
||||
print("join thread ... \n")
|
||||
debug.verbose("join thread ...")
|
||||
tmp.join()
|
||||
print "Exiting Main Thread"
|
||||
debug.verbose("Exiting ALL Threads")
|
||||
|
||||
|
||||
|
||||
def RunInPool(cmdLine, comment):
|
||||
def RunInPool(cmdLine, comment, storeCmdLine=""):
|
||||
if processorAvaillable <= 1:
|
||||
debug.printElement(comment[0], comment[1], comment[2], comment[3])
|
||||
RunCommand(cmdLine)
|
||||
RunCommand(cmdLine, storeCmdLine)
|
||||
return
|
||||
# multithreaded mode
|
||||
Init()
|
||||
# Fill the queue
|
||||
queueLock.acquire()
|
||||
debug.verbose("add : in pool cmdLine")
|
||||
workQueue.put(["cmdLine", cmdLine, comment])
|
||||
workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine])
|
||||
queueLock.release()
|
||||
|
||||
|
||||
@ -121,6 +144,9 @@ def PoolSynchrosize():
|
||||
while not workQueue.empty():
|
||||
time.sleep(0.2)
|
||||
pass
|
||||
# Wait all thread have ended their current process
|
||||
while currentThreadWorking != 0:
|
||||
time.sleep(0.2)
|
||||
pass
|
||||
debug.verbose("queue is empty")
|
||||
os.path.flush()
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user