atria-tools/lutin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[DEV] multiprocess availlable and stable (nearly as effitient as Make)

Browse Source
This commit is contained in:
Edouard DUPIN 2013-04-24 21:30:46 +02:00
parent 6c1649f36f
commit e0cebcb1a0
4 changed files with 128 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
lutin.py
View File

@ -7,6 +7,7 @@ import fnmatch
import lutinDebug as debug import lutinDebug as debug
import lutinEnv import lutinEnv
import lutinModule import lutinModule
import lutinMultiprocess
""" """
Display the help of this makefile Display the help of this makefile
@ -30,6 +31,8 @@ def usage():
print " Display makefile output in color" print " Display makefile output in color"
print " -f / --force" print " -f / --force"
print " Force the rebuild without checking the dependency" print " Force the rebuild without checking the dependency"
print " -j= / --jobs"
print " Specifies the number of jobs (commands) to run simultaneously."
print " [properties] : keep in the sequency of the cible" print " [properties] : keep in the sequency of the cible"
print " -t=... / --target=..." print " -t=... / --target=..."
print " (Android/Linux/MacOs/Windows) Select a target (by default the platform is the computer that compile this" print " (Android/Linux/MacOs/Windows) Select a target (by default the platform is the computer that compile this"
@ -53,33 +56,55 @@ def usage():
# preparse the argument to get the erbose element for debug mode # preparse the argument to get the erbose element for debug mode
def parseGenericArg(argument,active): def parseGenericArg(argument,active):
if argument == "-h" or argument == "--help": if argument == "-h" \
or argument == "--help":
#display help #display help
if active==False: if active==False:
usage() usage()
return True return True
elif argument[:2] == "-v": elif argument[:3] == "-j=" \
or argument[:2] == "-j" \
or argument[:7] == "--jobs=":
if active==True: if active==True:
val = "1"
if argument[:3] == "-j=":
val = argument[3:]
elif argument[:2] == "-j":
if len(argument) == 2: if len(argument) == 2:
debug.SetLevel(5) val = "1"
else: else:
debug.SetLevel(int(argument[2:])) val = argument[2:]
else:
val = argument[7:]
lutinMultiprocess.SetCoreNumber(int(val))
return True return True
elif argument[:9] == "--verbose": elif argument[:3] == "-v=" \
or argument[:2] == "-v" \
or argument[:10] == "--verbose=" \
or argument[:9] == "--verbose":
if active==True: if active==True:
val = "5"
if argument[:3] == "-v=":
val = argument[3:]
elif argument[:2] == "-v":
if len(argument) == 2:
val = "5"
else:
val = argument[2:]
else:
if len(argument) == 9: if len(argument) == 9:
debug.SetLevel(5) val = "5"
else: else:
if argument[:10] == "--verbose=": val = argument[10:]
debug.SetLevel(int(argument[10:])) debug.SetLevel(int(val))
else:
debug.SetLevel(int(argument[9:]))
return True return True
elif argument == "-c" or argument == "--color": elif argument == "-c" \
or argument == "--color":
if active==True: if active==True:
debug.EnableColor() debug.EnableColor()
return True return True
elif argument == "-f" or argument == "--force": elif argument == "-f" \
or argument == "--force":
if active==True: if active==True:
lutinEnv.SetForceMode(True) lutinEnv.SetForceMode(True)
return True return True
@ -169,6 +194,8 @@ def Start():
if target == None: if target == None:
target = lutinTarget.TargetLoad(targetName, compilator, mode) target = lutinTarget.TargetLoad(targetName, compilator, mode)
target.Build("all") target.Build("all")
# stop all started threads
lutinMultiprocess.UnInit()
""" """
When the user use with make.py we initialise ourself When the user use with make.py we initialise ourself

43
lutinDebug.py
View File

@ -2,6 +2,8 @@
import os import os
import thread import thread
import lutinMultiprocess import lutinMultiprocess
import threading
debugLevel=3 debugLevel=3
debugColor=False debugColor=False
@ -13,6 +15,9 @@ color_blue = ""
color_purple = "" color_purple = ""
color_cyan = "" color_cyan = ""
debugLock = threading.Lock()
def SetLevel(id): def SetLevel(id):
global debugLevel global debugLevel
debugLevel = id debugLevel = id
@ -37,24 +42,44 @@ def EnableColor():
color_cyan = "\033[36m" color_cyan = "\033[36m"
def verbose(input): def verbose(input):
global debugLock
global debugLevel
if debugLevel >= 5: if debugLevel >= 5:
print color_blue + input + color_default debugLock.acquire()
print(color_blue + input + color_default)
debugLock.release()
def debug(input): def debug(input):
global debugLock
global debugLevel
if debugLevel >= 4: if debugLevel >= 4:
print color_green + input + color_default debugLock.acquire()
print(color_green + input + color_default)
debugLock.release()
def info(input): def info(input):
global debugLock
global debugLevel
if debugLevel >= 3: if debugLevel >= 3:
print input + color_default debugLock.acquire()
print(input + color_default)
debugLock.release()
def warning(input): def warning(input):
global debugLock
global debugLevel
if debugLevel >= 2: if debugLevel >= 2:
print color_purple + "[WARNING] " + input + color_default debugLock.acquire()
print(color_purple + "[WARNING] " + input + color_default)
debugLock.release()
def error(input): def error(input):
global debugLock
global debugLevel
if debugLevel >= 1: if debugLevel >= 1:
print color_red + "[ERROR] " + input + color_default debugLock.acquire()
print(color_red + "[ERROR] " + input + color_default)
debugLock.release()
lutinMultiprocess.ErrorOccured() lutinMultiprocess.ErrorOccured()
thread.interrupt_main() thread.interrupt_main()
exit(-1) exit(-1)
@ -62,5 +87,11 @@ def error(input):
#raise "error happend" #raise "error happend"
def printElement(type, lib, dir, name): def printElement(type, lib, dir, name):
global debugLock
global debugLevel
if debugLevel >= 3: if debugLevel >= 3:
print color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default debugLock.acquire()
print(color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default)
debugLock.release()

4
lutinModule.py
View File

@ -168,7 +168,7 @@ class module:
lutinTools.CreateDirectoryOfFile(file_dst) lutinTools.CreateDirectoryOfFile(file_dst)
comment = ["c++", self.name, "<==", file] comment = ["c++", self.name, "<==", file]
#process element #process element
lutinMultiprocess.RunInPool(cmdLine, comment) lutinMultiprocess.RunInPool(cmdLine, comment, file_cmd)
return file_dst return file_dst
############################################################################### ###############################################################################
@ -197,7 +197,7 @@ class module:
lutinTools.CreateDirectoryOfFile(file_dst) lutinTools.CreateDirectoryOfFile(file_dst)
comment = ["c", self.name, "<==", file] comment = ["c", self.name, "<==", file]
# process element # process element
lutinMultiprocess.RunInPool(cmdLine, comment) lutinMultiprocess.RunInPool(cmdLine, comment, file_cmd)
return file_dst return file_dst

64
lutinMultiprocess.py
View File

@ -5,17 +5,30 @@ import threading
import time import time
import Queue import Queue
import os import os
import subprocess
def RunCommand(cmdLine, storeCmdLine=""):
def RunCommand(cmdLine):
debug.debug(cmdLine) debug.debug(cmdLine)
ret = os.system(cmdLine) retcode = -1
try:
retcode = subprocess.call(cmdLine, shell=True)
except OSError as e:
print >>sys.stderr, "Execution failed:", e
# write cmd line only after to prvent errors ...
if storeCmdLine!="":
file2 = open(storeCmdLine, "w")
file2.write(cmdLine)
file2.flush()
file2.close()
# TODO : Use "subprocess" instead ==> permit to pipline the renderings ... # TODO : Use "subprocess" instead ==> permit to pipline the renderings ...
if ret != 0:
if ret == 2: if retcode != 0:
if retcode == 2:
debug.error("can not compile file ... [keyboard interrrupt]") debug.error("can not compile file ... [keyboard interrrupt]")
else: else:
debug.error("can not compile file ... ret : " + str(ret)) debug.error("can not compile file ... ret : " + str(retcode))
exitFlag = False exitFlag = False
@ -27,32 +40,40 @@ class myThread(threading.Thread):
self.queue = queue self.queue = queue
self.lock = lock self.lock = lock
def run(self): def run(self):
print("Starting " + self.name) debug.verbose("Starting " + self.name)
global exitFlag global exitFlag
global queueLock global currentThreadWorking
global workQueue workingSet = False
while False==exitFlag: while False==exitFlag:
self.lock.acquire() self.lock.acquire()
if not self.queue.empty(): if not self.queue.empty():
if workingSet==False:
currentThreadWorking += 1
workingSet = True
data = self.queue.get() data = self.queue.get()
self.lock.release() self.lock.release()
print "%s processing %s" % (self.name, data[0]) debug.verbose(self.name + " processing '" + data[0] + "'")
if data[0]=="cmdLine": if data[0]=="cmdLine":
comment = data[2] comment = data[2]
cmdLine = data[1] cmdLine = data[1]
debug.printElement(comment[0], comment[1], comment[2], comment[3]) cmdStoreFile = data[3]
RunCommand(cmdLine) debug.printElement( "[" + str(self.threadID) + "] " + comment[0], comment[1], comment[2], comment[3])
RunCommand(cmdLine, cmdStoreFile)
else: else:
debug.warning("unknow request command : " + data[0]) debug.warning("unknow request command : " + data[0])
else: else:
if workingSet==True:
currentThreadWorking -= 1
workingSet=False
# no element to parse, just wait ... # no element to parse, just wait ...
self.lock.release() self.lock.release()
time.sleep(0.2) time.sleep(0.2)
# kill requested ... # kill requested ...
print("Exiting " + self.name) debug.verbose("Exiting " + self.name)
queueLock = threading.Lock() queueLock = threading.Lock()
workQueue = Queue.Queue() workQueue = Queue.Queue()
currentThreadWorking = 0
threads = [] threads = []
isInit = False isInit = False
@ -63,7 +84,9 @@ def ErrorOccured():
exitFlag = True exitFlag = True
def SetCoreNumber(numberOfcore): def SetCoreNumber(numberOfcore):
global processorAvaillable
processorAvaillable = numberOfcore processorAvaillable = numberOfcore
debug.debug(" set number of core for multi process compilation : " + str(processorAvaillable))
# nothing else to do # nothing else to do
def Init(): def Init():
@ -91,23 +114,23 @@ def UnInit():
if processorAvaillable > 1: if processorAvaillable > 1:
# Wait for all threads to complete # Wait for all threads to complete
for tmp in threads: for tmp in threads:
print("join thread ... \n") debug.verbose("join thread ...")
tmp.join() tmp.join()
print "Exiting Main Thread" debug.verbose("Exiting ALL Threads")
def RunInPool(cmdLine, comment): def RunInPool(cmdLine, comment, storeCmdLine=""):
if processorAvaillable <= 1: if processorAvaillable <= 1:
debug.printElement(comment[0], comment[1], comment[2], comment[3]) debug.printElement(comment[0], comment[1], comment[2], comment[3])
RunCommand(cmdLine) RunCommand(cmdLine, storeCmdLine)
return return
# multithreaded mode # multithreaded mode
Init() Init()
# Fill the queue # Fill the queue
queueLock.acquire() queueLock.acquire()
debug.verbose("add : in pool cmdLine") debug.verbose("add : in pool cmdLine")
workQueue.put(["cmdLine", cmdLine, comment]) workQueue.put(["cmdLine", cmdLine, comment, storeCmdLine])
queueLock.release() queueLock.release()
@ -121,6 +144,9 @@ def PoolSynchrosize():
while not workQueue.empty(): while not workQueue.empty():
time.sleep(0.2) time.sleep(0.2)
pass pass
# Wait all thread have ended their current process
while currentThreadWorking != 0:
time.sleep(0.2)
pass
debug.verbose("queue is empty") debug.verbose("queue is empty")
os.path.flush()