diff --git a/lutinDebug.py b/lutinDebug.py index 579e132..273bd32 100644 --- a/lutinDebug.py +++ b/lutinDebug.py @@ -1,5 +1,7 @@ #!/usr/bin/python - +import os +import thread +import lutinMultiprocess debugLevel=3 debugColor=False @@ -53,7 +55,10 @@ def warning(input): def error(input): if debugLevel >= 1: print color_red + "[ERROR] " + input + color_default + lutinMultiprocess.ErrorOccured() + thread.interrupt_main() exit(-1) + #os_exit(-1) #raise "error happend" def printElement(type, lib, dir, name): diff --git a/lutinDepend.py b/lutinDepend.py index b5b8ddd..4595429 100644 --- a/lutinDepend.py +++ b/lutinDepend.py @@ -4,11 +4,12 @@ import lutinDebug as debug import lutinEnv as environement -def NeedReBuild(dst, src, dependFile): +def NeedReBuild(dst, src, dependFile, file_cmd="", cmdLine=""): debug.verbose("Resuest check of dependency of :") debug.verbose(" dst='" + dst + "'") debug.verbose(" str='" + src + "'") debug.verbose(" dept='" + dependFile + "'") + debug.verbose(" cmd='" + file_cmd + "'") # if force mode selected ==> just force rebuild ... if environement.GetForceMode(): debug.verbose(" ==> must rebuild (force mode)") @@ -22,20 +23,36 @@ def NeedReBuild(dst, src, dependFile): if os.path.getmtime(src) > os.path.getmtime(dst): debug.verbose(" ==> must rebuild (source time greater)") return True - # now we need to parse the file to find all the dependency file - # file is done like - # file : \ - # dep1 \ - # dep2 - # - # dep2 : \ - # dep4 - # if False==os.path.exists(dependFile): debug.verbose(" ==> must rebuild (no depending file)") return True + if ""!=file_cmd: + if False==os.path.exists(file_cmd): + debug.verbose(" ==> must rebuild (no commandLine file)") + file2 = open(file_cmd, "w") + file2.write(cmdLine) + file2.flush() + file2.close() + return True + # check if the 2 cmdline are similar : + file2 = open(file_cmd, "r") + firstAndUniqueLine = file2.read() + if firstAndUniqueLine != cmdLine: + debug.verbose(" ==> must rebuild (cmdLines are not identical)") + debug.verbose(" ==> '" + cmdLine + "'") + debug.verbose(" ==> '" + firstAndUniqueLine + "'") + file2.close() + file2 = open(file_cmd, "w") + file2.write(cmdLine) + file2.flush() + file2.close() + return True + # the cmdfile is correct ... + file2.close() + + debug.verbose(" start parsing dependency file : '" + dependFile + "'") file = open(dependFile, "r") for curLine in file.readlines(): @@ -61,10 +78,14 @@ def NeedReBuild(dst, src, dependFile): debug.verbose(" ==> test"); if False==os.path.exists(testFile): debug.warning(" ==> must rebuild (a dependency file does not exist)") + file.close() return True if os.path.getmtime(testFile) > os.path.getmtime(dst): debug.warning(" ==> must rebuild (a dependency file time is newer)") + file.close() return True + # close the current file : + file.close() debug.verbose(" ==> Not rebuild (all dependency is OK)") return False diff --git a/lutinHeritage.py b/lutinHeritage.py index f43c6d7..1e4370e 100644 --- a/lutinHeritage.py +++ b/lutinHeritage.py @@ -28,13 +28,19 @@ class heritage: self.flags_mm=module.export_flags_mm self.path=module.export_path + def AppendAndCheck(self, listout, newElement): + for element in listout: + if element==newElement: + return + listout.append(newElement) + def AppendToInternalList(self, listout, list): if type(list) == type(str()): - listout.append(list) + self.AppendAndCheck(listout, list) else: # mulyiple imput in the list ... for elem in list: - listout.append(elem) + self.AppendAndCheck(listout, elem) def AddFlag_LD(self, list): self.AppendToInternalList(self.flags_ld, list) diff --git a/lutinModule.py b/lutinModule.py index d74abc3..d2abe8b 100644 --- a/lutinModule.py +++ b/lutinModule.py @@ -10,6 +10,7 @@ import lutinDebug as debug import lutinList as buildList import lutinHeritage as heritage import lutinDepend as dependency +import lutinMultiprocess def RunCommand(cmdLine): debug.debug(cmdLine) @@ -94,6 +95,7 @@ class module: } + ############################################################################### ## Commands for running gcc to compile a m++ file. ############################################################################### @@ -141,15 +143,11 @@ class module: ## Commands for running gcc to compile a C++ file. ############################################################################### def Compile_xx_to_o(self, file, binary, target, depancy): - tmpList = target.GenerateFile(binary, self.name,self.originFolder,file,"obj") - # check the dependency for this file : - if False==dependency.NeedReBuild(tmpList[1], tmpList[0], tmpList[2]): - return tmpList[1] - lutinTools.CreateDirectoryOfFile(tmpList[1]) - debug.printElement("c++", self.name, "<==", file) + file_src, file_dst, file_depend, file_cmd = target.fileGenerateObject(binary,self.name,self.originFolder,file) + # create the command line befor requesting start: cmdLine=lutinTools.ListToStr([ target.xx, - "-o", tmpList[1] , + "-o", file_dst , target.global_include_cc, lutinTools.AddPrefix("-I",self.export_path), lutinTools.AddPrefix("-I",self.local_path), @@ -158,37 +156,30 @@ class module: target.global_flags_xx, depancy.flags_cc, depancy.flags_xx, + self.flags_xx, self.flags_cc, + self.export_flags_xx, + self.export_flags_cc, " -c -MMD -MP -g ", - tmpList[0]]) - RunCommand(cmdLine) - """ - $(TARGET_CXX) \ - -o $@ \ - $(TARGET_GLOBAL_C_INCLUDES) \ - $(PRIVATE_C_INCLUDES) \ - $(TARGET_GLOBAL_CFLAGS_$(PRIVATE_ARM_MODE)) \ - $(TARGET_GLOBAL_CFLAGS) $(TARGET_GLOBAL_CPPFLAGS) $(CXX_FLAGS_WARNINGS) \ - $(PRIVATE_CFLAGS) $(PRIVATE_CPPFLAGS) \ - -D__EWOL_APPL_NAME__="$(PROJECT_NAME2)" \ - -c -MMD -MP -g \ - $(call path-from-top,$<) - """ - return tmpList[1] + file_src]) + # check the dependency for this file : + if False==dependency.NeedReBuild(file_dst, file_src, file_depend, file_cmd, cmdLine): + return file_dst + lutinTools.CreateDirectoryOfFile(file_dst) + comment = ["c++", self.name, "<==", file] + #process element + lutinMultiprocess.RunInPool(cmdLine, comment) + return file_dst ############################################################################### ## Commands for running gcc to compile a C file. ############################################################################### def Compile_cc_to_o(self, file, binary, target, depancy): - tmpList = target.GenerateFile(binary,self.name,self.originFolder,file,"obj") - # check the dependency for this file : - if False==dependency.NeedReBuild(tmpList[1], tmpList[0], tmpList[2]): - return tmpList[1] - lutinTools.CreateDirectoryOfFile(tmpList[1]) - debug.printElement("c", self.name, "<==", file) + file_src, file_dst, file_depend, file_cmd = target.fileGenerateObject(binary,self.name,self.originFolder,file) + # create the command line befor requesting start: cmdLine=lutinTools.ListToStr([ target.cc, - "-o", tmpList[1], + "-o", file_dst, target.global_include_cc, lutinTools.AddPrefix("-I",self.export_path), lutinTools.AddPrefix("-I",self.local_path), @@ -196,22 +187,18 @@ class module: target.global_flags_cc, depancy.flags_cc, self.flags_cc, + self.export_flags_cc, " -c -MMD -MP -g ", - tmpList[0]]) - RunCommand(cmdLine) - """ - $(TARGET_CC) \ - -o $@ \ - $(TARGET_GLOBAL_C_INCLUDES) \ - $(PRIVATE_C_INCLUDES) \ - $(TARGET_GLOBAL_CFLAGS_$(PRIVATE_ARM_MODE)) \ - $(TARGET_GLOBAL_CFLAGS) $(CC_FLAGS_WARNINGS) \ - $(PRIVATE_CFLAGS) \ - -D__EWOL_APPL_NAME__="$(PROJECT_NAME2)" \ - -c -MMD -MP -g \ - $(call path-from-top,$<) - """ - return tmpList[1] + file_src]) + + # check the dependency for this file : + if False==dependency.NeedReBuild(file_dst, file_src, file_depend, file_cmd, cmdLine): + return file_dst + lutinTools.CreateDirectoryOfFile(file_dst) + comment = ["c", self.name, "<==", file] + # process element + lutinMultiprocess.RunInPool(cmdLine, comment) + return file_dst ############################################################################### @@ -385,6 +372,9 @@ class module: listSubFileNeededToBuild.append(resFile) else: debug.verbose(" TODO : gcc " + self.originFolder + "/" + file) + # when multiprocess availlable, we need to synchronize here ... + lutinMultiprocess.PoolSynchrosize() + # generate end point: if self.type=='PREBUILD': # nothing to add ==> just dependence @@ -447,13 +437,20 @@ class module: else: debug.error("Dit not know the element type ... (impossible case) type=" + self.type) + def AppendAndCheck(self, listout, newElement): + for element in listout: + if element==newElement: + return + listout.append(newElement) + listout.sort() + def AppendToInternalList(self, listout, list): if type(list) == type(str()): - listout.append(list) + self.AppendAndCheck(listout, list) else: # mulyiple imput in the list ... for elem in list: - listout.append(elem) + self.AppendAndCheck(listout, elem) def AddModuleDepend(self, list): self.AppendToInternalList(self.depends, list) diff --git a/lutinMultiprocess.py b/lutinMultiprocess.py new file mode 100644 index 0000000..409c1d1 --- /dev/null +++ b/lutinMultiprocess.py @@ -0,0 +1,126 @@ +#!/usr/bin/python +import sys +import lutinDebug as debug +import threading +import time +import Queue +import os + + +def RunCommand(cmdLine): + debug.debug(cmdLine) + ret = os.system(cmdLine) + # TODO : Use "subprocess" instead ==> permit to pipline the renderings ... + if ret != 0: + if ret == 2: + debug.error("can not compile file ... [keyboard interrrupt]") + else: + debug.error("can not compile file ... ret : " + str(ret)) + +exitFlag = False + +class myThread(threading.Thread): + def __init__(self, threadID, lock, queue): + threading.Thread.__init__(self) + self.threadID = threadID + self.name = "Thread " + str(threadID) + self.queue = queue + self.lock = lock + def run(self): + print("Starting " + self.name) + global exitFlag + global queueLock + global workQueue + while False==exitFlag: + self.lock.acquire() + if not self.queue.empty(): + data = self.queue.get() + self.lock.release() + print "%s processing %s" % (self.name, data[0]) + if data[0]=="cmdLine": + comment = data[2] + cmdLine = data[1] + debug.printElement(comment[0], comment[1], comment[2], comment[3]) + RunCommand(cmdLine) + else: + debug.warning("unknow request command : " + data[0]) + else: + # no element to parse, just wait ... + self.lock.release() + time.sleep(0.2) + # kill requested ... + print("Exiting " + self.name) + +queueLock = threading.Lock() +workQueue = Queue.Queue() +threads = [] + +isInit = False +processorAvaillable = 1 + +def ErrorOccured(): + global exitFlag + exitFlag = True + +def SetCoreNumber(numberOfcore): + processorAvaillable = numberOfcore + # nothing else to do + +def Init(): + global exitFlag + global isInit + if isInit==False: + isInit=True + global threads + global queueLock + global workQueue + # Create all the new threads + threadID = 0 + while threadID < processorAvaillable: + thread = myThread(threadID, queueLock, workQueue) + thread.start() + threads.append(thread) + threadID += 1 + + + +def UnInit(): + global exitFlag + # Notify threads it's time to exit + exitFlag = True + if processorAvaillable > 1: + # Wait for all threads to complete + for tmp in threads: + print("join thread ... \n") + tmp.join() + print "Exiting Main Thread" + + + +def RunInPool(cmdLine, comment): + if processorAvaillable <= 1: + debug.printElement(comment[0], comment[1], comment[2], comment[3]) + RunCommand(cmdLine) + return + # multithreaded mode + Init() + # Fill the queue + queueLock.acquire() + debug.verbose("add : in pool cmdLine") + workQueue.put(["cmdLine", cmdLine, comment]) + queueLock.release() + + +def PoolSynchrosize(): + if processorAvaillable <= 1: + #in this case : nothing to synchronise + return + + debug.verbose("wait queue process ended\n") + # Wait for queue to empty + while not workQueue.empty(): + time.sleep(0.2) + pass + debug.verbose("queue is empty") + os.path.flush() + diff --git a/lutinTarget.py b/lutinTarget.py index 986d0d7..440edd8 100644 --- a/lutinTarget.py +++ b/lutinTarget.py @@ -38,6 +38,7 @@ class Target: self.global_sysroot="" + self.suffix_cmdLine='.cmd' self.suffix_dependence='.d' self.suffix_obj='.o' self.suffix_lib_static='.a' @@ -67,6 +68,14 @@ class Target: def SetEwolFolder(self, folder): self.folder_ewol = folder + + def fileGenerateObject(self,binaryName,moduleName,basePath,file): + list=[] + list.append(basePath + "/" + file) + list.append(self.GetBuildFolder(moduleName) + file + self.suffix_obj) + list.append(self.GetBuildFolder(moduleName) + file + self.suffix_dependence) + list.append(self.GetBuildFolder(moduleName) + file + self.suffix_cmdLine) + return list """ return a list of 3 elements : 0 : sources files (can be a list) @@ -79,10 +88,6 @@ class Target: list.append(file) list.append(self.GetStagingFolder(binaryName) + self.folder_bin + "/" + moduleName + self.suffix_binary) list.append(self.GetBuildFolder(moduleName) + moduleName + self.suffix_dependence) - elif (type=="obj"): - list.append(basePath + "/" + file) - list.append(self.GetBuildFolder(moduleName) + file + self.suffix_obj) - list.append(self.GetBuildFolder(moduleName) + file + self.suffix_dependence) elif (type=="lib-shared"): list.append(file) list.append(self.GetStagingFolder(binaryName) + self.folder_lib + "/" + moduleName + self.suffix_lib_dynamic) @@ -102,7 +107,7 @@ class Target: return lutinTools.GetRunFolder() + self.folder_out + self.folder_staging + "/" + binaryName + "/" def GetStagingFolderData(self, binaryName): - return self.GetStagingFolder(binaryName) + self.folder_data + "/" + return self.GetStagingFolder(binaryName) + self.folder_data + "/" + binaryName + "/" def GetBuildFolder(self, moduleName): return lutinTools.GetRunFolder() + self.folder_out + self.folder_build + "/" + moduleName + "/" diff --git a/lutinTargetLinux.py b/lutinTargetLinux.py index e40fc90..85b5571 100644 --- a/lutinTargetLinux.py +++ b/lutinTargetLinux.py @@ -26,7 +26,7 @@ class Target(lutinTarget.Target): debug.info("Generate package '" + pkgName + "'") debug.debug("------------------------------------------------------------------------") self.GetStagingFolder(pkgName) - targetOutFolderDebian=self.GetStagingFolder(pkgName) + "/" + pkgName + "/DEBIAN/" + targetOutFolderDebian=self.GetStagingFolder(pkgName) + "/DEBIAN/" finalFileControl = targetOutFolderDebian + "control" finalFilepostRm = targetOutFolderDebian + "postrm" # create the folders : @@ -50,20 +50,21 @@ class Target(lutinTarget.Target): tmpFile.write("#!/bin/bash\n") tmpFile.write("touch ~/." + pkgName + "\n") if pkgName != "": - tmpFile.write("rm -r ~/.local/" + pkgName + "\n") + tmpFile.write("touch ~/.local/share/" + pkgName + "\n") + tmpFile.write("rm -r ~/.local/share/" + pkgName + "\n") tmpFile.write("\n") tmpFile.flush() tmpFile.close() ## Enable Execution in script os.chmod(finalFilepostRm, stat.S_IRWXU + stat.S_IRGRP + stat.S_IXGRP + stat.S_IROTH + stat.S_IXOTH); # copy licence and information : - lutinTools.CopyFile("os-Linux/README", self.GetStagingFolder(pkgName) + "/usr/share/doc/README") - lutinTools.CopyFile("license.txt", self.GetStagingFolder(pkgName) + "/usr/share/doc/copyright") - lutinTools.CopyFile("changelog", self.GetStagingFolder(pkgName) + "/usr/share/doc/changelog") + lutinTools.CopyFile("os-Linux/README", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/README") + lutinTools.CopyFile("license.txt", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/copyright") + lutinTools.CopyFile("changelog", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/changelog") debug.debug("pachage : " + self.GetStagingFolder(pkgName) + "/" + pkgName + ".deb") - os.system("cd " + self.GetStagingFolder(pkgName) + " ; dpkg-deb --build " + pkgName) + os.system("cd " + self.GetStagingFolder("") + " ; dpkg-deb --build " + pkgName) lutinTools.CreateDirectoryOfFile(self.GetFinalFolder()) - lutinTools.CopyFile(self.GetStagingFolder(pkgName) + "/" + pkgName + self.suffix_package, self.GetFinalFolder() + "/" + pkgName + self.suffix_package) + lutinTools.CopyFile(self.GetStagingFolder("") + "/" + pkgName + self.suffix_package, self.GetFinalFolder() + "/" + pkgName + self.suffix_package) def InstallPackage(self, pkgName): debug.debug("------------------------------------------------------------------------")