atria-tools/lutin
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[DEV] add depend on the command liene to rebuild the element and basic start dev of a multiprocessor compilation mode

Browse Source
This commit is contained in:
Edouard DUPIN 2013-04-24 12:50:14 +02:00
parent 78d74b125f
commit 6c1649f36f
7 changed files with 232 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lutinDebug.py
View File

@ -1,5 +1,7 @@
#!/usr/bin/python
import os
import thread
import lutinMultiprocess
debugLevel=3
debugColor=False
@ -53,7 +55,10 @@ def warning(input):
def error(input):
if debugLevel >= 1:
print color_red + "[ERROR] " + input + color_default
lutinMultiprocess.ErrorOccured()
thread.interrupt_main()
exit(-1)
#os_exit(-1)
#raise "error happend"
def printElement(type, lib, dir, name):

41
lutinDepend.py
View File

@ -4,11 +4,12 @@ import lutinDebug as debug
import lutinEnv as environement
def NeedReBuild(dst, src, dependFile):
def NeedReBuild(dst, src, dependFile, file_cmd="", cmdLine=""):
debug.verbose("Resuest check of dependency of :")
debug.verbose(" dst='" + dst + "'")
debug.verbose(" str='" + src + "'")
debug.verbose(" dept='" + dependFile + "'")
debug.verbose(" cmd='" + file_cmd + "'")
# if force mode selected ==> just force rebuild ...
if environement.GetForceMode():
debug.verbose(" ==> must rebuild (force mode)")
@ -22,20 +23,36 @@ def NeedReBuild(dst, src, dependFile):
if os.path.getmtime(src) > os.path.getmtime(dst):
debug.verbose(" ==> must rebuild (source time greater)")
return True
# now we need to parse the file to find all the dependency file
# file is done like
# file : \
# dep1 \
# dep2
#
# dep2 : \
# dep4
#
if False==os.path.exists(dependFile):
debug.verbose(" ==> must rebuild (no depending file)")
return True
if ""!=file_cmd:
if False==os.path.exists(file_cmd):
debug.verbose(" ==> must rebuild (no commandLine file)")
file2 = open(file_cmd, "w")
file2.write(cmdLine)
file2.flush()
file2.close()
return True
# check if the 2 cmdline are similar :
file2 = open(file_cmd, "r")
firstAndUniqueLine = file2.read()
if firstAndUniqueLine != cmdLine:
debug.verbose(" ==> must rebuild (cmdLines are not identical)")
debug.verbose(" ==> '" + cmdLine + "'")
debug.verbose(" ==> '" + firstAndUniqueLine + "'")
file2.close()
file2 = open(file_cmd, "w")
file2.write(cmdLine)
file2.flush()
file2.close()
return True
# the cmdfile is correct ...
file2.close()
debug.verbose(" start parsing dependency file : '" + dependFile + "'")
file = open(dependFile, "r")
for curLine in file.readlines():
@ -61,10 +78,14 @@ def NeedReBuild(dst, src, dependFile):
debug.verbose(" ==> test");
if False==os.path.exists(testFile):
debug.warning(" ==> must rebuild (a dependency file does not exist)")
file.close()
return True
if os.path.getmtime(testFile) > os.path.getmtime(dst):
debug.warning(" ==> must rebuild (a dependency file time is newer)")
file.close()
return True
# close the current file :
file.close()
debug.verbose(" ==> Not rebuild (all dependency is OK)")
return False

10
lutinHeritage.py
View File

@ -28,13 +28,19 @@ class heritage:
self.flags_mm=module.export_flags_mm
self.path=module.export_path
def AppendAndCheck(self, listout, newElement):
for element in listout:
if element==newElement:
return
listout.append(newElement)
def AppendToInternalList(self, listout, list):
if type(list) == type(str()):
listout.append(list)
self.AppendAndCheck(listout, list)
else:
# mulyiple imput in the list ...
for elem in list:
listout.append(elem)
self.AppendAndCheck(listout, elem)
def AddFlag_LD(self, list):
self.AppendToInternalList(self.flags_ld, list)

89
lutinModule.py
View File

@ -10,6 +10,7 @@ import lutinDebug as debug
import lutinList as buildList
import lutinHeritage as heritage
import lutinDepend as dependency
import lutinMultiprocess
def RunCommand(cmdLine):
debug.debug(cmdLine)
@ -94,6 +95,7 @@ class module:
}
###############################################################################
## Commands for running gcc to compile a m++ file.
###############################################################################
@ -141,15 +143,11 @@ class module:
## Commands for running gcc to compile a C++ file.
###############################################################################
def Compile_xx_to_o(self, file, binary, target, depancy):
tmpList = target.GenerateFile(binary, self.name,self.originFolder,file,"obj")
# check the dependency for this file :
if False==dependency.NeedReBuild(tmpList[1], tmpList[0], tmpList[2]):
return tmpList[1]
lutinTools.CreateDirectoryOfFile(tmpList[1])
debug.printElement("c++", self.name, "<==", file)
file_src, file_dst, file_depend, file_cmd = target.fileGenerateObject(binary,self.name,self.originFolder,file)
# create the command line befor requesting start:
cmdLine=lutinTools.ListToStr([
target.xx,
"-o", tmpList[1] ,
"-o", file_dst ,
target.global_include_cc,
lutinTools.AddPrefix("-I",self.export_path),
lutinTools.AddPrefix("-I",self.local_path),
@ -158,37 +156,30 @@ class module:
target.global_flags_xx,
depancy.flags_cc,
depancy.flags_xx,
self.flags_xx,
self.flags_cc,
self.export_flags_xx,
self.export_flags_cc,
" -c -MMD -MP -g ",
tmpList[0]])
RunCommand(cmdLine)
"""
$(TARGET_CXX) \
-o $@ \
$(TARGET_GLOBAL_C_INCLUDES) \
$(PRIVATE_C_INCLUDES) \
$(TARGET_GLOBAL_CFLAGS_$(PRIVATE_ARM_MODE)) \
$(TARGET_GLOBAL_CFLAGS) $(TARGET_GLOBAL_CPPFLAGS) $(CXX_FLAGS_WARNINGS) \
$(PRIVATE_CFLAGS) $(PRIVATE_CPPFLAGS) \
-D__EWOL_APPL_NAME__="$(PROJECT_NAME2)" \
-c -MMD -MP -g \
$(call path-from-top,$<)
"""
return tmpList[1]
file_src])
# check the dependency for this file :
if False==dependency.NeedReBuild(file_dst, file_src, file_depend, file_cmd, cmdLine):
return file_dst
lutinTools.CreateDirectoryOfFile(file_dst)
comment = ["c++", self.name, "<==", file]
#process element
lutinMultiprocess.RunInPool(cmdLine, comment)
return file_dst
###############################################################################
## Commands for running gcc to compile a C file.
###############################################################################
def Compile_cc_to_o(self, file, binary, target, depancy):
tmpList = target.GenerateFile(binary,self.name,self.originFolder,file,"obj")
# check the dependency for this file :
if False==dependency.NeedReBuild(tmpList[1], tmpList[0], tmpList[2]):
return tmpList[1]
lutinTools.CreateDirectoryOfFile(tmpList[1])
debug.printElement("c", self.name, "<==", file)
file_src, file_dst, file_depend, file_cmd = target.fileGenerateObject(binary,self.name,self.originFolder,file)
# create the command line befor requesting start:
cmdLine=lutinTools.ListToStr([
target.cc,
"-o", tmpList[1],
"-o", file_dst,
target.global_include_cc,
lutinTools.AddPrefix("-I",self.export_path),
lutinTools.AddPrefix("-I",self.local_path),
@ -196,22 +187,18 @@ class module:
target.global_flags_cc,
depancy.flags_cc,
self.flags_cc,
self.export_flags_cc,
" -c -MMD -MP -g ",
tmpList[0]])
RunCommand(cmdLine)
"""
$(TARGET_CC) \
-o $@ \
$(TARGET_GLOBAL_C_INCLUDES) \
$(PRIVATE_C_INCLUDES) \
$(TARGET_GLOBAL_CFLAGS_$(PRIVATE_ARM_MODE)) \
$(TARGET_GLOBAL_CFLAGS) $(CC_FLAGS_WARNINGS) \
$(PRIVATE_CFLAGS) \
-D__EWOL_APPL_NAME__="$(PROJECT_NAME2)" \
-c -MMD -MP -g \
$(call path-from-top,$<)
"""
return tmpList[1]
file_src])
# check the dependency for this file :
if False==dependency.NeedReBuild(file_dst, file_src, file_depend, file_cmd, cmdLine):
return file_dst
lutinTools.CreateDirectoryOfFile(file_dst)
comment = ["c", self.name, "<==", file]
# process element
lutinMultiprocess.RunInPool(cmdLine, comment)
return file_dst
###############################################################################
@ -385,6 +372,9 @@ class module:
listSubFileNeededToBuild.append(resFile)
else:
debug.verbose(" TODO : gcc " + self.originFolder + "/" + file)
# when multiprocess availlable, we need to synchronize here ...
lutinMultiprocess.PoolSynchrosize()
# generate end point:
if self.type=='PREBUILD':
# nothing to add ==> just dependence
@ -447,13 +437,20 @@ class module:
else:
debug.error("Dit not know the element type ... (impossible case) type=" + self.type)
def AppendAndCheck(self, listout, newElement):
for element in listout:
if element==newElement:
return
listout.append(newElement)
listout.sort()
def AppendToInternalList(self, listout, list):
if type(list) == type(str()):
listout.append(list)
self.AppendAndCheck(listout, list)
else:
# mulyiple imput in the list ...
for elem in list:
listout.append(elem)
self.AppendAndCheck(listout, elem)
def AddModuleDepend(self, list):
self.AppendToInternalList(self.depends, list)

126
lutinMultiprocess.py Normal file
View File

@ -0,0 +1,126 @@
#!/usr/bin/python
import sys
import lutinDebug as debug
import threading
import time
import Queue
import os
def RunCommand(cmdLine):
debug.debug(cmdLine)
ret = os.system(cmdLine)
# TODO : Use "subprocess" instead ==> permit to pipline the renderings ...
if ret != 0:
if ret == 2:
debug.error("can not compile file ... [keyboard interrrupt]")
else:
debug.error("can not compile file ... ret : " + str(ret))
exitFlag = False
class myThread(threading.Thread):
def __init__(self, threadID, lock, queue):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = "Thread " + str(threadID)
self.queue = queue
self.lock = lock
def run(self):
print("Starting " + self.name)
global exitFlag
global queueLock
global workQueue
while False==exitFlag:
self.lock.acquire()
if not self.queue.empty():
data = self.queue.get()
self.lock.release()
print "%s processing %s" % (self.name, data[0])
if data[0]=="cmdLine":
comment = data[2]
cmdLine = data[1]
debug.printElement(comment[0], comment[1], comment[2], comment[3])
RunCommand(cmdLine)
else:
debug.warning("unknow request command : " + data[0])
else:
# no element to parse, just wait ...
self.lock.release()
time.sleep(0.2)
# kill requested ...
print("Exiting " + self.name)
queueLock = threading.Lock()
workQueue = Queue.Queue()
threads = []
isInit = False
processorAvaillable = 1
def ErrorOccured():
global exitFlag
exitFlag = True
def SetCoreNumber(numberOfcore):
processorAvaillable = numberOfcore
# nothing else to do
def Init():
global exitFlag
global isInit
if isInit==False:
isInit=True
global threads
global queueLock
global workQueue
# Create all the new threads
threadID = 0
while threadID < processorAvaillable:
thread = myThread(threadID, queueLock, workQueue)
thread.start()
threads.append(thread)
threadID += 1
def UnInit():
global exitFlag
# Notify threads it's time to exit
exitFlag = True
if processorAvaillable > 1:
# Wait for all threads to complete
for tmp in threads:
print("join thread ... \n")
tmp.join()
print "Exiting Main Thread"
def RunInPool(cmdLine, comment):
if processorAvaillable <= 1:
debug.printElement(comment[0], comment[1], comment[2], comment[3])
RunCommand(cmdLine)
return
# multithreaded mode
Init()
# Fill the queue
queueLock.acquire()
debug.verbose("add : in pool cmdLine")
workQueue.put(["cmdLine", cmdLine, comment])
queueLock.release()
def PoolSynchrosize():
if processorAvaillable <= 1:
#in this case : nothing to synchronise
return
debug.verbose("wait queue process ended\n")
# Wait for queue to empty
while not workQueue.empty():
time.sleep(0.2)
pass
debug.verbose("queue is empty")
os.path.flush()

15
lutinTarget.py
View File

@ -38,6 +38,7 @@ class Target:
self.global_sysroot=""
self.suffix_cmdLine='.cmd'
self.suffix_dependence='.d'
self.suffix_obj='.o'
self.suffix_lib_static='.a'
@ -67,6 +68,14 @@ class Target:
def SetEwolFolder(self, folder):
self.folder_ewol = folder
def fileGenerateObject(self,binaryName,moduleName,basePath,file):
list=[]
list.append(basePath + "/" + file)
list.append(self.GetBuildFolder(moduleName) + file + self.suffix_obj)
list.append(self.GetBuildFolder(moduleName) + file + self.suffix_dependence)
list.append(self.GetBuildFolder(moduleName) + file + self.suffix_cmdLine)
return list
"""
return a list of 3 elements :
0 : sources files (can be a list)
@ -79,10 +88,6 @@ class Target:
list.append(file)
list.append(self.GetStagingFolder(binaryName) + self.folder_bin + "/" + moduleName + self.suffix_binary)
list.append(self.GetBuildFolder(moduleName) + moduleName + self.suffix_dependence)
elif (type=="obj"):
list.append(basePath + "/" + file)
list.append(self.GetBuildFolder(moduleName) + file + self.suffix_obj)
list.append(self.GetBuildFolder(moduleName) + file + self.suffix_dependence)
elif (type=="lib-shared"):
list.append(file)
list.append(self.GetStagingFolder(binaryName) + self.folder_lib + "/" + moduleName + self.suffix_lib_dynamic)
@ -102,7 +107,7 @@ class Target:
return lutinTools.GetRunFolder() + self.folder_out + self.folder_staging + "/" + binaryName + "/"
def GetStagingFolderData(self, binaryName):
return self.GetStagingFolder(binaryName) + self.folder_data + "/"
return self.GetStagingFolder(binaryName) + self.folder_data + "/" + binaryName + "/"
def GetBuildFolder(self, moduleName):
return lutinTools.GetRunFolder() + self.folder_out + self.folder_build + "/" + moduleName + "/"

15
lutinTargetLinux.py
View File

@ -26,7 +26,7 @@ class Target(lutinTarget.Target):
debug.info("Generate package '" + pkgName + "'")
debug.debug("------------------------------------------------------------------------")
self.GetStagingFolder(pkgName)
targetOutFolderDebian=self.GetStagingFolder(pkgName) + "/" + pkgName + "/DEBIAN/"
targetOutFolderDebian=self.GetStagingFolder(pkgName) + "/DEBIAN/"
finalFileControl = targetOutFolderDebian + "control"
finalFilepostRm = targetOutFolderDebian + "postrm"
# create the folders :
@ -50,20 +50,21 @@ class Target(lutinTarget.Target):
tmpFile.write("#!/bin/bash\n")
tmpFile.write("touch ~/." + pkgName + "\n")
if pkgName != "":
tmpFile.write("rm -r ~/.local/" + pkgName + "\n")
tmpFile.write("touch ~/.local/share/" + pkgName + "\n")
tmpFile.write("rm -r ~/.local/share/" + pkgName + "\n")
tmpFile.write("\n")
tmpFile.flush()
tmpFile.close()
## Enable Execution in script
os.chmod(finalFilepostRm, stat.S_IRWXU + stat.S_IRGRP + stat.S_IXGRP + stat.S_IROTH + stat.S_IXOTH);
# copy licence and information :
lutinTools.CopyFile("os-Linux/README", self.GetStagingFolder(pkgName) + "/usr/share/doc/README")
lutinTools.CopyFile("license.txt", self.GetStagingFolder(pkgName) + "/usr/share/doc/copyright")
lutinTools.CopyFile("changelog", self.GetStagingFolder(pkgName) + "/usr/share/doc/changelog")
lutinTools.CopyFile("os-Linux/README", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/README")
lutinTools.CopyFile("license.txt", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/copyright")
lutinTools.CopyFile("changelog", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/changelog")
debug.debug("pachage : " + self.GetStagingFolder(pkgName) + "/" + pkgName + ".deb")
os.system("cd " + self.GetStagingFolder(pkgName) + " ; dpkg-deb --build " + pkgName)
os.system("cd " + self.GetStagingFolder("") + " ; dpkg-deb --build " + pkgName)
lutinTools.CreateDirectoryOfFile(self.GetFinalFolder())
lutinTools.CopyFile(self.GetStagingFolder(pkgName) + "/" + pkgName + self.suffix_package, self.GetFinalFolder() + "/" + pkgName + self.suffix_package)
lutinTools.CopyFile(self.GetStagingFolder("") + "/" + pkgName + self.suffix_package, self.GetFinalFolder() + "/" + pkgName + self.suffix_package)
def InstallPackage(self, pkgName):
debug.debug("------------------------------------------------------------------------")