[DEV] add depend on the command liene to rebuild the element and basic start dev of a multiprocessor compilation mode

lutinDebug.py

@@ -1,5 +1,7 @@
 #!/usr/bin/python
-
+import os
+import thread
+import lutinMultiprocess
 debugLevel=3
 debugColor=False
 
@@ -53,7 +55,10 @@ def warning(input):
 def error(input):
 	if debugLevel >= 1:
 		print color_red + "[ERROR] " + input + color_default
+	lutinMultiprocess.ErrorOccured()
+	thread.interrupt_main()
 	exit(-1)
+	#os_exit(-1)
 	#raise "error happend"
 
 def printElement(type, lib, dir, name):

lutinDepend.py

@@ -4,11 +4,12 @@ import lutinDebug as debug
 import lutinEnv as environement
 
 
-def NeedReBuild(dst, src, dependFile):
+def NeedReBuild(dst, src, dependFile, file_cmd="", cmdLine=""):
 	debug.verbose("Resuest check of dependency of :")
 	debug.verbose("		dst='" + dst + "'")
 	debug.verbose("		str='" + src + "'")
 	debug.verbose("		dept='" + dependFile + "'")
+	debug.verbose("		cmd='" + file_cmd + "'")
 	# if force mode selected ==> just force rebuild ...
 	if environement.GetForceMode():
 		debug.verbose("			==> must rebuild (force mode)")
@@ -22,20 +23,36 @@ def NeedReBuild(dst, src, dependFile):
 	if os.path.getmtime(src) > os.path.getmtime(dst):
 		debug.verbose("			==> must rebuild (source time greater)")
 		return True
-	# now we need to parse the file to find all the dependency file
-	#	file is done like
-	#		file : \
-	#		 dep1 \
-	#		 dep2
-	#		
-	#		dep2 : \
-	#		 dep4
-	#
 	
 	if False==os.path.exists(dependFile):
 		debug.verbose("			==> must rebuild (no depending file)")
 		return True
 	
+	if ""!=file_cmd:
+		if False==os.path.exists(file_cmd):
+			debug.verbose("			==> must rebuild (no commandLine file)")
+			file2 = open(file_cmd, "w")
+			file2.write(cmdLine)
+			file2.flush()
+			file2.close()
+			return True
+		# check if the 2 cmdline are similar :
+		file2 = open(file_cmd, "r")
+		firstAndUniqueLine = file2.read()
+		if firstAndUniqueLine != cmdLine:
+			debug.verbose("			==> must rebuild (cmdLines are not identical)")
+			debug.verbose("				==> '" + cmdLine + "'")
+			debug.verbose("				==> '" + firstAndUniqueLine + "'")
+			file2.close()
+			file2 = open(file_cmd, "w")
+			file2.write(cmdLine)
+			file2.flush()
+			file2.close()
+			return True
+		# the cmdfile is correct ...
+		file2.close()
+	
+	
 	debug.verbose("			start parsing dependency file : '" + dependFile + "'")
 	file = open(dependFile, "r")
 	for curLine in file.readlines():
@@ -61,10 +78,14 @@ def NeedReBuild(dst, src, dependFile):
 			debug.verbose("					==> test");
 			if False==os.path.exists(testFile):
 				debug.warning("			==> must rebuild (a dependency file does not exist)")
+				file.close()
 				return True
 			if os.path.getmtime(testFile) > os.path.getmtime(dst):
 				debug.warning("			==> must rebuild (a dependency file time is newer)")
+				file.close()
 				return True
+	# close the current file :
+	file.close()
 	
 	debug.verbose("			==> Not rebuild (all dependency is OK)")
 	return False

lutinHeritage.py

@@ -28,13 +28,19 @@ class heritage:
 			self.flags_mm=module.export_flags_mm
 			self.path=module.export_path
 	
+	def AppendAndCheck(self, listout, newElement):
+		for element in listout:
+			if element==newElement:
+				return
+		listout.append(newElement)
+	
 	def AppendToInternalList(self, listout, list):
 		if type(list) == type(str()):
-			listout.append(list)
+			self.AppendAndCheck(listout, list)
 		else:
 			# mulyiple imput in the list ...
 			for elem in list:
-				listout.append(elem)
+				self.AppendAndCheck(listout, elem)
 	
 	def AddFlag_LD(self, list):
 		self.AppendToInternalList(self.flags_ld, list)

lutinModule.py

@@ -10,6 +10,7 @@ import lutinDebug as debug
 import lutinList as buildList
 import lutinHeritage as heritage
 import lutinDepend as dependency
+import lutinMultiprocess
 
 def RunCommand(cmdLine):
 	debug.debug(cmdLine)
@@ -94,6 +95,7 @@ class module:
 		                    }
 		
 	
+	
 	###############################################################################
 	## Commands for running gcc to compile a m++ file.
 	###############################################################################
@@ -141,15 +143,11 @@ class module:
 	## Commands for running gcc to compile a C++ file.
 	###############################################################################
 	def Compile_xx_to_o(self, file, binary, target, depancy):
-		tmpList = target.GenerateFile(binary, self.name,self.originFolder,file,"obj")
+		file_src, file_dst, file_depend, file_cmd = target.fileGenerateObject(binary,self.name,self.originFolder,file)
-		# check the dependency for this file :
+		# create the command line befor requesting start:
-		if False==dependency.NeedReBuild(tmpList[1], tmpList[0], tmpList[2]):
-			return tmpList[1]
-		lutinTools.CreateDirectoryOfFile(tmpList[1])
-		debug.printElement("c++", self.name, "<==", file)
 		cmdLine=lutinTools.ListToStr([
 			target.xx,
-			"-o", tmpList[1] ,
+			"-o", file_dst ,
 			target.global_include_cc,
 			lutinTools.AddPrefix("-I",self.export_path),
 			lutinTools.AddPrefix("-I",self.local_path),
@@ -158,37 +156,30 @@ class module:
 			target.global_flags_xx,
 			depancy.flags_cc,
 			depancy.flags_xx,
+			self.flags_xx,
 			self.flags_cc,
+			self.export_flags_xx,
+			self.export_flags_cc,
 			" -c -MMD -MP -g ",
-			tmpList[0]])
+			file_src])
-		RunCommand(cmdLine)
+		# check the dependency for this file :
-		"""
+		if False==dependency.NeedReBuild(file_dst, file_src, file_depend, file_cmd, cmdLine):
-		$(TARGET_CXX) \
+			return file_dst
-		-o $@ \
+		lutinTools.CreateDirectoryOfFile(file_dst)
-		$(TARGET_GLOBAL_C_INCLUDES) \
+		comment = ["c++", self.name, "<==", file]
-		$(PRIVATE_C_INCLUDES) \
+		#process element
-		$(TARGET_GLOBAL_CFLAGS_$(PRIVATE_ARM_MODE)) \
+		lutinMultiprocess.RunInPool(cmdLine, comment)
-		$(TARGET_GLOBAL_CFLAGS) $(TARGET_GLOBAL_CPPFLAGS) $(CXX_FLAGS_WARNINGS) \
+		return file_dst
-		$(PRIVATE_CFLAGS) $(PRIVATE_CPPFLAGS) \
-		-D__EWOL_APPL_NAME__="$(PROJECT_NAME2)" \
-		-c -MMD -MP -g \
-		$(call path-from-top,$<)
-		"""
-		return tmpList[1]
 	
 	###############################################################################
 	## Commands for running gcc to compile a C file.
 	###############################################################################
 	def Compile_cc_to_o(self, file, binary, target, depancy):
-		tmpList = target.GenerateFile(binary,self.name,self.originFolder,file,"obj")
+		file_src, file_dst, file_depend, file_cmd = target.fileGenerateObject(binary,self.name,self.originFolder,file)
-		# check the dependency for this file :
+		# create the command line befor requesting start:
-		if False==dependency.NeedReBuild(tmpList[1], tmpList[0], tmpList[2]):
-			return tmpList[1]
-		lutinTools.CreateDirectoryOfFile(tmpList[1])
-		debug.printElement("c", self.name, "<==", file)
 		cmdLine=lutinTools.ListToStr([
 			target.cc,
-			"-o", tmpList[1],
+			"-o", file_dst,
 			target.global_include_cc,
 			lutinTools.AddPrefix("-I",self.export_path),
 			lutinTools.AddPrefix("-I",self.local_path),
@@ -196,22 +187,18 @@ class module:
 			target.global_flags_cc,
 			depancy.flags_cc,
 			self.flags_cc,
+			self.export_flags_cc,
 			" -c -MMD -MP -g ",
-			tmpList[0]])
+			file_src])
-		RunCommand(cmdLine)
+		
-		"""
+		# check the dependency for this file :
-		$(TARGET_CC) \
+		if False==dependency.NeedReBuild(file_dst, file_src, file_depend, file_cmd, cmdLine):
-		-o $@ \
+			return file_dst
-		$(TARGET_GLOBAL_C_INCLUDES) \
+		lutinTools.CreateDirectoryOfFile(file_dst)
-		$(PRIVATE_C_INCLUDES) \
+		comment = ["c", self.name, "<==", file]
-		$(TARGET_GLOBAL_CFLAGS_$(PRIVATE_ARM_MODE)) \
+		# process element
-		$(TARGET_GLOBAL_CFLAGS) $(CC_FLAGS_WARNINGS) \
+		lutinMultiprocess.RunInPool(cmdLine, comment)
-		$(PRIVATE_CFLAGS) \
+		return file_dst
-		-D__EWOL_APPL_NAME__="$(PROJECT_NAME2)" \
-		-c -MMD -MP -g \
-		$(call path-from-top,$<)
-		"""
-		return tmpList[1]
 	
 	
 	###############################################################################
@@ -385,6 +372,9 @@ class module:
 				listSubFileNeededToBuild.append(resFile)
 			else:
 				debug.verbose(" TODO : gcc " + self.originFolder + "/" + file)
+		# when multiprocess availlable, we need to synchronize here ...
+		lutinMultiprocess.PoolSynchrosize()
+		
 		# generate end point:
 		if self.type=='PREBUILD':
 			# nothing to add ==> just dependence
@@ -447,13 +437,20 @@ class module:
 		else:
 			debug.error("Dit not know the element type ... (impossible case) type=" + self.type)
 	
+	def AppendAndCheck(self, listout, newElement):
+		for element in listout:
+			if element==newElement:
+				return
+		listout.append(newElement)
+		listout.sort()
+	
 	def AppendToInternalList(self, listout, list):
 		if type(list) == type(str()):
-			listout.append(list)
+			self.AppendAndCheck(listout, list)
 		else:
 			# mulyiple imput in the list ...
 			for elem in list:
-				listout.append(elem)
+				self.AppendAndCheck(listout, elem)
 	
 	def AddModuleDepend(self, list):
 		self.AppendToInternalList(self.depends, list)

lutinMultiprocess.py

@@ -0,0 +1,126 @@
+#!/usr/bin/python
+import sys
+import lutinDebug as debug
+import threading
+import time
+import Queue
+import os
+
+
+def RunCommand(cmdLine):
+	debug.debug(cmdLine)
+	ret = os.system(cmdLine)
+	# TODO : Use "subprocess" instead ==> permit to pipline the renderings ...
+	if ret != 0:
+		if ret == 2:
+			debug.error("can not compile file ... [keyboard interrrupt]")
+		else:
+			debug.error("can not compile file ... ret : " + str(ret))
+
+exitFlag = False
+
+class myThread(threading.Thread):
+	def __init__(self, threadID, lock, queue):
+		threading.Thread.__init__(self)
+		self.threadID = threadID
+		self.name = "Thread " + str(threadID)
+		self.queue = queue
+		self.lock = lock
+	def run(self):
+		print("Starting " + self.name)
+		global exitFlag
+		global queueLock
+		global workQueue
+		while False==exitFlag:
+			self.lock.acquire()
+			if not self.queue.empty():
+				data = self.queue.get()
+				self.lock.release()
+				print "%s processing %s" % (self.name, data[0])
+				if data[0]=="cmdLine":
+					comment = data[2]
+					cmdLine = data[1]
+					debug.printElement(comment[0], comment[1], comment[2], comment[3])
+					RunCommand(cmdLine)
+				else:
+					debug.warning("unknow request command : " + data[0])
+			else:
+				# no element to parse, just wait ...
+				self.lock.release()
+				time.sleep(0.2)
+		# kill requested ...
+		print("Exiting " + self.name)
+
+queueLock = threading.Lock()
+workQueue = Queue.Queue()
+threads = []
+
+isInit = False
+processorAvaillable = 1
+
+def ErrorOccured():
+	global exitFlag
+	exitFlag = True
+
+def SetCoreNumber(numberOfcore):
+	processorAvaillable = numberOfcore
+	# nothing else to do
+
+def Init():
+	global exitFlag
+	global isInit
+	if isInit==False:
+		isInit=True
+		global threads
+		global queueLock
+		global workQueue
+		# Create all the new threads
+		threadID = 0
+		while threadID < processorAvaillable:
+			thread = myThread(threadID, queueLock, workQueue)
+			thread.start()
+			threads.append(thread)
+			threadID += 1
+		
+
+
+def UnInit():
+	global exitFlag
+	# Notify threads it's time to exit
+	exitFlag = True
+	if processorAvaillable > 1:
+		# Wait for all threads to complete
+		for tmp in threads:
+			print("join thread ... \n")
+			tmp.join()
+		print "Exiting Main Thread"
+
+
+
+def RunInPool(cmdLine, comment):
+	if processorAvaillable <= 1:
+		debug.printElement(comment[0], comment[1], comment[2], comment[3])
+		RunCommand(cmdLine)
+		return
+	# multithreaded mode
+	Init()
+	# Fill the queue
+	queueLock.acquire()
+	debug.verbose("add : in pool cmdLine")
+	workQueue.put(["cmdLine", cmdLine, comment])
+	queueLock.release()
+	
+
+def PoolSynchrosize():
+	if processorAvaillable <= 1:
+		#in this case : nothing to synchronise
+		return
+	
+	debug.verbose("wait queue process ended\n")
+	# Wait for queue to empty
+	while not workQueue.empty():
+		time.sleep(0.2)
+		pass
+	debug.verbose("queue is empty")
+	os.path.flush()
+	

lutinTarget.py

@@ -38,6 +38,7 @@ class Target:
 		
 		self.global_sysroot=""
 		
+		self.suffix_cmdLine='.cmd'
 		self.suffix_dependence='.d'
 		self.suffix_obj='.o'
 		self.suffix_lib_static='.a'
@@ -67,6 +68,14 @@ class Target:
 	def SetEwolFolder(self, folder):
 		self.folder_ewol = folder
 	
+	
+	def fileGenerateObject(self,binaryName,moduleName,basePath,file):
+		list=[]
+		list.append(basePath + "/" + file)
+		list.append(self.GetBuildFolder(moduleName) + file + self.suffix_obj)
+		list.append(self.GetBuildFolder(moduleName) + file + self.suffix_dependence)
+		list.append(self.GetBuildFolder(moduleName) + file + self.suffix_cmdLine)
+		return list
 	"""
 		return a list of 3 elements :
 			0 : sources files (can be a list)
@@ -79,10 +88,6 @@ class Target:
 			list.append(file)
 			list.append(self.GetStagingFolder(binaryName) + self.folder_bin + "/" + moduleName + self.suffix_binary)
 			list.append(self.GetBuildFolder(moduleName) + moduleName + self.suffix_dependence)
-		elif (type=="obj"):
-			list.append(basePath + "/" + file)
-			list.append(self.GetBuildFolder(moduleName) + file + self.suffix_obj)
-			list.append(self.GetBuildFolder(moduleName) + file + self.suffix_dependence)
 		elif (type=="lib-shared"):
 			list.append(file)
 			list.append(self.GetStagingFolder(binaryName) + self.folder_lib + "/" + moduleName + self.suffix_lib_dynamic)
@@ -102,7 +107,7 @@ class Target:
 		return lutinTools.GetRunFolder() + self.folder_out + self.folder_staging + "/" + binaryName + "/"
 	
 	def GetStagingFolderData(self, binaryName):
-		return self.GetStagingFolder(binaryName) + self.folder_data + "/"
+		return self.GetStagingFolder(binaryName) + self.folder_data + "/" + binaryName + "/"
 	
 	def GetBuildFolder(self, moduleName):
 		return lutinTools.GetRunFolder() + self.folder_out + self.folder_build + "/" + moduleName + "/"

lutinTargetLinux.py

@@ -26,7 +26,7 @@ class Target(lutinTarget.Target):
 		debug.info("Generate package '" + pkgName + "'")
 		debug.debug("------------------------------------------------------------------------")
 		self.GetStagingFolder(pkgName)
-		targetOutFolderDebian=self.GetStagingFolder(pkgName) + "/" + pkgName + "/DEBIAN/"
+		targetOutFolderDebian=self.GetStagingFolder(pkgName) + "/DEBIAN/"
 		finalFileControl = targetOutFolderDebian + "control"
 		finalFilepostRm = targetOutFolderDebian + "postrm"
 		# create the folders :
@@ -50,20 +50,21 @@ class Target(lutinTarget.Target):
 		tmpFile.write("#!/bin/bash\n")
 		tmpFile.write("touch ~/." + pkgName + "\n")
 		if pkgName != "":
-			tmpFile.write("rm -r ~/.local/" + pkgName + "\n")
+			tmpFile.write("touch ~/.local/share/" + pkgName + "\n")
+			tmpFile.write("rm -r ~/.local/share/" + pkgName + "\n")
 		tmpFile.write("\n")
 		tmpFile.flush()
 		tmpFile.close()
 		## Enable Execution in script
 		os.chmod(finalFilepostRm, stat.S_IRWXU + stat.S_IRGRP + stat.S_IXGRP + stat.S_IROTH + stat.S_IXOTH);
 		# copy licence and information : 
-		lutinTools.CopyFile("os-Linux/README", self.GetStagingFolder(pkgName) + "/usr/share/doc/README")
+		lutinTools.CopyFile("os-Linux/README", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/README")
-		lutinTools.CopyFile("license.txt", self.GetStagingFolder(pkgName) + "/usr/share/doc/copyright")
+		lutinTools.CopyFile("license.txt", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/copyright")
-		lutinTools.CopyFile("changelog", self.GetStagingFolder(pkgName) + "/usr/share/doc/changelog")
+		lutinTools.CopyFile("changelog", self.GetStagingFolder(pkgName) + "/usr/share/doc/"+ pkgName + "/changelog")
 		debug.debug("pachage : " + self.GetStagingFolder(pkgName) + "/" + pkgName + ".deb")
-		os.system("cd " + self.GetStagingFolder(pkgName) + " ; dpkg-deb --build " + pkgName)
+		os.system("cd " + self.GetStagingFolder("") + " ; dpkg-deb --build " + pkgName)
 		lutinTools.CreateDirectoryOfFile(self.GetFinalFolder())
-		lutinTools.CopyFile(self.GetStagingFolder(pkgName) + "/" + pkgName + self.suffix_package, self.GetFinalFolder() + "/" + pkgName + self.suffix_package)
+		lutinTools.CopyFile(self.GetStagingFolder("") + "/" + pkgName + self.suffix_package, self.GetFinalFolder() + "/" + pkgName + self.suffix_package)
 	
 	def InstallPackage(self, pkgName):
 		debug.debug("------------------------------------------------------------------------")