[DEV] faster search methode of the lutin element, and add configuration file lutinConfig.py

This commit is contained in:
Edouard DUPIN 2016-01-15 21:27:54 +01:00
parent d21217bfc4
commit a9dd50575a
10 changed files with 496 additions and 231 deletions

View File

@ -10,6 +10,7 @@
# for path inspection:
import sys
import os
import lutin
import lutin.debug as debug
import lutin.arg as arguments
import lutin.host as host
@ -18,6 +19,8 @@ import lutin.target as target
import lutin.env as env
import lutin.multiprocess as multiprocess
import lutin.tools as tools
import lutin.host as lutinHost
import lutin.tools as lutinTools
myArgs = arguments.LutinArg()
myArgs.add(arguments.ArgDefine("h", "help", desc="Display this help"))
@ -28,6 +31,7 @@ myArgs.add(arguments.ArgDefine("C", "color", desc="Display makefile output in co
myArgs.add(arguments.ArgDefine("B", "force-build", desc="Force the rebuild without checking the dependency"))
myArgs.add(arguments.ArgDefine("P", "pretty", desc="Print the debug has pretty display"))
myArgs.add(arguments.ArgDefine("j", "jobs", haveParam=True, desc="Specifies the number of jobs (commands) to run simultaneously"))
myArgs.add(arguments.ArgDefine("d", "depth", haveParam=True, desc="Depth of the search of sub element lutin_*.py (default=" + str(env.get_parse_depth()) + ")"))
myArgs.add(arguments.ArgDefine("s", "force-strip", desc="Force the stripping of the compile elements"))
myArgs.add(arguments.ArgDefine("w", "warning", desc="Store warning in a file build file"))
@ -150,6 +154,7 @@ def usage(full=False):
# preparse the argument to get the verbose element for debug mode
def parseGenericArg(argument, active):
debug.extreme_verbose("parse arg : " + argument.get_option_name() + " " + argument.get_arg() + " active=" + str(active))
if argument.get_option_name() == "help":
if active==False:
usage()
@ -184,13 +189,24 @@ def parseGenericArg(argument, active):
if active==True:
multiprocess.set_core_number(int(argument.get_arg()))
return True
elif argument.get_option_name()=="depth":
if active==True:
env.set_parse_depth(int(argument.get_arg()))
return True
elif argument.get_option_name() == "verbose":
if active==True:
debug.set_level(int(argument.get_arg()))
return True
elif argument.get_option_name() == "color":
if active==True:
debug.enable_color()
if argument.get_arg() == "" \
or argument.get_arg() == "1" \
or argument.get_arg() == "true" \
or argument.get_arg() == "True" \
or argument.get_arg() == True:
debug.enable_color()
else:
debug.disable_color()
return True
elif argument.get_option_name() == "force-build":
if active==True:
@ -198,28 +214,85 @@ def parseGenericArg(argument, active):
return True
elif argument.get_option_name() == "pretty":
if active==True:
env.set_print_pretty_mode(True)
if argument.get_arg() == "" \
or argument.get_arg() == "1" \
or argument.get_arg() == "true" \
or argument.get_arg() == "True" \
or argument.get_arg() == True:
env.set_print_pretty_mode(True)
else:
env.set_print_pretty_mode(False)
return True
elif argument.get_option_name() == "force-strip":
if active==True:
env.set_force_strip_mode(True)
if argument.get_arg() == "" \
or argument.get_arg() == "1" \
or argument.get_arg() == "true" \
or argument.get_arg() == "True" \
or argument.get_arg() == True:
env.set_force_strip_mode(True)
else:
env.set_force_strip_mode(False)
return True
elif argument.get_option_name() == "warning":
if active==True:
env.set_warning_mode(True)
if argument.get_arg() == "" \
or argument.get_arg() == "1" \
or argument.get_arg() == "true" \
or argument.get_arg() == "True" \
or argument.get_arg() == True:
env.set_warning_mode(True)
else:
env.set_warning_mode(False)
return True
return False
# open configuration of lutin:
config_file_name = "lutinConfig.py"
config_file = os.path.join(tools.get_run_path(), config_file_name)
if os.path.isfile(config_file) == True:
sys.path.append(os.path.dirname(config_file))
debug.debug("Find basic configuration file: '" + config_file + "'")
# the file exist, we can open it and get the initial configuration:
configuration_file = __import__(config_file_name[:-3])
if "get_exclude_path" in dir(configuration_file):
data = configuration_file.get_exclude_path()
debug.debug(" get default config 'get_exclude_path' val='" + str(data) + "'")
env.set_exclude_search_path(data)
if "get_parsing_depth" in dir(configuration_file):
data = configuration_file.get_parsing_depth()
debug.debug(" get default config 'get_parsing_depth' val='" + str(data) + "'")
parseGenericArg(arguments.ArgElement("depth", str(data)), True)
if "get_default_jobs" in dir(configuration_file):
data = configuration_file.get_default_jobs()
debug.debug(" get default config 'get_default_jobs' val='" + str(data) + "'")
parseGenericArg(arguments.ArgElement("jobs", str(data)), True)
if "get_default_color" in dir(configuration_file):
data = configuration_file.get_default_color()
debug.debug(" get default config 'get_default_color' val='" + str(data) + "'")
parseGenericArg(arguments.ArgElement("color", str(data)), True)
if "get_default_debug_level" in dir(configuration_file):
data = configuration_file.get_default_debug_level()
debug.debug(" get default config 'get_default_debug_level' val='" + str(data) + "'")
parseGenericArg(arguments.ArgElement("verbose", str(data)), True)
if "get_default_print_pretty" in dir(configuration_file):
data = configuration_file.get_default_print_pretty()
debug.debug(" get default config 'get_default_print_pretty' val='" + str(data) + "'")
parseGenericArg(arguments.ArgElement("pretty", str(data)), True)
# parse default unique argument:
for argument in localArgument:
parseGenericArg(argument, True)
import lutin
import lutin.host as lutinHost
import lutin.tools as lutinTools
# initialize the system ...
lutin.init()
#available target : Linux / MacOs / Windows / Android ...
targetName = host.OS

View File

@ -8,6 +8,7 @@
##
import os
import sys
import fnmatch
# Local import
from . import target
from . import builder
@ -16,32 +17,119 @@ from . import host
from . import tools
from . import debug
from . import module
from . import env
is_init = False
if is_init == False:
def filter_name_and_file(root, list_files, filter):
# filter elements:
tmp_list = fnmatch.filter(list_files, filter)
out = []
for elem in tmp_list:
if os.path.isfile(os.path.join(root, elem)) == True:
out.append(elem);
return out;
def filter_path(root, list_files):
out = []
for elem in list_files:
if len(elem) == 0 \
or elem[0] == '.':
continue
if os.path.isdir(os.path.join(root, elem)) == True:
out.append(elem);
return out;
def import_path_local(path, limit_sub_folder, exclude_path = [], base_name = ""):
out = []
debug.verbose("lutin files: " + str(path) + " [START]")
if limit_sub_folder == 0:
debug.debug("Subparsing limitation append ...")
return []
try:
list_files = os.listdir(path)
except:
# an error occure, maybe read error ...
debug.warning("error when getting subdirectory of '" + str(path) + "'")
return []
if path in exclude_path:
debug.debug("find '" + str(path) + "' in exclude_path=" + str(exclude_path))
return []
# filter elements:
tmp_list_lutin_file = filter_name_and_file(path, list_files, base_name + "*.py")
debug.verbose("lutin files: " + str(path) + " : " + str(tmp_list_lutin_file))
# Import the module:
for filename in tmp_list_lutin_file:
out.append(os.path.join(path, filename))
debug.extreme_verbose(" Find a file : '" + str(out[-1]) + "'")
need_parse_sub_folder = True
rm_value = -1
# check if we need to parse sub_folder
if len(tmp_list_lutin_file) != 0:
need_parse_sub_folder = False
# check if the file "lutin_parse_sub.py" is present ==> parse SubFolder (force and add +1 in the resursing
if base_name + "parse_sub.py" in list_files:
need_parse_sub_folder = True
rm_value = 0
if need_parse_sub_folder == True:
list_folders = filter_path(path, list_files)
for folder in list_folders:
tmp_out = import_path_local(os.path.join(path, folder),
limit_sub_folder - rm_value,
exclude_path,
base_name)
# add all the elements:
for elem in tmp_out:
out.append(elem)
return out
def init():
global is_init;
if is_init == True:
return
debug.verbose("Use Make as a make stadard")
sys.path.append(tools.get_run_path())
builder.import_path(tools.get_current_path(__file__))
module.import_path(tools.get_current_path(__file__))
system.import_path(tools.get_current_path(__file__))
target.import_path(tools.get_current_path(__file__))
debug.debug("missing file lutinBase.py ==> loading subPath...");
# create the list of basic folder:
basic_folder_list = []
basic_folder_list.append([tools.get_current_path(__file__), True])
# Import all sub path without out and archive
for path in os.listdir("."):
if os.path.isdir(path)==True:
if path.lower()!="android" \
and path.lower()!="archive" \
and path.lower()!="out" :
debug.debug("Automatic load path: '" + path + "'")
builder.import_path(path)
module.import_path(path)
system.import_path(path)
target.import_path(path)
for elem_path in os.listdir("."):
if os.path.isdir(elem_path) == False:
continue
if elem_path.lower() == "android" \
or elem_path == "out" :
continue
debug.debug("Automatic load path: '" + elem_path + "'")
basic_folder_list.append([elem_path, False])
# create in a single path the basic list of lutin files (all start with lutin and end with .py)
exclude_path = env.get_exclude_search_path()
limit_sub_folder = env.get_parse_depth()
list_of_lutin_files = []
for elem_path, is_system in basic_folder_list:
if is_system == True:
limit_sub_folder_tmp = 999999
else:
limit_sub_folder_tmp = limit_sub_folder
tmp_out = import_path_local(elem_path,
limit_sub_folder_tmp,
exclude_path,
env.get_build_system_base_name())
# add all the elements:
for elem in tmp_out:
list_of_lutin_files.append(elem)
debug.debug("Files specific lutin: ")
for elem_path in list_of_lutin_files:
debug.debug(" " + elem_path)
# simply import element from the basic list of files (single parse ...)
builder.import_path(list_of_lutin_files)
module.import_path(list_of_lutin_files)
system.import_path(list_of_lutin_files)
target.import_path(list_of_lutin_files)
builder.init()
is_init = True

View File

@ -14,6 +14,7 @@ import datetime
# Local import
from . import debug
from . import heritage
from . import env
##
## constitution of dictionnary:
@ -23,26 +24,37 @@ from . import heritage
## - "builder": pointer on the element
##
builder_list=[]
__start_builder_name="lutinBuilder_"
__start_builder_name="Builder_"
def import_path(path):
def import_path(path_list):
global builder_list
matches = []
debug.debug('BUILDER: Start find sub File : "%s"' %path)
for root, dirnames, filenames in os.walk(path):
tmpList = fnmatch.filter(filenames, __start_builder_name + "*.py")
# Import the module :
for filename in tmpList:
debug.debug('BUILDER: Find a file : "%s"' %os.path.join(root, filename))
#matches.append(os.path.join(root, filename))
sys.path.append(os.path.dirname(os.path.join(root, filename)) )
builder_name = filename.replace('.py', '')
the_builder = __import__(builder_name)
builder_list.append({"name":builder_name,
"element":the_builder
})
debug.debug('BUILDER: type=' + the_builder.get_type() + " in=" + str(the_builder.get_input_type()) + " out=" + str(the_builder.get_output_type()))
global_base = env.get_build_system_base_name()
debug.debug("BUILDER: Init with Files list:")
for elem in path_list:
sys.path.append(os.path.dirname(elem))
# Get file name:
filename = os.path.basename(elem)
# Remove .py at the end:
filename = filename[:-3]
base_file_name = filename
# Remove global base name:
filename = filename[len(global_base):]
# Check if it start with the local patern:
if filename[:len(__start_builder_name)] != __start_builder_name:
debug.extreme_verbose("BUILDER: Integrate: '" + filename + "' from '" + elem + "' ==> rejected")
continue
# Remove local patern
builder_name = filename[len(__start_builder_name):]
debug.verbose("BUILDER: Integrate: '" + builder_name + "' from '" + elem + "'")
the_builder = __import__(base_file_name)
builder_list.append({"name":builder_name,
"element":the_builder
})
debug.debug('BUILDER: type=' + the_builder.get_type() + " in=" + str(the_builder.get_input_type()) + " out=" + str(the_builder.get_output_type()))
debug.verbose("List of BUILDER: ")
for elem in builder_list:
debug.verbose(" " + str(elem["name"]))
# we must have call all import before ...
def init():

View File

@ -52,6 +52,24 @@ def enable_color():
global color_cyan
color_cyan = "\033[36m"
def disable_color():
global debugColor
debugColor = True
global color_default
color_default= ""
global color_red
color_red = ""
global color_green
color_green = ""
global color_yellow
color_yellow = ""
global color_blue
color_blue = ""
global color_purple
color_purple = ""
global color_cyan
color_cyan = ""
def extreme_verbose(input, force=False):
global debugLock
global debugLevel
@ -106,7 +124,7 @@ def todo(input, force=False):
print(color_purple + "[TODO] " + input + color_default)
debugLock.release()
def error(input, threadID=-1, force=False, crash=True):
def error(input, thread_id=-1, force=False, crash=True):
global debugLock
global debugLevel
if debugLevel >= 1 \
@ -114,10 +132,10 @@ def error(input, threadID=-1, force=False, crash=True):
debugLock.acquire()
print(color_red + "[ERROR] " + input + color_default)
debugLock.release()
if crash==True:
if crash == True:
from . import multiprocess
multiprocess.error_occured()
if threadID != -1:
multiprocess.set_error_occured()
if thread_id != -1:
threading.interrupt_main()
exit(-1)
#os_exit(-1)

View File

@ -25,6 +25,40 @@ def get_force_mode():
global force_mode
return force_mode
parse_depth = 9999999
def set_parse_depth(val):
global parse_depth
parse_depth = val
debug.debug("Set depth search element: " + str(parse_depth))
def get_parse_depth():
global parse_depth
return parse_depth
exclude_search_path = []
def set_exclude_search_path(val):
global exclude_search_path
exclude_search_path = val
debug.debug("Set depth search element: " + str(exclude_search_path))
def get_exclude_search_path():
global exclude_search_path
return exclude_search_path
build_system_base_name = "lutin"
def set_build_system_base_name(val):
global build_system_base_name
build_system_base_name = val
debug.debug("Set basename: '" + str(build_system_base_name) + "'")
def get_build_system_base_name():
global build_system_base_name
return build_system_base_name
print_pretty_mode=False

View File

@ -20,6 +20,7 @@ from . import builder
from . import multiprocess
from . import image
from . import license
from . import env
class Module:
@ -926,29 +927,32 @@ class Module:
self.ext_project_add_module(target, projectMng)
projectMng.generate_project_file()
module_list=[]
__start_module_name="lutin_"
__start_module_name="_"
def import_path(path):
def import_path(path_list):
global module_list
matches = []
debug.debug('MODULE: Start find sub File : "%s"' %path)
for root, dirnames, filenames in os.walk(path):
tmpList = fnmatch.filter(filenames, __start_module_name + "*.py")
# Import the module :
for filename in tmpList:
debug.debug('Module: Find a file : "%s"' %os.path.join(root, filename))
#matches.append(os.path.join(root, filename))
sys.path.append(os.path.dirname(os.path.join(root, filename)) )
module_name = filename.replace('.py', '')
module_name = module_name.replace(__start_module_name, '')
debug.debug("MODULE: Integrate module: '" + module_name + "' from '" + os.path.join(root, filename) + "'")
module_list.append([module_name, os.path.join(root, filename)])
global_base = env.get_build_system_base_name()
debug.debug("MODULE: Init with Files list:")
for elem in path_list:
sys.path.append(os.path.dirname(elem))
# Get file name:
filename = os.path.basename(elem)
# Remove .py at the end:
filename = filename[:-3]
# Remove global base name:
filename = filename[len(global_base):]
# Check if it start with the local patern:
if filename[:len(__start_module_name)] != __start_module_name:
debug.extreme_verbose("MODULE: NOT-Integrate: '" + filename + "' from '" + elem + "' ==> rejected")
continue
# Remove local patern
module_name = filename[len(__start_module_name):]
debug.verbose("MODULE: Integrate: '" + module_name + "' from '" + elem + "'")
module_list.append([module_name, elem])
debug.verbose("New list module: ")
for mod in module_list:
debug.verbose(" " + str(mod[0]))
for elem in module_list:
debug.verbose(" " + str(elem[0]))
def exist(target, name):
global module_list
@ -962,9 +966,9 @@ def load_module(target, name):
for mod in module_list:
if mod[0] == name:
sys.path.append(os.path.dirname(mod[1]))
debug.verbose("import module : '" + __start_module_name + name + "'")
debug.verbose("import module : '" + env.get_build_system_base_name() + __start_module_name + name + "'")
the_module_file = mod[1]
the_module = __import__(__start_module_name + name)
the_module = __import__(env.get_build_system_base_name() + __start_module_name + name)
# get basic module properties:
property = get_module_option(the_module, name)
# configure the module:
@ -1022,7 +1026,7 @@ def list_all_module_with_desc():
tmpList = []
for mod in module_list:
sys.path.append(os.path.dirname(mod[1]))
the_module = __import__("lutin_" + mod[0])
the_module = __import__(env.get_build_system_base_name() + __start_module_name + mod[0])
tmpList.append(get_module_option(the_module, mod[0]))
return tmpList
@ -1036,22 +1040,25 @@ def get_module_option(the_module, name):
compagny_name = None
maintainer = None
version = None
version_id = None
if "get_type" in dir(the_module):
list_of_function_in_factory = dir(the_module)
if "get_type" in list_of_function_in_factory:
type = the_module.get_type()
else:
debug.debug(" function get_type() must be provided in the module: " + name)
if "get_sub_type" in dir(the_module):
if "get_sub_type" in list_of_function_in_factory:
sub_type = the_module.get_sub_type()
if "get_desc" in dir(the_module):
if "get_desc" in list_of_function_in_factory:
description = the_module.get_desc()
if "get_licence" in dir(the_module):
if "get_licence" in list_of_function_in_factory:
license = the_module.get_licence()
if "get_compagny_type" in dir(the_module):
if "get_compagny_type" in list_of_function_in_factory:
compagny_type = the_module.get_compagny_type()
# com : Commercial
# net : Network??
@ -1065,15 +1072,18 @@ def get_module_option(the_module, name):
if compagny_type not in compagny_type_list:
debug.warning("[" + name + "] type of the company not normal: " + compagny_type + " not in " + str(compagny_type_list))
if "get_compagny_name" in dir(the_module):
if "get_compagny_name" in list_of_function_in_factory:
compagny_name = the_module.get_compagny_name()
if "get_maintainer" in dir(the_module):
if "get_maintainer" in list_of_function_in_factory:
maintainer = the_module.get_maintainer()
if "get_version" in dir(the_module):
if "get_version" in list_of_function_in_factory:
version = the_module.get_version()
if "get_version_id" in list_of_function_in_factory:
version_id = the_module.get_version_id()
return {
"name":name,
"description":description,
@ -1083,7 +1093,8 @@ def get_module_option(the_module, name):
"compagny-type":compagny_type,
"compagny-name":compagny_name,
"maintainer":maintainer,
"version":version
"version":version,
"version-id":version_id
}

View File

@ -24,13 +24,13 @@ from . import tools
from . import env
from . import depend
queueLock = threading.Lock()
workQueue = queue.Queue()
currentThreadWorking = 0
queue_lock = threading.Lock()
work_queue = queue.Queue()
current_thread_working = 0
threads = []
# To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation
currentIdExecution = 0
errorExecution = {
current_id_execution = 0
error_execution = {
"id":-1,
"cmd":"",
"return":0,
@ -38,10 +38,10 @@ errorExecution = {
"out":"",
}
exitFlag = False # resuest stop of the thread
isinit = False # the thread are initialized
errorOccured = False # a thread have an error
processorAvaillable = 1 # number of CPU core availlable
exit_flag = False # resuest stop of the thread
is_init = False # the thread are initialized
error_occured = False # a thread have an error
processor_availlable = 1 # number of CPU core availlable
##
## @brief Execute the command and ruturn generate data
@ -72,9 +72,10 @@ def run_command_direct(cmd_line):
def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_file="", depend_data=None):
global errorOccured
global exitFlag
global currentIdExecution
global error_occured
global exit_flag
global current_id_execution
global error_execution
# prepare command line:
args = shlex.split(cmd_line)
debug.verbose("cmd = " + str(args))
@ -95,7 +96,7 @@ def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_
# Check error :
if p.returncode == 0:
debug.debug(env.print_pretty(cmd_line))
queueLock.acquire()
queue_lock.acquire()
if depend_data != None:
depend.create_dependency_file(depend_data['file'], depend_data['data'])
# TODO : Print the output all the time .... ==> to show warnings ...
@ -105,10 +106,10 @@ def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_
debug.print_compilator(output)
if err != "":
debug.print_compilator(err)
queueLock.release()
queue_lock.release()
else:
errorOccured = True
exitFlag = True
error_occured = True
exit_flag = True
# if No ID : Not in a multiprocess mode ==> just stop here
if build_id < 0:
debug.debug(env.print_pretty(cmd_line), force=True)
@ -120,18 +121,18 @@ def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_
debug.error("can not compile file ... ret : " + str(p.returncode))
else:
# in multiprocess interface
queueLock.acquire()
queue_lock.acquire()
# if an other write an error before, check if the current process is started before ==> then is the first error
if errorExecution["id"] >= build_id:
if error_execution["id"] >= build_id:
# nothing to do ...
queueLock.release()
queue_lock.release()
return;
errorExecution["id"] = build_id
errorExecution["cmd"] = cmd_line
errorExecution["return"] = p.returncode
errorExecution["err"] = err,
errorExecution["out"] = output,
queueLock.release()
error_execution["id"] = build_id
error_execution["cmd"] = cmd_line
error_execution["return"] = p.returncode
error_execution["err"] = err,
error_execution["out"] = output,
queue_lock.release()
# not write the command file...
return
debug.verbose("done 3")
@ -141,38 +142,46 @@ def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_
class myThread(threading.Thread):
def __init__(self, threadID, lock, queue):
def __init__(self, thread_id, lock, queue):
threading.Thread.__init__(self)
self.thread_id = threadID
self.name = "Thread " + str(threadID)
self.thread_id = thread_id
self.name = "Thread " + str(thread_id)
self.queue = queue
self.lock = lock
def run(self):
debug.verbose("Starting " + self.name)
global exitFlag
global currentThreadWorking
workingSet = False
while exitFlag == False:
global exit_flag
global current_thread_working
working_set = False
while exit_flag == False:
self.lock.acquire()
if not self.queue.empty():
if workingSet==False:
currentThreadWorking += 1
workingSet = True
if working_set == False:
current_thread_working += 1
working_set = True
data = self.queue.get()
self.lock.release()
debug.verbose(self.name + " processing '" + data[0] + "'")
if data[0]=="cmdLine":
if data[0]=="cmd_line":
comment = data[2]
cmdLine = data[1]
cmdStoreFile = data[3]
debug.print_element( "[" + str(data[4]) + "][" + str(self.thread_id) + "] " + comment[0], comment[1], comment[2], comment[3])
run_command(cmdLine, cmdStoreFile, build_id=data[4], file=comment[3], store_output_file=data[5], depend_data=data[6])
cmd_line = data[1]
cmd_store_file = data[3]
debug.print_element("[" + str(data[4]) + "][" + str(self.thread_id) + "] " + comment[0],
comment[1],
comment[2],
comment[3])
run_command(cmd_line,
cmd_store_file,
build_id=data[4],
file=comment[3],
store_output_file=data[5],
depend_data=data[6])
else:
debug.warning("unknow request command : " + data[0])
else:
if workingSet==True:
currentThreadWorking -= 1
workingSet=False
if working_set==True:
current_thread_working -= 1
working_set=False
# no element to parse, just wait ...
self.lock.release()
time.sleep(0.2)
@ -180,39 +189,41 @@ class myThread(threading.Thread):
debug.verbose("Exiting " + self.name)
def error_occured():
global exitFlag
exitFlag = True
def set_error_occured():
global exit_flag
exit_flag = True
def set_core_number(numberOfcore):
global processorAvaillable
processorAvaillable = numberOfcore
debug.debug(" set number of core for multi process compilation : " + str(processorAvaillable))
def set_core_number(number_of_core):
global processor_availlable
processor_availlable = number_of_core
debug.debug(" set number of core for multi process compilation : " + str(processor_availlable))
# nothing else to do
def init():
global exitFlag
global isinit
if isinit==False:
isinit=True
global error_occured
global exit_flag
global is_init
if is_init == False:
is_init = True
error_occured = False
global threads
global queueLock
global workQueue
global queue_lock
global work_queue
# Create all the new threads
threadID = 0
while threadID < processorAvaillable:
thread = myThread(threadID, queueLock, workQueue)
thread_id = 0
while thread_id < processor_availlable:
thread = myThread(thread_id, queue_lock, work_queue)
thread.start()
threads.append(thread)
threadID += 1
thread_id += 1
def un_init():
global exitFlag
global exit_flag
# Notify threads it's time to exit
exitFlag = True
if processorAvaillable > 1:
exit_flag = True
if processor_availlable > 1:
# Wait for all threads to complete
for tmp in threads:
debug.verbose("join thread ...")
@ -222,53 +233,53 @@ def un_init():
def run_in_pool(cmd_line, comment, store_cmd_line="", store_output_file="", depend_data=None):
global currentIdExecution
if processorAvaillable <= 1:
global current_id_execution
if processor_availlable <= 1:
debug.print_element(comment[0], comment[1], comment[2], comment[3])
run_command(cmd_line, store_cmd_line, file=comment[3], store_output_file=store_output_file, depend_data=depend_data)
return
# multithreaded mode
init()
# Fill the queue
queueLock.acquire()
debug.verbose("add : in pool cmdLine")
workQueue.put(["cmdLine", cmd_line, comment, store_cmd_line, currentIdExecution, store_output_file, depend_data])
currentIdExecution +=1;
queueLock.release()
queue_lock.acquire()
debug.verbose("add : in pool cmd_line")
work_queue.put(["cmd_line", cmd_line, comment, store_cmd_line, current_id_execution, store_output_file, depend_data])
current_id_execution +=1;
queue_lock.release()
def pool_synchrosize():
global errorOccured
global errorExecution
if processorAvaillable <= 1:
global error_occured
global error_execution
if processor_availlable <= 1:
#in this case : nothing to synchronise
return
debug.verbose("wait queue process ended\n")
# Wait for queue to empty
while not workQueue.empty() \
and False==errorOccured:
while not work_queue.empty() \
and error_occured == False:
time.sleep(0.2)
pass
# Wait all thread have ended their current process
while currentThreadWorking != 0 \
and False==errorOccured:
while current_thread_working != 0 \
and error_occured == False:
time.sleep(0.2)
pass
if False==errorOccured:
if error_occured == False:
debug.verbose("queue is empty")
else:
un_init()
debug.debug("Thread return with error ... ==> stop all the pool")
if errorExecution["id"] == -1:
if error_execution["id"] == -1:
debug.error("Pool error occured ... (No return information on Pool)")
return
debug.error("Error in an pool element : [" + str(errorExecution["id"]) + "]", crash=False)
debug.debug(env.print_pretty(errorExecution["cmd"]), force=True)
debug.print_compilator(str(errorExecution["out"][0]))
debug.print_compilator(str(errorExecution["err"][0]))
if errorExecution["return"] == 2:
debug.error("Error in an pool element : [" + str(error_execution["id"]) + "]", crash=False)
debug.debug(env.print_pretty(error_execution["cmd"]), force=True)
debug.print_compilator(str(error_execution["out"][0]))
debug.print_compilator(str(error_execution["err"][0]))
if error_execution["return"] == 2:
debug.error("can not compile file ... [keyboard interrrupt]")
else:
debug.error("can not compile file ... return value : " + str(errorExecution["return"]))
debug.error("can not compile file ... return value : " + str(error_execution["return"]))

View File

@ -16,6 +16,7 @@ import datetime
from . import debug
from . import module
from . import tools
from . import env
class System:
def __init__(self):
@ -73,61 +74,70 @@ def create_module_from_system(target, dict):
# Dictionnaire of Target name
# inside table of ["Name of the lib", "path of the lib", boolean loaded, module loaded]
systemList={}
__start_system_name="lutinSystem_"
system_list={}
__start_system_name="System_"
def import_path(path_list):
global system_list
global_base = env.get_build_system_base_name()
debug.debug("SYSTEM: Init with Files list:")
for elem in path_list:
sys.path.append(os.path.dirname(elem))
# Get file name:
filename = os.path.basename(elem)
# Remove .py at the end:
filename = filename[:-3]
# Remove global base name:
filename = filename[len(global_base):]
# Check if it start with the local patern:
if filename[:len(__start_system_name)] != __start_system_name:
debug.extreme_verbose("SYSTEM: NOT-Integrate: '" + filename + "' from '" + elem + "' ==> rejected")
continue
# Remove local patern
system_name = filename[len(__start_system_name):]
system_type, system_name = system_name.split('_')
debug.verbose("SYSTEM: Integrate: '" + system_type + "':'" + system_name + "' from '" + elem + "'")
if system_type in system_list:
system_list[system_type].append({"name":system_name,
"path":elem,
"system":None,
"loaded":False,
"exist":False,
"module":None})
else:
system_list[system_type] = [{"name":system_name,
"path":elem,
"system":None,
"loaded":False,
"exist":False,
"module":None}]
debug.verbose("New list system: ")
for elem in system_list:
debug.verbose(" " + str(elem))
for val in system_list[elem]:
debug.verbose(" " + str(val["name"]))
def import_path(path):
global targetList
matches = []
debug.debug('Start find sub File : "%s"' %path)
for root, dirnames, filenames in os.walk(path):
tmpList = fnmatch.filter(filenames, __start_system_name + "*.py")
# Import the module :
for filename in tmpList:
debug.verbose(' Find a file : "%s"' %os.path.join(root, filename))
sys.path.append(os.path.dirname(os.path.join(root, filename)) )
systemName = filename.replace('.py', '')
systemName = systemName.replace(__start_system_name, '')
targetType, systemName = systemName.split('_')
debug.debug("integrate system: '" + targetType + "':'" + systemName + "' from '" + os.path.join(root, filename) + "'")
if targetType in systemList:
systemList[targetType].append({"name":systemName,
"path":os.path.join(root, filename),
"system":None,
"loaded":False,
"exist":False,
"module":None})
else:
systemList[targetType] = [{"name":systemName,
"path":os.path.join(root, filename),
"system":None,
"loaded":False,
"exist":False,
"module":None}]
debug.debug("list system=" + str(systemList))
def display():
global systemList
for elementName in systemList:
debug.info("integrate system: '" + elementName +"'")
for data in systemList[elementName]:
debug.info(" '" + data["name"] +"' in " + data["path"])
global system_list
for elementName in system_list:
debug.info("SYSTEM: Integrate system: '" + elementName +"'")
for data in system_list[elementName]:
debug.info("SYSTEM: '" + data["name"] +"' in " + data["path"])
def exist(lib_name, target_name, target) :
global systemList
global system_list
debug.verbose("exist= " + lib_name + " in " + target_name)
if target_name not in systemList:
if target_name not in system_list:
return False
for data in systemList[target_name]:
for data in system_list[target_name]:
if data["name"] == lib_name:
# we find it in the List ==> need to check if it is present in the system :
if data["loaded"] == False:
debug.verbose("add to path: '" + os.path.dirname(data["path"]) + "'")
sys.path.append(os.path.dirname(data["path"]))
debug.verbose("import system : '" + data["name"] + "'")
theSystem = __import__(__start_system_name + target_name + "_" + data["name"])
theSystem = __import__(env.get_build_system_base_name() + __start_system_name + target_name + "_" + data["name"])
#create the system module
try:
debug.info("call : " + data["name"])
@ -139,10 +149,10 @@ def exist(lib_name, target_name, target) :
return False
def load(target, lib_name, target_name):
global systemList
if target_name not in systemList:
global system_list
if target_name not in system_list:
debug.error("you must call this function after checking of the system exist() !1!")
for data in systemList[target_name]:
for data in system_list[target_name]:
if data["name"] == lib_name:
if data["exist"] == False:
debug.error("you must call this function after checking of the system exist() !2!")

View File

@ -19,6 +19,7 @@ from . import tools
from . import module
from . import system
from . import multiprocess
from . import env
class Target:
def __init__(self, name, config, arch):
@ -746,25 +747,32 @@ class Target:
target_list=[]
__start_target_name="lutinTarget_"
__start_target_name="Target_"
def import_path(path):
def import_path(path_list):
global target_list
matches = []
debug.debug('TARGET: Start find sub File : "%s"' %path)
for root, dirnames, filenames in os.walk(path):
tmpList = fnmatch.filter(filenames, __start_target_name + "*.py")
# Import the module :
for filename in tmpList:
debug.debug('TARGET: Find a file : "%s"' %os.path.join(root, filename))
#matches.append(os.path.join(root, filename))
sys.path.append(os.path.dirname(os.path.join(root, filename)) )
targetName = filename.replace('.py', '')
targetName = targetName.replace(__start_target_name, '')
debug.debug("TARGET: integrate module: '" + targetName + "' from '" + os.path.join(root, filename) + "'")
target_list.append([targetName,os.path.join(root, filename)])
global_base = env.get_build_system_base_name()
debug.debug("TARGET: Init with Files list:")
for elem in path_list:
sys.path.append(os.path.dirname(elem))
# Get file name:
filename = os.path.basename(elem)
# Remove .py at the end:
filename = filename[:-3]
# Remove global base name:
filename = filename[len(global_base):]
# Check if it start with the local patern:
if filename[:len(__start_target_name)] != __start_target_name:
debug.extreme_verbose("TARGET: NOT-Integrate: '" + filename + "' from '" + elem + "' ==> rejected")
continue
# Remove local patern
target_name = filename[len(__start_target_name):]
debug.verbose("TARGET: Integrate: '" + target_name + "' from '" + elem + "'")
target_list.append([target_name, elem])
debug.verbose("New list TARGET: ")
for elem in target_list:
debug.verbose(" " + str(elem[0]))
def load_target(name, config):
global target_list
@ -776,8 +784,8 @@ def load_target(name, config):
if mod[0] == name:
debug.verbose("add to path: '" + os.path.dirname(mod[1]) + "'")
sys.path.append(os.path.dirname(mod[1]))
debug.verbose("import target : '" + __start_target_name + name + "'")
theTarget = __import__(__start_target_name + name)
debug.verbose("import target : '" + env.get_build_system_base_name() + __start_target_name + name + "'")
theTarget = __import__(env.get_build_system_base_name() + __start_target_name + name)
#create the target
tmpTarget = theTarget.Target(config)
return tmpTarget
@ -795,7 +803,7 @@ def list_all_target_with_desc():
tmpList = []
for mod in target_list:
sys.path.append(os.path.dirname(mod[1]))
theTarget = __import__(__start_target_name + mod[0])
theTarget = __import__(env.get_build_system_base_name() + __start_target_name + mod[0])
try:
tmpdesc = theTarget.get_desc()
tmpList.append([mod[0], tmpdesc])

View File

@ -74,8 +74,8 @@ class Target(target.Target):
"-fobjc-link-runtime"
])
self.add_flag("m", ["-fobjc-arc")
#self.add_flag("m", ["-fmodules")
self.add_flag("m", ["-fobjc-arc"])
#self.add_flag("m", ["-fmodules"])
self.pkg_path_data = "share"
self.pkg_path_bin = ""