diff --git a/bin/lutin b/bin/lutin index 5234e13..7535fef 100755 --- a/bin/lutin +++ b/bin/lutin @@ -10,6 +10,7 @@ # for path inspection: import sys import os +import lutin import lutin.debug as debug import lutin.arg as arguments import lutin.host as host @@ -18,6 +19,8 @@ import lutin.target as target import lutin.env as env import lutin.multiprocess as multiprocess import lutin.tools as tools +import lutin.host as lutinHost +import lutin.tools as lutinTools myArgs = arguments.LutinArg() myArgs.add(arguments.ArgDefine("h", "help", desc="Display this help")) @@ -28,6 +31,7 @@ myArgs.add(arguments.ArgDefine("C", "color", desc="Display makefile output in co myArgs.add(arguments.ArgDefine("B", "force-build", desc="Force the rebuild without checking the dependency")) myArgs.add(arguments.ArgDefine("P", "pretty", desc="Print the debug has pretty display")) myArgs.add(arguments.ArgDefine("j", "jobs", haveParam=True, desc="Specifies the number of jobs (commands) to run simultaneously")) +myArgs.add(arguments.ArgDefine("d", "depth", haveParam=True, desc="Depth of the search of sub element lutin_*.py (default=" + str(env.get_parse_depth()) + ")")) myArgs.add(arguments.ArgDefine("s", "force-strip", desc="Force the stripping of the compile elements")) myArgs.add(arguments.ArgDefine("w", "warning", desc="Store warning in a file build file")) @@ -150,6 +154,7 @@ def usage(full=False): # preparse the argument to get the verbose element for debug mode def parseGenericArg(argument, active): + debug.extreme_verbose("parse arg : " + argument.get_option_name() + " " + argument.get_arg() + " active=" + str(active)) if argument.get_option_name() == "help": if active==False: usage() @@ -184,13 +189,24 @@ def parseGenericArg(argument, active): if active==True: multiprocess.set_core_number(int(argument.get_arg())) return True + elif argument.get_option_name()=="depth": + if active==True: + env.set_parse_depth(int(argument.get_arg())) + return True elif argument.get_option_name() == "verbose": if active==True: debug.set_level(int(argument.get_arg())) return True elif argument.get_option_name() == "color": if active==True: - debug.enable_color() + if argument.get_arg() == "" \ + or argument.get_arg() == "1" \ + or argument.get_arg() == "true" \ + or argument.get_arg() == "True" \ + or argument.get_arg() == True: + debug.enable_color() + else: + debug.disable_color() return True elif argument.get_option_name() == "force-build": if active==True: @@ -198,28 +214,85 @@ def parseGenericArg(argument, active): return True elif argument.get_option_name() == "pretty": if active==True: - env.set_print_pretty_mode(True) + if argument.get_arg() == "" \ + or argument.get_arg() == "1" \ + or argument.get_arg() == "true" \ + or argument.get_arg() == "True" \ + or argument.get_arg() == True: + env.set_print_pretty_mode(True) + else: + env.set_print_pretty_mode(False) return True elif argument.get_option_name() == "force-strip": if active==True: - env.set_force_strip_mode(True) + if argument.get_arg() == "" \ + or argument.get_arg() == "1" \ + or argument.get_arg() == "true" \ + or argument.get_arg() == "True" \ + or argument.get_arg() == True: + env.set_force_strip_mode(True) + else: + env.set_force_strip_mode(False) return True elif argument.get_option_name() == "warning": if active==True: - env.set_warning_mode(True) + if argument.get_arg() == "" \ + or argument.get_arg() == "1" \ + or argument.get_arg() == "true" \ + or argument.get_arg() == "True" \ + or argument.get_arg() == True: + env.set_warning_mode(True) + else: + env.set_warning_mode(False) return True return False + +# open configuration of lutin: +config_file_name = "lutinConfig.py" +config_file = os.path.join(tools.get_run_path(), config_file_name) +if os.path.isfile(config_file) == True: + sys.path.append(os.path.dirname(config_file)) + debug.debug("Find basic configuration file: '" + config_file + "'") + # the file exist, we can open it and get the initial configuration: + configuration_file = __import__(config_file_name[:-3]) + if "get_exclude_path" in dir(configuration_file): + data = configuration_file.get_exclude_path() + debug.debug(" get default config 'get_exclude_path' val='" + str(data) + "'") + env.set_exclude_search_path(data) + if "get_parsing_depth" in dir(configuration_file): + data = configuration_file.get_parsing_depth() + debug.debug(" get default config 'get_parsing_depth' val='" + str(data) + "'") + parseGenericArg(arguments.ArgElement("depth", str(data)), True) + + if "get_default_jobs" in dir(configuration_file): + data = configuration_file.get_default_jobs() + debug.debug(" get default config 'get_default_jobs' val='" + str(data) + "'") + parseGenericArg(arguments.ArgElement("jobs", str(data)), True) + + if "get_default_color" in dir(configuration_file): + data = configuration_file.get_default_color() + debug.debug(" get default config 'get_default_color' val='" + str(data) + "'") + parseGenericArg(arguments.ArgElement("color", str(data)), True) + + if "get_default_debug_level" in dir(configuration_file): + data = configuration_file.get_default_debug_level() + debug.debug(" get default config 'get_default_debug_level' val='" + str(data) + "'") + parseGenericArg(arguments.ArgElement("verbose", str(data)), True) + + if "get_default_print_pretty" in dir(configuration_file): + data = configuration_file.get_default_print_pretty() + debug.debug(" get default config 'get_default_print_pretty' val='" + str(data) + "'") + parseGenericArg(arguments.ArgElement("pretty", str(data)), True) + + + # parse default unique argument: for argument in localArgument: parseGenericArg(argument, True) - -import lutin - - -import lutin.host as lutinHost -import lutin.tools as lutinTools +# initialize the system ... +lutin.init() #available target : Linux / MacOs / Windows / Android ... targetName = host.OS diff --git a/lutin/__init__.py b/lutin/__init__.py index d821a10..919dbb5 100755 --- a/lutin/__init__.py +++ b/lutin/__init__.py @@ -8,6 +8,7 @@ ## import os import sys +import fnmatch # Local import from . import target from . import builder @@ -16,32 +17,119 @@ from . import host from . import tools from . import debug from . import module - +from . import env is_init = False -if is_init == False: + +def filter_name_and_file(root, list_files, filter): + # filter elements: + tmp_list = fnmatch.filter(list_files, filter) + out = [] + for elem in tmp_list: + if os.path.isfile(os.path.join(root, elem)) == True: + out.append(elem); + return out; + +def filter_path(root, list_files): + out = [] + for elem in list_files: + if len(elem) == 0 \ + or elem[0] == '.': + continue + if os.path.isdir(os.path.join(root, elem)) == True: + out.append(elem); + return out; + +def import_path_local(path, limit_sub_folder, exclude_path = [], base_name = ""): + out = [] + debug.verbose("lutin files: " + str(path) + " [START]") + if limit_sub_folder == 0: + debug.debug("Subparsing limitation append ...") + return [] + try: + list_files = os.listdir(path) + except: + # an error occure, maybe read error ... + debug.warning("error when getting subdirectory of '" + str(path) + "'") + return [] + if path in exclude_path: + debug.debug("find '" + str(path) + "' in exclude_path=" + str(exclude_path)) + return [] + # filter elements: + tmp_list_lutin_file = filter_name_and_file(path, list_files, base_name + "*.py") + debug.verbose("lutin files: " + str(path) + " : " + str(tmp_list_lutin_file)) + # Import the module: + for filename in tmp_list_lutin_file: + out.append(os.path.join(path, filename)) + debug.extreme_verbose(" Find a file : '" + str(out[-1]) + "'") + need_parse_sub_folder = True + rm_value = -1 + # check if we need to parse sub_folder + if len(tmp_list_lutin_file) != 0: + need_parse_sub_folder = False + # check if the file "lutin_parse_sub.py" is present ==> parse SubFolder (force and add +1 in the resursing + if base_name + "parse_sub.py" in list_files: + need_parse_sub_folder = True + rm_value = 0 + if need_parse_sub_folder == True: + list_folders = filter_path(path, list_files) + for folder in list_folders: + tmp_out = import_path_local(os.path.join(path, folder), + limit_sub_folder - rm_value, + exclude_path, + base_name) + # add all the elements: + for elem in tmp_out: + out.append(elem) + return out + + +def init(): + global is_init; + if is_init == True: + return debug.verbose("Use Make as a make stadard") sys.path.append(tools.get_run_path()) - builder.import_path(tools.get_current_path(__file__)) - module.import_path(tools.get_current_path(__file__)) - system.import_path(tools.get_current_path(__file__)) - target.import_path(tools.get_current_path(__file__)) - - debug.debug("missing file lutinBase.py ==> loading subPath..."); + # create the list of basic folder: + basic_folder_list = [] + basic_folder_list.append([tools.get_current_path(__file__), True]) # Import all sub path without out and archive - for path in os.listdir("."): - if os.path.isdir(path)==True: - if path.lower()!="android" \ - and path.lower()!="archive" \ - and path.lower()!="out" : - debug.debug("Automatic load path: '" + path + "'") - builder.import_path(path) - module.import_path(path) - system.import_path(path) - target.import_path(path) + for elem_path in os.listdir("."): + if os.path.isdir(elem_path) == False: + continue + if elem_path.lower() == "android" \ + or elem_path == "out" : + continue + debug.debug("Automatic load path: '" + elem_path + "'") + basic_folder_list.append([elem_path, False]) + + # create in a single path the basic list of lutin files (all start with lutin and end with .py) + exclude_path = env.get_exclude_search_path() + limit_sub_folder = env.get_parse_depth() + list_of_lutin_files = [] + for elem_path, is_system in basic_folder_list: + if is_system == True: + limit_sub_folder_tmp = 999999 + else: + limit_sub_folder_tmp = limit_sub_folder + tmp_out = import_path_local(elem_path, + limit_sub_folder_tmp, + exclude_path, + env.get_build_system_base_name()) + # add all the elements: + for elem in tmp_out: + list_of_lutin_files.append(elem) + + debug.debug("Files specific lutin: ") + for elem_path in list_of_lutin_files: + debug.debug(" " + elem_path) + # simply import element from the basic list of files (single parse ...) + builder.import_path(list_of_lutin_files) + module.import_path(list_of_lutin_files) + system.import_path(list_of_lutin_files) + target.import_path(list_of_lutin_files) builder.init() - is_init = True diff --git a/lutin/builder.py b/lutin/builder.py index d59813c..a0a9402 100644 --- a/lutin/builder.py +++ b/lutin/builder.py @@ -14,6 +14,7 @@ import datetime # Local import from . import debug from . import heritage +from . import env ## ## constitution of dictionnary: @@ -23,26 +24,37 @@ from . import heritage ## - "builder": pointer on the element ## builder_list=[] -__start_builder_name="lutinBuilder_" +__start_builder_name="Builder_" -def import_path(path): +def import_path(path_list): global builder_list - matches = [] - debug.debug('BUILDER: Start find sub File : "%s"' %path) - for root, dirnames, filenames in os.walk(path): - tmpList = fnmatch.filter(filenames, __start_builder_name + "*.py") - # Import the module : - for filename in tmpList: - debug.debug('BUILDER: Find a file : "%s"' %os.path.join(root, filename)) - #matches.append(os.path.join(root, filename)) - sys.path.append(os.path.dirname(os.path.join(root, filename)) ) - builder_name = filename.replace('.py', '') - the_builder = __import__(builder_name) - builder_list.append({"name":builder_name, - "element":the_builder - }) - debug.debug('BUILDER: type=' + the_builder.get_type() + " in=" + str(the_builder.get_input_type()) + " out=" + str(the_builder.get_output_type())) + global_base = env.get_build_system_base_name() + debug.debug("BUILDER: Init with Files list:") + for elem in path_list: + sys.path.append(os.path.dirname(elem)) + # Get file name: + filename = os.path.basename(elem) + # Remove .py at the end: + filename = filename[:-3] + base_file_name = filename + # Remove global base name: + filename = filename[len(global_base):] + # Check if it start with the local patern: + if filename[:len(__start_builder_name)] != __start_builder_name: + debug.extreme_verbose("BUILDER: Integrate: '" + filename + "' from '" + elem + "' ==> rejected") + continue + # Remove local patern + builder_name = filename[len(__start_builder_name):] + debug.verbose("BUILDER: Integrate: '" + builder_name + "' from '" + elem + "'") + the_builder = __import__(base_file_name) + builder_list.append({"name":builder_name, + "element":the_builder + }) + debug.debug('BUILDER: type=' + the_builder.get_type() + " in=" + str(the_builder.get_input_type()) + " out=" + str(the_builder.get_output_type())) + debug.verbose("List of BUILDER: ") + for elem in builder_list: + debug.verbose(" " + str(elem["name"])) # we must have call all import before ... def init(): diff --git a/lutin/debug.py b/lutin/debug.py index a6f8030..81229ec 100644 --- a/lutin/debug.py +++ b/lutin/debug.py @@ -52,6 +52,24 @@ def enable_color(): global color_cyan color_cyan = "\033[36m" +def disable_color(): + global debugColor + debugColor = True + global color_default + color_default= "" + global color_red + color_red = "" + global color_green + color_green = "" + global color_yellow + color_yellow = "" + global color_blue + color_blue = "" + global color_purple + color_purple = "" + global color_cyan + color_cyan = "" + def extreme_verbose(input, force=False): global debugLock global debugLevel @@ -106,7 +124,7 @@ def todo(input, force=False): print(color_purple + "[TODO] " + input + color_default) debugLock.release() -def error(input, threadID=-1, force=False, crash=True): +def error(input, thread_id=-1, force=False, crash=True): global debugLock global debugLevel if debugLevel >= 1 \ @@ -114,10 +132,10 @@ def error(input, threadID=-1, force=False, crash=True): debugLock.acquire() print(color_red + "[ERROR] " + input + color_default) debugLock.release() - if crash==True: + if crash == True: from . import multiprocess - multiprocess.error_occured() - if threadID != -1: + multiprocess.set_error_occured() + if thread_id != -1: threading.interrupt_main() exit(-1) #os_exit(-1) diff --git a/lutin/env.py b/lutin/env.py index 2064a93..fccbd10 100644 --- a/lutin/env.py +++ b/lutin/env.py @@ -25,6 +25,40 @@ def get_force_mode(): global force_mode return force_mode +parse_depth = 9999999 + +def set_parse_depth(val): + global parse_depth + parse_depth = val + debug.debug("Set depth search element: " + str(parse_depth)) + +def get_parse_depth(): + global parse_depth + return parse_depth + +exclude_search_path = [] + +def set_exclude_search_path(val): + global exclude_search_path + exclude_search_path = val + debug.debug("Set depth search element: " + str(exclude_search_path)) + +def get_exclude_search_path(): + global exclude_search_path + return exclude_search_path + + +build_system_base_name = "lutin" + +def set_build_system_base_name(val): + global build_system_base_name + build_system_base_name = val + debug.debug("Set basename: '" + str(build_system_base_name) + "'") + +def get_build_system_base_name(): + global build_system_base_name + return build_system_base_name + print_pretty_mode=False diff --git a/lutin/module.py b/lutin/module.py index f16aa63..5366367 100644 --- a/lutin/module.py +++ b/lutin/module.py @@ -20,6 +20,7 @@ from . import builder from . import multiprocess from . import image from . import license +from . import env class Module: @@ -926,29 +927,32 @@ class Module: self.ext_project_add_module(target, projectMng) projectMng.generate_project_file() - - module_list=[] -__start_module_name="lutin_" +__start_module_name="_" -def import_path(path): +def import_path(path_list): global module_list - matches = [] - debug.debug('MODULE: Start find sub File : "%s"' %path) - for root, dirnames, filenames in os.walk(path): - tmpList = fnmatch.filter(filenames, __start_module_name + "*.py") - # Import the module : - for filename in tmpList: - debug.debug('Module: Find a file : "%s"' %os.path.join(root, filename)) - #matches.append(os.path.join(root, filename)) - sys.path.append(os.path.dirname(os.path.join(root, filename)) ) - module_name = filename.replace('.py', '') - module_name = module_name.replace(__start_module_name, '') - debug.debug("MODULE: Integrate module: '" + module_name + "' from '" + os.path.join(root, filename) + "'") - module_list.append([module_name, os.path.join(root, filename)]) + global_base = env.get_build_system_base_name() + debug.debug("MODULE: Init with Files list:") + for elem in path_list: + sys.path.append(os.path.dirname(elem)) + # Get file name: + filename = os.path.basename(elem) + # Remove .py at the end: + filename = filename[:-3] + # Remove global base name: + filename = filename[len(global_base):] + # Check if it start with the local patern: + if filename[:len(__start_module_name)] != __start_module_name: + debug.extreme_verbose("MODULE: NOT-Integrate: '" + filename + "' from '" + elem + "' ==> rejected") + continue + # Remove local patern + module_name = filename[len(__start_module_name):] + debug.verbose("MODULE: Integrate: '" + module_name + "' from '" + elem + "'") + module_list.append([module_name, elem]) debug.verbose("New list module: ") - for mod in module_list: - debug.verbose(" " + str(mod[0])) + for elem in module_list: + debug.verbose(" " + str(elem[0])) def exist(target, name): global module_list @@ -962,9 +966,9 @@ def load_module(target, name): for mod in module_list: if mod[0] == name: sys.path.append(os.path.dirname(mod[1])) - debug.verbose("import module : '" + __start_module_name + name + "'") + debug.verbose("import module : '" + env.get_build_system_base_name() + __start_module_name + name + "'") the_module_file = mod[1] - the_module = __import__(__start_module_name + name) + the_module = __import__(env.get_build_system_base_name() + __start_module_name + name) # get basic module properties: property = get_module_option(the_module, name) # configure the module: @@ -1022,7 +1026,7 @@ def list_all_module_with_desc(): tmpList = [] for mod in module_list: sys.path.append(os.path.dirname(mod[1])) - the_module = __import__("lutin_" + mod[0]) + the_module = __import__(env.get_build_system_base_name() + __start_module_name + mod[0]) tmpList.append(get_module_option(the_module, mod[0])) return tmpList @@ -1036,22 +1040,25 @@ def get_module_option(the_module, name): compagny_name = None maintainer = None version = None + version_id = None - if "get_type" in dir(the_module): + list_of_function_in_factory = dir(the_module) + + if "get_type" in list_of_function_in_factory: type = the_module.get_type() else: debug.debug(" function get_type() must be provided in the module: " + name) - if "get_sub_type" in dir(the_module): + if "get_sub_type" in list_of_function_in_factory: sub_type = the_module.get_sub_type() - if "get_desc" in dir(the_module): + if "get_desc" in list_of_function_in_factory: description = the_module.get_desc() - if "get_licence" in dir(the_module): + if "get_licence" in list_of_function_in_factory: license = the_module.get_licence() - if "get_compagny_type" in dir(the_module): + if "get_compagny_type" in list_of_function_in_factory: compagny_type = the_module.get_compagny_type() # com : Commercial # net : Network?? @@ -1065,15 +1072,18 @@ def get_module_option(the_module, name): if compagny_type not in compagny_type_list: debug.warning("[" + name + "] type of the company not normal: " + compagny_type + " not in " + str(compagny_type_list)) - if "get_compagny_name" in dir(the_module): + if "get_compagny_name" in list_of_function_in_factory: compagny_name = the_module.get_compagny_name() - if "get_maintainer" in dir(the_module): + if "get_maintainer" in list_of_function_in_factory: maintainer = the_module.get_maintainer() - if "get_version" in dir(the_module): + if "get_version" in list_of_function_in_factory: version = the_module.get_version() + if "get_version_id" in list_of_function_in_factory: + version_id = the_module.get_version_id() + return { "name":name, "description":description, @@ -1083,7 +1093,8 @@ def get_module_option(the_module, name): "compagny-type":compagny_type, "compagny-name":compagny_name, "maintainer":maintainer, - "version":version + "version":version, + "version-id":version_id } diff --git a/lutin/multiprocess.py b/lutin/multiprocess.py index a945991..c518a40 100644 --- a/lutin/multiprocess.py +++ b/lutin/multiprocess.py @@ -24,13 +24,13 @@ from . import tools from . import env from . import depend -queueLock = threading.Lock() -workQueue = queue.Queue() -currentThreadWorking = 0 +queue_lock = threading.Lock() +work_queue = queue.Queue() +current_thread_working = 0 threads = [] # To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation -currentIdExecution = 0 -errorExecution = { +current_id_execution = 0 +error_execution = { "id":-1, "cmd":"", "return":0, @@ -38,10 +38,10 @@ errorExecution = { "out":"", } -exitFlag = False # resuest stop of the thread -isinit = False # the thread are initialized -errorOccured = False # a thread have an error -processorAvaillable = 1 # number of CPU core availlable +exit_flag = False # resuest stop of the thread +is_init = False # the thread are initialized +error_occured = False # a thread have an error +processor_availlable = 1 # number of CPU core availlable ## ## @brief Execute the command and ruturn generate data @@ -72,9 +72,10 @@ def run_command_direct(cmd_line): def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_file="", depend_data=None): - global errorOccured - global exitFlag - global currentIdExecution + global error_occured + global exit_flag + global current_id_execution + global error_execution # prepare command line: args = shlex.split(cmd_line) debug.verbose("cmd = " + str(args)) @@ -95,7 +96,7 @@ def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_ # Check error : if p.returncode == 0: debug.debug(env.print_pretty(cmd_line)) - queueLock.acquire() + queue_lock.acquire() if depend_data != None: depend.create_dependency_file(depend_data['file'], depend_data['data']) # TODO : Print the output all the time .... ==> to show warnings ... @@ -105,10 +106,10 @@ def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_ debug.print_compilator(output) if err != "": debug.print_compilator(err) - queueLock.release() + queue_lock.release() else: - errorOccured = True - exitFlag = True + error_occured = True + exit_flag = True # if No ID : Not in a multiprocess mode ==> just stop here if build_id < 0: debug.debug(env.print_pretty(cmd_line), force=True) @@ -120,18 +121,18 @@ def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_ debug.error("can not compile file ... ret : " + str(p.returncode)) else: # in multiprocess interface - queueLock.acquire() + queue_lock.acquire() # if an other write an error before, check if the current process is started before ==> then is the first error - if errorExecution["id"] >= build_id: + if error_execution["id"] >= build_id: # nothing to do ... - queueLock.release() + queue_lock.release() return; - errorExecution["id"] = build_id - errorExecution["cmd"] = cmd_line - errorExecution["return"] = p.returncode - errorExecution["err"] = err, - errorExecution["out"] = output, - queueLock.release() + error_execution["id"] = build_id + error_execution["cmd"] = cmd_line + error_execution["return"] = p.returncode + error_execution["err"] = err, + error_execution["out"] = output, + queue_lock.release() # not write the command file... return debug.verbose("done 3") @@ -141,38 +142,46 @@ def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_ class myThread(threading.Thread): - def __init__(self, threadID, lock, queue): + def __init__(self, thread_id, lock, queue): threading.Thread.__init__(self) - self.thread_id = threadID - self.name = "Thread " + str(threadID) + self.thread_id = thread_id + self.name = "Thread " + str(thread_id) self.queue = queue self.lock = lock def run(self): debug.verbose("Starting " + self.name) - global exitFlag - global currentThreadWorking - workingSet = False - while exitFlag == False: + global exit_flag + global current_thread_working + working_set = False + while exit_flag == False: self.lock.acquire() if not self.queue.empty(): - if workingSet==False: - currentThreadWorking += 1 - workingSet = True + if working_set == False: + current_thread_working += 1 + working_set = True data = self.queue.get() self.lock.release() debug.verbose(self.name + " processing '" + data[0] + "'") - if data[0]=="cmdLine": + if data[0]=="cmd_line": comment = data[2] - cmdLine = data[1] - cmdStoreFile = data[3] - debug.print_element( "[" + str(data[4]) + "][" + str(self.thread_id) + "] " + comment[0], comment[1], comment[2], comment[3]) - run_command(cmdLine, cmdStoreFile, build_id=data[4], file=comment[3], store_output_file=data[5], depend_data=data[6]) + cmd_line = data[1] + cmd_store_file = data[3] + debug.print_element("[" + str(data[4]) + "][" + str(self.thread_id) + "] " + comment[0], + comment[1], + comment[2], + comment[3]) + run_command(cmd_line, + cmd_store_file, + build_id=data[4], + file=comment[3], + store_output_file=data[5], + depend_data=data[6]) else: debug.warning("unknow request command : " + data[0]) else: - if workingSet==True: - currentThreadWorking -= 1 - workingSet=False + if working_set==True: + current_thread_working -= 1 + working_set=False # no element to parse, just wait ... self.lock.release() time.sleep(0.2) @@ -180,39 +189,41 @@ class myThread(threading.Thread): debug.verbose("Exiting " + self.name) -def error_occured(): - global exitFlag - exitFlag = True +def set_error_occured(): + global exit_flag + exit_flag = True -def set_core_number(numberOfcore): - global processorAvaillable - processorAvaillable = numberOfcore - debug.debug(" set number of core for multi process compilation : " + str(processorAvaillable)) +def set_core_number(number_of_core): + global processor_availlable + processor_availlable = number_of_core + debug.debug(" set number of core for multi process compilation : " + str(processor_availlable)) # nothing else to do def init(): - global exitFlag - global isinit - if isinit==False: - isinit=True + global error_occured + global exit_flag + global is_init + if is_init == False: + is_init = True + error_occured = False global threads - global queueLock - global workQueue + global queue_lock + global work_queue # Create all the new threads - threadID = 0 - while threadID < processorAvaillable: - thread = myThread(threadID, queueLock, workQueue) + thread_id = 0 + while thread_id < processor_availlable: + thread = myThread(thread_id, queue_lock, work_queue) thread.start() threads.append(thread) - threadID += 1 + thread_id += 1 def un_init(): - global exitFlag + global exit_flag # Notify threads it's time to exit - exitFlag = True - if processorAvaillable > 1: + exit_flag = True + if processor_availlable > 1: # Wait for all threads to complete for tmp in threads: debug.verbose("join thread ...") @@ -222,53 +233,53 @@ def un_init(): def run_in_pool(cmd_line, comment, store_cmd_line="", store_output_file="", depend_data=None): - global currentIdExecution - if processorAvaillable <= 1: + global current_id_execution + if processor_availlable <= 1: debug.print_element(comment[0], comment[1], comment[2], comment[3]) run_command(cmd_line, store_cmd_line, file=comment[3], store_output_file=store_output_file, depend_data=depend_data) return # multithreaded mode init() # Fill the queue - queueLock.acquire() - debug.verbose("add : in pool cmdLine") - workQueue.put(["cmdLine", cmd_line, comment, store_cmd_line, currentIdExecution, store_output_file, depend_data]) - currentIdExecution +=1; - queueLock.release() + queue_lock.acquire() + debug.verbose("add : in pool cmd_line") + work_queue.put(["cmd_line", cmd_line, comment, store_cmd_line, current_id_execution, store_output_file, depend_data]) + current_id_execution +=1; + queue_lock.release() def pool_synchrosize(): - global errorOccured - global errorExecution - if processorAvaillable <= 1: + global error_occured + global error_execution + if processor_availlable <= 1: #in this case : nothing to synchronise return debug.verbose("wait queue process ended\n") # Wait for queue to empty - while not workQueue.empty() \ - and False==errorOccured: + while not work_queue.empty() \ + and error_occured == False: time.sleep(0.2) pass # Wait all thread have ended their current process - while currentThreadWorking != 0 \ - and False==errorOccured: + while current_thread_working != 0 \ + and error_occured == False: time.sleep(0.2) pass - if False==errorOccured: + if error_occured == False: debug.verbose("queue is empty") else: un_init() debug.debug("Thread return with error ... ==> stop all the pool") - if errorExecution["id"] == -1: + if error_execution["id"] == -1: debug.error("Pool error occured ... (No return information on Pool)") return - debug.error("Error in an pool element : [" + str(errorExecution["id"]) + "]", crash=False) - debug.debug(env.print_pretty(errorExecution["cmd"]), force=True) - debug.print_compilator(str(errorExecution["out"][0])) - debug.print_compilator(str(errorExecution["err"][0])) - if errorExecution["return"] == 2: + debug.error("Error in an pool element : [" + str(error_execution["id"]) + "]", crash=False) + debug.debug(env.print_pretty(error_execution["cmd"]), force=True) + debug.print_compilator(str(error_execution["out"][0])) + debug.print_compilator(str(error_execution["err"][0])) + if error_execution["return"] == 2: debug.error("can not compile file ... [keyboard interrrupt]") else: - debug.error("can not compile file ... return value : " + str(errorExecution["return"])) + debug.error("can not compile file ... return value : " + str(error_execution["return"])) diff --git a/lutin/system.py b/lutin/system.py index 3d241a4..9dcbce3 100644 --- a/lutin/system.py +++ b/lutin/system.py @@ -16,6 +16,7 @@ import datetime from . import debug from . import module from . import tools +from . import env class System: def __init__(self): @@ -73,61 +74,70 @@ def create_module_from_system(target, dict): # Dictionnaire of Target name # inside table of ["Name of the lib", "path of the lib", boolean loaded, module loaded] -systemList={} -__start_system_name="lutinSystem_" +system_list={} +__start_system_name="System_" +def import_path(path_list): + global system_list + global_base = env.get_build_system_base_name() + debug.debug("SYSTEM: Init with Files list:") + for elem in path_list: + sys.path.append(os.path.dirname(elem)) + # Get file name: + filename = os.path.basename(elem) + # Remove .py at the end: + filename = filename[:-3] + # Remove global base name: + filename = filename[len(global_base):] + # Check if it start with the local patern: + if filename[:len(__start_system_name)] != __start_system_name: + debug.extreme_verbose("SYSTEM: NOT-Integrate: '" + filename + "' from '" + elem + "' ==> rejected") + continue + # Remove local patern + system_name = filename[len(__start_system_name):] + system_type, system_name = system_name.split('_') + debug.verbose("SYSTEM: Integrate: '" + system_type + "':'" + system_name + "' from '" + elem + "'") + if system_type in system_list: + system_list[system_type].append({"name":system_name, + "path":elem, + "system":None, + "loaded":False, + "exist":False, + "module":None}) + else: + system_list[system_type] = [{"name":system_name, + "path":elem, + "system":None, + "loaded":False, + "exist":False, + "module":None}] + debug.verbose("New list system: ") + for elem in system_list: + debug.verbose(" " + str(elem)) + for val in system_list[elem]: + debug.verbose(" " + str(val["name"])) -def import_path(path): - global targetList - matches = [] - debug.debug('Start find sub File : "%s"' %path) - for root, dirnames, filenames in os.walk(path): - tmpList = fnmatch.filter(filenames, __start_system_name + "*.py") - # Import the module : - for filename in tmpList: - debug.verbose(' Find a file : "%s"' %os.path.join(root, filename)) - sys.path.append(os.path.dirname(os.path.join(root, filename)) ) - systemName = filename.replace('.py', '') - systemName = systemName.replace(__start_system_name, '') - targetType, systemName = systemName.split('_') - debug.debug("integrate system: '" + targetType + "':'" + systemName + "' from '" + os.path.join(root, filename) + "'") - if targetType in systemList: - systemList[targetType].append({"name":systemName, - "path":os.path.join(root, filename), - "system":None, - "loaded":False, - "exist":False, - "module":None}) - else: - systemList[targetType] = [{"name":systemName, - "path":os.path.join(root, filename), - "system":None, - "loaded":False, - "exist":False, - "module":None}] - debug.debug("list system=" + str(systemList)) def display(): - global systemList - for elementName in systemList: - debug.info("integrate system: '" + elementName +"'") - for data in systemList[elementName]: - debug.info(" '" + data["name"] +"' in " + data["path"]) - + global system_list + for elementName in system_list: + debug.info("SYSTEM: Integrate system: '" + elementName +"'") + for data in system_list[elementName]: + debug.info("SYSTEM: '" + data["name"] +"' in " + data["path"]) def exist(lib_name, target_name, target) : - global systemList + global system_list debug.verbose("exist= " + lib_name + " in " + target_name) - if target_name not in systemList: + if target_name not in system_list: return False - for data in systemList[target_name]: + for data in system_list[target_name]: if data["name"] == lib_name: # we find it in the List ==> need to check if it is present in the system : if data["loaded"] == False: debug.verbose("add to path: '" + os.path.dirname(data["path"]) + "'") sys.path.append(os.path.dirname(data["path"])) debug.verbose("import system : '" + data["name"] + "'") - theSystem = __import__(__start_system_name + target_name + "_" + data["name"]) + theSystem = __import__(env.get_build_system_base_name() + __start_system_name + target_name + "_" + data["name"]) #create the system module try: debug.info("call : " + data["name"]) @@ -139,10 +149,10 @@ def exist(lib_name, target_name, target) : return False def load(target, lib_name, target_name): - global systemList - if target_name not in systemList: + global system_list + if target_name not in system_list: debug.error("you must call this function after checking of the system exist() !1!") - for data in systemList[target_name]: + for data in system_list[target_name]: if data["name"] == lib_name: if data["exist"] == False: debug.error("you must call this function after checking of the system exist() !2!") diff --git a/lutin/target.py b/lutin/target.py index a93d1c6..7731db0 100644 --- a/lutin/target.py +++ b/lutin/target.py @@ -19,6 +19,7 @@ from . import tools from . import module from . import system from . import multiprocess +from . import env class Target: def __init__(self, name, config, arch): @@ -746,25 +747,32 @@ class Target: target_list=[] -__start_target_name="lutinTarget_" +__start_target_name="Target_" -def import_path(path): +def import_path(path_list): global target_list - matches = [] - debug.debug('TARGET: Start find sub File : "%s"' %path) - for root, dirnames, filenames in os.walk(path): - tmpList = fnmatch.filter(filenames, __start_target_name + "*.py") - # Import the module : - for filename in tmpList: - debug.debug('TARGET: Find a file : "%s"' %os.path.join(root, filename)) - #matches.append(os.path.join(root, filename)) - sys.path.append(os.path.dirname(os.path.join(root, filename)) ) - targetName = filename.replace('.py', '') - targetName = targetName.replace(__start_target_name, '') - debug.debug("TARGET: integrate module: '" + targetName + "' from '" + os.path.join(root, filename) + "'") - target_list.append([targetName,os.path.join(root, filename)]) - + global_base = env.get_build_system_base_name() + debug.debug("TARGET: Init with Files list:") + for elem in path_list: + sys.path.append(os.path.dirname(elem)) + # Get file name: + filename = os.path.basename(elem) + # Remove .py at the end: + filename = filename[:-3] + # Remove global base name: + filename = filename[len(global_base):] + # Check if it start with the local patern: + if filename[:len(__start_target_name)] != __start_target_name: + debug.extreme_verbose("TARGET: NOT-Integrate: '" + filename + "' from '" + elem + "' ==> rejected") + continue + # Remove local patern + target_name = filename[len(__start_target_name):] + debug.verbose("TARGET: Integrate: '" + target_name + "' from '" + elem + "'") + target_list.append([target_name, elem]) + debug.verbose("New list TARGET: ") + for elem in target_list: + debug.verbose(" " + str(elem[0])) def load_target(name, config): global target_list @@ -776,8 +784,8 @@ def load_target(name, config): if mod[0] == name: debug.verbose("add to path: '" + os.path.dirname(mod[1]) + "'") sys.path.append(os.path.dirname(mod[1])) - debug.verbose("import target : '" + __start_target_name + name + "'") - theTarget = __import__(__start_target_name + name) + debug.verbose("import target : '" + env.get_build_system_base_name() + __start_target_name + name + "'") + theTarget = __import__(env.get_build_system_base_name() + __start_target_name + name) #create the target tmpTarget = theTarget.Target(config) return tmpTarget @@ -795,7 +803,7 @@ def list_all_target_with_desc(): tmpList = [] for mod in target_list: sys.path.append(os.path.dirname(mod[1])) - theTarget = __import__(__start_target_name + mod[0]) + theTarget = __import__(env.get_build_system_base_name() + __start_target_name + mod[0]) try: tmpdesc = theTarget.get_desc() tmpList.append([mod[0], tmpdesc]) diff --git a/lutin/z_target/lutinTarget_IOs.py b/lutin/z_target/lutinTarget_IOs.py index 300c370..3822897 100644 --- a/lutin/z_target/lutinTarget_IOs.py +++ b/lutin/z_target/lutinTarget_IOs.py @@ -74,8 +74,8 @@ class Target(target.Target): "-fobjc-link-runtime" ]) - self.add_flag("m", ["-fobjc-arc") - #self.add_flag("m", ["-fmodules") + self.add_flag("m", ["-fobjc-arc"]) + #self.add_flag("m", ["-fmodules"]) self.pkg_path_data = "share" self.pkg_path_bin = ""