From 21af5be1a30f12d429d03adc37320afa829fee2c Mon Sep 17 00:00:00 2001 From: Edouard DUPIN Date: Sun, 13 Sep 2015 23:28:01 +0200 Subject: [PATCH] [DEV] start compile in shered object mode --- lutin/depend.py | 115 ++++++++++++++----------- lutin/heritage.py | 78 +++++++++++------ lutin/module.py | 79 +++++++++++------ lutin/target.py | 3 +- lutin/tools.py | 2 +- lutin/z_builder/lutinBuilder_binary.py | 55 +++++++++--- lutin/z_target/lutinTarget_Linux.py | 3 +- 7 files changed, 218 insertions(+), 117 deletions(-) diff --git a/lutin/depend.py b/lutin/depend.py index 01f3202..7aecfb0 100644 --- a/lutin/depend.py +++ b/lutin/depend.py @@ -29,11 +29,11 @@ def _file_read_data(path, binary=False): return data_file -def need_re_build(dst, src, dependFile=None, file_cmd="", cmdLine="", force_identical=False): +def need_re_build(dst, src, depend_file=None, file_cmd="", cmd_line="", force_identical=False): debug.extreme_verbose("Request check of dependency of :") debug.extreme_verbose(" dst='" + str(dst) + "'") - debug.extreme_verbose(" str='" + str(src) + "'") - debug.extreme_verbose(" dept='" + str(dependFile) + "'") + debug.extreme_verbose(" src='" + str(src) + "'") + debug.extreme_verbose(" dept='" + str(depend_file) + "'") debug.extreme_verbose(" cmd='" + str(file_cmd) + "'") debug.extreme_verbose(" force_identical='" + str(force_identical) + "'") # if force mode selected ==> just force rebuild ... @@ -59,9 +59,9 @@ def need_re_build(dst, src, dependFile=None, file_cmd="", cmdLine="", force_iden debug.extreme_verbose(" ==> must rebuild (source time greater)") return True - if dependFile != "" \ - and dependFile != None \ - and os.path.exists(dependFile) == False: + if depend_file != "" \ + and depend_file != None \ + and os.path.exists(depend_file) == False: debug.extreme_verbose(" ==> must rebuild (no depending file)") return True @@ -70,50 +70,50 @@ def need_re_build(dst, src, dependFile=None, file_cmd="", cmdLine="", force_iden if os.path.exists(file_cmd) == False: debug.extreme_verbose(" ==> must rebuild (no commandLine file)") return True - # check if the 2 cmdline are similar : + # check if the 2 cmd_line are similar : file2 = open(file_cmd, "r") - firstAndUniqueLine = file2.read() - if firstAndUniqueLine != cmdLine: - debug.extreme_verbose(" ==> must rebuild (cmdLines are not identical)") - debug.extreme_verbose(" ==> '" + cmdLine + "'") - debug.extreme_verbose(" ==> '" + firstAndUniqueLine + "'") + first_and_unique_line = file2.read() + if first_and_unique_line != cmd_line: + debug.extreme_verbose(" ==> must rebuild (cmd_lines are not identical)") + debug.extreme_verbose(" ==> '" + cmd_line + "'") + debug.extreme_verbose(" ==> '" + first_and_unique_line + "'") file2.close() return True # the cmdfile is correct ... file2.close() - if dependFile != "" \ - and dependFile != None: - debug.extreme_verbose(" start parsing dependency file : '" + dependFile + "'") - file = open(dependFile, "r") - for curLine in file.readlines(): + if depend_file != "" \ + and depend_file != None: + debug.extreme_verbose(" start parsing dependency file : '" + depend_file + "'") + file = open(depend_file, "r") + for cur_line in file.readlines(): # normal file : end with : ": \\n" - curLine = curLine[:len(curLine)-1] + cur_line = cur_line[:len(cur_line)-1] # removing last \ ... - if curLine[len(curLine)-1:] == '\\' : - curLine = curLine[:len(curLine)-1] + if cur_line[len(cur_line)-1:] == '\\' : + cur_line = cur_line[:len(cur_line)-1] # remove white space : - #debug.verbose(" Line (read) : '" + curLine + "'"); - curLine = curLine.strip() - #debug.verbose(" Line (strip) : '" + curLine + "'"); + #debug.verbose(" Line (read) : '" + cur_line + "'"); + cur_line = cur_line.strip() + #debug.verbose(" Line (strip) : '" + cur_line + "'"); - testFile="" - if curLine[len(curLine)-1:] == ':': - debug.extreme_verbose(" Line (no check (already done) : '" + curLine + "'"); - elif len(curLine) == 0 \ - or curLine == '\\': - debug.extreme_verbose(" Line (Not parsed) : '" + curLine + "'"); + test_file="" + if cur_line[len(cur_line)-1:] == ':': + debug.extreme_verbose(" Line (no check (already done) : '" + cur_line + "'"); + elif len(cur_line) == 0 \ + or cur_line == '\\': + debug.extreme_verbose(" Line (Not parsed) : '" + cur_line + "'"); else: - testFile = curLine - debug.extreme_verbose(" Line (might check) : '" + testFile + "'"); + test_file = cur_line + debug.extreme_verbose(" Line (might check) : '" + test_file + "'"); # really check files: - if testFile!="": + if test_file != "": debug.extreme_verbose(" ==> test"); - if False==os.path.exists(testFile): + if False==os.path.exists(test_file): debug.extreme_verbose(" ==> must rebuild (a dependency file does not exist)") file.close() return True - if os.path.getmtime(testFile) > os.path.getmtime(dst): + if os.path.getmtime(test_file) > os.path.getmtime(dst): debug.extreme_verbose(" ==> must rebuild (a dependency file time is newer)") file.close() return True @@ -121,7 +121,7 @@ def need_re_build(dst, src, dependFile=None, file_cmd="", cmdLine="", force_iden file.close() # check the 2 files are identical: if force_identical == True: - # check if the 2 cmdline are similar : + # check if the 2 cmd_line are similar : size_src = _file_size(src) size_dst = _file_size(dst) if size_src != size_dst: @@ -138,14 +138,27 @@ def need_re_build(dst, src, dependFile=None, file_cmd="", cmdLine="", force_iden -def need_re_package(dst, srcList, mustHaveSrc, file_cmd="", cmdLine=""): - debug.extreme_verbose("Resuest check of dependency of :") - debug.extreme_verbose(" dst='" + dst + "'") - debug.extreme_verbose(" src()=") - for src in srcList: - debug.verbose(" '" + src + "'") +def need_re_package(dst, src_list, must_have_src, file_cmd="", cmd_line=""): + debug.extreme_verbose("Request check of dependency of :") + debug.extreme_verbose(" dst='" + str(dst) + "'") + compleate_list = [] + debug.extreme_verbose(" src:") + if type(src_list) == str: + compleate_list.append(src_list) + debug.extreme_verbose(" '" + src_list + "'") + elif type(src_list) == list: + for src in src_list: + compleate_list.append(src) + debug.extreme_verbose(" '" + str(src) + "'") + elif type(src_list) == dict: + for key in src_list: + debug.extreme_verbose(" '" + str(key) + "'") + for src in src_list[key]: + compleate_list.append(src) + debug.extreme_verbose(" '" + str(src) + "'") - if mustHaveSrc==False and len(srcList)==0: + if must_have_src == False \ + and len(compleate_list) == 0: return False # if force mode selected ==> just force rebuild ... @@ -154,14 +167,14 @@ def need_re_package(dst, srcList, mustHaveSrc, file_cmd="", cmdLine=""): return True # check if the destination existed: - if False==os.path.exists(dst): + if os.path.exists(dst) == False: debug.extreme_verbose(" ==> must re-package (dst does not exist)") return True # chek the basic date if the 2 files - if len(srcList)==0: + if len(compleate_list) == 0: debug.extreme_verbose(" ==> must re-package (no source ???)") return True - for src in srcList: + for src in compleate_list: if os.path.getmtime(src) > os.path.getmtime(dst): debug.extreme_verbose(" ==> must re-package (source time greater) : '" + src + "'") return True @@ -170,13 +183,13 @@ def need_re_package(dst, srcList, mustHaveSrc, file_cmd="", cmdLine=""): if False==os.path.exists(file_cmd): debug.extreme_verbose(" ==> must rebuild (no commandLine file)") return True - # check if the 2 cmdline are similar : + # check if the 2 cmd_line are similar : file2 = open(file_cmd, "r") - firstAndUniqueLine = file2.read() - if firstAndUniqueLine != cmdLine: - debug.extreme_verbose(" ==> must rebuild (cmdLines are not identical)") - debug.extreme_verbose(" ==> '" + cmdLine + "'") - debug.extreme_verbose(" ==> '" + firstAndUniqueLine + "'") + first_and_unique_line = file2.read() + if first_and_unique_line != cmd_line: + debug.extreme_verbose(" ==> must rebuild (cmd_lines are not identical)") + debug.extreme_verbose(" ==> '" + cmd_line + "'") + debug.extreme_verbose(" ==> '" + first_and_unique_line + "'") file2.close() return True # the cmdfile is correct ... diff --git a/lutin/heritage.py b/lutin/heritage.py index 4488143..be68d03 100644 --- a/lutin/heritage.py +++ b/lutin/heritage.py @@ -12,26 +12,28 @@ import copy from . import debug -def append_to_list(listout, list): - if type(list) == type(str()): - if list not in listout: - listout.append(list) +def append_to_list(list_out, elem): + if type(elem) == str: + if elem not in list_out: + list_out.append(elem) else: # mulyiple imput in the list ... - for elem in list: - if elem not in listout: - listout.append(elem) + for element in elem: + if element not in list_out: + list_out.append(element) class HeritageList: def __init__(self, heritage = None): - self.flags={} + self.flags = {} # sources list: - self.src=[] - self.path={} - - self.list_heritage=[] + self.src = { 'src':[], + 'dynamic':[], + 'static':[] + } + self.path = {} + self.list_heritage = [] if heritage != None: self.add_heritage(heritage) @@ -58,10 +60,13 @@ class HeritageList: self.regenerate_tree() def regenerate_tree(self): - self.flags={} + self.flags = {} # sources list: - self.src=[] - self.path={} + self.src = { 'src':[], + 'dynamic':[], + 'static':[] + } + self.path = {} # reorder heritage list : listHeritage = self.list_heritage self.list_heritage = [] @@ -105,7 +110,9 @@ class HeritageList: self.path[ppp] = value else: append_to_list(self.path[ppp], value) - append_to_list(self.src, element.src) + append_to_list(self.src['src'], element.src['src']) + append_to_list(self.src['dynamic'], element.src['dynamic']) + append_to_list(self.src['static'], element.src['static']) if "c-version" in element.flags: ver = element.flags["c-version"] if "c-version" in self.flags: @@ -122,14 +129,17 @@ class HeritageList: class heritage: def __init__(self, module): - self.name="" - self.depends=[] + self.name = "" + self.depends = [] ## Remove all variable to prevent error of multiple definition # all the parameter that the upper classe need when build - self.flags={} + self.flags = {} # sources list: - self.src=[] - self.path={} + self.src = { 'src':[], + 'dynamic':[], + 'static':[] + } + self.path = {} # update is set at true when data are newly created ==> force upper element to update self.hasBeenUpdated=False @@ -141,16 +151,32 @@ class heritage: self.flags = module.flags["export"] self.path = module.path["export"] - def add_depends(self, depend): - self.depends.append(depend) + def add_depends(self, elements): + self.depends.append(elements) def add_import_path(self, list): append_to_list(self.path, list) - def add_sources(self, list): - if type(list) == type(None): + def add_sources(self, elements): + if type(elements) == type(None): debug.error("try add element none in a list ...") - append_to_list(self.src, list) + append_to_list(self.src['src'], elements) + + def add_lib_static(self, elements): + if type(elements) == type(None): + debug.error("try add element none in a list ...") + append_to_list(self.src['static'], elements) + + def add_lib_dynamic(self, elements): + if type(elements) == type(None): + debug.error("try add element none in a list ...") + append_to_list(self.src['dynamic'], elements) + + def add_lib_interpreted(self, type_interpretation, elements): + debug.error("TODO ...") + if type(elements) == type(None): + debug.error("try add element none in a list ...") + append_to_list(self.src, elements) def need_update(self, list): self.hasBeenUpdated=True diff --git a/lutin/module.py b/lutin/module.py index a2d1fc6..9f758a4 100644 --- a/lutin/module.py +++ b/lutin/module.py @@ -63,9 +63,10 @@ class Module: self.paths = [] # The module has been already build ... self.isbuild = False - ## end of basic INIT ... if moduleType == 'BINARY' \ + or moduleType == 'BINARY_SHARED' \ + or moduleType == 'BINARY_STAND_ALONE' \ or moduleType == 'LIBRARY' \ or moduleType == 'LIBRARY_DYNAMIC' \ or moduleType == 'LIBRARY_STATIC' \ @@ -316,11 +317,13 @@ class Module: self.local_heritage = heritage.heritage(self) if package_name==None \ - and ( self.type=="BINARY" \ - or self.type=="PACKAGE" ) : + and ( self.type == 'BINARY' + or self.type == 'BINARY_SHARED' \ + or self.type == 'BINARY_STAND_ALONE' \ + or self.type == 'PACKAGE' ) : # this is the endpoint binary ... package_name = self.name - else : + else: pass # build dependency before list_sub_file_needed_to_build = [] @@ -362,7 +365,11 @@ class Module: if self.type=='LIBRARY_STATIC': debug.print_element("Library(static)", self.name, "", "") if self.type=='BINARY': - debug.print_element("Binary", self.name, "", "") + debug.print_element("Binary(auto)", self.name, "", "") + if self.type=='BINARY_SHARED': + debug.print_element("Binary (shared)", self.name, "", "") + if self.type=='BINARY_STAND_ALONE': + debug.print_element("Binary (stand alone)", self.name, "", "") if self.type=='PACKAGE': debug.print_element("Package", self.name, "", "") # ---------------------------------------------------- @@ -401,13 +408,13 @@ class Module: try: tmp_builder = builder.get_builder(fileExt); res_file = tmp_builder.compile(file, - package_name, - target, - self.sub_heritage_list, - flags = self.flags, - path = self.path, - name = self.name, - basic_path = self.origin_path) + package_name, + target, + self.sub_heritage_list, + flags = self.flags, + path = self.path, + name = self.name, + basic_path = self.origin_path) if res_file["action"] == "add": list_sub_file_needed_to_build.append(res_file["file"]) elif res_file["action"] == "path": @@ -423,11 +430,12 @@ class Module: # ---------------------------------------------------- if self.type=='PREBUILD': self.local_heritage.add_sources(self.src) - elif self.type=='LIBRARY' \ - or self.type=='LIBRARY_DYNAMIC' \ - or self.type=='LIBRARY_STATIC': - if self.type=='LIBRARY' \ - or self.type=='LIBRARY_STATIC': + elif self.type == 'LIBRARY' \ + or self.type == 'LIBRARY_DYNAMIC' \ + or self.type == 'LIBRARY_STATIC': + res_file_out = [] + if self.type == 'LIBRARY' \ + or self.type == 'LIBRARY_STATIC': try: tmp_builder = builder.get_builder_with_output("a"); list_file = tools.filter_extention(list_sub_file_needed_to_build, tmp_builder.get_input_type()) @@ -438,11 +446,11 @@ class Module: self.sub_heritage_list, name = self.name, basic_path = self.origin_path) - self.local_heritage.add_sources(res_file) + self.local_heritage.add_lib_static(res_file) except ValueError: debug.error(" UN-SUPPORTED link format: '.a'") - if self.type=='LIBRARY' \ - or self.type=='LIBRARY_DYNAMIC': + if self.type == 'LIBRARY' \ + or self.type == 'LIBRARY_DYNAMIC': try: tmp_builder = builder.get_builder_with_output("so"); list_file = tools.filter_extention(list_sub_file_needed_to_build, tmp_builder.get_input_type()) @@ -453,7 +461,7 @@ class Module: self.sub_heritage_list, name = self.name, basic_path = self.origin_path) - # TODO : self.local_heritage.add_sources(res_file) + self.local_heritage.add_lib_dynamic(res_file) except ValueError: debug.error(" UN-SUPPORTED link format: '.so'/'.dynlib'/'.dll'") try: @@ -466,21 +474,42 @@ class Module: self.sub_heritage_list, name = self.name, basic_path = self.origin_path) - self.local_heritage.add_sources(res_file) + self.local_heritage.add_lib_interpreted('java', res_file) except ValueError: debug.error(" UN-SUPPORTED link format: '.jar'") - elif self.type=='BINARY': + elif self.type == 'BINARY' \ + or self.type == 'BINARY_SHARED' \ + or self.type == 'BINARY_STAND_ALONE': try: tmp_builder = builder.get_builder_with_output("bin"); + parents = { 'src':[], + 'dynamic':[], + 'static':[] + } + parents_dynamic = [] + parents_static = [] + #extract libs ... + for elem in self.sub_heritage_list.src: + if elem[-len(target.suffix_lib_static):] == target.suffix_lib_static: + parents_static.append(elem) + elif elem[-len(target.suffix_lib_dynamic):] == target.suffix_lib_dynamic: + parents_dynamic.append(elem) + else: + parents['src'].append(elem) + + static_mode = True + if self.type == 'BINARY_SHARED': + static_mode = False res_file = tmp_builder.link(list_sub_file_needed_to_build, package_name, target, self.sub_heritage_list, name = self.name, - basic_path = self.origin_path) + basic_path = self.origin_path, + static = static_mode) except ValueError: debug.error(" UN-SUPPORTED link format: '.bin'") - elif self.type=="PACKAGE": + elif self.type == "PACKAGE": if target.name=="Android": # special case for android wrapper: try: diff --git a/lutin/target.py b/lutin/target.py index a05160e..2ff1746 100644 --- a/lutin/target.py +++ b/lutin/target.py @@ -79,6 +79,7 @@ class Target: self.suffix_dependence='.d' self.suffix_obj='.o' self.suffix_lib_static='.a' + self.prefix_lib_dynamic='lib' self.suffix_lib_dynamic='.so' self.suffix_binary='' self.suffix_package='.deb' @@ -280,7 +281,7 @@ class Target: list.append(os.path.join(self.get_build_path(binary_name), self.path_bin, module_name + self.suffix_binary + self.suffix_warning)) elif (type=="lib-shared"): list.append(file) - list.append(os.path.join(self.get_build_path(module_name), self.path_lib, module_name + self.suffix_lib_dynamic)) + list.append(os.path.join(self.get_build_path(module_name), self.path_lib, self.prefix_lib_dynamic + module_name + self.suffix_lib_dynamic)) list.append(os.path.join(self.get_build_path(module_name), self.path_lib, module_name + self.suffix_dependence)) list.append(os.path.join(self.get_build_path(module_name), self.path_lib, module_name + self.suffix_lib_dynamic + self.suffix_cmd_line)) list.append(os.path.join(self.get_build_path(module_name), self.path_lib, module_name + self.suffix_lib_dynamic + self.suffix_warning)) diff --git a/lutin/tools.py b/lutin/tools.py index e37f8f0..6d2f45e 100644 --- a/lutin/tools.py +++ b/lutin/tools.py @@ -102,7 +102,7 @@ def copy_file(src, dst, cmd_file=None, force=False, force_identical=False): debug.error("Request a copy a file that does not existed : '" + src + "'") cmd_line = "copy \"" + src + "\" \"" + dst + "\"" if force == False \ - and depend.need_re_build(dst, src, file_cmd=cmd_file , cmdLine=cmd_line, force_identical=force_identical) == False: + and depend.need_re_build(dst, src, file_cmd=cmd_file , cmd_line=cmd_line, force_identical=force_identical) == False: debug.verbose ("no need to copy ...") return debug.print_element("copy file ", os.path.relpath(src), "==>", os.path.relpath(dst)) diff --git a/lutin/z_builder/lutinBuilder_binary.py b/lutin/z_builder/lutinBuilder_binary.py index 6957ab3..5d48d93 100644 --- a/lutin/z_builder/lutinBuilder_binary.py +++ b/lutin/z_builder/lutinBuilder_binary.py @@ -40,9 +40,27 @@ def get_output_type(): ## ## @brief Commands for running gcc to link an executable. ## -def link(file, binary, target, depancy, name, basic_path): +def link(file, binary, target, depancy, name, basic_path, static = False): file_src, file_dst, file_depend, file_cmd, file_warning = target.generate_file(binary, name, basic_path, file, "bin") - #create comdLine : + debug.extreme_verbose("list files = " + str(depancy.src)) + list_static = [] + list_dynamic = [] + if static == True: + #get all parent static libs + list_static = depancy.src['static'] + # get only parent shared that is not static + for elem in depancy.src['dynamic']: + if elem[:-len(target.suffix_lib_dynamic)] + target.suffix_lib_static not in depancy.src['static']: + list_dynamic.append(elem) + else: + #get all parent dynamic libs + list_dynamic = depancy.src['dynamic'] + # get only parent shared that is not static + for elem in depancy.src['static']: + if elem[:-len(target.suffix_lib_static)] + target.suffix_lib_dynamic not in depancy.src['dynamic']: + list_static.append(elem) + + #create comand line: cmd = [ target.xx ] @@ -67,7 +85,20 @@ def link(file, binary, target, depancy, name, basic_path): except: pass try: - cmd.append(depancy.src) + cmd.append(depancy.src['src']) + except: + pass + try: + cmd.append(list_static) + except: + pass + try: + for elem in list_dynamic: + lib_path = os.path.dirname(elem) + lib_name = elem[len(lib_path)+len(target.prefix_lib_dynamic)+1:-len(target.suffix_lib_dynamic)] + cmd.append("-L" + lib_path) + cmd.append("-l" + lib_name) + #cmd.append(parents['dynamic']) except: pass try: @@ -82,27 +113,27 @@ def link(file, binary, target, depancy, name, basic_path): cmd.append(target.global_flags_ld) except: pass - cmdLine=tools.list_to_str(cmd) + cmd_line = tools.list_to_str(cmd) # check the dependency for this file : - if depend.need_re_package(file_dst, file_src, True, file_cmd, cmdLine) == False \ - and depend.need_re_package(file_dst, depancy.src, False, file_cmd, cmdLine) == False: + if depend.need_re_package(file_dst, file_src, True, file_cmd=file_cmd, cmd_line=cmd_line) == False \ + and depend.need_re_package(file_dst, depancy.src, False, file_cmd=file_cmd, cmd_line=cmd_line) == False: return file_dst tools.create_directory_of_file(file_dst) debug.print_element("Executable", name, "==>", os.path.relpath(file_dst)) - multiprocess.run_command(cmdLine, store_output_file=file_warning) + multiprocess.run_command(cmd_line, store_output_file=file_warning) if target.config["mode"] == "release"\ or env.get_force_strip_mode() == True: # get the file size of the non strip file originSize = tools.file_size(file_dst); debug.print_element("Executable(strip)", name, "", "") - cmdLineStrip=tools.list_to_str([ + cmd_lineStrip=tools.list_to_str([ target.strip, file_dst]) - multiprocess.run_command(cmdLineStrip, store_output_file=file_warning) + multiprocess.run_command(cmd_lineStrip, store_output_file=file_warning) # get the stip size of the binary - stripSize = tools.file_size(file_dst) - debug.debug("file reduce size : " + str(originSize/1024) + "ko ==> " + str(stripSize/1024) + "ko") + strip_size = tools.file_size(file_dst) + debug.debug("file reduce size : " + str(originSize/1024) + "ko ==> " + str(strip_size/1024) + "ko") # write cmd line only after to prevent errors ... - tools.store_command(cmdLine, file_cmd) + tools.store_command(cmd_line, file_cmd) diff --git a/lutin/z_target/lutinTarget_Linux.py b/lutin/z_target/lutinTarget_Linux.py index 4bb1bc6..470f51a 100644 --- a/lutin/z_target/lutinTarget_Linux.py +++ b/lutin/z_target/lutinTarget_Linux.py @@ -118,7 +118,7 @@ class Target(target.Target): tools.create_directory_of_file(target_outpath) ## Create share datas - target_outpath_data = os.path.join(target_outpath, "share") + target_outpath_data = os.path.join(target_outpath, "share", pkgName) tools.create_directory_of_file(target_outpath_data) debug.debug("heritage for " + str(pkgName) + ":") for heritage in heritage_list.list_heritage: @@ -139,6 +139,7 @@ class Target(target.Target): path_dst = os.path.join(target_outpath_bin, pkgName + self.suffix_binary) debug.warning("path_dst: " + str(path_dst)) tools.copy_file(path_src, path_dst) + ## Create libraries