diff --git a/island/actions/islandAction_deliver.py b/island/actions/islandAction_deliver.py index 854fb5c..5348dcc 100644 --- a/island/actions/islandAction_deliver.py +++ b/island/actions/islandAction_deliver.py @@ -152,7 +152,7 @@ def execute(_arguments): version_path_file = os.path.join(git_repo_path, "version.txt") # update version file: - tools.file_write_data(version_path_file, tools.version_to_string(new_version_description)) + tools.file_write_data(version_path_file, tools.version_to_string(new_version_description) + "\n") commands.add_file(git_repo_path, version_path_file) commands.commit_all( git_repo_path, @@ -162,7 +162,7 @@ def execute(_arguments): commands.checkout(git_repo_path, source_branch) commands.reset_hard(git_repo_path, destination_branch) new_version_description.append("dev") - tools.file_write_data(version_path_file, tools.version_to_string(new_version_description)) + tools.file_write_data(version_path_file, tools.version_to_string(new_version_description) + "\n") commands.add_file(git_repo_path, version_path_file) commands.commit_all(git_repo_path, status.default_update_message) commands.checkout(git_repo_path, destination_branch) diff --git a/island/manifest.py b/island/manifest.py index c2ac567..123dbb5 100644 --- a/island/manifest.py +++ b/island/manifest.py @@ -6,10 +6,15 @@ @copyright 2012, Edouard DUPIN, all right reserved @license MPL v2.0 (see license file) """ - +import json import copy import os -from typing import List, Dict +from typing import List, Dict, Any + +# import pyyaml module +import yaml +from yaml.loader import SafeLoader + from lxml import etree @@ -44,19 +49,19 @@ def check_lutin_is_init(): exit(-1) -class Manifest: - def __init__(self, manifest_xml: str) -> None: - self.manifest_xml = manifest_xml +class ManifestVSC: + def __init__(self, manifest_yaml: str) -> None: + self.manifest_yaml = manifest_yaml self.projects: List[Dict] = [] self.default = None self.default_base = { "remote": "origin", - "revision": "master", + "revision": "master", # todo: rename 'branch-release' "sync": False, + "branch-develop": "dev", + "default-branch": "dev", } self.remotes: List[Dict] = [] - self.includes: List[Dict] = [] - self.links: List[Dict] = [] self.deliver_master = "master" self.deliver_develop = "develop" self.deliver_mode = "merge" @@ -66,10 +71,154 @@ class Manifest: self._check_double_path([]) def get_links(self): - return self.links + return [] def _load(self): - debug.debug("manifest : '{self.manifest_xml}'") + debug.debug(f"manifest VSC: '{self.manifest_yaml}'") + data = {} + with open(self.manifest_yaml) as f: + data = yaml.load(f, Loader=SafeLoader) + if "repositories" not in data: + debug.error(f"in '{self.manifest_yaml}' VSC manifest: missing root key: repositories ") + for name, value in data["repositories"].items(): + if "type" in value and value["type"] != "git": + debug.error(f"in '{self.manifest_yaml}' VSC manifest: unsupported type: '{value['type']}' for {name}") + if "url" not in value: + debug.error(f"in '{self.manifest_yaml}' VSC manifest: missing 'url' for {name}") + url = value["url"] + # TODO: Manage URL remote element ==> dynamic add !!! and manage http(s)://xxx.yyy/* + url_split = url.split(":") + if len(url_split) > 1: + url = url_split[-1] + version = None + if "version" not in value: + version = value["version"] + self.projects.append( + { + "name": url, + "path": name, + "tag": version, + } + ) + + def _create_path_with_elem(self, element): + # debug.info(f"create path : {json.dumps(element)}") + path = element["path"] + if path == "": + path = element["name"] + if len(path) >= 4 and path[-4:] == ".git": + path = path[:-4] + # debug.info(f" generate path {path}") + return path + + def _check_double_path(self, list_path=[], space=""): + # debug.debug(f"{space}check path : '{self.manifest_yaml}'") + for elem in self.projects: + path = self._create_path_with_elem(elem) + debug.debug(f"{space} check path:'{path}'") + if path in list_path: + debug.error(f"Check Manifest error : double use of the path '{path}'") + list_path.append(path) + + def get_all_configs(self, default=None, upper_remotes=[]): + out = [] + if default is None: + if self.default is not None: + default = copy.deepcopy(self.default) + else: + default = copy.deepcopy(self.default_base) + # debug.error(f" self.default={self.default}") + # add all local project + for elem in self.projects: + debug.verbose(f"parse element {elem}") + if env.need_process_with_filter(elem["name"]) is False: + debug.info(f"Filter repository: {elem['name']}") + continue + conf = repo_config.RepoConfig() + conf.name = elem["name"] + conf.tag = elem["tag"] + conf.path = self._create_path_with_elem(elem) + + # add default remote for the project (search in inherited element) + for remote in self.remotes: + debug.verbose(f" Local Remote: {remote}") + if remote["name"] == default["remote"]: + conf.remotes.append(remote) + if len(conf.remotes) == 0: + for remote in upper_remotes: + debug.verbose(f" upper Remote: {remote}") + if remote["name"] == default["remote"]: + conf.remotes.append(remote) + if len(conf.remotes) == 0: + debug.error( + f" No remote detected: {len(conf.remotes)} for {conf.name} with default remote name : {default['remote']} self remote: {self.remotes}" + ) + + # select default remote: + conf.select_remote = None + debug.debug(f" remotes count: {len(conf.remotes)}") + for remote in conf.remotes: + debug.debug(f" remote={remote}") + debug.debug(f" Check remote : {remote['name']} == {default['remote']}") + debug.verbose(f" remote={remote}") + debug.verbose(f" default={default}") + if remote["name"] == default["remote"]: + conf.select_remote = copy.deepcopy(remote) + debug.debug(f" copy select={conf.select_remote}") + + # copy the submodule synchronization + conf.select_remote["sync"] = default["sync"] + break + if conf.select_remote == None: + debug.error(f"missing remote for project: {conf.name}") + + conf.branch = default["revision"] + out.append(conf) + # create a temporary variable to transmit the remote to includes + upper_remotes_forward = copy.deepcopy(upper_remotes) + for remote in self.remotes: + upper_remotes_forward.append(remote) + + if False: + debug.info("list of all repo:") + for elem in out: + debug.info(f" '{elem.name}'") + debug.info(f" path: {elem.path}") + debug.info(f" remotes: {elem.remotes}") + debug.info(f" select_remote: {elem.select_remote}") + debug.info(f" branch: {elem.branch}") + return out + + +class Manifest: + def __init__(self, manifest_xml: str) -> None: + self.manifest_xml = manifest_xml + self.projects: List[Dict] = [] + self.default = None + self.default_base = { + "remote": "origin", + "revision": "master", # todo: rename 'branch-release' + "sync": False, + "branch-develop": "dev", + "default-branch": "dev", + } + self.remotes: List[Dict] = [] + self.includes: List[Dict] = [] + self.imports: List[Dict] = [] + self.links: List[Dict] = [] + self.deliver_master = "master" + self.deliver_develop = "develop" + self.deliver_mode = "merge" + # load the manifest + self._load() + # check error in manifest (double path ...) + self._check_double_path([]) + + def get_links(self) -> Dict[str, Any]: + return self.links + + def _load(self) -> None: + debug.debug(f"manifest : '{self.manifest_xml}'") tree = etree.parse(self.manifest_xml) root = tree.getroot() if root.tag != "manifest": @@ -151,6 +300,25 @@ class Manifest: self.remotes.append({"name": name, "fetch": fetch, "mirror": mirror_list}) continue + if child.tag == "import": + type_manifest = "vcs" + name = "" + for attr in child.attrib: + if attr == "type": + type_manifest = child.attrib[attr] + if type_manifest not in ["vcs"]: + debug.error( + f"(l:{child.sourceline}) Parsing the manifest: {child.tag} attribute '{attr}={type_manifest}' value available: [vcs]" + ) + elif attr == "name": + name = child.attrib[attr] + else: + debug.error(f"(l:{child.sourceline}) Parsing the manifest : unknown '{child.tag}' attribute : '{attr}', available:[name]") + new_name_yaml = os.path.join(os.path.dirname(self.manifest_xml), name) + if os.path.exists(new_name_yaml) is False: + debug.error(f"(l:{child.sourceline}) The file does not exist : '{new_name_yaml}'") + self.imports.append({"name": name, "path": new_name_yaml, "type": type_manifest}) + continue if child.tag == "include": name = "" for attr in child.attrib: @@ -165,37 +333,59 @@ class Manifest: debug.error(f"(l:{child.sourceline}) The file does not exist : '{new_name_xml}'") self.includes.append({"name": name, "path": new_name_xml, "manifest": None}) continue - if child.tag == "default": + if child.tag == "option": remote = "origin" - revision = "master" + deliver_master = "master" sync = False - for attr in child.attrib: - if attr == "remote": - remote = child.attrib[attr] - elif attr == "revision": - revision = child.attrib[attr] - elif attr == "sync-s": # synchronize submodule ... automaticaly - sync = child.attrib[attr] - if sync.lower() == "true" or sync == "1" or sync.lower() == "yes": + deliver_source = "dev" + default_branch = "dev" + deliver_mode = "fast_forward" + + for child_2 in child: + if child_2.tag == "branch-release": + deliver_master = child_2.text + elif child_2.tag == "branch-develop": + deliver_source = child_2.text + elif child_2.tag == "default-branch": + default_branch = child_2.text + elif child_2.tag == "default-remote": + remote = child_2.text + elif child_2.tag == "deliver-mode": + deliver_mode = child_2.text + if deliver_mode not in ["merge", "fast_forward"]: + debug.error(f"(l:{child.sourceline}) Parsing the manifest: option 'deliver-mode' value available: [merge,fast_forward]") + elif child_2.tag == "synchronize-submodule": + sync_tmp = child_2.text + if sync_tmp.lower() == "true" or sync_tmp == "1" or sync_tmp.lower() == "yes": sync = True - elif sync.lower() == "false" or sync == "0" or sync.lower() == "no": + elif sync_tmp.lower() == "false" or sync_tmp == "0" or sync_tmp.lower() == "no": sync = False else: debug.error( - f"(l:{child.sourceline}) Parsing the manifest : unknown '{child.tag}' attribute : '{attr}', value:'{sync}' available:[true,1,yes,false,0,no]" + f"(l:{child.sourceline}) Parsing the manifest : unknown '{child.tag}/{child2.tag}', value:'{sync}' available:[true,1,yes,false,0,no]" ) else: debug.error( - f"(l:{child.sourceline}) Parsing the manifest : unknown '{child.tag}' attribute : '{attr}', available:[remote,revision,sync-s]" + f"(l:{child_2.sourceline}) Parsing the manifest : unknown '{child.tag}/{child_2.tag}', available:[branch-release,branch-develop,default-branch,default-remote,synchronize-submodule]" ) - if self.default != None: - debug.error(f"(l:{child.sourceline}) Parsing the manifest : Node '{child.tag}' already set") self.default = { "remote": remote, - "revision": revision, + "revision": deliver_master, "sync": sync, + "branch-develop": deliver_source, + "default-branch": default_branch, + "deliver-mode": deliver_mode, } - debug.debug(f"(l:{child.sourceline}) find '{child.tag}' : remote='{remote}' revision='{revision}' sync={sync}") + self.deliver_master = deliver_master + self.deliver_develop = deliver_source + self.deliver_mode = deliver_mode + debug.debug(f"(l:{child.sourceline}) find '{child.tag}':") + debug.debug(f" - default-branch:'{default_branch}':") + debug.debug(f" - default-remote:'{remote}':") + debug.debug(f" - synchronize-submodule:'{sync}':") + debug.debug(f" - branch-release:'{deliver_master}':") + debug.debug(f" - branch-develop:'{deliver_source}':") + debug.debug(f" - deliver-mode:'{deliver_mode}':") continue if child.tag == "project": name = "" @@ -225,32 +415,6 @@ class Manifest: ) debug.debug(f"(l:{child.sourceline}) find '{child.tag}' : name='{name}' path='{path}' tag='{str(tag_sha1)}'") continue - if child.tag == "option": - # not managed ==> future use - type_option = "" - value_option = "" - for attr in child.attrib: - if attr == "type": - type_option = child.attrib[attr] - elif attr == "value": - value_option = child.attrib[attr] - else: - debug.error( - f"(l:{child.sourceline}) Parsing the manifest: unknown '{child.tag}' attribute : '{attr}', available:[type,value]" - ) - if type_option == "deliver_master": - self.deliver_master = value_option - elif type_option == "deliver_develop": - self.deliver_develop = value_option - elif type_option == "deliver_mode": - self.deliver_mode = value_option - if self.deliver_mode not in ["merge", "fast_forward"]: - debug.error(f"(l:{child.sourceline}) Parsing the manifest: option 'deliver_mode' value available: [merge,fast_forward]") - else: - debug.error( - f"(l:{child.sourceline}) Parsing the manifest: unknown 'type' value available: [deliver_master,deliver_develop,deliver_mode]" - ) - continue if child.tag == "link": # not managed ==> future use source = "" @@ -285,6 +449,8 @@ class Manifest: # now we parse all sub repo: for elem in self.includes: elem["manifest"] = Manifest(elem["path"]) + for elem in self.imports: + elem["manifest"] = ManifestVSC(elem["path"]) # inside data child.text @@ -370,6 +536,11 @@ class Manifest: list_project = elem["manifest"].get_all_configs(default, upper_remotes_forward) for elem_proj in list_project: out.append(elem_proj) + # add all import project + for elem in self.imports: + list_project = elem["manifest"].get_all_configs(default, upper_remotes_forward) + for elem_proj in list_project: + out.append(elem_proj) # ------------------------------------------------------------- # -- add Volatile ... @@ -410,7 +581,7 @@ def tag_manifest(manifest_xml_filename, all_tags): root = tree.getroot() includes = [] if root.tag != "manifest": - debug.error(f"(l:{child.sourceline}) in '{file}' have not main xml node='manifest'") + debug.error("(l:{child.sourceline}) in '{file}' have not main xml node='manifest'") return False for child in root: if type(child) == etree._Comment: @@ -474,7 +645,7 @@ def tag_clear(manifest_xml_filename): root = tree.getroot() includes = [] if root.tag != "manifest": - debug.error(f"(l:{child.sourceline}) in '{file}' have not main xml node='manifest'") + debug.error("(l:{child.sourceline}) in '{file}' have not main xml node='manifest'") return False for child in root: if type(child) == etree._Comment: diff --git a/setup.py b/setup.py index bcf07f6..f9c3b7e 100755 --- a/setup.py +++ b/setup.py @@ -55,9 +55,9 @@ setup( "lxml", "realog", "death", + "PyYAML", ], include_package_data=True, - zip_safe=False, ) # To developp: sudo ./setup.py install