[DEV] basic version with global mirror added only at the first sync

README.rst

@@ -3,6 +3,12 @@ Lutin
 
 `island` is a generic source downloader and syncronizer is a FREE software tool.
 
+It is compatible with basic format of repo-git manifest. This project is created to be easyest to read with simple interface
+(no internal git usage, but user level git usage) The main point to create the "fork" is the non-support of repo of relativity
+in submodule of git (submodule reference with ../xxx.git) This point is really important when you want to have a rellocable
+manifest and project with submodule. The simpl eexample is the atria-soft / generic-library / musicdsp that are availlable on
+github, gitlab, bitbucket and personal server.
+
 
 .. image:: https://badge.fury.io/py/island.png
     :target: https://pypi.python.org/pypi/island
@@ -61,6 +67,45 @@ Install pip on MacOs:
 
   sudo easy_install pip
 
+Usage
+-----
+
+Select a manifest:
+
+  island init http://github.com/atria-soft/manifest.git
+
+Download and synchronize the sources:
+
+  island sync
+
+Select all branch availlable in the worktree: (checkout origin/dev in dev branch and track it, do nothing if the branch does not exist)
+
+  island checkout dev
+
+Show the status of the workspace
+
+  island status
+
+TODO list
+---------
+
+  - When sync checkout the new manifest
+  - status: show how many time late we are on the branch
+  - sync: filter the apply of this cmd
+  - create snapshot
+  - use a snapshot
+  - commit all change in a single commit name and date
+  - push all change in the origin branch
+  - stash/unstash all change
+  - permit to set the pasword when requested by git
+  - sync: show download progress
+  - support single project mirror
+  - support submodule mirror
+  - support project upstream
+  - support submodule add upstream
+  - push modilfication in all late mirorr (force mode optionnal) ==> for automatic server synchronisation in 4 lines
+  - a good documation of the format and the usage
+  - parallele download / sync / push ...
 
 License (MPL v2.0)
 ---------------------

island/actions/islandAction_checkout.py

@@ -0,0 +1,137 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+##
+## @author Edouard DUPIN
+##
+## @copyright 2012, Edouard DUPIN, all right reserved
+##
+## @license MPL v2.0 (see license file)
+##
+
+from island import debug
+from island import tools
+from island import env
+from island import multiprocess
+from island import manifest
+import os
+
+
+def help():
+	return "plop"
+
+
+
+
+
+def execute(arguments):
+	debug.info("execute:")
+	for elem in arguments:
+		debug.info("    '" + str(elem.get_arg()) + "'")
+	if len(arguments) != 1:
+		debug.error("checkout: missing argument to select the new branch ...")
+	branch_to_checkout = ""
+	for elem in arguments:
+		if elem.get_option_name() == "":
+			if branch_to_checkout != "":
+				debug.error("checkout branch already set : '" + branch_to_checkout + "' !!! '" + elem.get_arg() + "'")
+			branch_to_checkout = elem.get_arg()
+		else:
+			debug.error("Wrong argument: '" + elem.get_option_name() + "' '" + elem.get_arg() + "'")
+	
+	# check if .XXX exist (create it if needed)
+	if    os.path.exists(env.get_island_path()) == False \
+	   or os.path.exists(env.get_island_path_config()) == False \
+	   or os.path.exists(env.get_island_path_manifest()) == False:
+		debug.error("System already init have an error: missing data: '" + str(env.get_island_path()) + "'")
+	
+	configuration = manifest.load_config()
+	
+	file_source_manifest = os.path.join(env.get_island_path_manifest(), configuration["file"])
+	if os.path.exists(file_source_manifest) == False:
+		debug.error("Missing manifest file : '" + str(file_source_manifest) + "'")
+	
+	mani = manifest.Manifest(file_source_manifest)
+	
+	all_project = mani.get_all_configs()
+	debug.info("checkout of: " + str(len(all_project)) + " projects")
+	id_element = 0
+	for elem in all_project:
+		id_element += 1
+		debug.verbose("checkout : " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name))
+		git_repo_path = os.path.join(env.get_island_root_path(), elem.path)
+		if os.path.exists(git_repo_path) == False:
+			debug.warning("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> repository does not exist ...")
+			continue
+		
+		# check if the repository is modify
+		cmd = "git diff --quiet"
+		debug.verbose("execute : " + cmd)
+		ret_diff = multiprocess.run_command(cmd, cwd=git_repo_path)
+		# get local branch
+		cmd = "git branch"
+		debug.verbose("execute : " + cmd)
+		ret_branch = multiprocess.run_command(cmd, cwd=git_repo_path)
+		
+		# get  tracking branch
+		cmd = "git rev-parse --abbrev-ref --symbolic-full-name @{u}"
+		debug.verbose("execute : " + cmd)
+		ret_track = multiprocess.run_command(cmd, cwd=git_repo_path)
+		
+		is_modify = True
+		if ret_diff[0] == 0:
+			is_modify = False
+		
+		if is_modify == True:
+			debug.warning("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> modify data can not checkout new branch")
+			continue
+		
+		list_branch = ret_branch[1].split('\n')
+		list_branch2 = []
+		select_branch = ""
+		for elem_branch in list_branch:
+			if elem_branch[:2] == "* ":
+				list_branch2.append([elem_branch[2:], True])
+				select_branch = elem_branch[2:]
+			else:
+				list_branch2.append([elem_branch[2:], False])
+		
+		
+		# check if we are on the good branch:
+		if branch_to_checkout == select_branch:
+			debug.info("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> No change already on good branch")
+			continue
+		
+		# check if we have already checkout the branch before
+		if branch_to_checkout in list_branch2:
+			cmd = "git checkout " + branch_to_checkout
+			debug.verbose("execute : " + cmd)
+			ret = multiprocess.run_command(cmd, cwd=git_repo_path)
+			if     ret[1] != "" \
+			   and ret != False:
+				debug.info("'" + ret + "'")
+				debug.error("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> Can not checkout to the corest branch")
+				continue
+			debug.info("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> switch branch")
+			# TODO : Check the number of commit to the origin/XXX branch ....
+			
+			continue
+			
+		
+		# TODO: Check if the remote branch exist ...
+		
+		
+		# checkout the new branch:
+		cmd = "git checkout --quiet " + elem.select_remote["name"] + "/" + branch_to_checkout + " -b " + branch_to_checkout
+		# + " --track " + elem.select_remote["name"] + "/" + branch_to_checkout
+		debug.verbose("execute : " + cmd)
+		ret = multiprocess.run_command(cmd, cwd=git_repo_path)
+		if     ret[1] != "" \
+		   and ret != False:
+			debug.info("'" + str(ret) + "'")
+			debug.error("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> Can not checkout to the correct branch")
+			continue
+		debug.info("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> create new branch")
+		continue
+
+
+

island/actions/islandAction_status.py

@@ -28,7 +28,7 @@ def execute(arguments):
 	for elem in arguments:
 		debug.info("    '" + str(elem.get_arg()) + "'")
 	if len(arguments) != 0:
-		debug.error("Sync have not parameter")
+		debug.error("status have not parameter")
 	
 	# check if .XXX exist (create it if needed)
 	if    os.path.exists(env.get_island_path()) == False \

island/actions/islandAction_sync.py

@@ -69,9 +69,21 @@ def execute(arguments):
 			if     ret != "" \
 			   and ret != False:
 				# all is good, ready to get the system work corectly
-				debug.info("'" + ret + "'")
+				debug.info("'" + str(ret) + "'")
 				debug.error("Clone repository does not work ... ")
 				continue
+			# add global mirror list
+			for mirror in elem.select_remote["mirror"]:
+				debug.verbose("Add global mirror: " + str(mirror))
+				cmd = "git remote add " + mirror["name"] + " " + mirror["fetch"] + "/" + elem.name 
+				ret = multiprocess.run_command_direct(cmd, cwd=git_repo_path)
+				if     ret != "" \
+				   and ret != False:
+					# all is good, ready to get the system work corectly
+					debug.info("'" + str(ret) + "'")
+					debug.warning("Can not add global mirror ... ")
+					continue
+				debug.verbose("Add global mirror: " + str(mirror) + " (done)")
 			#debug.info("plop " + str(elem.select_remote.keys()))
 			# check submodule if requested:
 			if     elem.select_remote["sync"] == True \
@@ -82,7 +94,7 @@ def execute(arguments):
 				if     ret != "" \
 				   and ret != False:
 					# all is good, ready to get the system work corectly
-					debug.info("'" + ret + "'")
+					debug.info("'" + str(ret) + "'")
 					debug.error("Can not init submodules ... ")
 					continue
 				cmd = "git submodule update"
@@ -93,7 +105,7 @@ def execute(arguments):
 				elif     ret != "" \
 				     and ret != False:
 					# all is good, ready to get the system work corectly
-					debug.info("'" + ret + "'")
+					debug.info("'" + str(ret) + "'")
 					debug.error("Can not init submodules ... ")
 					continue
 				
@@ -153,8 +165,11 @@ def execute(arguments):
 			debug.info("    ==> sync submodule")
 			cmd = "git submodule sync"
 			ret = multiprocess.run_command_direct(cmd, cwd=git_repo_path)
-			if     ret != "" \
-			   and ret != False:
+			if ret[:31] == "Synchronizing submodule url for":
+				#all is good ...
+				debug.info("    " + ret)
+			elif     ret != "" \
+			     and ret != False:
 				# all is good, ready to get the system work corectly
 				debug.info("'" + ret + "'")
 				debug.error("Can not sync submodules ... ")
@@ -179,4 +194,4 @@ def execute(arguments):
 				debug.info("'" + ret + "'")
 				debug.error("Can not init submodules ... ")
 				continue
-			"""
+			"""

island/manifest.py

@@ -15,6 +15,7 @@ import copy
 from . import debug
 from . import tools
 from . import env
+from . import multiprocess
 
 from lxml import etree
 
@@ -83,6 +84,29 @@ class Manifest():
 						name = child.attrib[attr]
 					elif attr == "fetch":
 						fetch = child.attrib[attr]
+						if     len(fetch) >= 2 \
+						   and fetch[:2] == "..":
+							# we have a relative island manifest ==> use local manifest origin to get the full origin
+							cmd = "git remote get-url origin"
+							debug.verbose("execute : " + cmd)
+							base_origin = multiprocess.run_command(cmd, cwd=env.get_island_path_manifest())
+							debug.verbose("base_origin=" + base_origin[1])
+							base_origin = base_origin[1]
+							while     len(fetch) >= 2 \
+							      and fetch[:2] == "..":
+								fetch = fetch[2:]
+								while      len(fetch) >= 1 \
+								       and (    fetch[0] == "/" \
+								             or fetch[0] == "\\"):
+									fetch = fetch[1:]
+								base_origin = base_origin[:base_origin.rfind('/')]
+							debug.verbose("new base_origin=" + base_origin)
+							debug.verbose("tmp fetch=" + fetch)
+							if fetch != "":
+								fetch = base_origin + "/" + fetch
+							else:
+								fetch = base_origin
+							debug.verbose("new fetch=" + fetch)
 						while     len(fetch) > 1 \
 						      and (    fetch[-1] == "\\" \
 						            or fetch[-1] == "/") :
@@ -90,11 +114,42 @@ class Manifest():
 					else:
 						debug.error("(l:" + str(child.sourceline) + ") Parsing the manifest : Unknow '" + child.tag + "'  attibute : '" + attr + "', availlable:[name,fetch]")
 				debug.debug("(l:" + str(child.sourceline) + ")     find '" + child.tag + "' : name='" + name + "' fetch='" + fetch + "'");
+				# parse the sub global mirror list
+				mirror_list = []
+				for child_2 in child:
+					if child_2.tag == "mirror":
+						# find a new mirror
+						mirror_name = ""
+						mirror_fetch = ""
+						for attr_2 in child_2.attrib:
+							if attr_2 == "name":
+								mirror_name = child_2.attrib[attr_2]
+							elif attr_2 == "fetch":
+								mirror_fetch = child_2.attrib[attr_2]
+								while     len(mirror_fetch) > 1 \
+								      and (    mirror_fetch[-1] == "\\" \
+								            or mirror_fetch[-1] == "/") :
+									mirror_fetch = mirror_fetch[:-1]
+							else:
+								debug.error("(l:" + str(child_2.sourceline) + ") Parsing the manifest : Unknow '" + child_2.tag + "'  attibute : '" + attr_2 + "', availlable:[name,fetch]")
+						debug.debug("mirror: '" + mirror_name + "' '" + mirror_fetch + "'")
+						if mirror_name == "":
+							debug.error("(l:" + str(child_2.sourceline) + ") Missing mirrot 'name'")
+						if mirror_fetch == "":
+							debug.error("(l:" + str(child_2.sourceline) + ") Missing mirror 'fetch'")
+						mirror_list.append({
+						    "name":mirror_name,
+						    "fetch":mirror_fetch,
+						    })
+					else:
+						debug.error("(l:" + str(child_2.sourceline) + ") Parsing the manifest : Unknow '" + child_2.tag + "', availlable:[mirror]")
 				self.remotes.append({
 				    "name":name,
-				    "fetch":fetch
+				    "fetch":fetch,
+				    "mirror":mirror_list
 				    })
 				continue
+				
 			if child.tag == "include":
 				name = ""
 				for attr in child.attrib:
@@ -200,18 +255,22 @@ class Manifest():
 				tmp_default = copy.deepcopy(self.default)
 			else:
 				tmp_default = copy.deepcopy(self.default_base)
+		# debug.error(" self.default=" + str(self.default))
 		# add all local project
 		for elem in self.projects:
+			debug.verbose("parse element " + str(elem))
 			conf = RepoConfig()
 			conf.name = elem["name"]
 			conf.path = self._create_path_with_elem(elem)
 			
 			# add default remote for the project (search in herited element)
 			for remote in self.remotes:
+				debug.verbose("    Local Remote: " + str(remote))
 				if remote["name"] == default["remote"]:
 					conf.remotes.append(remote)
 			if len(conf.remotes) == 0:
 				for remote in upper_remotes:
+					debug.verbose("    upper Remote: " + str(remote))
 					if remote["name"] == default["remote"]:
 						conf.remotes.append(remote)
 			if len(conf.remotes) == 0:
@@ -221,11 +280,14 @@ class Manifest():
 			conf.select_remote = None
 			debug.debug("    remotes count: " + str(len(conf.remotes)))
 			for remote in conf.remotes:
+				debug.debug("    remote=" + str(remote))
 				debug.debug("    Ckeck remote : " + remote["name"] + " == " + default["remote"])
 				debug.verbose("          remote=" + str(remote))
 				debug.verbose("          default=" + str(default))
 				if remote["name"] == default["remote"]:
 					conf.select_remote = copy.deepcopy(remote)
+					debug.debug("    copy select=" + str(conf.select_remote))
+					
 					# copy the submodule synchronisation
 					conf.select_remote["sync"] = default["sync"]
 					break

setup.py

@@ -16,7 +16,7 @@ def readme():
 
 # https://pypi.python.org/pypi?%3Aaction=list_classifiers
 setup(name='island',
-      version='0.2.0',
+      version='0.3.2',
       description='island generic source manager (like repo in simple mode)',
       long_description=readme(),
       url='http://github.com/HeeroYui/island',