16 Commits
0.1.0 ... 0.4.0

22 changed files with 1414 additions and 609 deletions

View File

@@ -1,25 +1,31 @@
Lutin
=====
`maestro` is a generic source downloader and syncronizer is a FREE software tool.
`island` is a generic source downloader and syncronizer is a FREE software tool.
It is compatible with basic format of repo-git manifest. This project is created to be easyest to read with simple interface
(no internal git usage, but user level git usage) The main point to create the "fork" is the non-support of repo of relativity
in submodule of git (submodule reference with ../xxx.git) This point is really important when you want to have a rellocable
manifest and project with submodule. The simpl eexample is the atria-soft / generic-library / musicdsp that are availlable on
github, gitlab, bitbucket and personal server.
.. image:: https://badge.fury.io/py/maestro.png
:target: https://pypi.python.org/pypi/maestro
.. image:: https://badge.fury.io/py/island.png
:target: https://pypi.python.org/pypi/island
Release (master)
----------------
.. image:: https://travis-ci.org/HeeroYui/maestro.svg?branch=master
:target: https://travis-ci.org/HeeroYui/maestro
.. image:: https://travis-ci.org/HeeroYui/island.svg?branch=master
:target: https://travis-ci.org/HeeroYui/island
Developement (dev)
------------------
.. image:: https://travis-ci.org/HeeroYui/maestro.svg?branch=dev
:target: https://travis-ci.org/HeeroYui/maestro
.. image:: https://travis-ci.org/HeeroYui/island.svg?branch=dev
:target: https://travis-ci.org/HeeroYui/island
Instructions
@@ -27,18 +33,18 @@ Instructions
This is a tool to download ```git``` source repositiry in a versatile worktree
Maestro is under a FREE license that can be found in the LICENSE file.
island is under a FREE license that can be found in the LICENSE file.
Any contribution is more than welcome ;)
git repository
--------------
http://github.com/HeeroYui/maestro/
http://github.com/HeeroYui/island/
Documentation
-------------
http://HeeroYui.github.io/maestro/
http://HeeroYui.github.io/island/
Installation
------------
@@ -47,7 +53,7 @@ Requirements: ``Python >= 2.7`` and ``pip``
Just run:
pip install maestro
pip install island
Install pip on debian/ubuntu:
@@ -61,11 +67,50 @@ Install pip on MacOs:
sudo easy_install pip
Usage
-----
Select a manifest:
island init http://github.com/atria-soft/manifest.git
Download and synchronize the sources:
island sync
Select all branch availlable in the worktree: (checkout origin/dev in dev branch and track it, do nothing if the branch does not exist)
island checkout dev
Show the status of the workspace
island status
TODO list
---------
- When sync checkout the new manifest
- status: show how many time late we are on the branch
- sync: filter the apply of this cmd
- create snapshot
- use a snapshot
- commit all change in a single commit name and date
- push all change in the origin branch
- stash/unstash all change
- permit to set the pasword when requested by git
- sync: show download progress
- support single project mirror
- support submodule mirror
- support project upstream
- support submodule add upstream
- push modilfication in all late mirorr (force mode optionnal) ==> for automatic server synchronisation in 4 lines
- a good documation of the format and the usage
- parallele download / sync / push ...
License (MPL v2.0)
---------------------
Copyright maestro Edouard DUPIN
Copyright island Edouard DUPIN
Licensed under the Mozilla Public License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

11
bin/island Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import island

View File

@@ -7,25 +7,63 @@
##
## @license MPL v2.0 (see license file)
##
# for path inspection:
import sys
import os
import sys
import fnmatch
import copy
import maestro
import maestro.debug as debug
import maestro.arg as arguments
import maestro.host as host
import maestro.env as env
import maestro.tools as tools
import maestro.host as maestroHost
import maestro.tools as maestroTools
# Local import
from . import host
from . import tools
from . import debug
from . import env
from . import actions
from . import arguments
is_init = False
myArgs = arguments.maestroArg()
def filter_name_and_file(root, list_files, filter):
# filter elements:
tmp_list = fnmatch.filter(list_files, filter)
out = []
for elem in tmp_list:
if os.path.isfile(os.path.join(root, elem)) == True:
out.append(elem);
return out;
def import_path_local(path):
out = []
debug.verbose("island files: " + str(path) + " [START]")
list_files = os.listdir(path)
# filter elements:
tmp_list_island_file = filter_name_and_file(path, list_files, env.get_system_base_name() + "*.py")
debug.verbose("island files: " + str(path) + " : " + str(tmp_list_island_file))
# Import the module:
for filename in tmp_list_island_file:
out.append(os.path.join(path, filename))
debug.verbose(" Find a file : '" + str(out[-1]) + "'")
return out
def init():
global is_init;
if is_init == True:
return
list_of_island_files = import_path_local(os.path.join(tools.get_current_path(__file__), 'actions'))
actions.init(list_of_island_files)
is_init = True
myArgs = arguments.islandArg()
myArgs.add_section("option", "Can be set one time in all case")
myArgs.add("h", "help", desc="Display this help")
myArgs.add("v", "verbose", list=[["0","None"],["1","error"],["2","warning"],["3","info"],["4","debug"],["5","verbose"],["6","extreme_verbose"]], desc="display debug level (verbose) default =2")
myArgs.add("c", "color", desc="Display message in color")
# for init only
#myArgs.add("h", "help", desc="Help of this action")
myArgs.add("b", "branch", haveParam=True, desc="Select branch to display")
myArgs.add("m", "manifest", haveParam=True, desc="Name of the manifest")
"""
myArgs.add("j", "jobs", haveParam=True, desc="Specifies the number of jobs (commands) to run simultaneously")
myArgs.add("d", "depth", haveParam=True, desc="Depth to clone all the repository")
@@ -42,12 +80,17 @@ def usage():
# generic argument displayed :
myArgs.display()
print(" Action availlable" )
list_actions = actions.get_list_of_action();
for elem in list_actions:
print(" " + color['green'] + elem + color['default'])
"""
print(" " + color['green'] + "init" + color['default'])
print(" initialize a 'maestro' interface with a manifest in a git ")
print(" initialize a 'island' interface with a manifest in a git ")
print(" " + color['green'] + "sync" + color['default'])
print(" Syncronise the currect environement")
print(" " + color['green'] + "status" + color['default'])
print(" Dump the status of the environement")
"""
print(" ex: " + sys.argv[0] + " -c init http://github.com/atria-soft/manifest.git")
print(" ex: " + sys.argv[0] + " sync")
exit(0)
@@ -91,8 +134,8 @@ def parseGenericArg(argument, active):
return False
"""
# open configuration of maestro:
config_file_name = "maestroConfig.py"
# open configuration of island:
config_file_name = "islandConfig.py"
config_file = os.path.join(tools.get_run_path(), config_file_name)
if os.path.isfile(config_file) == True:
sys.path.append(os.path.dirname(config_file))
@@ -147,7 +190,7 @@ for argument in localArgument:
parseGenericArg(argument, True)
# initialize the system ...
maestro.init()
init()
# remove all generic arguments:
new_argument_list = []
@@ -163,32 +206,26 @@ if len(new_argument_list) == 0:
debug.warning("--------------------------------------")
usage()
list_of_action_availlable=["init","sync","status"]
# TODO : move tin in actions ...
list_actions = actions.get_list_of_action();
action_to_do = new_argument_list[0].get_arg()
new_argument_list = new_argument_list[1:]
if action_to_do not in list_of_action_availlable:
if action_to_do not in list_actions:
debug.warning("--------------------------------------")
debug.warning("Wrong action type : '" + str(action_to_do) + "' availlable list: " + str(list_of_action_availlable) )
debug.warning("Wrong action type : '" + str(action_to_do) + "' availlable list: " + str(list_actions) )
debug.warning("--------------------------------------")
usage()
# todo : Remove this
if action_to_do != "init" \
and os.path.exists("." + env.get_system_base_name()) == False:
debug.error("Can not execute a maestro cmd if we have not initialize a config: '" + str("." + env.get_system_base_name()) + "'")
and os.path.exists(env.get_island_path()) == False:
debug.error("Can not execute a island cmd if we have not initialize a config: '" + str("." + env.get_system_base_name()) + "' in upper 6 parent path")
exit(-1)
if action_to_do == "init":
debug.info("action: init");
elif action_to_do == "sync":
debug.info("action: sync");
elif action_to_do == "status":
debug.info("action: status");
else:
debug.error("Can not do the action...")
actions.execute(action_to_do, new_argument_list)
# stop all started threads;
#multiprocess.un_init()

85
island/actions.py Normal file
View File

@@ -0,0 +1,85 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
# Local import
from . import debug
import os
import sys
from . import env
list_actions = []
__base_action_name = env.get_system_base_name() + "Action_"
def init(files):
global list_actions;
debug.debug("List of action for island: ")
for elem_path in files :
base_name = os.path.basename(elem_path)
if len(base_name) <= 3 + len(__base_action_name):
# reject it, too small
continue
base_name = base_name[:-3]
if base_name[:len(__base_action_name)] != __base_action_name:
# reject it, wrong start file
continue
name_action = base_name[len(__base_action_name):]
debug.debug(" '" + os.path.basename(elem_path)[:-3] + "' file=" + elem_path)
list_actions.append({
"name":name_action,
"path":elem_path,
})
##
## @brief Get the wall list of action availlable
## @return ([string]) the list of action name
##
def get_list_of_action():
global list_actions;
out = []
for elem in list_actions:
out.append(elem["name"])
return out
##
## @brief Get a description of an action
## @param[in] action_name (string) Name of the action
## @return (string/None) A descriptive string or None
##
def get_desc(action_name):
global list_actions;
for elem in list_actions:
if elem["name"] == action_name:
# finish the parsing
sys.path.append(os.path.dirname(elem["path"]))
the_action = __import__(__base_action_name + action_name)
if "get_desc" not in dir(the_action):
debug.error("execute is not implmented for this action ... '" + str(action_name) + "'")
return None
return the_action.get_desc()
return None
def execute(action_name, argument_list):
global list_actions;
# TODO: Move here the check if action is availlable
for elem in list_actions:
if elem["name"] == action_name:
debug.info("action: " + str(elem));
# finish the parsing
sys.path.append(os.path.dirname(elem["path"]))
the_action = __import__(__base_action_name + action_name)
if "execute" not in dir(the_action):
debug.error("execute is not implmented for this action ... '" + str(action_name) + "'")
return False
return the_action.execute(argument_list)
debug.error("Can not do the action...")
return False

View File

@@ -0,0 +1,153 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
from island import debug
from island import tools
from island import env
from island import config
from island import multiprocess
from island import manifest
import os
def help():
return "plop"
def execute(arguments):
debug.info("execute:")
for elem in arguments:
debug.info(" '" + str(elem.get_arg()) + "'")
if len(arguments) != 1:
debug.error("checkout: missing argument to select the new branch ...")
branch_to_checkout = ""
for elem in arguments:
if elem.get_option_name() == "":
if branch_to_checkout != "":
debug.error("checkout branch already set : '" + branch_to_checkout + "' !!! '" + elem.get_arg() + "'")
branch_to_checkout = elem.get_arg()
else:
debug.error("Wrong argument: '" + elem.get_option_name() + "' '" + elem.get_arg() + "'")
# check if .XXX exist (create it if needed)
if os.path.exists(env.get_island_path()) == False \
or os.path.exists(env.get_island_path_config()) == False \
or os.path.exists(env.get_island_path_manifest()) == False:
debug.error("System already init have an error: missing data: '" + str(env.get_island_path()) + "'")
configuration = config.Config()
# update the local configuration file:
configuration.set_branch(branch_to_checkout)
configuration.store()
file_source_manifest = os.path.join(env.get_island_path_manifest(), configuration.get_manifest_name())
if os.path.exists(file_source_manifest) == False:
debug.error("Missing manifest file : '" + str(file_source_manifest) + "'")
mani = manifest.Manifest(file_source_manifest)
all_project = mani.get_all_configs()
debug.info("checkout of: " + str(len(all_project)) + " projects")
id_element = 0
for elem in all_project:
id_element += 1
debug.verbose("checkout : " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name))
git_repo_path = os.path.join(env.get_island_root_path(), elem.path)
if os.path.exists(git_repo_path) == False:
debug.warning("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> repository does not exist ...")
continue
# check if the repository is modify
cmd = "git diff --quiet"
debug.verbose("execute : " + cmd)
ret_diff = multiprocess.run_command(cmd, cwd=git_repo_path)
# get local branch
cmd = "git branch"
debug.verbose("execute : " + cmd)
ret_branch = multiprocess.run_command(cmd, cwd=git_repo_path)
# get tracking branch
cmd = "git rev-parse --abbrev-ref --symbolic-full-name @{u}"
debug.verbose("execute : " + cmd)
ret_track = multiprocess.run_command(cmd, cwd=git_repo_path)
is_modify = True
if ret_diff[0] == 0:
is_modify = False
if is_modify == True:
debug.warning("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> modify data can not checkout new branch")
continue
list_branch = ret_branch[1].split('\n')
list_branch2 = []
select_branch = ""
for elem_branch in list_branch:
list_branch2.append(elem_branch[2:])
if elem_branch[:2] == "* ":
select_branch = elem_branch[2:]
# check if we are on the good branch:
if branch_to_checkout == select_branch:
debug.info("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> No change already on good branch")
continue
# check if we have already checkout the branch before
debug.verbose(" check : " + branch_to_checkout + " in " + str(list_branch2))
if branch_to_checkout in list_branch2:
cmd = "git checkout " + branch_to_checkout
debug.verbose("execute : " + cmd)
ret = multiprocess.run_command(cmd, cwd=git_repo_path)
if ret[0] != 0 \
and ret[1] != "" \
and ret != False:
debug.info("'" + str(ret) + "'")
debug.error("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> Can not checkout to the correct branch")
continue
debug.info("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> switch branch")
# TODO : Check the number of commit to the origin/XXX branch ....
continue
# Check if the remote branch exist ...
cmd = "git branch -a"
debug.verbose("execute : " + cmd)
ret_branch_all = multiprocess.run_command(cmd, cwd=git_repo_path)
list_branch_all = ret_branch_all[1].split('\n')
exist = False
for elem_branch in list_branch_all:
debug.verbose(" check : '" + elem_branch + "' == '" + " remotes/" + elem.select_remote["name"] + "/" + branch_to_checkout + "'")
if elem_branch == " remotes/" + elem.select_remote["name"] + "/" + branch_to_checkout:
exist = True
debug.info(" ==> find ...")
break
if exist == False:
debug.info("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> NO remote branch")
continue
# checkout the new branch:
cmd = "git checkout --quiet " + elem.select_remote["name"] + "/" + branch_to_checkout + " -b " + branch_to_checkout
# + " --track " + elem.select_remote["name"] + "/" + branch_to_checkout
debug.verbose("execute : " + cmd)
ret = multiprocess.run_command(cmd, cwd=git_repo_path)
if ret[1] != "" \
and ret != False:
debug.info("'" + str(ret) + "'")
debug.error("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> Can not checkout to the correct branch")
continue
debug.info("checkout " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + " ==> create new branch")
continue

View File

@@ -0,0 +1,75 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
from island import debug
from island import tools
from island import env
from island import config
from island import multiprocess
from island import manifest
import os
def help():
return "plop"
def execute(arguments):
debug.info("execute:")
for elem in arguments:
debug.info(" '" + str(elem.get_arg()) + "'")
if len(arguments) != 0:
debug.error("Sync have not parameter")
# check if .XXX exist (create it if needed)
if os.path.exists(env.get_island_path()) == False \
or os.path.exists(env.get_island_path_config()) == False \
or os.path.exists(env.get_island_path_manifest()) == False:
debug.error("System already init have an error: missing data: '" + str(env.get_island_path()) + "'")
configuration = config.Config()
debug.info("update manifest : '" + str(env.get_island_path_manifest()) + "'")
# update manifest
cmd = "git fetch --all"
multiprocess.run_command_direct(cmd, cwd=env.get_island_path_manifest())
file_source_manifest = os.path.join(env.get_island_path_manifest(), configuration.get_manifest_name())
if os.path.exists(file_source_manifest) == False:
debug.error("Missing manifest file : '" + str(file_source_manifest) + "'")
mani = manifest.Manifest(file_source_manifest)
all_project = mani.get_all_configs()
debug.info("fetch : " + str(len(all_project)) + " projects")
id_element = 0
for elem in all_project:
id_element += 1
debug.info("fetch : " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name))
#debug.debug("elem : " + str(elem))
git_repo_path = os.path.join(env.get_island_root_path(), elem.path)
if os.path.exists(git_repo_path) == False:
debug.error("can not fetch project that not exist")
continue
if os.path.exists(os.path.join(git_repo_path,".git")) == False:
# path already exist but it is not used to as a git repo ==> this is an error
debug.error("path '" + git_repo_path + "' is already existing but not used for a git repository. Clean it and restart")
# simply update the repository ...
debug.verbose("Fetching project: ")
# fetch the repository
cmd = "git fetch " + elem.select_remote["name"]
debug.verbose("execute : " + cmd)
multiprocess.run_command_direct(cmd, cwd=git_repo_path)

View File

@@ -0,0 +1,83 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
from island import debug
from island import tools
from island import env
from island import config
from island import multiprocess
import os
def help():
return "plop"
def execute(arguments):
debug.info("execute:")
for elem in arguments:
debug.info(" '" + str(elem.get_arg()) + "'")
if len(arguments) == 0:
debug.error("Missing argument to execute the current action ...")
# the configuration availlable:
branch = "master"
manifest_name = "default.xml"
address_manifest = ""
for elem in arguments:
if elem.get_option_name() == "branch":
debug.info("find branch name: '" + elem.get_arg() + "'")
branch = elem.get_arg()
elif elem.get_option_name() == "manifest":
debug.info("find mmanifest name: '" + elem.get_arg() + "'")
manifest_name = elem.get_arg()
elif elem.get_option_name() == "":
if address_manifest != "":
debug.error("Manifest adress already set : '" + address_manifest + "' !!! '" + elem.get_arg() + "'")
address_manifest = elem.get_arg()
else:
debug.error("Wrong argument: '" + elem.get_option_name() + "' '" + elem.get_arg() + "'")
if address_manifest == "":
debug.error("Init: Missing manifest name")
debug.info("Init with: '" + address_manifest + "' branch='" + branch + "' name of manifest='" + manifest_name + "'")
# check if .XXX exist (create it if needed)
if os.path.exists(env.get_island_path()) == True \
and os.path.exists(env.get_island_path_config()) == True \
and os.path.exists(env.get_island_path_manifest()) == True:
debug.error("System already init: path already exist: '" + str(env.get_island_path()) + "'")
tools.create_directory(env.get_island_path())
# check if the git of the manifest if availlable
# create the file configuration:
conf = config.Config()
conf.set_manifest(address_manifest)
conf.set_branch(branch)
conf.set_manifest_name(manifest_name)
conf.store()
#clone the manifest repository
cmd = "git clone " + address_manifest + " --branch " + branch + " " + env.get_island_path_manifest()
debug.info("clone the manifest")
ret = multiprocess.run_command_direct(cmd)
if ret == "":
return True
if ret == False:
# all is good, ready to get the system work corectly
return True
debug.info("'" + ret + "'")
debug.error("Init does not work")
return False

View File

@@ -0,0 +1,125 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
from island import debug
from island import tools
from island import env
from island import multiprocess
from island import config
from island import manifest
import os
def help():
return "plop"
def execute(arguments):
debug.info("execute:")
for elem in arguments:
debug.info(" '" + str(elem.get_arg()) + "'")
if len(arguments) != 0:
debug.error("status have not parameter")
# check if .XXX exist (create it if needed)
if os.path.exists(env.get_island_path()) == False \
or os.path.exists(env.get_island_path_config()) == False \
or os.path.exists(env.get_island_path_manifest()) == False:
debug.error("System already init have an error: missing data: '" + str(env.get_island_path()) + "'")
configuration = config.Config()
file_source_manifest = os.path.join(env.get_island_path_manifest(), configuration.get_manifest_name())
if os.path.exists(file_source_manifest) == False:
debug.error("Missing manifest file : '" + str(file_source_manifest) + "'")
mani = manifest.Manifest(file_source_manifest)
all_project = mani.get_all_configs()
debug.info("status of: " + str(len(all_project)) + " projects")
id_element = 0
for elem in all_project:
id_element += 1
debug.verbose("status : " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name))
#debug.debug("elem : " + str(elem))
git_repo_path = os.path.join(env.get_island_root_path(), elem.path)
if os.path.exists(git_repo_path) == False:
debug.info("" + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + "\r\t\t\t\t\t\t\t\t\t" + " (not download)")
continue
# check if the repository is modify
cmd = "git diff --quiet"
debug.verbose("execute : " + cmd)
ret_diff = multiprocess.run_command(cmd, cwd=git_repo_path)
# get local branch
cmd = "git branch"
debug.verbose("execute : " + cmd)
ret_branch = multiprocess.run_command(cmd, cwd=git_repo_path)
# get tracking branch
cmd = "git rev-parse --abbrev-ref --symbolic-full-name @{u}"
debug.verbose("execute : " + cmd)
ret_track = multiprocess.run_command(cmd, cwd=git_repo_path)
is_modify = True
if ret_diff[0] == 0:
is_modify = False
list_branch = ret_branch[1].split('\n')
list_branch2 = []
select_branch = ""
for elem_branch in list_branch:
if elem_branch[:2] == "* ":
list_branch2.append([elem_branch[2:], True])
select_branch = elem_branch[2:]
else:
list_branch2.append([elem_branch[2:], False])
modify_status = " "
if is_modify == True:
modify_status = " *** "
debug.verbose("select branch = '" + select_branch + "' is modify : " + str(is_modify) + " track: '" + str(ret_track[1]) + "'")
cmd = "git rev-list " + select_branch
debug.verbose("execute : " + cmd)
ret_current_branch_sha1 = multiprocess.run_command(cmd, cwd=git_repo_path)[1].split('\n')
cmd = "git rev-list " + ret_track[1]
debug.verbose("execute : " + cmd)
ret_track_branch_sha1 = multiprocess.run_command(cmd, cwd=git_repo_path)[1].split('\n')
# remove all identical sha1 ==> not needed for this
in_forward = 0
for elem_sha1 in ret_current_branch_sha1:
if elem_sha1 not in ret_track_branch_sha1:
in_forward += 1
in_behind = 0
for elem_sha1 in ret_track_branch_sha1:
if elem_sha1 not in ret_current_branch_sha1:
in_behind += 1
behind_forward_comment = ""
if in_forward != 0:
behind_forward_comment += "forward=" + str(in_forward)
if in_behind != 0:
behind_forward_comment += "behind=" + str(in_behind)
if behind_forward_comment != "":
behind_forward_comment = "\r\t\t\t\t\t\t\t\t\t\t\t\t[" + behind_forward_comment + "]"
#debug.info("" + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + "\r\t\t\t\t\t\t\t" + modify_status + "(" + select_branch + " -> " + ret_track[1] + " -> " + elem.select_remote["name"] + "/" + elem.branch + ")")
debug.info("" + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name) + "\r\t\t\t\t\t\t\t" + modify_status + "(" + select_branch + " -> " + ret_track[1] + ")" + behind_forward_comment)
if is_modify == True:
cmd = "git status --short"
debug.verbose("execute : " + cmd)
ret_diff = multiprocess.run_command(cmd, cwd=git_repo_path)
tmp_color_red = "\033[31m"
tmp_color_default= "\033[00m"
debug.info(tmp_color_red + ret_diff[1] + tmp_color_default)

View File

@@ -0,0 +1,213 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
from island import debug
from island import tools
from island import env
from island import config
from island import multiprocess
from island import manifest
import os
def help():
return "plop"
def execute(arguments):
debug.info("execute:")
for elem in arguments:
debug.info(" '" + str(elem.get_arg()) + "'")
if len(arguments) != 0:
debug.error("Sync have not parameter")
# check if .XXX exist (create it if needed)
if os.path.exists(env.get_island_path()) == False \
or os.path.exists(env.get_island_path_config()) == False \
or os.path.exists(env.get_island_path_manifest()) == False:
debug.error("System already init have an error: missing data: '" + str(env.get_island_path()) + "'")
configuration = config.Config()
debug.info("update manifest : '" + str(env.get_island_path_manifest()) + "'")
# update manifest
cmd = "git fetch --all"
multiprocess.run_command_direct(cmd, cwd=env.get_island_path_manifest())
file_source_manifest = os.path.join(env.get_island_path_manifest(), configuration.get_manifest_name())
if os.path.exists(file_source_manifest) == False:
debug.error("Missing manifest file : '" + str(file_source_manifest) + "'")
mani = manifest.Manifest(file_source_manifest)
all_project = mani.get_all_configs()
debug.info("synchronize : " + str(len(all_project)) + " projects")
id_element = 0
for elem in all_project:
id_element += 1
debug.info("sync : " + str(id_element) + "/" + str(len(all_project)) + " : " + str(elem.name))
#debug.debug("elem : " + str(elem))
git_repo_path = os.path.join(env.get_island_root_path(), elem.path)
if os.path.exists(git_repo_path) == False:
# this is a new clone ==> this is easy ...
#clone the manifest repository
address_manifest = ""
cmd = "git clone " + elem.select_remote["fetch"] + "/" + elem.name + " --branch " + elem.branch + " --origin " + elem.select_remote["name"] + " " + git_repo_path
debug.info("clone the repo")
ret = multiprocess.run_command_direct(cmd)
if ret != "" \
and ret != False:
# all is good, ready to get the system work corectly
debug.info("'" + str(ret) + "'")
debug.error("Clone repository does not work ... ")
continue
# add global mirror list
for mirror in elem.select_remote["mirror"]:
debug.verbose("Add global mirror: " + str(mirror))
cmd = "git remote add " + mirror["name"] + " " + mirror["fetch"] + "/" + elem.name
ret = multiprocess.run_command_direct(cmd, cwd=git_repo_path)
if ret != "" \
and ret != False:
# all is good, ready to get the system work corectly
debug.info("'" + str(ret) + "'")
debug.warning("Can not add global mirror ... ")
continue
debug.verbose("Add global mirror: " + str(mirror) + " (done)")
#debug.info("plop " + str(elem.select_remote.keys()))
# check submodule if requested:
if elem.select_remote["sync"] == True \
and os.path.exists(os.path.join(git_repo_path, ".gitmodules")) == True:
debug.info(" ==> update submodule")
cmd = "git submodule init"
ret = multiprocess.run_command_direct(cmd, cwd=git_repo_path)
if ret != "" \
and ret != False:
# all is good, ready to get the system work corectly
debug.info("'" + str(ret) + "'")
debug.error("Can not init submodules ... ")
continue
cmd = "git submodule update"
ret = multiprocess.run_command_direct(cmd, cwd=git_repo_path)
if ret[:16] == "Submodule path '":
#all is good ...
debug.info(" " + ret)
elif ret != "" \
and ret != False:
# all is good, ready to get the system work corectly
debug.info("'" + str(ret) + "'")
debug.error("Can not init submodules ... ")
continue
continue
if os.path.exists(os.path.join(git_repo_path,".git")) == False:
# path already exist but it is not used to as a git repo ==> this is an error
debug.error("path '" + git_repo_path + "' is already existing but not used for a git repository. Clean it and restart")
# simply update the repository ...
debug.verbose("Fetching project: ")
# fetch the repository
cmd = "git fetch " + elem.select_remote["name"]
debug.verbose("execute : " + cmd)
multiprocess.run_command_direct(cmd, cwd=git_repo_path)
# check if the repository is modify
cmd = "git diff --quiet"
debug.verbose("execute : " + cmd)
ret_diff = multiprocess.run_command(cmd, cwd=git_repo_path)
# get local branch
cmd = "git branch"
debug.verbose("execute : " + cmd)
ret_branch = multiprocess.run_command(cmd, cwd=git_repo_path)
# get tracking branch
cmd = "git rev-parse --abbrev-ref --symbolic-full-name @{u}"
debug.verbose("execute : " + cmd)
ret_track = multiprocess.run_command(cmd, cwd=git_repo_path)
is_modify = True
if ret_diff[0] == 0:
is_modify = False
list_branch = ret_branch[1].split('\n')
list_branch2 = []
select_branch = ""
for elem_branch in list_branch:
if elem_branch[:2] == "* ":
list_branch2.append([elem_branch[2:], True])
select_branch = elem_branch[2:]
else:
list_branch2.append([elem_branch[2:], False])
if is_modify == True:
debug.warning("[" + elem.name + "] Not update ==> the repository is modified")
continue
""" # TODO: this does not work ...
if ret_track[1] != elem.select_remote["name"] + "/" + elem.branch:
debug.warning("[" + elem.name + "] Not update ==> the current branch does not track the correct branch : track '" + ret_track[1] + "' instead of '" + elem.select_remote["name"] + "/" + elem.branch + "'")
continue
"""
cmd = "git pull"
debug.verbose("execute : " + cmd)
ret_pull = multiprocess.run_command(cmd, cwd=git_repo_path)
if ret_pull[0] == 0:
if ret_pull[1] == "Already up-to-date.":
pass
elif ret_pull[1] != "":
debug.info(ret_pull[1])
else:
if ret_pull[1] != "":
debug.warning("ERROR GIT: " + ret_pull[1])
else:
debug.warning("ERROR GIT: in pull")
debug.verbose("select branch = '" + select_branch + "' is modify : " + str(is_modify) + " track: '" + str(ret_track[1]) + "'")
# check submodule if requested:
if elem.select_remote["sync"] == True \
and os.path.exists(os.path.join(git_repo_path, ".gitmodules")) == True:
debug.info(" ==> sync submodule")
cmd = "git submodule sync"
ret = multiprocess.run_command_direct(cmd, cwd=git_repo_path)
if ret[:31] == "Synchronizing submodule url for":
#all is good ...
debug.info(" " + ret)
elif ret != "" \
and ret != False:
# all is good, ready to get the system work corectly
debug.info("'" + ret + "'")
debug.error("Can not sync submodules ... ")
continue
"""
cmd = "git submodule init"
ret = multiprocess.run_command_direct(cmd, cwd=git_repo_path)
if ret != "" \
and ret != False:
# all is good, ready to get the system work corectly
debug.info("'" + ret + "'")
debug.error("Can not init submodules ... ")
continue
cmd = "git submodule update"
ret = multiprocess.run_command_direct(cmd, cwd=git_repo_path)
if ret[:16] == "Submodule path '":
#all is good ...
debug.info(" " + ret)
elif ret != "" \
and ret != False:
# all is good, ready to get the system work corectly
debug.info("'" + ret + "'")
debug.error("Can not init submodules ... ")
continue
"""

View File

@@ -218,7 +218,7 @@ class ArgSection:
##
## @brief Class to define the agmument list availlable for a program
##
class maestroArg:
class islandArg:
##
## @brief Constructor.
## @param[in] self Class handle

72
island/config.py Normal file
View File

@@ -0,0 +1,72 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import platform
import sys
import os
import copy
# Local import
from . import debug
from . import tools
from . import env
from . import multiprocess
env.get_island_path_config()
class Config():
def __init__(self):
self._repo = ""
self._branch = "master"
self._manifest_name = "default.xml"
self.load()
def load(self):
config_property = tools.file_read_data(env.get_island_path_config())
element_config = config_property.split("\n")
for line in element_config:
if len(line) == 0 \
or line[0] == "#":
# simple comment line ==> pass
pass
elif line[:5] == "repo=":
self._repo = line[5:]
elif line[:7] == "branch=":
self._branch = line[7:]
elif line[:5] == "file=":
self._manifest_name = line[5:]
else:
debug.warning("island config error: can not parse: '" + str(line) + "'")
return True
def store(self):
data = "repo=" + self._repo + "\nbranch=" + self._branch + "\nfile=" + self._manifest_name
tools.file_write_data(env.get_island_path_config(), data)
def set_manifest(self, value):
self._repo = value
def get_manifest(self):
return self._repo
def set_branch(self, value):
self._branch = value
def get_branch(self):
return self._branch
def set_manifest_name(self, value):
self._manifest_name = value
def get_manifest_name(self):
return self._manifest_name

73
island/env.py Normal file
View File

@@ -0,0 +1,73 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
# Local import
from . import debug
import os
system_base_name = "island"
def set_system_base_name(val):
global system_base_name
system_base_name = val
debug.debug("Set basename: '" + str(system_base_name) + "'")
def get_system_base_name():
global system_base_name
return system_base_name
island_root_path = os.path.join(os.getcwd())
if os.path.exists(os.path.join(island_root_path, "." + get_system_base_name())) == True:
# all is good ...
pass
elif os.path.exists(os.path.join(island_root_path, "..", "." + get_system_base_name())) == True:
island_root_path = os.path.join(os.getcwd(), "..")
elif os.path.exists(os.path.join(island_root_path, "..", "..", "." + get_system_base_name())) == True:
island_root_path = os.path.join(os.getcwd(), "..", "..")
elif os.path.exists(os.path.join(island_root_path, "..", "..", "..", "." + get_system_base_name())) == True:
island_root_path = os.path.join(os.getcwd(), "..", "..", "..")
elif os.path.exists(os.path.join(island_root_path, "..", "..", "..", "." + get_system_base_name())) == True:
island_root_path = os.path.join(os.getcwd(), "..", "..", "..")
elif os.path.exists(os.path.join(island_root_path, "..", "..", "..", "..", "." + get_system_base_name())) == True:
island_root_path = os.path.join(os.getcwd(), "..", "..", "..", "..")
elif os.path.exists(os.path.join(island_root_path, "..", "..", "..", "..", "..", "." + get_system_base_name())) == True:
island_root_path = os.path.join(os.getcwd(), "..", "..", "..", "..", "..")
else:
#debug.error("the root path of " + get_system_base_name() + " must not be upper that 6 parent path")
pass
island_path = os.path.join(island_root_path, "." + get_system_base_name())
island_path_config = os.path.join(island_path, "config.txt")
island_path_manifest = os.path.join(island_path, "manifest")
##
## @brief to use later to know where the ".island" parent path is ...
## @return the parent path of the ".island"
##
def get_island_root_path():
global island_root_path
return island_root_path
def get_island_path():
global island_path
return island_path
def get_island_path_config():
global island_path_config
return island_path_config
def get_island_path_manifest():
global island_path_manifest
return island_path_manifest

View File

@@ -15,23 +15,12 @@ from . import debug
# print os.name # ==> 'posix'
if platform.system() == "Linux":
OS = "Linux"
HOST_DEFAULT_COMPILATOR = "gcc"
elif platform.system() == "Windows":
OS = "Windows"
HOST_DEFAULT_COMPILATOR = "gcc"
elif platform.system() == "Darwin":
OS = "MacOs"
HOST_DEFAULT_COMPILATOR = "clang"
else:
debug.error("Unknow the Host OS ... '" + platform.system() + "'")
debug.debug("host.OS = " + OS)
if sys.maxsize > 2**32:
BUS_SIZE = 64
else:
BUS_SIZE = 32
debug.debug("host.BUS_SIZE = " + str(BUS_SIZE))

289
island/manifest.py Normal file
View File

@@ -0,0 +1,289 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import platform
import sys
import os
import copy
# Local import
from . import debug
from . import tools
from . import env
from . import multiprocess
from lxml import etree
class RepoConfig():
def __init__(self):
self.name = ""
self.path = ""
self.remotes = [] # list of all remotes, with the upstream elements (needed for third party integrations)
self.select_remote = ""
self.branch = ""
class Manifest():
def __init__(self, manifest_xml):
self.manifest_xml = manifest_xml
self.projects = []
self.default = None
self.default_base = {
"remote":"origin",
"revision":"master",
"sync":False,
}
self.remotes = []
self.includes = []
# load the manifest
self._load()
# check error in manifest (double path ...)
self._check_double_path([])
def _load(self):
tree = etree.parse(self.manifest_xml)
debug.debug("manifest : '" + self.manifest_xml + "'")
root = tree.getroot()
if root.tag != "manifest":
debug.error("(l:" + str(child.sourceline) + ") in '" + str(file) + "' have not main xml node='manifest'")
for child in root:
if type(child) == etree._Comment:
debug.verbose("(l:" + str(child.sourceline) + ") comment='" + str(child.text) + "'");
continue
if child.tag == "remote":
name = "origin"
fetch = ""
for attr in child.attrib:
if attr == "name":
name = child.attrib[attr]
elif attr == "fetch":
fetch = child.attrib[attr]
if len(fetch) >= 2 \
and fetch[:2] == "..":
# we have a relative island manifest ==> use local manifest origin to get the full origin
cmd = "git remote get-url origin"
debug.verbose("execute : " + cmd)
base_origin = multiprocess.run_command(cmd, cwd=env.get_island_path_manifest())
debug.verbose("base_origin=" + base_origin[1])
base_origin = base_origin[1]
while len(fetch) >= 2 \
and fetch[:2] == "..":
fetch = fetch[2:]
while len(fetch) >= 1 \
and ( fetch[0] == "/" \
or fetch[0] == "\\"):
fetch = fetch[1:]
base_origin = base_origin[:base_origin.rfind('/')]
debug.verbose("new base_origin=" + base_origin)
debug.verbose("tmp fetch=" + fetch)
if fetch != "":
fetch = base_origin + "/" + fetch
else:
fetch = base_origin
debug.verbose("new fetch=" + fetch)
while len(fetch) > 1 \
and ( fetch[-1] == "\\" \
or fetch[-1] == "/") :
fetch = fetch[:-1]
else:
debug.error("(l:" + str(child.sourceline) + ") Parsing the manifest : Unknow '" + child.tag + "' attibute : '" + attr + "', availlable:[name,fetch]")
debug.debug("(l:" + str(child.sourceline) + ") find '" + child.tag + "' : name='" + name + "' fetch='" + fetch + "'");
# parse the sub global mirror list
mirror_list = []
for child_2 in child:
if child_2.tag == "mirror":
# find a new mirror
mirror_name = ""
mirror_fetch = ""
for attr_2 in child_2.attrib:
if attr_2 == "name":
mirror_name = child_2.attrib[attr_2]
elif attr_2 == "fetch":
mirror_fetch = child_2.attrib[attr_2]
while len(mirror_fetch) > 1 \
and ( mirror_fetch[-1] == "\\" \
or mirror_fetch[-1] == "/") :
mirror_fetch = mirror_fetch[:-1]
else:
debug.error("(l:" + str(child_2.sourceline) + ") Parsing the manifest : Unknow '" + child_2.tag + "' attibute : '" + attr_2 + "', availlable:[name,fetch]")
debug.debug("mirror: '" + mirror_name + "' '" + mirror_fetch + "'")
if mirror_name == "":
debug.error("(l:" + str(child_2.sourceline) + ") Missing mirrot 'name'")
if mirror_fetch == "":
debug.error("(l:" + str(child_2.sourceline) + ") Missing mirror 'fetch'")
mirror_list.append({
"name":mirror_name,
"fetch":mirror_fetch,
})
else:
debug.error("(l:" + str(child_2.sourceline) + ") Parsing the manifest : Unknow '" + child_2.tag + "', availlable:[mirror]")
self.remotes.append({
"name":name,
"fetch":fetch,
"mirror":mirror_list
})
continue
if child.tag == "include":
name = ""
for attr in child.attrib:
if attr == "name":
name = child.attrib[attr]
else:
debug.error("(l:" + str(child.sourceline) + ") Parsing the manifest : Unknow '" + child.tag + "' attibute : '" + attr + "', availlable:[name]")
debug.debug("(l:" + str(child.sourceline) + ") find '" + child.tag + "' : name='" + name + "'");
# check if the file exist ...
new_name_xml = os.path.join(os.path.dirname(self.manifest_xml),name)
if os.path.exists(new_name_xml) == False:
debug.error("(l:" + str(child.sourceline) + ") The file does not exist : '" + new_name_xml + "'")
self.includes.append({
"name":name,
"path":new_name_xml,
"manifest":None
})
continue
if child.tag == "default":
remote = "origin"
revision = "master"
sync = False
for attr in child.attrib:
if attr == "remote":
remote = child.attrib[attr]
elif attr == "revision":
revision = child.attrib[attr]
elif attr == "sync-s": # synchronize submodule ... automaticaly
sync = child.attrib[attr]
if sync.lower() == "true" \
or sync == "1" \
or sync.lower() == "yes":
sync = True
elif sync.lower() == "false" \
or sync == "0" \
or sync.lower() == "no":
sync = False
else:
debug.error("(l:" + str(child.sourceline) + ") Parsing the manifest : Unknow '" + child.tag + "' attbute : '" + attr + "', value:'" + sync + "' availlable:[true,1,yes,false,0,no]")
else:
debug.error("(l:" + str(child.sourceline) + ") Parsing the manifest : Unknow '" + child.tag + "' attibute : '" + attr + "', availlable:[remote,revision,sync-s]")
if self.default != None:
debug.error("(l:" + str(child.sourceline) + ") Parsing the manifest : Node '" + child.tag + "' already set")
self.default = {
"remote":remote,
"revision":revision,
"sync":sync,
}
debug.debug("(l:" + str(child.sourceline) + ") find '" + child.tag + "' : remote='" + remote + "' revision='" + revision + "' sync=" + str(sync));
continue
if child.tag == "project":
name = ""
path = ""
for attr in child.attrib:
if attr == "name":
name = child.attrib[attr]
elif attr == "path":
path = child.attrib[attr]
else:
debug.error("(l:" + str(child.sourceline) + ") Parsing the manifest: Unknow '" + child.tag + "' attibute : '" + attr + "', availlable:[name,revision,sync-s]")
if name == "":
debug.error("(l:" + str(child.sourceline) + ") Parsing the manifest: '" + child.tag + "' missing attribute: 'name' ==> specify the git to clone ...")
self.projects.append({
"name":name,
"path":path,
})
debug.debug("(l:" + str(child.sourceline) + ") find '" + child.tag + "' : name='" + name + "' path='" + path + "'");
continue
debug.info("(l:" + str(child.sourceline) + ") '" + str(child.tag) + "' values=" + str(child.attrib));
debug.error("(l:" + str(child.sourceline) + ") Parsing error Unknow NODE : '" + str(child.tag) + "' availlable:[remote,include,default,project]")
# now we parse all sub repo:
for elem in self.includes:
elem["manifest"] = Manifest(elem["path"])
# inside data child.text
def _create_path_with_elem(self, element):
path = element["path"]
if path == "":
path = element["name"]
if len(path) >= 4 \
and path[-4:] == ".git":
path = path[:-4]
return path
def _check_double_path(self, list_path = [], space=""):
debug.debug(space + "check path : '" + self.manifest_xml + "'")
for elem in self.projects:
path = self._create_path_with_elem(elem)
debug.debug(space + " check path:'" + str(path) + "'")
if path in list_path:
debug.error("Check Manifest error : double use of the path '" + str(path) + "'")
list_path.append(path)
for elem in self.includes:
elem["manifest"]._check_double_path(list_path, space + " ")
def get_all_configs(self, default=None, upper_remotes=[]):
out = []
if default == None:
if self.default != None:
default = copy.deepcopy(self.default)
else:
default = copy.deepcopy(self.default_base)
# debug.error(" self.default=" + str(self.default))
# add all local project
for elem in self.projects:
debug.verbose("parse element " + str(elem))
conf = RepoConfig()
conf.name = elem["name"]
conf.path = self._create_path_with_elem(elem)
# add default remote for the project (search in herited element)
for remote in self.remotes:
debug.verbose(" Local Remote: " + str(remote))
if remote["name"] == default["remote"]:
conf.remotes.append(remote)
if len(conf.remotes) == 0:
for remote in upper_remotes:
debug.verbose(" upper Remote: " + str(remote))
if remote["name"] == default["remote"]:
conf.remotes.append(remote)
if len(conf.remotes) == 0:
debug.error(" No remote detected: " + str(len(conf.remotes)) + " for " + conf.name + " with default remote name : " + default["remote"] + " self remote: " + str(self.remotes))
# select default remote:
conf.select_remote = None
debug.debug(" remotes count: " + str(len(conf.remotes)))
for remote in conf.remotes:
debug.debug(" remote=" + str(remote))
debug.debug(" Ckeck remote : " + remote["name"] + " == " + default["remote"])
debug.verbose(" remote=" + str(remote))
debug.verbose(" default=" + str(default))
if remote["name"] == default["remote"]:
conf.select_remote = copy.deepcopy(remote)
debug.debug(" copy select=" + str(conf.select_remote))
# copy the submodule synchronisation
conf.select_remote["sync"] = default["sync"]
break
if conf.select_remote == None:
debug.error("missing remote for project: " + str(conf.name))
conf.branch = default["revision"]
out.append(conf)
# create a temporary variable to transmit the remote to includes
upper_remotes_forward = copy.deepcopy(upper_remotes)
for remote in self.remotes:
upper_remotes_forward.append(remote)
# add all include project
for elem in self.includes:
list_project = elem["manifest"].get_all_configs(default, upper_remotes_forward)
for elem_proj in list_project:
out.append(elem_proj)
return out

91
island/multiprocess.py Normal file
View File

@@ -0,0 +1,91 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import sys
import threading
import time
import sys
import os
import subprocess
import shlex
# Local import
from . import debug
from . import tools
from . import env
def run_command_direct_shell(cmd_line, cwd=None, shell=False):
# prepare command line:
args = shlex.split(cmd_line)
debug.verbose("cmd = " + str(args))
subprocess.check_call(args, shell=shell)
return ""
##
## @brief Execute the command and ruturn generate data
##
def run_command_direct(cmd_line, cwd=None):
# prepare command line:
args = shlex.split(cmd_line)
debug.verbose("cmd = " + str(args))
"""
if True:
subprocess.check_call(args)
return ""
"""
try:
# create the subprocess
#p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#p = subprocess.check_call(args)
"""
if cwd != None:
debug.info("path = " + cwd)
"""
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
except subprocess.CalledProcessError as e:
debug.error("subprocess.CalledProcessError : " + str(args))
except:
debug.error("Exception on : " + str(args))
# launch the subprocess:
output, err = p.communicate()
if sys.version_info >= (3, 0):
output = output.decode("utf-8")
err = err.decode("utf-8")
# Check error :
if p.returncode == 0:
if output == None:
return err[:-1];
return output[:-1];
else:
return False
def run_command(cmd_line, cwd=None):
# prepare command line:
args = shlex.split(cmd_line)
debug.verbose("cmd = " + str(args))
try:
# create the subprocess
"""
if cwd != None:
debug.info("path = " + cwd)
"""
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
except subprocess.CalledProcessError as e:
debug.error("subprocess.CalledProcessError : " + str(args))
except:
debug.error("Exception on : " + str(args))
# launch the subprocess:
output, err = p.communicate()
if sys.version_info >= (3, 0):
output = output.decode("utf-8")
err = err.decode("utf-8")
# Check error :
return [p.returncode, output[:-1], err[:-1]]

View File

@@ -29,13 +29,18 @@ def get_run_path():
def get_current_path(file):
return os.path.dirname(os.path.realpath(file))
def create_directory_of_file(file):
path = os.path.dirname(file)
def create_directory(path):
try:
os.stat(path)
except:
os.makedirs(path)
def create_directory_of_file(file):
path = os.path.dirname(file)
create_directory(path)
def get_list_sub_path(path):
# TODO : os.listdir(path)
for dirname, dirnames, filenames in os.walk(path):

View File

@@ -1,154 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import os
import sys
import fnmatch
# Local import
from . import host
from . import tools
from . import debug
from . import env
is_init = False
"""
def filter_name_and_file(root, list_files, filter):
# filter elements:
tmp_list = fnmatch.filter(list_files, filter)
out = []
for elem in tmp_list:
if os.path.isfile(os.path.join(root, elem)) == True:
out.append(elem);
return out;
def filter_path(root, list_files):
out = []
for elem in list_files:
if len(elem) == 0 \
or elem[0] == '.':
continue
if os.path.isdir(os.path.join(root, elem)) == True:
out.append(elem);
return out;
def import_path_local(path, limit_sub_folder, exclude_path = [], base_name = ""):
out = []
debug.verbose("maestro files: " + str(path) + " [START]")
if limit_sub_folder == 0:
debug.debug("Subparsing limitation append ...")
return []
try:
list_files = os.listdir(path)
except:
# an error occure, maybe read error ...
debug.warning("error when getting subdirectory of '" + str(path) + "'")
return []
if path in exclude_path:
debug.debug("find '" + str(path) + "' in exclude_path=" + str(exclude_path))
return []
# filter elements:
tmp_list_maestro_file = filter_name_and_file(path, list_files, base_name + "*.py")
debug.verbose("maestro files: " + str(path) + " : " + str(tmp_list_maestro_file))
# Import the module:
for filename in tmp_list_maestro_file:
out.append(os.path.join(path, filename))
debug.extreme_verbose(" Find a file : '" + str(out[-1]) + "'")
need_parse_sub_folder = True
rm_value = -1
# check if we need to parse sub_folder
if len(tmp_list_maestro_file) != 0:
need_parse_sub_folder = False
# check if the file "maestro_parse_sub.py" is present ==> parse SubFolder (force and add +1 in the resursing
if base_name + "ParseSubFolders.txt" in list_files:
debug.debug("find SubParser ... " + str(base_name + "ParseSubFolders.txt") + " " + path)
data_file_sub = tools.file_read_data(os.path.join(path, base_name + "ParseSubFolders.txt"))
if data_file_sub == "":
debug.debug(" Empty file Load all subfolder in the worktree in '" + str(path) + "'")
need_parse_sub_folder = True
rm_value = 0
else:
list_sub = data_file_sub.split("\n")
debug.debug(" Parse selected folders " + str(list_sub) + " no parse local folder directory")
need_parse_sub_folder = False
for folder in list_sub:
if folder == "" \
or folder == "/":
continue;
tmp_out = import_path_local(os.path.join(path, folder),
1,
exclude_path,
base_name)
# add all the elements:
for elem in tmp_out:
out.append(elem)
if need_parse_sub_folder == True:
list_folders = filter_path(path, list_files)
for folder in list_folders:
tmp_out = import_path_local(os.path.join(path, folder),
limit_sub_folder - rm_value,
exclude_path,
base_name)
# add all the elements:
for elem in tmp_out:
out.append(elem)
return out
"""
def init():
global is_init;
if is_init == True:
return
"""
debug.verbose("Use Make as a make stadard")
sys.path.append(tools.get_run_path())
# create the list of basic folder:
basic_folder_list = []
basic_folder_list.append([tools.get_current_path(__file__), True])
# Import all sub path without out and archive
for elem_path in os.listdir("."):
if os.path.isdir(elem_path) == False:
continue
if elem_path.lower() == "android" \
or elem_path == "out" :
continue
debug.debug("Automatic load path: '" + elem_path + "'")
basic_folder_list.append([elem_path, False])
# create in a single path the basic list of maestro files (all start with maestro and end with .py)
exclude_path = env.get_exclude_search_path()
limit_sub_folder = env.get_parse_depth()
list_of_maestro_files = []
for elem_path, is_system in basic_folder_list:
if is_system == True:
limit_sub_folder_tmp = 999999
else:
limit_sub_folder_tmp = limit_sub_folder
tmp_out = import_path_local(elem_path,
limit_sub_folder_tmp,
exclude_path,
env.get_build_system_base_name())
# add all the elements:
for elem in tmp_out:
list_of_maestro_files.append(elem)
debug.debug("Files specific maestro: ")
for elem_path in list_of_maestro_files:
debug.debug(" " + elem_path)
# simply import element from the basic list of files (single parse ...)
builder.import_path(list_of_maestro_files)
module.import_path(list_of_maestro_files)
system.import_path(list_of_maestro_files)
target.import_path(list_of_maestro_files)
macro.import_path(list_of_maestro_files)
builder.init()
"""
is_init = True

View File

@@ -1,88 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
# Local import
from . import debug
force_mode=False
def set_force_mode(val):
global force_mode
if val==1:
force_mode = 1
else:
force_mode = 0
def get_force_mode():
global force_mode
return force_mode
force_optimisation=False
def set_force_optimisation(val):
global force_optimisation
if val==1:
force_optimisation = 1
else:
force_optimisation = 0
def get_force_optimisation():
global force_optimisation
return force_optimisation
isolate_system=False
def set_isolate_system(val):
global isolate_system
if val==1:
isolate_system = 1
else:
isolate_system = 0
def get_isolate_system():
global isolate_system
return isolate_system
parse_depth = 9999999
def set_parse_depth(val):
global parse_depth
parse_depth = val
debug.debug("Set depth search element: " + str(parse_depth))
def get_parse_depth():
global parse_depth
return parse_depth
exclude_search_path = []
def set_exclude_search_path(val):
global exclude_search_path
exclude_search_path = val
debug.debug("Set depth search element: " + str(exclude_search_path))
def get_exclude_search_path():
global exclude_search_path
return exclude_search_path
system_base_name = "maestro"
def set_system_base_name(val):
global system_base_name
system_base_name = val
debug.debug("Set basename: '" + str(system_base_name) + "'")
def get_system_base_name():
global system_base_name
return system_base_name

View File

@@ -1,303 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import sys
import threading
import time
import sys
if sys.version_info >= (3, 0):
import queue
else:
import Queue as queue
import os
import subprocess
import shlex
# Local import
from . import debug
from . import tools
from . import env
from . import depend
queue_lock = threading.Lock()
work_queue = queue.Queue()
current_thread_working = 0
threads = []
# To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation
current_id_execution = 0
error_execution = {
"id":-1,
"cmd":"",
"return":0,
"err":"",
"out":"",
}
exit_flag = False # resuest stop of the thread
is_init = False # the thread are initialized
error_occured = False # a thread have an error
processor_availlable = 1 # number of CPU core availlable
##
## @brief Execute the command with no get of output
##
def run_command_no_lock_out(cmd_line):
# prepare command line:
args = shlex.split(cmd_line)
debug.info("cmd = " + str(args))
try:
# create the subprocess
p = subprocess.Popen(args)
except subprocess.CalledProcessError as e:
debug.error("subprocess.CalledProcessError : " + str(args))
return
#except:
# debug.error("Exception on : " + str(args))
# launch the subprocess:
p.communicate()
##
## @brief Execute the command and ruturn generate data
##
def run_command_direct(cmd_line):
# prepare command line:
args = shlex.split(cmd_line)
debug.verbose("cmd = " + str(args))
try:
# create the subprocess
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
debug.error("subprocess.CalledProcessError : " + str(args))
except:
debug.error("Exception on : " + str(args))
# launch the subprocess:
output, err = p.communicate()
if sys.version_info >= (3, 0):
output = output.decode("utf-8")
err = err.decode("utf-8")
# Check error :
if p.returncode == 0:
if output == None:
return err[:-1];
return output[:-1];
else:
return False
def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_file="", depend_data=None):
global error_occured
global exit_flag
global current_id_execution
global error_execution
# prepare command line:
args = shlex.split(cmd_line)
debug.verbose("cmd = " + str(args))
try:
# create the subprocess
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
debug.error("subprocess.CalledProcessError : TODO ...")
except:
debug.error("Exception on : " + str(args))
# launch the subprocess:
output, err = p.communicate()
if sys.version_info >= (3, 0):
output = output.decode("utf-8")
err = err.decode("utf-8")
# store error if needed:
tools.store_warning(store_output_file, output, err)
# Check error :
if p.returncode == 0:
debug.debug(env.print_pretty(cmd_line))
queue_lock.acquire()
if depend_data != None:
depend.create_dependency_file(depend_data['file'], depend_data['data'])
# TODO : Print the output all the time .... ==> to show warnings ...
if build_id >= 0 and (output != "" or err != ""):
debug.warning("output in subprocess compiling: '" + file + "'")
if output != "":
debug.print_compilator(output)
if err != "":
debug.print_compilator(err)
queue_lock.release()
else:
error_occured = True
exit_flag = True
# if No ID : Not in a multiprocess mode ==> just stop here
if build_id < 0:
debug.debug(env.print_pretty(cmd_line), force=True)
debug.print_compilator(output)
debug.print_compilator(err)
if p.returncode == 2:
debug.error("can not compile file ... [keyboard interrrupt]")
else:
debug.error("can not compile file ... ret : " + str(p.returncode))
else:
# in multiprocess interface
queue_lock.acquire()
# if an other write an error before, check if the current process is started before ==> then is the first error
if error_execution["id"] >= build_id:
# nothing to do ...
queue_lock.release()
return;
error_execution["id"] = build_id
error_execution["cmd"] = cmd_line
error_execution["return"] = p.returncode
error_execution["err"] = err,
error_execution["out"] = output,
queue_lock.release()
# not write the command file...
return
debug.verbose("done 3")
# write cmd line only after to prevent errors ...
tools.store_command(cmd_line, store_cmd_line)
class myThread(threading.Thread):
def __init__(self, thread_id, lock, queue):
threading.Thread.__init__(self)
self.thread_id = thread_id
self.name = "Thread " + str(thread_id)
self.queue = queue
self.lock = lock
def run(self):
debug.verbose("Starting " + self.name)
global exit_flag
global current_thread_working
working_set = False
while exit_flag == False:
self.lock.acquire()
if not self.queue.empty():
if working_set == False:
current_thread_working += 1
working_set = True
data = self.queue.get()
self.lock.release()
debug.verbose(self.name + " processing '" + data[0] + "'")
if data[0]=="cmd_line":
comment = data[2]
cmd_line = data[1]
cmd_store_file = data[3]
debug.print_element("[" + str(data[4]) + "][" + str(self.thread_id) + "] " + comment[0],
comment[1],
comment[2],
comment[3])
run_command(cmd_line,
cmd_store_file,
build_id=data[4],
file=comment[3],
store_output_file=data[5],
depend_data=data[6])
else:
debug.warning("unknow request command : " + data[0])
else:
if working_set==True:
current_thread_working -= 1
working_set=False
# no element to parse, just wait ...
self.lock.release()
time.sleep(0.2)
# kill requested ...
debug.verbose("Exiting " + self.name)
def set_error_occured():
global exit_flag
exit_flag = True
def set_core_number(number_of_core):
global processor_availlable
processor_availlable = number_of_core
debug.debug(" set number of core for multi process compilation : " + str(processor_availlable))
# nothing else to do
def init():
global error_occured
global exit_flag
global is_init
if is_init == False:
is_init = True
error_occured = False
global threads
global queue_lock
global work_queue
# Create all the new threads
thread_id = 0
while thread_id < processor_availlable:
thread = myThread(thread_id, queue_lock, work_queue)
thread.start()
threads.append(thread)
thread_id += 1
def un_init():
global exit_flag
# Notify threads it's time to exit
exit_flag = True
if processor_availlable > 1:
# Wait for all threads to complete
for tmp in threads:
debug.verbose("join thread ...")
tmp.join()
debug.verbose("Exiting ALL Threads")
def run_in_pool(cmd_line, comment, store_cmd_line="", store_output_file="", depend_data=None):
global current_id_execution
if processor_availlable <= 1:
debug.print_element(comment[0], comment[1], comment[2], comment[3])
run_command(cmd_line, store_cmd_line, file=comment[3], store_output_file=store_output_file, depend_data=depend_data)
return
# multithreaded mode
init()
# Fill the queue
queue_lock.acquire()
debug.verbose("add : in pool cmd_line")
work_queue.put(["cmd_line", cmd_line, comment, store_cmd_line, current_id_execution, store_output_file, depend_data])
current_id_execution +=1;
queue_lock.release()
def pool_synchrosize():
global error_occured
global error_execution
if processor_availlable <= 1:
#in this case : nothing to synchronise
return
debug.verbose("wait queue process ended\n")
# Wait for queue to empty
while not work_queue.empty() \
and error_occured == False:
time.sleep(0.2)
pass
# Wait all thread have ended their current process
while current_thread_working != 0 \
and error_occured == False:
time.sleep(0.2)
pass
if error_occured == False:
debug.verbose("queue is empty")
else:
un_init()
debug.debug("Thread return with error ... ==> stop all the pool")
if error_execution["id"] == -1:
debug.error("Pool error occured ... (No return information on Pool)")
return
debug.error("Error in an pool element : [" + str(error_execution["id"]) + "]", crash=False)
debug.debug(env.print_pretty(error_execution["cmd"]), force=True)
debug.print_compilator(str(error_execution["out"][0]))
debug.print_compilator(str(error_execution["err"][0]))
if error_execution["return"] == 2:
debug.error("can not compile file ... [keyboard interrrupt]")
else:
debug.error("can not compile file ... return value : " + str(error_execution["return"]))

View File

@@ -15,15 +15,16 @@ def readme():
return f.read()
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
setup(name='maestro',
version='0.1.0',
description='Maestro generic source manager (like repo in simple mode)',
setup(name='island',
version='0.4.0',
description='island generic source manager (like repo in simple mode)',
long_description=readme(),
url='http://github.com/HeeroYui/maestro',
url='http://github.com/HeeroYui/island',
author='Edouard DUPIN',
author_email='yui.heero@gmail.com',
license='MPL-2',
packages=['maestro'],
packages=['island',
'island/actions'],
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
@@ -31,11 +32,14 @@ setup(name='maestro',
'Topic :: Software Development :: Build Tools'
],
keywords='source manager repo qisrc lutin',
scripts=['bin/maestro'],
scripts=['bin/island'],
# Does not work on MacOs
#data_file=[
# ('/etc/bash_completion.d', ['bash-autocompletion/lutin']),
#],
install_requires=[
'lxml',
],
include_package_data = True,
zip_safe=False)