[DEV] init is done, continue integration of sync
This commit is contained in:
parent
aeb7089a00
commit
69ba49e66b
30
bin/maestro
30
bin/maestro
@ -20,12 +20,17 @@ import maestro.env as env
|
|||||||
import maestro.tools as tools
|
import maestro.tools as tools
|
||||||
import maestro.host as maestroHost
|
import maestro.host as maestroHost
|
||||||
import maestro.tools as maestroTools
|
import maestro.tools as maestroTools
|
||||||
|
import maestro.actions as actions
|
||||||
|
|
||||||
myArgs = arguments.maestroArg()
|
myArgs = arguments.maestroArg()
|
||||||
myArgs.add_section("option", "Can be set one time in all case")
|
myArgs.add_section("option", "Can be set one time in all case")
|
||||||
myArgs.add("h", "help", desc="Display this help")
|
myArgs.add("h", "help", desc="Display this help")
|
||||||
myArgs.add("v", "verbose", list=[["0","None"],["1","error"],["2","warning"],["3","info"],["4","debug"],["5","verbose"],["6","extreme_verbose"]], desc="display debug level (verbose) default =2")
|
myArgs.add("v", "verbose", list=[["0","None"],["1","error"],["2","warning"],["3","info"],["4","debug"],["5","verbose"],["6","extreme_verbose"]], desc="display debug level (verbose) default =2")
|
||||||
myArgs.add("c", "color", desc="Display message in color")
|
myArgs.add("c", "color", desc="Display message in color")
|
||||||
|
# for init only
|
||||||
|
#myArgs.add("h", "help", desc="Help of this action")
|
||||||
|
myArgs.add("b", "branch", haveParam=True, desc="Select branch to display")
|
||||||
|
myArgs.add("m", "manifest", haveParam=True, desc="Name of the manifest")
|
||||||
"""
|
"""
|
||||||
myArgs.add("j", "jobs", haveParam=True, desc="Specifies the number of jobs (commands) to run simultaneously")
|
myArgs.add("j", "jobs", haveParam=True, desc="Specifies the number of jobs (commands) to run simultaneously")
|
||||||
myArgs.add("d", "depth", haveParam=True, desc="Depth to clone all the repository")
|
myArgs.add("d", "depth", haveParam=True, desc="Depth to clone all the repository")
|
||||||
@ -42,12 +47,17 @@ def usage():
|
|||||||
# generic argument displayed :
|
# generic argument displayed :
|
||||||
myArgs.display()
|
myArgs.display()
|
||||||
print(" Action availlable" )
|
print(" Action availlable" )
|
||||||
|
list_actions = actions.get_list_of_action();
|
||||||
|
for elem in list_actions:
|
||||||
|
print(" " + color['green'] + elem + color['default'])
|
||||||
|
"""
|
||||||
print(" " + color['green'] + "init" + color['default'])
|
print(" " + color['green'] + "init" + color['default'])
|
||||||
print(" initialize a 'maestro' interface with a manifest in a git ")
|
print(" initialize a 'maestro' interface with a manifest in a git ")
|
||||||
print(" " + color['green'] + "sync" + color['default'])
|
print(" " + color['green'] + "sync" + color['default'])
|
||||||
print(" Syncronise the currect environement")
|
print(" Syncronise the currect environement")
|
||||||
print(" " + color['green'] + "status" + color['default'])
|
print(" " + color['green'] + "status" + color['default'])
|
||||||
print(" Dump the status of the environement")
|
print(" Dump the status of the environement")
|
||||||
|
"""
|
||||||
print(" ex: " + sys.argv[0] + " -c init http://github.com/atria-soft/manifest.git")
|
print(" ex: " + sys.argv[0] + " -c init http://github.com/atria-soft/manifest.git")
|
||||||
print(" ex: " + sys.argv[0] + " sync")
|
print(" ex: " + sys.argv[0] + " sync")
|
||||||
exit(0)
|
exit(0)
|
||||||
@ -163,32 +173,26 @@ if len(new_argument_list) == 0:
|
|||||||
debug.warning("--------------------------------------")
|
debug.warning("--------------------------------------")
|
||||||
usage()
|
usage()
|
||||||
|
|
||||||
list_of_action_availlable=["init","sync","status"]
|
|
||||||
|
# TODO : move tin in actions ...
|
||||||
|
list_actions = actions.get_list_of_action();
|
||||||
|
|
||||||
action_to_do = new_argument_list[0].get_arg()
|
action_to_do = new_argument_list[0].get_arg()
|
||||||
new_argument_list = new_argument_list[1:]
|
new_argument_list = new_argument_list[1:]
|
||||||
if action_to_do not in list_of_action_availlable:
|
if action_to_do not in list_actions:
|
||||||
debug.warning("--------------------------------------")
|
debug.warning("--------------------------------------")
|
||||||
debug.warning("Wrong action type : '" + str(action_to_do) + "' availlable list: " + str(list_of_action_availlable) )
|
debug.warning("Wrong action type : '" + str(action_to_do) + "' availlable list: " + str(list_actions) )
|
||||||
debug.warning("--------------------------------------")
|
debug.warning("--------------------------------------")
|
||||||
usage()
|
usage()
|
||||||
|
|
||||||
|
# todo : Remove this
|
||||||
if action_to_do != "init" \
|
if action_to_do != "init" \
|
||||||
and os.path.exists("." + env.get_system_base_name()) == False:
|
and os.path.exists("." + env.get_system_base_name()) == False:
|
||||||
debug.error("Can not execute a maestro cmd if we have not initialize a config: '" + str("." + env.get_system_base_name()) + "'")
|
debug.error("Can not execute a maestro cmd if we have not initialize a config: '" + str("." + env.get_system_base_name()) + "'")
|
||||||
exit(-1)
|
exit(-1)
|
||||||
|
|
||||||
if action_to_do == "init":
|
|
||||||
debug.info("action: init");
|
|
||||||
|
|
||||||
elif action_to_do == "sync":
|
actions.execute(action_to_do, new_argument_list)
|
||||||
debug.info("action: sync");
|
|
||||||
|
|
||||||
elif action_to_do == "status":
|
|
||||||
debug.info("action: status");
|
|
||||||
|
|
||||||
else:
|
|
||||||
debug.error("Can not do the action...")
|
|
||||||
|
|
||||||
# stop all started threads;
|
# stop all started threads;
|
||||||
#multiprocess.un_init()
|
#multiprocess.un_init()
|
||||||
|
@ -15,9 +15,10 @@ from . import host
|
|||||||
from . import tools
|
from . import tools
|
||||||
from . import debug
|
from . import debug
|
||||||
from . import env
|
from . import env
|
||||||
|
from . import actions
|
||||||
is_init = False
|
is_init = False
|
||||||
|
|
||||||
"""
|
|
||||||
def filter_name_and_file(root, list_files, filter):
|
def filter_name_and_file(root, list_files, filter):
|
||||||
# filter elements:
|
# filter elements:
|
||||||
tmp_list = fnmatch.filter(list_files, filter)
|
tmp_list = fnmatch.filter(list_files, filter)
|
||||||
@ -27,128 +28,28 @@ def filter_name_and_file(root, list_files, filter):
|
|||||||
out.append(elem);
|
out.append(elem);
|
||||||
return out;
|
return out;
|
||||||
|
|
||||||
def filter_path(root, list_files):
|
def import_path_local(path):
|
||||||
out = []
|
|
||||||
for elem in list_files:
|
|
||||||
if len(elem) == 0 \
|
|
||||||
or elem[0] == '.':
|
|
||||||
continue
|
|
||||||
if os.path.isdir(os.path.join(root, elem)) == True:
|
|
||||||
out.append(elem);
|
|
||||||
return out;
|
|
||||||
|
|
||||||
def import_path_local(path, limit_sub_folder, exclude_path = [], base_name = ""):
|
|
||||||
out = []
|
out = []
|
||||||
debug.verbose("maestro files: " + str(path) + " [START]")
|
debug.verbose("maestro files: " + str(path) + " [START]")
|
||||||
if limit_sub_folder == 0:
|
|
||||||
debug.debug("Subparsing limitation append ...")
|
|
||||||
return []
|
|
||||||
try:
|
|
||||||
list_files = os.listdir(path)
|
list_files = os.listdir(path)
|
||||||
except:
|
|
||||||
# an error occure, maybe read error ...
|
|
||||||
debug.warning("error when getting subdirectory of '" + str(path) + "'")
|
|
||||||
return []
|
|
||||||
if path in exclude_path:
|
|
||||||
debug.debug("find '" + str(path) + "' in exclude_path=" + str(exclude_path))
|
|
||||||
return []
|
|
||||||
# filter elements:
|
# filter elements:
|
||||||
tmp_list_maestro_file = filter_name_and_file(path, list_files, base_name + "*.py")
|
tmp_list_maestro_file = filter_name_and_file(path, list_files, "*.py")
|
||||||
debug.verbose("maestro files: " + str(path) + " : " + str(tmp_list_maestro_file))
|
debug.verbose("maestro files: " + str(path) + " : " + str(tmp_list_maestro_file))
|
||||||
# Import the module:
|
# Import the module:
|
||||||
for filename in tmp_list_maestro_file:
|
for filename in tmp_list_maestro_file:
|
||||||
out.append(os.path.join(path, filename))
|
out.append(os.path.join(path, filename))
|
||||||
debug.extreme_verbose(" Find a file : '" + str(out[-1]) + "'")
|
debug.verbose(" Find a file : '" + str(out[-1]) + "'")
|
||||||
need_parse_sub_folder = True
|
|
||||||
rm_value = -1
|
|
||||||
# check if we need to parse sub_folder
|
|
||||||
if len(tmp_list_maestro_file) != 0:
|
|
||||||
need_parse_sub_folder = False
|
|
||||||
# check if the file "maestro_parse_sub.py" is present ==> parse SubFolder (force and add +1 in the resursing
|
|
||||||
if base_name + "ParseSubFolders.txt" in list_files:
|
|
||||||
debug.debug("find SubParser ... " + str(base_name + "ParseSubFolders.txt") + " " + path)
|
|
||||||
data_file_sub = tools.file_read_data(os.path.join(path, base_name + "ParseSubFolders.txt"))
|
|
||||||
if data_file_sub == "":
|
|
||||||
debug.debug(" Empty file Load all subfolder in the worktree in '" + str(path) + "'")
|
|
||||||
need_parse_sub_folder = True
|
|
||||||
rm_value = 0
|
|
||||||
else:
|
|
||||||
list_sub = data_file_sub.split("\n")
|
|
||||||
debug.debug(" Parse selected folders " + str(list_sub) + " no parse local folder directory")
|
|
||||||
need_parse_sub_folder = False
|
|
||||||
for folder in list_sub:
|
|
||||||
if folder == "" \
|
|
||||||
or folder == "/":
|
|
||||||
continue;
|
|
||||||
tmp_out = import_path_local(os.path.join(path, folder),
|
|
||||||
1,
|
|
||||||
exclude_path,
|
|
||||||
base_name)
|
|
||||||
# add all the elements:
|
|
||||||
for elem in tmp_out:
|
|
||||||
out.append(elem)
|
|
||||||
if need_parse_sub_folder == True:
|
|
||||||
list_folders = filter_path(path, list_files)
|
|
||||||
for folder in list_folders:
|
|
||||||
tmp_out = import_path_local(os.path.join(path, folder),
|
|
||||||
limit_sub_folder - rm_value,
|
|
||||||
exclude_path,
|
|
||||||
base_name)
|
|
||||||
# add all the elements:
|
|
||||||
for elem in tmp_out:
|
|
||||||
out.append(elem)
|
|
||||||
return out
|
return out
|
||||||
"""
|
|
||||||
|
|
||||||
def init():
|
def init():
|
||||||
global is_init;
|
global is_init;
|
||||||
if is_init == True:
|
if is_init == True:
|
||||||
return
|
return
|
||||||
"""
|
list_of_maestro_files = import_path_local(os.path.join(tools.get_current_path(__file__), 'actions'))
|
||||||
debug.verbose("Use Make as a make stadard")
|
|
||||||
sys.path.append(tools.get_run_path())
|
|
||||||
# create the list of basic folder:
|
|
||||||
basic_folder_list = []
|
|
||||||
basic_folder_list.append([tools.get_current_path(__file__), True])
|
|
||||||
# Import all sub path without out and archive
|
|
||||||
for elem_path in os.listdir("."):
|
|
||||||
if os.path.isdir(elem_path) == False:
|
|
||||||
continue
|
|
||||||
if elem_path.lower() == "android" \
|
|
||||||
or elem_path == "out" :
|
|
||||||
continue
|
|
||||||
debug.debug("Automatic load path: '" + elem_path + "'")
|
|
||||||
basic_folder_list.append([elem_path, False])
|
|
||||||
|
|
||||||
# create in a single path the basic list of maestro files (all start with maestro and end with .py)
|
actions.init(list_of_maestro_files)
|
||||||
exclude_path = env.get_exclude_search_path()
|
|
||||||
limit_sub_folder = env.get_parse_depth()
|
|
||||||
list_of_maestro_files = []
|
|
||||||
for elem_path, is_system in basic_folder_list:
|
|
||||||
if is_system == True:
|
|
||||||
limit_sub_folder_tmp = 999999
|
|
||||||
else:
|
|
||||||
limit_sub_folder_tmp = limit_sub_folder
|
|
||||||
tmp_out = import_path_local(elem_path,
|
|
||||||
limit_sub_folder_tmp,
|
|
||||||
exclude_path,
|
|
||||||
env.get_build_system_base_name())
|
|
||||||
# add all the elements:
|
|
||||||
for elem in tmp_out:
|
|
||||||
list_of_maestro_files.append(elem)
|
|
||||||
|
|
||||||
debug.debug("Files specific maestro: ")
|
|
||||||
for elem_path in list_of_maestro_files:
|
|
||||||
debug.debug(" " + elem_path)
|
|
||||||
# simply import element from the basic list of files (single parse ...)
|
|
||||||
builder.import_path(list_of_maestro_files)
|
|
||||||
module.import_path(list_of_maestro_files)
|
|
||||||
system.import_path(list_of_maestro_files)
|
|
||||||
target.import_path(list_of_maestro_files)
|
|
||||||
macro.import_path(list_of_maestro_files)
|
|
||||||
|
|
||||||
builder.init()
|
|
||||||
"""
|
|
||||||
is_init = True
|
is_init = True
|
||||||
|
|
||||||
|
|
||||||
|
53
maestro/actions.py
Normal file
53
maestro/actions.py
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
##
|
||||||
|
## @author Edouard DUPIN
|
||||||
|
##
|
||||||
|
## @copyright 2012, Edouard DUPIN, all right reserved
|
||||||
|
##
|
||||||
|
## @license MPL v2.0 (see license file)
|
||||||
|
##
|
||||||
|
|
||||||
|
# Local import
|
||||||
|
from . import debug
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
list_actions = []
|
||||||
|
|
||||||
|
def init(files):
|
||||||
|
global list_actions;
|
||||||
|
debug.debug("List of action for maestro: ")
|
||||||
|
for elem_path in files :
|
||||||
|
debug.debug(" '" + os.path.basename(elem_path)[:-3] + "' file=" + elem_path)
|
||||||
|
list_actions.append({
|
||||||
|
"name":os.path.basename(elem_path)[:-3],
|
||||||
|
"path":elem_path,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def get_list_of_action():
|
||||||
|
global list_actions;
|
||||||
|
out = []
|
||||||
|
for elem in list_actions:
|
||||||
|
out.append(elem["name"])
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def execute(action_to_do, argument_list):
|
||||||
|
global list_actions;
|
||||||
|
# TODO: Move here the check if action is availlable
|
||||||
|
|
||||||
|
for elem in list_actions:
|
||||||
|
if elem["name"] == action_to_do:
|
||||||
|
debug.info("action: " + str(elem));
|
||||||
|
# finish the parsing
|
||||||
|
sys.path.append(os.path.dirname(elem["path"]))
|
||||||
|
the_action = __import__(action_to_do)
|
||||||
|
if "execute" not in dir(the_action):
|
||||||
|
debug.error("execute is not implmented for this action ... '" + str(action_to_do) + "'")
|
||||||
|
return False
|
||||||
|
return the_action.execute(argument_list)
|
||||||
|
debug.error("Can not do the action...")
|
||||||
|
return False
|
82
maestro/actions/init.py
Normal file
82
maestro/actions/init.py
Normal file
@ -0,0 +1,82 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
##
|
||||||
|
## @author Edouard DUPIN
|
||||||
|
##
|
||||||
|
## @copyright 2012, Edouard DUPIN, all right reserved
|
||||||
|
##
|
||||||
|
## @license MPL v2.0 (see license file)
|
||||||
|
##
|
||||||
|
|
||||||
|
from maestro import debug
|
||||||
|
from maestro import tools
|
||||||
|
from maestro import env
|
||||||
|
from maestro import multiprocess
|
||||||
|
import os
|
||||||
|
|
||||||
|
def help():
|
||||||
|
return "plop"
|
||||||
|
|
||||||
|
def execute(arguments):
|
||||||
|
debug.info("execute:")
|
||||||
|
for elem in arguments:
|
||||||
|
debug.info(" '" + str(elem.get_arg()) + "'")
|
||||||
|
if len(arguments) == 0:
|
||||||
|
debug.error("Missing argument to execute the current action ...")
|
||||||
|
|
||||||
|
# the configuration availlable:
|
||||||
|
branch = "master"
|
||||||
|
manifest_name = "default.xml"
|
||||||
|
address_manifest = ""
|
||||||
|
for elem in arguments:
|
||||||
|
if elem.get_option_name() == "branch":
|
||||||
|
debug.info("find branch name: '" + elem.get_arg() + "'")
|
||||||
|
branch = elem.get_arg()
|
||||||
|
elif elem.get_option_name() == "manifest":
|
||||||
|
debug.info("find mmanifest name: '" + elem.get_arg() + "'")
|
||||||
|
manifest_name = elem.get_arg()
|
||||||
|
elif elem.get_option_name() == "":
|
||||||
|
if address_manifest != "":
|
||||||
|
debug.error("Manifest adress already set : '" + address_manifest + "' !!! '" + elem.get_arg() + "'")
|
||||||
|
address_manifest = elem.get_arg()
|
||||||
|
else:
|
||||||
|
debug.error("Wrong argument: '" + elem.get_option_name() + "' '" + elem.get_arg() + "'")
|
||||||
|
|
||||||
|
if address_manifest == "":
|
||||||
|
debug.error("Init: Missing manifest name")
|
||||||
|
|
||||||
|
debug.info("Init with: '" + address_manifest + "' branch='" + branch + "' name of manifest='" + manifest_name + "'")
|
||||||
|
|
||||||
|
|
||||||
|
# check if .XXX exist (create it if needed)
|
||||||
|
base_path = os.path.join(tools.get_run_path(), "." + env.get_system_base_name())
|
||||||
|
base_config = os.path.join(base_path, "config.txt")
|
||||||
|
base_manifest_repo = os.path.join(base_path, "manifest")
|
||||||
|
if os.path.exists(base_path) == True \
|
||||||
|
and os.path.exists(base_config) == True \
|
||||||
|
and os.path.exists(base_manifest_repo) == True:
|
||||||
|
debug.error("System already init: path already exist: '" + str(base_path) + "'")
|
||||||
|
tools.create_directory(base_path)
|
||||||
|
# check if the git of the manifest if availlable
|
||||||
|
|
||||||
|
# create the file configuration:
|
||||||
|
data = "repo=" + address_manifest + "\nbranch=" + branch + "\nfile=" + manifest_name
|
||||||
|
tools.file_write_data(base_config, data)
|
||||||
|
|
||||||
|
#clone the manifest repository
|
||||||
|
cmd = "git clone " + address_manifest + " --branch " + branch + " " + base_manifest_repo
|
||||||
|
|
||||||
|
debug.info("clone the manifest")
|
||||||
|
ret = multiprocess.run_command_direct(cmd)
|
||||||
|
|
||||||
|
if ret == "":
|
||||||
|
return True
|
||||||
|
|
||||||
|
if ret == False:
|
||||||
|
# all is good, ready to get the system work corectly
|
||||||
|
return True
|
||||||
|
debug.info("'" + ret + "'")
|
||||||
|
debug.error("Init does not work")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
0
maestro/actions/status.py
Normal file
0
maestro/actions/status.py
Normal file
76
maestro/actions/sync.py
Normal file
76
maestro/actions/sync.py
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
##
|
||||||
|
## @author Edouard DUPIN
|
||||||
|
##
|
||||||
|
## @copyright 2012, Edouard DUPIN, all right reserved
|
||||||
|
##
|
||||||
|
## @license MPL v2.0 (see license file)
|
||||||
|
##
|
||||||
|
|
||||||
|
from maestro import debug
|
||||||
|
from maestro import tools
|
||||||
|
from maestro import env
|
||||||
|
from maestro import multiprocess
|
||||||
|
import os
|
||||||
|
from lxml import etree
|
||||||
|
|
||||||
|
|
||||||
|
def help():
|
||||||
|
return "plop"
|
||||||
|
|
||||||
|
def load_manifest(file):
|
||||||
|
tree = etree.parse(file)
|
||||||
|
debug.info("manifest:")
|
||||||
|
root = tree.getroot()
|
||||||
|
if root.tag != "manifest":
|
||||||
|
debug.error("in '" + str(file) + "' have not main xml node='manifest'")
|
||||||
|
for child in root:
|
||||||
|
if type(child) == etree._Comment:
|
||||||
|
debug.info(" comment='" + str(child.text) + "'");
|
||||||
|
else:
|
||||||
|
debug.info(" '" + str(child.tag) + "' values=" + str(child.attrib));
|
||||||
|
# inside data child.text
|
||||||
|
return "";
|
||||||
|
|
||||||
|
|
||||||
|
def execute(arguments):
|
||||||
|
debug.info("execute:")
|
||||||
|
for elem in arguments:
|
||||||
|
debug.info(" '" + str(elem.get_arg()) + "'")
|
||||||
|
if len(arguments) != 0:
|
||||||
|
debug.error("Sync have not parameter")
|
||||||
|
|
||||||
|
# check if .XXX exist (create it if needed)
|
||||||
|
base_path = os.path.join(tools.get_run_path(), "." + env.get_system_base_name())
|
||||||
|
base_config = os.path.join(base_path, "config.txt")
|
||||||
|
base_manifest_repo = os.path.join(base_path, "manifest")
|
||||||
|
if os.path.exists(base_path) == False \
|
||||||
|
or os.path.exists(base_config) == False \
|
||||||
|
or os.path.exists(base_manifest_repo) == False:
|
||||||
|
debug.error("System already init have an error: missing data: '" + str(base_path) + "'")
|
||||||
|
|
||||||
|
config_property = tools.file_read_data(base_config)
|
||||||
|
|
||||||
|
element_config = config_property.split("\n")
|
||||||
|
if len(element_config) != 3:
|
||||||
|
debug.error("error in configuration property")
|
||||||
|
if element_config[0][:5] != "repo=":
|
||||||
|
debug.error("error in configuration property (2)")
|
||||||
|
if element_config[1][:7] != "branch=":
|
||||||
|
debug.error("error in configuration property (3)")
|
||||||
|
if element_config[2][:5] != "file=":
|
||||||
|
debug.error("error in configuration property (4)")
|
||||||
|
configuration = {
|
||||||
|
"repo":element_config[0][5:],
|
||||||
|
"branch":element_config[1][7:],
|
||||||
|
"file":element_config[2][5:]
|
||||||
|
}
|
||||||
|
debug.info("configuration property: " + str(configuration))
|
||||||
|
|
||||||
|
file_source_manifest = os.path.join(base_manifest_repo, configuration["file"])
|
||||||
|
if os.path.exists(file_source_manifest) == False:
|
||||||
|
debug.error("Missing manifest file : '" + str(file_source_manifest) + "'")
|
||||||
|
|
||||||
|
manifest = load_manifest(file_source_manifest)
|
||||||
|
|
@ -12,10 +12,6 @@ import sys
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import sys
|
import sys
|
||||||
if sys.version_info >= (3, 0):
|
|
||||||
import queue
|
|
||||||
else:
|
|
||||||
import Queue as queue
|
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import shlex
|
import shlex
|
||||||
@ -23,43 +19,7 @@ import shlex
|
|||||||
from . import debug
|
from . import debug
|
||||||
from . import tools
|
from . import tools
|
||||||
from . import env
|
from . import env
|
||||||
from . import depend
|
|
||||||
|
|
||||||
queue_lock = threading.Lock()
|
|
||||||
work_queue = queue.Queue()
|
|
||||||
current_thread_working = 0
|
|
||||||
threads = []
|
|
||||||
# To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation
|
|
||||||
current_id_execution = 0
|
|
||||||
error_execution = {
|
|
||||||
"id":-1,
|
|
||||||
"cmd":"",
|
|
||||||
"return":0,
|
|
||||||
"err":"",
|
|
||||||
"out":"",
|
|
||||||
}
|
|
||||||
|
|
||||||
exit_flag = False # resuest stop of the thread
|
|
||||||
is_init = False # the thread are initialized
|
|
||||||
error_occured = False # a thread have an error
|
|
||||||
processor_availlable = 1 # number of CPU core availlable
|
|
||||||
##
|
|
||||||
## @brief Execute the command with no get of output
|
|
||||||
##
|
|
||||||
def run_command_no_lock_out(cmd_line):
|
|
||||||
# prepare command line:
|
|
||||||
args = shlex.split(cmd_line)
|
|
||||||
debug.info("cmd = " + str(args))
|
|
||||||
try:
|
|
||||||
# create the subprocess
|
|
||||||
p = subprocess.Popen(args)
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
debug.error("subprocess.CalledProcessError : " + str(args))
|
|
||||||
return
|
|
||||||
#except:
|
|
||||||
# debug.error("Exception on : " + str(args))
|
|
||||||
# launch the subprocess:
|
|
||||||
p.communicate()
|
|
||||||
|
|
||||||
##
|
##
|
||||||
## @brief Execute the command and ruturn generate data
|
## @brief Execute the command and ruturn generate data
|
||||||
@ -89,215 +49,3 @@ def run_command_direct(cmd_line):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_file="", depend_data=None):
|
|
||||||
global error_occured
|
|
||||||
global exit_flag
|
|
||||||
global current_id_execution
|
|
||||||
global error_execution
|
|
||||||
# prepare command line:
|
|
||||||
args = shlex.split(cmd_line)
|
|
||||||
debug.verbose("cmd = " + str(args))
|
|
||||||
try:
|
|
||||||
# create the subprocess
|
|
||||||
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
debug.error("subprocess.CalledProcessError : TODO ...")
|
|
||||||
except:
|
|
||||||
debug.error("Exception on : " + str(args))
|
|
||||||
# launch the subprocess:
|
|
||||||
output, err = p.communicate()
|
|
||||||
if sys.version_info >= (3, 0):
|
|
||||||
output = output.decode("utf-8")
|
|
||||||
err = err.decode("utf-8")
|
|
||||||
# store error if needed:
|
|
||||||
tools.store_warning(store_output_file, output, err)
|
|
||||||
# Check error :
|
|
||||||
if p.returncode == 0:
|
|
||||||
debug.debug(env.print_pretty(cmd_line))
|
|
||||||
queue_lock.acquire()
|
|
||||||
if depend_data != None:
|
|
||||||
depend.create_dependency_file(depend_data['file'], depend_data['data'])
|
|
||||||
# TODO : Print the output all the time .... ==> to show warnings ...
|
|
||||||
if build_id >= 0 and (output != "" or err != ""):
|
|
||||||
debug.warning("output in subprocess compiling: '" + file + "'")
|
|
||||||
if output != "":
|
|
||||||
debug.print_compilator(output)
|
|
||||||
if err != "":
|
|
||||||
debug.print_compilator(err)
|
|
||||||
queue_lock.release()
|
|
||||||
else:
|
|
||||||
error_occured = True
|
|
||||||
exit_flag = True
|
|
||||||
# if No ID : Not in a multiprocess mode ==> just stop here
|
|
||||||
if build_id < 0:
|
|
||||||
debug.debug(env.print_pretty(cmd_line), force=True)
|
|
||||||
debug.print_compilator(output)
|
|
||||||
debug.print_compilator(err)
|
|
||||||
if p.returncode == 2:
|
|
||||||
debug.error("can not compile file ... [keyboard interrrupt]")
|
|
||||||
else:
|
|
||||||
debug.error("can not compile file ... ret : " + str(p.returncode))
|
|
||||||
else:
|
|
||||||
# in multiprocess interface
|
|
||||||
queue_lock.acquire()
|
|
||||||
# if an other write an error before, check if the current process is started before ==> then is the first error
|
|
||||||
if error_execution["id"] >= build_id:
|
|
||||||
# nothing to do ...
|
|
||||||
queue_lock.release()
|
|
||||||
return;
|
|
||||||
error_execution["id"] = build_id
|
|
||||||
error_execution["cmd"] = cmd_line
|
|
||||||
error_execution["return"] = p.returncode
|
|
||||||
error_execution["err"] = err,
|
|
||||||
error_execution["out"] = output,
|
|
||||||
queue_lock.release()
|
|
||||||
# not write the command file...
|
|
||||||
return
|
|
||||||
debug.verbose("done 3")
|
|
||||||
# write cmd line only after to prevent errors ...
|
|
||||||
tools.store_command(cmd_line, store_cmd_line)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class myThread(threading.Thread):
|
|
||||||
def __init__(self, thread_id, lock, queue):
|
|
||||||
threading.Thread.__init__(self)
|
|
||||||
self.thread_id = thread_id
|
|
||||||
self.name = "Thread " + str(thread_id)
|
|
||||||
self.queue = queue
|
|
||||||
self.lock = lock
|
|
||||||
def run(self):
|
|
||||||
debug.verbose("Starting " + self.name)
|
|
||||||
global exit_flag
|
|
||||||
global current_thread_working
|
|
||||||
working_set = False
|
|
||||||
while exit_flag == False:
|
|
||||||
self.lock.acquire()
|
|
||||||
if not self.queue.empty():
|
|
||||||
if working_set == False:
|
|
||||||
current_thread_working += 1
|
|
||||||
working_set = True
|
|
||||||
data = self.queue.get()
|
|
||||||
self.lock.release()
|
|
||||||
debug.verbose(self.name + " processing '" + data[0] + "'")
|
|
||||||
if data[0]=="cmd_line":
|
|
||||||
comment = data[2]
|
|
||||||
cmd_line = data[1]
|
|
||||||
cmd_store_file = data[3]
|
|
||||||
debug.print_element("[" + str(data[4]) + "][" + str(self.thread_id) + "] " + comment[0],
|
|
||||||
comment[1],
|
|
||||||
comment[2],
|
|
||||||
comment[3])
|
|
||||||
run_command(cmd_line,
|
|
||||||
cmd_store_file,
|
|
||||||
build_id=data[4],
|
|
||||||
file=comment[3],
|
|
||||||
store_output_file=data[5],
|
|
||||||
depend_data=data[6])
|
|
||||||
else:
|
|
||||||
debug.warning("unknow request command : " + data[0])
|
|
||||||
else:
|
|
||||||
if working_set==True:
|
|
||||||
current_thread_working -= 1
|
|
||||||
working_set=False
|
|
||||||
# no element to parse, just wait ...
|
|
||||||
self.lock.release()
|
|
||||||
time.sleep(0.2)
|
|
||||||
# kill requested ...
|
|
||||||
debug.verbose("Exiting " + self.name)
|
|
||||||
|
|
||||||
|
|
||||||
def set_error_occured():
|
|
||||||
global exit_flag
|
|
||||||
exit_flag = True
|
|
||||||
|
|
||||||
def set_core_number(number_of_core):
|
|
||||||
global processor_availlable
|
|
||||||
processor_availlable = number_of_core
|
|
||||||
debug.debug(" set number of core for multi process compilation : " + str(processor_availlable))
|
|
||||||
# nothing else to do
|
|
||||||
|
|
||||||
def init():
|
|
||||||
global error_occured
|
|
||||||
global exit_flag
|
|
||||||
global is_init
|
|
||||||
if is_init == False:
|
|
||||||
is_init = True
|
|
||||||
error_occured = False
|
|
||||||
global threads
|
|
||||||
global queue_lock
|
|
||||||
global work_queue
|
|
||||||
# Create all the new threads
|
|
||||||
thread_id = 0
|
|
||||||
while thread_id < processor_availlable:
|
|
||||||
thread = myThread(thread_id, queue_lock, work_queue)
|
|
||||||
thread.start()
|
|
||||||
threads.append(thread)
|
|
||||||
thread_id += 1
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def un_init():
|
|
||||||
global exit_flag
|
|
||||||
# Notify threads it's time to exit
|
|
||||||
exit_flag = True
|
|
||||||
if processor_availlable > 1:
|
|
||||||
# Wait for all threads to complete
|
|
||||||
for tmp in threads:
|
|
||||||
debug.verbose("join thread ...")
|
|
||||||
tmp.join()
|
|
||||||
debug.verbose("Exiting ALL Threads")
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def run_in_pool(cmd_line, comment, store_cmd_line="", store_output_file="", depend_data=None):
|
|
||||||
global current_id_execution
|
|
||||||
if processor_availlable <= 1:
|
|
||||||
debug.print_element(comment[0], comment[1], comment[2], comment[3])
|
|
||||||
run_command(cmd_line, store_cmd_line, file=comment[3], store_output_file=store_output_file, depend_data=depend_data)
|
|
||||||
return
|
|
||||||
# multithreaded mode
|
|
||||||
init()
|
|
||||||
# Fill the queue
|
|
||||||
queue_lock.acquire()
|
|
||||||
debug.verbose("add : in pool cmd_line")
|
|
||||||
work_queue.put(["cmd_line", cmd_line, comment, store_cmd_line, current_id_execution, store_output_file, depend_data])
|
|
||||||
current_id_execution +=1;
|
|
||||||
queue_lock.release()
|
|
||||||
|
|
||||||
|
|
||||||
def pool_synchrosize():
|
|
||||||
global error_occured
|
|
||||||
global error_execution
|
|
||||||
if processor_availlable <= 1:
|
|
||||||
#in this case : nothing to synchronise
|
|
||||||
return
|
|
||||||
|
|
||||||
debug.verbose("wait queue process ended\n")
|
|
||||||
# Wait for queue to empty
|
|
||||||
while not work_queue.empty() \
|
|
||||||
and error_occured == False:
|
|
||||||
time.sleep(0.2)
|
|
||||||
pass
|
|
||||||
# Wait all thread have ended their current process
|
|
||||||
while current_thread_working != 0 \
|
|
||||||
and error_occured == False:
|
|
||||||
time.sleep(0.2)
|
|
||||||
pass
|
|
||||||
if error_occured == False:
|
|
||||||
debug.verbose("queue is empty")
|
|
||||||
else:
|
|
||||||
un_init()
|
|
||||||
debug.debug("Thread return with error ... ==> stop all the pool")
|
|
||||||
if error_execution["id"] == -1:
|
|
||||||
debug.error("Pool error occured ... (No return information on Pool)")
|
|
||||||
return
|
|
||||||
debug.error("Error in an pool element : [" + str(error_execution["id"]) + "]", crash=False)
|
|
||||||
debug.debug(env.print_pretty(error_execution["cmd"]), force=True)
|
|
||||||
debug.print_compilator(str(error_execution["out"][0]))
|
|
||||||
debug.print_compilator(str(error_execution["err"][0]))
|
|
||||||
if error_execution["return"] == 2:
|
|
||||||
debug.error("can not compile file ... [keyboard interrrupt]")
|
|
||||||
else:
|
|
||||||
debug.error("can not compile file ... return value : " + str(error_execution["return"]))
|
|
||||||
|
|
||||||
|
@ -29,13 +29,18 @@ def get_run_path():
|
|||||||
def get_current_path(file):
|
def get_current_path(file):
|
||||||
return os.path.dirname(os.path.realpath(file))
|
return os.path.dirname(os.path.realpath(file))
|
||||||
|
|
||||||
def create_directory_of_file(file):
|
|
||||||
path = os.path.dirname(file)
|
def create_directory(path):
|
||||||
try:
|
try:
|
||||||
os.stat(path)
|
os.stat(path)
|
||||||
except:
|
except:
|
||||||
os.makedirs(path)
|
os.makedirs(path)
|
||||||
|
|
||||||
|
def create_directory_of_file(file):
|
||||||
|
path = os.path.dirname(file)
|
||||||
|
create_directory(path)
|
||||||
|
|
||||||
|
|
||||||
def get_list_sub_path(path):
|
def get_list_sub_path(path):
|
||||||
# TODO : os.listdir(path)
|
# TODO : os.listdir(path)
|
||||||
for dirname, dirnames, filenames in os.walk(path):
|
for dirname, dirnames, filenames in os.walk(path):
|
||||||
|
3
setup.py
3
setup.py
@ -23,7 +23,8 @@ setup(name='maestro',
|
|||||||
author='Edouard DUPIN',
|
author='Edouard DUPIN',
|
||||||
author_email='yui.heero@gmail.com',
|
author_email='yui.heero@gmail.com',
|
||||||
license='MPL-2',
|
license='MPL-2',
|
||||||
packages=['maestro'],
|
packages=['maestro',
|
||||||
|
'maestro/actions'],
|
||||||
classifiers=[
|
classifiers=[
|
||||||
'Development Status :: 2 - Pre-Alpha',
|
'Development Status :: 2 - Pre-Alpha',
|
||||||
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
|
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
|
||||||
|
Loading…
Reference in New Issue
Block a user