[DEV] init is done, continue integration of sync
This commit is contained in:
parent
aeb7089a00
commit
69ba49e66b
32
bin/maestro
32
bin/maestro
@ -20,12 +20,17 @@ import maestro.env as env
|
||||
import maestro.tools as tools
|
||||
import maestro.host as maestroHost
|
||||
import maestro.tools as maestroTools
|
||||
import maestro.actions as actions
|
||||
|
||||
myArgs = arguments.maestroArg()
|
||||
myArgs.add_section("option", "Can be set one time in all case")
|
||||
myArgs.add("h", "help", desc="Display this help")
|
||||
myArgs.add("v", "verbose", list=[["0","None"],["1","error"],["2","warning"],["3","info"],["4","debug"],["5","verbose"],["6","extreme_verbose"]], desc="display debug level (verbose) default =2")
|
||||
myArgs.add("c", "color", desc="Display message in color")
|
||||
# for init only
|
||||
#myArgs.add("h", "help", desc="Help of this action")
|
||||
myArgs.add("b", "branch", haveParam=True, desc="Select branch to display")
|
||||
myArgs.add("m", "manifest", haveParam=True, desc="Name of the manifest")
|
||||
"""
|
||||
myArgs.add("j", "jobs", haveParam=True, desc="Specifies the number of jobs (commands) to run simultaneously")
|
||||
myArgs.add("d", "depth", haveParam=True, desc="Depth to clone all the repository")
|
||||
@ -42,12 +47,17 @@ def usage():
|
||||
# generic argument displayed :
|
||||
myArgs.display()
|
||||
print(" Action availlable" )
|
||||
list_actions = actions.get_list_of_action();
|
||||
for elem in list_actions:
|
||||
print(" " + color['green'] + elem + color['default'])
|
||||
"""
|
||||
print(" " + color['green'] + "init" + color['default'])
|
||||
print(" initialize a 'maestro' interface with a manifest in a git ")
|
||||
print(" " + color['green'] + "sync" + color['default'])
|
||||
print(" Syncronise the currect environement")
|
||||
print(" " + color['green'] + "status" + color['default'])
|
||||
print(" Dump the status of the environement")
|
||||
"""
|
||||
print(" ex: " + sys.argv[0] + " -c init http://github.com/atria-soft/manifest.git")
|
||||
print(" ex: " + sys.argv[0] + " sync")
|
||||
exit(0)
|
||||
@ -163,32 +173,26 @@ if len(new_argument_list) == 0:
|
||||
debug.warning("--------------------------------------")
|
||||
usage()
|
||||
|
||||
list_of_action_availlable=["init","sync","status"]
|
||||
|
||||
# TODO : move tin in actions ...
|
||||
list_actions = actions.get_list_of_action();
|
||||
|
||||
action_to_do = new_argument_list[0].get_arg()
|
||||
new_argument_list = new_argument_list[1:]
|
||||
if action_to_do not in list_of_action_availlable:
|
||||
if action_to_do not in list_actions:
|
||||
debug.warning("--------------------------------------")
|
||||
debug.warning("Wrong action type : '" + str(action_to_do) + "' availlable list: " + str(list_of_action_availlable) )
|
||||
debug.warning("Wrong action type : '" + str(action_to_do) + "' availlable list: " + str(list_actions) )
|
||||
debug.warning("--------------------------------------")
|
||||
usage()
|
||||
|
||||
# todo : Remove this
|
||||
if action_to_do != "init" \
|
||||
and os.path.exists("." + env.get_system_base_name()) == False:
|
||||
debug.error("Can not execute a maestro cmd if we have not initialize a config: '" + str("." + env.get_system_base_name()) + "'")
|
||||
exit(-1)
|
||||
|
||||
if action_to_do == "init":
|
||||
debug.info("action: init");
|
||||
|
||||
elif action_to_do == "sync":
|
||||
debug.info("action: sync");
|
||||
|
||||
elif action_to_do == "status":
|
||||
debug.info("action: status");
|
||||
|
||||
else:
|
||||
debug.error("Can not do the action...")
|
||||
|
||||
actions.execute(action_to_do, new_argument_list)
|
||||
|
||||
# stop all started threads;
|
||||
#multiprocess.un_init()
|
||||
|
||||
@ -15,9 +15,10 @@ from . import host
|
||||
from . import tools
|
||||
from . import debug
|
||||
from . import env
|
||||
from . import actions
|
||||
is_init = False
|
||||
|
||||
"""
|
||||
|
||||
def filter_name_and_file(root, list_files, filter):
|
||||
# filter elements:
|
||||
tmp_list = fnmatch.filter(list_files, filter)
|
||||
@ -27,128 +28,28 @@ def filter_name_and_file(root, list_files, filter):
|
||||
out.append(elem);
|
||||
return out;
|
||||
|
||||
def filter_path(root, list_files):
|
||||
out = []
|
||||
for elem in list_files:
|
||||
if len(elem) == 0 \
|
||||
or elem[0] == '.':
|
||||
continue
|
||||
if os.path.isdir(os.path.join(root, elem)) == True:
|
||||
out.append(elem);
|
||||
return out;
|
||||
|
||||
def import_path_local(path, limit_sub_folder, exclude_path = [], base_name = ""):
|
||||
def import_path_local(path):
|
||||
out = []
|
||||
debug.verbose("maestro files: " + str(path) + " [START]")
|
||||
if limit_sub_folder == 0:
|
||||
debug.debug("Subparsing limitation append ...")
|
||||
return []
|
||||
try:
|
||||
list_files = os.listdir(path)
|
||||
except:
|
||||
# an error occure, maybe read error ...
|
||||
debug.warning("error when getting subdirectory of '" + str(path) + "'")
|
||||
return []
|
||||
if path in exclude_path:
|
||||
debug.debug("find '" + str(path) + "' in exclude_path=" + str(exclude_path))
|
||||
return []
|
||||
list_files = os.listdir(path)
|
||||
# filter elements:
|
||||
tmp_list_maestro_file = filter_name_and_file(path, list_files, base_name + "*.py")
|
||||
tmp_list_maestro_file = filter_name_and_file(path, list_files, "*.py")
|
||||
debug.verbose("maestro files: " + str(path) + " : " + str(tmp_list_maestro_file))
|
||||
# Import the module:
|
||||
for filename in tmp_list_maestro_file:
|
||||
out.append(os.path.join(path, filename))
|
||||
debug.extreme_verbose(" Find a file : '" + str(out[-1]) + "'")
|
||||
need_parse_sub_folder = True
|
||||
rm_value = -1
|
||||
# check if we need to parse sub_folder
|
||||
if len(tmp_list_maestro_file) != 0:
|
||||
need_parse_sub_folder = False
|
||||
# check if the file "maestro_parse_sub.py" is present ==> parse SubFolder (force and add +1 in the resursing
|
||||
if base_name + "ParseSubFolders.txt" in list_files:
|
||||
debug.debug("find SubParser ... " + str(base_name + "ParseSubFolders.txt") + " " + path)
|
||||
data_file_sub = tools.file_read_data(os.path.join(path, base_name + "ParseSubFolders.txt"))
|
||||
if data_file_sub == "":
|
||||
debug.debug(" Empty file Load all subfolder in the worktree in '" + str(path) + "'")
|
||||
need_parse_sub_folder = True
|
||||
rm_value = 0
|
||||
else:
|
||||
list_sub = data_file_sub.split("\n")
|
||||
debug.debug(" Parse selected folders " + str(list_sub) + " no parse local folder directory")
|
||||
need_parse_sub_folder = False
|
||||
for folder in list_sub:
|
||||
if folder == "" \
|
||||
or folder == "/":
|
||||
continue;
|
||||
tmp_out = import_path_local(os.path.join(path, folder),
|
||||
1,
|
||||
exclude_path,
|
||||
base_name)
|
||||
# add all the elements:
|
||||
for elem in tmp_out:
|
||||
out.append(elem)
|
||||
if need_parse_sub_folder == True:
|
||||
list_folders = filter_path(path, list_files)
|
||||
for folder in list_folders:
|
||||
tmp_out = import_path_local(os.path.join(path, folder),
|
||||
limit_sub_folder - rm_value,
|
||||
exclude_path,
|
||||
base_name)
|
||||
# add all the elements:
|
||||
for elem in tmp_out:
|
||||
out.append(elem)
|
||||
debug.verbose(" Find a file : '" + str(out[-1]) + "'")
|
||||
return out
|
||||
"""
|
||||
|
||||
|
||||
def init():
|
||||
global is_init;
|
||||
if is_init == True:
|
||||
return
|
||||
"""
|
||||
debug.verbose("Use Make as a make stadard")
|
||||
sys.path.append(tools.get_run_path())
|
||||
# create the list of basic folder:
|
||||
basic_folder_list = []
|
||||
basic_folder_list.append([tools.get_current_path(__file__), True])
|
||||
# Import all sub path without out and archive
|
||||
for elem_path in os.listdir("."):
|
||||
if os.path.isdir(elem_path) == False:
|
||||
continue
|
||||
if elem_path.lower() == "android" \
|
||||
or elem_path == "out" :
|
||||
continue
|
||||
debug.debug("Automatic load path: '" + elem_path + "'")
|
||||
basic_folder_list.append([elem_path, False])
|
||||
list_of_maestro_files = import_path_local(os.path.join(tools.get_current_path(__file__), 'actions'))
|
||||
|
||||
# create in a single path the basic list of maestro files (all start with maestro and end with .py)
|
||||
exclude_path = env.get_exclude_search_path()
|
||||
limit_sub_folder = env.get_parse_depth()
|
||||
list_of_maestro_files = []
|
||||
for elem_path, is_system in basic_folder_list:
|
||||
if is_system == True:
|
||||
limit_sub_folder_tmp = 999999
|
||||
else:
|
||||
limit_sub_folder_tmp = limit_sub_folder
|
||||
tmp_out = import_path_local(elem_path,
|
||||
limit_sub_folder_tmp,
|
||||
exclude_path,
|
||||
env.get_build_system_base_name())
|
||||
# add all the elements:
|
||||
for elem in tmp_out:
|
||||
list_of_maestro_files.append(elem)
|
||||
actions.init(list_of_maestro_files)
|
||||
|
||||
debug.debug("Files specific maestro: ")
|
||||
for elem_path in list_of_maestro_files:
|
||||
debug.debug(" " + elem_path)
|
||||
# simply import element from the basic list of files (single parse ...)
|
||||
builder.import_path(list_of_maestro_files)
|
||||
module.import_path(list_of_maestro_files)
|
||||
system.import_path(list_of_maestro_files)
|
||||
target.import_path(list_of_maestro_files)
|
||||
macro.import_path(list_of_maestro_files)
|
||||
|
||||
builder.init()
|
||||
"""
|
||||
is_init = True
|
||||
|
||||
|
||||
|
||||
53
maestro/actions.py
Normal file
53
maestro/actions.py
Normal file
@ -0,0 +1,53 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
##
|
||||
## @author Edouard DUPIN
|
||||
##
|
||||
## @copyright 2012, Edouard DUPIN, all right reserved
|
||||
##
|
||||
## @license MPL v2.0 (see license file)
|
||||
##
|
||||
|
||||
# Local import
|
||||
from . import debug
|
||||
import os
|
||||
import sys
|
||||
|
||||
list_actions = []
|
||||
|
||||
def init(files):
|
||||
global list_actions;
|
||||
debug.debug("List of action for maestro: ")
|
||||
for elem_path in files :
|
||||
debug.debug(" '" + os.path.basename(elem_path)[:-3] + "' file=" + elem_path)
|
||||
list_actions.append({
|
||||
"name":os.path.basename(elem_path)[:-3],
|
||||
"path":elem_path,
|
||||
})
|
||||
|
||||
|
||||
def get_list_of_action():
|
||||
global list_actions;
|
||||
out = []
|
||||
for elem in list_actions:
|
||||
out.append(elem["name"])
|
||||
return out
|
||||
|
||||
|
||||
|
||||
def execute(action_to_do, argument_list):
|
||||
global list_actions;
|
||||
# TODO: Move here the check if action is availlable
|
||||
|
||||
for elem in list_actions:
|
||||
if elem["name"] == action_to_do:
|
||||
debug.info("action: " + str(elem));
|
||||
# finish the parsing
|
||||
sys.path.append(os.path.dirname(elem["path"]))
|
||||
the_action = __import__(action_to_do)
|
||||
if "execute" not in dir(the_action):
|
||||
debug.error("execute is not implmented for this action ... '" + str(action_to_do) + "'")
|
||||
return False
|
||||
return the_action.execute(argument_list)
|
||||
debug.error("Can not do the action...")
|
||||
return False
|
||||
82
maestro/actions/init.py
Normal file
82
maestro/actions/init.py
Normal file
@ -0,0 +1,82 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
##
|
||||
## @author Edouard DUPIN
|
||||
##
|
||||
## @copyright 2012, Edouard DUPIN, all right reserved
|
||||
##
|
||||
## @license MPL v2.0 (see license file)
|
||||
##
|
||||
|
||||
from maestro import debug
|
||||
from maestro import tools
|
||||
from maestro import env
|
||||
from maestro import multiprocess
|
||||
import os
|
||||
|
||||
def help():
|
||||
return "plop"
|
||||
|
||||
def execute(arguments):
|
||||
debug.info("execute:")
|
||||
for elem in arguments:
|
||||
debug.info(" '" + str(elem.get_arg()) + "'")
|
||||
if len(arguments) == 0:
|
||||
debug.error("Missing argument to execute the current action ...")
|
||||
|
||||
# the configuration availlable:
|
||||
branch = "master"
|
||||
manifest_name = "default.xml"
|
||||
address_manifest = ""
|
||||
for elem in arguments:
|
||||
if elem.get_option_name() == "branch":
|
||||
debug.info("find branch name: '" + elem.get_arg() + "'")
|
||||
branch = elem.get_arg()
|
||||
elif elem.get_option_name() == "manifest":
|
||||
debug.info("find mmanifest name: '" + elem.get_arg() + "'")
|
||||
manifest_name = elem.get_arg()
|
||||
elif elem.get_option_name() == "":
|
||||
if address_manifest != "":
|
||||
debug.error("Manifest adress already set : '" + address_manifest + "' !!! '" + elem.get_arg() + "'")
|
||||
address_manifest = elem.get_arg()
|
||||
else:
|
||||
debug.error("Wrong argument: '" + elem.get_option_name() + "' '" + elem.get_arg() + "'")
|
||||
|
||||
if address_manifest == "":
|
||||
debug.error("Init: Missing manifest name")
|
||||
|
||||
debug.info("Init with: '" + address_manifest + "' branch='" + branch + "' name of manifest='" + manifest_name + "'")
|
||||
|
||||
|
||||
# check if .XXX exist (create it if needed)
|
||||
base_path = os.path.join(tools.get_run_path(), "." + env.get_system_base_name())
|
||||
base_config = os.path.join(base_path, "config.txt")
|
||||
base_manifest_repo = os.path.join(base_path, "manifest")
|
||||
if os.path.exists(base_path) == True \
|
||||
and os.path.exists(base_config) == True \
|
||||
and os.path.exists(base_manifest_repo) == True:
|
||||
debug.error("System already init: path already exist: '" + str(base_path) + "'")
|
||||
tools.create_directory(base_path)
|
||||
# check if the git of the manifest if availlable
|
||||
|
||||
# create the file configuration:
|
||||
data = "repo=" + address_manifest + "\nbranch=" + branch + "\nfile=" + manifest_name
|
||||
tools.file_write_data(base_config, data)
|
||||
|
||||
#clone the manifest repository
|
||||
cmd = "git clone " + address_manifest + " --branch " + branch + " " + base_manifest_repo
|
||||
|
||||
debug.info("clone the manifest")
|
||||
ret = multiprocess.run_command_direct(cmd)
|
||||
|
||||
if ret == "":
|
||||
return True
|
||||
|
||||
if ret == False:
|
||||
# all is good, ready to get the system work corectly
|
||||
return True
|
||||
debug.info("'" + ret + "'")
|
||||
debug.error("Init does not work")
|
||||
return False
|
||||
|
||||
|
||||
0
maestro/actions/status.py
Normal file
0
maestro/actions/status.py
Normal file
76
maestro/actions/sync.py
Normal file
76
maestro/actions/sync.py
Normal file
@ -0,0 +1,76 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
##
|
||||
## @author Edouard DUPIN
|
||||
##
|
||||
## @copyright 2012, Edouard DUPIN, all right reserved
|
||||
##
|
||||
## @license MPL v2.0 (see license file)
|
||||
##
|
||||
|
||||
from maestro import debug
|
||||
from maestro import tools
|
||||
from maestro import env
|
||||
from maestro import multiprocess
|
||||
import os
|
||||
from lxml import etree
|
||||
|
||||
|
||||
def help():
|
||||
return "plop"
|
||||
|
||||
def load_manifest(file):
|
||||
tree = etree.parse(file)
|
||||
debug.info("manifest:")
|
||||
root = tree.getroot()
|
||||
if root.tag != "manifest":
|
||||
debug.error("in '" + str(file) + "' have not main xml node='manifest'")
|
||||
for child in root:
|
||||
if type(child) == etree._Comment:
|
||||
debug.info(" comment='" + str(child.text) + "'");
|
||||
else:
|
||||
debug.info(" '" + str(child.tag) + "' values=" + str(child.attrib));
|
||||
# inside data child.text
|
||||
return "";
|
||||
|
||||
|
||||
def execute(arguments):
|
||||
debug.info("execute:")
|
||||
for elem in arguments:
|
||||
debug.info(" '" + str(elem.get_arg()) + "'")
|
||||
if len(arguments) != 0:
|
||||
debug.error("Sync have not parameter")
|
||||
|
||||
# check if .XXX exist (create it if needed)
|
||||
base_path = os.path.join(tools.get_run_path(), "." + env.get_system_base_name())
|
||||
base_config = os.path.join(base_path, "config.txt")
|
||||
base_manifest_repo = os.path.join(base_path, "manifest")
|
||||
if os.path.exists(base_path) == False \
|
||||
or os.path.exists(base_config) == False \
|
||||
or os.path.exists(base_manifest_repo) == False:
|
||||
debug.error("System already init have an error: missing data: '" + str(base_path) + "'")
|
||||
|
||||
config_property = tools.file_read_data(base_config)
|
||||
|
||||
element_config = config_property.split("\n")
|
||||
if len(element_config) != 3:
|
||||
debug.error("error in configuration property")
|
||||
if element_config[0][:5] != "repo=":
|
||||
debug.error("error in configuration property (2)")
|
||||
if element_config[1][:7] != "branch=":
|
||||
debug.error("error in configuration property (3)")
|
||||
if element_config[2][:5] != "file=":
|
||||
debug.error("error in configuration property (4)")
|
||||
configuration = {
|
||||
"repo":element_config[0][5:],
|
||||
"branch":element_config[1][7:],
|
||||
"file":element_config[2][5:]
|
||||
}
|
||||
debug.info("configuration property: " + str(configuration))
|
||||
|
||||
file_source_manifest = os.path.join(base_manifest_repo, configuration["file"])
|
||||
if os.path.exists(file_source_manifest) == False:
|
||||
debug.error("Missing manifest file : '" + str(file_source_manifest) + "'")
|
||||
|
||||
manifest = load_manifest(file_source_manifest)
|
||||
|
||||
@ -12,10 +12,6 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
import sys
|
||||
if sys.version_info >= (3, 0):
|
||||
import queue
|
||||
else:
|
||||
import Queue as queue
|
||||
import os
|
||||
import subprocess
|
||||
import shlex
|
||||
@ -23,43 +19,7 @@ import shlex
|
||||
from . import debug
|
||||
from . import tools
|
||||
from . import env
|
||||
from . import depend
|
||||
|
||||
queue_lock = threading.Lock()
|
||||
work_queue = queue.Queue()
|
||||
current_thread_working = 0
|
||||
threads = []
|
||||
# To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation
|
||||
current_id_execution = 0
|
||||
error_execution = {
|
||||
"id":-1,
|
||||
"cmd":"",
|
||||
"return":0,
|
||||
"err":"",
|
||||
"out":"",
|
||||
}
|
||||
|
||||
exit_flag = False # resuest stop of the thread
|
||||
is_init = False # the thread are initialized
|
||||
error_occured = False # a thread have an error
|
||||
processor_availlable = 1 # number of CPU core availlable
|
||||
##
|
||||
## @brief Execute the command with no get of output
|
||||
##
|
||||
def run_command_no_lock_out(cmd_line):
|
||||
# prepare command line:
|
||||
args = shlex.split(cmd_line)
|
||||
debug.info("cmd = " + str(args))
|
||||
try:
|
||||
# create the subprocess
|
||||
p = subprocess.Popen(args)
|
||||
except subprocess.CalledProcessError as e:
|
||||
debug.error("subprocess.CalledProcessError : " + str(args))
|
||||
return
|
||||
#except:
|
||||
# debug.error("Exception on : " + str(args))
|
||||
# launch the subprocess:
|
||||
p.communicate()
|
||||
|
||||
##
|
||||
## @brief Execute the command and ruturn generate data
|
||||
@ -89,215 +49,3 @@ def run_command_direct(cmd_line):
|
||||
return False
|
||||
|
||||
|
||||
def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_file="", depend_data=None):
|
||||
global error_occured
|
||||
global exit_flag
|
||||
global current_id_execution
|
||||
global error_execution
|
||||
# prepare command line:
|
||||
args = shlex.split(cmd_line)
|
||||
debug.verbose("cmd = " + str(args))
|
||||
try:
|
||||
# create the subprocess
|
||||
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
except subprocess.CalledProcessError as e:
|
||||
debug.error("subprocess.CalledProcessError : TODO ...")
|
||||
except:
|
||||
debug.error("Exception on : " + str(args))
|
||||
# launch the subprocess:
|
||||
output, err = p.communicate()
|
||||
if sys.version_info >= (3, 0):
|
||||
output = output.decode("utf-8")
|
||||
err = err.decode("utf-8")
|
||||
# store error if needed:
|
||||
tools.store_warning(store_output_file, output, err)
|
||||
# Check error :
|
||||
if p.returncode == 0:
|
||||
debug.debug(env.print_pretty(cmd_line))
|
||||
queue_lock.acquire()
|
||||
if depend_data != None:
|
||||
depend.create_dependency_file(depend_data['file'], depend_data['data'])
|
||||
# TODO : Print the output all the time .... ==> to show warnings ...
|
||||
if build_id >= 0 and (output != "" or err != ""):
|
||||
debug.warning("output in subprocess compiling: '" + file + "'")
|
||||
if output != "":
|
||||
debug.print_compilator(output)
|
||||
if err != "":
|
||||
debug.print_compilator(err)
|
||||
queue_lock.release()
|
||||
else:
|
||||
error_occured = True
|
||||
exit_flag = True
|
||||
# if No ID : Not in a multiprocess mode ==> just stop here
|
||||
if build_id < 0:
|
||||
debug.debug(env.print_pretty(cmd_line), force=True)
|
||||
debug.print_compilator(output)
|
||||
debug.print_compilator(err)
|
||||
if p.returncode == 2:
|
||||
debug.error("can not compile file ... [keyboard interrrupt]")
|
||||
else:
|
||||
debug.error("can not compile file ... ret : " + str(p.returncode))
|
||||
else:
|
||||
# in multiprocess interface
|
||||
queue_lock.acquire()
|
||||
# if an other write an error before, check if the current process is started before ==> then is the first error
|
||||
if error_execution["id"] >= build_id:
|
||||
# nothing to do ...
|
||||
queue_lock.release()
|
||||
return;
|
||||
error_execution["id"] = build_id
|
||||
error_execution["cmd"] = cmd_line
|
||||
error_execution["return"] = p.returncode
|
||||
error_execution["err"] = err,
|
||||
error_execution["out"] = output,
|
||||
queue_lock.release()
|
||||
# not write the command file...
|
||||
return
|
||||
debug.verbose("done 3")
|
||||
# write cmd line only after to prevent errors ...
|
||||
tools.store_command(cmd_line, store_cmd_line)
|
||||
|
||||
|
||||
|
||||
class myThread(threading.Thread):
|
||||
def __init__(self, thread_id, lock, queue):
|
||||
threading.Thread.__init__(self)
|
||||
self.thread_id = thread_id
|
||||
self.name = "Thread " + str(thread_id)
|
||||
self.queue = queue
|
||||
self.lock = lock
|
||||
def run(self):
|
||||
debug.verbose("Starting " + self.name)
|
||||
global exit_flag
|
||||
global current_thread_working
|
||||
working_set = False
|
||||
while exit_flag == False:
|
||||
self.lock.acquire()
|
||||
if not self.queue.empty():
|
||||
if working_set == False:
|
||||
current_thread_working += 1
|
||||
working_set = True
|
||||
data = self.queue.get()
|
||||
self.lock.release()
|
||||
debug.verbose(self.name + " processing '" + data[0] + "'")
|
||||
if data[0]=="cmd_line":
|
||||
comment = data[2]
|
||||
cmd_line = data[1]
|
||||
cmd_store_file = data[3]
|
||||
debug.print_element("[" + str(data[4]) + "][" + str(self.thread_id) + "] " + comment[0],
|
||||
comment[1],
|
||||
comment[2],
|
||||
comment[3])
|
||||
run_command(cmd_line,
|
||||
cmd_store_file,
|
||||
build_id=data[4],
|
||||
file=comment[3],
|
||||
store_output_file=data[5],
|
||||
depend_data=data[6])
|
||||
else:
|
||||
debug.warning("unknow request command : " + data[0])
|
||||
else:
|
||||
if working_set==True:
|
||||
current_thread_working -= 1
|
||||
working_set=False
|
||||
# no element to parse, just wait ...
|
||||
self.lock.release()
|
||||
time.sleep(0.2)
|
||||
# kill requested ...
|
||||
debug.verbose("Exiting " + self.name)
|
||||
|
||||
|
||||
def set_error_occured():
|
||||
global exit_flag
|
||||
exit_flag = True
|
||||
|
||||
def set_core_number(number_of_core):
|
||||
global processor_availlable
|
||||
processor_availlable = number_of_core
|
||||
debug.debug(" set number of core for multi process compilation : " + str(processor_availlable))
|
||||
# nothing else to do
|
||||
|
||||
def init():
|
||||
global error_occured
|
||||
global exit_flag
|
||||
global is_init
|
||||
if is_init == False:
|
||||
is_init = True
|
||||
error_occured = False
|
||||
global threads
|
||||
global queue_lock
|
||||
global work_queue
|
||||
# Create all the new threads
|
||||
thread_id = 0
|
||||
while thread_id < processor_availlable:
|
||||
thread = myThread(thread_id, queue_lock, work_queue)
|
||||
thread.start()
|
||||
threads.append(thread)
|
||||
thread_id += 1
|
||||
|
||||
|
||||
|
||||
def un_init():
|
||||
global exit_flag
|
||||
# Notify threads it's time to exit
|
||||
exit_flag = True
|
||||
if processor_availlable > 1:
|
||||
# Wait for all threads to complete
|
||||
for tmp in threads:
|
||||
debug.verbose("join thread ...")
|
||||
tmp.join()
|
||||
debug.verbose("Exiting ALL Threads")
|
||||
|
||||
|
||||
|
||||
def run_in_pool(cmd_line, comment, store_cmd_line="", store_output_file="", depend_data=None):
|
||||
global current_id_execution
|
||||
if processor_availlable <= 1:
|
||||
debug.print_element(comment[0], comment[1], comment[2], comment[3])
|
||||
run_command(cmd_line, store_cmd_line, file=comment[3], store_output_file=store_output_file, depend_data=depend_data)
|
||||
return
|
||||
# multithreaded mode
|
||||
init()
|
||||
# Fill the queue
|
||||
queue_lock.acquire()
|
||||
debug.verbose("add : in pool cmd_line")
|
||||
work_queue.put(["cmd_line", cmd_line, comment, store_cmd_line, current_id_execution, store_output_file, depend_data])
|
||||
current_id_execution +=1;
|
||||
queue_lock.release()
|
||||
|
||||
|
||||
def pool_synchrosize():
|
||||
global error_occured
|
||||
global error_execution
|
||||
if processor_availlable <= 1:
|
||||
#in this case : nothing to synchronise
|
||||
return
|
||||
|
||||
debug.verbose("wait queue process ended\n")
|
||||
# Wait for queue to empty
|
||||
while not work_queue.empty() \
|
||||
and error_occured == False:
|
||||
time.sleep(0.2)
|
||||
pass
|
||||
# Wait all thread have ended their current process
|
||||
while current_thread_working != 0 \
|
||||
and error_occured == False:
|
||||
time.sleep(0.2)
|
||||
pass
|
||||
if error_occured == False:
|
||||
debug.verbose("queue is empty")
|
||||
else:
|
||||
un_init()
|
||||
debug.debug("Thread return with error ... ==> stop all the pool")
|
||||
if error_execution["id"] == -1:
|
||||
debug.error("Pool error occured ... (No return information on Pool)")
|
||||
return
|
||||
debug.error("Error in an pool element : [" + str(error_execution["id"]) + "]", crash=False)
|
||||
debug.debug(env.print_pretty(error_execution["cmd"]), force=True)
|
||||
debug.print_compilator(str(error_execution["out"][0]))
|
||||
debug.print_compilator(str(error_execution["err"][0]))
|
||||
if error_execution["return"] == 2:
|
||||
debug.error("can not compile file ... [keyboard interrrupt]")
|
||||
else:
|
||||
debug.error("can not compile file ... return value : " + str(error_execution["return"]))
|
||||
|
||||
|
||||
@ -29,13 +29,18 @@ def get_run_path():
|
||||
def get_current_path(file):
|
||||
return os.path.dirname(os.path.realpath(file))
|
||||
|
||||
def create_directory_of_file(file):
|
||||
path = os.path.dirname(file)
|
||||
|
||||
def create_directory(path):
|
||||
try:
|
||||
os.stat(path)
|
||||
except:
|
||||
os.makedirs(path)
|
||||
|
||||
def create_directory_of_file(file):
|
||||
path = os.path.dirname(file)
|
||||
create_directory(path)
|
||||
|
||||
|
||||
def get_list_sub_path(path):
|
||||
# TODO : os.listdir(path)
|
||||
for dirname, dirnames, filenames in os.walk(path):
|
||||
|
||||
3
setup.py
3
setup.py
@ -23,7 +23,8 @@ setup(name='maestro',
|
||||
author='Edouard DUPIN',
|
||||
author_email='yui.heero@gmail.com',
|
||||
license='MPL-2',
|
||||
packages=['maestro'],
|
||||
packages=['maestro',
|
||||
'maestro/actions'],
|
||||
classifiers=[
|
||||
'Development Status :: 2 - Pre-Alpha',
|
||||
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user