diff --git a/bin/maestro b/bin/maestro index be1f338..0d37f2c 100755 --- a/bin/maestro +++ b/bin/maestro @@ -20,12 +20,17 @@ import maestro.env as env import maestro.tools as tools import maestro.host as maestroHost import maestro.tools as maestroTools +import maestro.actions as actions myArgs = arguments.maestroArg() myArgs.add_section("option", "Can be set one time in all case") myArgs.add("h", "help", desc="Display this help") myArgs.add("v", "verbose", list=[["0","None"],["1","error"],["2","warning"],["3","info"],["4","debug"],["5","verbose"],["6","extreme_verbose"]], desc="display debug level (verbose) default =2") myArgs.add("c", "color", desc="Display message in color") +# for init only +#myArgs.add("h", "help", desc="Help of this action") +myArgs.add("b", "branch", haveParam=True, desc="Select branch to display") +myArgs.add("m", "manifest", haveParam=True, desc="Name of the manifest") """ myArgs.add("j", "jobs", haveParam=True, desc="Specifies the number of jobs (commands) to run simultaneously") myArgs.add("d", "depth", haveParam=True, desc="Depth to clone all the repository") @@ -42,12 +47,17 @@ def usage(): # generic argument displayed : myArgs.display() print(" Action availlable" ) + list_actions = actions.get_list_of_action(); + for elem in list_actions: + print(" " + color['green'] + elem + color['default']) + """ print(" " + color['green'] + "init" + color['default']) print(" initialize a 'maestro' interface with a manifest in a git ") print(" " + color['green'] + "sync" + color['default']) print(" Syncronise the currect environement") print(" " + color['green'] + "status" + color['default']) print(" Dump the status of the environement") + """ print(" ex: " + sys.argv[0] + " -c init http://github.com/atria-soft/manifest.git") print(" ex: " + sys.argv[0] + " sync") exit(0) @@ -163,32 +173,26 @@ if len(new_argument_list) == 0: debug.warning("--------------------------------------") usage() -list_of_action_availlable=["init","sync","status"] + +# TODO : move tin in actions ... +list_actions = actions.get_list_of_action(); action_to_do = new_argument_list[0].get_arg() new_argument_list = new_argument_list[1:] -if action_to_do not in list_of_action_availlable: +if action_to_do not in list_actions: debug.warning("--------------------------------------") - debug.warning("Wrong action type : '" + str(action_to_do) + "' availlable list: " + str(list_of_action_availlable) ) + debug.warning("Wrong action type : '" + str(action_to_do) + "' availlable list: " + str(list_actions) ) debug.warning("--------------------------------------") usage() +# todo : Remove this if action_to_do != "init" \ and os.path.exists("." + env.get_system_base_name()) == False: debug.error("Can not execute a maestro cmd if we have not initialize a config: '" + str("." + env.get_system_base_name()) + "'") exit(-1) -if action_to_do == "init": - debug.info("action: init"); - -elif action_to_do == "sync": - debug.info("action: sync"); - -elif action_to_do == "status": - debug.info("action: status"); - -else: - debug.error("Can not do the action...") + +actions.execute(action_to_do, new_argument_list) # stop all started threads; #multiprocess.un_init() diff --git a/maestro/__init__.py b/maestro/__init__.py index 51049f0..058f657 100755 --- a/maestro/__init__.py +++ b/maestro/__init__.py @@ -15,9 +15,10 @@ from . import host from . import tools from . import debug from . import env +from . import actions is_init = False -""" + def filter_name_and_file(root, list_files, filter): # filter elements: tmp_list = fnmatch.filter(list_files, filter) @@ -27,128 +28,28 @@ def filter_name_and_file(root, list_files, filter): out.append(elem); return out; -def filter_path(root, list_files): - out = [] - for elem in list_files: - if len(elem) == 0 \ - or elem[0] == '.': - continue - if os.path.isdir(os.path.join(root, elem)) == True: - out.append(elem); - return out; - -def import_path_local(path, limit_sub_folder, exclude_path = [], base_name = ""): +def import_path_local(path): out = [] debug.verbose("maestro files: " + str(path) + " [START]") - if limit_sub_folder == 0: - debug.debug("Subparsing limitation append ...") - return [] - try: - list_files = os.listdir(path) - except: - # an error occure, maybe read error ... - debug.warning("error when getting subdirectory of '" + str(path) + "'") - return [] - if path in exclude_path: - debug.debug("find '" + str(path) + "' in exclude_path=" + str(exclude_path)) - return [] + list_files = os.listdir(path) # filter elements: - tmp_list_maestro_file = filter_name_and_file(path, list_files, base_name + "*.py") + tmp_list_maestro_file = filter_name_and_file(path, list_files, "*.py") debug.verbose("maestro files: " + str(path) + " : " + str(tmp_list_maestro_file)) # Import the module: for filename in tmp_list_maestro_file: out.append(os.path.join(path, filename)) - debug.extreme_verbose(" Find a file : '" + str(out[-1]) + "'") - need_parse_sub_folder = True - rm_value = -1 - # check if we need to parse sub_folder - if len(tmp_list_maestro_file) != 0: - need_parse_sub_folder = False - # check if the file "maestro_parse_sub.py" is present ==> parse SubFolder (force and add +1 in the resursing - if base_name + "ParseSubFolders.txt" in list_files: - debug.debug("find SubParser ... " + str(base_name + "ParseSubFolders.txt") + " " + path) - data_file_sub = tools.file_read_data(os.path.join(path, base_name + "ParseSubFolders.txt")) - if data_file_sub == "": - debug.debug(" Empty file Load all subfolder in the worktree in '" + str(path) + "'") - need_parse_sub_folder = True - rm_value = 0 - else: - list_sub = data_file_sub.split("\n") - debug.debug(" Parse selected folders " + str(list_sub) + " no parse local folder directory") - need_parse_sub_folder = False - for folder in list_sub: - if folder == "" \ - or folder == "/": - continue; - tmp_out = import_path_local(os.path.join(path, folder), - 1, - exclude_path, - base_name) - # add all the elements: - for elem in tmp_out: - out.append(elem) - if need_parse_sub_folder == True: - list_folders = filter_path(path, list_files) - for folder in list_folders: - tmp_out = import_path_local(os.path.join(path, folder), - limit_sub_folder - rm_value, - exclude_path, - base_name) - # add all the elements: - for elem in tmp_out: - out.append(elem) + debug.verbose(" Find a file : '" + str(out[-1]) + "'") return out -""" + def init(): global is_init; if is_init == True: return - """ - debug.verbose("Use Make as a make stadard") - sys.path.append(tools.get_run_path()) - # create the list of basic folder: - basic_folder_list = [] - basic_folder_list.append([tools.get_current_path(__file__), True]) - # Import all sub path without out and archive - for elem_path in os.listdir("."): - if os.path.isdir(elem_path) == False: - continue - if elem_path.lower() == "android" \ - or elem_path == "out" : - continue - debug.debug("Automatic load path: '" + elem_path + "'") - basic_folder_list.append([elem_path, False]) + list_of_maestro_files = import_path_local(os.path.join(tools.get_current_path(__file__), 'actions')) - # create in a single path the basic list of maestro files (all start with maestro and end with .py) - exclude_path = env.get_exclude_search_path() - limit_sub_folder = env.get_parse_depth() - list_of_maestro_files = [] - for elem_path, is_system in basic_folder_list: - if is_system == True: - limit_sub_folder_tmp = 999999 - else: - limit_sub_folder_tmp = limit_sub_folder - tmp_out = import_path_local(elem_path, - limit_sub_folder_tmp, - exclude_path, - env.get_build_system_base_name()) - # add all the elements: - for elem in tmp_out: - list_of_maestro_files.append(elem) + actions.init(list_of_maestro_files) - debug.debug("Files specific maestro: ") - for elem_path in list_of_maestro_files: - debug.debug(" " + elem_path) - # simply import element from the basic list of files (single parse ...) - builder.import_path(list_of_maestro_files) - module.import_path(list_of_maestro_files) - system.import_path(list_of_maestro_files) - target.import_path(list_of_maestro_files) - macro.import_path(list_of_maestro_files) - - builder.init() - """ is_init = True diff --git a/maestro/actions.py b/maestro/actions.py new file mode 100644 index 0000000..fce8c04 --- /dev/null +++ b/maestro/actions.py @@ -0,0 +1,53 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2012, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +# Local import +from . import debug +import os +import sys + +list_actions = [] + +def init(files): + global list_actions; + debug.debug("List of action for maestro: ") + for elem_path in files : + debug.debug(" '" + os.path.basename(elem_path)[:-3] + "' file=" + elem_path) + list_actions.append({ + "name":os.path.basename(elem_path)[:-3], + "path":elem_path, + }) + + +def get_list_of_action(): + global list_actions; + out = [] + for elem in list_actions: + out.append(elem["name"]) + return out + + + +def execute(action_to_do, argument_list): + global list_actions; + # TODO: Move here the check if action is availlable + + for elem in list_actions: + if elem["name"] == action_to_do: + debug.info("action: " + str(elem)); + # finish the parsing + sys.path.append(os.path.dirname(elem["path"])) + the_action = __import__(action_to_do) + if "execute" not in dir(the_action): + debug.error("execute is not implmented for this action ... '" + str(action_to_do) + "'") + return False + return the_action.execute(argument_list) + debug.error("Can not do the action...") + return False diff --git a/maestro/actions/init.py b/maestro/actions/init.py new file mode 100644 index 0000000..a07fdd3 --- /dev/null +++ b/maestro/actions/init.py @@ -0,0 +1,82 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2012, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +from maestro import debug +from maestro import tools +from maestro import env +from maestro import multiprocess +import os + +def help(): + return "plop" + +def execute(arguments): + debug.info("execute:") + for elem in arguments: + debug.info(" '" + str(elem.get_arg()) + "'") + if len(arguments) == 0: + debug.error("Missing argument to execute the current action ...") + + # the configuration availlable: + branch = "master" + manifest_name = "default.xml" + address_manifest = "" + for elem in arguments: + if elem.get_option_name() == "branch": + debug.info("find branch name: '" + elem.get_arg() + "'") + branch = elem.get_arg() + elif elem.get_option_name() == "manifest": + debug.info("find mmanifest name: '" + elem.get_arg() + "'") + manifest_name = elem.get_arg() + elif elem.get_option_name() == "": + if address_manifest != "": + debug.error("Manifest adress already set : '" + address_manifest + "' !!! '" + elem.get_arg() + "'") + address_manifest = elem.get_arg() + else: + debug.error("Wrong argument: '" + elem.get_option_name() + "' '" + elem.get_arg() + "'") + + if address_manifest == "": + debug.error("Init: Missing manifest name") + + debug.info("Init with: '" + address_manifest + "' branch='" + branch + "' name of manifest='" + manifest_name + "'") + + + # check if .XXX exist (create it if needed) + base_path = os.path.join(tools.get_run_path(), "." + env.get_system_base_name()) + base_config = os.path.join(base_path, "config.txt") + base_manifest_repo = os.path.join(base_path, "manifest") + if os.path.exists(base_path) == True \ + and os.path.exists(base_config) == True \ + and os.path.exists(base_manifest_repo) == True: + debug.error("System already init: path already exist: '" + str(base_path) + "'") + tools.create_directory(base_path) + # check if the git of the manifest if availlable + + # create the file configuration: + data = "repo=" + address_manifest + "\nbranch=" + branch + "\nfile=" + manifest_name + tools.file_write_data(base_config, data) + + #clone the manifest repository + cmd = "git clone " + address_manifest + " --branch " + branch + " " + base_manifest_repo + + debug.info("clone the manifest") + ret = multiprocess.run_command_direct(cmd) + + if ret == "": + return True + + if ret == False: + # all is good, ready to get the system work corectly + return True + debug.info("'" + ret + "'") + debug.error("Init does not work") + return False + + diff --git a/maestro/actions/status.py b/maestro/actions/status.py new file mode 100644 index 0000000..e69de29 diff --git a/maestro/actions/sync.py b/maestro/actions/sync.py new file mode 100644 index 0000000..e18a917 --- /dev/null +++ b/maestro/actions/sync.py @@ -0,0 +1,76 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2012, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +from maestro import debug +from maestro import tools +from maestro import env +from maestro import multiprocess +import os +from lxml import etree + + +def help(): + return "plop" + +def load_manifest(file): + tree = etree.parse(file) + debug.info("manifest:") + root = tree.getroot() + if root.tag != "manifest": + debug.error("in '" + str(file) + "' have not main xml node='manifest'") + for child in root: + if type(child) == etree._Comment: + debug.info(" comment='" + str(child.text) + "'"); + else: + debug.info(" '" + str(child.tag) + "' values=" + str(child.attrib)); + # inside data child.text + return ""; + + +def execute(arguments): + debug.info("execute:") + for elem in arguments: + debug.info(" '" + str(elem.get_arg()) + "'") + if len(arguments) != 0: + debug.error("Sync have not parameter") + + # check if .XXX exist (create it if needed) + base_path = os.path.join(tools.get_run_path(), "." + env.get_system_base_name()) + base_config = os.path.join(base_path, "config.txt") + base_manifest_repo = os.path.join(base_path, "manifest") + if os.path.exists(base_path) == False \ + or os.path.exists(base_config) == False \ + or os.path.exists(base_manifest_repo) == False: + debug.error("System already init have an error: missing data: '" + str(base_path) + "'") + + config_property = tools.file_read_data(base_config) + + element_config = config_property.split("\n") + if len(element_config) != 3: + debug.error("error in configuration property") + if element_config[0][:5] != "repo=": + debug.error("error in configuration property (2)") + if element_config[1][:7] != "branch=": + debug.error("error in configuration property (3)") + if element_config[2][:5] != "file=": + debug.error("error in configuration property (4)") + configuration = { + "repo":element_config[0][5:], + "branch":element_config[1][7:], + "file":element_config[2][5:] + } + debug.info("configuration property: " + str(configuration)) + + file_source_manifest = os.path.join(base_manifest_repo, configuration["file"]) + if os.path.exists(file_source_manifest) == False: + debug.error("Missing manifest file : '" + str(file_source_manifest) + "'") + + manifest = load_manifest(file_source_manifest) + \ No newline at end of file diff --git a/maestro/multiprocess.py b/maestro/multiprocess.py index 79b3af5..465853c 100644 --- a/maestro/multiprocess.py +++ b/maestro/multiprocess.py @@ -12,10 +12,6 @@ import sys import threading import time import sys -if sys.version_info >= (3, 0): - import queue -else: - import Queue as queue import os import subprocess import shlex @@ -23,43 +19,7 @@ import shlex from . import debug from . import tools from . import env -from . import depend -queue_lock = threading.Lock() -work_queue = queue.Queue() -current_thread_working = 0 -threads = [] -# To know the first error arrive in the pool ==> to display all the time the same error file when multiple compilation -current_id_execution = 0 -error_execution = { - "id":-1, - "cmd":"", - "return":0, - "err":"", - "out":"", -} - -exit_flag = False # resuest stop of the thread -is_init = False # the thread are initialized -error_occured = False # a thread have an error -processor_availlable = 1 # number of CPU core availlable -## -## @brief Execute the command with no get of output -## -def run_command_no_lock_out(cmd_line): - # prepare command line: - args = shlex.split(cmd_line) - debug.info("cmd = " + str(args)) - try: - # create the subprocess - p = subprocess.Popen(args) - except subprocess.CalledProcessError as e: - debug.error("subprocess.CalledProcessError : " + str(args)) - return - #except: - # debug.error("Exception on : " + str(args)) - # launch the subprocess: - p.communicate() ## ## @brief Execute the command and ruturn generate data @@ -89,215 +49,3 @@ def run_command_direct(cmd_line): return False -def run_command(cmd_line, store_cmd_line="", build_id=-1, file="", store_output_file="", depend_data=None): - global error_occured - global exit_flag - global current_id_execution - global error_execution - # prepare command line: - args = shlex.split(cmd_line) - debug.verbose("cmd = " + str(args)) - try: - # create the subprocess - p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except subprocess.CalledProcessError as e: - debug.error("subprocess.CalledProcessError : TODO ...") - except: - debug.error("Exception on : " + str(args)) - # launch the subprocess: - output, err = p.communicate() - if sys.version_info >= (3, 0): - output = output.decode("utf-8") - err = err.decode("utf-8") - # store error if needed: - tools.store_warning(store_output_file, output, err) - # Check error : - if p.returncode == 0: - debug.debug(env.print_pretty(cmd_line)) - queue_lock.acquire() - if depend_data != None: - depend.create_dependency_file(depend_data['file'], depend_data['data']) - # TODO : Print the output all the time .... ==> to show warnings ... - if build_id >= 0 and (output != "" or err != ""): - debug.warning("output in subprocess compiling: '" + file + "'") - if output != "": - debug.print_compilator(output) - if err != "": - debug.print_compilator(err) - queue_lock.release() - else: - error_occured = True - exit_flag = True - # if No ID : Not in a multiprocess mode ==> just stop here - if build_id < 0: - debug.debug(env.print_pretty(cmd_line), force=True) - debug.print_compilator(output) - debug.print_compilator(err) - if p.returncode == 2: - debug.error("can not compile file ... [keyboard interrrupt]") - else: - debug.error("can not compile file ... ret : " + str(p.returncode)) - else: - # in multiprocess interface - queue_lock.acquire() - # if an other write an error before, check if the current process is started before ==> then is the first error - if error_execution["id"] >= build_id: - # nothing to do ... - queue_lock.release() - return; - error_execution["id"] = build_id - error_execution["cmd"] = cmd_line - error_execution["return"] = p.returncode - error_execution["err"] = err, - error_execution["out"] = output, - queue_lock.release() - # not write the command file... - return - debug.verbose("done 3") - # write cmd line only after to prevent errors ... - tools.store_command(cmd_line, store_cmd_line) - - - -class myThread(threading.Thread): - def __init__(self, thread_id, lock, queue): - threading.Thread.__init__(self) - self.thread_id = thread_id - self.name = "Thread " + str(thread_id) - self.queue = queue - self.lock = lock - def run(self): - debug.verbose("Starting " + self.name) - global exit_flag - global current_thread_working - working_set = False - while exit_flag == False: - self.lock.acquire() - if not self.queue.empty(): - if working_set == False: - current_thread_working += 1 - working_set = True - data = self.queue.get() - self.lock.release() - debug.verbose(self.name + " processing '" + data[0] + "'") - if data[0]=="cmd_line": - comment = data[2] - cmd_line = data[1] - cmd_store_file = data[3] - debug.print_element("[" + str(data[4]) + "][" + str(self.thread_id) + "] " + comment[0], - comment[1], - comment[2], - comment[3]) - run_command(cmd_line, - cmd_store_file, - build_id=data[4], - file=comment[3], - store_output_file=data[5], - depend_data=data[6]) - else: - debug.warning("unknow request command : " + data[0]) - else: - if working_set==True: - current_thread_working -= 1 - working_set=False - # no element to parse, just wait ... - self.lock.release() - time.sleep(0.2) - # kill requested ... - debug.verbose("Exiting " + self.name) - - -def set_error_occured(): - global exit_flag - exit_flag = True - -def set_core_number(number_of_core): - global processor_availlable - processor_availlable = number_of_core - debug.debug(" set number of core for multi process compilation : " + str(processor_availlable)) - # nothing else to do - -def init(): - global error_occured - global exit_flag - global is_init - if is_init == False: - is_init = True - error_occured = False - global threads - global queue_lock - global work_queue - # Create all the new threads - thread_id = 0 - while thread_id < processor_availlable: - thread = myThread(thread_id, queue_lock, work_queue) - thread.start() - threads.append(thread) - thread_id += 1 - - - -def un_init(): - global exit_flag - # Notify threads it's time to exit - exit_flag = True - if processor_availlable > 1: - # Wait for all threads to complete - for tmp in threads: - debug.verbose("join thread ...") - tmp.join() - debug.verbose("Exiting ALL Threads") - - - -def run_in_pool(cmd_line, comment, store_cmd_line="", store_output_file="", depend_data=None): - global current_id_execution - if processor_availlable <= 1: - debug.print_element(comment[0], comment[1], comment[2], comment[3]) - run_command(cmd_line, store_cmd_line, file=comment[3], store_output_file=store_output_file, depend_data=depend_data) - return - # multithreaded mode - init() - # Fill the queue - queue_lock.acquire() - debug.verbose("add : in pool cmd_line") - work_queue.put(["cmd_line", cmd_line, comment, store_cmd_line, current_id_execution, store_output_file, depend_data]) - current_id_execution +=1; - queue_lock.release() - - -def pool_synchrosize(): - global error_occured - global error_execution - if processor_availlable <= 1: - #in this case : nothing to synchronise - return - - debug.verbose("wait queue process ended\n") - # Wait for queue to empty - while not work_queue.empty() \ - and error_occured == False: - time.sleep(0.2) - pass - # Wait all thread have ended their current process - while current_thread_working != 0 \ - and error_occured == False: - time.sleep(0.2) - pass - if error_occured == False: - debug.verbose("queue is empty") - else: - un_init() - debug.debug("Thread return with error ... ==> stop all the pool") - if error_execution["id"] == -1: - debug.error("Pool error occured ... (No return information on Pool)") - return - debug.error("Error in an pool element : [" + str(error_execution["id"]) + "]", crash=False) - debug.debug(env.print_pretty(error_execution["cmd"]), force=True) - debug.print_compilator(str(error_execution["out"][0])) - debug.print_compilator(str(error_execution["err"][0])) - if error_execution["return"] == 2: - debug.error("can not compile file ... [keyboard interrrupt]") - else: - debug.error("can not compile file ... return value : " + str(error_execution["return"])) - diff --git a/maestro/tools.py b/maestro/tools.py index 68e6834..2eb3962 100644 --- a/maestro/tools.py +++ b/maestro/tools.py @@ -29,13 +29,18 @@ def get_run_path(): def get_current_path(file): return os.path.dirname(os.path.realpath(file)) -def create_directory_of_file(file): - path = os.path.dirname(file) + +def create_directory(path): try: os.stat(path) except: os.makedirs(path) +def create_directory_of_file(file): + path = os.path.dirname(file) + create_directory(path) + + def get_list_sub_path(path): # TODO : os.listdir(path) for dirname, dirnames, filenames in os.walk(path): diff --git a/setup.py b/setup.py index a076b96..e411bdd 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,8 @@ setup(name='maestro', author='Edouard DUPIN', author_email='yui.heero@gmail.com', license='MPL-2', - packages=['maestro'], + packages=['maestro', + 'maestro/actions'], classifiers=[ 'Development Status :: 2 - Pre-Alpha', 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',