diff --git a/Dockerfile b/Dockerfile index f53645d..85b80b6 100755 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,8 @@ RUN pip install python-dateutil RUN pip install realog +RUN pip install python-magic + EXPOSE 80 ADD src /application/ diff --git a/src/app_video.py b/src/app_video.py index 1d737e4..8901081 100755 --- a/src/app_video.py +++ b/src/app_video.py @@ -23,6 +23,7 @@ import dateutil.parser import time +import json import os import sys import datetime @@ -74,6 +75,7 @@ API_SAISON = "saison" add_interface(API_SAISON) API_VIDEO = "video" add_interface(API_VIDEO) +API_DATA = "data" def add_type(_app, _name_api): elem_blueprint = Blueprint(_name_api) @@ -241,7 +243,8 @@ def add_video(_app, _name_api): class DataModel: type_id = int - saison_id = int + saison = int + episode = int group_id = int name = str description = str @@ -297,41 +300,98 @@ def add_video(_app, _name_api): add_video(app, API_VIDEO) -import hashlib -import shutil - - tmp_value = 0 #curl -F 'file=@Totally_Spies.mp4;type=application/octet-stream' -H 'transfer-encoding:chunked' 127.0.0.1:15080/data -X POST -O; echo ; -@app.post('/data', stream=True) -async def handler(_request): - debug.info("request streaming " + str(_request)); - async def streaming(_response): - debug.info("streaming " + str(_response)); - total_size = 0 - temporary_file = os.path.join(rest_config["tmp_data"], str(tmp_value) + ".tmp") - if not os.path.exists(rest_config["tmp_data"]): - os.makedirs(rest_config["tmp_data"]) - if not os.path.exists(rest_config["data_media"]): - os.makedirs(rest_config["data_media"]) - file_stream = open(temporary_file,"wb") - sha1 = hashlib.sha512() - while True: - body = await _request.stream.read() - if body is None: - debug.warning("empty body"); - break - total_size += len(body) - debug.warning("body " + str(len(body)) + "/" + str(total_size)) - file_stream.write(body) - sha1.update(body) - file_stream.close() - print("SHA512: " + str(sha1.hexdigest())) - await _response.write('{"size":' + str(total_size) + ', "sha512":"' + str(sha1.hexdigest()) + '"}') - shutil.move(temporary_file, os.path.join(rest_config["data_media"], str(sha1.hexdigest()))) - return response.stream(streaming, content_type='application/json') +def add_data(_app, _name_api): + elem_blueprint = Blueprint(_name_api) + """ + @elem_blueprint.get('/' + _name_api, strict_slashes=True) + @doc.summary("Show saisons") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def list(request): + return response.json(data_global_elements.get_interface(_name_api).gets()) + """ + + @elem_blueprint.post('/' + _name_api, strict_slashes=True, stream=True) + @doc.summary("send new file data") + @doc.description("Create a new data file (associated with his sha512.") + #@doc.consumes(DataModel, location='body')#, required=True) + @doc.response_success(status=201, description='If successful created') + async def create(_request): + debug.info("request streaming " + str(_request)); + args_with_blank_values = _request.headers + debug.info("List arguments: " + str(args_with_blank_values)); + async def streaming(_response): + debug.info("streaming " + str(_response)); + total_size = 0 + temporary_file = os.path.join(rest_config["tmp_data"], str(tmp_value) + ".tmp") + if not os.path.exists(rest_config["tmp_data"]): + os.makedirs(rest_config["tmp_data"]) + if not os.path.exists(rest_config["data_media"]): + os.makedirs(rest_config["data_media"]) + file_stream = open(temporary_file,"wb") + sha1 = hashlib.sha512() + while True: + body = await _request.stream.read() + if body is None: + debug.warning("empty body"); + break + total_size += len(body) + debug.verbose("body " + str(len(body)) + "/" + str(total_size)) + file_stream.write(body) + sha1.update(body) + file_stream.close() + print("SHA512: " + str(sha1.hexdigest())) + destination_filename = os.path.join(rest_config["data_media"], str(sha1.hexdigest())) + if os.path.isfile(destination_filename) == True: + answer_data = { + "size": total_size, + "sha512": str(sha1.hexdigest()), + "already_exist": True, + } + + await _response.write(json.dumps(answer_data, sort_keys=True, indent=4)) + return + shutil.move(temporary_file, destination_filename) + data_metafile = { + "sha512": str(sha1.hexdigest()), + "size": total_size, + 'filename': _request.headers["filename"], + 'mime-type': _request.headers["mime-type"], + } + tools.file_write_data(destination_filename + ".meta", json.dumps(data_metafile, sort_keys=True, indent=4)) + answer_data = { + "size": total_size, + "sha512": str(sha1.hexdigest()), + "already_exist": True, + } + await _response.write(json.dumps(answer_data, sort_keys=True, indent=4)) + return response.stream(streaming, content_type='application/json') + + """ + @elem_blueprint.get('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Show resources") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def retrive(request, id): + value = data_global_elements.get_interface(_name_api).get(id) + if value != None: + return response.json(value) + raise ServerError("No data found", status_code=404) + """ + + + _app.blueprint(elem_blueprint) + +add_data(app, API_DATA) + +import hashlib +import shutil + + if __name__ == "__main__": debug.info("Start REST application: " + str(rest_config["host"]) + ":" + str(rest_config["port"])) diff --git a/tools/sendLocalData.py b/tools/sendLocalData.py index efa7843..f2ad7b5 100755 --- a/tools/sendLocalData.py +++ b/tools/sendLocalData.py @@ -13,7 +13,8 @@ import sys import hashlib import requests # pip install requests import realog.debug as debug - +import magic +import json class upload_in_chunks(object): def __init__(self, filename, chunksize=1 + 13): @@ -41,40 +42,27 @@ class upload_in_chunks(object): #result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(filename, chunksize=4096)) #print("result : " + str(result) + " " + result.text)#str(dir(result))) -""" - -static etk::String extractAndRemove(const etk::String& _inputValue, const char _startMark, const char _stopMark, etk::Vector& _values: - _values.clear(); - etk::String out; - bool inside=False; - etk::String insideData; - for (auto &it : _inputValue: - if inside == False - and it == _startMark: - inside = True; - elif inside == True - and it == _stopMark: - inside = False; - _values.pushBack(insideData); - insideData.clear(); +def extract_and_remove(_input_value, _start_mark, _stop_mark): + values = [] + out = "" + inside = False + inside_data = "" + for it in _input_value: + if inside == False \ + and it == _start_mark: + inside = True + elif inside == True \ + and it == _stop_mark: + inside = False + _values.append(inside_data) + inside_data = "" elif inside == True: - insideData += it; + insideData += it else: - out += it; - - - return out; + out += it + return (out, values) - -bool progressCall(const etk::String& _value: - return False; - - -void progressCallback(const etk::String& _value: - debug.info("plop " + _value); - -""" def create_directory_of_file(_file): path = os.path.dirname(_file) try: @@ -129,6 +117,7 @@ def calculate_sha512(_path): def push_video_file(_path, _basic_key={}): file_name, file_extension = os.path.splitext(_path); + debug.info("Send file: '" + file_name + "' with extention " + file_extension) # internal file_extension ... if file_extension == "sha512": debug.verbose("file: '" + _path + "' sha512 extention ...") @@ -207,6 +196,7 @@ def push_video_file(_path, _basic_key={}): debug.info("check file existance: sha='" + storedSha512 + "'"); """ + # push only if the file exist """ # TODO : Check the metadata updating ... @@ -227,176 +217,164 @@ def push_video_file(_path, _basic_key={}): return False; """ - result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(_path, chunksize=4096)) - print("result *********** : " + str(result) + " " + result.text)#str(dir(result))) - """ - # Get the media - zeus::ProxyMedia media = _srv.get(mediaId).waitFor(echrono::seconds(2000)).get(); - if media.exist() == False: - debug.error("get media error"); - return False; - - - # TODO: if the media have meta data ==> this mean that the media already added before ... + mime = magic.Magic(mime=True) + mime_type = mime.from_file(_path) + headers_values = {'filename': _path, 'mime-type': mime_type} + result_send_data = requests.post("http://127.0.0.1:15080/data", headers=headers_values, data=upload_in_chunks(_path, chunksize=4096)) + print("result *********** : " + str(result_send_data) + " " + result_send_data.text) + file_name = os.path.basename(file_name) debug.info("Find file_name : '" + file_name + "'"); - # Remove Date (XXXX) or other title - etk::Vector dates; - file_name = extractAndRemove(file_name, '(', ')', dates); - bool haveDate = False; - bool haveTitle = False; - for (auto &it: dates: - if it.size() == 0: - continue; - - if it[0] == '0' - or it[0] == '1' - or it[0] == '2' - or it[0] == '3' - or it[0] == '4' - or it[0] == '5' - or it[0] == '6' - or it[0] == '7' - or it[0] == '8' + # Remove Date (XXXX) or other titreadsofarle + file_name, dates = extract_and_remove(file_name, '(', ')'); + have_date = False + have_Title = False + for it in dates: + if len(it) == 0: + continue + if it[0] == '0' \ + or it[0] == '1' \ + or it[0] == '2' \ + or it[0] == '3' \ + or it[0] == '4' \ + or it[0] == '5' \ + or it[0] == '6' \ + or it[0] == '7' \ + or it[0] == '8' \ or it[0] == '9': # find a date ... - if haveDate == True: - debug.info(" '" + file_name + "'"); - debug.error("Parse Date error : () : " + it + " ==> multiple date"); - continue; - - haveDate = True; - _basic_key.set("date", it); + if have_date == True: + debug.info(" '" + file_name + "'") + debug.error("Parse Date error : () : " + it + " ==> multiple date") + continue + have_date = True + _basic_key["date"] = it else: - if haveTitle == True: - debug.info(" '" + file_name + "'"); - debug.error("Parse Title error : () : " + it + " ==> multiple title"); - continue; - - haveTitle = True; + if have_Title == True: + debug.info(" '" + file_name + "'") + debug.error("Parse Title error : () : " + it + " ==> multiple title") + continue + have_Title = True # Other title - _basic_key.set("title2", it); - + _basic_key.set["title2"] = it; # remove unneeded date - if haveDate == False: - _basic_key.set("date", ""); + if have_date == False: + _basic_key["date"] = "" # remove unneeded title 2 - if haveTitle == False: - _basic_key.set("title2", ""); + if have_Title == False: + _basic_key["title2"] = "" # Remove the actors [XXX YYY][EEE TTT]... - etk::Vector acthors; - file_name = extractAndRemove(file_name, '[', ']', acthors); - if acthors.size() > 0: - debug.info(" '" + file_name + "'"); - etk::String actorList; - for (auto &itActor : acthors: - if actorList != "": - actorList += ";"; - - actorList += itActor; - - _basic_key.set("acthors", actorList); + file_name, acthors = extract_and_remove(file_name, '[', ']'); + if len(acthors) > 0: + debug.info(" '" + file_name + "'") + actor_list = [] + for it_actor in acthors: + if actor_list != "": + actor_list += ";" + actor_list.append(it_actor) + _basic_key["acthors"] = actor_list + list_element_base = file_name.split('-') - # remove file_extension - file_name = etk::String(file_name.begin(), file_name.begin() + file_name.size() - (file_extension.size()+1)); - - etk::Vector listElementBase = etk::split(file_name, '-'); - - etk::Vector listElement; - etk::String tmpStartString; - for (size_t iii=0; iii 3 - and listElement[1][0] == 's' - and listElement[2][0] == 'e': + if len(list_element) > 3 \ + and list_element[1][0] == 's' \ + and list_element[2][0] == 'e': # internal formalisme ... - int32_t saison = -1; - int32_t episode = -1; - etk::String seriesName = listElement[0]; + saison = -1; + episode = -1; + series_name = list_element[0]; - _basic_key.set("series-name", etk::toString(seriesName)); - etk::String fullEpisodeName = listElement[3]; - for (int32_t yyy=4; yyy nothing to do ... + pass else: - saison = etk::string_to_int32_t(&listElement[1][1]); + saison = int(list_element[1][1:]); - if etk::String(&listElement[2][1]) == "XX": + if list_element[2][1:] == "XX": # episode unknow ... ==> nothing to do ... + pass else: - episode = etk::string_to_int32_t(&listElement[2][1]); - - _basic_key.set("episode", etk::toString(episode)); + episode = int(list_element[2][1:]); + _basic_key["episode"] = episode debug.info("Find a internal mode series: :"); debug.info(" origin : '" + file_name + "'"); - etk::String saisonPrint = "XX"; - etk::String episodePrint = "XX"; + saisonPrint = "XX"; + episodePrint = "XX"; if saison < 0: # nothing to do + pass elif saison < 10: - saisonPrint = "0" + etk::toString(saison); - _basic_key.set("saison", etk::toString(saison)); + saisonPrint = "0" + str(saison) + _basic_key["saison"] = str(saison) else: - saisonPrint = etk::toString(saison); - _basic_key.set("saison", etk::toString(saison)); + saisonPrint = str(saison) + _basic_key["saison"] = str(saison) if episode < 0: # nothing to do + pass elif episode < 10: - episodePrint = "0" + etk::toString(episode); - _basic_key.set("episode", etk::toString(episode)); + episodePrint = "0" + str(episode); + _basic_key["episode"] = str(episode) else: - episodePrint = etk::toString(episode); - _basic_key.set("episode", etk::toString(episode)); + episodePrint = str(episode); + _basic_key["episode"] = str(episode) - debug.info(" ==> '" + seriesName + "-s" + saisonPrint + "-e" + episodePrint + "-" + fullEpisodeName + "'"); - + debug.info(" ==> '" + series_name + "-s" + saisonPrint + "-e" + episodePrint + "-" + full_episode_name + "'"); - # send all meta data: - zeus::FutureGroup group; - for (auto &itKey : _basic_key: - if itKey.second != "": - APPL_WARNING("Set metaData: " + itKey.first + " : " + itKey.second); - - group.add(media.setMetadata(itKey.first, itKey.second)); - group.wait(); - """ - return True; + result_send_data_json = json.loads(result_send_data.text) + debug.info("pared meta data: " + json.dumps(_basic_key, sort_keys=True, indent=4)) + data_model = { + "type_id": _basic_key["type"], + "sha512": result_send_data_json["sha512"], + "saison": _basic_key["saison"], + "episode": _basic_key["episode"], + "group_name": _basic_key["series-name"], + #"group_id": int, + "name": _basic_key["title"], + "description": None, + # creating time + "date": _basic_key["date"], + "actors": _basic_key["acthors"], + # number of second + "time": None, + } + result_send_data = requests.post("http://127.0.0.1:15080/video", data=json.dumps(data_model, sort_keys=True, indent=4)) + print("result *********** : " + str(result_send_data) + " " + result_send_data.text) + + return True def install_video_path( _path, _basic_key = {}):