[DEV] better version

Dockerfile

@@ -10,6 +10,8 @@ RUN pip install python-dateutil
 
 RUN pip install realog
 
+RUN pip install python-magic
+
 EXPOSE 80
 
 ADD src /application/

src/app_video.py

@@ -23,6 +23,7 @@ import dateutil.parser
 
 
 import time
+import json
 import os
 import sys
 import datetime
@@ -74,6 +75,7 @@ API_SAISON = "saison"
 add_interface(API_SAISON)
 API_VIDEO = "video"
 add_interface(API_VIDEO)
+API_DATA = "data"
 
 def add_type(_app, _name_api):
 	elem_blueprint = Blueprint(_name_api)
@@ -241,7 +243,8 @@ def add_video(_app, _name_api):
 	
 	class DataModel:
 		type_id = int
-		saison_id = int
+		saison = int
+		episode = int
 		group_id = int
 		name = str
 		description = str
@@ -297,41 +300,98 @@ def add_video(_app, _name_api):
 
 add_video(app, API_VIDEO)
 
-import hashlib
-import shutil
-
-
 tmp_value = 0
 
 #curl  -F 'file=@Totally_Spies.mp4;type=application/octet-stream' -H 'transfer-encoding:chunked' 127.0.0.1:15080/data -X POST -O; echo ;
 
-@app.post('/data', stream=True)
+def add_data(_app, _name_api):
-async def handler(_request):
+	elem_blueprint = Blueprint(_name_api)
-	debug.info("request streaming " + str(_request));
+	"""
-	async def streaming(_response):
+	@elem_blueprint.get('/' + _name_api, strict_slashes=True)
-		debug.info("streaming " + str(_response));
+	@doc.summary("Show saisons")
-		total_size = 0
+	@doc.description("Display a listing of the resource.")
-		temporary_file = os.path.join(rest_config["tmp_data"], str(tmp_value) + ".tmp")
+	@doc.produces(content_type='application/json')
-		if not os.path.exists(rest_config["tmp_data"]):
+	async def list(request):
-			os.makedirs(rest_config["tmp_data"])
+		return response.json(data_global_elements.get_interface(_name_api).gets())
-		if not os.path.exists(rest_config["data_media"]):
+	"""
-			os.makedirs(rest_config["data_media"])
+	
-		file_stream = open(temporary_file,"wb")
+	@elem_blueprint.post('/' + _name_api, strict_slashes=True, stream=True)
-		sha1 = hashlib.sha512()
+	@doc.summary("send new file data")
-		while True:
+	@doc.description("Create a new data file (associated with his sha512.")
-			body = await _request.stream.read()
+	#@doc.consumes(DataModel, location='body')#, required=True)
-			if body is None:
+	@doc.response_success(status=201, description='If successful created')
-				debug.warning("empty body");
+	async def create(_request):
-				break
+		debug.info("request streaming " + str(_request));
-			total_size += len(body)
+		args_with_blank_values = _request.headers
-			debug.warning("body " + str(len(body)) + "/" + str(total_size))
+		debug.info("List arguments: " + str(args_with_blank_values));
-			file_stream.write(body)
+		async def streaming(_response):
-			sha1.update(body)
+			debug.info("streaming " + str(_response));
-		file_stream.close()
+			total_size = 0
-		print("SHA512: " + str(sha1.hexdigest()))
+			temporary_file = os.path.join(rest_config["tmp_data"], str(tmp_value) + ".tmp")
-		await _response.write('{"size":' + str(total_size) + ', "sha512":"' + str(sha1.hexdigest()) + '"}')
+			if not os.path.exists(rest_config["tmp_data"]):
-		shutil.move(temporary_file, os.path.join(rest_config["data_media"], str(sha1.hexdigest())))
+				os.makedirs(rest_config["tmp_data"])
-	return response.stream(streaming, content_type='application/json')
+			if not os.path.exists(rest_config["data_media"]):
+				os.makedirs(rest_config["data_media"])
+			file_stream = open(temporary_file,"wb")
+			sha1 = hashlib.sha512()
+			while True:
+				body = await _request.stream.read()
+				if body is None:
+					debug.warning("empty body");
+					break
+				total_size += len(body)
+				debug.verbose("body " + str(len(body)) + "/" + str(total_size))
+				file_stream.write(body)
+				sha1.update(body)
+			file_stream.close()
+			print("SHA512: " + str(sha1.hexdigest()))
+			destination_filename = os.path.join(rest_config["data_media"], str(sha1.hexdigest()))
+			if os.path.isfile(destination_filename) == True:
+				answer_data = {
+					"size": total_size,
+					"sha512": str(sha1.hexdigest()),
+					"already_exist": True,
+				}
+				
+				await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))
+				return
+			shutil.move(temporary_file, destination_filename)
+			data_metafile = {
+				"sha512": str(sha1.hexdigest()),
+				"size": total_size,
+				'filename': _request.headers["filename"],
+				'mime-type': _request.headers["mime-type"],
+			}
+			tools.file_write_data(destination_filename + ".meta", json.dumps(data_metafile, sort_keys=True, indent=4))
+			answer_data = {
+				"size": total_size,
+				"sha512": str(sha1.hexdigest()),
+				"already_exist": True,
+			}
+			await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))
+		return response.stream(streaming, content_type='application/json')
+	
+	"""
+	@elem_blueprint.get('/' + _name_api + '/<id:int>', strict_slashes=True)
+	@doc.summary("Show resources")
+	@doc.description("Display a listing of the resource.")
+	@doc.produces(content_type='application/json')
+	async def retrive(request, id):
+		value = data_global_elements.get_interface(_name_api).get(id)
+		if value != None:
+			return response.json(value)
+		raise ServerError("No data found", status_code=404)
+	"""
+	
+	
+	_app.blueprint(elem_blueprint)
+
+add_data(app, API_DATA)
+
+import hashlib
+import shutil
+
+
 
 if __name__ == "__main__":
 	debug.info("Start REST application: " + str(rest_config["host"]) + ":" + str(rest_config["port"]))

tools/sendLocalData.py

@@ -13,7 +13,8 @@ import sys
 import hashlib
 import requests  # pip install requests
 import realog.debug as debug
-
+import magic
+import json
 
 class upload_in_chunks(object):
     def __init__(self, filename, chunksize=1 + 13):
@@ -41,40 +42,27 @@ class upload_in_chunks(object):
 #result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(filename, chunksize=4096))
 #print("result : " + str(result) + "  " + result.text)#str(dir(result)))
 
-"""
 
-
+def extract_and_remove(_input_value, _start_mark, _stop_mark):
-static etk::String extractAndRemove(const etk::String& _inputValue, const char _startMark, const char _stopMark, etk::Vector<etk::String>& _values:
+	values = []
-	_values.clear();
+	out = ""
-	etk::String out;
+	inside = False
-	bool inside=False;
+	inside_data = ""
-	etk::String insideData;
+	for it in _input_value:
-	for (auto &it : _inputValue:
+		if     inside == False \
-		if     inside == False
+		   and it == _start_mark:
-		   and it == _startMark:
+			inside = True
-			inside = True;
+		elif     inside == True \
-		elif     inside == True
+		     and it == _stop_mark:
-		          and it == _stopMark:
+			inside = False
-			inside = False;
+			_values.append(inside_data)
-			_values.pushBack(insideData);
+			inside_data = ""
-			insideData.clear();
 		elif inside == True:
-			insideData += it;
+			insideData += it
 		else:
-			out += it;
+			out += it
-		
+	return (out, values)
-	
-	return out;
 
-
-bool progressCall(const etk::String& _value:
-	return False;
-
-
-void progressCallback(const etk::String& _value:
-	debug.info("plop " + _value);
-
-"""
 def create_directory_of_file(_file):
 	path = os.path.dirname(_file)
 	try:
@@ -129,6 +117,7 @@ def calculate_sha512(_path):
 
 def push_video_file(_path, _basic_key={}):
 	file_name, file_extension = os.path.splitext(_path);
+	debug.info("Send file: '" + file_name + "'  with extention " + file_extension)
 	# internal file_extension ...
 	if file_extension == "sha512":
 		debug.verbose("file: '" + _path + "' sha512 extention ...")
@@ -207,6 +196,7 @@ def push_video_file(_path, _basic_key={}):
 	debug.info("check file existance: sha='" + storedSha512 + "'");
 	"""
 	
+	
 	# push only if the file exist
 	"""
 	# TODO : Check the metadata updating ...
@@ -227,176 +217,164 @@ def push_video_file(_path, _basic_key={}):
 		return False;
 	
 	"""
-	result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(_path, chunksize=4096))
+	mime = magic.Magic(mime=True)
-	print("result *********** : " + str(result) + "  " + result.text)#str(dir(result)))
+	mime_type = mime.from_file(_path)
-	"""
+	headers_values = {'filename': _path, 'mime-type': mime_type}
-	# Get the media
+	result_send_data = requests.post("http://127.0.0.1:15080/data", headers=headers_values, data=upload_in_chunks(_path, chunksize=4096))
-	zeus::ProxyMedia media = _srv.get(mediaId).waitFor(echrono::seconds(2000)).get();
+	print("result *********** : " + str(result_send_data) + "  " + result_send_data.text)
-	if media.exist() == False:
+	file_name = os.path.basename(file_name)
-		debug.error("get media error");
-		return False;
-	
-	
-	# TODO: if the media have meta data ==> this mean that the media already added before ...
 	debug.info("Find file_name : '" + file_name + "'");
-	# Remove Date (XXXX) or other title
+	# Remove Date (XXXX) or other titreadsofarle
-	etk::Vector<etk::String> dates;
+	file_name, dates = extract_and_remove(file_name, '(', ')');
-	file_name = extractAndRemove(file_name, '(', ')', dates);
+	have_date = False
-	bool haveDate = False;
+	have_Title = False
-	bool haveTitle = False;
+	for it in dates:
-	for (auto &it: dates:
+		if len(it) == 0:
-		if it.size() == 0:
+			continue
-			continue;
+		if      it[0] == '0' \
-		
+		     or it[0] == '1' \
-		if     it[0] == '0'
+		     or it[0] == '2' \
-		     or it[0] == '1'
+		     or it[0] == '3' \
-		     or it[0] == '2'
+		     or it[0] == '4' \
-		     or it[0] == '3'
+		     or it[0] == '5' \
-		     or it[0] == '4'
+		     or it[0] == '6' \
-		     or it[0] == '5'
+		     or it[0] == '7' \
-		     or it[0] == '6'
+		     or it[0] == '8' \
-		     or it[0] == '7'
-		     or it[0] == '8'
 		     or it[0] == '9':
 			# find a date ...
-			if haveDate == True:
+			if have_date == True:
-				debug.info("                '" + file_name + "'");
+				debug.info("                '" + file_name + "'")
-				debug.error("Parse Date error : () : " + it + " ==> multiple date");
+				debug.error("Parse Date error : () : " + it + " ==> multiple date")
-				continue;
+				continue
-			
+			have_date = True
-			haveDate = True;
+			_basic_key["date"] = it
-			_basic_key.set("date", it);
 		else:
-			if haveTitle == True:
+			if have_Title == True:
-				debug.info("                '" + file_name + "'");
+				debug.info("                '" + file_name + "'")
-				debug.error("Parse Title error : () : " + it + " ==> multiple title");
+				debug.error("Parse Title error : () : " + it + " ==> multiple title")
-				continue;
+				continue
-			
+			have_Title = True
-			haveTitle = True;
 			# Other title
-			_basic_key.set("title2", it);
+			_basic_key.set["title2"] = it;
-		
 	
 	# remove unneeded date
-	if haveDate == False:
+	if have_date == False:
-		_basic_key.set("date", "");
+		_basic_key["date"] = ""
 	
 	# remove unneeded title 2
-	if haveTitle == False:
+	if have_Title == False:
-		_basic_key.set("title2", "");
+		_basic_key["title2"] = ""
 	
 	# Remove the actors [XXX YYY][EEE TTT]...
-	etk::Vector<etk::String> acthors;
+	file_name, acthors = extract_and_remove(file_name, '[', ']');
-	file_name = extractAndRemove(file_name, '[', ']', acthors);
+	if len(acthors) > 0:
-	if acthors.size() > 0:
+		debug.info("                '" + file_name + "'")
-		debug.info("                '" + file_name + "'");
+		actor_list = []
-		etk::String actorList;
+		for it_actor in acthors:
-		for (auto &itActor : acthors:
+			if actor_list != "":
-			if actorList != "":
+				actor_list += ";"
-				actorList += ";";
+			actor_list.append(it_actor)
-			
+		_basic_key["acthors"] = actor_list
-			actorList += itActor;
-		
-		_basic_key.set("acthors", actorList);
 	
+	list_element_base = file_name.split('-')
 	
-	# remove file_extension
+	list_element = [];
-	file_name = etk::String(file_name.begin(), file_name.begin() + file_name.size() - (file_extension.size()+1));
+	tmp_start_string = "";
-	
+	for iii in range(0,len(list_element_base)):
-	etk::Vector<etk::String> listElementBase = etk::split(file_name, '-');
+		if     list_element_base[iii][0] != 's' \
-	
+		   and list_element_base[iii][0] != 'e':
-	etk::Vector<etk::String> listElement;
+			if tmp_start_string != "":
-	etk::String tmpStartString;
+				tmp_start_string += '-'
-	for (size_t iii=0; iii<listElementBase.size(); ++iii:
+			tmp_start_string += list_element_base[iii]
-		if     listElementBase[iii][0] != 's'
-		   and listElementBase[iii][0] != 'e':
-			if tmpStartString != "":
-				tmpStartString += '-';
-			
-			tmpStartString += listElementBase[iii];
 		else:
-			listElement.pushBack(tmpStartString);
+			list_element.append(tmp_start_string)
-			tmpStartString = "";
+			tmp_start_string = ""
-			for (/* nothing to do */; iii<listElementBase.size(); ++iii:
+			while iii<len(list_element_base):
-				listElement.pushBack(listElementBase[iii]);
+				list_element.append(list_element_base[iii])
-			
+				iii += 1
-		
-		
 	
-	if tmpStartString != "":
+	if tmp_start_string != "":
-		listElement.pushBack(tmpStartString);
+		list_element.append(tmp_start_string)
 	
-	if listElement.size() == 1:
+	if len(list_element) == 1:
 		# nothing to do , it might be a film ...
-		_basic_key.set("title", etk::toString(listElement[0]));
+		_basic_key["title"] = list_element[0]
 	else:
-		/*
+		if     len(list_element) > 3 \
-		for (auto &itt : listElement:
+		   and list_element[1][0] == 's' \
-			debug.info("    " + itt);
+		   and list_element[2][0] == 'e':
-		
-		*/
-		if     listElement.size() > 3
-		   and listElement[1][0] == 's'
-		   and listElement[2][0] == 'e':
 			# internal formalisme ...
-			int32_t saison = -1;
+			saison = -1;
-			int32_t episode = -1;
+			episode = -1;
-			etk::String seriesName = listElement[0];
+			series_name = list_element[0];
 			
-			_basic_key.set("series-name", etk::toString(seriesName));
+			_basic_key["series-name"] = series_name
-			etk::String fullEpisodeName = listElement[3];
+			full_episode_name = list_element[3]
-			for (int32_t yyy=4; yyy<listElement.size(); ++yyy:
+			for yyy in range(4, len(list_element)):
-				fullEpisodeName += "-" + listElement[yyy];
+				full_episode_name += "-" + list_element[yyy]
 			
-			_basic_key.set("title", etk::toString(fullEpisodeName));
+			_basic_key["title"] = full_episode_name
-			if etk::String(&listElement[1][1]) == "XX":
+			if list_element[1][1:] == "XX":
 				# saison unknow ... ==> nothing to do ...
+				pass
 			else:
-				saison = etk::string_to_int32_t(&listElement[1][1]);
+				saison = int(list_element[1][1:]);
 			
-			if etk::String(&listElement[2][1]) == "XX":
+			if list_element[2][1:] == "XX":
 				# episode unknow ... ==> nothing to do ...
+				pass
 			else:
-				episode = etk::string_to_int32_t(&listElement[2][1]);
+				episode = int(list_element[2][1:]);
-				
+				_basic_key["episode"] = episode
-				_basic_key.set("episode", etk::toString(episode));
 			
 			debug.info("Find a internal mode series: :");
 			debug.info("    origin       : '" + file_name + "'");
-			etk::String saisonPrint = "XX";
+			saisonPrint = "XX";
-			etk::String episodePrint = "XX";
+			episodePrint = "XX";
 			if saison < 0:
 				# nothing to do
+				pass
 			elif saison < 10:
-				saisonPrint = "0" + etk::toString(saison);
+				saisonPrint = "0" + str(saison)
-				_basic_key.set("saison", etk::toString(saison));
+				_basic_key["saison"] = str(saison)
 			else:
-				saisonPrint = etk::toString(saison);
+				saisonPrint = str(saison)
-				_basic_key.set("saison", etk::toString(saison));
+				_basic_key["saison"] = str(saison)
 			
 			if episode < 0:
 				# nothing to do
+				pass
 			elif episode < 10:
-				episodePrint = "0" + etk::toString(episode);
+				episodePrint = "0" + str(episode);
-				_basic_key.set("episode", etk::toString(episode));
+				_basic_key["episode"] = str(episode)
 			else:
-				episodePrint = etk::toString(episode);
+				episodePrint = str(episode);
-				_basic_key.set("episode", etk::toString(episode));
+				_basic_key["episode"] = str(episode)
 			
-			debug.info("     ==> '" + seriesName + "-s" + saisonPrint + "-e" + episodePrint + "-" + fullEpisodeName + "'");
+			debug.info("     ==> '" + series_name + "-s" + saisonPrint + "-e" + episodePrint + "-" + full_episode_name + "'");
-		
 	
-	# send all meta data:
-	zeus::FutureGroup group;
-	for (auto &itKey : _basic_key:
-		if itKey.second != "":
-			APPL_WARNING("Set metaData: " + itKey.first + " : " + itKey.second);
-		
-		group.add(media.setMetadata(itKey.first, itKey.second));
 	
-	group.wait();
+	result_send_data_json = json.loads(result_send_data.text)
-	"""
+	debug.info("pared meta data: " + json.dumps(_basic_key, sort_keys=True, indent=4))
-	return True;
+	data_model = {
+		"type_id": _basic_key["type"],
+		"sha512": result_send_data_json["sha512"],
+		"saison": _basic_key["saison"],
+		"episode": _basic_key["episode"],
+		"group_name": _basic_key["series-name"],
+		#"group_id": int,
+		"name": _basic_key["title"],
+		"description": None,
+		# creating time
+		"date": _basic_key["date"],
+		"actors": _basic_key["acthors"],
+		# number of second
+		"time": None,
+	}
+	result_send_data = requests.post("http://127.0.0.1:15080/video", data=json.dumps(data_model, sort_keys=True, indent=4))
+	print("result *********** : " + str(result_send_data) + "  " + result_send_data.text)
+	
+	return True
 
 
 def install_video_path( _path, _basic_key = {}):