[DEV] rework path of data with id only for reference

.gitignore

@@ -6,6 +6,9 @@
 /tmp
 /out-tsc
 
+dataPush
+node_modules
+
 # dependencies
 applicationFront/node_modules
 

back/src/api/data.py

@@ -52,6 +52,19 @@ def add(_app, _name_api):
 		return response.json(data_global_elements.get_interface(_name_api).gets())
 	"""
 	
+	class DataModelBdd:
+		id = int
+		size = int
+		sha512 = str
+		mime_type = str
+		original_name = [str, type(None)]
+		# creating time
+		create_date = str
+	
+	data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd)
+	
+	
+	
 	@elem_blueprint.post('/' + _name_api, strict_slashes=True, stream=True)
 	@doc.summary("send new file data")
 	@doc.description("Create a new data file (associated with his sha512.")
@@ -82,38 +95,49 @@ def add(_app, _name_api):
 				sha1.update(body)
 			file_stream.close()
 			print("SHA512: " + str(sha1.hexdigest()))
-			destination_filename = os.path.join(_app.config['REST_MEDIA_DATA'], str(sha1.hexdigest()))
+			
+			new_data = {
+					"size": total_size,
+					"sha512": str(sha1.hexdigest()),
+					'original_name': _request.headers["filename"],
+					'mime_type': _request.headers["mime-type"],
+					'create_date': datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
+				}
+			# TODO: Check if the element already exist ...
+			
+			return_bdd = data_global_elements.get_interface(_name_api).post(new_data)
+			
+			basic_data_path = os.path.join(_app.config['REST_MEDIA_DATA'], str(return_bdd["id"]))
+			
+			if not os.path.exists(basic_data_path):
+				os.makedirs(basic_data_path)
+			destination_filename = os.path.join(basic_data_path, "video")
+			"""
 			if os.path.isfile(destination_filename) == True:
 				answer_data = {
 					"size": total_size,
 					"sha512": str(sha1.hexdigest()),
 					'filename': _request.headers["filename"],
-					'mime-type': _request.headers["mime-type"],
+					'mime_type': _request.headers["mime-type"],
 					"already_exist": True,
 				}
 				await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))
 				return
+			"""
+			
 			# move the file
 			shutil.move(temporary_file, destination_filename)
-			
 			# collect media info ...
 			media_info = MediaInfo.parse(destination_filename)
 			data_metafile = {
 				"sha512": str(sha1.hexdigest()),
 				"size": total_size,
 				'filename': _request.headers["filename"],
-				'mime-type': _request.headers["mime-type"],
+				'mime_type': _request.headers["mime-type"],
-				'media-info': json.loads(media_info.to_json())
+				'media_info': json.loads(media_info.to_json())
 			}
-			tools.file_write_data(destination_filename + ".meta", json.dumps(data_metafile, sort_keys=True, indent=4))
+			tools.file_write_data(os.path.join(basic_data_path, "meta.json"), json.dumps(data_metafile, sort_keys=True, indent=4))
-			answer_data = {
+			await _response.write(json.dumps(return_bdd, sort_keys=True, indent=4))
-				"size": total_size,
-				"sha512": str(sha1.hexdigest()),
-				'filename': _request.headers["filename"],
-				'mime-type': _request.headers["mime-type"],
-				"already_exist": True,
-			}
-			await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))
 		return response.stream(streaming, content_type='application/json')
 	
 	@elem_blueprint.get('/' + _name_api + '/<id:string>', strict_slashes=True)
@@ -130,9 +154,10 @@ def add(_app, _name_api):
 			id = id[:-4]
 		if id[-4:] == ".ts":
 			id = id[:-3]
-		filename = os.path.join(_app.config['REST_MEDIA_DATA'], id)
+		filename = os.path.join(_app.config['REST_MEDIA_DATA'], id, "video")
+		value = data_global_elements.get_interface(_name_api).get(id)
 		headers = {
-			'Content-Type': 'video/x-matroska',
+			'Content-Type': value["mime_type"],
 			'Accept-Ranges': 'Accept-Ranges: bytes'
 			}
 		try:

back/src/api/video.py

@@ -66,7 +66,7 @@ def add(_app, _name_api):
 	
 	class DataModelBdd:
 		id = int
-		sha512 = str
+		data_id = int
 		type_id = int
 		saison_id = [int, type(None)]
 		episode = [int, type(None)]
@@ -109,7 +109,7 @@ def add(_app, _name_api):
 	@doc.consumes(DataModel, location='body')#, required=True)
 	@doc.response_success(status=201, description='If successful created')
 	async def create(request):
-		for type_key in ["sha512","type_id","name"]:
+		for type_key in ["data_id","type_id","name"]:
 			if type_key not in request.json.keys():
 				raise ServerError("Bad Request: Missing Key '" + type_key + "'", status_code=400)
 		for type_key in ["create_date"]:
@@ -120,7 +120,7 @@ def add(_app, _name_api):
 				request.json[type_key] = None
 		request.json["create_date"] = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
 		#Find if already exist
-		list_elem = data_global_elements.get_interface(_name_api).find(["group_id", "sha512"], request.json);
+		list_elem = data_global_elements.get_interface(_name_api).find(["group_id", "data_id"], request.json);
 		for elem in list_elem:
 			return response.json(elem)
 		

back/src/app_video.py

@@ -118,6 +118,7 @@ def add_interface(_name, _default_value = None):
 		interface.reset_with_value(_default_value);
 	data_global_elements.add_interface(_name, interface)
 
+add_interface(data_global_elements.API_DATA)
 add_interface(data_global_elements.API_TYPE, default_values_type)
 add_interface(data_global_elements.API_GROUP)
 add_interface(data_global_elements.API_SAISON)

back/tools/sendLocalData.py

@@ -126,8 +126,9 @@ def push_video_file(_path, _basic_key={}):
 		return True
 	
 	debug.info("Add media : '" + _path + "'")
-	if     file_extension[1:] not in ["webm", "avi", "mkv", "mov", "mp4", "ts"] \
+	# "avi", , "mov", , "ts", "cover_1.tiff", "cover_1.bmp", "cover_1.tga"] copy only file that is supported by the html5 video player (chrome mode only)
-	   and file_name not in ["cover_1.jpg","cover_1.png", "cover_1.till", "cover_1.bmp", "cover_1.tga"]:
+	if     file_extension[1:] not in ["webm", "mkv", "mp4"] \
+	   and file_name not in ["cover_1.jpg","cover_1.png"]:
 		debug.warning("Not send file : " + _path + " Not manage file_extension... " + file_extension)
 		return False
 	
@@ -217,10 +218,25 @@ def push_video_file(_path, _basic_key={}):
 	if mediaId == 0:
 		debug.error("Get media ID = 0 With no error");
 		return False;
-	
 	"""
 	mime = magic.Magic(mime=True)
 	mime_type = mime.from_file(_path)
+	# do it by myself .. it is better ...
+	filename___, file_extension = os.path.splitext(_path)
+	if file_extension == "mkv":
+		mime_type = "video/x-matroska"
+	elif file_extension == "mka":
+		mime_type = "audio/x-matroska"
+	elif file_extension == "mp4":
+		mime_type = "video/mp4"
+	elif file_extension == "webm":
+		mime_type = "video/webm"
+	elif file_extension == "json":
+		mime_type = "application/json"
+	elif file_extension == "jpeg":
+		mime_type = "image/jpeg"
+	elif file_extension == "png":
+		mime_type = "image/png"
 	headers_values = {'filename': _path, 'mime-type': mime_type}
 	result_send_data = requests.post("http://127.0.0.1:15080/data", headers=headers_values, data=upload_in_chunks(_path, chunksize=4096))
 	debug.info("result *********** : " + str(result_send_data) + "  " + result_send_data.text)

back/tools/videoTranscode.py

@@ -0,0 +1,287 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+##
+## @author Edouard DUPIN
+##
+## @copyright 2016, Edouard DUPIN, all right reserved
+##
+## @license APACHE v2.0 (see license file)
+##
+import os
+import fnmatch
+import sys
+import subprocess
+import shlex
+##
+## @brief Execute the command with no get of output
+##
+def run_command(cmd_line):
+	# prepare command line:
+	args = shlex.split(cmd_line)
+	print("[INFO] cmd = " + str(args))
+	try:
+		# create the subprocess
+		p = subprocess.Popen(args)
+	except subprocess.CalledProcessError as e:
+		print("[ERROR] subprocess.CalledProcessError : " + str(args))
+		return False
+	#except:
+	#	debug.error("Exception on : " + str(args))
+	# launch the subprocess:
+	output, err = p.communicate()
+	# Check error :
+	if p.returncode == 0:
+		return True
+	else:
+		return False
+
+
+##
+## @brief Get list of all Files in a specific path (with a regex)
+## @param[in] path (string) Full path of the machine to search files (start with / or x:)
+## @param[in] regex (string) Regular expression to search data
+## @param[in] recursive (bool) List file with recursive search
+## @param[in] remove_path (string) Data to remove in the path
+## @return (list) return files requested
+##
+def get_list_of_file_in_path(path, regex="*", recursive = False, remove_path=""):
+	out = []
+	if os.path.isdir(os.path.realpath(path)):
+		tmp_path = os.path.realpath(path)
+		tmp_rule = regex
+	else:
+		debug.error("path does not exist : '" + str(path) + "'")
+	
+	for root, dirnames, filenames in os.walk(tmp_path):
+		deltaRoot = root[len(tmp_path):]
+		while     len(deltaRoot) > 0 \
+		      and (    deltaRoot[0] == '/' \
+		            or deltaRoot[0] == '\\' ):
+			deltaRoot = deltaRoot[1:]
+		if     recursive == False \
+		   and deltaRoot != "":
+			return out
+		tmpList = filenames
+		if len(tmp_rule) > 0:
+			tmpList = fnmatch.filter(filenames, tmp_rule)
+		# Import the module :
+		for cycleFile in tmpList:
+			#for cycleFile in filenames:
+			add_file = os.path.join(tmp_path, deltaRoot, cycleFile)
+			if len(remove_path) != 0:
+				if add_file[:len(remove_path)] != remove_path:
+					print("ERROR : Request remove start of a path that is not the same: '" + add_file[:len(remove_path)] + "' demand remove of '" + str(remove_path) + "'")
+				else:
+					add_file = add_file[len(remove_path)+1:]
+			out.append(add_file)
+	return out;
+
+#ffmpeg -i 000.ts -threads 0 -vcodec libx264 -crf 20 -force_key_frames expr:gte\(t,n_forced*1\) -s 720x540 -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace 000_transcoded.ts
+#ffmpeg -i 000.ts -threads 0 -vcodec libx264 -crf 20 -force_key_frames expr:gte\(t,n_forced*1\) -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace 000_transcoded.ts
+
+list_files_ts = get_list_of_file_in_path('.', "*.ts")
+list_files_flv = get_list_of_file_in_path('.', "*.flv")
+list_files_mp4 = get_list_of_file_in_path('.', "*.mp4")
+list_files_avi = get_list_of_file_in_path('.', "*.avi")
+list_files_mkv = get_list_of_file_in_path('.', "*.mkv")
+list_files_wmv = get_list_of_file_in_path('.', "*.wmv")
+list_files_divx = get_list_of_file_in_path('.', "*.divx")
+list_files_webm = get_list_of_file_in_path('.', "*.webm")
+
+# remove all encoded element in the other files (TS)
+for elem_mkv in list_files_mkv:
+	index = 0
+	for elem_ts in list_files_ts:
+		if elem_mkv[:-3]+"ts" == elem_ts:
+			break;
+		index += 1
+	if index != len(list_files_ts):
+		print("[INFO] remove from list '" + list_files_ts[index] + "' ==> already transcoded")
+		del list_files_ts[index]
+
+
+
+# remove all encoded element in the other files (FLV)
+for elem_mkv in list_files_mkv:
+	index = 0
+	for elem_flv in list_files_flv:
+		if elem_mkv[:-3]+"flv" == elem_flv:
+			break;
+		index += 1
+	if index != len(list_files_flv):
+		print("[INFO] remove from list '" + list_files_flv[index] + "' ==> already transcoded")
+		del list_files_flv[index]
+
+
+# remove all encoded element in the other files (mp4)
+for elem_mkv in list_files_mkv:
+	index = 0
+	for elem_mp4 in list_files_mp4:
+		if elem_mkv[:-3]+"mp4" == elem_mp4:
+			break;
+		index += 1
+	if index != len(list_files_mp4):
+		print("[INFO] remove from list '" + list_files_mp4[index] + "' ==> already transcoded")
+		del list_files_mp4[index]
+
+
+# remove all encoded element in the other files (TS)
+for elem_mkv in list_files_mkv:
+	index = 0
+	for elem_avi in list_files_avi:
+		if elem_mkv[:-3]+"ts" == elem_avi:
+			break;
+		index += 1
+	if index != len(list_files_avi):
+		print("[INFO] remove from list '" + list_files_avi[index] + "' ==> already transcoded")
+		del list_files_avi[index]
+
+
+# remove all encoded element in the other files (wmv)
+for elem_mkv in list_files_mkv:
+	index = 0
+	for elem_wmv in list_files_wmv:
+		if elem_mkv[:-3]+"wmv" == elem_wmv:
+			break;
+		index += 1
+	if index != len(list_files_wmv):
+		print("[INFO] remove from list '" + list_files_wmv[index] + "' ==> already transcoded")
+		del list_files_wmv[index]
+
+# remove all encoded element in the other files (divx)
+for elem_mkv in list_files_mkv:
+	index = 0
+	for elem_divx in list_files_divx:
+		if elem_mkv[:-3]+"divx" == elem_divx:
+			break;
+		index += 1
+	if index != len(list_files_divx):
+		print("[INFO] remove from list '" + list_files_divx[index] + "' ==> already transcoded")
+		del list_files_divx[index]
+
+print("list of elements TS : ")
+for elem in list_files_ts:
+	print("    - '" + elem + "'")
+print("list of elements MP4 : ")
+for elem in list_files_mp4:
+	print("    - '" + elem + "'")
+print("list of elements FLV : ")
+for elem in list_files_flv:
+	print("    - '" + elem + "'")
+print("list of elements AVI : ")
+for elem in list_files_avi:
+	print("    - '" + elem + "'")
+print("list of elements WMV : ")
+for elem in list_files_wmv:
+	print("    - '" + elem + "'")
+print("list of elements MKV : ")
+for elem in list_files_mkv:
+	print("    - '" + elem + "'")
+print("list of elements divx : ")
+for elem in list_files_divx:
+	print("    - '" + elem + "'")
+print("list of elements webm : ")
+for elem in list_files_webm:
+	print("    - '" + elem + "'")
+
+
+from pymediainfo import MediaInfo
+
+def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offset=0) :
+	print("Start strancoding: '." + extention + "' ... " + str(len(list_of_file)))
+	id_elem = 0
+	for elem in list_of_file:
+		id_elem += 1
+		print("    ========================================================================================")
+		print("    == " + str(offset+id_elem) + " / " + str(total_count_of_file))
+		print("    == Trancode: '" + elem.replace("'", "\'") + "'")
+		print("    ========================================================================================")
+		# collect media info ...
+		#if it is a mk: .. chack the opus format...
+		if extention == "mkv":
+			media_info = MediaInfo.parse(elem)
+			print("media-info: ...   " + str(len(media_info.tracks)))
+			need_trascode_audio = False
+			for elem_track in media_info.tracks:
+				data_print = "[" + str(elem_track.track_id) + "] " + str(elem_track.track_type)
+				#print('track_id     = ' + str(elem_track.track_id))
+				#print('track_type   = ' + str(elem_track.track_type))
+				if elem_track.track_type == "Audio":
+					data_print += " (" + str(elem_track.language) + ") enc=" + str(elem_track.format);
+					#print('language     = ' + str(elem_track.language))
+					#print('format       = ' + str(elem_track.format))
+					if elem_track.format != "Opus":
+						need_trascode_audio = True
+				elif elem_track.track_type == "Video":
+					data_print += " enc=" + str(elem_track.format);
+				print("    - " + data_print)
+				#print("media-info: ..." + str(dir(elem_track)))
+			if need_trascode_audio == False:
+				print("  ==> No transcoding, already in the good format...")
+				continue
+		
+		
+		"""
+		media_info = MediaInfo.parse(elem)
+		print("media-info: ..." + str(len(media_info.tracks)))
+		for elem_track in media_info.tracks:
+			print('track_type   = ' + str(elem_track.track_type))
+			print('track_id   = ' + str(elem_track.track_id))
+			print('language   = ' + str(elem_track.language))
+			#print("media-info: ..." + str(dir(elem_track)))
+		continue
+		"""
+		
+		
+		cmd_line = "ffmpeg -i "
+		cmd_line += elem.replace(" ", "\ ").replace("'", "\\'")
+		#cmd_line += " -threads 4 -vcodec libx264 -crf 22 -force_key_frames expr:gte\(t,n_forced*1\) -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace zzz_transcoded.mkv_tmp"
+		#cmd_line += " -threads 4 -vcodec copy -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace tmp_transcoded.avi"
+		#cmd_line += " -threads 4 -vcodec copy -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace tmp_transcoded.mp4"
+		#cmd_line += " -threads 6 -c:v libvpx-vp9 -lossless 1 -c:a libopus -b:a 128k -deinterlace tmp_transcoded.webm"
+		#cmd_line += " -threads 6 -c:v libvpx-vp9 -row-mt 1 -c:a libopus -b:a 128k -deinterlace tmp_transcoded.webm"
+		# -map 0:v ==> copy all video stream
+		# -map 0:a ==> copy all audio stream
+		# -map 0:s ==> copy all subtitle stream
+		
+		cmd_line += " -map 0:v -map 0:a -c:v copy -c:a libopus -b:a 128k -deinterlace -threads 6 zzz_transcoded.mkv_tmp"
+		#cmd_line += " -threads 4 -vcodec copy -acodec copy tmp_transcoded.webm"
+		ret = run_command(cmd_line)
+		print(" ret value = " + str(ret))
+		if ret == False:
+			print("[ERROR]    Trancode: error occured ...")
+			exit(-1)
+		print("    move in: '" + elem[:-len(extention)] + "mkv'")
+		cmd_line = "mv " + elem.replace(" ", "\ ").replace("'", "\\'") + " " + elem.replace(" ", "\ ").replace("'", "\\'") + "__"
+		ret = run_command(cmd_line)
+		cmd_line = "mv zzz_transcoded.mkv_tmp " + elem.replace(" ", "\ ").replace("'", "\\'")[:-len(extention)] + "mkv"
+		ret = run_command(cmd_line)
+		
+		
+		#cmd_line = "mv " + elem.replace(" ", "\ ").replace("'", "\\'") + " last_transcoded.ts"
+		#ret = run_command(cmd_line)
+		#break
+
+full_list_size = len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv) + len(list_files_divx)
+offset = 0;
+
+trancode_local(list_files_ts  , "ts",  full_list_size, offset)
+offset += len(list_files_ts)
+trancode_local(list_files_mp4 , "mp4", full_list_size, offset)
+offset += len(list_files_mp4)
+trancode_local(list_files_flv , "flv", full_list_size, offset)
+offset += len(list_files_flv)
+trancode_local(list_files_avi , "avi", full_list_size, offset)
+offset += len(list_files_avi)
+trancode_local(list_files_wmv , "wmv", full_list_size, offset)
+offset += len(list_files_wmv)
+trancode_local(list_files_divx , "divx", full_list_size, offset)
+offset += len(list_files_divx)
+trancode_local(list_files_mkv , "mkv", full_list_size, offset)
+offset += len(list_files_mkv)
+#trancode_local(list_files_webm , "webm", full_list_size, offset)
+#offset += len(list_files_webm)
+
+## extract a thumb from a video
+## ffmpeg -i Passenger.mkv -ss 00:05:00 -f image2 -vframes 1 thumb.jpg
+

front/src/app/element-video/element-video.component.ts

@@ -30,7 +30,7 @@ export class ElementVideoComponent implements OnInit {
 	episode:number = undefined
 	group_id:number = undefined
 	saison_id:number = undefined
-	sha512:string = ""
+	data_id:number = -1
 	time:number = undefined
 	type_id:number = undefined
 	generated_name:string = ""
@@ -53,11 +53,11 @@ export class ElementVideoComponent implements OnInit {
 				self.episode = response.episode;
 				self.group_id = response.group_id;
 				self.saison_id = response.saison_id;
-				self.sha512 = response.sha512;
+				self.data_id = response.data_id;
 				self.time = response.time;
 				self.generated_name = response.generated_name;
-				if (self.sha512 != "") {
+				if (self.data_id != -1) {
-					self.video_source = "http://localhost:15080/data/" + self.sha512 + ".mp4";
+					self.video_source = "http://localhost:15080/data/" + self.data_id + ".mp4";
 					self.video_enable = true;
 				} else {
 					self.video_source = "";
@@ -72,7 +72,7 @@ export class ElementVideoComponent implements OnInit {
 				self.episode = undefined;
 				self.group_id = undefined;
 				self.saison_id = undefined;
-				self.sha512 = "";
+				self.data_id = -1;
 				self.time = undefined;
 				self.generated_name = "";
 				self.video_source = "";

front/src/app/video-detail/video-detail.component.ts

@@ -29,7 +29,7 @@ export class VideoDetailComponent implements OnInit {
 	episode:number = undefined
 	group_id:number = undefined
 	saison_id:number = undefined
-	sha512:string = ""
+	data_id:number = -1
 	time:number = undefined
 	type_id:number = undefined
 	generated_name:string = ""
@@ -54,11 +54,11 @@ export class VideoDetailComponent implements OnInit {
 				self.episode = response.episode;
 				self.group_id = response.group_id;
 				self.saison_id = response.saison_id;
-				self.sha512 = response.sha512;
+				self.data_id = response.data_id;
 				self.time = response.time;
 				self.generated_name = response.generated_name;
-				if (self.sha512 != "") {
+				if (self.data_id != -1) {
-					self.video_source = "http://localhost:15080/data/" + self.sha512 + ".mp4";
+					self.video_source = "http://localhost:15080/data/" + self.data_id + ".mp4";
 				} else {
 					self.video_source = "";
 				}
@@ -71,7 +71,7 @@ export class VideoDetailComponent implements OnInit {
 				self.episode = undefined;
 				self.group_id = undefined;
 				self.saison_id = undefined;
-				self.sha512 = "";
+				self.data_id = -1;
 				self.time = undefined;
 				self.generated_name = "";
 				self.video_source = "";