kangaroo-and-rabbit/karideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[DEV] rework path of data with id only for reference

Browse Source
This commit is contained in:
Edouard DUPIN 2019-12-28 18:14:31 +01:00
parent 682b36c167
commit 6f19f0cf11
8 changed files with 364 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -6,6 +6,9 @@
/tmp
/out-tsc
dataPush
node_modules
# dependencies
applicationFront/node_modules

57
back/src/api/data.py
View File

@ -52,6 +52,19 @@ def add(_app, _name_api):
return response.json(data_global_elements.get_interface(_name_api).gets())
"""
class DataModelBdd:
id = int
size = int
sha512 = str
mime_type = str
original_name = [str, type(None)]
# creating time
create_date = str
data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd)
@elem_blueprint.post('/' + _name_api, strict_slashes=True, stream=True)
@doc.summary("send new file data")
@doc.description("Create a new data file (associated with his sha512.")
@ -82,38 +95,49 @@ def add(_app, _name_api):
sha1.update(body)
file_stream.close()
print("SHA512: " + str(sha1.hexdigest()))
destination_filename = os.path.join(_app.config['REST_MEDIA_DATA'], str(sha1.hexdigest()))
new_data = {
"size": total_size,
"sha512": str(sha1.hexdigest()),
'original_name': _request.headers["filename"],
'mime_type': _request.headers["mime-type"],
'create_date': datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
}
# TODO: Check if the element already exist ...
return_bdd = data_global_elements.get_interface(_name_api).post(new_data)
basic_data_path = os.path.join(_app.config['REST_MEDIA_DATA'], str(return_bdd["id"]))
if not os.path.exists(basic_data_path):
os.makedirs(basic_data_path)
destination_filename = os.path.join(basic_data_path, "video")
"""
if os.path.isfile(destination_filename) == True:
answer_data = {
"size": total_size,
"sha512": str(sha1.hexdigest()),
'filename': _request.headers["filename"],
'mime-type': _request.headers["mime-type"],
'mime_type': _request.headers["mime-type"],
"already_exist": True,
}
await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))
return
"""
# move the file
shutil.move(temporary_file, destination_filename)
# collect media info ...
media_info = MediaInfo.parse(destination_filename)
data_metafile = {
"sha512": str(sha1.hexdigest()),
"size": total_size,
'filename': _request.headers["filename"],
'mime-type': _request.headers["mime-type"],
'media-info': json.loads(media_info.to_json())
'mime_type': _request.headers["mime-type"],
'media_info': json.loads(media_info.to_json())
}
tools.file_write_data(destination_filename + ".meta", json.dumps(data_metafile, sort_keys=True, indent=4))
answer_data = {
"size": total_size,
"sha512": str(sha1.hexdigest()),
'filename': _request.headers["filename"],
'mime-type': _request.headers["mime-type"],
"already_exist": True,
}
await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))
tools.file_write_data(os.path.join(basic_data_path, "meta.json"), json.dumps(data_metafile, sort_keys=True, indent=4))
await _response.write(json.dumps(return_bdd, sort_keys=True, indent=4))
return response.stream(streaming, content_type='application/json')
@elem_blueprint.get('/' + _name_api + '/<id:string>', strict_slashes=True)
@ -130,9 +154,10 @@ def add(_app, _name_api):
id = id[:-4]
if id[-4:] == ".ts":
id = id[:-3]
filename = os.path.join(_app.config['REST_MEDIA_DATA'], id)
filename = os.path.join(_app.config['REST_MEDIA_DATA'], id, "video")
value = data_global_elements.get_interface(_name_api).get(id)
headers = {
'Content-Type': 'video/x-matroska',
'Content-Type': value["mime_type"],
'Accept-Ranges': 'Accept-Ranges: bytes'
}
try:

6
back/src/api/video.py
View File

@ -66,7 +66,7 @@ def add(_app, _name_api):
class DataModelBdd:
id = int
sha512 = str
data_id = int
type_id = int
saison_id = [int, type(None)]
episode = [int, type(None)]
@ -109,7 +109,7 @@ def add(_app, _name_api):
@doc.consumes(DataModel, location='body')#, required=True)
@doc.response_success(status=201, description='If successful created')
async def create(request):
for type_key in ["sha512","type_id","name"]:
for type_key in ["data_id","type_id","name"]:
if type_key not in request.json.keys():
raise ServerError("Bad Request: Missing Key '" + type_key + "'", status_code=400)
for type_key in ["create_date"]:
@ -120,7 +120,7 @@ def add(_app, _name_api):
request.json[type_key] = None
request.json["create_date"] = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
#Find if already exist
list_elem = data_global_elements.get_interface(_name_api).find(["group_id", "sha512"], request.json);
list_elem = data_global_elements.get_interface(_name_api).find(["group_id", "data_id"], request.json);
for elem in list_elem:
return response.json(elem)

1
back/src/app_video.py
View File

@ -118,6 +118,7 @@ def add_interface(_name, _default_value = None):
interface.reset_with_value(_default_value);
data_global_elements.add_interface(_name, interface)
add_interface(data_global_elements.API_DATA)
add_interface(data_global_elements.API_TYPE, default_values_type)
add_interface(data_global_elements.API_GROUP)
add_interface(data_global_elements.API_SAISON)

22
back/tools/sendLocalData.py
View File

@ -126,8 +126,9 @@ def push_video_file(_path, _basic_key={}):
return True
debug.info("Add media : '" + _path + "'")
if file_extension[1:] not in ["webm", "avi", "mkv", "mov", "mp4", "ts"] \
and file_name not in ["cover_1.jpg","cover_1.png", "cover_1.till", "cover_1.bmp", "cover_1.tga"]:
# "avi", , "mov", , "ts", "cover_1.tiff", "cover_1.bmp", "cover_1.tga"] copy only file that is supported by the html5 video player (chrome mode only)
if file_extension[1:] not in ["webm", "mkv", "mp4"] \
and file_name not in ["cover_1.jpg","cover_1.png"]:
debug.warning("Not send file : " + _path + " Not manage file_extension... " + file_extension)
return False
@ -217,10 +218,25 @@ def push_video_file(_path, _basic_key={}):
if mediaId == 0:
debug.error("Get media ID = 0 With no error");
return False;
"""
mime = magic.Magic(mime=True)
mime_type = mime.from_file(_path)
# do it by myself .. it is better ...
filename___, file_extension = os.path.splitext(_path)
if file_extension == "mkv":
mime_type = "video/x-matroska"
elif file_extension == "mka":
mime_type = "audio/x-matroska"
elif file_extension == "mp4":
mime_type = "video/mp4"
elif file_extension == "webm":
mime_type = "video/webm"
elif file_extension == "json":
mime_type = "application/json"
elif file_extension == "jpeg":
mime_type = "image/jpeg"
elif file_extension == "png":
mime_type = "image/png"
headers_values = {'filename': _path, 'mime-type': mime_type}
result_send_data = requests.post("http://127.0.0.1:15080/data", headers=headers_values, data=upload_in_chunks(_path, chunksize=4096))
debug.info("result *********** : " + str(result_send_data) + " " + result_send_data.text)

287
back/tools/videoTranscode.py Executable file
View File

@ -0,0 +1,287 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2016, Edouard DUPIN, all right reserved
##
## @license APACHE v2.0 (see license file)
##
import os
import fnmatch
import sys
import subprocess
import shlex
##
## @brief Execute the command with no get of output
##
def run_command(cmd_line):
# prepare command line:
args = shlex.split(cmd_line)
print("[INFO] cmd = " + str(args))
try:
# create the subprocess
p = subprocess.Popen(args)
except subprocess.CalledProcessError as e:
print("[ERROR] subprocess.CalledProcessError : " + str(args))
return False
#except:
# debug.error("Exception on : " + str(args))
# launch the subprocess:
output, err = p.communicate()
# Check error :
if p.returncode == 0:
return True
else:
return False
##
## @brief Get list of all Files in a specific path (with a regex)
## @param[in] path (string) Full path of the machine to search files (start with / or x:)
## @param[in] regex (string) Regular expression to search data
## @param[in] recursive (bool) List file with recursive search
## @param[in] remove_path (string) Data to remove in the path
## @return (list) return files requested
##
def get_list_of_file_in_path(path, regex="*", recursive = False, remove_path=""):
out = []
if os.path.isdir(os.path.realpath(path)):
tmp_path = os.path.realpath(path)
tmp_rule = regex
else:
debug.error("path does not exist : '" + str(path) + "'")
for root, dirnames, filenames in os.walk(tmp_path):
deltaRoot = root[len(tmp_path):]
while len(deltaRoot) > 0 \
and ( deltaRoot[0] == '/' \
or deltaRoot[0] == '\\' ):
deltaRoot = deltaRoot[1:]
if recursive == False \
and deltaRoot != "":
return out
tmpList = filenames
if len(tmp_rule) > 0:
tmpList = fnmatch.filter(filenames, tmp_rule)
# Import the module :
for cycleFile in tmpList:
#for cycleFile in filenames:
add_file = os.path.join(tmp_path, deltaRoot, cycleFile)
if len(remove_path) != 0:
if add_file[:len(remove_path)] != remove_path:
print("ERROR : Request remove start of a path that is not the same: '" + add_file[:len(remove_path)] + "' demand remove of '" + str(remove_path) + "'")
else:
add_file = add_file[len(remove_path)+1:]
out.append(add_file)
return out;
#ffmpeg -i 000.ts -threads 0 -vcodec libx264 -crf 20 -force_key_frames expr:gte\(t,n_forced*1\) -s 720x540 -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace 000_transcoded.ts
#ffmpeg -i 000.ts -threads 0 -vcodec libx264 -crf 20 -force_key_frames expr:gte\(t,n_forced*1\) -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace 000_transcoded.ts
list_files_ts = get_list_of_file_in_path('.', "*.ts")
list_files_flv = get_list_of_file_in_path('.', "*.flv")
list_files_mp4 = get_list_of_file_in_path('.', "*.mp4")
list_files_avi = get_list_of_file_in_path('.', "*.avi")
list_files_mkv = get_list_of_file_in_path('.', "*.mkv")
list_files_wmv = get_list_of_file_in_path('.', "*.wmv")
list_files_divx = get_list_of_file_in_path('.', "*.divx")
list_files_webm = get_list_of_file_in_path('.', "*.webm")
# remove all encoded element in the other files (TS)
for elem_mkv in list_files_mkv:
index = 0
for elem_ts in list_files_ts:
if elem_mkv[:-3]+"ts" == elem_ts:
break;
index += 1
if index != len(list_files_ts):
print("[INFO] remove from list '" + list_files_ts[index] + "' ==> already transcoded")
del list_files_ts[index]
# remove all encoded element in the other files (FLV)
for elem_mkv in list_files_mkv:
index = 0
for elem_flv in list_files_flv:
if elem_mkv[:-3]+"flv" == elem_flv:
break;
index += 1
if index != len(list_files_flv):
print("[INFO] remove from list '" + list_files_flv[index] + "' ==> already transcoded")
del list_files_flv[index]
# remove all encoded element in the other files (mp4)
for elem_mkv in list_files_mkv:
index = 0
for elem_mp4 in list_files_mp4:
if elem_mkv[:-3]+"mp4" == elem_mp4:
break;
index += 1
if index != len(list_files_mp4):
print("[INFO] remove from list '" + list_files_mp4[index] + "' ==> already transcoded")
del list_files_mp4[index]
# remove all encoded element in the other files (TS)
for elem_mkv in list_files_mkv:
index = 0
for elem_avi in list_files_avi:
if elem_mkv[:-3]+"ts" == elem_avi:
break;
index += 1
if index != len(list_files_avi):
print("[INFO] remove from list '" + list_files_avi[index] + "' ==> already transcoded")
del list_files_avi[index]
# remove all encoded element in the other files (wmv)
for elem_mkv in list_files_mkv:
index = 0
for elem_wmv in list_files_wmv:
if elem_mkv[:-3]+"wmv" == elem_wmv:
break;
index += 1
if index != len(list_files_wmv):
print("[INFO] remove from list '" + list_files_wmv[index] + "' ==> already transcoded")
del list_files_wmv[index]
# remove all encoded element in the other files (divx)
for elem_mkv in list_files_mkv:
index = 0
for elem_divx in list_files_divx:
if elem_mkv[:-3]+"divx" == elem_divx:
break;
index += 1
if index != len(list_files_divx):
print("[INFO] remove from list '" + list_files_divx[index] + "' ==> already transcoded")
del list_files_divx[index]
print("list of elements TS : ")
for elem in list_files_ts:
print(" - '" + elem + "'")
print("list of elements MP4 : ")
for elem in list_files_mp4:
print(" - '" + elem + "'")
print("list of elements FLV : ")
for elem in list_files_flv:
print(" - '" + elem + "'")
print("list of elements AVI : ")
for elem in list_files_avi:
print(" - '" + elem + "'")
print("list of elements WMV : ")
for elem in list_files_wmv:
print(" - '" + elem + "'")
print("list of elements MKV : ")
for elem in list_files_mkv:
print(" - '" + elem + "'")
print("list of elements divx : ")
for elem in list_files_divx:
print(" - '" + elem + "'")
print("list of elements webm : ")
for elem in list_files_webm:
print(" - '" + elem + "'")
from pymediainfo import MediaInfo
def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offset=0) :
print("Start strancoding: '." + extention + "' ... " + str(len(list_of_file)))
id_elem = 0
for elem in list_of_file:
id_elem += 1
print(" ========================================================================================")
print(" == " + str(offset+id_elem) + " / " + str(total_count_of_file))
print(" == Trancode: '" + elem.replace("'", "\'") + "'")
print(" ========================================================================================")
# collect media info ...
#if it is a mk: .. chack the opus format...
if extention == "mkv":
media_info = MediaInfo.parse(elem)
print("media-info: ... " + str(len(media_info.tracks)))
need_trascode_audio = False
for elem_track in media_info.tracks:
data_print = "[" + str(elem_track.track_id) + "] " + str(elem_track.track_type)
#print('track_id = ' + str(elem_track.track_id))
#print('track_type = ' + str(elem_track.track_type))
if elem_track.track_type == "Audio":
data_print += " (" + str(elem_track.language) + ") enc=" + str(elem_track.format);
#print('language = ' + str(elem_track.language))
#print('format = ' + str(elem_track.format))
if elem_track.format != "Opus":
need_trascode_audio = True
elif elem_track.track_type == "Video":
data_print += " enc=" + str(elem_track.format);
print(" - " + data_print)
#print("media-info: ..." + str(dir(elem_track)))
if need_trascode_audio == False:
print(" ==> No transcoding, already in the good format...")
continue
"""
media_info = MediaInfo.parse(elem)
print("media-info: ..." + str(len(media_info.tracks)))
for elem_track in media_info.tracks:
print('track_type = ' + str(elem_track.track_type))
print('track_id = ' + str(elem_track.track_id))
print('language = ' + str(elem_track.language))
#print("media-info: ..." + str(dir(elem_track)))
continue
"""
cmd_line = "ffmpeg -i "
cmd_line += elem.replace(" ", "\ ").replace("'", "\\'")
#cmd_line += " -threads 4 -vcodec libx264 -crf 22 -force_key_frames expr:gte\(t,n_forced*1\) -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace zzz_transcoded.mkv_tmp"
#cmd_line += " -threads 4 -vcodec copy -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace tmp_transcoded.avi"
#cmd_line += " -threads 4 -vcodec copy -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace tmp_transcoded.mp4"
#cmd_line += " -threads 6 -c:v libvpx-vp9 -lossless 1 -c:a libopus -b:a 128k -deinterlace tmp_transcoded.webm"
#cmd_line += " -threads 6 -c:v libvpx-vp9 -row-mt 1 -c:a libopus -b:a 128k -deinterlace tmp_transcoded.webm"
# -map 0:v ==> copy all video stream
# -map 0:a ==> copy all audio stream
# -map 0:s ==> copy all subtitle stream
cmd_line += " -map 0:v -map 0:a -c:v copy -c:a libopus -b:a 128k -deinterlace -threads 6 zzz_transcoded.mkv_tmp"
#cmd_line += " -threads 4 -vcodec copy -acodec copy tmp_transcoded.webm"
ret = run_command(cmd_line)
print(" ret value = " + str(ret))
if ret == False:
print("[ERROR] Trancode: error occured ...")
exit(-1)
print(" move in: '" + elem[:-len(extention)] + "mkv'")
cmd_line = "mv " + elem.replace(" ", "\ ").replace("'", "\\'") + " " + elem.replace(" ", "\ ").replace("'", "\\'") + "__"
ret = run_command(cmd_line)
cmd_line = "mv zzz_transcoded.mkv_tmp " + elem.replace(" ", "\ ").replace("'", "\\'")[:-len(extention)] + "mkv"
ret = run_command(cmd_line)
#cmd_line = "mv " + elem.replace(" ", "\ ").replace("'", "\\'") + " last_transcoded.ts"
#ret = run_command(cmd_line)
#break
full_list_size = len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv) + len(list_files_divx)
offset = 0;
trancode_local(list_files_ts , "ts", full_list_size, offset)
offset += len(list_files_ts)
trancode_local(list_files_mp4 , "mp4", full_list_size, offset)
offset += len(list_files_mp4)
trancode_local(list_files_flv , "flv", full_list_size, offset)
offset += len(list_files_flv)
trancode_local(list_files_avi , "avi", full_list_size, offset)
offset += len(list_files_avi)
trancode_local(list_files_wmv , "wmv", full_list_size, offset)
offset += len(list_files_wmv)
trancode_local(list_files_divx , "divx", full_list_size, offset)
offset += len(list_files_divx)
trancode_local(list_files_mkv , "mkv", full_list_size, offset)
offset += len(list_files_mkv)
#trancode_local(list_files_webm , "webm", full_list_size, offset)
#offset += len(list_files_webm)
## extract a thumb from a video
## ffmpeg -i Passenger.mkv -ss 00:05:00 -f image2 -vframes 1 thumb.jpg

10
front/src/app/element-video/element-video.component.ts
View File

@ -30,7 +30,7 @@ export class ElementVideoComponent implements OnInit {
episode:number = undefined
group_id:number = undefined
saison_id:number = undefined
sha512:string = ""
data_id:number = -1
time:number = undefined
type_id:number = undefined
generated_name:string = ""
@ -53,11 +53,11 @@ export class ElementVideoComponent implements OnInit {
self.episode = response.episode;
self.group_id = response.group_id;
self.saison_id = response.saison_id;
self.sha512 = response.sha512;
self.data_id = response.data_id;
self.time = response.time;
self.generated_name = response.generated_name;
if (self.sha512 != "") {
self.video_source = "http://localhost:15080/data/" + self.sha512 + ".mp4";
if (self.data_id != -1) {
self.video_source = "http://localhost:15080/data/" + self.data_id + ".mp4";
self.video_enable = true;
} else {
self.video_source = "";
@ -72,7 +72,7 @@ export class ElementVideoComponent implements OnInit {
self.episode = undefined;
self.group_id = undefined;
self.saison_id = undefined;
self.sha512 = "";
self.data_id = -1;
self.time = undefined;
self.generated_name = "";
self.video_source = "";

10
front/src/app/video-detail/video-detail.component.ts
View File

@ -29,7 +29,7 @@ export class VideoDetailComponent implements OnInit {
episode:number = undefined
group_id:number = undefined
saison_id:number = undefined
sha512:string = ""
data_id:number = -1
time:number = undefined
type_id:number = undefined
generated_name:string = ""
@ -54,11 +54,11 @@ export class VideoDetailComponent implements OnInit {
self.episode = response.episode;
self.group_id = response.group_id;
self.saison_id = response.saison_id;
self.sha512 = response.sha512;
self.data_id = response.data_id;
self.time = response.time;
self.generated_name = response.generated_name;
if (self.sha512 != "") {
self.video_source = "http://localhost:15080/data/" + self.sha512 + ".mp4";
if (self.data_id != -1) {
self.video_source = "http://localhost:15080/data/" + self.data_id + ".mp4";
} else {
self.video_source = "";
}
@ -71,7 +71,7 @@ export class VideoDetailComponent implements OnInit {
self.episode = undefined;
self.group_id = undefined;
self.saison_id = undefined;
self.sha512 = "";
self.data_id = -1;
self.time = undefined;
self.generated_name = "";
self.video_source = "";