kangaroo-and-rabbit/karideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[DEV] add tools

Browse Source
This commit is contained in:
Edouard DUPIN 2020-01-11 23:28:18 +00:00
parent 15c2362f97
commit a7d340922f
4 changed files with 277 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
back/src/tools.py
View File

@ -122,6 +122,14 @@ def file_write_data_safe(path, data):
shutil.move(path + ".tmp", path)
return True
def file_move(path_src, path_dst):
#real write of data:
create_directory_of_file(path_dst)
shutil.move(path_src, path_dst)
return True
def list_to_str(list):
if type(list) == type(str()):
return list + " "

9
back/tools/sendLocalData.py
View File

@ -14,7 +14,7 @@ import datetime
import hashlib
import requests # pip install requests
import realog.debug as debug
import magic
#import magic
import json
debug.enable_color();
@ -252,9 +252,10 @@ def push_video_file(_path, _basic_key={}):
debug.error("Get media ID = 0 With no error");
return False;
"""
mime = magic.Magic(mime=True)
mime_type = mime.from_file(_path)
# do it by myself .. it is better ...
#mime = magic.Magic(mime=True)
#mime_type = mime.from_file(_path)
mime_type = "unknown"
# do it by myself .. it is better ...
filename___, file_extension = os.path.splitext(_path)
if file_extension == "mkv":
mime_type = "video/x-matroska"

115
back/tools/videoTranscode.py
View File

@ -79,15 +79,32 @@ def get_list_of_file_in_path(path, regex="*", recursive = False, remove_path="")
#ffmpeg -i 000.ts -threads 0 -vcodec libx264 -crf 20 -force_key_frames expr:gte\(t,n_forced*1\) -s 720x540 -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace 000_transcoded.ts
#ffmpeg -i 000.ts -threads 0 -vcodec libx264 -crf 20 -force_key_frames expr:gte\(t,n_forced*1\) -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace 000_transcoded.ts
list_files_ts = get_list_of_file_in_path('.', "*.ts")
list_files_flv = get_list_of_file_in_path('.', "*.flv")
list_files_mp4 = get_list_of_file_in_path('.', "*.mp4")
list_files_avi = get_list_of_file_in_path('.', "*.avi")
list_files_mkv = get_list_of_file_in_path('.', "*.mkv")
list_files_wmv = get_list_of_file_in_path('.', "*.wmv")
list_files_divx = get_list_of_file_in_path('.', "*.divx")
list_files_webm = get_list_of_file_in_path('.', "*.webm")
"""
def remove_group(list_of_file=[], total_count_of_file=0):
id_elem = 0
for elem in list_of_file:
id_elem += 1
tmpfile_name = elem.replace(" ", "\ ").replace("!", "\\!").replace("'", "\\'")
print(" [" + str(id_elem) + " / " + str(total_count_of_file) + "] " + tmpfile_name)
cmd_line = "rm " + tmpfile_name
ret = run_command(cmd_line)
list_files_tmp = get_list_of_file_in_path('.', "*__", recursive = True)
remove_group(list_files_tmp, len(list_files_tmp))
list_files_sha512 = get_list_of_file_in_path('.', "*.sha512", recursive = True)
remove_group(list_files_sha512, len(list_files_sha512))
exit(0)
"""
list_files_ts = get_list_of_file_in_path('.', "*.ts", recursive = True)
list_files_flv = get_list_of_file_in_path('.', "*.flv", recursive = True)
list_files_mp4 = get_list_of_file_in_path('.', "*.mp4", recursive = True)
list_files_avi = get_list_of_file_in_path('.', "*.avi", recursive = True)
list_files_mkv = get_list_of_file_in_path('.', "*.mkv", recursive = True)
list_files_wmv = get_list_of_file_in_path('.', "*.wmv", recursive = True)
list_files_divx = get_list_of_file_in_path('.', "*.divx", recursive = True)
list_files_webm = get_list_of_file_in_path('.', "*.webm", recursive = True)
"""
# remove all encoded element in the other files (TS)
for elem_mkv in list_files_mkv:
index = 0
@ -158,36 +175,52 @@ for elem_mkv in list_files_mkv:
if index != len(list_files_divx):
print("[INFO] remove from list '" + list_files_divx[index] + "' ==> already transcoded")
del list_files_divx[index]
"""
print("list of elements TS : ")
for elem in list_files_ts:
print(" - '" + elem + "'")
print(" - '" + str(elem) + "'")
print("list of elements MP4 : ")
for elem in list_files_mp4:
print(" - '" + elem + "'")
print(" - '" + str(elem) + "'")
print("list of elements FLV : ")
for elem in list_files_flv:
print(" - '" + elem + "'")
print(" - '" + str(elem) + "'")
print("list of elements AVI : ")
for elem in list_files_avi:
print(" - '" + elem + "'")
print(" - '" + str(elem) + "'")
print("list of elements WMV : ")
for elem in list_files_wmv:
print(" - '" + elem + "'")
print(" - '" + str(elem) + "'")
print("list of elements MKV : ")
for elem in list_files_mkv:
print(" - '" + elem + "'")
print(" - '" + str(elem) + "'")
print("list of elements divx : ")
for elem in list_files_divx:
print(" - '" + elem + "'")
print(" - '" + str(elem) + "'")
print("list of elements webm : ")
for elem in list_files_webm:
print(" - '" + elem + "'")
print(" - '" + str(elem) + "'")
import random
from pymediainfo import MediaInfo
for arg in sys.argv:
print("arg: " + arg)
id_value = 0
if len(sys.argv) == 2:
id_value = int(sys.argv[1])
tmp_name_encoded_file = "zzz_transcoded_" + str(id_value) + ".mkv"
print("lement name: " + tmp_name_encoded_file)
element_error=[]
def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offset=0) :
global element_error;
print("Start strancoding: '." + extention + "' ... " + str(len(list_of_file)))
id_elem = 0
for elem in list_of_file:
@ -196,6 +229,13 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse
print(" == " + str(offset+id_elem) + " / " + str(total_count_of_file))
print(" == Trancode: '" + elem.replace("'", "\'") + "'")
print(" ========================================================================================")
if not os.path.isfile(elem):
print(" ==> file does not exist")
continue
cmd_line = "rm " + tmp_name_encoded_file
ret = run_command(cmd_line)
# collect media info ...
#if it is a mk: .. chack the opus format...
if extention == "mkv":
@ -232,8 +272,11 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse
continue
"""
cmd_line = "ffmpeg -i "
if extention != "mkv":
cmd_line = "ffmpeg -fflags +genpts -i "
#cmd_line = "ffmpeg -fflags +igndts -i "
else:
cmd_line = "ffmpeg -i "
cmd_line += elem.replace(" ", "\ ").replace("'", "\\'")
#cmd_line += " -threads 4 -vcodec libx264 -crf 22 -force_key_frames expr:gte\(t,n_forced*1\) -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace zzz_transcoded.mkv_tmp"
#cmd_line += " -threads 4 -vcodec copy -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace tmp_transcoded.avi"
@ -244,17 +287,20 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse
# -map 0:a ==> copy all audio stream
# -map 0:s ==> copy all subtitle stream
cmd_line += " -map 0:v -map 0:a -c:v copy -c:a libopus -b:a 128k -deinterlace -threads 6 zzz_transcoded.mkv_tmp"
cmd_line += " -map 0:v -map 0:a -c:v copy -c:a libopus -ac 2 -b:a 192k -r:a 48000 -deinterlace -threads 6 " + tmp_name_encoded_file
#cmd_line += " -threads 4 -vcodec copy -acodec copy tmp_transcoded.webm"
ret = run_command(cmd_line)
print(" ret value = " + str(ret))
if ret == False:
print("[ERROR] Trancode: error occured ...")
exit(-1)
element_error.append(elem)
#exit(-1)
continue
print(" move in: '" + elem[:-len(extention)] + "mkv'")
cmd_line = "mv " + elem.replace(" ", "\ ").replace("'", "\\'") + " " + elem.replace(" ", "\ ").replace("'", "\\'") + "__"
# cmd_line = "mv " + elem.replace(" ", "\ ").replace("'", "\\'") + " last_transcoded.xx"
cmd_line = "mv " + elem.replace(" ", "\ ").replace("!", "\\!").replace("'", "\\'") + " last_transcoded"
ret = run_command(cmd_line)
cmd_line = "mv zzz_transcoded.mkv_tmp " + elem.replace(" ", "\ ").replace("'", "\\'")[:-len(extention)] + "mkv"
cmd_line = "mv " + tmp_name_encoded_file + " " + elem.replace(" ", "\ ").replace("!", "\\!").replace("'", "\\'")[:-len(extention)] + "mkv"
ret = run_command(cmd_line)
@ -262,9 +308,25 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse
#ret = run_command(cmd_line)
#break
full_list_size = len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv) + len(list_files_divx)
full_list_size = len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv) + len(list_files_divx) + len(list_files_mkv) + len(list_files_webm)
offset = 0;
reverse_sort = False
list_files_ts.sort(reverse=reverse_sort)
list_files_mp4.sort(reverse=reverse_sort)
list_files_flv.sort(reverse=reverse_sort)
list_files_avi.sort(reverse=reverse_sort)
list_files_wmv.sort(reverse=reverse_sort)
list_files_divx.sort(reverse=reverse_sort)
list_files_mkv.sort(reverse=reverse_sort)
list_files_webm.sort(reverse=reverse_sort)
random.shuffle(list_files_mp4)
random.shuffle(list_files_avi)
random.shuffle(list_files_mkv)
trancode_local(list_files_ts , "ts", full_list_size, offset)
offset += len(list_files_ts)
trancode_local(list_files_mp4 , "mp4", full_list_size, offset)
@ -282,6 +344,11 @@ offset += len(list_files_mkv)
#trancode_local(list_files_webm , "webm", full_list_size, offset)
#offset += len(list_files_webm)
print("List error transcode: " + len(element_error))
for elem in element_error:
print(" == Trancode: '" + elem.replace("'", "\'") + "'")
## extract a thumb from a video
## ffmpeg -i Passenger.mkv -ss 00:05:00 -f image2 -vframes 1 thumb.jpg

173
back/tools/video_move_correct.py Executable file
View File

@ -0,0 +1,173 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2016, Edouard DUPIN, all right reserved
##
## @license APACHE v2.0 (see license file)
##
import os
import fnmatch
import sys
import subprocess
import shlex
import shutil
##
## @brief Execute the command with no get of output
##
def run_command(cmd_line):
# prepare command line:
args = shlex.split(cmd_line)
print("[INFO] cmd = " + str(args))
try:
# create the subprocess
p = subprocess.Popen(args)
except subprocess.CalledProcessError as e:
print("[ERROR] subprocess.CalledProcessError : " + str(args))
return False
#except:
# debug.error("Exception on : " + str(args))
# launch the subprocess:
output, err = p.communicate()
# Check error :
if p.returncode == 0:
return True
else:
return False
##
## @brief Get list of all Files in a specific path (with a regex)
## @param[in] path (string) Full path of the machine to search files (start with / or x:)
## @param[in] regex (string) Regular expression to search data
## @param[in] recursive (bool) List file with recursive search
## @param[in] remove_path (string) Data to remove in the path
## @return (list) return files requested
##
def get_list_of_file_in_path(path, regex="*", recursive = False, remove_path=""):
out = []
if os.path.isdir(os.path.realpath(path)):
tmp_path = os.path.realpath(path)
tmp_rule = regex
else:
debug.error("path does not exist : '" + str(path) + "'")
for root, dirnames, filenames in os.walk(tmp_path):
deltaRoot = root[len(tmp_path):]
while len(deltaRoot) > 0 \
and ( deltaRoot[0] == '/' \
or deltaRoot[0] == '\\' ):
deltaRoot = deltaRoot[1:]
if recursive == False \
and deltaRoot != "":
return out
tmpList = filenames
if len(tmp_rule) > 0:
tmpList = fnmatch.filter(filenames, tmp_rule)
# Import the module :
for cycleFile in tmpList:
#for cycleFile in filenames:
add_file = os.path.join(tmp_path, deltaRoot, cycleFile)
if len(remove_path) != 0:
if add_file[:len(remove_path)] != remove_path:
print("ERROR : Request remove start of a path that is not the same: '" + add_file[:len(remove_path)] + "' demand remove of '" + str(remove_path) + "'")
else:
add_file = add_file[len(remove_path)+1:]
out.append(add_file)
return out;
def get_run_path():
return os.getcwd()
src_path = get_run_path()
dst_path = os.path.join(src_path, "..", "zzz_video_push_correct")
list_files_mkv = get_list_of_file_in_path(src_path, "*.mkv", recursive = True)
list_files_webm = get_list_of_file_in_path(src_path, "*.webm", recursive = True)
list_files_jpg = get_list_of_file_in_path(src_path, "*.jpg", recursive = True)
list_files_png = get_list_of_file_in_path(src_path, "*.png", recursive = True)
print("list of elements MKV : ")
for elem in list_files_mkv:
print(" - '" + str(elem) + "'")
print("list of elements webm : ")
for elem in list_files_webm:
print(" - '" + str(elem) + "'")
import random
from pymediainfo import MediaInfo
for arg in sys.argv:
print("arg: " + arg)
id_value = 0
if len(sys.argv) == 2:
id_value = int(sys.argv[1])
def create_directory_of_file(file):
path = os.path.dirname(file)
try:
os.stat(path)
except:
os.makedirs(path)
def file_move(path_src, path_dst):
#real write of data:
print("kljlkjlkjklj " + path_src)
print("kljlkjlkjklj " + path_dst)
create_directory_of_file(path_dst)
shutil.move(path_src, path_dst)
return True
def move_local(list_of_file=[], extention="mkv") :
global element_error;
print("Start strancoding: '." + extention + "' ... " + str(len(list_of_file)))
id_elem = 0
total_count_of_file = len(list_of_file)
for elem in list_of_file:
id_elem += 1
print(" ========================================================================================")
print(" == " + str(id_elem) + " / " + str(total_count_of_file))
print(" == Trancode: '" + elem.replace("'", "\'") + "'")
print(" ========================================================================================")
if not os.path.isfile(elem):
print(" ==> file does not exist")
continue
# collect media info ...
#if it is a mk: .. chack the opus format...
if extention == "mkv":
media_info = MediaInfo.parse(elem)
print("media-info: ... " + str(len(media_info.tracks)))
need_move_file = True
for elem_track in media_info.tracks:
data_print = "[" + str(elem_track.track_id) + "] " + str(elem_track.track_type)
#print('track_id = ' + str(elem_track.track_id))
#print('track_type = ' + str(elem_track.track_type))
if elem_track.track_type == "Audio":
data_print += " (" + str(elem_track.language) + ") enc=" + str(elem_track.format);
#print('language = ' + str(elem_track.language))
#print('format = ' + str(elem_track.format))
if elem_track.format != "Opus":
need_move_file = False
elif elem_track.track_type == "Video":
data_print += " enc=" + str(elem_track.format);
if elem_track.format != "AVC":
need_move_file = False
print(" - " + data_print)
#print("media-info: ..." + str(dir(elem_track)))
if need_move_file == False:
print(" ==> Need transcode, NOT already in the good format...")
continue
file_move(elem, os.path.join(dst_path, elem[len(src_path)+1:]))
move_local(list_files_mkv, "mkv")
move_local(list_files_webm, "webm")
move_local(list_files_jpg, "jpg")
move_local(list_files_png, "png")