From a7d340922f1ab60e551eaa385a2029c6b10ab0e8 Mon Sep 17 00:00:00 2001 From: Edouard DUPIN Date: Sat, 11 Jan 2020 23:28:18 +0000 Subject: [PATCH] [DEV] add tools --- back/src/tools.py | 8 ++ back/tools/sendLocalData.py | 9 +- back/tools/videoTranscode.py | 115 +++++++++++++++----- back/tools/video_move_correct.py | 173 +++++++++++++++++++++++++++++++ 4 files changed, 277 insertions(+), 28 deletions(-) create mode 100755 back/tools/video_move_correct.py diff --git a/back/src/tools.py b/back/src/tools.py index 9851f86..225fb38 100644 --- a/back/src/tools.py +++ b/back/src/tools.py @@ -122,6 +122,14 @@ def file_write_data_safe(path, data): shutil.move(path + ".tmp", path) return True + +def file_move(path_src, path_dst): + #real write of data: + create_directory_of_file(path_dst) + shutil.move(path_src, path_dst) + return True + + def list_to_str(list): if type(list) == type(str()): return list + " " diff --git a/back/tools/sendLocalData.py b/back/tools/sendLocalData.py index 60fd281..7cfab24 100755 --- a/back/tools/sendLocalData.py +++ b/back/tools/sendLocalData.py @@ -14,7 +14,7 @@ import datetime import hashlib import requests # pip install requests import realog.debug as debug -import magic +#import magic import json debug.enable_color(); @@ -252,9 +252,10 @@ def push_video_file(_path, _basic_key={}): debug.error("Get media ID = 0 With no error"); return False; """ - mime = magic.Magic(mime=True) - mime_type = mime.from_file(_path) - # do it by myself .. it is better ... + #mime = magic.Magic(mime=True) + #mime_type = mime.from_file(_path) + mime_type = "unknown" + # do it by myself .. it is better ... filename___, file_extension = os.path.splitext(_path) if file_extension == "mkv": mime_type = "video/x-matroska" diff --git a/back/tools/videoTranscode.py b/back/tools/videoTranscode.py index bdf656e..180af63 100755 --- a/back/tools/videoTranscode.py +++ b/back/tools/videoTranscode.py @@ -79,15 +79,32 @@ def get_list_of_file_in_path(path, regex="*", recursive = False, remove_path="") #ffmpeg -i 000.ts -threads 0 -vcodec libx264 -crf 20 -force_key_frames expr:gte\(t,n_forced*1\) -s 720x540 -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace 000_transcoded.ts #ffmpeg -i 000.ts -threads 0 -vcodec libx264 -crf 20 -force_key_frames expr:gte\(t,n_forced*1\) -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace 000_transcoded.ts -list_files_ts = get_list_of_file_in_path('.', "*.ts") -list_files_flv = get_list_of_file_in_path('.', "*.flv") -list_files_mp4 = get_list_of_file_in_path('.', "*.mp4") -list_files_avi = get_list_of_file_in_path('.', "*.avi") -list_files_mkv = get_list_of_file_in_path('.', "*.mkv") -list_files_wmv = get_list_of_file_in_path('.', "*.wmv") -list_files_divx = get_list_of_file_in_path('.', "*.divx") -list_files_webm = get_list_of_file_in_path('.', "*.webm") +""" +def remove_group(list_of_file=[], total_count_of_file=0): + id_elem = 0 + for elem in list_of_file: + id_elem += 1 + tmpfile_name = elem.replace(" ", "\ ").replace("!", "\\!").replace("'", "\\'") + print(" [" + str(id_elem) + " / " + str(total_count_of_file) + "] " + tmpfile_name) + cmd_line = "rm " + tmpfile_name + ret = run_command(cmd_line) +list_files_tmp = get_list_of_file_in_path('.', "*__", recursive = True) +remove_group(list_files_tmp, len(list_files_tmp)) +list_files_sha512 = get_list_of_file_in_path('.', "*.sha512", recursive = True) +remove_group(list_files_sha512, len(list_files_sha512)) +exit(0) +""" + +list_files_ts = get_list_of_file_in_path('.', "*.ts", recursive = True) +list_files_flv = get_list_of_file_in_path('.', "*.flv", recursive = True) +list_files_mp4 = get_list_of_file_in_path('.', "*.mp4", recursive = True) +list_files_avi = get_list_of_file_in_path('.', "*.avi", recursive = True) +list_files_mkv = get_list_of_file_in_path('.', "*.mkv", recursive = True) +list_files_wmv = get_list_of_file_in_path('.', "*.wmv", recursive = True) +list_files_divx = get_list_of_file_in_path('.', "*.divx", recursive = True) +list_files_webm = get_list_of_file_in_path('.', "*.webm", recursive = True) +""" # remove all encoded element in the other files (TS) for elem_mkv in list_files_mkv: index = 0 @@ -158,36 +175,52 @@ for elem_mkv in list_files_mkv: if index != len(list_files_divx): print("[INFO] remove from list '" + list_files_divx[index] + "' ==> already transcoded") del list_files_divx[index] +""" print("list of elements TS : ") for elem in list_files_ts: - print(" - '" + elem + "'") + print(" - '" + str(elem) + "'") print("list of elements MP4 : ") for elem in list_files_mp4: - print(" - '" + elem + "'") + print(" - '" + str(elem) + "'") print("list of elements FLV : ") for elem in list_files_flv: - print(" - '" + elem + "'") + print(" - '" + str(elem) + "'") print("list of elements AVI : ") for elem in list_files_avi: - print(" - '" + elem + "'") + print(" - '" + str(elem) + "'") print("list of elements WMV : ") for elem in list_files_wmv: - print(" - '" + elem + "'") + print(" - '" + str(elem) + "'") print("list of elements MKV : ") for elem in list_files_mkv: - print(" - '" + elem + "'") + print(" - '" + str(elem) + "'") print("list of elements divx : ") for elem in list_files_divx: - print(" - '" + elem + "'") + print(" - '" + str(elem) + "'") print("list of elements webm : ") for elem in list_files_webm: - print(" - '" + elem + "'") - + print(" - '" + str(elem) + "'") +import random from pymediainfo import MediaInfo +for arg in sys.argv: + print("arg: " + arg) + +id_value = 0 +if len(sys.argv) == 2: + id_value = int(sys.argv[1]) + +tmp_name_encoded_file = "zzz_transcoded_" + str(id_value) + ".mkv" + +print("lement name: " + tmp_name_encoded_file) + +element_error=[] + + def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offset=0) : + global element_error; print("Start strancoding: '." + extention + "' ... " + str(len(list_of_file))) id_elem = 0 for elem in list_of_file: @@ -196,6 +229,13 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse print(" == " + str(offset+id_elem) + " / " + str(total_count_of_file)) print(" == Trancode: '" + elem.replace("'", "\'") + "'") print(" ========================================================================================") + if not os.path.isfile(elem): + print(" ==> file does not exist") + continue + + cmd_line = "rm " + tmp_name_encoded_file + ret = run_command(cmd_line) + # collect media info ... #if it is a mk: .. chack the opus format... if extention == "mkv": @@ -232,8 +272,11 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse continue """ - - cmd_line = "ffmpeg -i " + if extention != "mkv": + cmd_line = "ffmpeg -fflags +genpts -i " + #cmd_line = "ffmpeg -fflags +igndts -i " + else: + cmd_line = "ffmpeg -i " cmd_line += elem.replace(" ", "\ ").replace("'", "\\'") #cmd_line += " -threads 4 -vcodec libx264 -crf 22 -force_key_frames expr:gte\(t,n_forced*1\) -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace zzz_transcoded.mkv_tmp" #cmd_line += " -threads 4 -vcodec copy -acodec mp2 -ac 2 -ab 192k -ar 48000 -async 1 -deinterlace tmp_transcoded.avi" @@ -244,17 +287,20 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse # -map 0:a ==> copy all audio stream # -map 0:s ==> copy all subtitle stream - cmd_line += " -map 0:v -map 0:a -c:v copy -c:a libopus -b:a 128k -deinterlace -threads 6 zzz_transcoded.mkv_tmp" + cmd_line += " -map 0:v -map 0:a -c:v copy -c:a libopus -ac 2 -b:a 192k -r:a 48000 -deinterlace -threads 6 " + tmp_name_encoded_file #cmd_line += " -threads 4 -vcodec copy -acodec copy tmp_transcoded.webm" ret = run_command(cmd_line) print(" ret value = " + str(ret)) if ret == False: print("[ERROR] Trancode: error occured ...") - exit(-1) + element_error.append(elem) + #exit(-1) + continue print(" move in: '" + elem[:-len(extention)] + "mkv'") - cmd_line = "mv " + elem.replace(" ", "\ ").replace("'", "\\'") + " " + elem.replace(" ", "\ ").replace("'", "\\'") + "__" + # cmd_line = "mv " + elem.replace(" ", "\ ").replace("'", "\\'") + " last_transcoded.xx" + cmd_line = "mv " + elem.replace(" ", "\ ").replace("!", "\\!").replace("'", "\\'") + " last_transcoded" ret = run_command(cmd_line) - cmd_line = "mv zzz_transcoded.mkv_tmp " + elem.replace(" ", "\ ").replace("'", "\\'")[:-len(extention)] + "mkv" + cmd_line = "mv " + tmp_name_encoded_file + " " + elem.replace(" ", "\ ").replace("!", "\\!").replace("'", "\\'")[:-len(extention)] + "mkv" ret = run_command(cmd_line) @@ -262,9 +308,25 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse #ret = run_command(cmd_line) #break -full_list_size = len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv) + len(list_files_divx) +full_list_size = len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv) + len(list_files_divx) + len(list_files_mkv) + len(list_files_webm) offset = 0; + +reverse_sort = False + +list_files_ts.sort(reverse=reverse_sort) +list_files_mp4.sort(reverse=reverse_sort) +list_files_flv.sort(reverse=reverse_sort) +list_files_avi.sort(reverse=reverse_sort) +list_files_wmv.sort(reverse=reverse_sort) +list_files_divx.sort(reverse=reverse_sort) +list_files_mkv.sort(reverse=reverse_sort) +list_files_webm.sort(reverse=reverse_sort) + +random.shuffle(list_files_mp4) +random.shuffle(list_files_avi) +random.shuffle(list_files_mkv) + trancode_local(list_files_ts , "ts", full_list_size, offset) offset += len(list_files_ts) trancode_local(list_files_mp4 , "mp4", full_list_size, offset) @@ -282,6 +344,11 @@ offset += len(list_files_mkv) #trancode_local(list_files_webm , "webm", full_list_size, offset) #offset += len(list_files_webm) +print("List error transcode: " + len(element_error)) +for elem in element_error: + print(" == Trancode: '" + elem.replace("'", "\'") + "'") + + ## extract a thumb from a video ## ffmpeg -i Passenger.mkv -ss 00:05:00 -f image2 -vframes 1 thumb.jpg diff --git a/back/tools/video_move_correct.py b/back/tools/video_move_correct.py new file mode 100755 index 0000000..af99340 --- /dev/null +++ b/back/tools/video_move_correct.py @@ -0,0 +1,173 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2016, Edouard DUPIN, all right reserved +## +## @license APACHE v2.0 (see license file) +## +import os +import fnmatch +import sys +import subprocess +import shlex +import shutil + +## +## @brief Execute the command with no get of output +## +def run_command(cmd_line): + # prepare command line: + args = shlex.split(cmd_line) + print("[INFO] cmd = " + str(args)) + try: + # create the subprocess + p = subprocess.Popen(args) + except subprocess.CalledProcessError as e: + print("[ERROR] subprocess.CalledProcessError : " + str(args)) + return False + #except: + # debug.error("Exception on : " + str(args)) + # launch the subprocess: + output, err = p.communicate() + # Check error : + if p.returncode == 0: + return True + else: + return False + + +## +## @brief Get list of all Files in a specific path (with a regex) +## @param[in] path (string) Full path of the machine to search files (start with / or x:) +## @param[in] regex (string) Regular expression to search data +## @param[in] recursive (bool) List file with recursive search +## @param[in] remove_path (string) Data to remove in the path +## @return (list) return files requested +## +def get_list_of_file_in_path(path, regex="*", recursive = False, remove_path=""): + out = [] + if os.path.isdir(os.path.realpath(path)): + tmp_path = os.path.realpath(path) + tmp_rule = regex + else: + debug.error("path does not exist : '" + str(path) + "'") + + for root, dirnames, filenames in os.walk(tmp_path): + deltaRoot = root[len(tmp_path):] + while len(deltaRoot) > 0 \ + and ( deltaRoot[0] == '/' \ + or deltaRoot[0] == '\\' ): + deltaRoot = deltaRoot[1:] + if recursive == False \ + and deltaRoot != "": + return out + tmpList = filenames + if len(tmp_rule) > 0: + tmpList = fnmatch.filter(filenames, tmp_rule) + # Import the module : + for cycleFile in tmpList: + #for cycleFile in filenames: + add_file = os.path.join(tmp_path, deltaRoot, cycleFile) + if len(remove_path) != 0: + if add_file[:len(remove_path)] != remove_path: + print("ERROR : Request remove start of a path that is not the same: '" + add_file[:len(remove_path)] + "' demand remove of '" + str(remove_path) + "'") + else: + add_file = add_file[len(remove_path)+1:] + out.append(add_file) + return out; + +def get_run_path(): + return os.getcwd() + +src_path = get_run_path() +dst_path = os.path.join(src_path, "..", "zzz_video_push_correct") +list_files_mkv = get_list_of_file_in_path(src_path, "*.mkv", recursive = True) +list_files_webm = get_list_of_file_in_path(src_path, "*.webm", recursive = True) +list_files_jpg = get_list_of_file_in_path(src_path, "*.jpg", recursive = True) +list_files_png = get_list_of_file_in_path(src_path, "*.png", recursive = True) + +print("list of elements MKV : ") +for elem in list_files_mkv: + print(" - '" + str(elem) + "'") +print("list of elements webm : ") +for elem in list_files_webm: + print(" - '" + str(elem) + "'") + +import random +from pymediainfo import MediaInfo + +for arg in sys.argv: + print("arg: " + arg) + +id_value = 0 +if len(sys.argv) == 2: + id_value = int(sys.argv[1]) + + + +def create_directory_of_file(file): + path = os.path.dirname(file) + try: + os.stat(path) + except: + os.makedirs(path) + +def file_move(path_src, path_dst): + #real write of data: + print("kljlkjlkjklj " + path_src) + print("kljlkjlkjklj " + path_dst) + create_directory_of_file(path_dst) + shutil.move(path_src, path_dst) + return True + + +def move_local(list_of_file=[], extention="mkv") : + global element_error; + print("Start strancoding: '." + extention + "' ... " + str(len(list_of_file))) + id_elem = 0 + total_count_of_file = len(list_of_file) + for elem in list_of_file: + id_elem += 1 + print(" ========================================================================================") + print(" == " + str(id_elem) + " / " + str(total_count_of_file)) + print(" == Trancode: '" + elem.replace("'", "\'") + "'") + print(" ========================================================================================") + if not os.path.isfile(elem): + print(" ==> file does not exist") + continue + + # collect media info ... + #if it is a mk: .. chack the opus format... + if extention == "mkv": + media_info = MediaInfo.parse(elem) + print("media-info: ... " + str(len(media_info.tracks))) + need_move_file = True + for elem_track in media_info.tracks: + data_print = "[" + str(elem_track.track_id) + "] " + str(elem_track.track_type) + #print('track_id = ' + str(elem_track.track_id)) + #print('track_type = ' + str(elem_track.track_type)) + if elem_track.track_type == "Audio": + data_print += " (" + str(elem_track.language) + ") enc=" + str(elem_track.format); + #print('language = ' + str(elem_track.language)) + #print('format = ' + str(elem_track.format)) + if elem_track.format != "Opus": + need_move_file = False + elif elem_track.track_type == "Video": + data_print += " enc=" + str(elem_track.format); + if elem_track.format != "AVC": + need_move_file = False + print(" - " + data_print) + #print("media-info: ..." + str(dir(elem_track))) + if need_move_file == False: + print(" ==> Need transcode, NOT already in the good format...") + continue + + file_move(elem, os.path.join(dst_path, elem[len(src_path)+1:])) + +move_local(list_files_mkv, "mkv") +move_local(list_files_webm, "webm") +move_local(list_files_jpg, "jpg") +move_local(list_files_png, "png") +