diff --git a/.bin/pathShaChecker.py b/.bin/pathShaChecker.py new file mode 100755 index 0000000..7118208 --- /dev/null +++ b/.bin/pathShaChecker.py @@ -0,0 +1,261 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2012, Edouard DUPIN, all right reserved +## +## @license APACHE v2.0 (see license file) +## + +## +## read a path (ref) reference all file sith their sha1 and chek the second folder (src) +## to check if this qhaone exist or not if it exist, it can move to the dst folder or just remove it. +## + + +import os +import shutil +import errno +import fnmatch +import stat +import sys +import subprocess +import shlex +import re +import copy + +import hashlib + + +def get_sha512(filename, reduce=True): + BLOCKSIZE = 16000 + hasher = hashlib.sha512() + #hasher = hashlib.md5() + with open(filename, 'rb') as afile: + buf = afile.read(BLOCKSIZE) + while len(buf) > 0: + hasher.update(buf) + if reduce == True: + return hasher.hexdigest() + buf = afile.read(BLOCKSIZE) + return hasher.hexdigest() + + +## +## @brief Get list of all Files in a specific path (with a regex) +## @param[in] path (string) Full path of the machine to search files (start with / or x:) +## @param[in] regex (string) Regular expression to search data +## @param[in] recursive (bool) List file with recursive search +## @param[in] remove_path (string) Data to remove in the path +## @return (list) return files requested +## +def get_list_of_file_in_path(path, filter="*", recursive = False, remove_path=""): + out = [] + if os.path.isdir(os.path.realpath(path)): + tmp_path = os.path.realpath(path) + else: + print("[E] path does not exist : '" + str(path) + "'") + last_x = 0 + for root, dirnames, filenames in os.walk(tmp_path): + deltaRoot = root[len(tmp_path):] + while len(deltaRoot) > 0 \ + and ( deltaRoot[0] == '/' \ + or deltaRoot[0] == '\\' ): + deltaRoot = deltaRoot[1:] + print("\r" + " " * (last_x+12), end="") + last_x = len(str(deltaRoot)) + print("\r[I] path: '" + str(deltaRoot) + "'", end="") + #ilter some stupid path ... thumbnails=>perso @eaDir synology + if ".thumbnails" in deltaRoot \ + or "@eaDir" in deltaRoot: + continue + if recursive == False \ + and deltaRoot != "": + return out + tmpList = [] + for elem in filter: + tmpppp = fnmatch.filter(filenames, elem) + for elemmm in tmpppp: + tmpList.append(elemmm) + # Import the module : + for cycleFile in tmpList: + #for cycleFile in filenames: + add_file = os.path.join(tmp_path, deltaRoot, cycleFile) + if len(remove_path) != 0: + if add_file[:len(remove_path)] != remove_path: + print("[E] Request remove start of a path that is not the same: '" + add_file[:len(remove_path)] + "' demand remove of '" + str(remove_path) + "'") + else: + add_file = add_file[len(remove_path)+1:] + out.append(add_file) + print("") + return out; + + +import argparse + +parser = argparse.ArgumentParser(description='Check comparison between 2 path.') +parser.add_argument('--ref', + type=str, + action='store', + default="", + help='Reference path (the sha512 sources)') +parser.add_argument('--src', + type=str, + action='store', + default="", + help='Path to check that data is duplicated (remove data that is duplicated)') +parser.add_argument('--dst', + type=str, + action='store', + default="out_store_rebut", + help='Path to move duplication (move here data that is duplicated)') +parser.add_argument('--ref_remove_double', + action='store_true', + default=False, + help='remove double find in the base reference') +parser.add_argument('--src_remove_double', + action='store_true', + default=False, + help='remove double find in the src reference') + +args = parser.parse_args() + +print("[I] ref=" + args.ref) +print("[I] src=" + args.src) +print("[I] dst=" + args.dst) + +if args.ref == "": + print("[ERROR] Missing ref") + exit(-1) +if args.src == "": + print("[ERROR] Missing src") + exit(-1) +if args.dst == "": + print("[ERROR] Missing dst") + exit(-1) + +def check_file(data): + out = "" + for elem in str(data): + if elem in "azertyuiopqsdfghjklmwxcvbn1234567890AZERTYUIOPQSDFGHJKLMWXCVBNéèçà@.- _?:!*+/\<>()[]{}=°|#~²": + out += elem + return out + +print("===================================================") +print("[I] Create List of files REF ...") +print("===================================================") +list_files_ref = get_list_of_file_in_path(args.ref, recursive=True) + +curent_DB = {} + +print("===================================================") +print("[I] Generate sha512 ...") +print("===================================================") +ref_duplicate = 0 +src_missing = 0 + +file_in_double = open("fileInDouble.txt", "w") +file_not_in_ref = open("fileNotInRef.txt", "w") + +last_x = 0 +iii = 0 +for elem in list_files_ref: + iii += 1 + value_progress = float(iii) / float(len(list_files_ref)) + value_progress1 = int(value_progress*100) + value_progress2 = int(value_progress*10000 - value_progress1*100) + print("\r[I] processing " + str(iii) + "/" + str(len(list_files_ref)) + " " + str(value_progress1) + "." + str(value_progress2) + "/100 " + check_file(elem) + " "*(last_x-len(elem)), end='') + last_x = len(elem) + sys.stdout.flush() + value_sha512 = get_sha512(elem) + # check if the element is not a duplication ... + if value_sha512 in curent_DB.keys(): + ## print("\n[ERROR] Double element in the data-base (reduced ...):") + print(" ... ", end="") + # need to check with a not reduce sha512 + value_src_sha512_full = get_sha512(elem,reduce=False) + # a reduce sha512 can have many distint file depending on it ... + for elem_previous in curent_DB[value_sha512]: + if "sha512" not in elem_previous: + elem_previous["sha512"] = get_sha512(elem_previous["file"],reduce=False) + find_double = False + for elem_previous in curent_DB[value_sha512]: + if elem_previous["sha512"] == value_src_sha512_full: + if args.ref_remove_double == True: + print("\n ==> remove double in reference: '" + check_file(elem) + "'") + os.remove(elem) + else: + ref_duplicate += 1 + print("\n[ERROR] Double element in the data-base: " + check_file(ref_duplicate)) + print(" sha512=" + str(value_sha512)) + print(" ref =" + check_file(elem_previous["file"])) + print(" ref(copy)=" + check_file(elem)) + file_in_double.write(check_file(elem_previous["file"]) + "\n") + file_in_double.write(check_file(elem) + "\n") + file_in_double.write(str("--------------------------------------------------------------------\n")) + find_double = True + break + if find_double == True: + curent_DB[value_sha512].append({"file":elem,"sha512":value_src_sha512_full}) + else: + curent_DB[value_sha512] = [{"file":elem}] +print("") + +""" +for elem in curent_DB.keys(): + print(" " + str(elem) + " - " + str(curent_DB[elem])) +""" + +print("===================================================") +print("[I] Create List of files SRC ...") +print("===================================================") +list_files_src = get_list_of_file_in_path(args.src, recursive=True) + +print("===================================================") +print("[I] Generate sha512 ...") +print("===================================================") +last_x = 0 +iii = 0 +for elem in list_files_src: + iii += 1 + value_progress = float(iii) / float(len(list_files_ref)) + value_progress1 = int(value_progress*100) + value_progress2 = int(value_progress*10000 - value_progress1*100) + print("\r[I] processing " + str(iii) + "/" + str(len(list_files_ref)) + " " + str(value_progress1) + "." + str(value_progress2) + "/100 " + check_file(elem) + " "*(last_x-len(elem)), end='') + last_x = len(elem) + sys.stdout.flush() + value_sha512 = get_sha512(elem) + # check if the element is not a duplication ... + if value_sha512 not in curent_DB.keys(): + src_missing += 1 + print("\n[INFO] Find element not in the dB: " + str(src_missing)) + print(" sha512=" + str(value_sha512)) + print(" src=" + check_file(elem)) + file_not_in_ref.write(str(elem + "\n")) + else: + value_src_sha512_full = get_sha512(elem,reduce=False) + for elem_previous in curent_DB[value_sha512]: + if "sha512" not in elem_previous: + elem_previous["sha512"] = get_sha512(elem_previous["file"],reduce=False) + find_double = False + for elem_previous in curent_DB[value_sha512]: + if elem_previous["sha512"] == value_src_sha512_full: + find_double = True + if args.src_remove_double == True: + print("\n ==> remove double in source: '" + check_file(elem) + "'") + os.remove(elem) + else: + print("\n ==> must move double in source in destination: '" + check_file(elem) + "'") + break + if find_double == False: + src_missing += 1 + print("\n[INFO] Find element not in the dB (FULL): " + str(src_missing)) + print(" sha512=" + str(value_sha512)) + print(" src=" + check_file(elem)) + file_not_in_ref.write(check_file(elem) + "\n") +print("") + +file_not_in_ref.close() +print("Duplicate file in reference : " + str(ref_duplicate)) +print("Missing file in reference : " + str(src_missing)) diff --git a/.bin/renameMP3.py b/.bin/renameMP3.py new file mode 100755 index 0000000..0bc65d3 --- /dev/null +++ b/.bin/renameMP3.py @@ -0,0 +1,353 @@ +#!/usr/bin/python +# +# Desmond Cox +# April 10, 2008 + +"""Project Music + +Renames audio files based on metadata + +Usage: renameMP3.py [options] + +Options: + -d ..., --directory=... Specify which directory to work in + (default is the current directory) + -f ..., --format=... Specify the naming format + -l, --flatten Move all files into the same root + directory + -r, --recursive Work recursively on the specified + directory + -t, --test Only display the new file names; nothing + will be renamed + -h, --help Display this help + +Formatting: + The following information is available to be used in the file name: + album artist title track + + To specify a file name format, enter the desired format enclosed in quotation + marks. The words album, artist, title, and track will be replaced by values + retrieved from the audio file's metadata. + + For example, --format="artist - album [track] title" will rename music files + with the name format: + Sample Artist - Sample Album [1] Sample Title + + The following characters are of special importance to the operating system + and cannot be used in the file name: + \ : * ? " < > | + + (=) is replaced by the directory path separator, so to move files into + artist and album subdirectories, the following format can be used: + "artist(=)album(=)track - title" + + If no format is provided, the default format is the same as used in the above + example. + +Examples: + renameMP3.py Renames music files in the current + directory + renameMP3.py -d /music/path/ Renames music files in /music/path/ + renameMP3.py -f "title -- artist" Renames music files in the current + directory with the name format: + Sample Title -- Sample Artist.mp3 + renameMP3.py -d . -r --flatten + +pip install mutagen --user +pip install easyid3 --user +pip install soundscrape --user + +""" + +### Imports ### + +import time +import re +import os +import getopt +import sys +import fnmatch + +import mutagen.easyid3 +import mutagen.oggvorbis + +### Exceptions ### +class FormatError(Exception): + """ + Exception raised due to improper formatting + """ + pass + +class DirectoryError(Exception): + """ + Exception raised due to a non-existent directory + """ + pass + +### Definitions ### + +def create_directory_of_file(file): + path = os.path.dirname(file) + try: + os.stat(path) + except: + os.makedirs(path) + +## +## @brief Get list of all Files in a specific path (with a regex) +## @param[in] path (string) Full path of the machine to search files (start with / or x:) +## @param[in] regex (string) Regular expression to search data +## @param[in] recursive (bool) List file with recursive search +## @param[in] remove_path (string) Data to remove in the path +## @return (list) return files requested +## +## get_list_of_file_in_path +def scanDirectory(path, filter="*", recursive = False, remove_path=""): + print(" ******** " + path) + out = [] + if os.path.isdir(os.path.realpath(path)): + tmp_path = os.path.realpath(path) + else: + print("[E] path does not exist : '" + str(path) + "'") + last_x = 0 + for root, dirnames, filenames in os.walk(tmp_path): + deltaRoot = root[len(tmp_path):] + while len(deltaRoot) > 0 \ + and ( deltaRoot[0] == '/' \ + or deltaRoot[0] == '\\' ): + deltaRoot = deltaRoot[1:] + print("\r" + " " * (last_x+12), end="") + last_x = len(str(deltaRoot)) + print("\r[I] path: '" + str(deltaRoot) + "'", end="") + #ilter some stupid path ... thumbnails=>perso @eaDir synology + if ".thumbnails" in deltaRoot \ + or "@eaDir" in deltaRoot: + continue + if recursive == False \ + and deltaRoot != "": + return out + tmpList = [] + for elem in filter: + tmpppp = fnmatch.filter(filenames, elem) + for elemmm in tmpppp: + tmpList.append(elemmm) + # Import the module : + for cycleFile in tmpList: + #for cycleFile in filenames: + add_file = os.path.join(tmp_path, deltaRoot, cycleFile) + if len(remove_path) != 0: + if add_file[:len(remove_path)] != remove_path: + print("[E] Request remove start of a path that is not the same: '" + add_file[:len(remove_path)] + "' demand remove of '" + str(remove_path) + "'") + else: + add_file = add_file[len(remove_path)+1:] + out.append(add_file) + print(" len out " + str(len(out))) + return out; + + + +class AudioFile: + """ + A generic audio file + """ + def __init__(self, fileName): + self.fileName = fileName + self.fileExt = os.path.splitext(fileName)[1].lower() + self.filePath = os.path.split(fileName)[0] + os.path.sep + self.data = getattr(self, "parse_%s" % self.fileExt[1:])() + # call the appropriate method based on the file type + self.generate() + + def generate(self): + def lookup(key, default): + return self.data[key][0] if ( key in self.data.keys() and + self.data[key][0] ) else default + self.artist = lookup("artist", "No Artist") + self.album = lookup("album", "No Album") + self.title = lookup("title", "No Title") + self.track = lookup("tracknumber", "0") + if self.track != "0": + self.track = self.track.split("/")[0].lstrip("0") + # In regards to track numbers, self.data["tracknumber"] returns numbers + # in several different formats: 1, 1/10, 01, or 01/10. Wanting a + # consistent format, the returned string is split at the "/" and leading + # zeros are stripped. + if int(self.track) < 10: + self.track = "0" + self.track + + def parse_mp3(self): + return mutagen.easyid3.EasyID3(self.fileName) + + def parse_ogg(self): + return mutagen.oggvorbis.Open(self.fileName) + + def rename(self, newFileName, flatten=False): + def uniqueName(newFileName, count=0): + """ + Returns a unique name if a file already exists with the supplied + name + """ + c = "_(%s)" % str(count) if count else "" + prefix = directory + os.path.sep if flatten else self.filePath + testFileName = prefix + newFileName + c + self.fileExt + if os.path.isfile(testFileName): + count += 1 + return uniqueName(newFileName, count) + else: + return testFileName + new_name = uniqueName(newFileName) + create_directory_of_file(new_name) + os.renames(self.fileName, new_name) + # Note: this function is quite simple at the moment; it does not support + # multiple file extensions, such as "sample.txt.backup", which would + # only retain the ".backup" file extension. + + def cleanFileName(self, format): + """ + Generate a clean file name based on metadata + """ + rawFileName = format % {"artist": self.artist, + "album": self.album, + "title": self.title, + "track": self.track} + rawFileName.encode("ascii", "replace") + # encode is used to override the default encode error-handing mode; + # which is to raise a UnicodeDecodeError + cleanFileName = re.sub(restrictedCharPattern, "+", rawFileName) + # remove restricted filename characters (\, /, :, *, ?, ", <, >, |) from + # the supplied string + + return cleanFileName.replace("(=)", os.path.sep) + +### Main ### + +def main(argv): + global directory + directory = os.getcwd() + format = "%(artist)s/%(album)s/%(track)s-%(title)s" + flatten = False + recursive = False + test = False + + def verifyFormat(format): + """ + Verify the supplied filename format + """ + if re.search(restrictedCharPattern, format): + raise(FormatError, "supplied format contains restricted characters") + + if not re.search(formatPattern, format): + raise(FormatError, "supplied format does not contain any metadata keys") + # the supplied format must contain at least one of "artist", + # "album", "title", or "track", or all files will be named + # identically + + format = format.replace("artist", "%(artist)s") + format = format.replace("album", "%(album)s") + format = format.replace("title", "%(title)s") + format = format.replace("track", "%(track)s") + return format + + def verifyDirectory(directory): + """ + Verify the supplied directory path + """ + if os.path.isdir(directory): + return os.path.abspath(directory) + + else: + raise(DirectoryError, "supplied directory cannot be found") + + def usage(): + print(__doc__) + + try: + opts, args = getopt.getopt(argv, "d:f:hlrt", ["directory=", + "format=", + "help", + "flatten", + "recursive", + "test"]) + except getopt.error: + usage() + print("\n***Error: %s***" % error) + sys.exit(1) + except error: + usage() + print("\n***Error: %s***" % error) + sys.exit(1) + for opt, arg in opts: + if opt in ("-h", "--help"): + usage() + sys.exit() + elif opt in ("-f", "--format"): + try: + format = verifyFormat(arg) + except FormatError: + print("\n***Error: %s***" % error) + sys.exit(2) + except error: + print("\n***Error: %s***" % error) + sys.exit(2) + elif opt in ("-d", "--directory"): + """ + try: + directory = verifyDirectory(arg) + + except DirectoryError: + print("\n***Error: %s***" % error) + sys.exit(3) + except error: + print("\n***Error: %s***" % error) + sys.exit(3) + """ + directory = arg + elif opt in ("-l", "--flatten"): + flatten = True + elif opt in ("-r", "--recursive"): + recursive = True + elif opt in ("-t", "--test"): + test = True + work(directory, format, flatten, recursive, test) + +def safety(message): + print("\n***Attention: %s***" % message) + #safety = raw_input("Enter 'ok' to continue (any other response will abort): ") + safety = "ok" + if safety.lower().strip() != "ok": + print("\n***Attention: aborting***") + sys.exit() + +def work(directory, format, flatten, recursive, test): + #fileList = scanDirectory(directory, [".mp3", ".ogg"], recursive) + fileList = scanDirectory(directory, ["*.mp3"], recursive=recursive) + try: + if test: + safety("testing mode; nothing will be renamed") + print("\n***Attention: starting***") + for f in fileList: + current = AudioFile(f) + print(current.cleanFileName(format)) + + else: + count = 0 + total = len(fileList) + safety("all audio files in %s will be renamed : %d " % (directory, total)) + print("\n***Attention: starting***") + start = time.time() + for file in fileList: + count += 1 + current = AudioFile(file) + current.rename(current.cleanFileName(format), flatten) + print("Renamed %d of %d : " % (count, total)) + print("\n%d files renamed in %f seconds" % (len(fileList), + time.time() - start)) + except StandardError: + print("\n***Error: %s***" % f) + raise + +if __name__ == "__main__": + restrictedCharPattern = re.compile('[\\\:\*\?"<>\|]') + formatPattern = re.compile('artist|album|title|track') + + main(sys.argv[1:]) diff --git a/.bin/sha_checker_test/plop/cou/hello2/jhkjh.exe b/.bin/sha_checker_test/plop/cou/hello2/jhkjh.exe new file mode 100644 index 0000000..8a96904 --- /dev/null +++ b/.bin/sha_checker_test/plop/cou/hello2/jhkjh.exe @@ -0,0 +1 @@ +ssdfdgsfgsdsdfgfgsxcvdbfx bfds bvcxgbsxvcbdfg sdfgsdf gsqsddsfgsdfg sdfgsdfg fdgdfgdfg diff --git a/.bin/sha_checker_test/plop/cou/hello2/jhkjhsdfgsdfg b/.bin/sha_checker_test/plop/cou/hello2/jhkjhsdfgsdfg new file mode 100644 index 0000000..1d06b33 --- /dev/null +++ b/.bin/sha_checker_test/plop/cou/hello2/jhkjhsdfgsdfg @@ -0,0 +1 @@ +ssdfdgsfgsdsdfdbfx bfds bvcxgbsxvcbdfg sdfgsdf gsqsddsfgsdfg sdfgsdfg diff --git a/.bin/sha_checker_test/plop/cou/hellsgdfgsdfg b/.bin/sha_checker_test/plop/cou/hellsgdfgsdfg new file mode 100644 index 0000000..26df92b --- /dev/null +++ b/.bin/sha_checker_test/plop/cou/hellsgdfgsdfg @@ -0,0 +1 @@ +ssdfdgsfgsdsdfgfgsxcvdbfx bfds bvcxgbsxvcbdfg sdfgsdf gsqsd diff --git a/.bin/sha_checker_test/plop/cou/ssdfgsdfg b/.bin/sha_checker_test/plop/cou/ssdfgsdfg new file mode 100644 index 0000000..17871ad --- /dev/null +++ b/.bin/sha_checker_test/plop/cou/ssdfgsdfg @@ -0,0 +1 @@ +ssdfdgsfgsdsdfgfgsxcvdbfx bfds bvcxgbsxvcbdfg sdfgsdf gsqsddsfgsdfg sdfgsdfg diff --git a/.bin/sha_checker_test/plop/coucou/hello/dddddd.txt b/.bin/sha_checker_test/plop/coucou/hello/dddddd.txt new file mode 100644 index 0000000..6797700 --- /dev/null +++ b/.bin/sha_checker_test/plop/coucou/hello/dddddd.txt @@ -0,0 +1 @@ +sdfgsdfgsdf fds gsdfg sdfgsdf gsqsd diff --git a/.bin/sha_checker_test/plop/coucou/hello2/dddddd b/.bin/sha_checker_test/plop/coucou/hello2/dddddd new file mode 100644 index 0000000..c6a0f0d --- /dev/null +++ b/.bin/sha_checker_test/plop/coucou/hello2/dddddd @@ -0,0 +1 @@ +sdfgsdfgsdf fds gsdfg sdfgsdf gsqsdfqsdf dfgs dfgsdf sqsdfqsdfdf hs df g sdf g diff --git a/.bin/sha_checker_test/plop/coucou/hello2/kjlkj b/.bin/sha_checker_test/plop/coucou/hello2/kjlkj new file mode 100644 index 0000000..21f0a91 --- /dev/null +++ b/.bin/sha_checker_test/plop/coucou/hello2/kjlkj @@ -0,0 +1 @@ +sdfgsdfgsdf fds gsdfg sdfgsdf gs dfgs dfgsdf sdf hs df g sdf g diff --git a/.bin/sha_checker_test/plop/coudd/sdfsdfs.plo b/.bin/sha_checker_test/plop/coudd/sdfsdfs.plo new file mode 100644 index 0000000..18b4191 --- /dev/null +++ b/.bin/sha_checker_test/plop/coudd/sdfsdfs.plo @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaasdfgaaaaaaaaaasdfgaaaaaaaaa diff --git a/.bin/sha_checker_test/plop/coudd/sdfsdfs.plop b/.bin/sha_checker_test/plop/coudd/sdfsdfs.plop new file mode 100644 index 0000000..a46adba --- /dev/null +++ b/.bin/sha_checker_test/plop/coudd/sdfsdfs.plop @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/.bin/sha_checker_test/plop/sdfsdfs.plsdfkjho b/.bin/sha_checker_test/plop/sdfsdfs.plsdfkjho new file mode 100644 index 0000000..9855624 --- /dev/null +++ b/.bin/sha_checker_test/plop/sdfsdfs.plsdfkjho @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaatyuityuiaaaaaaaaaaaaaaaaaaaaaaaaasdfgaaaaaakhkjhaaaasdfgaaaaasdfgsaaaa diff --git a/.bin/sha_checker_test/plop/sdfsdfs.plsdfo b/.bin/sha_checker_test/plop/sdfsdfs.plsdfo new file mode 100644 index 0000000..c2d1166 --- /dev/null +++ b/.bin/sha_checker_test/plop/sdfsdfs.plsdfo @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaatyuityuiaaaaaaaaaaaaaaaaaaaaaaaaasdfgaaaaaaaaaasdfgaaaaasdfgsaaaa diff --git a/.bin/sha_checker_test/plop2/cou/hello2/jhkjh.exe b/.bin/sha_checker_test/plop2/cou/hello2/jhkjh.exe new file mode 100644 index 0000000..8a96904 --- /dev/null +++ b/.bin/sha_checker_test/plop2/cou/hello2/jhkjh.exe @@ -0,0 +1 @@ +ssdfdgsfgsdsdfgfgsxcvdbfx bfds bvcxgbsxvcbdfg sdfgsdf gsqsddsfgsdfg sdfgsdfg fdgdfgdfg diff --git a/.bin/sha_checker_test/plop2/cou/hello2/jhkjhsdfgsdfg b/.bin/sha_checker_test/plop2/cou/hello2/jhkjhsdfgsdfg new file mode 100644 index 0000000..1d06b33 --- /dev/null +++ b/.bin/sha_checker_test/plop2/cou/hello2/jhkjhsdfgsdfg @@ -0,0 +1 @@ +ssdfdgsfgsdsdfdbfx bfds bvcxgbsxvcbdfg sdfgsdf gsqsddsfgsdfg sdfgsdfg diff --git a/.bin/sha_checker_test/plop2/cou/hellsgdfgsdfg b/.bin/sha_checker_test/plop2/cou/hellsgdfgsdfg new file mode 100644 index 0000000..26df92b --- /dev/null +++ b/.bin/sha_checker_test/plop2/cou/hellsgdfgsdfg @@ -0,0 +1 @@ +ssdfdgsfgsdsdfgfgsxcvdbfx bfds bvcxgbsxvcbdfg sdfgsdf gsqsd diff --git a/.bin/sha_checker_test/plop2/cou/ssdfgsdfg b/.bin/sha_checker_test/plop2/cou/ssdfgsdfg new file mode 100644 index 0000000..17871ad --- /dev/null +++ b/.bin/sha_checker_test/plop2/cou/ssdfgsdfg @@ -0,0 +1 @@ +ssdfdgsfgsdsdfgfgsxcvdbfx bfds bvcxgbsxvcbdfg sdfgsdf gsqsddsfgsdfg sdfgsdfg diff --git a/.bin/sha_checker_test/plop2/coucou/hello/dddddd (copie 1).txt b/.bin/sha_checker_test/plop2/coucou/hello/dddddd (copie 1).txt new file mode 100644 index 0000000..ee04059 --- /dev/null +++ b/.bin/sha_checker_test/plop2/coucou/hello/dddddd (copie 1).txt @@ -0,0 +1,6 @@ +sdfgsdfgsdf fds gsdfg sdfgsdf gsqsd +qsdfsd +qsdf + + +qsdfqsdf \ No newline at end of file diff --git a/.bin/sha_checker_test/plop2/coucou/hello/dddddd.txt b/.bin/sha_checker_test/plop2/coucou/hello/dddddd.txt new file mode 100644 index 0000000..6797700 --- /dev/null +++ b/.bin/sha_checker_test/plop2/coucou/hello/dddddd.txt @@ -0,0 +1 @@ +sdfgsdfgsdf fds gsdfg sdfgsdf gsqsd diff --git a/.bin/sha_checker_test/plop2/coucou/hello2/dddddd b/.bin/sha_checker_test/plop2/coucou/hello2/dddddd new file mode 100644 index 0000000..c6a0f0d --- /dev/null +++ b/.bin/sha_checker_test/plop2/coucou/hello2/dddddd @@ -0,0 +1 @@ +sdfgsdfgsdf fds gsdfg sdfgsdf gsqsdfqsdf dfgs dfgsdf sqsdfqsdfdf hs df g sdf g diff --git a/.bin/sha_checker_test/plop2/coudd/sdfsdfs.plo b/.bin/sha_checker_test/plop2/coudd/sdfsdfs.plo new file mode 100644 index 0000000..18b4191 --- /dev/null +++ b/.bin/sha_checker_test/plop2/coudd/sdfsdfs.plo @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaasdfgaaaaaaaaaasdfgaaaaaaaaa diff --git a/.bin/sha_checker_test/plop2/coudd/sdfsdfs.plop b/.bin/sha_checker_test/plop2/coudd/sdfsdfs.plop new file mode 100644 index 0000000..a46adba --- /dev/null +++ b/.bin/sha_checker_test/plop2/coudd/sdfsdfs.plop @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/.bin/sha_checker_test/plop2/sdfsdfs.plsdfo b/.bin/sha_checker_test/plop2/sdfsdfs.plsdfo new file mode 100644 index 0000000..c2d1166 --- /dev/null +++ b/.bin/sha_checker_test/plop2/sdfsdfs.plsdfo @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaatyuityuiaaaaaaaaaaaaaaaaaaaaaaaaasdfgaaaaaaaaaasdfgaaaaasdfgsaaaa diff --git a/.bin/videoCheckDoubleName.py b/.bin/videoCheckDoubleName.py new file mode 100755 index 0000000..ea9d13d --- /dev/null +++ b/.bin/videoCheckDoubleName.py @@ -0,0 +1,155 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2012, Edouard DUPIN, all right reserved +## +## @license APACHE v2.0 (see license file) +## + +import os +import shutil +import errno +import fnmatch +import stat +import sys +import subprocess +import shlex +import re +import copy + +## +## @brief Execute the command with no get of output +## +def run_command(cmd_line): + # prepare command line: + args = shlex.split(cmd_line) + print("[INFO] cmd = " + str(args)) + try: + # create the subprocess + p = subprocess.Popen(args) + except subprocess.CalledProcessError as e: + print("[ERROR] subprocess.CalledProcessError : " + str(args)) + return False + #except: + # debug.error("Exception on : " + str(args)) + # launch the subprocess: + output, err = p.communicate() + # Check error : + if p.returncode == 0: + return True + else: + return False + + + +## +## @brief Get list of all Files in a specific path (with a regex) +## @param[in] path (string) Full path of the machine to search files (start with / or x:) +## @param[in] regex (string) Regular expression to search data +## @param[in] recursive (bool) List file with recursive search +## @param[in] remove_path (string) Data to remove in the path +## @return (list) return files requested +## +def get_list_of_file_in_path(path, filter, recursive = False, remove_path=""): + out = [] + if os.path.isdir(os.path.realpath(path)): + tmp_path = os.path.realpath(path) + else: + print("[E] path does not exist : '" + str(path) + "'") + + for root, dirnames, filenames in os.walk(tmp_path): + deltaRoot = root[len(tmp_path):] + while len(deltaRoot) > 0 \ + and ( deltaRoot[0] == '/' \ + or deltaRoot[0] == '\\' ): + deltaRoot = deltaRoot[1:] + if recursive == False \ + and deltaRoot != "": + return out + tmpList = [] + for elem in filter: + tmpppp = fnmatch.filter(filenames, elem) + for elemmm in tmpppp: + tmpList.append(elemmm) + # Import the module : + for cycleFile in tmpList: + #for cycleFile in filenames: + add_file = os.path.join(tmp_path, deltaRoot, cycleFile) + if len(remove_path) != 0: + if add_file[:len(remove_path)] != remove_path: + print("[E] Request remove start of a path that is not the same: '" + add_file[:len(remove_path)] + "' demand remove of '" + str(remove_path) + "'") + else: + add_file = add_file[len(remove_path)+1:] + out.append(add_file) + return out; + + +full_list = [] + +for extention in ["ts","avi","mkv","mp4"]: + list_files = get_list_of_file_in_path(".", ["*."+extention], recursive=True) + for elem in list_files: + # find format -sXX-eXX- + tmp = elem.split("/")[-1].split("-s") + if len(tmp) != 2: + print("[W] 1 ??? " + elem) + print(" ==> " + str(tmp)) + continue + base = tmp[0] + tmp = tmp[1].split("-e") + if len(tmp) != 2: + print("[W] 2 ??? " + elem) + print(" ==> " + str(tmp)) + continue + if tmp[1][2] == "-": + name = tmp[1][2:-len(extention)] + else: + print("[W] 3 missing '-' at pos 2 " + elem) + print(" ==> " + str(tmp[1])) + continue + + full_list.append({ + "name":name, + "base":base, + "file":elem + }) + + +def file_size(path): + if not os.path.isfile(path): + return 0 + statinfo = os.stat(path) + return statinfo.st_size + +def get_list(name): + out = [] + for elem in full_list: + if elem["name"] == name: + out.append(elem) + return out + + +# check naming correllation +for elem in full_list: + tmp = get_list(elem["name"]) + if len(tmp) == 1: + # normal case ... + continue + print("Duplicate Name:") + first = True + for elem_tmp in tmp: + if elem["file"] == elem_tmp["file"]: + print(" * " + elem_tmp["file"] + " " + str(int(file_size(elem_tmp["file"])/1024/1024)) + " MB") + first = False + else: + if first == True: + break + print(" - " + elem_tmp["file"] + " " + str(int(file_size(elem_tmp["file"])/1024/1024)) + " MB") + + + + + + diff --git a/.bin/videoRename.py b/.bin/videoRename.py index 1a200f1..48613c4 100755 --- a/.bin/videoRename.py +++ b/.bin/videoRename.py @@ -16,6 +16,8 @@ import stat import sys import subprocess import shlex +import re +import copy ## ## @brief Execute the command with no get of output @@ -84,8 +86,36 @@ def get_list_of_file_in_path(path, filter, recursive = False, remove_path=""): return out; +list_auto_replace = [ + "Chuggington", + "Caliméro", + "Le livre de la jungle", + "Barbapapa", + "Oum le dauphin blanc", + "Les Minijusticiers", + "Octonauts", + "Les Enquêtes de Mirette", + "Julius Jr", + "Paw Patrol, la Pat'Patrouille", + "Rusty Rivets" + ] - +def change_order_special(data): + basic_dir = os.path.dirname(data) + tmp = os.path.basename(data) + for elem in list_auto_replace: + if tmp[-len(elem)-3:] == " - " + elem: + tmp = elem + " - " + tmp[:-len(elem)-3] + print("") + print("") + print("") + print("") + print("") + print("*************************************************************************") + print("replace : " + data) + print(" : " + os.path.join(basic_dir,tmp)) + return os.path.join(basic_dir,tmp) + return data def rm_date(data): @@ -114,14 +144,121 @@ def rm_date(data): if len(val) == 2: return val[0] val = data.split("_Gulli_") + if len(val) == 2: + return val[0] + val = data.split("_Arte_") if len(val) == 2: return val[0] return data +def replace_generic_saison_and_name(data): + for elem in [ "1", "2", "3", "4", "5", "6", "7", "8", "9"]: + data = data.replace("S0" + elem + "E01", "- Saison " + elem + " Episode 1 -"); + data = data.replace("S0" + elem + "E02", "- Saison " + elem + " Episode 2 -"); + data = data.replace("S0" + elem + "E03", "- Saison " + elem + " Episode 3 -"); + data = data.replace("S0" + elem + "E04", "- Saison " + elem + " Episode 4 -"); + data = data.replace("S0" + elem + "E05", "- Saison " + elem + " Episode 5 -"); + data = data.replace("S0" + elem + "E06", "- Saison " + elem + " Episode 6 -"); + data = data.replace("S0" + elem + "E07", "- Saison " + elem + " Episode 7 -"); + data = data.replace("S0" + elem + "E08", "- Saison " + elem + " Episode 8 -"); + data = data.replace("S0" + elem + "E09", "- Saison " + elem + " Episode 9 -"); + data = data.replace("S0" + elem + "E10", "- Saison " + elem + " Episode 10 -"); + data = data.replace("S0" + elem + "E11", "- Saison " + elem + " Episode 11 -"); + data = data.replace("S0" + elem + "E12", "- Saison " + elem + " Episode 12 -"); + data = data.replace("S0" + elem + "E13", "- Saison " + elem + " Episode 13 -"); + data = data.replace("S0" + elem + "E14", "- Saison " + elem + " Episode 14 -"); + data = data.replace("S0" + elem + "E15", "- Saison " + elem + " Episode 15 -"); + data = data.replace("S0" + elem + "E16", "- Saison " + elem + " Episode 16 -"); + data = data.replace("S0" + elem + "E17", "- Saison " + elem + " Episode 17 -"); + data = data.replace("S0" + elem + "E18", "- Saison " + elem + " Episode 18 -"); + data = data.replace("S0" + elem + "E19", "- Saison " + elem + " Episode 19 -"); + data = data.replace("S0" + elem + "E20", "- Saison " + elem + " Episode 20 -"); + data = data.replace("S0" + elem + "E21", "- Saison " + elem + " Episode 21 -"); + data = data.replace("S0" + elem + "E22", "- Saison " + elem + " Episode 22 -"); + data = data.replace("S0" + elem + "E23", "- Saison " + elem + " Episode 23 -"); + data = data.replace("S0" + elem + "E24", "- Saison " + elem + " Episode 24 -"); + data = data.replace("S0" + elem + "E25", "- Saison " + elem + " Episode 25 -"); + data = data.replace("S0" + elem + "E26", "- Saison " + elem + " Episode 26 -"); + data = data.replace("S0" + elem + "E27", "- Saison " + elem + " Episode 27 -"); + data = data.replace("S0" + elem + "E28", "- Saison " + elem + " Episode 28 -"); + data = data.replace("S0" + elem + "E29", "- Saison " + elem + " Episode 29 -"); + data = data.replace("S0" + elem + "E30", "- Saison " + elem + " Episode 30 -"); + data = data.replace("S0" + elem + "E31", "- Saison " + elem + " Episode 31 -"); + data = data.replace("S0" + elem + "E32", "- Saison " + elem + " Episode 32 -"); + data = data.replace("S0" + elem + "E33", "- Saison " + elem + " Episode 33 -"); + data = data.replace("S0" + elem + "E34", "- Saison " + elem + " Episode 34 -"); + data = data.replace("S0" + elem + "E35", "- Saison " + elem + " Episode 35 -"); + data = data.replace("S0" + elem + "E36", "- Saison " + elem + " Episode 36 -"); + data = data.replace("S0" + elem + "E37", "- Saison " + elem + " Episode 37 -"); + data = data.replace("S0" + elem + "E38", "- Saison " + elem + " Episode 38 -"); + data = data.replace("S0" + elem + "E39", "- Saison " + elem + " Episode 39 -"); + data = data.replace("S0" + elem + "E40", "- Saison " + elem + " Episode 40 -"); + data = data.replace("S0" + elem + "E41", "- Saison " + elem + " Episode 41 -"); + data = data.replace("S0" + elem + "E42", "- Saison " + elem + " Episode 42 -"); + data = data.replace("S0" + elem + "E43", "- Saison " + elem + " Episode 43 -"); + data = data.replace("S0" + elem + "E44", "- Saison " + elem + " Episode 44 -"); + data = data.replace("S0" + elem + "E45", "- Saison " + elem + " Episode 45 -"); + data = data.replace("S0" + elem + "E46", "- Saison " + elem + " Episode 46 -"); + data = data.replace("S0" + elem + "E47", "- Saison " + elem + " Episode 47 -"); + data = data.replace("S0" + elem + "E48", "- Saison " + elem + " Episode 48 -"); + data = data.replace("S0" + elem + "E49", "- Saison " + elem + " Episode 49 -"); + data = data.replace("S0" + elem + "E50", "- Saison " + elem + " Episode 50 -"); + data = data.replace("S0" + elem + "E51", "- Saison " + elem + " Episode 51 -"); + data = data.replace("S0" + elem + "E52", "- Saison " + elem + " Episode 52 -"); + data = data.replace("S0" + elem + "E53", "- Saison " + elem + " Episode 53 -"); + data = data.replace("S0" + elem + "E54", "- Saison " + elem + " Episode 54 -"); + data = data.replace("S0" + elem + "E55", "- Saison " + elem + " Episode 55 -"); + data = data.replace("S0" + elem + "E56", "- Saison " + elem + " Episode 56 -"); + data = data.replace("S0" + elem + "E57", "- Saison " + elem + " Episode 57 -"); + data = data.replace("S0" + elem + "E58", "- Saison " + elem + " Episode 58 -"); + data = data.replace("S0" + elem + "E59", "- Saison " + elem + " Episode 59 -"); + data = data.replace("S0" + elem + "E60", "- Saison " + elem + " Episode 60 -"); + data = data.replace("S0" + elem + "E61", "- Saison " + elem + " Episode 61 -"); + data = data.replace("S0" + elem + "E62", "- Saison " + elem + " Episode 62 -"); + data = data.replace("S0" + elem + "E63", "- Saison " + elem + " Episode 63 -"); + data = data.replace("S0" + elem + "E64", "- Saison " + elem + " Episode 64 -"); + data = data.replace("S0" + elem + "E65", "- Saison " + elem + " Episode 65 -"); + data = data.replace("S0" + elem + "E66", "- Saison " + elem + " Episode 66 -"); + data = data.replace("S0" + elem + "E67", "- Saison " + elem + " Episode 67 -"); + data = data.replace("S0" + elem + "E68", "- Saison " + elem + " Episode 68 -"); + data = data.replace("S0" + elem + "E69", "- Saison " + elem + " Episode 69 -"); + data = data.replace("S0" + elem + "E70", "- Saison " + elem + " Episode 70 -"); + data = data.replace("S0" + elem + "E71", "- Saison " + elem + " Episode 71 -"); + data = data.replace("S0" + elem + "E72", "- Saison " + elem + " Episode 72 -"); + data = data.replace("S0" + elem + "E73", "- Saison " + elem + " Episode 73 -"); + data = data.replace("S0" + elem + "E74", "- Saison " + elem + " Episode 74 -"); + data = data.replace("S0" + elem + "E75", "- Saison " + elem + " Episode 75 -"); + data = data.replace("S0" + elem + "E76", "- Saison " + elem + " Episode 76 -"); + data = data.replace("S0" + elem + "E77", "- Saison " + elem + " Episode 77 -"); + data = data.replace("S0" + elem + "E78", "- Saison " + elem + " Episode 78 -"); + data = data.replace("S0" + elem + "E79", "- Saison " + elem + " Episode 79 -"); + data = data.replace("S0" + elem + "E80", "- Saison " + elem + " Episode 80 -"); + data = data.replace("S0" + elem + "E81", "- Saison " + elem + " Episode 81 -"); + data = data.replace("S0" + elem + "E82", "- Saison " + elem + " Episode 82 -"); + data = data.replace("S0" + elem + "E83", "- Saison " + elem + " Episode 83 -"); + data = data.replace("S0" + elem + "E84", "- Saison " + elem + " Episode 84 -"); + data = data.replace("S0" + elem + "E85", "- Saison " + elem + " Episode 85 -"); + data = data.replace("S0" + elem + "E86", "- Saison " + elem + " Episode 86 -"); + data = data.replace("S0" + elem + "E87", "- Saison " + elem + " Episode 87 -"); + data = data.replace("S0" + elem + "E88", "- Saison " + elem + " Episode 88 -"); + data = data.replace("S0" + elem + "E89", "- Saison " + elem + " Episode 89 -"); + data = data.replace("S0" + elem + "E90", "- Saison " + elem + " Episode 90 -"); + data = data.replace("S0" + elem + "E91", "- Saison " + elem + " Episode 91 -"); + data = data.replace("S0" + elem + "E92", "- Saison " + elem + " Episode 92 -"); + data = data.replace("S0" + elem + "E93", "- Saison " + elem + " Episode 93 -"); + data = data.replace("S0" + elem + "E94", "- Saison " + elem + " Episode 94 -"); + data = data.replace("S0" + elem + "E95", "- Saison " + elem + " Episode 95 -"); + data = data.replace("S0" + elem + "E96", "- Saison " + elem + " Episode 96 -"); + data = data.replace("S0" + elem + "E97", "- Saison " + elem + " Episode 97 -"); + data = data.replace("S0" + elem + "E98", "- Saison " + elem + " Episode 98 -"); + data = data.replace("S0" + elem + "E99", "- Saison " + elem + " Episode 99 -"); + return data def rename_group(list_element, extention): for elem in list_files: reduced_name = rm_date(elem[:-(len(extention)+1)]) + reduced_name = change_order_special(reduced_name) + reduced_name = replace_generic_saison_and_name(reduced_name) print("file: " + reduced_name) # remove the date in the channel ... val = reduced_name.split(" - ") diff --git a/.bin/videoTranscode.py b/.bin/videoTranscode.py index c368780..2a29409 100755 --- a/.bin/videoTranscode.py +++ b/.bin/videoTranscode.py @@ -85,6 +85,7 @@ list_files_mp4 = get_list_of_file_in_path('.', "*.mp4") list_files_avi = get_list_of_file_in_path('.', "*.avi") list_files_mkv = get_list_of_file_in_path('.', "*.mkv") list_files_wmv = get_list_of_file_in_path('.', "*.wmv") +list_files_divx = get_list_of_file_in_path('.', "*.divx") # remove all encoded element in the other files (TS) for elem_mkv in list_files_mkv: @@ -135,7 +136,7 @@ for elem_mkv in list_files_mkv: del list_files_avi[index] -# remove all encoded element in the other files (TS) +# remove all encoded element in the other files (wmv) for elem_mkv in list_files_mkv: index = 0 for elem_wmv in list_files_wmv: @@ -146,6 +147,17 @@ for elem_mkv in list_files_mkv: print("[INFO] remove from list '" + list_files_wmv[index] + "' ==> already transcoded") del list_files_wmv[index] +# remove all encoded element in the other files (divx) +for elem_mkv in list_files_mkv: + index = 0 + for elem_divx in list_files_divx: + if elem_mkv[:-3]+"divx" == elem_divx: + break; + index += 1 + if index != len(list_files_divx): + print("[INFO] remove from list '" + list_files_divx[index] + "' ==> already transcoded") + del list_files_divx[index] + print("list of elements TS : ") for elem in list_files_ts: print(" - '" + elem + "'") @@ -164,6 +176,9 @@ for elem in list_files_wmv: print("list of elements MKV : ") for elem in list_files_mkv: print(" - '" + elem + "'") +print("list of elements divx : ") +for elem in list_files_divx: + print(" - '" + elem + "'") @@ -192,13 +207,17 @@ def trancode_local(list_of_file=[], extention="ts", total_count_of_file=0, offse ret = run_command(cmd_line) +full_list_size = len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv) + len(list_files_divx) offset = 0; -trancode_local(list_files_ts , "ts", len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv), offset) -offset += len(list_files_ts); -trancode_local(list_files_mp4 , "mp4", len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv), offset) -offset += len(list_files_mp4); -trancode_local(list_files_flv , "flv", len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv), offset) -offset += len(list_files_flv); -trancode_local(list_files_avi , "avi", len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv), offset) -offset += len(list_files_avi); -trancode_local(list_files_wmv , "wmv", len(list_files_ts) + len(list_files_mp4) + len(list_files_flv) + len(list_files_avi) + len(list_files_wmv), offset) +trancode_local(list_files_ts , "ts", full_list_size, offset) +offset += len(list_files_ts) +trancode_local(list_files_mp4 , "mp4", full_list_size, offset) +offset += len(list_files_mp4) +trancode_local(list_files_flv , "flv", full_list_size, offset) +offset += len(list_files_flv) +trancode_local(list_files_avi , "avi", full_list_size, offset) +offset += len(list_files_avi) +trancode_local(list_files_wmv , "wmv", full_list_size, offset) +offset += len(list_files_wmv) +trancode_local(list_files_divx , "divx", full_list_size, offset) +offset += len(list_files_divx)