[DEV] encode in 16k personal corpus
This commit is contained in:
parent
d64ac95dd9
commit
d8dcf70f1b
269
debug.py
Normal file
269
debug.py
Normal file
@ -0,0 +1,269 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
##
|
||||||
|
## @author Edouard DUPIN
|
||||||
|
##
|
||||||
|
## @copyright 2012, Edouard DUPIN, all right reserved
|
||||||
|
##
|
||||||
|
## @license MPL v2.0 (see license file)
|
||||||
|
##
|
||||||
|
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
import re
|
||||||
|
|
||||||
|
debug_level=3
|
||||||
|
debug_color=False
|
||||||
|
|
||||||
|
color_default= ""
|
||||||
|
color_red = ""
|
||||||
|
color_green = ""
|
||||||
|
color_yellow = ""
|
||||||
|
color_blue = ""
|
||||||
|
color_purple = ""
|
||||||
|
color_cyan = ""
|
||||||
|
|
||||||
|
|
||||||
|
debug_lock = threading.Lock()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Set log level of the console log system
|
||||||
|
## @param[in] id (int) Value of the log level:
|
||||||
|
## 0: None
|
||||||
|
## 1: error
|
||||||
|
## 2: warning
|
||||||
|
## 3: info
|
||||||
|
## 4: debug
|
||||||
|
## 5: verbose
|
||||||
|
## 6: extreme_verbose
|
||||||
|
##
|
||||||
|
def set_level(id):
|
||||||
|
global debug_level
|
||||||
|
debug_level = id
|
||||||
|
#print "SetDebug level at " + str(debug_level)
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Get the current debug leval
|
||||||
|
## @return The value of the log level. Show: @ref set_level
|
||||||
|
##
|
||||||
|
def get_level():
|
||||||
|
global debug_level
|
||||||
|
return debug_level
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Enable color of the console Log system
|
||||||
|
##
|
||||||
|
def enable_color():
|
||||||
|
global debug_color
|
||||||
|
debug_color = True
|
||||||
|
global color_default
|
||||||
|
color_default= "\033[00m"
|
||||||
|
global color_red
|
||||||
|
color_red = "\033[31m"
|
||||||
|
global color_green
|
||||||
|
color_green = "\033[32m"
|
||||||
|
global color_yellow
|
||||||
|
color_yellow = "\033[33m"
|
||||||
|
global color_blue
|
||||||
|
color_blue = "\033[01;34m"
|
||||||
|
global color_purple
|
||||||
|
color_purple = "\033[35m"
|
||||||
|
global color_cyan
|
||||||
|
color_cyan = "\033[36m"
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Disable color of the console Log system
|
||||||
|
##
|
||||||
|
def disable_color():
|
||||||
|
global debug_color
|
||||||
|
debug_color = True
|
||||||
|
global color_default
|
||||||
|
color_default= ""
|
||||||
|
global color_red
|
||||||
|
color_red = ""
|
||||||
|
global color_green
|
||||||
|
color_green = ""
|
||||||
|
global color_yellow
|
||||||
|
color_yellow = ""
|
||||||
|
global color_blue
|
||||||
|
color_blue = ""
|
||||||
|
global color_purple
|
||||||
|
color_purple = ""
|
||||||
|
global color_cyan
|
||||||
|
color_cyan = ""
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print a extreme verbose log
|
||||||
|
## @param[in] input (string) Value to print if level is enough
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
##
|
||||||
|
def extreme_verbose(input, force=False):
|
||||||
|
global debug_lock
|
||||||
|
global debug_level
|
||||||
|
if debug_level >= 6 \
|
||||||
|
or force == True:
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(color_blue + input + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print a verbose log
|
||||||
|
## @param[in] input (string) Value to print if level is enough
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
##
|
||||||
|
def verbose(input, force=False):
|
||||||
|
global debug_lock
|
||||||
|
global debug_level
|
||||||
|
if debug_level >= 5 \
|
||||||
|
or force == True:
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(color_blue + input + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print a log (every time
|
||||||
|
## @param[in] input (string) Value to print if level is enough
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
##
|
||||||
|
def display(input, force=False):
|
||||||
|
global debug_lock
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(color_blue + input + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print a debug log
|
||||||
|
## @param[in] input (string) Value to print if level is enough
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
##
|
||||||
|
def debug(input, force=False):
|
||||||
|
global debug_lock
|
||||||
|
global debug_level
|
||||||
|
if debug_level >= 4 \
|
||||||
|
or force == True:
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(color_green + input + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print an info log
|
||||||
|
## @param[in] input (string) Value to print if level is enough
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
##
|
||||||
|
def info(input, force=False):
|
||||||
|
global debug_lock
|
||||||
|
global debug_level
|
||||||
|
if debug_level >= 3 \
|
||||||
|
or force == True:
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(input + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print a warning log
|
||||||
|
## @param[in] input (string) Value to print if level is enough
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
##
|
||||||
|
def warning(input, force=False):
|
||||||
|
global debug_lock
|
||||||
|
global debug_level
|
||||||
|
if debug_level >= 2 \
|
||||||
|
or force == True:
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(color_purple + "[WARNING] " + input + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print a todo log
|
||||||
|
## @param[in] input (string) Value to print if level is enough
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
##
|
||||||
|
def todo(input, force=False):
|
||||||
|
global debug_lock
|
||||||
|
global debug_level
|
||||||
|
if debug_level >= 3 \
|
||||||
|
or force == True:
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(color_purple + "[TODO] " + input + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print an error log
|
||||||
|
## @param[in] input (string) Value to print if level is enough
|
||||||
|
## @param[in] thread_id (int) Current thead ID of the builder thread
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
## @param[in] crash (bool) build error has appear ==> request stop of all builds
|
||||||
|
##
|
||||||
|
def error(input, thread_id=-1, force=False, crash=True):
|
||||||
|
global debug_lock
|
||||||
|
global debug_level
|
||||||
|
if debug_level >= 1 \
|
||||||
|
or force == True:
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(color_red + "[ERROR] " + input + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
if crash == True:
|
||||||
|
exit(-1)
|
||||||
|
#os_exit(-1)
|
||||||
|
#raise "error happend"
|
||||||
|
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print a log for a specific element action like generateing .so or binary ...
|
||||||
|
## @param[in] type (string) type of action. Like: "copy file", "StaticLib", "Prebuild", "Library" ...
|
||||||
|
## @param[in] lib (string) Name of the library/binary/package that action is done
|
||||||
|
## @param[in] dir (string) build direction. ex: "<==", "==>" ...
|
||||||
|
## @param[in] name (string) Destination of the data
|
||||||
|
## @param[in] force (bool) force display (no check of log level)
|
||||||
|
##
|
||||||
|
def print_element(type, lib, dir, name, force=False):
|
||||||
|
global debug_lock
|
||||||
|
global debug_level
|
||||||
|
if debug_level >= 3 \
|
||||||
|
or force == True:
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(color_cyan + type + color_default + " : " + color_yellow + lib + color_default + " " + dir + " " + color_blue + name + color_default)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Print a compilation return (output)
|
||||||
|
## @param[in] my_string (string) Std-error/std-info that is generate by the build system
|
||||||
|
##
|
||||||
|
def print_compilator(my_string):
|
||||||
|
global debug_color
|
||||||
|
global debug_lock
|
||||||
|
if debug_color == True:
|
||||||
|
my_string = my_string.replace('\\n', '\n')
|
||||||
|
my_string = my_string.replace('\\t', '\t')
|
||||||
|
my_string = my_string.replace('error:', color_red+'error:'+color_default)
|
||||||
|
my_string = my_string.replace('warning:', color_purple+'warning:'+color_default)
|
||||||
|
my_string = my_string.replace('note:', color_green+'note:'+color_default)
|
||||||
|
my_string = re.sub(r'([/\w_-]+\.\w+):', r'-COLORIN-\1-COLOROUT-:', my_string)
|
||||||
|
my_string = my_string.replace('-COLORIN-', color_yellow)
|
||||||
|
my_string = my_string.replace('-COLOROUT-', color_default)
|
||||||
|
|
||||||
|
debug_lock.acquire()
|
||||||
|
print(my_string)
|
||||||
|
debug_lock.release()
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Get the list of default color
|
||||||
|
## @return A map with keys: "default","red","green","yellow","blue","purple","cyan"
|
||||||
|
##
|
||||||
|
def get_color_set() :
|
||||||
|
global color_default
|
||||||
|
global color_red
|
||||||
|
global color_green
|
||||||
|
global color_yellow
|
||||||
|
global color_blue
|
||||||
|
global color_purple
|
||||||
|
global color_cyan
|
||||||
|
return {
|
||||||
|
"default": color_default,
|
||||||
|
"red": color_red,
|
||||||
|
"green": color_green,
|
||||||
|
"yellow": color_yellow,
|
||||||
|
"blue": color_blue,
|
||||||
|
"purple": color_purple,
|
||||||
|
"cyan": color_cyan,
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
102
proprocessCorpus.py
Executable file
102
proprocessCorpus.py
Executable file
@ -0,0 +1,102 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
##
|
||||||
|
## @author Edouard DUPIN
|
||||||
|
##
|
||||||
|
## @license MPL v2.0 (see license file)
|
||||||
|
##
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tools
|
||||||
|
import debug
|
||||||
|
import argparse
|
||||||
|
import math
|
||||||
|
import json
|
||||||
|
import resampy
|
||||||
|
import numpy as np
|
||||||
|
import scipy.io.wavfile as wavfile
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("-i", "--input", help="Input directory",
|
||||||
|
default="")
|
||||||
|
parser.add_argument("-o", "--output", help="Output directory",
|
||||||
|
default="out")
|
||||||
|
parser.add_argument("-v", "--verbose", help="display all LOGS",
|
||||||
|
default=False,
|
||||||
|
action='store_true')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
debug.info("***********************************")
|
||||||
|
|
||||||
|
# ---------------------------------------------
|
||||||
|
# -- check input
|
||||||
|
# ---------------------------------------------
|
||||||
|
if args.verbose == True:
|
||||||
|
debug.set_level(6);
|
||||||
|
if args.input == None \
|
||||||
|
or args.input == "" :
|
||||||
|
debug.error("must set an input directory")
|
||||||
|
|
||||||
|
tools.create_directory(args.output)
|
||||||
|
|
||||||
|
debug.info("==================================================================================================")
|
||||||
|
debug.info("== Preprocess corpus data: " + args.input + " to " + args.output)
|
||||||
|
debug.info("==================================================================================================")
|
||||||
|
|
||||||
|
debug.info("Get list of corpus files:")
|
||||||
|
audio_corpus_element = tools.get_list_of_file_in_path(args.input, ["*.json"], recursive=True)
|
||||||
|
|
||||||
|
debug.info("Corpus count " + str(len(audio_corpus_element)) + " element(s)")
|
||||||
|
elem_id = 0
|
||||||
|
for elem in audio_corpus_element:
|
||||||
|
debug.info("---------------------------[ " + str(elem_id) + " / " + str(len(audio_corpus_element)) + " ]---------------------------------------")
|
||||||
|
elem_id += 1
|
||||||
|
debug.info("Element: " + elem)
|
||||||
|
with open(elem) as file:
|
||||||
|
input_data = json.load(file)
|
||||||
|
"""{
|
||||||
|
"user": "Edouard DUPIN",
|
||||||
|
"value": "bonjour",
|
||||||
|
"language": "FR_fr",
|
||||||
|
"time": 3088499332851,
|
||||||
|
"audio_format": "int16",
|
||||||
|
"audio_channel": 1,
|
||||||
|
"audio_sample_rate": 48000,
|
||||||
|
"audio_filename": "FR_fr_Edouard DUPIN_3088499332851.raw"
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
if "audio_format" not in input_data.keys():
|
||||||
|
debug.error(" ==> missing field 'audio_format' ...")
|
||||||
|
if input_data["audio_format"] != "int16":
|
||||||
|
debug.error(" ==> field 'audio_format' have wrong value: '" + str(input_data["audio_format"]) + "' suported: [int16]")
|
||||||
|
if "audio_channel" not in input_data.keys():
|
||||||
|
debug.error(" ==> missing field 'audio_channel' ...")
|
||||||
|
if input_data["audio_channel"] != 1:
|
||||||
|
debug.error(" ==> field 'audio_channel' have wrong value: '" + str(input_data["audio_channel"]) + "' suported: [1]")
|
||||||
|
if "audio_sample_rate" not in input_data.keys():
|
||||||
|
debug.error(" ==> missing field 'audio_sample_rate' ...")
|
||||||
|
if input_data["audio_sample_rate"] != 48000:
|
||||||
|
debug.error(" ==> field 'audio_sample_rate' have wrong value: '" + str(input_data["audio_sample_rate"]) + "' suported: [48000]")
|
||||||
|
if "audio_filename" not in input_data.keys():
|
||||||
|
debug.error(" ==> missing field 'audio_filename' ...")
|
||||||
|
filename = os.path.join(os.path.dirname(elem), input_data["audio_filename"])
|
||||||
|
if filename[-3:] == "wav":
|
||||||
|
sample_rate, audio_data = wavfile.read(filename)
|
||||||
|
else:
|
||||||
|
debug.error("Not supported file type: '" + str(filename[-3:]) + "' suported: [wav]")
|
||||||
|
debug.info("Read: " + str(len(audio_data)) + " sample(s)")
|
||||||
|
|
||||||
|
audio_16k = resampy.resample(audio_data, 48000, 16000)
|
||||||
|
|
||||||
|
debug.info("write: " + str(len(audio_16k)) + " sample(s)")
|
||||||
|
|
||||||
|
filename_out = os.path.join(args.output, input_data["audio_filename"])
|
||||||
|
wavfile.write(filename_out, 16000, audio_16k)
|
||||||
|
input_data["audio_sample_rate"] = 16000
|
||||||
|
filename_out_json = os.path.join(args.output, os.path.basename(elem))
|
||||||
|
with open(filename_out_json, 'w') as outfile:
|
||||||
|
json.dump(input_data, outfile, indent="\t")
|
||||||
|
|
||||||
|
|
||||||
|
debug.info("Finish")
|
||||||
|
|
||||||
126
tools.py
Normal file
126
tools.py
Normal file
@ -0,0 +1,126 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
##
|
||||||
|
## @author Edouard DUPIN
|
||||||
|
##
|
||||||
|
## @copyright 2012, Edouard DUPIN, all right reserved
|
||||||
|
##
|
||||||
|
## @license MPL v2.0 (see license file)
|
||||||
|
##
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import errno
|
||||||
|
import fnmatch
|
||||||
|
import stat
|
||||||
|
|
||||||
|
|
||||||
|
def create_directory(path):
|
||||||
|
try:
|
||||||
|
os.stat(path)
|
||||||
|
except:
|
||||||
|
os.makedirs(path)
|
||||||
|
|
||||||
|
def create_directory_of_file(file):
|
||||||
|
path = os.path.dirname(file)
|
||||||
|
create_directory(path)
|
||||||
|
|
||||||
|
def file_write_data(path, data):
|
||||||
|
print(" write file: " + path)
|
||||||
|
create_directory_of_file(path)
|
||||||
|
file = open(path, "w")
|
||||||
|
file.write(data)
|
||||||
|
file.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
def remove_file(path):
|
||||||
|
if os.path.isfile(path):
|
||||||
|
os.remove(path)
|
||||||
|
elif os.path.islink(path):
|
||||||
|
os.remove(path)
|
||||||
|
|
||||||
|
def file_read_data(path, binary=False):
|
||||||
|
print("path= " + path)
|
||||||
|
if not os.path.isfile(path):
|
||||||
|
return ""
|
||||||
|
if binary == True:
|
||||||
|
file = open(path, "rb")
|
||||||
|
else:
|
||||||
|
file = open(path, "r")
|
||||||
|
data_file = file.read()
|
||||||
|
file.close()
|
||||||
|
return data_file
|
||||||
|
|
||||||
|
def copy_file(src, dst):
|
||||||
|
print("copy " + src + " ==> " + dst)
|
||||||
|
create_directory_of_file(dst)
|
||||||
|
shutil.copyfile(src, dst)
|
||||||
|
|
||||||
|
def copy_anything(src, dst):
|
||||||
|
print(" copy anything : '" + str(src) + "'")
|
||||||
|
print(" to : '" + str(dst) + "'")
|
||||||
|
if os.path.isdir(os.path.realpath(src)):
|
||||||
|
tmp_path = os.path.realpath(src)
|
||||||
|
tmp_rule = ""
|
||||||
|
else:
|
||||||
|
tmp_path = os.path.dirname(os.path.realpath(src))
|
||||||
|
tmp_rule = os.path.basename(src)
|
||||||
|
|
||||||
|
for root, dirnames, filenames in os.walk(tmp_path):
|
||||||
|
deltaRoot = root[len(tmp_path):]
|
||||||
|
while len(deltaRoot) > 0 \
|
||||||
|
and ( deltaRoot[0] == '/' \
|
||||||
|
or deltaRoot[0] == '\\' ):
|
||||||
|
deltaRoot = deltaRoot[1:]
|
||||||
|
if deltaRoot != "":
|
||||||
|
return
|
||||||
|
tmpList = filenames
|
||||||
|
if len(tmp_rule) > 0:
|
||||||
|
tmpList = fnmatch.filter(filenames, tmp_rule)
|
||||||
|
# Import the module :
|
||||||
|
for cycleFile in tmpList:
|
||||||
|
#for cycleFile in filenames:
|
||||||
|
copy_file(os.path.join(tmp_path, deltaRoot, cycleFile),
|
||||||
|
os.path.join(dst, deltaRoot, cycleFile))
|
||||||
|
|
||||||
|
|
||||||
|
##
|
||||||
|
## @brief Get list of all Files in a specific path (with a regex)
|
||||||
|
## @param[in] path (string) Full path of the machine to search files (start with / or x:)
|
||||||
|
## @param[in] regex (string) Regular expression to search data
|
||||||
|
## @param[in] recursive (bool) List file with recursive search
|
||||||
|
## @param[in] remove_path (string) Data to remove in the path
|
||||||
|
## @return (list) return files requested
|
||||||
|
##
|
||||||
|
def get_list_of_file_in_path(path, filter, recursive = False, remove_path=""):
|
||||||
|
out = []
|
||||||
|
if os.path.isdir(os.path.realpath(path)):
|
||||||
|
tmp_path = os.path.realpath(path)
|
||||||
|
else:
|
||||||
|
print("[E] path does not exist : '" + str(path) + "'")
|
||||||
|
|
||||||
|
for root, dirnames, filenames in os.walk(tmp_path):
|
||||||
|
deltaRoot = root[len(tmp_path):]
|
||||||
|
while len(deltaRoot) > 0 \
|
||||||
|
and ( deltaRoot[0] == '/' \
|
||||||
|
or deltaRoot[0] == '\\' ):
|
||||||
|
deltaRoot = deltaRoot[1:]
|
||||||
|
if recursive == False \
|
||||||
|
and deltaRoot != "":
|
||||||
|
return out
|
||||||
|
tmpList = []
|
||||||
|
for elem in filter:
|
||||||
|
tmpppp = fnmatch.filter(filenames, elem)
|
||||||
|
for elemmm in tmpppp:
|
||||||
|
tmpList.append(elemmm)
|
||||||
|
# Import the module :
|
||||||
|
for cycleFile in tmpList:
|
||||||
|
#for cycleFile in filenames:
|
||||||
|
add_file = os.path.join(tmp_path, deltaRoot, cycleFile)
|
||||||
|
if len(remove_path) != 0:
|
||||||
|
if add_file[:len(remove_path)] != remove_path:
|
||||||
|
print("[E] Request remove start of a path that is not the same: '" + add_file[:len(remove_path)] + "' demand remove of '" + str(remove_path) + "'")
|
||||||
|
else:
|
||||||
|
add_file = add_file[len(remove_path)+1:]
|
||||||
|
out.append(add_file)
|
||||||
|
return out;
|
||||||
Loading…
x
Reference in New Issue
Block a user