diff --git a/tools/create_supplement_gypi.py b/tools/create_supplement_gypi.py new file mode 100644 index 000000000..f5a08e619 --- /dev/null +++ b/tools/create_supplement_gypi.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright (c) 2011 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import sys + +supplement_gypi = """#!/usr/bin/env python +# This file is generated by %s. Not for check-in. +# Please see the WebRTC DEPS file for details. +{ + 'variables': { + 'build_with_chromium': 0, + 'inside_chromium_build': 0, + } +} +""" + +def main(argv): + open(argv[1], 'w').write(supplement_gypi % argv[0]) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/tools/refactoring/addfileheader.py b/tools/refactoring/addfileheader.py new file mode 100644 index 000000000..01c8a8b4e --- /dev/null +++ b/tools/refactoring/addfileheader.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python + +import stringmanipulation +import filemanagement +import sys + +extensions = ['.h','.cc','.c','.cpp'] + +ignore_these = ['my_ignore_header.h'] + +if((len(sys.argv) != 2) and (len(sys.argv) != 3)): + print 'parameters are: directory [--commit]' + quit() + +directory = sys.argv[1]; +if(not filemanagement.pathexist(directory)): + print 'path ' + directory + ' does not exist' + quit() + +if((len(sys.argv) == 3) and (sys.argv[2] != '--commit')): + print 'parameters are: parent directory extension new extension [--commit]' + quit() + +commit = False +if(len(sys.argv) == 3): + commit = True + +files_to_fix = [] +for extension in extensions: + files_to_fix.extend(filemanagement.listallfilesinfolder(directory,\ + extension)) + +# Just steal the header from the template +def fileheaderasstring(): + template_file_name = 'license_template.txt' + if (not filemanagement.fileexist(template_file_name)): + print 'File ' + template_file_name + ' not found!' + quit() + template_file = open(template_file_name,'r') + return_string = '' + for line in template_file: + return_string += line + return return_string + +# Just steal the header from the template +def fileheaderasarray(): + template_file_name = 'license_template.txt' + if (not filemanagement.fileexist(template_file_name)): + print 'File ' + template_file_name + ' not found!' + quit() + template_file = open(template_file_name,'r') + return_value = [] + for line in template_file: + return_value.append(line) + return return_value + + +def findheader(path, file_name): + full_file_name = path + file_name + if (not filemanagement.fileexist(full_file_name)): + print 'File ' + file_name + ' not found!' + print 'Unexpected error!' + quit() + file_handle = open(full_file_name) + template_file_content = fileheaderasarray() + compare_content = [] + # load the same number of lines from file as the fileheader + for index in range(len(template_file_content)): + line = file_handle.readline() + if (line == ''): + return False + compare_content.append(line) + + while (True): + found = True + for index in range(len(template_file_content)): + line1 = template_file_content[index] + line2 = compare_content[index] + if(line1 != line2): + found = False + break + if (found): + return True + compare_content = compare_content[1:len(compare_content)] + line = file_handle.readline() + if (line == ''): + return False + compare_content.append(line) + return False + +# Used to store temporary result before flushing to real file when finished +def temporaryfilename(old_file_name): + return old_file_name + '.deleteme' + +def updatefile(path, old_file_name): + full_old_file_name = path + old_file_name + if (not filemanagement.fileexist(full_old_file_name)): + print 'File ' + full_old_file_name + ' is not found.' + print 'Should not happen! Ever!' + quit() + + full_temporary_file_name = path + temporaryfilename(old_file_name) + + # Make sure that the files are closed by putting them out of scope + old_file = open(full_old_file_name,'r') + temporary_file = open(full_temporary_file_name,'w') + + temporary_file.writelines(fileheaderasstring()) + remove_whitespaces = True + for line in old_file: + if (remove_whitespaces and (len(line.split()) == 0)): + continue + else: + remove_whitespaces = False + temporary_file.writelines(line) + old_file.close() + temporary_file.close() + + filemanagement.copyfile(full_old_file_name,full_temporary_file_name) + filemanagement.deletefile(full_temporary_file_name) + + +failed_files = [] +skipped_files = [] +for index in range(len(files_to_fix)): + if(commit): + print (100*index)/len(files_to_fix) + path_dir = files_to_fix[index][0] + filename = files_to_fix[index][1] + is_ignore = False + for ignore_names in ignore_these: + if(filename == ignore_names): + is_ignore = True + break + if(is_ignore): + continue + +# Let the word copyright be our sanity, i.e. make sure there is only one +# copy right occurance or report that there will be no change + if(filemanagement.findstringinfile(path_dir,filename,'Copyright') or + filemanagement.findstringinfile(path_dir,filename,'copyright') or + filemanagement.findstringinfile(path_dir,filename,'COPYRIGHT')): + if(findheader(path_dir,filename)): + skipped_files.append(path_dir + filename) + else: + failed_files.append(path_dir + filename) + continue + + if (not commit): + print 'File ' + path_dir + filename + ' will be updated' + continue + updatefile(path_dir,filename) + +tense = 'will be' +if (commit): + tense = 'has been' +if (len(skipped_files) > 0): + print str(len(skipped_files)) + ' file(s) ' + tense + ' skipped since they already have the correct header' + +if (len(failed_files) > 0): + print 'Following files seem to have an invalid file header:' +for line in failed_files: + print line diff --git a/tools/refactoring/filemanagement.py b/tools/refactoring/filemanagement.py new file mode 100644 index 000000000..4ff64ceb2 --- /dev/null +++ b/tools/refactoring/filemanagement.py @@ -0,0 +1,72 @@ +import fnmatch +import os +import stringmanipulation + +def fileexist( file_name ): + return os.path.isfile(file_name) + +def pathexist( path ): + return os.path.exists(path) + +def fixpath( path ): + return_value = path + if( return_value[len(return_value) - 1] != '/'): + return_value = return_value + '/' + return return_value + +def listallfilesinfolder( path, extension ): + matches = [] + signature = '*' + extension + for root, dirnames, filenames in os.walk(path): + for filename in fnmatch.filter(filenames, signature): + matches.append([fixpath(root), filename]) + return matches + +def copyfile(to_file, from_file): + if(not fileexist(from_file)): + return + command = 'cp -f ' + from_file + ' ' + to_file + os.system(command) + #print command + +def deletefile(file_to_delete): + if(not fileexist(file_to_delete)): + return + os.system('rm ' + file_to_delete) + +# very ugly but works, so keep for now +def findstringinfile(path,file_name,search_string): + command = 'grep \'' + search_string + '\' ' + path + file_name + ' > deleteme.txt' + return_value = os.system(command) +# print command + return (return_value == 0) + +def replacestringinfolder( path, old_string, new_string, extension ): + if(not stringmanipulation.isextension(extension)): + print 'failed to search and replace' + return + if(len(old_string) == 0): + print 'failed to search and replace' + return + find_command = 'ls '+ path + '/*' + extension + sed_command = 'sed -i \'s/' + old_string + '/' + new_string +\ + '/g\' *' + extension + command_string = find_command + ' | xargs ' + sed_command + ' 2> deleteme.txt' + os.system(command_string) + #print command_string + +#find ./ -name "*.h" -type f | xargs -P 0 sed -i 's/process_thread_wrapper.h/process_thread.h/g' *.h deleteme.txt +def replacestringinallsubfolders( old_string, new_string, extension): + if(not stringmanipulation.isextension(extension)): + print 'failed to search and replace' + return + if(len(old_string) == 0): + print 'failed to search and replace' + return + + find_command = 'find ./ -name \"*' + extension + '\" -type f' + sed_command = 'sed -i \'s/' + old_string + '/' + new_string +\ + '/g\' *' + extension + command_string = find_command + ' | xargs -P 0 ' + sed_command + ' 2> deleteme.txt' + os.system(command_string) + #print command_string diff --git a/tools/refactoring/fixincludeguards.py b/tools/refactoring/fixincludeguards.py new file mode 100644 index 000000000..0b563556b --- /dev/null +++ b/tools/refactoring/fixincludeguards.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python + +import stringmanipulation +import filemanagement +import sys + +extensions = ['.h'] + +ignore_these = ['my_ignore_header.h'] + +if((len(sys.argv) != 2) and (len(sys.argv) != 3)): + print 'parameters are: directory [--commit]' + quit() + +directory = sys.argv[1]; +if(not filemanagement.pathexist(directory)): + print 'path ' + directory + ' does not exist' + quit() + +if((len(sys.argv) == 3) and (sys.argv[2] != '--commit')): + print 'parameters are: parent directory extension new extension [--commit]' + quit() + +commit = False +if(len(sys.argv) == 3): + commit = True + +for extension in extensions: + files_to_fix = filemanagement.listallfilesinfolder(directory,\ + extension) + +def buildincludeguardname(path,filename): + full_file_name = 'WEBRTC_' + path + filename + full_file_name = full_file_name.upper() + full_file_name = stringmanipulation.replaceoccurances(full_file_name, '/', '_') + full_file_name = stringmanipulation.replaceoccurances(full_file_name, '\\', '_') + full_file_name = stringmanipulation.replaceoccurances(full_file_name, '.', '_') + full_file_name += '_' + return full_file_name + +def buildnewincludeguardset(path,filename): + include_guard_name = buildincludeguardname(path,filename) + if(include_guard_name == ''): + return [] + return_value = [] + return_value.append('#ifndef ' + include_guard_name) + return_value.append('#define ' + include_guard_name) + return_value.append(include_guard_name) + return return_value + +def printincludeguardset(include_guard_set): + print 'First line: ' + include_guard_set[0] + print 'Second line: ' + include_guard_set[1] + print 'Last line: ' + include_guard_set[2] + return + +include_guard_begin_identifier = ['#ifndef', '#if !defined'] +include_guard_second_identifier = ['#define'] +def findincludeguardidentifier(line): + for begin_identifier in include_guard_begin_identifier: + line = stringmanipulation.removealloccurances(line,begin_identifier) + for second_identifier in include_guard_begin_identifier: + line = stringmanipulation.removealloccurances(line,second_identifier) + removed_prefix = [True,''] + line = stringmanipulation.whitespacestoonespace(line) + while(removed_prefix[0]): + removed_prefix = stringmanipulation.removeprefix(line,' ') + line = removed_prefix[1] + line = stringmanipulation.removealloccurances(line,'(') + if(line == ''): + return '' + word_pos = stringmanipulation.getword(line,0) + return_value = line[0:word_pos[1]] + return_value = return_value.rstrip('\r\n') + return return_value + +def findoldincludeguardset(path,filename): + return_value = [] + full_file_name = path + filename + file_pointer = open(full_file_name,'r') + include_guard_name = '' + for line in file_pointer: + if (include_guard_name == ''): + for compare_string in include_guard_begin_identifier: + if (stringmanipulation.issubstring(compare_string, line) != -1): + include_guard_name = findincludeguardidentifier(line) + if (include_guard_name == ''): + break + line = line.rstrip('\r\n') + return_value.append(line) + break + else: + for compare_string in include_guard_second_identifier: + if (stringmanipulation.issubstring(compare_string, line) != -1): + if (stringmanipulation.issubstring(include_guard_name, line) != -1): + line = line.rstrip('\r\n') + return_value.append(line) + return_value.append(include_guard_name) + return return_value + include_guard_name = '' + return_value = [] + return [] + +failed_files = [] +for index in range(len(files_to_fix)): + if(commit): + print (100*index)/len(files_to_fix) + path_dir = files_to_fix[index][0] + filename = files_to_fix[index][1] + is_ignore = False + for ignore_names in ignore_these: + if(filename == ignore_names): + is_ignore = True + break + if(is_ignore): + continue + old_include_guard_set = findoldincludeguardset(path_dir,filename) + if (len(old_include_guard_set) != 3) : + failed_files.append('unable to figure out the include guards for ' + filename) + continue + + new_include_guard_set = buildnewincludeguardset(path_dir,filename) + if (len(new_include_guard_set) != 3) : + failed_files.append('unable to figure out new the include guards for ' + filename) + continue + + if(not commit): + print 'old guard: ' + old_include_guard_set[2] + print 'new guard: ' + new_include_guard_set[2] + continue + + for index in range(2): + # enough to only replace for file. However, no function for that + for extension in extensions: + filemanagement.replacestringinfolder(path_dir,old_include_guard_set[index],new_include_guard_set[index],extension) + # special case for last to avoid complications + for extension in extensions: + filemanagement.replacestringinfolder(path_dir,' ' + old_include_guard_set[2],' ' + new_include_guard_set[2],extension) + filemanagement.replacestringinfolder(path_dir,'\\/\\/' + old_include_guard_set[2],'\\/\\/ ' + new_include_guard_set[2],extension) + + +if(len(failed_files) > 0): + print 'Following failures should be investigated manually:' +for line in failed_files: + print line diff --git a/tools/refactoring/fixnames.py b/tools/refactoring/fixnames.py new file mode 100644 index 000000000..15381e38d --- /dev/null +++ b/tools/refactoring/fixnames.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python + +import stringmanipulation +import filemanagement +import p4commands +import sys + +name_space_to_ignore = 'GIPS::' +#only allow one prefix to be removed since allowing multiple will complicate +# things +prefix_to_filter = 'gips' +#words_to_filter = ['Module'] +# it might be dangerous to remove GIPS but keep it default +words_to_filter = ['Module','GIPS'] + +# This script finds all the words that should be replaced in an h-file. Once +# all words that should be replaced are found it does a global search and +# replace. + +extensions_to_edit = ['.cpp','.cc','.h'] + +#line = ' ~hiGIPSCriticalSectionScoped()' +#print line +#position = stringmanipulation.getword(line,11) +#old_word = line[position[0]:position[0]+position[1]] +#result = stringmanipulation.removealloccurances(old_word,'gips') +#new_word = result +#print old_word +#print position[0] +#print position[0]+position[1] +#print new_word +#quit() + +# Ignore whole line if any item in this table is a substring of the line +do_not_replace_line_table = [] +do_not_replace_line_table.append('namespace GIPS') + +# [old_string,new_string] +# List of things to remove that are static: +manual_replace_table = [] +#manual_replace_table.append(['using namespace GIPS;','']) +#manual_replace_table.append(['CreateGipsEvent','CreateEvent']) +#manual_replace_table.append(['CreateGIPSTrace','CreateTrace']) +#manual_replace_table.append(['ReturnGIPSTrace','ReturnTrace']) +#manual_replace_table.append(['CreateGIPSFile','CreateFile']) +replace_table = manual_replace_table +#replace_table.append(['GIPS::','webrtc::']) +# List of things to not remove that are static, i.e. exceptions: +# don't replace any of the GIPS_Words since that will affect all files +# do that in a separate script! +do_not_replace_table = [] +do_not_replace_table.append('GIPS_CipherTypes') +do_not_replace_table.append('GIPS_AuthenticationTypes') +do_not_replace_table.append('GIPS_SecurityLevels') +do_not_replace_table.append('GIPS_encryption') +do_not_replace_table.append('~GIPS_encryption') +do_not_replace_table.append('GIPS_transport') +do_not_replace_table.append('~GIPS_transport') +do_not_replace_table.append('GIPSTraceCallback') +do_not_replace_table.append('~GIPSTraceCallback') +do_not_replace_table.append('GIPS_RTP_CSRC_SIZE') +do_not_replace_table.append('GIPS_RTPDirections') +do_not_replace_table.append('GIPS_RTP_INCOMING') +do_not_replace_table.append('GIPS_RTP_OUTGOING') +do_not_replace_table.append('GIPSFrameType') +do_not_replace_table.append('GIPS_FRAME_EMPTY') +do_not_replace_table.append('GIPS_AUDIO_FRAME_SPEECH') +do_not_replace_table.append('GIPS_AUDIO_FRAME_CN') +do_not_replace_table.append('GIPS_VIDEO_FRAME_KEY') +do_not_replace_table.append('GIPS_VIDEO_FRAME_DELTA') +do_not_replace_table.append('GIPS_VIDEO_FRAME_GOLDEN') +do_not_replace_table.append('GIPS_VIDEO_FRAME_DELTA_KEY') +do_not_replace_table.append('GIPS_PacketType') +do_not_replace_table.append('GIPS_PACKET_TYPE_RTP') +do_not_replace_table.append('GIPS_PACKET_TYPE_KEEP_ALIVE') +do_not_replace_table.append('GIPS_AudioLayers') +do_not_replace_table.append('GIPS_AUDIO_PLATFORM_DEFAULT') +do_not_replace_table.append('GIPS_AUDIO_WINDOWS_WAVE') +do_not_replace_table.append('GIPS_AUDIO_WINDOWS_CORE') +do_not_replace_table.append('GIPS_AUDIO_LINUX_ALSA') +do_not_replace_table.append('GIPS_AUDIO_LINUX_PULSE') +do_not_replace_table.append('GIPS_AUDIO_FORMAT') +do_not_replace_table.append('GIPS_PCM_16_16KHZ') +do_not_replace_table.append('GIPS_PCM_16_8KHZ') +do_not_replace_table.append('GIPS_G729') +do_not_replace_table.append('GIPSAMRmode') +do_not_replace_table.append('GIPS_RFC3267_BWEFFICIENT') +do_not_replace_table.append('GIPS_RFC3267_OCTETALIGNED') +do_not_replace_table.append('GIPS_RFC3267_FILESTORAGE') +do_not_replace_table.append('GIPS_NCModes') +do_not_replace_table.append('GIPS_NC_OFF') +do_not_replace_table.append('GIPS_NC_MILD') +do_not_replace_table.append('GIPS_NC_MODERATE') +do_not_replace_table.append('GIPS_NC_AGGRESSIVE') +do_not_replace_table.append('GIPS_NC_VERY_AGGRESSIVE') +do_not_replace_table.append('GIPS_AGCModes') +do_not_replace_table.append('GIPS_AGC_OFF') +do_not_replace_table.append('GIPS_AGC_ANALOG') +do_not_replace_table.append('GIPS_AGC_DIGITAL') +do_not_replace_table.append('GIPS_AGC_STANDALONE_DIG') +do_not_replace_table.append('GIPS_ECModes') +do_not_replace_table.append('GIPS_EC_UNCHANGED') +do_not_replace_table.append('GIPS_EC_DEFAULT') +do_not_replace_table.append('GIPS_EC_CONFERENCE') +do_not_replace_table.append('GIPS_EC_AEC') +do_not_replace_table.append('GIPS_EC_AES') +do_not_replace_table.append('GIPS_EC_AECM') +do_not_replace_table.append('GIPS_EC_NEC_IAD') +do_not_replace_table.append('GIPS_AESModes') +do_not_replace_table.append('GIPS_AES_DEFAULT') +do_not_replace_table.append('GIPS_AES_NORMAL') +do_not_replace_table.append('GIPS_AES_HIGH') +do_not_replace_table.append('GIPS_AES_ATTENUATE') +do_not_replace_table.append('GIPS_AES_NORMAL_SOFT_TRANS') +do_not_replace_table.append('GIPS_AES_HIGH_SOFT_TRANS') +do_not_replace_table.append('GIPS_AES_ATTENUATE_SOFT_TRANS') +do_not_replace_table.append('GIPS_AECMModes') +do_not_replace_table.append('GIPS_AECM_QUIET_EARPIECE_OR_HEADSET') +do_not_replace_table.append('GIPS_AECM_EARPIECE') +do_not_replace_table.append('GIPS_AECM_LOUD_EARPIECE') +do_not_replace_table.append('GIPS_AECM_SPEAKERPHONE') +do_not_replace_table.append('GIPS_AECM_LOUD_SPEAKERPHONE') +do_not_replace_table.append('AECM_LOUD_SPEAKERPHONE') +do_not_replace_table.append('GIPS_VAD_CONVENTIONAL') +do_not_replace_table.append('GIPS_VAD_AGGRESSIVE_LOW') +do_not_replace_table.append('GIPS_VAD_AGGRESSIVE_MID') +do_not_replace_table.append('GIPS_VAD_AGGRESSIVE_HIGH') +do_not_replace_table.append('GIPS_NetEQModes') +do_not_replace_table.append('GIPS_NETEQ_DEFAULT') +do_not_replace_table.append('GIPS_NETEQ_STREAMING') +do_not_replace_table.append('GIPS_NETEQ_FAX') +do_not_replace_table.append('GIPS_NetEQBGNModes') +do_not_replace_table.append('GIPS_BGN_ON') +do_not_replace_table.append('GIPS_BGN_FADE') +do_not_replace_table.append('GIPS_BGN_OFF') +do_not_replace_table.append('GIPS_OnHoldModes') +do_not_replace_table.append('GIPS_HOLD_SEND_AND_PLAY') +do_not_replace_table.append('GIPS_HOLD_SEND_ONLY') +do_not_replace_table.append('GIPS_HOLD_PLAY_ONLY') +do_not_replace_table.append('GIPS_PayloadFrequencies') +do_not_replace_table.append('GIPS_FREQ_8000_HZ') +do_not_replace_table.append('GIPS_FREQ_16000_HZ') +do_not_replace_table.append('GIPS_FREQ_32000_HZ') +do_not_replace_table.append('GIPS_TelephoneEventDetectionMethods') +do_not_replace_table.append('GIPS_IN_BAND') +do_not_replace_table.append('GIPS_OUT_OF_BAND') +do_not_replace_table.append('GIPS_IN_AND_OUT_OF_BAND') +do_not_replace_table.append('GIPS_ProcessingTypes') +do_not_replace_table.append('GIPS_PLAYBACK_PER_CHANNEL') +do_not_replace_table.append('GIPS_PLAYBACK_ALL_CHANNELS_MIXED') +do_not_replace_table.append('GIPS_RECORDING_PER_CHANNEL') +do_not_replace_table.append('GIPS_RECORDING_ALL_CHANNELS_MIXED') +do_not_replace_table.append('GIPS_StereoChannel') +do_not_replace_table.append('GIPS_StereoLeft') +do_not_replace_table.append('GIPS_StereoRight') +do_not_replace_table.append('GIPS_StereoBoth') +do_not_replace_table.append('GIPS_stat_val') +do_not_replace_table.append('GIPS_P56_statistics') +do_not_replace_table.append('GIPS_echo_statistics') +do_not_replace_table.append('GIPS_NetworkStatistics') +do_not_replace_table.append('GIPS_JitterStatistics') +do_not_replace_table.append('GIPSVideoRawType') +do_not_replace_table.append('GIPS_VIDEO_I420') +do_not_replace_table.append('GIPS_VIDEO_YV12') +do_not_replace_table.append('GIPS_VIDEO_YUY2') +do_not_replace_table.append('GIPS_VIDEO_UYVY') +do_not_replace_table.append('GIPS_VIDEO_IYUV') +do_not_replace_table.append('GIPS_VIDEO_ARGB') +do_not_replace_table.append('GIPS_VIDEO_RGB24') +do_not_replace_table.append('GIPS_VIDEO_RGB565') +do_not_replace_table.append('GIPS_VIDEO_ARGB4444') +do_not_replace_table.append('GIPS_VIDEO_ARGB1555') +do_not_replace_table.append('GIPS_VIDEO_MJPG') +do_not_replace_table.append('GIPS_VIDEO_NV12') +do_not_replace_table.append('GIPS_VIDEO_NV21') +do_not_replace_table.append('GIPS_VIDEO_Unknown') +do_not_replace_table.append('GIPSVideoLayouts') +do_not_replace_table.append('GIPS_LAYOUT_NONE') +do_not_replace_table.append('GIPS_LAYOUT_DEFAULT') +do_not_replace_table.append('GIPS_LAYOUT_ADVANCED1') +do_not_replace_table.append('GIPS_LAYOUT_ADVANCED2') +do_not_replace_table.append('GIPS_LAYOUT_ADVANCED3') +do_not_replace_table.append('GIPS_LAYOUT_ADVANCED4') +do_not_replace_table.append('GIPS_LAYOUT_FULL') +do_not_replace_table.append('KGIPSConfigParameterSize') +do_not_replace_table.append('KGIPSPayloadNameSize') +do_not_replace_table.append('GIPSVideoCodecH263') +do_not_replace_table.append('GIPSVideoH264Packetization') +do_not_replace_table.append('GIPS_H264_SingleMode') +do_not_replace_table.append('GIPS_H264_NonInterleavedMode') +do_not_replace_table.append('GIPSVideoCodecComplexity') +do_not_replace_table.append('GIPSVideoCodec_Complexity_Normal') +do_not_replace_table.append('GIPSVideoCodec_Comlexity_High') +do_not_replace_table.append('GIPSVideoCodec_Comlexity_Higher') +do_not_replace_table.append('GIPSVideoCodec_Comlexity_Max') +do_not_replace_table.append('GIPSVideoCodecH264') +do_not_replace_table.append('GIPSVideoH264Packetization') +do_not_replace_table.append('GIPSVideoCodecComplexity') +do_not_replace_table.append('GIPSVideoCodecProfile') +do_not_replace_table.append('KGIPSConfigParameterSize') +do_not_replace_table.append('KGIPSMaxSVCLayers') +do_not_replace_table.append('GIPSVideoH264LayerTypes') +do_not_replace_table.append('GIPS_H264SVC_Base') +do_not_replace_table.append('GIPS_H264SVC_Extend_2X2') +do_not_replace_table.append('GIPS_H264SVC_Extend_1X1') +do_not_replace_table.append('GIPS_H264SVC_Extend_MGS') +do_not_replace_table.append('GIPS_H264SVC_Extend_1_5') +do_not_replace_table.append('GIPS_H264SVC_Extend_Custom') +do_not_replace_table.append('GIPSVideoH264LayersProperties') +do_not_replace_table.append('GIPSVideoH264LayerTypes') +do_not_replace_table.append('GIPSVideoH264Layers') +do_not_replace_table.append('GIPSVideoH264LayersProperties') +do_not_replace_table.append('GIPSVideoCodecH264SVC') +do_not_replace_table.append('GIPSVideoCodecComplexity') +do_not_replace_table.append('GIPSVideoCodecProfile') +do_not_replace_table.append('GIPSVideoH264Layers') +do_not_replace_table.append('GIPSVideoCodecVP8') +do_not_replace_table.append('GIPSVideoCodecComplexity') +do_not_replace_table.append('GIPSVideoCodecMPEG') +do_not_replace_table.append('GIPSVideoCodecGeneric') +do_not_replace_table.append('GIPSVideoCodecType') +do_not_replace_table.append('GIPSVideoCodec_H263') +do_not_replace_table.append('GIPSVideoCodec_H264') +do_not_replace_table.append('GIPSVideoCodec_H264SVC') +do_not_replace_table.append('GIPSVideoCodec_VP8') +do_not_replace_table.append('GIPSVideoCodec_MPEG4') +do_not_replace_table.append('GIPSVideoCodec_I420') +do_not_replace_table.append('GIPSVideoCodec_RED') +do_not_replace_table.append('GIPSVideoCodec_ULPFEC') +do_not_replace_table.append('GIPSVideoCodec_Unknown') +do_not_replace_table.append('GIPSVideoCodecUnion') +do_not_replace_table.append('GIPSVideoCodecH263') +do_not_replace_table.append('GIPSVideoCodecH264') +do_not_replace_table.append('GIPSVideoCodecH264SVC') +do_not_replace_table.append('GIPSVideoCodecVP8') +do_not_replace_table.append('GIPSVideoCodecMPEG4') +do_not_replace_table.append('GIPSVideoCodecGeneric') +do_not_replace_table.append('GIPSVideoCodec') +do_not_replace_table.append('GIPSVideoCodecType') +do_not_replace_table.append('GIPSVideoCodecUnion') +do_not_replace_table.append('GIPSAudioFrame') +do_not_replace_table.append('GIPS_CodecInst') +do_not_replace_table.append('GIPS_FileFormats') +do_not_replace_table.append('GIPSTickTime') +do_not_replace_table.append('GIPS_Word64') +do_not_replace_table.append('GIPS_UWord64') +do_not_replace_table.append('GIPS_Word32') +do_not_replace_table.append('GIPS_UWord32') +do_not_replace_table.append('GIPS_Word16') +do_not_replace_table.append('GIPS_UWord16') +do_not_replace_table.append('GIPS_Word8') +do_not_replace_table.append('GIPS_UWord8') + +if((len(sys.argv) != 2) and (len(sys.argv) != 3)): + print 'parameters are: parent directory [--commit]' + quit() + +if((len(sys.argv) == 3) and (sys.argv[2] != '--commit')): + print 'parameters are: parent directory [--commit]' + quit() + +commit = (len(sys.argv) == 3) + +directory = sys.argv[1]; +if(not filemanagement.pathexist(directory)): + print 'path ' + directory + ' does not exist' + quit() + +# APIs are all in h-files +extension = '.h' + +# All h-files +files_to_modify = filemanagement.listallfilesinfolder(directory,\ + extension) + +def isinmanualremovetable( compare_word ): + for old_word, new_word in manual_replace_table: + if(old_word == compare_word): + return True + return False + +# Begin +# This function looks at each line and decides which words should be replaced +# that is this is the only part of the script that you will ever want to change! +def findstringstoreplace(line): + original_line = line +# Dont replace compiler directives + if(line[0] == '#'): + return [] +# Dont allow global removal of namespace gips since it is very intrusive + for sub_string_compare in do_not_replace_line_table: + index = stringmanipulation.issubstring(line,sub_string_compare) + if(index != -1): + return [] + + return_value = [] + + line = stringmanipulation.removeccomment(line) + line = stringmanipulation.whitespacestoonespace(line) + if(len(line) == 0): + return [] + if(line[0] == '*'): + return [] + index = stringmanipulation.issubstring(line,prefix_to_filter) + while index >= 0: + dont_store_hit = False + word_position = stringmanipulation.getword(line, index) + start_of_word = word_position[0] + size_of_word = word_position[1] + end_of_word = start_of_word + size_of_word + old_word = line[start_of_word:end_of_word] + if(isinmanualremovetable(old_word)): + dont_store_hit = True + if((end_of_word + 2 < len(line)) and\ + name_space_to_ignore == line[start_of_word:end_of_word+2]): + dont_store_hit = True + + result = stringmanipulation.removeprefix(old_word,prefix_to_filter) + new_word = result[1] + for word_to_filter in words_to_filter: + new_word = stringmanipulation.removealloccurances(new_word,word_to_filter) + result = stringmanipulation.removeprefix(new_word,'_') + new_word = result[1] + new_word = stringmanipulation.fixabbreviations(new_word) + new_word = stringmanipulation.removealloccurances(new_word,'_') + if(not dont_store_hit): + return_value.append([old_word,new_word]) +# remove the word we found from the string so we dont find it again + line = line[0:start_of_word] + line[end_of_word:len(line)] + index = stringmanipulation.issubstring(line,'GIPS') + + return return_value +# End + +# loop through all files +for path, file_name in files_to_modify: +# if(file_name != 'GIPSTickUtil.h'): +# continue + full_file_name = path + file_name + file_pointer = open(full_file_name,'r') +# print file_name +#loop through all lines + for line in file_pointer: +# print line + local_replace_string = findstringstoreplace(line) + #print local_replace_string + if(len(local_replace_string) != 0): + replace_table.extend(local_replace_string) + + +# we have built our replace table now +replace_table = stringmanipulation.removeduplicates( replace_table ) +replace_table = stringmanipulation.ordertablesizefirst( replace_table ) +replace_table = stringmanipulation.complement(replace_table,\ + do_not_replace_table) + +def replaceoriginal( path,my_table ): + size_of_table = len(my_table) + for index in range(len(my_table)): + old_name = my_table[index][0] + new_name = my_table[index][1] + filemanagement.replacestringinfolder(path, old_name, new_name,\ + ".h") + print (100*index) / (size_of_table*2) + +def replaceall( my_table, extension_list ): + size_of_table = len(my_table) + for index in range(len(my_table)): + old_name = my_table[index][0] + new_name = my_table[index][1] + new_name = new_name + for extension in extensions_to_edit: + filemanagement.replacestringinallsubfolders(old_name, new_name, + extension) + print 100*(size_of_table + index) / (size_of_table*2) + + +if(commit): + print 'commiting' + replace_table = stringmanipulation.removenochange(replace_table) + p4commands.checkoutallfiles() + replaceoriginal(directory,replace_table) + replaceall(replace_table,extensions_to_edit) + p4commands.revertunchangedfiles() +else: + for old_name, new_name in replace_table: + print 'Going to replace [' + old_name + '] with [' + new_name + ']' diff --git a/tools/refactoring/integratefiles.py b/tools/refactoring/integratefiles.py new file mode 100644 index 000000000..c5cc89209 --- /dev/null +++ b/tools/refactoring/integratefiles.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +import stringmanipulation +import filemanagement +import p4commands +import sys + +extensions = ['.h', '.cpp', '.cc', '.gyp'] + +ignore_these = ['list_no_stl.h','map_no_stl.h','constructor_magic.h'] + +exceptions = [ +['GIPSRWLock.h','rw_lock.h'], +['GIPSCriticalsection.h','critical_section.h'], +] + +if((len(sys.argv) != 4) and (len(sys.argv) != 5)): + print 'parameters are: parent directory extension new extension [--commit]' + quit() + +directory = sys.argv[1]; +if(not filemanagement.pathexist(directory)): + print 'path ' + directory + ' does not exist' + quit() + +old_extension = sys.argv[2] +if(not stringmanipulation.isextension(old_extension)): + print old_extension + ' is not a valid extension' + quit() + +new_extension = sys.argv[3] +if(not stringmanipulation.isextension(new_extension)): + print new_extension + ' is not a valid extension' + quit() + +if((len(sys.argv) == 5) and (sys.argv[4] != '--commit')): + print 'parameters are: parent directory extension new extension [--commit]' + quit() + +commit = False +if(len(sys.argv) == 5): + commit = True + +files_to_integrate = filemanagement.listallfilesinfolder(directory,\ + old_extension) + +if(commit): + p4commands.checkoutallfiles() +for index in range(len(files_to_integrate)): + if(commit): + print (100*index)/len(files_to_integrate) + path_dir = files_to_integrate[index][0] + filename = files_to_integrate[index][1] + is_ignore = False + for ignore_names in ignore_these: + if(filename == ignore_names): + is_ignore = True + break + if(is_ignore): + continue + + new_file_name = '' + is_exception = False + for exception_name,exception_name_new in exceptions: + if(filename == exception_name): + is_exception = True + new_file_name = exception_name_new + break + + if(not is_exception): + new_file_name = filename + + new_file_name = stringmanipulation.removeallprefix(new_file_name,\ + 'gips') + new_file_name = stringmanipulation.removealloccurances(new_file_name,\ + 'module') + new_file_name = stringmanipulation.changeextension(new_file_name,\ + old_extension,\ + new_extension) + new_file_name = stringmanipulation.fixabbreviations( new_file_name ) + new_file_name = stringmanipulation.lowercasewithunderscore(new_file_name) + if(not commit): + print 'File ' + filename + ' will be replaced with ' + new_file_name + continue + full_new_file_name = path_dir + new_file_name + full_old_file_name = path_dir + filename + if(full_new_file_name != full_old_file_name): + p4commands.integratefile(full_old_file_name,full_new_file_name) + else: + print 'skipping ' + new_file_name + ' due to no change' + for extension in extensions: + print 'replacing ' + filename + if (extension == ".gyp"): + filemanagement.replacestringinallsubfolders( + filename,new_file_name,extension) + else: + filemanagement.replacestringinallsubfolders( + '\"' + filename + '\"', '\"' + new_file_name + '\"', extension) +if(commit): + p4commands.revertunchangedfiles() diff --git a/tools/refactoring/p4commands.py b/tools/refactoring/p4commands.py new file mode 100644 index 000000000..71ac31b0d --- /dev/null +++ b/tools/refactoring/p4commands.py @@ -0,0 +1,31 @@ +import os +import filemanagement + +# checks out entire p4 repository +def checkoutallfiles(): + os.system('p4 edit //depotGoogle/...') + return + +# reverts all unchanged files, this is completely innoculus +def revertunchangedfiles(): + os.system('p4 revert -a //depotGoogle/...') + return + +def integratefile( old_name, new_name): + if(old_name == new_name): + return + if(not filemanagement.fileexist(old_name)): + return + integrate_command = 'p4 integrate -o -f ' +\ + old_name +\ + ' ' +\ + new_name +\ + ' > p4summary.txt 2> error.txt' + os.system(integrate_command) + #print integrate_command + delete_command = 'p4 delete -c default ' +\ + old_name +\ + ' > p4summary.txt 2> error.txt' + os.system(delete_command) + #print delete_command + return diff --git a/tools/refactoring/stringmanipulation.py b/tools/refactoring/stringmanipulation.py new file mode 100644 index 000000000..0d9e0ff3a --- /dev/null +++ b/tools/refactoring/stringmanipulation.py @@ -0,0 +1,303 @@ +import string + +# returns tuple, [success,updated_string] where the updated string has +# has one less (the first) occurance of match string +def removefirstoccurance( remove_string, match_string ): + lowercase_string = remove_string.lower() + lowercase_match_string = match_string.lower() + lowest_index = lowercase_string.find(lowercase_match_string) + if(lowest_index == -1): + return [False,remove_string] + past_match_index = lowest_index + len(lowercase_match_string) + highest_index = len(remove_string) + remove_string = remove_string[0:lowest_index] + remove_string[past_match_index: highest_index] + return [True,remove_string] + +# returns a string with all occurances of match_string removed +def removealloccurances( remove_string, match_string ): + return_value = [True, remove_string] + while(return_value[0]): + return_value = removefirstoccurance(return_value[1],match_string) + return return_value[1] + +# removes an occurance of match_string only if it's first in the string +# returns tuple [succes, new_string] +def removeprefix( remove_string, match_string ): + lowercase_string = remove_string.lower() + lowercase_match_string = match_string.lower() + lowest_index = lowercase_string.find(lowercase_match_string) + if(lowest_index == -1): + return [False,remove_string] + if(lowest_index != 0): + return [False,remove_string] + past_match_index = lowest_index + len(lowercase_match_string) + highest_index = len(remove_string) + remove_string = remove_string[0:lowest_index] + remove_string[past_match_index: highest_index] +# print lowest_index +# print past_match_index + return [True,remove_string] + +# removes multiple occurances of match string as long as they are first in +# the string +def removeallprefix( remove_string, match_string ): + return_value = [True, remove_string] + while(return_value[0]): + return_value = removeprefix(return_value[1],match_string) + return return_value[1] + +# returns true if extensionstring is a correct extension +def isextension( extensionstring ): + if(len(extensionstring) < 2): + return False + if(extensionstring[0] != '.'): + return False + if(extensionstring[1:len(extensionstring)-1].find('.') != -1): + return False + return True + +# returns the index of start of the last occurance of match_string +def findlastoccurance( original_string, match_string ): + search_index = original_string.find(match_string) + found_index = search_index + last_index = len(original_string) - 1 + while((search_index != -1) and (search_index < last_index)): + search_index = original_string[search_index+1:last_index].find(match_string) + if(search_index != -1): + found_index = search_index + return found_index + +# changes extension from original_extension to new_extension +def changeextension( original_string, original_extension, new_extension): + if(not isextension(original_extension)): + return original_string + if(not isextension(new_extension)): + return original_string + index = findlastoccurance(original_string, original_extension) + if(index == -1): + return original_string + return_value = original_string[0:index] + new_extension + return return_value + +# wanted to do this with str.find however didnt seem to work so do it manually +# returns the index of the first capital letter +def findfirstcapitalletter( original_string ): + for index in range(len(original_string)): + if(original_string[index].lower() != original_string[index]): + return index + return -1 + + +# replaces capital letters with underscore and lower case letter (except very +# first +def lowercasewithunderscore( original_string ): +# ignore the first letter since there should be no underscore in front of it + if(len(original_string) < 2): + return original_string + return_value = original_string[1:len(original_string)] + index = findfirstcapitalletter(return_value) + while(index != -1): + return_value = return_value[0:index] + \ + '_' + \ + return_value[index].lower() + \ + return_value[index+1:len(return_value)] + index = findfirstcapitalletter(return_value) + return_value = original_string[0].lower() + return_value + return return_value + +# my table is a duplicate of strings +def removeduplicates( my_table ): + new_table = [] + for old_string1, new_string1 in my_table: + found = 0 + for old_string2, new_string2 in new_table: + if(old_string1 == old_string2): + found += 1 + if(new_string1 == new_string2): + if(new_string1 == ''): + found += found + else: + found += 1 + if(found == 1): + print 'missmatching set, terminating program' + print old_string1 + print new_string1 + print old_string2 + print new_string2 + quit() + if(found == 2): + break + if(found == 0): + new_table.append([old_string1,new_string1]) + return new_table + +def removenochange( my_table ): + new_table = [] + for old_string, new_string in my_table: + if(old_string != new_string): + new_table.append([old_string,new_string]) + return new_table + +# order table after size of the string (can be used to replace bigger strings +# first which is useful since smaller strings can be inside the bigger string) +# E.g. GIPS is a sub string of GIPSVE if we remove GIPS first GIPSVE will never +# be removed. N is small so no need for fancy sort algorithm. Use selection sort +def ordertablesizefirst( my_table ): + for current_index in range(len(my_table)): + biggest_string = 0 + biggest_string_index = -1 + for search_index in range(len(my_table)): + if(search_index < current_index): + continue + length_of_string = len(my_table[search_index][0]) + if(length_of_string > biggest_string): + biggest_string = length_of_string + biggest_string_index = search_index + if(biggest_string_index == -1): + print 'sorting algorithm failed, program exit' + quit() + old_value = my_table[current_index] + my_table[current_index] = my_table[biggest_string_index] + my_table[biggest_string_index] = old_value + return my_table + +# returns true if string 1 or 2 is a substring of the other, assuming neither +# has whitespaces +def issubstring( string1, string2 ): + if(len(string1) == 0): + return -1 + if(len(string2) == 0): + return -1 + large_string = string1 + small_string = string2 + if(len(string1) < len(string2)): + large_string = string2 + small_string = string1 + + for index in range(len(large_string)): + large_sub_string = large_string[index:index+len(small_string)].lower() + if(large_sub_string ==\ + small_string.lower()): + return index + return -1 + +#not_part_of_word_table = [' ','(',')','{','}',':','\t','*','&','/','[',']','.',',','\n'] +#def ispartofword( char ): +# for item in not_part_of_word_table: +# if(char == item): +# return False +# return True + +# must be numerical,_ or charachter +def ispartofword( char ): + if(char.isalpha()): + return True + if(char.isalnum()): + return True + if(char == '_'): + return True + return False + +# returns the index of the first letter in the word that the current_index +# is pointing to and the size of the word +def getword( line, current_index): + if(current_index < 0): + return [] + line = line.rstrip() + if(len(line) <= current_index): + return [] + if(line[current_index] == ' '): + return [] + start_pos = current_index + while start_pos >= 0: + if(not ispartofword(line[start_pos])): + start_pos += 1 + break + start_pos -= 1 + if(start_pos == -1): + start_pos = 0 + end_pos = current_index + while end_pos < len(line): + if(not ispartofword(line[end_pos])): + break + end_pos += 1 + return [start_pos,end_pos - start_pos] + +# my table is a tuple [string1,string2] complement_to_table is just a list +# of strings to compare to string1 +def complement( my_table, complement_to_table ): + new_table = [] + for index in range(len(my_table)): + found = False; + for compare_string in complement_to_table: + if(my_table[index][0].lower() == compare_string.lower()): + found = True + if(not found): + new_table.append(my_table[index]) + return new_table + +def removestringfromhead( line, remove_string): + for index in range(len(line)): + if(line[index:index+len(remove_string)] != remove_string): + return line[index:index+len(line)] + return '' + +def removeccomment( line ): + comment_string = '//' + for index in range(len(line)): + if(line[index:index+len(comment_string)] == comment_string): + return line[0:index] + return line + +def whitespacestoonespace( line ): + return ' '.join(line.split()) + +def fixabbreviations( original_string ): + previouswascapital = (original_string[0].upper() == original_string[0]) + new_string = '' + for index in range(len(original_string)): + if(index == 0): + new_string += original_string[index] + continue + if(original_string[index] == '_'): + new_string += original_string[index] + previouswascapital = False + continue + if(original_string[index].isdigit()): + new_string += original_string[index] + previouswascapital = False + continue + currentiscapital = (original_string[index].upper() == original_string[index]) + letter_to_add = original_string[index] + if(previouswascapital and currentiscapital): + letter_to_add = letter_to_add.lower() + if(previouswascapital and (not currentiscapital)): + old_letter = new_string[len(new_string)-1] + new_string = new_string[0:len(new_string)-1] + new_string += old_letter.upper() + previouswascapital = currentiscapital + new_string += letter_to_add + return new_string + +def replaceoccurances(old_string, replace_string, replace_with_string): + if (len(replace_string) == 0): + return old_string + if (len(old_string) < len(replace_string)): + return old_string + # Simple implementation, could proably be done smarter + new_string = '' + for index in range(len(old_string)): + #print new_string + if(len(replace_string) > (len(old_string) - index)): + new_string += old_string[index:index + len(old_string)] + break + match = (len(replace_string) > 0) + for replace_index in range(len(replace_string)): + if (replace_string[replace_index] != old_string[index + replace_index]): + match = False + break + if (match): + new_string += replace_with_string + index =+ len(replace_string) + else: + new_string += old_string[index] + return new_string diff --git a/tools/refactoring/trim.py b/tools/refactoring/trim.py new file mode 100644 index 000000000..5539f5fef --- /dev/null +++ b/tools/refactoring/trim.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +import sys +import fileinput + +# Defaults +TABSIZE = 4 + +usage = """ +Replaces all TAB characters with %(TABSIZE)d space characters. +In addition, all trailing space characters are removed. +usage: trim file ... +file ... : files are changed in place without taking any backup. +""" % vars() + +def main(): + + if len(sys.argv) == 1: + sys.stderr.write(usage) + sys.exit(2) + + # Iterate over the lines of all files listed in sys.argv[1:] + for line in fileinput.input(sys.argv[1:], inplace=True): + line = line.replace('\t',' '*TABSIZE); # replace TABs + line = line.rstrip(None) # remove trailing whitespaces + print line # modify the file + +if __name__ == '__main__': + main() diff --git a/tools/refactoring/trimall.py b/tools/refactoring/trimall.py new file mode 100644 index 000000000..7a1c458af --- /dev/null +++ b/tools/refactoring/trimall.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +import sys +import fileinput +import filemanagement +import p4commands + +# Defaults +TABSIZE = 4 + +extensions = ['.h','.cc','.c','.cpp'] + +ignore_these = ['my_ignore_header.h'] + +usage = """ +Replaces all TAB characters with %(TABSIZE)d space characters. +In addition, all trailing space characters are removed. +usage: trim directory +""" % vars() + +if((len(sys.argv) != 2) and (len(sys.argv) != 3)): + sys.stderr.write(usage) + sys.exit(2) + +directory = sys.argv[1]; +if(not filemanagement.pathexist(directory)): + sys.stderr.write(usage) + sys.exit(2) + +if((len(sys.argv) == 3) and (sys.argv[2] != '--commit')): + sys.stderr.write(usage) + sys.exit(2) + +commit = False +if(len(sys.argv) == 3): + commit = True + +files_to_fix = [] +for extension in extensions: + files_to_fix.extend(filemanagement.listallfilesinfolder(directory,\ + extension)) + +def main(): + if (commit): + p4commands.checkoutallfiles() + for path,file_name in files_to_fix: + full_file_name = path + file_name + if (not commit): + print full_file_name + ' will be edited' + continue + for line in fileinput.input(full_file_name, inplace=True): + line = line.replace('\t',' '*TABSIZE); # replace TABs + line = line.rstrip(None) # remove trailing whitespaces + print line # modify the file + if (commit): + p4commands.revertunchangedfiles() + +if __name__ == '__main__': + main()