diff --git a/libc/kernel/tools/clean_header.py b/libc/kernel/tools/clean_header.py index 0e0ed7658..e84bcf959 100755 --- a/libc/kernel/tools/clean_header.py +++ b/libc/kernel/tools/clean_header.py @@ -73,90 +73,77 @@ import sys, cpp, kernel, glob, os, re, getopt from defaults import * from utils import * -noUpdate = 1 +def print_error(no_update, msg): + if no_update: + panic(msg) + sys.stderr.write("warning: " + msg) -def cleanupFile(path, original_path): + +def cleanupFile(dst_dir, src_dir, rel_path, no_update = True): """reads an original header and perform the cleanup operation on it this functions returns the destination path and the clean header as a single string""" # check the header path - src_path = path + full_path = os.path.join(src_dir, rel_path) - if not os.path.exists(src_path): - if noUpdate: - panic( "file does not exist: '%s'\n" % path ) - sys.stderr.write( "warning: file does not exit: %s\n" % path ) + if not os.path.exists(full_path): + print_error(no_update, "file does not exist: '%s'\n" % full_path) return None, None - if not os.path.isfile(src_path): - if noUpdate: - panic( "path is not a file: '%s'\n" % path ) - sys.stderr.write( "warning: not a file: %s\n" % path ) + if not os.path.isfile(full_path): + print_error(no_update, "path is not a file: '%s'\n" % full_path) return None, None - if os.path.commonprefix( [ src_path, original_path ] ) != original_path: - if noUpdate: - panic( "file is not in 'original' directory: %s\n" % path ); - sys.stderr.write( "warning: file not in 'original' ignored: %s\n" % path ) - return None, None - - src_path = src_path[len(original_path):] - if len(src_path) > 0 and src_path[0] == '/': - src_path = src_path[1:] - - if len(src_path) == 0: - panic( "oops, internal error, can't extract correct relative path\n" ) - # convert into destination path, extracting architecture if needed # and the corresponding list of known static functions # arch = None statics = kernel_known_generic_statics - m = re.match(r"asm-([\w\d_\+\.\-]+)(/.*)", src_path) + m = re.match(r"asm-([\w\d_\+\.\-]+)(/.*)", rel_path) if m and m.group(1) != 'generic': dst_path = "arch-%s/asm/%s" % m.groups() - arch = m.group(1) - statics = statics.union( kernel_known_statics.get( arch, set() ) ) + arch = m.group(1) + statics = statics.union(kernel_known_statics.get(arch, set())) else: # process headers under the uapi directory # note the "asm" level has been explicitly added in the original # kernel header tree for architectural-dependent uapi headers - m_uapi = re.match(r"(uapi)/([\w\d_\+\.\-]+)(/.*)", src_path) + m_uapi = re.match(r"(uapi)/([\w\d_\+\.\-]+)(/.*)", rel_path) if m_uapi: - dst_path = src_path + dst_path = rel_path m_uapi_arch = re.match(r"asm-([\w\d_\+\.\-]+)", m_uapi.group(2)) if m_uapi_arch and m_uapi_arch.group(1) != 'generic': - arch = m_uapi_arch.group(1) - statics = statics.union( kernel_known_statics.get( arch, set() ) ) + arch = m_uapi_arch.group(1) + statics = statics.union(kernel_known_statics.get(arch, set())) # common headers (ie non-asm and non-uapi) else: - dst_path = "common/" + src_path + dst_path = os.path.join("common", rel_path) - dst_path = os.path.normpath( kernel_cleaned_path + "/" + dst_path ) + dst_path = os.path.join(dst_dir, dst_path) # now, let's parse the file # parser = cpp.BlockParser() - blocks = parser.parseFile(path) + blocks = parser.parseFile(full_path) if not parser.parsed: - sys.stderr.write( "error: can't parse '%s'" % path ) - sys.exit(1) + print_error(no_update, "can't parse '%s'%" % full_path) + return None, None macros = kernel_known_macros.copy() if arch and arch in kernel_default_arch_macros: macros.update(kernel_default_arch_macros[arch]) if arch and arch in kernel_arch_token_replacements: - blocks.replaceTokens( kernel_arch_token_replacements[arch] ) + blocks.replaceTokens(kernel_arch_token_replacements[arch]) - blocks.optimizeMacros( macros ) + blocks.optimizeMacros(macros) blocks.optimizeIf01() - blocks.removeVarsAndFuncs( statics ) - blocks.replaceTokens( kernel_token_replacements ) - blocks.removeMacroDefines( kernel_ignored_macros ) + blocks.removeVarsAndFuncs(statics) + blocks.replaceTokens(kernel_token_replacements) + blocks.removeMacroDefines(kernel_ignored_macros) out = StringOutput() - out.write( kernel_disclaimer ) + out.write(kernel_disclaimer) blocks.writeWithWarning(out, kernel_warning, 4) return dst_path, out.get() @@ -183,28 +170,31 @@ if __name__ == "__main__": sys.exit(1) try: - optlist, args = getopt.getopt( sys.argv[1:], 'uvk:d:' ) + optlist, args = getopt.getopt(sys.argv[1:], 'uvk:d:') except: # unrecognized option - sys.stderr.write( "error: unrecognized option\n" ) + sys.stderr.write("error: unrecognized option\n") usage() + no_update = True + dst_dir = get_kernel_dir() + src_dir = get_kernel_headers_original_dir() for opt, arg in optlist: if opt == '-u': - noUpdate = 0 + no_update = False elif opt == '-v': logging.basicConfig(level=logging.DEBUG) elif opt == '-k': - kernel_original_path = arg + src_dir = arg elif opt == '-d': - kernel_cleaned_path = arg + dst_dir = arg if len(args) == 0: usage() - if noUpdate: + if no_update: for path in args: - dst_path, newdata = cleanupFile(path,kernel_original_path) + dst_path, newdata = cleanupFile(dst_dir, src_dir, path) print newdata sys.exit(0) @@ -214,12 +204,12 @@ if __name__ == "__main__": b = BatchFileUpdater() for path in args: - dst_path, newdata = cleanupFile(path,kernel_original_path) + dst_path, newdata = cleanupFile(dst_dir, src_dir, path, no_update) if not dst_path: continue - b.readFile( dst_path ) - r = b.editFile( dst_path, newdata ) + b.readFile(dst_path) + r = b.editFile(dst_path, newdata) if r == 0: r = "unchanged" elif r == 1: @@ -227,7 +217,7 @@ if __name__ == "__main__": else: r = "added" - print "cleaning: %-*s -> %-*s (%s)" % ( 35, path, 35, dst_path, r ) + print "cleaning: %-*s -> %-*s (%s)" % (35, path, 35, dst_path, r) b.updateGitFiles() diff --git a/libc/kernel/tools/defaults.py b/libc/kernel/tools/defaults.py index 8aba9985a..1b6853e51 100644 --- a/libc/kernel/tools/defaults.py +++ b/libc/kernel/tools/defaults.py @@ -12,12 +12,6 @@ kernel_archs = [ 'arm', 'arm64', 'mips', 'x86' ] # tree. used when looking for sources... kernel_dirs = [ "linux", "asm", "asm-generic", "mtd" ] -# path to the directory containing the original kernel headers -kernel_original_path = os.path.normpath( find_program_dir() + '/../../../../external/kernel-headers/original' ) - -# path to the default location of the cleaned-up headers -kernel_cleaned_path = os.path.normpath( find_program_dir() + '/..' ) - # a special value that is used to indicate that a given macro is known to be # undefined during optimization kCppUndefinedMacro = "<<>>" diff --git a/libc/kernel/tools/generate_uapi_headers.sh b/libc/kernel/tools/generate_uapi_headers.sh index 90ba0ed3d..3c80d9f53 100755 --- a/libc/kernel/tools/generate_uapi_headers.sh +++ b/libc/kernel/tools/generate_uapi_headers.sh @@ -99,6 +99,35 @@ function copy_if_exists () { done } +function check_hdrs () { + local src_dir=$1 + local tgt_dir=$2 + local kernel_dir=$3 + + local search_dirs=() + + # This only works if none of the filenames have spaces. + for file in $(ls -d ${src_dir}/* 2> /dev/null); do + if [[ -d "${file}" ]]; then + search_dirs+=("${file}") + elif [[ -f "${file}" ]] && [[ "${file}" =~ .h$ ]]; then + tgt_file=${tgt_dir}/$(basename ${file}) + if [[ -e ${tgt_file} ]] && ! diff "${file}" "${tgt_file}" > /dev/null; then + if [[ ${file} =~ ${kernel_dir}/*(.+) ]]; then + echo "New version of ${BASH_REMATCH[1]} found in kernel headers." + else + echo "New version of ${file} found in kernel headers." + fi + echo "This file needs to be updated manually." + fi + fi + done + + for dir in "${search_dirs[@]}"; do + check_hdrs "${dir}" ${tgt_dir}/$(basename ${dir}) "${kernel_dir}" + done +} + trap cleanup EXIT # This automatically triggers a call to cleanup. trap "exit 1" HUP INT TERM TSTP @@ -207,3 +236,8 @@ for arch in "${ARCH_LIST[@]}"; do "${KERNEL_DIR}/${src_dir}/arch/${arch}/include/generated/asm" \ "${ANDROID_KERNEL_DIR}/uapi/asm-${arch}/asm" done + +# Verify if modified headers have changed. +check_hdrs "${KERNEL_DIR}/${src_dir}/include/scsi" \ + "${ANDROID_KERNEL_DIR}/scsi" \ + "${KERNEL_DIR}/${src_dir}" diff --git a/libc/kernel/tools/update_all.py b/libc/kernel/tools/update_all.py index f45d4e057..7f3657c20 100755 --- a/libc/kernel/tools/update_all.py +++ b/libc/kernel/tools/update_all.py @@ -6,72 +6,93 @@ from utils import * def usage(): print """\ - usage: %(progname)s [kernel-original-path] + usage: %(progname)s [kernel-original-path] [kernel-modified-path] this program is used to update all the auto-generated clean headers used by the Bionic C library. it assumes the following: - - a set of source kernel headers is located in '../original', - relative to the program's directory + - a set of source kernel headers is located in + 'external/kernel-headers/original', relative to the current + android tree - - the clean headers will be placed in '../arch-/asm', - '../common/linux', '../common/asm-generic', etc.. + - a set of manually modified kernel header files located in + 'external/kernel-headers/modified', relative to the current + android tree + + - the clean headers will be placed in 'bionic/libc/kernel/arch-/asm', + 'bionic/libc/kernel/common', etc.. """ % { "progname" : os.path.basename(sys.argv[0]) } sys.exit(0) try: - optlist, args = getopt.getopt( sys.argv[1:], '' ) + optlist, args = getopt.getopt(sys.argv[1:], '') except: # unrecognized option - sys.stderr.write( "error: unrecognized option\n" ) + sys.stderr.write("error: unrecognized option\n") usage() -if len(optlist) > 0 or len(args) > 1: +if len(optlist) > 0 or len(args) > 2: usage() -progdir = find_program_dir() - -if len(args) == 1: +modified_dir = get_kernel_headers_modified_dir() +if len(args) == 1 or len(args) == 2: original_dir = args[0] if not os.path.isdir(original_dir): - panic( "Not a directory: %s\n" % original_dir ) + panic("Not a directory: %s\n" % original_dir) + + if len(args) == 2: + modified_dir = args[1] + if not os.path.isdir(modified_dir): + panic("Not a directory: %s\n" % modified_dir) else: - original_dir = kernel_original_path + original_dir = get_kernel_headers_original_dir() if not os.path.isdir(original_dir): - panic( "Missing directory, please specify one through command-line: %s\n" % original_dir ) + panic("Missing directory, please specify one through command-line: %s\n" % original_dir) -skip_ion = False +if not os.path.isdir(modified_dir): + modified_dir = None -# find all source files in 'original' -# -sources = [] -warning_ion = [] -for root, dirs, files in os.walk( original_dir ): +# Find all source files in 'original'. +sources = dict() +original_dir = os.path.normpath(original_dir) +original_dir_len = len(original_dir) + 1 +for root, _, files in os.walk(original_dir): for file in files: - if skip_ion and (file == "ion.h" or file == "ion_test.h"): - warning_ion.append(" Skipped file %s/%s" % (root, file)) - continue - base, ext = os.path.splitext(file) + _, ext = os.path.splitext(file) if ext == ".h": - sources.append( "%s/%s" % (root,file) ) + rel_path = os.path.normpath(os.path.join(root, file)) + rel_path = rel_path[original_dir_len:] + # Check to see if there is a modified header to use instead. + if modified_dir and os.path.exists(os.path.join(modified_dir, rel_path)): + sources[rel_path] = False + else: + sources[rel_path] = True + b = BatchFileUpdater() +kernel_dir = get_kernel_dir() for arch in kernel_archs: - b.readDir( os.path.normpath( progdir + "/../arch-%s" % arch ) ) + b.readDir(os.path.join(kernel_dir, "arch-%s" % arch)) -b.readDir( os.path.normpath( progdir + "/../common" ) ) - -#print "OLD " + repr(b.old_files) +b.readDir(os.path.join(kernel_dir, "common")) oldlen = 120 -for path in sources: - dst_path, newdata = clean_header.cleanupFile(path, original_dir) +android_root_len = len(get_android_root()) + 1 +for rel_path in sorted(sources): + if sources[rel_path]: + src_dir = original_dir + src_str = "/" + else: + src_dir = modified_dir + src_str = "/" + dst_path, newdata = clean_header.cleanupFile(kernel_dir, src_dir, rel_path) if not dst_path: continue - b.readFile( dst_path ) - r = b.editFile( dst_path, newdata ) + dst_path = os.path.join(kernel_dir, dst_path) + b.readFile(dst_path) + r = b.editFile(dst_path, newdata) if r == 0: state = "unchanged" elif r == 1: @@ -79,9 +100,11 @@ for path in sources: else: state = "added" - str = "cleaning: %-*s -> %-*s (%s)" % ( 35, "" + path[len(original_dir):], 35, dst_path, state ) + # dst_path is guaranteed to include android root. + rel_dst_path = dst_path[android_root_len:] + str = "cleaning: %-*s -> %-*s (%s)" % (35, src_str + rel_path, 35, rel_dst_path, state) if sys.stdout.isatty(): - print "%-*s" % (oldlen,str), + print "%-*s" % (oldlen, str), if (r == 0): print "\r", else: @@ -92,11 +115,8 @@ for path in sources: oldlen = len(str) -print "%-*s" % (oldlen,"Done!") +print "%-*s" % (oldlen, "Done!") b.updateGitFiles() -if warning_ion: - print "NOTE: Due to import into aosp, some files were not processed." - print "\n".join(warning_ion) sys.exit(0) diff --git a/libc/kernel/tools/utils.py b/libc/kernel/tools/utils.py index e5a310e03..e2cc9ce10 100644 --- a/libc/kernel/tools/utils.py +++ b/libc/kernel/tools/utils.py @@ -13,8 +13,26 @@ def panic(msg): sys.exit(1) -def find_program_dir(): - return os.path.dirname(sys.argv[0]) +def get_kernel_headers_dir(): + return os.path.join(get_android_root(), "external/kernel-headers") + + +def get_kernel_headers_original_dir(): + return os.path.join(get_kernel_headers_dir(), "original") + + +def get_kernel_headers_modified_dir(): + return os.path.join(get_kernel_headers_dir(), "modified") + + +def get_kernel_dir(): + return os.path.join(get_android_root(), "bionic/libc/kernel") + + +def get_android_root(): + if "ANDROID_BUILD_TOP" in os.environ: + return os.environ["ANDROID_BUILD_TOP"] + panic("Unable to find root of tree, did you forget to lunch a target?") class StringOutput: