From d12c332018143e731337292910b03fa0f41b2ca2 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 15 Sep 2015 14:13:17 -0700 Subject: [PATCH] Add support for manually modified kernel headers. This changes the scripts so that if some kernel files exists in external/kernel-headers/modified, that they will be preferred over the same files found in original. This is to support the case where the kernel headers cannot be taken without some small modifications. Included with this change, is a general cleanup of the python scripts. This also modifies the generate uapi headers script to indicate if the source of the modified headers has changed. Change-Id: Id13523b244ced52a2ecd9f1399c43996dd8296fa --- libc/kernel/tools/clean_header.py | 96 ++++++++++----------- libc/kernel/tools/defaults.py | 6 -- libc/kernel/tools/generate_uapi_headers.sh | 34 ++++++++ libc/kernel/tools/update_all.py | 98 +++++++++++++--------- libc/kernel/tools/utils.py | 22 ++++- 5 files changed, 156 insertions(+), 100 deletions(-) diff --git a/libc/kernel/tools/clean_header.py b/libc/kernel/tools/clean_header.py index 0e0ed7658..e84bcf959 100755 --- a/libc/kernel/tools/clean_header.py +++ b/libc/kernel/tools/clean_header.py @@ -73,90 +73,77 @@ import sys, cpp, kernel, glob, os, re, getopt from defaults import * from utils import * -noUpdate = 1 +def print_error(no_update, msg): + if no_update: + panic(msg) + sys.stderr.write("warning: " + msg) -def cleanupFile(path, original_path): + +def cleanupFile(dst_dir, src_dir, rel_path, no_update = True): """reads an original header and perform the cleanup operation on it this functions returns the destination path and the clean header as a single string""" # check the header path - src_path = path + full_path = os.path.join(src_dir, rel_path) - if not os.path.exists(src_path): - if noUpdate: - panic( "file does not exist: '%s'\n" % path ) - sys.stderr.write( "warning: file does not exit: %s\n" % path ) + if not os.path.exists(full_path): + print_error(no_update, "file does not exist: '%s'\n" % full_path) return None, None - if not os.path.isfile(src_path): - if noUpdate: - panic( "path is not a file: '%s'\n" % path ) - sys.stderr.write( "warning: not a file: %s\n" % path ) + if not os.path.isfile(full_path): + print_error(no_update, "path is not a file: '%s'\n" % full_path) return None, None - if os.path.commonprefix( [ src_path, original_path ] ) != original_path: - if noUpdate: - panic( "file is not in 'original' directory: %s\n" % path ); - sys.stderr.write( "warning: file not in 'original' ignored: %s\n" % path ) - return None, None - - src_path = src_path[len(original_path):] - if len(src_path) > 0 and src_path[0] == '/': - src_path = src_path[1:] - - if len(src_path) == 0: - panic( "oops, internal error, can't extract correct relative path\n" ) - # convert into destination path, extracting architecture if needed # and the corresponding list of known static functions # arch = None statics = kernel_known_generic_statics - m = re.match(r"asm-([\w\d_\+\.\-]+)(/.*)", src_path) + m = re.match(r"asm-([\w\d_\+\.\-]+)(/.*)", rel_path) if m and m.group(1) != 'generic': dst_path = "arch-%s/asm/%s" % m.groups() - arch = m.group(1) - statics = statics.union( kernel_known_statics.get( arch, set() ) ) + arch = m.group(1) + statics = statics.union(kernel_known_statics.get(arch, set())) else: # process headers under the uapi directory # note the "asm" level has been explicitly added in the original # kernel header tree for architectural-dependent uapi headers - m_uapi = re.match(r"(uapi)/([\w\d_\+\.\-]+)(/.*)", src_path) + m_uapi = re.match(r"(uapi)/([\w\d_\+\.\-]+)(/.*)", rel_path) if m_uapi: - dst_path = src_path + dst_path = rel_path m_uapi_arch = re.match(r"asm-([\w\d_\+\.\-]+)", m_uapi.group(2)) if m_uapi_arch and m_uapi_arch.group(1) != 'generic': - arch = m_uapi_arch.group(1) - statics = statics.union( kernel_known_statics.get( arch, set() ) ) + arch = m_uapi_arch.group(1) + statics = statics.union(kernel_known_statics.get(arch, set())) # common headers (ie non-asm and non-uapi) else: - dst_path = "common/" + src_path + dst_path = os.path.join("common", rel_path) - dst_path = os.path.normpath( kernel_cleaned_path + "/" + dst_path ) + dst_path = os.path.join(dst_dir, dst_path) # now, let's parse the file # parser = cpp.BlockParser() - blocks = parser.parseFile(path) + blocks = parser.parseFile(full_path) if not parser.parsed: - sys.stderr.write( "error: can't parse '%s'" % path ) - sys.exit(1) + print_error(no_update, "can't parse '%s'%" % full_path) + return None, None macros = kernel_known_macros.copy() if arch and arch in kernel_default_arch_macros: macros.update(kernel_default_arch_macros[arch]) if arch and arch in kernel_arch_token_replacements: - blocks.replaceTokens( kernel_arch_token_replacements[arch] ) + blocks.replaceTokens(kernel_arch_token_replacements[arch]) - blocks.optimizeMacros( macros ) + blocks.optimizeMacros(macros) blocks.optimizeIf01() - blocks.removeVarsAndFuncs( statics ) - blocks.replaceTokens( kernel_token_replacements ) - blocks.removeMacroDefines( kernel_ignored_macros ) + blocks.removeVarsAndFuncs(statics) + blocks.replaceTokens(kernel_token_replacements) + blocks.removeMacroDefines(kernel_ignored_macros) out = StringOutput() - out.write( kernel_disclaimer ) + out.write(kernel_disclaimer) blocks.writeWithWarning(out, kernel_warning, 4) return dst_path, out.get() @@ -183,28 +170,31 @@ if __name__ == "__main__": sys.exit(1) try: - optlist, args = getopt.getopt( sys.argv[1:], 'uvk:d:' ) + optlist, args = getopt.getopt(sys.argv[1:], 'uvk:d:') except: # unrecognized option - sys.stderr.write( "error: unrecognized option\n" ) + sys.stderr.write("error: unrecognized option\n") usage() + no_update = True + dst_dir = get_kernel_dir() + src_dir = get_kernel_headers_original_dir() for opt, arg in optlist: if opt == '-u': - noUpdate = 0 + no_update = False elif opt == '-v': logging.basicConfig(level=logging.DEBUG) elif opt == '-k': - kernel_original_path = arg + src_dir = arg elif opt == '-d': - kernel_cleaned_path = arg + dst_dir = arg if len(args) == 0: usage() - if noUpdate: + if no_update: for path in args: - dst_path, newdata = cleanupFile(path,kernel_original_path) + dst_path, newdata = cleanupFile(dst_dir, src_dir, path) print newdata sys.exit(0) @@ -214,12 +204,12 @@ if __name__ == "__main__": b = BatchFileUpdater() for path in args: - dst_path, newdata = cleanupFile(path,kernel_original_path) + dst_path, newdata = cleanupFile(dst_dir, src_dir, path, no_update) if not dst_path: continue - b.readFile( dst_path ) - r = b.editFile( dst_path, newdata ) + b.readFile(dst_path) + r = b.editFile(dst_path, newdata) if r == 0: r = "unchanged" elif r == 1: @@ -227,7 +217,7 @@ if __name__ == "__main__": else: r = "added" - print "cleaning: %-*s -> %-*s (%s)" % ( 35, path, 35, dst_path, r ) + print "cleaning: %-*s -> %-*s (%s)" % (35, path, 35, dst_path, r) b.updateGitFiles() diff --git a/libc/kernel/tools/defaults.py b/libc/kernel/tools/defaults.py index 8aba9985a..1b6853e51 100644 --- a/libc/kernel/tools/defaults.py +++ b/libc/kernel/tools/defaults.py @@ -12,12 +12,6 @@ kernel_archs = [ 'arm', 'arm64', 'mips', 'x86' ] # tree. used when looking for sources... kernel_dirs = [ "linux", "asm", "asm-generic", "mtd" ] -# path to the directory containing the original kernel headers -kernel_original_path = os.path.normpath( find_program_dir() + '/../../../../external/kernel-headers/original' ) - -# path to the default location of the cleaned-up headers -kernel_cleaned_path = os.path.normpath( find_program_dir() + '/..' ) - # a special value that is used to indicate that a given macro is known to be # undefined during optimization kCppUndefinedMacro = "<<>>" diff --git a/libc/kernel/tools/generate_uapi_headers.sh b/libc/kernel/tools/generate_uapi_headers.sh index 90ba0ed3d..3c80d9f53 100755 --- a/libc/kernel/tools/generate_uapi_headers.sh +++ b/libc/kernel/tools/generate_uapi_headers.sh @@ -99,6 +99,35 @@ function copy_if_exists () { done } +function check_hdrs () { + local src_dir=$1 + local tgt_dir=$2 + local kernel_dir=$3 + + local search_dirs=() + + # This only works if none of the filenames have spaces. + for file in $(ls -d ${src_dir}/* 2> /dev/null); do + if [[ -d "${file}" ]]; then + search_dirs+=("${file}") + elif [[ -f "${file}" ]] && [[ "${file}" =~ .h$ ]]; then + tgt_file=${tgt_dir}/$(basename ${file}) + if [[ -e ${tgt_file} ]] && ! diff "${file}" "${tgt_file}" > /dev/null; then + if [[ ${file} =~ ${kernel_dir}/*(.+) ]]; then + echo "New version of ${BASH_REMATCH[1]} found in kernel headers." + else + echo "New version of ${file} found in kernel headers." + fi + echo "This file needs to be updated manually." + fi + fi + done + + for dir in "${search_dirs[@]}"; do + check_hdrs "${dir}" ${tgt_dir}/$(basename ${dir}) "${kernel_dir}" + done +} + trap cleanup EXIT # This automatically triggers a call to cleanup. trap "exit 1" HUP INT TERM TSTP @@ -207,3 +236,8 @@ for arch in "${ARCH_LIST[@]}"; do "${KERNEL_DIR}/${src_dir}/arch/${arch}/include/generated/asm" \ "${ANDROID_KERNEL_DIR}/uapi/asm-${arch}/asm" done + +# Verify if modified headers have changed. +check_hdrs "${KERNEL_DIR}/${src_dir}/include/scsi" \ + "${ANDROID_KERNEL_DIR}/scsi" \ + "${KERNEL_DIR}/${src_dir}" diff --git a/libc/kernel/tools/update_all.py b/libc/kernel/tools/update_all.py index f45d4e057..7f3657c20 100755 --- a/libc/kernel/tools/update_all.py +++ b/libc/kernel/tools/update_all.py @@ -6,72 +6,93 @@ from utils import * def usage(): print """\ - usage: %(progname)s [kernel-original-path] + usage: %(progname)s [kernel-original-path] [kernel-modified-path] this program is used to update all the auto-generated clean headers used by the Bionic C library. it assumes the following: - - a set of source kernel headers is located in '../original', - relative to the program's directory + - a set of source kernel headers is located in + 'external/kernel-headers/original', relative to the current + android tree - - the clean headers will be placed in '../arch-/asm', - '../common/linux', '../common/asm-generic', etc.. + - a set of manually modified kernel header files located in + 'external/kernel-headers/modified', relative to the current + android tree + + - the clean headers will be placed in 'bionic/libc/kernel/arch-/asm', + 'bionic/libc/kernel/common', etc.. """ % { "progname" : os.path.basename(sys.argv[0]) } sys.exit(0) try: - optlist, args = getopt.getopt( sys.argv[1:], '' ) + optlist, args = getopt.getopt(sys.argv[1:], '') except: # unrecognized option - sys.stderr.write( "error: unrecognized option\n" ) + sys.stderr.write("error: unrecognized option\n") usage() -if len(optlist) > 0 or len(args) > 1: +if len(optlist) > 0 or len(args) > 2: usage() -progdir = find_program_dir() - -if len(args) == 1: +modified_dir = get_kernel_headers_modified_dir() +if len(args) == 1 or len(args) == 2: original_dir = args[0] if not os.path.isdir(original_dir): - panic( "Not a directory: %s\n" % original_dir ) + panic("Not a directory: %s\n" % original_dir) + + if len(args) == 2: + modified_dir = args[1] + if not os.path.isdir(modified_dir): + panic("Not a directory: %s\n" % modified_dir) else: - original_dir = kernel_original_path + original_dir = get_kernel_headers_original_dir() if not os.path.isdir(original_dir): - panic( "Missing directory, please specify one through command-line: %s\n" % original_dir ) + panic("Missing directory, please specify one through command-line: %s\n" % original_dir) -skip_ion = False +if not os.path.isdir(modified_dir): + modified_dir = None -# find all source files in 'original' -# -sources = [] -warning_ion = [] -for root, dirs, files in os.walk( original_dir ): +# Find all source files in 'original'. +sources = dict() +original_dir = os.path.normpath(original_dir) +original_dir_len = len(original_dir) + 1 +for root, _, files in os.walk(original_dir): for file in files: - if skip_ion and (file == "ion.h" or file == "ion_test.h"): - warning_ion.append(" Skipped file %s/%s" % (root, file)) - continue - base, ext = os.path.splitext(file) + _, ext = os.path.splitext(file) if ext == ".h": - sources.append( "%s/%s" % (root,file) ) + rel_path = os.path.normpath(os.path.join(root, file)) + rel_path = rel_path[original_dir_len:] + # Check to see if there is a modified header to use instead. + if modified_dir and os.path.exists(os.path.join(modified_dir, rel_path)): + sources[rel_path] = False + else: + sources[rel_path] = True + b = BatchFileUpdater() +kernel_dir = get_kernel_dir() for arch in kernel_archs: - b.readDir( os.path.normpath( progdir + "/../arch-%s" % arch ) ) + b.readDir(os.path.join(kernel_dir, "arch-%s" % arch)) -b.readDir( os.path.normpath( progdir + "/../common" ) ) - -#print "OLD " + repr(b.old_files) +b.readDir(os.path.join(kernel_dir, "common")) oldlen = 120 -for path in sources: - dst_path, newdata = clean_header.cleanupFile(path, original_dir) +android_root_len = len(get_android_root()) + 1 +for rel_path in sorted(sources): + if sources[rel_path]: + src_dir = original_dir + src_str = "/" + else: + src_dir = modified_dir + src_str = "/" + dst_path, newdata = clean_header.cleanupFile(kernel_dir, src_dir, rel_path) if not dst_path: continue - b.readFile( dst_path ) - r = b.editFile( dst_path, newdata ) + dst_path = os.path.join(kernel_dir, dst_path) + b.readFile(dst_path) + r = b.editFile(dst_path, newdata) if r == 0: state = "unchanged" elif r == 1: @@ -79,9 +100,11 @@ for path in sources: else: state = "added" - str = "cleaning: %-*s -> %-*s (%s)" % ( 35, "" + path[len(original_dir):], 35, dst_path, state ) + # dst_path is guaranteed to include android root. + rel_dst_path = dst_path[android_root_len:] + str = "cleaning: %-*s -> %-*s (%s)" % (35, src_str + rel_path, 35, rel_dst_path, state) if sys.stdout.isatty(): - print "%-*s" % (oldlen,str), + print "%-*s" % (oldlen, str), if (r == 0): print "\r", else: @@ -92,11 +115,8 @@ for path in sources: oldlen = len(str) -print "%-*s" % (oldlen,"Done!") +print "%-*s" % (oldlen, "Done!") b.updateGitFiles() -if warning_ion: - print "NOTE: Due to import into aosp, some files were not processed." - print "\n".join(warning_ion) sys.exit(0) diff --git a/libc/kernel/tools/utils.py b/libc/kernel/tools/utils.py index e5a310e03..e2cc9ce10 100644 --- a/libc/kernel/tools/utils.py +++ b/libc/kernel/tools/utils.py @@ -13,8 +13,26 @@ def panic(msg): sys.exit(1) -def find_program_dir(): - return os.path.dirname(sys.argv[0]) +def get_kernel_headers_dir(): + return os.path.join(get_android_root(), "external/kernel-headers") + + +def get_kernel_headers_original_dir(): + return os.path.join(get_kernel_headers_dir(), "original") + + +def get_kernel_headers_modified_dir(): + return os.path.join(get_kernel_headers_dir(), "modified") + + +def get_kernel_dir(): + return os.path.join(get_android_root(), "bionic/libc/kernel") + + +def get_android_root(): + if "ANDROID_BUILD_TOP" in os.environ: + return os.environ["ANDROID_BUILD_TOP"] + panic("Unable to find root of tree, did you forget to lunch a target?") class StringOutput: