generic-library/bionic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Clean up some of our python scripts.

Browse Source 
Change-Id: Ifa75345db43434298cfb6113fbe2f7a33b88c79d
This commit is contained in:
Elliott Hughes 2014-08-20 11:16:11 -07:00
parent 55244a9bdb
commit dc1fb7000a
5 changed files with 151 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
libc/kernel/tools/clean_header.py
View File

@ -199,8 +199,7 @@ if __name__ == "__main__":
if opt == '-u': if opt == '-u':
noUpdate = 0 noUpdate = 0
elif opt == '-v': elif opt == '-v':
verbose = 1 logging.basicConfig(level=logging.DEBUG)
D_setlevel(1)
elif opt == '-k': elif opt == '-k':
kernel_original_path = arg kernel_original_path = arg
elif opt == '-d': elif opt == '-d':

17
libc/kernel/tools/cpp.py
View File

@ -1711,7 +1711,7 @@ def optimize_if01( blocks ):
while j < n and not blocks[j].isIf(): while j < n and not blocks[j].isIf():
j += 1 j += 1
if j > i: if j > i:
D2("appending lines %d to %d" % (blocks[i].lineno, blocks[j-1].lineno)) logging.debug("appending lines %d to %d" % (blocks[i].lineno, blocks[j-1].lineno))
result += blocks[i:j] result += blocks[i:j]
if j >= n: if j >= n:
break break
@ -1730,17 +1730,17 @@ def optimize_if01( blocks ):
break break
dir = blocks[j].directive dir = blocks[j].directive
if dir == "endif": if dir == "endif":
D2("remove 'if 0' .. 'endif' (lines %d to %d)" % (blocks[i].lineno, blocks[j].lineno)) logging.debug("remove 'if 0' .. 'endif' (lines %d to %d)" % (blocks[i].lineno, blocks[j].lineno))
i = j + 1 i = j + 1
elif dir == "else": elif dir == "else":
# convert 'else' into 'if 1' # convert 'else' into 'if 1'
D2("convert 'if 0' .. 'else' into 'if 1' (lines %d to %d)" % (blocks[i].lineno, blocks[j-1].lineno)) logging.debug("convert 'if 0' .. 'else' into 'if 1' (lines %d to %d)" % (blocks[i].lineno, blocks[j-1].lineno))
blocks[j].directive = "if" blocks[j].directive = "if"
blocks[j].expr = CppExpr( CppLineTokenizer("1").toTokenList() ) blocks[j].expr = CppExpr( CppLineTokenizer("1").toTokenList() )
i = j i = j
elif dir == "elif": elif dir == "elif":
# convert 'elif' into 'if' # convert 'elif' into 'if'
D2("convert 'if 0' .. 'elif' into 'if'") logging.debug("convert 'if 0' .. 'elif' into 'if'")
blocks[j].directive = "if" blocks[j].directive = "if"
i = j i = j
continue continue
@ -1749,25 +1749,25 @@ def optimize_if01( blocks ):
k = find_matching_endif( blocks, j+1 ) k = find_matching_endif( blocks, j+1 )
if k >= n: if k >= n:
# unterminated #if 1, finish here # unterminated #if 1, finish here
D2("unterminated 'if 1'") logging.debug("unterminated 'if 1'")
result += blocks[j+1:k] result += blocks[j+1:k]
break break
dir = blocks[k].directive dir = blocks[k].directive
if dir == "endif": if dir == "endif":
D2("convert 'if 1' .. 'endif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) logging.debug("convert 'if 1' .. 'endif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
result += optimize_if01(blocks[j+1:k]) result += optimize_if01(blocks[j+1:k])
i = k+1 i = k+1
elif dir == "else": elif dir == "else":
# convert 'else' into 'if 0' # convert 'else' into 'if 0'
D2("convert 'if 1' .. 'else' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) logging.debug("convert 'if 1' .. 'else' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
result += optimize_if01(blocks[j+1:k]) result += optimize_if01(blocks[j+1:k])
blocks[k].directive = "if" blocks[k].directive = "if"
blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() ) blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() )
i = k i = k
elif dir == "elif": elif dir == "elif":
# convert 'elif' into 'if 0' # convert 'elif' into 'if 0'
D2("convert 'if 1' .. 'elif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) logging.debug("convert 'if 1' .. 'elif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
result += optimize_if01(blocks[j+1:k]) result += optimize_if01(blocks[j+1:k])
blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() ) blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() )
i = k i = k
@ -1835,7 +1835,6 @@ def test_optimizeAll():
out = StringOutput() out = StringOutput()
lines = string.split(text, '\n') lines = string.split(text, '\n')
list = BlockParser().parse( CppLinesTokenizer(lines) ) list = BlockParser().parse( CppLinesTokenizer(lines) )
#D_setlevel(2)
list.replaceTokens( kernel_token_replacements ) list.replaceTokens( kernel_token_replacements )
list.optimizeAll( {"__KERNEL__":kCppUndefinedMacro} ) list.optimizeAll( {"__KERNEL__":kCppUndefinedMacro} )
list.write(out) list.write(out)

89
libc/kernel/tools/utils.py
View File

@ -1,59 +1,29 @@
# common python utility routines for the Bionic tool scripts # common python utility routines for the Bionic tool scripts
import sys, os, commands, string, commands import commands
import logging
import os
import string
import sys
# basic debugging trace support
# call D_setlevel to set the verbosity level
# and D(), D2(), D3(), D4() to add traces
#
verbose = 0
def panic(msg): def panic(msg):
sys.stderr.write( find_program_name() + ": error: " ) sys.stderr.write(os.path.basename(sys.argv[0]) + ": error: ")
sys.stderr.write( msg ) sys.stderr.write(msg)
sys.exit(1) sys.exit(1)
def D(msg):
global verbose
if verbose > 0:
print msg
def D2(msg):
global verbose
if verbose >= 2:
print msg
def D3(msg):
global verbose
if verbose >= 3:
print msg
def D4(msg):
global verbose
if verbose >= 4:
print msg
def D_setlevel(level):
global verbose
verbose = level
# other stuff
#
#
def find_program_name():
return os.path.basename(sys.argv[0])
def find_program_dir(): def find_program_dir():
return os.path.dirname(sys.argv[0]) return os.path.dirname(sys.argv[0])
class StringOutput: class StringOutput:
def __init__(self): def __init__(self):
self.line = "" self.line = ""
def write(self,msg): def write(self,msg):
self.line += msg self.line += msg
D2("write '%s'" % msg) logging.debug("write '%s'" % msg)
def get(self): def get(self):
return self.line return self.line
@ -76,47 +46,6 @@ def create_file_path(path):
continue continue
os.mkdir(dir) os.mkdir(dir)
def walk_source_files(paths,callback,args,excludes=[]):
"""recursively walk a list of paths and files, only keeping the source files in directories"""
for path in paths:
if len(path) > 0 and path[0] == '@':
# this is the name of another file, include it and parse it
path = path[1:]
if os.path.exists(path):
for line in open(path):
if len(line) > 0 and line[-1] == '\n':
line = line[:-1]
walk_source_files([line],callback,args,excludes)
continue
if not os.path.isdir(path):
callback(path,args)
else:
for root, dirs, files in os.walk(path):
#print "w-- %s (ex: %s)" % (repr((root,dirs)), repr(excludes))
if len(excludes):
for d in dirs[:]:
if os.path.join(root,d) in excludes:
dirs.remove(d)
for f in files:
r, ext = os.path.splitext(f)
if ext in [ ".h", ".c", ".cpp", ".S" ]:
callback( "%s/%s" % (root,f), args )
def cleanup_dir(path):
"""create a directory if needed, and ensure that it is totally empty
by removing any existing content in it"""
if not os.path.exists(path):
os.mkdir(path)
else:
for root, dirs, files in os.walk(path, topdown=False):
if root.endswith("kernel_headers/"):
# skip 'kernel_headers'
continue
for name in files:
os.remove(os.path.join(root, name))
for name in dirs:
os.rmdir(os.path.join(root, name))
class BatchFileUpdater: class BatchFileUpdater:
"""a class used to edit several files at once""" """a class used to edit several files at once"""

165
libc/tools/bionic_utils.py
View File

@ -1,165 +0,0 @@
# common python utility routines for the Bionic tool scripts
import sys, os, commands, string
all_arches = [ "arm", "arm64", "mips", "mips64", "x86", "x86_64" ]
# basic debugging trace support
# call D_setlevel to set the verbosity level
# and D(), D2(), D3(), D4() to add traces
#
verbose = 0
def D(msg):
global verbose
if verbose > 0:
print msg
def D2(msg):
global verbose
if verbose >= 2:
print msg
def D3(msg):
global verbose
if verbose >= 3:
print msg
def D4(msg):
global verbose
if verbose >= 4:
print msg
def D_setlevel(level):
global verbose
verbose = level
# parser for the SYSCALLS.TXT file
#
class SysCallsTxtParser:
def __init__(self):
self.syscalls = []
self.lineno = 0
def E(self, msg):
print "%d: %s" % (self.lineno, msg)
def parse_line(self, line):
""" parse a syscall spec line.
line processing, format is
return type func_name[|alias_list][:syscall_name[:socketcall_id]] ( [paramlist] ) architecture_list
"""
pos_lparen = line.find('(')
E = self.E
if pos_lparen < 0:
E("missing left parenthesis in '%s'" % line)
return
pos_rparen = line.rfind(')')
if pos_rparen < 0 or pos_rparen <= pos_lparen:
E("missing or misplaced right parenthesis in '%s'" % line)
return
return_type = line[:pos_lparen].strip().split()
if len(return_type) < 2:
E("missing return type in '%s'" % line)
return
syscall_func = return_type[-1]
return_type = string.join(return_type[:-1],' ')
socketcall_id = -1
pos_colon = syscall_func.find(':')
if pos_colon < 0:
syscall_name = syscall_func
else:
if pos_colon == 0 or pos_colon+1 >= len(syscall_func):
E("misplaced colon in '%s'" % line)
return
# now find if there is a socketcall_id for a dispatch-type syscall
# after the optional 2nd colon
pos_colon2 = syscall_func.find(':', pos_colon + 1)
if pos_colon2 < 0:
syscall_name = syscall_func[pos_colon+1:]
syscall_func = syscall_func[:pos_colon]
else:
if pos_colon2+1 >= len(syscall_func):
E("misplaced colon2 in '%s'" % line)
return
syscall_name = syscall_func[(pos_colon+1):pos_colon2]
socketcall_id = int(syscall_func[pos_colon2+1:])
syscall_func = syscall_func[:pos_colon]
alias_delim = syscall_func.find('|')
if alias_delim > 0:
alias_list = syscall_func[alias_delim+1:].strip()
syscall_func = syscall_func[:alias_delim]
alias_delim = syscall_name.find('|')
if alias_delim > 0:
syscall_name = syscall_name[:alias_delim]
syscall_aliases = string.split(alias_list, ',')
else:
syscall_aliases = []
if pos_rparen > pos_lparen+1:
syscall_params = line[pos_lparen+1:pos_rparen].split(',')
params = string.join(syscall_params,',')
else:
syscall_params = []
params = "void"
t = {
"name" : syscall_name,
"func" : syscall_func,
"aliases" : syscall_aliases,
"params" : syscall_params,
"decl" : "%-15s %s (%s);" % (return_type, syscall_func, params),
"socketcall_id" : socketcall_id
}
# Parse the architecture list.
arch_list = line[pos_rparen+1:].strip()
if arch_list == "all":
for arch in all_arches:
t[arch] = True
else:
for arch in string.split(arch_list, ','):
if arch in all_arches:
t[arch] = True
else:
E("invalid syscall architecture '%s' in '%s'" % (arch, line))
return
self.syscalls.append(t)
global verbose
if verbose >= 2:
print t
def parse_file(self, file_path):
D2("parse_file: %s" % file_path)
fp = open(file_path)
for line in fp.xreadlines():
self.lineno += 1
line = line.strip()
if not line: continue
if line[0] == '#': continue
self.parse_line(line)
fp.close()
class StringOutput:
def __init__(self):
self.line = ""
def write(self,msg):
self.line += msg
D2("write '%s'" % msg)
def get(self):
return self.line

148
libc/tools/gensyscalls.py
View File

@ -8,14 +8,18 @@ import atexit
import commands import commands
import filecmp import filecmp
import glob import glob
import logging
import os.path import os.path
import re import re
import shutil import shutil
import stat import stat
import string
import sys import sys
import tempfile import tempfile
from bionic_utils import *
all_arches = [ "arm", "arm64", "mips", "mips64", "x86", "x86_64" ]
# temp directory where we store all intermediate files # temp directory where we store all intermediate files
bionic_temp = tempfile.mkdtemp(prefix="bionic_gensyscalls"); bionic_temp = tempfile.mkdtemp(prefix="bionic_gensyscalls");
@ -387,6 +391,120 @@ def x86_64_genstub(syscall):
return result return result
class SysCallsTxtParser:
def __init__(self):
self.syscalls = []
self.lineno = 0
def E(self, msg):
print "%d: %s" % (self.lineno, msg)
def parse_line(self, line):
""" parse a syscall spec line.
line processing, format is
return type func_name[|alias_list][:syscall_name[:socketcall_id]] ( [paramlist] ) architecture_list
"""
pos_lparen = line.find('(')
E = self.E
if pos_lparen < 0:
E("missing left parenthesis in '%s'" % line)
return
pos_rparen = line.rfind(')')
if pos_rparen < 0 or pos_rparen <= pos_lparen:
E("missing or misplaced right parenthesis in '%s'" % line)
return
return_type = line[:pos_lparen].strip().split()
if len(return_type) < 2:
E("missing return type in '%s'" % line)
return
syscall_func = return_type[-1]
return_type = string.join(return_type[:-1],' ')
socketcall_id = -1
pos_colon = syscall_func.find(':')
if pos_colon < 0:
syscall_name = syscall_func
else:
if pos_colon == 0 or pos_colon+1 >= len(syscall_func):
E("misplaced colon in '%s'" % line)
return
# now find if there is a socketcall_id for a dispatch-type syscall
# after the optional 2nd colon
pos_colon2 = syscall_func.find(':', pos_colon + 1)
if pos_colon2 < 0:
syscall_name = syscall_func[pos_colon+1:]
syscall_func = syscall_func[:pos_colon]
else:
if pos_colon2+1 >= len(syscall_func):
E("misplaced colon2 in '%s'" % line)
return
syscall_name = syscall_func[(pos_colon+1):pos_colon2]
socketcall_id = int(syscall_func[pos_colon2+1:])
syscall_func = syscall_func[:pos_colon]
alias_delim = syscall_func.find('|')
if alias_delim > 0:
alias_list = syscall_func[alias_delim+1:].strip()
syscall_func = syscall_func[:alias_delim]
alias_delim = syscall_name.find('|')
if alias_delim > 0:
syscall_name = syscall_name[:alias_delim]
syscall_aliases = string.split(alias_list, ',')
else:
syscall_aliases = []
if pos_rparen > pos_lparen+1:
syscall_params = line[pos_lparen+1:pos_rparen].split(',')
params = string.join(syscall_params,',')
else:
syscall_params = []
params = "void"
t = {
"name" : syscall_name,
"func" : syscall_func,
"aliases" : syscall_aliases,
"params" : syscall_params,
"decl" : "%-15s %s (%s);" % (return_type, syscall_func, params),
"socketcall_id" : socketcall_id
}
# Parse the architecture list.
arch_list = line[pos_rparen+1:].strip()
if arch_list == "all":
for arch in all_arches:
t[arch] = True
else:
for arch in string.split(arch_list, ','):
if arch in all_arches:
t[arch] = True
else:
E("invalid syscall architecture '%s' in '%s'" % (arch, line))
return
self.syscalls.append(t)
logging.debug(t)
def parse_file(self, file_path):
logging.debug("parse_file: %s" % file_path)
fp = open(file_path)
for line in fp.xreadlines():
self.lineno += 1
line = line.strip()
if not line: continue
if line[0] == '#': continue
self.parse_line(line)
fp.close()
class State: class State:
def __init__(self): def __init__(self):
self.old_stubs = [] self.old_stubs = []
@ -444,7 +562,7 @@ class State:
def gen_glibc_syscalls_h(self): def gen_glibc_syscalls_h(self):
# TODO: generate a separate file for each architecture, like glibc's bits/syscall.h. # TODO: generate a separate file for each architecture, like glibc's bits/syscall.h.
glibc_syscalls_h_path = "include/sys/glibc-syscalls.h" glibc_syscalls_h_path = "include/sys/glibc-syscalls.h"
D("generating " + glibc_syscalls_h_path) logging.info("generating " + glibc_syscalls_h_path)
glibc_fp = create_file(glibc_syscalls_h_path) glibc_fp = create_file(glibc_syscalls_h_path)
glibc_fp.write("/* %s */\n" % warning) glibc_fp.write("/* %s */\n" % warning)
glibc_fp.write("#ifndef _BIONIC_GLIBC_SYSCALLS_H_\n") glibc_fp.write("#ifndef _BIONIC_GLIBC_SYSCALLS_H_\n")
@ -473,7 +591,7 @@ class State:
for arch in all_arches: for arch in all_arches:
if syscall.has_key("asm-%s" % arch): if syscall.has_key("asm-%s" % arch):
filename = "arch-%s/syscalls/%s.S" % (arch, syscall["func"]) filename = "arch-%s/syscalls/%s.S" % (arch, syscall["func"])
D2(">>> generating " + filename) logging.info(">>> generating " + filename)
fp = create_file(filename) fp = create_file(filename)
fp.write(syscall["asm-%s" % arch]) fp.write(syscall["asm-%s" % arch])
fp.close() fp.close()
@ -481,28 +599,28 @@ class State:
def regenerate(self): def regenerate(self):
D("scanning for existing architecture-specific stub files...") logging.info("scanning for existing architecture-specific stub files...")
for arch in all_arches: for arch in all_arches:
arch_dir = "arch-" + arch arch_dir = "arch-" + arch
D("scanning " + os.path.join(bionic_libc_root, arch_dir)) logging.info("scanning " + os.path.join(bionic_libc_root, arch_dir))
rel_path = os.path.join(arch_dir, "syscalls") rel_path = os.path.join(arch_dir, "syscalls")
for file in os.listdir(os.path.join(bionic_libc_root, rel_path)): for file in os.listdir(os.path.join(bionic_libc_root, rel_path)):
if file.endswith(".S"): if file.endswith(".S"):
self.old_stubs.append(os.path.join(rel_path, file)) self.old_stubs.append(os.path.join(rel_path, file))
D("found %d stub files" % len(self.old_stubs)) logging.info("found %d stub files" % len(self.old_stubs))
if not os.path.exists(bionic_temp): if not os.path.exists(bionic_temp):
D("creating %s..." % bionic_temp) logging.info("creating %s..." % bionic_temp)
make_dir(bionic_temp) make_dir(bionic_temp)
D("re-generating stubs and support files...") logging.info("re-generating stubs and support files...")
self.gen_glibc_syscalls_h() self.gen_glibc_syscalls_h()
self.gen_syscall_stubs() self.gen_syscall_stubs()
D("comparing files...") logging.info("comparing files...")
adds = [] adds = []
edits = [] edits = []
@ -511,18 +629,18 @@ class State:
libc_file = os.path.join(bionic_libc_root, stub) libc_file = os.path.join(bionic_libc_root, stub)
if not os.path.exists(libc_file): if not os.path.exists(libc_file):
# new file, git add it # new file, git add it
D("new file: " + stub) logging.info("new file: " + stub)
adds.append(libc_file) adds.append(libc_file)
shutil.copyfile(tmp_file, libc_file) shutil.copyfile(tmp_file, libc_file)
elif not filecmp.cmp(tmp_file, libc_file): elif not filecmp.cmp(tmp_file, libc_file):
D("changed file: " + stub) logging.info("changed file: " + stub)
edits.append(stub) edits.append(stub)
deletes = [] deletes = []
for stub in self.old_stubs: for stub in self.old_stubs:
if not stub in self.new_stubs: if not stub in self.new_stubs:
D("deleted file: " + stub) logging.info("deleted file: " + stub)
deletes.append(os.path.join(bionic_libc_root, stub)) deletes.append(os.path.join(bionic_libc_root, stub))
if not DRY_RUN: if not DRY_RUN:
@ -539,11 +657,11 @@ class State:
commands.getoutput("git add %s" % (os.path.join(bionic_libc_root, "SYSCALLS.TXT"))) commands.getoutput("git add %s" % (os.path.join(bionic_libc_root, "SYSCALLS.TXT")))
if (not adds) and (not deletes) and (not edits): if (not adds) and (not deletes) and (not edits):
D("no changes detected!") logging.info("no changes detected!")
else: else:
D("ready to go!!") logging.info("ready to go!!")
D_setlevel(1) logging.basicConfig(level=logging.INFO)
state = State() state = State()
state.process_file(os.path.join(bionic_libc_root, "SYSCALLS.TXT")) state.process_file(os.path.join(bionic_libc_root, "SYSCALLS.TXT"))