From 29bba01e2f90fe58a7ecf04156d264dce4a30f69 Mon Sep 17 00:00:00 2001 From: "phoglund@webrtc.org" Date: Fri, 24 Feb 2012 12:12:31 +0000 Subject: [PATCH] The clean script can now also delete directories. BUG= TEST= Review URL: https://webrtc-codereview.appspot.com/399004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1763 4adac7df-926f-26a2-2b94-8c16560cd09d --- tools/continuous_build/clean_old_files.py | 96 ++++++++++++++++++----- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/tools/continuous_build/clean_old_files.py b/tools/continuous_build/clean_old_files.py index e1c29fa49..e87526edc 100755 --- a/tools/continuous_build/clean_old_files.py +++ b/tools/continuous_build/clean_old_files.py @@ -9,6 +9,16 @@ __author__ = "ivinnichenko@webrtc.org (Illya Vinnichenko)" +"""This script will prune sufficiently old files and empty directories. + + The algorithm is to look into the provided directory and delete any files + that is older than x days, recursively. Then all empty directories will be + deleted (we can't look at timestamps there since the act of deleting a file + will refresh the directory's timestamp). + + Note: This script has only been tested on Linux. +""" + from optparse import OptionParser import os import sys @@ -23,7 +33,8 @@ WHITELIST = ["buildbot.tac", "master.cfg", "public_html", "changes.pck", def is_whitelisted(path): """Check if file is whitelisted. - path: file path. + Args: + path: file path. """ for entry in WHITELIST: if entry in path: @@ -31,38 +42,79 @@ def is_whitelisted(path): return False -def remove_old_filenames(path, num_days, verbose): - """Remove old files. +def delete_directory(directory): + try: + os.rmdir(directory) + return True + except OSError as exception: + # The directory probably contains newer files. + print "Could not remove directory %s: reason %s." % (directory, exception) + return False - path: base directory for removal. - num_days: days limit for removal. - verbose: print every cmd? + +def delete_file(file): + try: + os.remove(file) + except OSError as exception: + print "Unexpectedly failed to remove file %s: reason %s." % (file, + exception) + + +def log_removal(file_or_directory, time_stamp, verbose): + if verbose: + str_stamp = time.strftime("%a, %d %b %Y %H:%M:%S +0000", + time.gmtime(time_stamp)) + print "Removing [%s], stamped on %s" % (file_or_directory, str_stamp) + + +def remove_old_files_and_directories(path, num_days, verbose, skip_dirs): + """Removes all files under path that are older than num_days days. + The algorithm also tried to delete all directories, except for those who + contain files that are sufficiently new. + + Implementation note: it doesn't make sense to look at timestamps for + directories since their timestamps are updated when a file is deleted. + + Args: + path: The starting point. + num_days: days limit for removal. + verbose: print every cmd? """ - print "Cleaning up everything in %s older than %s days" % (path, num_days) current_time = time.time() - limit = 60*60*24*num_days - for root, unused_dirs, files in os.walk(path): - for filename in files: - current_file = os.path.join(root, filename) + limit = 60 * 60 * 24 * num_days + + # Walk bottom-up so directories are deleted in the right order. + for root, directories, files in os.walk(path, topdown=False): + for file in files: + current_file = os.path.join(root, file) + time_stamp = os.stat(current_file).st_mtime + if is_whitelisted(current_file): continue - time_stamp = os.stat(current_file).st_mtime + if (current_time - time_stamp) > limit: - str_stamp = time.strftime("%a, %d %b %Y %H:%M:%S +0000", - time.gmtime(time_stamp)) - if verbose: - print "Removing [%s], stamped on %s" % (current_file, str_stamp) - os.remove(current_file) + delete_file(current_file) + log_removal(current_file, time_stamp, verbose) + + if not skip_dirs: + for directory in directories: + current_directory = os.path.join(root, directory) + time_stamp = os.stat(current_directory).st_mtime + if delete_directory(current_directory): + log_removal(current_directory, time_stamp, verbose) def main(): - usage = "usage: %prog [options] arg" + usage = "usage: %prog -p -n [-q] [-d]" parser = OptionParser(usage) parser.add_option("-p", "--path", dest="cleanup_path", help="base directory") parser.add_option("-n", "--num_days", dest="num_days", help="number of days") parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True, help="don't print status messages to stdout") + parser.add_option("-d", "--delete-dirs-too", + action="store_false", dest="skip_dirs", default=True, + help="number of days") options, args = parser.parse_args() if not options.cleanup_path: @@ -71,8 +123,12 @@ def main(): if not options.num_days: print "You must specify number of days old" sys.exit(2) - remove_old_filenames(options.cleanup_path, int(options.num_days), - options.verbose) + + if options.verbose: + print "Cleaning up everything in %s older than %s days" % ( + options.cleanup_path, options.num_days) + remove_old_files_and_directories(options.cleanup_path, int(options.num_days), + options.verbose, options.skip_dirs) if __name__ == "__main__": main()