4d3abcb033
The ICU4J changes are not necessary for use on Android (since we use the ICU4C .dat file), but updating them ensures that the .jars in sync with everything else and the jars are currently required for host tests. Change-Id: Ie56b31af87e8fbd27a6489af8287e4b6a7be6b8f
263 lines
8.6 KiB
Python
Executable File
263 lines
8.6 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
"""Updates the timezone data held in bionic and ICU."""
|
|
|
|
import ftplib
|
|
import glob
|
|
import httplib
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
|
|
regions = ['africa', 'antarctica', 'asia', 'australasia',
|
|
'etcetera', 'europe', 'northamerica', 'southamerica',
|
|
# These two deliberately come last so they override what came
|
|
# before (and each other).
|
|
'backward', 'backzone' ]
|
|
|
|
def CheckDirExists(dir, dirname):
|
|
if not os.path.isdir(dir):
|
|
print "Couldn't find %s (%s)!" % (dirname, dir)
|
|
sys.exit(1)
|
|
|
|
bionic_libc_tools_zoneinfo_dir = os.path.realpath(os.path.dirname(sys.argv[0]))
|
|
|
|
# Find the bionic directory, searching upward from this script.
|
|
bionic_dir = os.path.realpath('%s/../../..' % bionic_libc_tools_zoneinfo_dir)
|
|
bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir
|
|
CheckDirExists(bionic_libc_zoneinfo_dir, 'bionic/libc/zoneinfo')
|
|
CheckDirExists(bionic_libc_tools_zoneinfo_dir, 'bionic/libc/tools/zoneinfo')
|
|
print 'Found bionic in %s ...' % bionic_dir
|
|
|
|
# Find the icu directory.
|
|
icu_dir = os.path.realpath('%s/../external/icu' % bionic_dir)
|
|
icu4c_dir = os.path.realpath('%s/icu4c/source' % icu_dir)
|
|
icu4j_dir = os.path.realpath('%s/icu4j' % icu_dir)
|
|
CheckDirExists(icu4c_dir, 'external/icu/icu4c/source')
|
|
CheckDirExists(icu4j_dir, 'external/icu/icu4j')
|
|
print 'Found icu in %s ...' % icu_dir
|
|
|
|
|
|
def GetCurrentTzDataVersion():
|
|
return open('%s/tzdata' % bionic_libc_zoneinfo_dir).read().split('\x00', 1)[0]
|
|
|
|
|
|
def WriteSetupFile():
|
|
"""Writes the list of zones that ZoneCompactor should process."""
|
|
links = []
|
|
zones = []
|
|
for region in regions:
|
|
for line in open('extracted/%s' % region):
|
|
fields = line.split()
|
|
if fields:
|
|
if fields[0] == 'Link':
|
|
links.append('%s %s %s' % (fields[0], fields[1], fields[2]))
|
|
zones.append(fields[2])
|
|
elif fields[0] == 'Zone':
|
|
zones.append(fields[1])
|
|
zones.sort()
|
|
|
|
setup = open('setup', 'w')
|
|
for link in sorted(set(links)):
|
|
setup.write('%s\n' % link)
|
|
for zone in sorted(set(zones)):
|
|
setup.write('%s\n' % zone)
|
|
setup.close()
|
|
|
|
|
|
def SwitchToNewTemporaryDirectory():
|
|
tmp_dir = tempfile.mkdtemp('-tzdata')
|
|
os.chdir(tmp_dir)
|
|
print 'Created temporary directory "%s"...' % tmp_dir
|
|
|
|
|
|
def FtpRetrieveFile(ftp, filename):
|
|
ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write)
|
|
|
|
|
|
def FtpRetrieveFileAndSignature(ftp, data_filename):
|
|
"""Downloads and repackages the given data from the given FTP server."""
|
|
print 'Downloading data...'
|
|
FtpRetrieveFile(ftp, data_filename)
|
|
|
|
print 'Downloading signature...'
|
|
signature_filename = '%s.asc' % data_filename
|
|
FtpRetrieveFile(ftp, signature_filename)
|
|
|
|
|
|
def HttpRetrieveFile(http, path, output_filename):
|
|
http.request("GET", path)
|
|
f = open(output_filename, 'wb')
|
|
f.write(http.getresponse().read())
|
|
f.close()
|
|
|
|
|
|
def HttpRetrieveFileAndSignature(http, data_filename):
|
|
"""Downloads and repackages the given data from the given HTTP server."""
|
|
path = "/time-zones/repository/releases/%s" % data_filename
|
|
|
|
print 'Downloading data...'
|
|
HttpRetrieveFile(http, path, data_filename)
|
|
|
|
print 'Downloading signature...'
|
|
signature_filename = '%s.asc' % data_filename
|
|
HttpRetrievefile(http, "%s.asc" % path, signature_filename)
|
|
|
|
|
|
def BuildIcuToolsAndData(data_filename):
|
|
# Keep track of the original cwd so we can go back to it at the end.
|
|
original_working_dir = os.getcwd()
|
|
|
|
# Create a directory to run 'make' from.
|
|
icu_working_dir = '%s/icu' % original_working_dir
|
|
os.mkdir(icu_working_dir)
|
|
os.chdir(icu_working_dir)
|
|
|
|
# Build the ICU tools.
|
|
print 'Configuring ICU tools...'
|
|
subprocess.check_call(['%s/runConfigureICU' % icu4c_dir, 'Linux'])
|
|
|
|
# Run the ICU tools.
|
|
os.chdir('tools/tzcode')
|
|
|
|
# The tz2icu tool only picks up icuregions and icuzones in they are in the CWD
|
|
for icu_data_file in [ 'icuregions', 'icuzones']:
|
|
icu_data_file_source = '%s/tools/tzcode/%s' % (icu4c_dir, icu_data_file)
|
|
icu_data_file_symlink = './%s' % icu_data_file
|
|
os.symlink(icu_data_file_source, icu_data_file_symlink)
|
|
|
|
shutil.copyfile('%s/%s' % (original_working_dir, data_filename), data_filename)
|
|
print 'Making ICU data...'
|
|
# The Makefile assumes the existence of the bin directory.
|
|
os.mkdir('%s/bin' % icu_working_dir)
|
|
subprocess.check_call(['make'])
|
|
|
|
# Copy the source file to its ultimate destination.
|
|
icu_txt_data_dir = '%s/data/misc' % icu4c_dir
|
|
print 'Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir
|
|
shutil.copy('zoneinfo64.txt', icu_txt_data_dir)
|
|
|
|
# Regenerate the .dat file.
|
|
os.chdir(icu_working_dir)
|
|
subprocess.check_call(['make', 'INCLUDE_UNI_CORE_DATA=1', '-j32'])
|
|
|
|
# Copy the .dat file to its ultimate destination.
|
|
icu_dat_data_dir = '%s/stubdata' % icu4c_dir
|
|
datfiles = glob.glob('data/out/tmp/icudt??l.dat')
|
|
if len(datfiles) != 1:
|
|
print 'ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles)
|
|
sys.exit(1)
|
|
datfile = datfiles[0]
|
|
print 'Copying %s to %s ...' % (datfile, icu_dat_data_dir)
|
|
shutil.copy(datfile, icu_dat_data_dir)
|
|
|
|
# Generate the ICU4J .jar files
|
|
os.chdir('%s/data' % icu_working_dir)
|
|
subprocess.check_call(['make', 'icu4j-data'])
|
|
|
|
# Copy the ICU4J .jar files to their ultimate destination.
|
|
icu_jar_data_dir = '%s/main/shared/data' % icu4j_dir
|
|
jarfiles = glob.glob('out/icu4j/*.jar')
|
|
if len(jarfiles) != 2:
|
|
print 'ERROR: Unexpectedly found %d .jar files (%s). Halting.' % (len(jarfiles), jarfiles)
|
|
sys.exit(1)
|
|
for jarfile in jarfiles:
|
|
print 'Copying %s to %s ...' % (jarfile, icu_jar_data_dir)
|
|
shutil.copy(jarfile, icu_jar_data_dir)
|
|
|
|
# Switch back to the original working cwd.
|
|
os.chdir(original_working_dir)
|
|
|
|
|
|
def CheckSignature(data_filename):
|
|
signature_filename = '%s.asc' % data_filename
|
|
print 'Verifying signature...'
|
|
# If this fails for you, you probably need to import Paul Eggert's public key:
|
|
# gpg --recv-keys ED97E90E62AA7E34
|
|
subprocess.check_call(['gpg', '--trusted-key=ED97E90E62AA7E34', '--verify',
|
|
signature_filename, data_filename])
|
|
|
|
|
|
def BuildBionicToolsAndData(data_filename):
|
|
new_version = re.search('(tzdata.+)\\.tar\\.gz', data_filename).group(1)
|
|
|
|
print 'Extracting...'
|
|
os.mkdir('extracted')
|
|
tar = tarfile.open(data_filename, 'r')
|
|
tar.extractall('extracted')
|
|
|
|
print 'Calling zic(1)...'
|
|
os.mkdir('data')
|
|
zic_inputs = [ 'extracted/%s' % x for x in regions ]
|
|
zic_cmd = ['zic', '-d', 'data' ]
|
|
zic_cmd.extend(zic_inputs)
|
|
subprocess.check_call(zic_cmd)
|
|
|
|
WriteSetupFile()
|
|
|
|
print 'Calling ZoneCompactor to update bionic to %s...' % new_version
|
|
subprocess.check_call(['javac', '-d', '.',
|
|
'%s/ZoneCompactor.java' % bionic_libc_tools_zoneinfo_dir])
|
|
subprocess.check_call(['java', 'ZoneCompactor',
|
|
'setup', 'data', 'extracted/zone.tab',
|
|
bionic_libc_zoneinfo_dir, new_version])
|
|
|
|
|
|
# Run with no arguments from any directory, with no special setup required.
|
|
# See http://www.iana.org/time-zones/ for more about the source of this data.
|
|
def main():
|
|
print 'Looking for new tzdata...'
|
|
|
|
tzdata_filenames = []
|
|
|
|
# The FTP server lets you download intermediate releases, and also lets you
|
|
# download the signatures for verification, so it's your best choice.
|
|
use_ftp = True
|
|
|
|
if use_ftp:
|
|
ftp = ftplib.FTP('ftp.iana.org')
|
|
ftp.login()
|
|
ftp.cwd('tz/releases')
|
|
for filename in ftp.nlst():
|
|
if filename.startswith('tzdata20') and filename.endswith('.tar.gz'):
|
|
tzdata_filenames.append(filename)
|
|
tzdata_filenames.sort()
|
|
else:
|
|
http = httplib.HTTPConnection('www.iana.org')
|
|
http.request("GET", "/time-zones")
|
|
index_lines = http.getresponse().read().split('\n')
|
|
for line in index_lines:
|
|
m = re.compile('.*href="/time-zones/repository/releases/(tzdata20\d\d\c\.tar\.gz)".*').match(line)
|
|
if m:
|
|
tzdata_filenames.append(m.group(1))
|
|
|
|
# If you're several releases behind, we'll walk you through the upgrades
|
|
# one by one.
|
|
current_version = GetCurrentTzDataVersion()
|
|
current_filename = '%s.tar.gz' % current_version
|
|
for filename in tzdata_filenames:
|
|
if filename > current_filename:
|
|
print 'Found new tzdata: %s' % filename
|
|
SwitchToNewTemporaryDirectory()
|
|
if use_ftp:
|
|
FtpRetrieveFileAndSignature(ftp, filename)
|
|
else:
|
|
HttpRetrieveFileAndSignature(http, filename)
|
|
|
|
CheckSignature(filename)
|
|
BuildIcuToolsAndData(filename)
|
|
BuildBionicToolsAndData(filename)
|
|
print 'Look in %s and %s for new data files' % (bionic_dir, icu_dir)
|
|
sys.exit(0)
|
|
|
|
print 'You already have the latest tzdata (%s)!' % current_version
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|