generic-library/bionic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge "Move tzdata scripts from bionic to external/icu"

Browse Source
This commit is contained in:
Neil Fuller 2015-08-25 09:22:36 +00:00 committed by Gerrit Code Review
commit f7286a26de
2 changed files with 0 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
libc/tools/zoneinfo/ZoneCompactor.java
View File

@ -1,192 +0,0 @@
import java.io.*;
import java.util.*;
// usage: java ZoneCompiler <setup file> <data directory> <output directory> <tzdata version>
//
// Compile a set of tzfile-formatted files into a single file containing an index.
//
// The compilation is controlled by a setup file, which is provided as a
// command-line argument. The setup file has the form:
//
// Link <toName> <fromName>
// ...
// <zone filename>
// ...
//
// Note that the links must be declared prior to the zone names.
// A zone name is a filename relative to the source directory such as
// 'GMT', 'Africa/Dakar', or 'America/Argentina/Jujuy'.
//
// Use the 'zic' command-line tool to convert from flat files
// (such as 'africa' or 'northamerica') to a directory
// hierarchy suitable for this tool (containing files such as 'data/Africa/Abidjan').
//
public class ZoneCompactor {
// Maximum number of characters in a zone name, including '\0' terminator.
private static final int MAXNAME = 40;
// Zone name synonyms.
private Map<String,String> links = new HashMap<String,String>();
// File offsets by zone name.
private Map<String,Integer> offsets = new HashMap<String,Integer>();
// File lengths by zone name.
private Map<String,Integer> lengths = new HashMap<String,Integer>();
// Concatenate the contents of 'inFile' onto 'out'.
private static void copyFile(File inFile, OutputStream out) throws Exception {
byte[] ret = new byte[0];
InputStream in = new FileInputStream(inFile);
byte[] buf = new byte[8192];
while (true) {
int nbytes = in.read(buf);
if (nbytes == -1) {
break;
}
out.write(buf, 0, nbytes);
byte[] nret = new byte[ret.length + nbytes];
System.arraycopy(ret, 0, nret, 0, ret.length);
System.arraycopy(buf, 0, nret, ret.length, nbytes);
ret = nret;
}
out.flush();
}
public ZoneCompactor(String setupFile, String dataDirectory, String zoneTabFile, String outputDirectory, String version) throws Exception {
// Read the setup file and concatenate all the data.
ByteArrayOutputStream allData = new ByteArrayOutputStream();
BufferedReader reader = new BufferedReader(new FileReader(setupFile));
String s;
int offset = 0;
while ((s = reader.readLine()) != null) {
s = s.trim();
if (s.startsWith("Link")) {
StringTokenizer st = new StringTokenizer(s);
st.nextToken();
String to = st.nextToken();
String from = st.nextToken();
links.put(from, to);
} else {
String link = links.get(s);
if (link == null) {
File sourceFile = new File(dataDirectory, s);
long length = sourceFile.length();
offsets.put(s, offset);
lengths.put(s, (int) length);
offset += length;
copyFile(sourceFile, allData);
}
}
}
reader.close();
// Fill in fields for links.
Iterator<String> it = links.keySet().iterator();
while (it.hasNext()) {
String from = it.next();
String to = links.get(from);
offsets.put(from, offsets.get(to));
lengths.put(from, lengths.get(to));
}
// Create/truncate the destination file.
RandomAccessFile f = new RandomAccessFile(new File(outputDirectory, "tzdata"), "rw");
f.setLength(0);
// Write the header.
// byte[12] tzdata_version -- 'tzdata2012f\0'
// int index_offset -- so we can slip in extra header fields in a backwards-compatible way
// int data_offset
// int zonetab_offset
// tzdata_version
f.write(toAscii(new byte[12], version));
// Write dummy values for the three offsets, and remember where we need to seek back to later
// when we have the real values.
int index_offset_offset = (int) f.getFilePointer();
f.writeInt(0);
int data_offset_offset = (int) f.getFilePointer();
f.writeInt(0);
int zonetab_offset_offset = (int) f.getFilePointer();
f.writeInt(0);
int index_offset = (int) f.getFilePointer();
// Write the index.
ArrayList<String> sortedOlsonIds = new ArrayList<String>();
sortedOlsonIds.addAll(offsets.keySet());
Collections.sort(sortedOlsonIds);
it = sortedOlsonIds.iterator();
while (it.hasNext()) {
String zoneName = it.next();
if (zoneName.length() >= MAXNAME) {
throw new RuntimeException("zone filename too long: " + zoneName.length());
}
// Follow the chain of links to work out where the real data for this zone lives.
String actualZoneName = zoneName;
while (links.get(actualZoneName) != null) {
actualZoneName = links.get(actualZoneName);
}
f.write(toAscii(new byte[MAXNAME], zoneName));
f.writeInt(offsets.get(actualZoneName));
f.writeInt(lengths.get(actualZoneName));
f.writeInt(0); // Used to be raw GMT offset. No longer used.
}
int data_offset = (int) f.getFilePointer();
// Write the data.
f.write(allData.toByteArray());
int zonetab_offset = (int) f.getFilePointer();
// Copy the zone.tab.
reader = new BufferedReader(new FileReader(zoneTabFile));
while ((s = reader.readLine()) != null) {
if (!s.startsWith("#")) {
f.writeBytes(s);
f.write('\n');
}
}
reader.close();
// Go back and fix up the offsets in the header.
f.seek(index_offset_offset);
f.writeInt(index_offset);
f.seek(data_offset_offset);
f.writeInt(data_offset);
f.seek(zonetab_offset_offset);
f.writeInt(zonetab_offset);
f.close();
}
private static byte[] toAscii(byte[] dst, String src) {
for (int i = 0; i < src.length(); ++i) {
if (src.charAt(i) > '~') {
throw new RuntimeException("non-ASCII string: " + src);
}
dst[i] = (byte) src.charAt(i);
}
return dst;
}
public static void main(String[] args) throws Exception {
if (args.length != 5) {
System.err.println("usage: java ZoneCompactor <setup file> <data directory> <zone.tab file> <output directory> <tzdata version>");
System.exit(0);
}
new ZoneCompactor(args[0], args[1], args[2], args[3], args[4]);
}
}

262
libc/tools/zoneinfo/update-tzdata.py
View File

@ -1,262 +0,0 @@
#!/usr/bin/python
"""Updates the timezone data held in bionic and ICU."""
import ftplib
import glob
import httplib
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
regions = ['africa', 'antarctica', 'asia', 'australasia',
'etcetera', 'europe', 'northamerica', 'southamerica',
# These two deliberately come last so they override what came
# before (and each other).
'backward', 'backzone' ]
def CheckDirExists(dir, dirname):
if not os.path.isdir(dir):
print "Couldn't find %s (%s)!" % (dirname, dir)
sys.exit(1)
bionic_libc_tools_zoneinfo_dir = os.path.realpath(os.path.dirname(sys.argv[0]))
# Find the bionic directory, searching upward from this script.
bionic_dir = os.path.realpath('%s/../../..' % bionic_libc_tools_zoneinfo_dir)
bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir
CheckDirExists(bionic_libc_zoneinfo_dir, 'bionic/libc/zoneinfo')
CheckDirExists(bionic_libc_tools_zoneinfo_dir, 'bionic/libc/tools/zoneinfo')
print 'Found bionic in %s ...' % bionic_dir
# Find the icu directory.
icu_dir = os.path.realpath('%s/../external/icu' % bionic_dir)
icu4c_dir = os.path.realpath('%s/icu4c/source' % icu_dir)
icu4j_dir = os.path.realpath('%s/icu4j' % icu_dir)
CheckDirExists(icu4c_dir, 'external/icu/icu4c/source')
CheckDirExists(icu4j_dir, 'external/icu/icu4j')
print 'Found icu in %s ...' % icu_dir
def GetCurrentTzDataVersion():
return open('%s/tzdata' % bionic_libc_zoneinfo_dir).read().split('\x00', 1)[0]
def WriteSetupFile():
"""Writes the list of zones that ZoneCompactor should process."""
links = []
zones = []
for region in regions:
for line in open('extracted/%s' % region):
fields = line.split()
if fields:
if fields[0] == 'Link':
links.append('%s %s %s' % (fields[0], fields[1], fields[2]))
zones.append(fields[2])
elif fields[0] == 'Zone':
zones.append(fields[1])
zones.sort()
setup = open('setup', 'w')
for link in sorted(set(links)):
setup.write('%s\n' % link)
for zone in sorted(set(zones)):
setup.write('%s\n' % zone)
setup.close()
def SwitchToNewTemporaryDirectory():
tmp_dir = tempfile.mkdtemp('-tzdata')
os.chdir(tmp_dir)
print 'Created temporary directory "%s"...' % tmp_dir
def FtpRetrieveFile(ftp, filename):
ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write)
def FtpRetrieveFileAndSignature(ftp, data_filename):
"""Downloads and repackages the given data from the given FTP server."""
print 'Downloading data...'
FtpRetrieveFile(ftp, data_filename)
print 'Downloading signature...'
signature_filename = '%s.asc' % data_filename
FtpRetrieveFile(ftp, signature_filename)
def HttpRetrieveFile(http, path, output_filename):
http.request("GET", path)
f = open(output_filename, 'wb')
f.write(http.getresponse().read())
f.close()
def HttpRetrieveFileAndSignature(http, data_filename):
"""Downloads and repackages the given data from the given HTTP server."""
path = "/time-zones/repository/releases/%s" % data_filename
print 'Downloading data...'
HttpRetrieveFile(http, path, data_filename)
print 'Downloading signature...'
signature_filename = '%s.asc' % data_filename
HttpRetrievefile(http, "%s.asc" % path, signature_filename)
def BuildIcuToolsAndData(data_filename):
# Keep track of the original cwd so we can go back to it at the end.
original_working_dir = os.getcwd()
# Create a directory to run 'make' from.
icu_working_dir = '%s/icu' % original_working_dir
os.mkdir(icu_working_dir)
os.chdir(icu_working_dir)
# Build the ICU tools.
print 'Configuring ICU tools...'
subprocess.check_call(['%s/runConfigureICU' % icu4c_dir, 'Linux'])
# Run the ICU tools.
os.chdir('tools/tzcode')
# The tz2icu tool only picks up icuregions and icuzones in they are in the CWD
for icu_data_file in [ 'icuregions', 'icuzones']:
icu_data_file_source = '%s/tools/tzcode/%s' % (icu4c_dir, icu_data_file)
icu_data_file_symlink = './%s' % icu_data_file
os.symlink(icu_data_file_source, icu_data_file_symlink)
shutil.copyfile('%s/%s' % (original_working_dir, data_filename), data_filename)
print 'Making ICU data...'
# The Makefile assumes the existence of the bin directory.
os.mkdir('%s/bin' % icu_working_dir)
subprocess.check_call(['make'])
# Copy the source file to its ultimate destination.
icu_txt_data_dir = '%s/data/misc' % icu4c_dir
print 'Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir
shutil.copy('zoneinfo64.txt', icu_txt_data_dir)
# Regenerate the .dat file.
os.chdir(icu_working_dir)
subprocess.check_call(['make', 'INCLUDE_UNI_CORE_DATA=1', '-j32'])
# Copy the .dat file to its ultimate destination.
icu_dat_data_dir = '%s/stubdata' % icu4c_dir
datfiles = glob.glob('data/out/tmp/icudt??l.dat')
if len(datfiles) != 1:
print 'ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles)
sys.exit(1)
datfile = datfiles[0]
print 'Copying %s to %s ...' % (datfile, icu_dat_data_dir)
shutil.copy(datfile, icu_dat_data_dir)
# Generate the ICU4J .jar files
os.chdir('%s/data' % icu_working_dir)
subprocess.check_call(['make', 'icu4j-data'])
# Copy the ICU4J .jar files to their ultimate destination.
icu_jar_data_dir = '%s/main/shared/data' % icu4j_dir
jarfiles = glob.glob('out/icu4j/*.jar')
if len(jarfiles) != 2:
print 'ERROR: Unexpectedly found %d .jar files (%s). Halting.' % (len(jarfiles), jarfiles)
sys.exit(1)
for jarfile in jarfiles:
print 'Copying %s to %s ...' % (jarfile, icu_jar_data_dir)
shutil.copy(jarfile, icu_jar_data_dir)
# Switch back to the original working cwd.
os.chdir(original_working_dir)
def CheckSignature(data_filename):
signature_filename = '%s.asc' % data_filename
print 'Verifying signature...'
# If this fails for you, you probably need to import Paul Eggert's public key:
# gpg --recv-keys ED97E90E62AA7E34
subprocess.check_call(['gpg', '--trusted-key=ED97E90E62AA7E34', '--verify',
signature_filename, data_filename])
def BuildBionicToolsAndData(data_filename):
new_version = re.search('(tzdata.+)\\.tar\\.gz', data_filename).group(1)
print 'Extracting...'
os.mkdir('extracted')
tar = tarfile.open(data_filename, 'r')
tar.extractall('extracted')
print 'Calling zic(1)...'
os.mkdir('data')
zic_inputs = [ 'extracted/%s' % x for x in regions ]
zic_cmd = ['zic', '-d', 'data' ]
zic_cmd.extend(zic_inputs)
subprocess.check_call(zic_cmd)
WriteSetupFile()
print 'Calling ZoneCompactor to update bionic to %s...' % new_version
subprocess.check_call(['javac', '-d', '.',
'%s/ZoneCompactor.java' % bionic_libc_tools_zoneinfo_dir])
subprocess.check_call(['java', 'ZoneCompactor',
'setup', 'data', 'extracted/zone.tab',
bionic_libc_zoneinfo_dir, new_version])
# Run with no arguments from any directory, with no special setup required.
# See http://www.iana.org/time-zones/ for more about the source of this data.
def main():
print 'Looking for new tzdata...'
tzdata_filenames = []
# The FTP server lets you download intermediate releases, and also lets you
# download the signatures for verification, so it's your best choice.
use_ftp = True
if use_ftp:
ftp = ftplib.FTP('ftp.iana.org')
ftp.login()
ftp.cwd('tz/releases')
for filename in ftp.nlst():
if filename.startswith('tzdata20') and filename.endswith('.tar.gz'):
tzdata_filenames.append(filename)
tzdata_filenames.sort()
else:
http = httplib.HTTPConnection('www.iana.org')
http.request("GET", "/time-zones")
index_lines = http.getresponse().read().split('\n')
for line in index_lines:
m = re.compile('.*href="/time-zones/repository/releases/(tzdata20\d\d\c\.tar\.gz)".*').match(line)
if m:
tzdata_filenames.append(m.group(1))
# If you're several releases behind, we'll walk you through the upgrades
# one by one.
current_version = GetCurrentTzDataVersion()
current_filename = '%s.tar.gz' % current_version
for filename in tzdata_filenames:
if filename > current_filename:
print 'Found new tzdata: %s' % filename
SwitchToNewTemporaryDirectory()
if use_ftp:
FtpRetrieveFileAndSignature(ftp, filename)
else:
HttpRetrieveFileAndSignature(http, filename)
CheckSignature(filename)
BuildIcuToolsAndData(filename)
BuildBionicToolsAndData(filename)
print 'Look in %s and %s for new data files' % (bionic_dir, icu_dir)
sys.exit(0)
print 'You already have the latest tzdata (%s)!' % current_version
sys.exit(0)
if __name__ == '__main__':
main()