generic-library/bionic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Change ZoneCompator to generate the single "tzdata" file.

Browse Source 
Also remove the obsolete individual files, and the temporary script
that converted between the formats.

Bug: 7012465
Change-Id: I5a4030098e4d53e747fd6d395df2679d1567ee1f
This commit is contained in:
Elliott Hughes 2012-10-19 14:47:37 -07:00
parent e75c273ac4
commit 5b1497acdb
6 changed files with 268 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
libc/tools/zoneinfo/ZoneCompactor.java
View File

@ -7,8 +7,7 @@ import libcore.util.ZoneInfo;
// usage: java ZoneCompiler <setup file> <data directory> <output directory> <tzdata version> // usage: java ZoneCompiler <setup file> <data directory> <output directory> <tzdata version>
// //
// Compile a set of tzfile-formatted files into a single file plus // Compile a set of tzfile-formatted files into a single file containing an index.
// an index file.
// //
// The compilation is controlled by a setup file, which is provided as a // The compilation is controlled by a setup file, which is provided as a
// command-line argument. The setup file has the form: // command-line argument. The setup file has the form:
@ -18,19 +17,14 @@ import libcore.util.ZoneInfo;
// <zone filename> // <zone filename>
// ... // ...
// //
// Note that the links must be declared prior to the zone names. A // Note that the links must be declared prior to the zone names.
// zone name is a filename relative to the source directory such as // A zone name is a filename relative to the source directory such as
// 'GMT', 'Africa/Dakar', or 'America/Argentina/Jujuy'. // 'GMT', 'Africa/Dakar', or 'America/Argentina/Jujuy'.
// //
// Use the 'zic' command-line tool to convert from flat files // Use the 'zic' command-line tool to convert from flat files
// (e.g., 'africa', 'northamerica') into a suitable source directory // (such as 'africa' or 'northamerica') to a directory
// hierarchy for this tool (e.g., 'data/Africa/Abidjan'). // hierarchy suitable for this tool (containing files such as 'data/Africa/Abidjan').
// //
// Example:
// zic -d data tz2007h
// javac ZoneCompactor.java
// java ZoneCompactor setup data
// <produces zoneinfo.dat and zoneinfo.idx>
public class ZoneCompactor { public class ZoneCompactor {
public static class ByteArrayBufferIteratorBE extends BufferIterator { public static class ByteArrayBufferIteratorBE extends BufferIterator {
@ -113,23 +107,12 @@ public class ZoneCompactor {
return ret; return ret;
} }
// Write a 32-bit integer in network byte order
private void writeInt(OutputStream os, int x) throws IOException {
os.write((x >> 24) & 0xff);
os.write((x >> 16) & 0xff);
os.write((x >> 8) & 0xff);
os.write( x & 0xff);
}
public ZoneCompactor(String setupFile, String dataDirectory, String outputDirectory, String version) throws Exception { public ZoneCompactor(String setupFile, String dataDirectory, String outputDirectory, String version) throws Exception {
File zoneInfoFile = new File(outputDirectory, "zoneinfo.dat"); // Read the setup file, and concatenate all the data.
zoneInfoFile.delete(); ByteArrayOutputStream allData = new ByteArrayOutputStream();
OutputStream zoneInfo = new FileOutputStream(zoneInfoFile); BufferedReader reader = new BufferedReader(new FileReader(setupFile));
BufferedReader rdr = new BufferedReader(new FileReader(setupFile));
String s; String s;
while ((s = rdr.readLine()) != null) { while ((s = reader.readLine()) != null) {
s = s.trim(); s = s.trim();
if (s.startsWith("Link")) { if (s.startsWith("Link")) {
StringTokenizer st = new StringTokenizer(s); StringTokenizer st = new StringTokenizer(s);
@ -140,27 +123,26 @@ public class ZoneCompactor {
} else { } else {
String link = links.get(s); String link = links.get(s);
if (link == null) { if (link == null) {
File f = new File(dataDirectory, s); File sourceFile = new File(dataDirectory, s);
long length = f.length(); long length = sourceFile.length();
starts.put(s, new Integer(start)); starts.put(s, start);
lengths.put(s, new Integer((int)length)); lengths.put(s, (int) length);
start += length; start += length;
byte[] data = copyFile(f, zoneInfo); byte[] data = copyFile(sourceFile, allData);
BufferIterator it = new ByteArrayBufferIteratorBE(data); BufferIterator it = new ByteArrayBufferIteratorBE(data);
TimeZone tz = ZoneInfo.makeTimeZone(s, it); TimeZone tz = ZoneInfo.makeTimeZone(s, it);
int gmtOffset = tz.getRawOffset(); int gmtOffset = tz.getRawOffset();
offsets.put(s, new Integer(gmtOffset)); offsets.put(s, gmtOffset);
} }
} }
} }
zoneInfo.close();
// Fill in fields for links // Fill in fields for links.
Iterator<String> iter = links.keySet().iterator(); Iterator<String> it = links.keySet().iterator();
while (iter.hasNext()) { while (it.hasNext()) {
String from = iter.next(); String from = it.next();
String to = links.get(from); String to = links.get(from);
starts.put(from, starts.get(to)); starts.put(from, starts.get(to));
@ -168,38 +150,79 @@ public class ZoneCompactor {
offsets.put(from, offsets.get(to)); offsets.put(from, offsets.get(to));
} }
File idxFile = new File(outputDirectory, "zoneinfo.idx"); // Create/truncate the destination file.
idxFile.delete(); RandomAccessFile f = new RandomAccessFile(new File(outputDirectory, "tzdata"), "rw");
FileOutputStream idx = new FileOutputStream(idxFile); f.setLength(0);
ArrayList<String> l = new ArrayList<String>(); // Write the header.
l.addAll(starts.keySet());
Collections.sort(l); // byte[12] tzdata_version -- 'tzdata2012f\0'
Iterator<String> ziter = l.iterator(); // int file_format_version -- probably won't need this, but just in case
while (ziter.hasNext()) { // int index_offset -- likewise
String zname = ziter.next(); // int data_offset
if (zname.length() >= MAXNAME) { // int zonetab_offset
System.err.println("Error - zone filename exceeds " +
(MAXNAME - 1) + " characters!"); // tzdata_version
f.write(toAscii(new byte[12], version));
// file_format_version
f.writeInt(1);
// Write dummy values for the three offsets, and remember where we need to seek back to later
// when we have the real values.
int index_offset_offset = (int) f.getFilePointer();
f.writeInt(0);
int data_offset_offset = (int) f.getFilePointer();
f.writeInt(0);
int zonetab_offset_offset = (int) f.getFilePointer();
f.writeInt(0);
int index_offset = (int) f.getFilePointer();
// Write the index.
ArrayList<String> sortedOlsonIds = new ArrayList<String>();
sortedOlsonIds.addAll(starts.keySet());
Collections.sort(sortedOlsonIds);
it = sortedOlsonIds.iterator();
while (it.hasNext()) {
String zoneName = it.next();
if (zoneName.length() >= MAXNAME) {
throw new RuntimeException("zone filename too long: " + zoneName.length());
} }
byte[] znameBuf = new byte[MAXNAME]; f.write(toAscii(new byte[MAXNAME], zoneName));
for (int i = 0; i < zname.length(); i++) { f.writeInt(starts.get(zoneName));
znameBuf[i] = (byte)zname.charAt(i); f.writeInt(lengths.get(zoneName));
f.writeInt(offsets.get(zoneName));
} }
idx.write(znameBuf);
writeInt(idx, starts.get(zname).intValue()); int data_offset = (int) f.getFilePointer();
writeInt(idx, lengths.get(zname).intValue());
writeInt(idx, offsets.get(zname).intValue()); // Write the data.
f.write(allData.toByteArray());
// TODO: append the zonetab.
int zonetab_offset = 0;
// Go back and fix up the offsets in the header.
f.seek(index_offset_offset);
f.writeInt(index_offset);
f.seek(data_offset_offset);
f.writeInt(data_offset);
f.seek(zonetab_offset_offset);
f.writeInt(zonetab_offset);
f.close();
} }
idx.close();
OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(new File(outputDirectory, "zoneinfo.version")), "US-ASCII"); private static byte[] toAscii(byte[] dst, String src) {
writer.write(version); for (int i = 0; i < src.length(); ++i) {
writer.write('\n'); if (src.charAt(i) > '~') {
writer.close(); throw new RuntimeException("non-ASCII string: " + src);
}
// System.out.println("maxLength = " + maxLength); dst[i] = (byte) src.charAt(i);
}
return dst;
} }
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {

101
libc/tools/zoneinfo/generate
View File

@ -1,12 +1,10 @@
#!/usr/bin/python #!/usr/bin/python
# Run with no arguments from any directory, with no special setup required.
"""Updates the tzdata file."""
import ftplib import ftplib
import hashlib
import os import os
import re import re
import shutil
import string
import subprocess import subprocess
import sys import sys
import tarfile import tarfile
@ -18,41 +16,58 @@ bionic_libc_tools_dir = os.path.dirname(bionic_libc_tools_zoneinfo_dir)
bionic_libc_dir = os.path.dirname(bionic_libc_tools_dir) bionic_libc_dir = os.path.dirname(bionic_libc_tools_dir)
bionic_dir = os.path.dirname(bionic_libc_dir) bionic_dir = os.path.dirname(bionic_libc_dir)
bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir
if not os.path.isdir(bionic_libc_tools_zoneinfo_dir) or not os.path.isdir(bionic_libc_zoneinfo_dir):
if not os.path.isdir(bionic_libc_tools_zoneinfo_dir):
print "Couldn't find bionic/libc/tools/zoneinfo!" print "Couldn't find bionic/libc/tools/zoneinfo!"
sys.exit(1) sys.exit(1)
if not os.path.isdir(bionic_libc_zoneinfo_dir):
print "Couldn't find bionic/libc/zoneinfo!"
sys.exit(1)
print 'Found bionic in %s...' % bionic_dir print 'Found bionic in %s...' % bionic_dir
regions = ['africa', 'antarctica', 'asia', 'australasia', 'backward', 'etcetera', 'europe', 'northamerica', 'southamerica'] regions = ['africa', 'antarctica', 'asia', 'australasia', 'backward',
'etcetera', 'europe', 'northamerica', 'southamerica']
def current_tzdata_version(): def GetCurrentTzDataVersion():
return open('%s/zoneinfo.version' % bionic_libc_zoneinfo_dir).readline().rstrip('\n') return open('%s/tzdata' % bionic_libc_zoneinfo_dir).read().split('\0', 1)[0]
def md5_file(filename): def WriteSetupFile():
md5 = hashlib.md5() links = []
f = open(filename, 'rb') zones = []
while True: for region in regions:
data = f.read(8192) for line in open('extracted/%s' % region):
if not data: fields = line.split()
break if len(fields) == 0:
md5.update(data) continue
return md5.hexdigest() elif fields[0] == 'Link':
links.append('%s %s %s\n' % (fields[0], fields[1], fields[2]))
zones.append(fields[2])
elif fields[0] == 'Zone':
zones.append(fields[1])
zones.sort()
setup = open('setup', 'w')
for link in links:
setup.write(link)
for zone in zones:
setup.write('%s\n' % zone)
setup.close()
def upgrade_to(ftp, filename): def UpgradeTo(ftp, filename):
version = re.search('tzdata(.+)\.tar\.gz', filename).group(1) new_version = re.search('(tzdata.+)\.tar\.gz', filename).group(1)
# Switch to a temporary directory. # Switch to a temporary directory.
tmp_dir = tempfile.mkdtemp('-tzdata') tmp_dir = tempfile.mkdtemp('-tzdata')
os.chdir(tmp_dir) os.chdir(tmp_dir)
print 'Created temporary directory "%s"...' % tmp_dir print 'Created temporary directory "%s"...' % tmp_dir
print 'Downloading %s...' % filename print 'Downloading...'
ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write) ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write)
print 'MD5: %s' % md5_file(filename)
print 'Extracting...' print 'Extracting...'
os.mkdir('extracted') os.mkdir('extracted')
@ -65,38 +80,20 @@ def upgrade_to(ftp, filename):
if region != 'backward': if region != 'backward':
subprocess.check_call(['zic', '-d', 'data', 'extracted/%s' % region]) subprocess.check_call(['zic', '-d', 'data', 'extracted/%s' % region])
# Collect the data ZoneCompactor needs. WriteSetupFile()
links = []
zones = []
for region in regions:
for line in open('extracted/%s' % region).readlines():
fields = string.split(line)
if len(fields) == 0:
continue
elif fields[0] == 'Link':
links.append('%s %s %s\n' % (fields[0], fields[1], fields[2]))
zones.append(fields[2])
elif fields[0] == 'Zone':
zones.append(fields[1])
zones.sort()
# Write it into the "setup" file. print 'Calling ZoneCompactor to update bionic to %s...' % new_version
setup = open('setup', 'w')
for link in links:
setup.write(link)
for zone in zones:
setup.write('%s\n' % zone)
setup.close()
print 'Calling ZoneCompactor to update bionic from %s to %s...' % (current_tzdata_version(), version)
libcore_src_dir = '%s/../libcore/luni/src/main/java/' % bionic_dir libcore_src_dir = '%s/../libcore/luni/src/main/java/' % bionic_dir
subprocess.check_call(['javac', '-d', '.', subprocess.check_call(['javac', '-d', '.',
'%s/ZoneCompactor.java' % bionic_libc_tools_zoneinfo_dir, '%s/ZoneCompactor.java' % bionic_libc_tools_zoneinfo_dir,
'%s/libcore/util/ZoneInfo.java' % libcore_src_dir, '%s/libcore/util/ZoneInfo.java' % libcore_src_dir,
'%s/libcore/io/BufferIterator.java' % libcore_src_dir]) '%s/libcore/io/BufferIterator.java' % libcore_src_dir])
subprocess.check_call(['java', 'ZoneCompactor', 'setup', 'data', bionic_libc_zoneinfo_dir, version]) subprocess.check_call(['java', 'ZoneCompactor',
'setup', 'data', bionic_libc_zoneinfo_dir, new_version])
# Run with no arguments from any directory, with no special setup required.
def main():
# URL from "Sources for Time Zone and Daylight Saving Time Data" # URL from "Sources for Time Zone and Daylight Saving Time Data"
# http://www.twinsun.com/tz/tz-link.htm # http://www.twinsun.com/tz/tz-link.htm
@ -110,13 +107,19 @@ for filename in ftp.nlst():
tzdata_filenames.append(filename) tzdata_filenames.append(filename)
tzdata_filenames.sort() tzdata_filenames.sort()
# If you're several releases behind, we'll walk you through the upgrades one by one. # If you're several releases behind, we'll walk you through the upgrades
current_version = current_tzdata_version() # one by one.
current_filename = 'tzdata%s.tar.gz' % current_version current_version = GetCurrentTzDataVersion()
current_filename = '%s.tar.gz' % current_version
for filename in tzdata_filenames: for filename in tzdata_filenames:
if filename > current_filename: if filename > current_filename:
upgrade_to(ftp, filename) print 'Found new tzdata: %s' % filename
UpgradeTo(ftp, filename)
sys.exit(0) sys.exit(0)
print 'You already have the latest tzdata (%s)!' % current_version print 'You already have the latest tzdata (%s)!' % current_version
sys.exit(0) sys.exit(0)
if __name__ == '__main__':
main()

77
libc/tools/zoneinfo/generate-single-file
View File

@ -1,77 +0,0 @@
#!/usr/bin/python
# Run with no arguments from any directory, with no special setup required.
import ftplib
import hashlib
import os
import re
import shutil
import string
import struct
import subprocess
import sys
import tarfile
import tempfile
# Find the bionic directory, searching upward from this script.
bionic_libc_tools_zoneinfo_dir = os.path.realpath(os.path.dirname(sys.argv[0]))
bionic_libc_tools_dir = os.path.dirname(bionic_libc_tools_zoneinfo_dir)
bionic_libc_dir = os.path.dirname(bionic_libc_tools_dir)
bionic_dir = os.path.dirname(bionic_libc_dir)
bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir
if not os.path.isdir(bionic_libc_tools_zoneinfo_dir) or not os.path.isdir(bionic_libc_zoneinfo_dir):
print "Couldn't find bionic/libc/tools/zoneinfo!"
sys.exit(1)
def current_tzdata_version():
return open('%s/zoneinfo.version' % bionic_libc_zoneinfo_dir).readline().rstrip('\n')
# TODO: make the regular "generate" script just output this format directly.
# Open the output file.
f = open('%s/tzdata' % bionic_libc_zoneinfo_dir, 'wb+')
# -- header
# char[12] tzdata_version -- 'tzdata2012f\0'
# u32 file_format_version -- probably won't need this, but just in case
# u32 index_offset -- likewise
# u32 data_offset
# u32 zonetab_offset
header_format = "! 12s i i i i"
header_size = struct.calcsize(header_format)
index_offset = header_size
index_bytes = open('%s/zoneinfo.idx' % bionic_libc_zoneinfo_dir, "rb").read()
index_size = len(index_bytes)
data_offset = index_offset + index_size
data_bytes = open('%s/zoneinfo.dat' % bionic_libc_zoneinfo_dir).read()
data_size = len(data_bytes)
zonetab_offset = 0 # TODO: data_offset + data_size
tzdata_version = current_tzdata_version()
file_format_version = 1
header = struct.pack(header_format, 'tzdata%s' % tzdata_version, file_format_version, index_offset, data_offset, zonetab_offset)
f.write(header)
# -- index (@index_offset)
# u8* index_bytes
f.write(index_bytes)
# -- data (@data_offset)
# u8* data_bytes
f.write(data_bytes)
# TODO: zonetab
# -- zonetab (@zonetab_offset)
# u8* zonetab_bytes
f.close()
sys.exit(0)

BIN
libc/zoneinfo/zoneinfo.dat
View File

Binary file not shown.

BIN
libc/zoneinfo/zoneinfo.idx
View File

Binary file not shown.

1
libc/zoneinfo/zoneinfo.version
View File

@ -1 +0,0 @@
2012g