1#!/usr/bin/python
2
3"""Updates the timezone data held in bionic and ICU."""
4
5import ftplib
6import glob
7import httplib
8import os
9import re
10import shutil
11import subprocess
12import sys
13import tarfile
14import tempfile
15
16regions = ['africa', 'antarctica', 'asia', 'australasia',
17           'etcetera', 'europe', 'northamerica', 'southamerica',
18           # These two deliberately come last so they override what came
19           # before (and each other).
20           'backward', 'backzone' ]
21
22def CheckDirExists(dir, dirname):
23  if not os.path.isdir(dir):
24    print "Couldn't find %s (%s)!" % (dirname, dir)
25    sys.exit(1)
26
27bionic_libc_tools_zoneinfo_dir = os.path.realpath(os.path.dirname(sys.argv[0]))
28
29# Find the bionic directory, searching upward from this script.
30bionic_dir = os.path.realpath('%s/../../..' % bionic_libc_tools_zoneinfo_dir)
31bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir
32CheckDirExists(bionic_libc_zoneinfo_dir, 'bionic/libc/zoneinfo')
33CheckDirExists(bionic_libc_tools_zoneinfo_dir, 'bionic/libc/tools/zoneinfo')
34print 'Found bionic in %s ...' % bionic_dir
35
36# Find the icu directory.
37icu_dir = os.path.realpath('%s/../external/icu' % bionic_dir)
38icu4c_dir = os.path.realpath('%s/icu4c/source' % icu_dir)
39icu4j_dir = os.path.realpath('%s/icu4j' % icu_dir)
40CheckDirExists(icu4c_dir, 'external/icu/icu4c/source')
41CheckDirExists(icu4j_dir, 'external/icu/icu4j')
42print 'Found icu in %s ...' % icu_dir
43
44
45def GetCurrentTzDataVersion():
46  return open('%s/tzdata' % bionic_libc_zoneinfo_dir).read().split('\x00', 1)[0]
47
48
49def WriteSetupFile():
50  """Writes the list of zones that ZoneCompactor should process."""
51  links = []
52  zones = []
53  for region in regions:
54    for line in open('extracted/%s' % region):
55      fields = line.split()
56      if fields:
57        if fields[0] == 'Link':
58          links.append('%s %s %s' % (fields[0], fields[1], fields[2]))
59          zones.append(fields[2])
60        elif fields[0] == 'Zone':
61          zones.append(fields[1])
62  zones.sort()
63
64  setup = open('setup', 'w')
65  for link in sorted(set(links)):
66    setup.write('%s\n' % link)
67  for zone in sorted(set(zones)):
68    setup.write('%s\n' % zone)
69  setup.close()
70
71
72def SwitchToNewTemporaryDirectory():
73  tmp_dir = tempfile.mkdtemp('-tzdata')
74  os.chdir(tmp_dir)
75  print 'Created temporary directory "%s"...' % tmp_dir
76
77
78def FtpRetrieveFile(ftp, filename):
79  ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write)
80
81
82def FtpRetrieveFileAndSignature(ftp, data_filename):
83  """Downloads and repackages the given data from the given FTP server."""
84  print 'Downloading data...'
85  FtpRetrieveFile(ftp, data_filename)
86
87  print 'Downloading signature...'
88  signature_filename = '%s.asc' % data_filename
89  FtpRetrieveFile(ftp, signature_filename)
90
91
92def HttpRetrieveFile(http, path, output_filename):
93  http.request("GET", path)
94  f = open(output_filename, 'wb')
95  f.write(http.getresponse().read())
96  f.close()
97
98
99def HttpRetrieveFileAndSignature(http, data_filename):
100  """Downloads and repackages the given data from the given HTTP server."""
101  path = "/time-zones/repository/releases/%s" % data_filename
102
103  print 'Downloading data...'
104  HttpRetrieveFile(http, path, data_filename)
105
106  print 'Downloading signature...'
107  signature_filename = '%s.asc' % data_filename
108  HttpRetrievefile(http, "%s.asc" % path, signature_filename)
109
110
111def BuildIcuToolsAndData(data_filename):
112  # Keep track of the original cwd so we can go back to it at the end.
113  original_working_dir = os.getcwd()
114
115  # Create a directory to run 'make' from.
116  icu_working_dir = '%s/icu' % original_working_dir
117  os.mkdir(icu_working_dir)
118  os.chdir(icu_working_dir)
119
120  # Build the ICU tools.
121  print 'Configuring ICU tools...'
122  subprocess.check_call(['%s/runConfigureICU' % icu4c_dir, 'Linux'])
123
124  # Run the ICU tools.
125  os.chdir('tools/tzcode')
126
127  # The tz2icu tool only picks up icuregions and icuzones in they are in the CWD
128  for icu_data_file in [ 'icuregions', 'icuzones']:
129    icu_data_file_source = '%s/tools/tzcode/%s' % (icu4c_dir, icu_data_file)
130    icu_data_file_symlink = './%s' % icu_data_file
131    os.symlink(icu_data_file_source, icu_data_file_symlink)
132
133  shutil.copyfile('%s/%s' % (original_working_dir, data_filename), data_filename)
134  print 'Making ICU data...'
135  # The Makefile assumes the existence of the bin directory.
136  os.mkdir('%s/bin' % icu_working_dir)
137  subprocess.check_call(['make'])
138
139  # Copy the source file to its ultimate destination.
140  icu_txt_data_dir = '%s/data/misc' % icu4c_dir
141  print 'Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir
142  shutil.copy('zoneinfo64.txt', icu_txt_data_dir)
143
144  # Regenerate the .dat file.
145  os.chdir(icu_working_dir)
146  subprocess.check_call(['make', 'INCLUDE_UNI_CORE_DATA=1', '-j32'])
147
148  # Copy the .dat file to its ultimate destination.
149  icu_dat_data_dir = '%s/stubdata' % icu4c_dir
150  datfiles = glob.glob('data/out/tmp/icudt??l.dat')
151  if len(datfiles) != 1:
152    print 'ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles)
153    sys.exit(1)
154  datfile = datfiles[0]
155  print 'Copying %s to %s ...' % (datfile, icu_dat_data_dir)
156  shutil.copy(datfile, icu_dat_data_dir)
157
158  # Generate the ICU4J .jar files
159  os.chdir('%s/data' % icu_working_dir)
160  subprocess.check_call(['make', 'icu4j-data'])
161
162  # Copy the ICU4J .jar files to their ultimate destination.
163  icu_jar_data_dir = '%s/main/shared/data' % icu4j_dir
164  jarfiles = glob.glob('out/icu4j/*.jar')
165  if len(jarfiles) != 2:
166    print 'ERROR: Unexpectedly found %d .jar files (%s). Halting.' % (len(jarfiles), jarfiles)
167    sys.exit(1)
168  for jarfile in jarfiles:
169    print 'Copying %s to %s ...' % (jarfile, icu_jar_data_dir)
170    shutil.copy(jarfile, icu_jar_data_dir)
171
172  # Switch back to the original working cwd.
173  os.chdir(original_working_dir)
174
175
176def CheckSignature(data_filename):
177  signature_filename = '%s.asc' % data_filename
178  print 'Verifying signature...'
179  # If this fails for you, you probably need to import Paul Eggert's public key:
180  # gpg --recv-keys ED97E90E62AA7E34
181  subprocess.check_call(['gpg', '--trusted-key=ED97E90E62AA7E34', '--verify',
182                         signature_filename, data_filename])
183
184
185def BuildBionicToolsAndData(data_filename):
186  new_version = re.search('(tzdata.+)\\.tar\\.gz', data_filename).group(1)
187
188  print 'Extracting...'
189  os.mkdir('extracted')
190  tar = tarfile.open(data_filename, 'r')
191  tar.extractall('extracted')
192
193  print 'Calling zic(1)...'
194  os.mkdir('data')
195  zic_inputs = [ 'extracted/%s' % x for x in regions ]
196  zic_cmd = ['zic', '-d', 'data' ]
197  zic_cmd.extend(zic_inputs)
198  subprocess.check_call(zic_cmd)
199
200  WriteSetupFile()
201
202  print 'Calling ZoneCompactor to update bionic to %s...' % new_version
203  subprocess.check_call(['javac', '-d', '.',
204                         '%s/ZoneCompactor.java' % bionic_libc_tools_zoneinfo_dir])
205  subprocess.check_call(['java', 'ZoneCompactor',
206                         'setup', 'data', 'extracted/zone.tab',
207                         bionic_libc_zoneinfo_dir, new_version])
208
209
210# Run with no arguments from any directory, with no special setup required.
211# See http://www.iana.org/time-zones/ for more about the source of this data.
212def main():
213  print 'Looking for new tzdata...'
214
215  tzdata_filenames = []
216
217  # The FTP server lets you download intermediate releases, and also lets you
218  # download the signatures for verification, so it's your best choice.
219  use_ftp = True
220
221  if use_ftp:
222    ftp = ftplib.FTP('ftp.iana.org')
223    ftp.login()
224    ftp.cwd('tz/releases')
225    for filename in ftp.nlst():
226      if filename.startswith('tzdata20') and filename.endswith('.tar.gz'):
227        tzdata_filenames.append(filename)
228    tzdata_filenames.sort()
229  else:
230    http = httplib.HTTPConnection('www.iana.org')
231    http.request("GET", "/time-zones")
232    index_lines = http.getresponse().read().split('\n')
233    for line in index_lines:
234      m = re.compile('.*href="/time-zones/repository/releases/(tzdata20\d\d\c\.tar\.gz)".*').match(line)
235      if m:
236        tzdata_filenames.append(m.group(1))
237
238  # If you're several releases behind, we'll walk you through the upgrades
239  # one by one.
240  current_version = GetCurrentTzDataVersion()
241  current_filename = '%s.tar.gz' % current_version
242  for filename in tzdata_filenames:
243    if filename > current_filename:
244      print 'Found new tzdata: %s' % filename
245      SwitchToNewTemporaryDirectory()
246      if use_ftp:
247        FtpRetrieveFileAndSignature(ftp, filename)
248      else:
249        HttpRetrieveFileAndSignature(http, filename)
250
251      CheckSignature(filename)
252      BuildIcuToolsAndData(filename)
253      BuildBionicToolsAndData(filename)
254      print 'Look in %s and %s for new data files' % (bionic_dir, icu_dir)
255      sys.exit(0)
256
257  print 'You already have the latest tzdata (%s)!' % current_version
258  sys.exit(0)
259
260
261if __name__ == '__main__':
262  main()
263