update-tzdata.py revision 90cb5ffb85a9bc2e725824b3ca8db932d02c45db 
1#!/usr/bin/python
2
3"""Updates the timezone data held in bionic and ICU."""
4
5import ftplib
6import glob
7import httplib
8import os
9import re
10import shutil
11import subprocess
12import sys
13import tarfile
14import tempfile
15
16regions = ['africa', 'antarctica', 'asia', 'australasia', 'backward',
17           'etcetera', 'europe', 'northamerica', 'southamerica']
18
19def CheckDirExists(dir, dirname):
20  if not os.path.isdir(dir):
21    print "Couldn't find %s (%s)!" % (dirname, dir)
22    sys.exit(1)
23
24bionic_libc_tools_zoneinfo_dir = os.path.realpath(os.path.dirname(sys.argv[0]))
25
26# Find the bionic directory, searching upward from this script.
27bionic_dir = os.path.realpath('%s/../../..' % bionic_libc_tools_zoneinfo_dir)
28bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir
29CheckDirExists(bionic_libc_zoneinfo_dir, 'bionic/libc/zoneinfo')
30CheckDirExists(bionic_libc_tools_zoneinfo_dir, 'bionic/libc/tools/zoneinfo')
31print 'Found bionic in %s ...' % bionic_dir
32
33# Find the icu4c directory.
34icu_dir = os.path.realpath('%s/../external/icu/icu4c/source' % bionic_dir)
35CheckDirExists(icu_dir, 'external/icu/icu4c/source')
36print 'Found icu in %s ...' % icu_dir
37
38
39def GetCurrentTzDataVersion():
40  return open('%s/tzdata' % bionic_libc_zoneinfo_dir).read().split('\x00', 1)[0]
41
42
43def WriteSetupFile():
44  """Writes the list of zones that ZoneCompactor should process."""
45  links = []
46  zones = []
47  for region in regions:
48    for line in open('extracted/%s' % region):
49      fields = line.split()
50      if fields:
51        if fields[0] == 'Link':
52          links.append('%s %s %s\n' % (fields[0], fields[1], fields[2]))
53          zones.append(fields[2])
54        elif fields[0] == 'Zone':
55          zones.append(fields[1])
56  zones.sort()
57
58  setup = open('setup', 'w')
59  for link in links:
60    setup.write(link)
61  for zone in zones:
62    setup.write('%s\n' % zone)
63  setup.close()
64
65
66def SwitchToNewTemporaryDirectory():
67  tmp_dir = tempfile.mkdtemp('-tzdata')
68  os.chdir(tmp_dir)
69  print 'Created temporary directory "%s"...' % tmp_dir
70
71
72def FtpRetrieveFile(ftp, filename):
73  ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write)
74
75
76def FtpRetrieveFileAndSignature(ftp, data_filename):
77  """Downloads and repackages the given data from the given FTP server."""
78  print 'Downloading data...'
79  FtpRetrieveFile(ftp, data_filename)
80
81  print 'Downloading signature...'
82  signature_filename = '%s.asc' % data_filename
83  FtpRetrieveFile(ftp, signature_filename)
84
85
86def HttpRetrieveFile(http, path, output_filename):
87  http.request("GET", path)
88  f = open(output_filename, 'wb')
89  f.write(http.getresponse().read())
90  f.close()
91
92
93def HttpRetrieveFileAndSignature(http, data_filename):
94  """Downloads and repackages the given data from the given HTTP server."""
95  path = "/time-zones/repository/releases/%s" % data_filename
96
97  print 'Downloading data...'
98  HttpRetrieveFile(http, path, data_filename)
99
100  print 'Downloading signature...'
101  signature_filename = '%s.asc' % data_filename
102  HttpRetrievefile(http, "%s.asc" % path, signature_filename)
103
104
105def BuildIcuToolsAndData(data_filename):
106  # Keep track of the original cwd so we can go back to it at the end.
107  original_working_dir = os.getcwd()
108
109  # Create a directory to run 'make' from.
110  icu_working_dir = '%s/icu' % original_working_dir
111  os.mkdir(icu_working_dir)
112  os.chdir(icu_working_dir)
113
114  # Build the ICU tools.
115  print 'Configuring ICU tools...'
116  subprocess.check_call(['%s/runConfigureICU' % icu_dir, 'Linux'])
117  print 'Making ICU tools...'
118  subprocess.check_call(['make', '-j32'])
119
120  # Run the ICU tools.
121  os.chdir('tools/tzcode')
122  shutil.copyfile('%s/%s' % (original_working_dir, data_filename), data_filename)
123  print 'Making ICU data...'
124  subprocess.check_call(['make'])
125
126  # Copy the output files to their ultimate destination.
127  icu_txt_data_dir = '%s/data/misc' % icu_dir
128  print 'Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir
129  shutil.copy('zoneinfo64.txt', icu_txt_data_dir)
130
131  os.chdir(icu_working_dir)
132  icu_dat_data_dir = '%s/stubdata' % icu_dir
133  datfiles = glob.glob('data/out/tmp/icudt??l.dat')
134  if len(datfiles) != 1:
135    print 'ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles)
136    sys.exit(1)
137
138  datfile = datfiles[0]
139  print 'Copying %s to %s ...' % (datfile, icu_dat_data_dir)
140  shutil.copy(datfile, icu_dat_data_dir)
141
142  # Switch back to the original working cwd.
143  os.chdir(original_working_dir)
144
145
146def CheckSignature(data_filename):
147  signature_filename = '%s.asc' % data_filename
148  print 'Verifying signature...'
149  # If this fails for you, you probably need to import Paul Eggert's public key:
150  # gpg --recv-keys ED97E90E62AA7E34
151  subprocess.check_call(['gpg', '--trusted-key=ED97E90E62AA7E34', '--verify',
152                         signature_filename, data_filename])
153
154
155def BuildBionicToolsAndData(data_filename):
156  new_version = re.search('(tzdata.+)\\.tar\\.gz', data_filename).group(1)
157
158  print 'Extracting...'
159  os.mkdir('extracted')
160  tar = tarfile.open(data_filename, 'r')
161  tar.extractall('extracted')
162
163  print 'Calling zic(1)...'
164  os.mkdir('data')
165  for region in regions:
166    if region != 'backward':
167      subprocess.check_call(['zic', '-d', 'data', 'extracted/%s' % region])
168
169  WriteSetupFile()
170
171  print 'Calling ZoneCompactor to update bionic to %s...' % new_version
172  subprocess.check_call(['javac', '-d', '.',
173                         '%s/ZoneCompactor.java' % bionic_libc_tools_zoneinfo_dir])
174  subprocess.check_call(['java', 'ZoneCompactor',
175                         'setup', 'data', 'extracted/zone.tab',
176                         bionic_libc_zoneinfo_dir, new_version])
177
178
179# Run with no arguments from any directory, with no special setup required.
180# See http://www.iana.org/time-zones/ for more about the source of this data.
181def main():
182  print 'Looking for new tzdata...'
183
184  tzdata_filenames = []
185
186  # The FTP server lets you download intermediate releases, and also lets you
187  # download the signatures for verification, so it's your best choice.
188  use_ftp = True
189
190  if use_ftp:
191    ftp = ftplib.FTP('ftp.iana.org')
192    ftp.login()
193    ftp.cwd('tz/releases')
194    for filename in ftp.nlst():
195      if filename.startswith('tzdata20') and filename.endswith('.tar.gz'):
196        tzdata_filenames.append(filename)
197    tzdata_filenames.sort()
198  else:
199    http = httplib.HTTPConnection('www.iana.org')
200    http.request("GET", "/time-zones")
201    index_lines = http.getresponse().read().split('\n')
202    for line in index_lines:
203      m = re.compile('.*href="/time-zones/repository/releases/(tzdata20\d\d\c\.tar\.gz)".*').match(line)
204      if m:
205        tzdata_filenames.append(m.group(1))
206
207  # If you're several releases behind, we'll walk you through the upgrades
208  # one by one.
209  current_version = GetCurrentTzDataVersion()
210  current_filename = '%s.tar.gz' % current_version
211  for filename in tzdata_filenames:
212    if filename > current_filename:
213      print 'Found new tzdata: %s' % filename
214      SwitchToNewTemporaryDirectory()
215      if use_ftp:
216        FtpRetrieveFileAndSignature(ftp, filename)
217      else:
218        HttpRetrieveFileAndSignature(http, filename)
219
220      CheckSignature(filename)
221      BuildIcuToolsAndData(filename)
222      BuildBionicToolsAndData(filename)
223      print 'Look in %s and %s for new data files' % (bionic_dir, icu_dir)
224      sys.exit(0)
225
226  print 'You already have the latest tzdata (%s)!' % current_version
227  sys.exit(0)
228
229
230if __name__ == '__main__':
231  main()
232