update-tzdata.py revision 246c6880207539d01e84a6f049cd639139a0691a
1#!/usr/bin/python 2 3"""Updates the timezone data held in bionic and ICU.""" 4 5import ftplib 6import glob 7import httplib 8import os 9import re 10import shutil 11import subprocess 12import sys 13import tarfile 14import tempfile 15 16regions = ['africa', 'antarctica', 'asia', 'australasia', 'backward', 17 'etcetera', 'europe', 'northamerica', 'southamerica'] 18 19def CheckDirExists(dir, dirname): 20 if not os.path.isdir(dir): 21 print "Couldn't find %s (%s)!" % (dirname, dir) 22 sys.exit(1) 23 24bionic_libc_tools_zoneinfo_dir = os.path.realpath(os.path.dirname(sys.argv[0])) 25 26# Find the bionic directory, searching upward from this script. 27bionic_dir = os.path.realpath('%s/../../..' % bionic_libc_tools_zoneinfo_dir) 28bionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir 29CheckDirExists(bionic_libc_zoneinfo_dir, 'bionic/libc/zoneinfo') 30CheckDirExists(bionic_libc_tools_zoneinfo_dir, 'bionic/libc/tools/zoneinfo') 31print 'Found bionic in %s ...' % bionic_dir 32 33# Find the icu4c directory. 34icu_dir = os.path.realpath('%s/../external/icu4c' % bionic_dir) 35CheckDirExists(icu_dir, 'external/icu4c') 36print 'Found icu in %s ...' % icu_dir 37 38 39def GetCurrentTzDataVersion(): 40 return open('%s/tzdata' % bionic_libc_zoneinfo_dir).read().split('\x00', 1)[0] 41 42 43def WriteSetupFile(): 44 """Writes the list of zones that ZoneCompactor should process.""" 45 links = [] 46 zones = [] 47 for region in regions: 48 for line in open('extracted/%s' % region): 49 fields = line.split() 50 if fields: 51 if fields[0] == 'Link': 52 links.append('%s %s %s\n' % (fields[0], fields[1], fields[2])) 53 zones.append(fields[2]) 54 elif fields[0] == 'Zone': 55 zones.append(fields[1]) 56 zones.sort() 57 58 setup = open('setup', 'w') 59 for link in links: 60 setup.write(link) 61 for zone in zones: 62 setup.write('%s\n' % zone) 63 setup.close() 64 65 66def SwitchToNewTemporaryDirectory(): 67 tmp_dir = tempfile.mkdtemp('-tzdata') 68 os.chdir(tmp_dir) 69 print 'Created temporary directory "%s"...' % tmp_dir 70 71 72def FtpRetrieveFile(ftp, filename): 73 ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write) 74 75 76def FtpRetrieveFileAndSignature(ftp, data_filename): 77 """Downloads and repackages the given data from the given FTP server.""" 78 print 'Downloading data...' 79 FtpRetrieveFile(ftp, data_filename) 80 81 print 'Downloading signature...' 82 signature_filename = '%s.asc' % data_filename 83 FtpRetrieveFile(ftp, signature_filename) 84 85 86def HttpRetrieveFile(http, path, output_filename): 87 http.request("GET", path) 88 f = open(output_filename, 'wb') 89 f.write(http.getresponse().read()) 90 f.close() 91 92 93def HttpRetrieveFileAndSignature(http, data_filename): 94 """Downloads and repackages the given data from the given HTTP server.""" 95 path = "/time-zones/repository/releases/%s" % data_filename 96 97 print 'Downloading data...' 98 HttpRetrieveFile(http, path, data_filename) 99 100 print 'Downloading signature...' 101 signature_filename = '%s.asc' % data_filename 102 HttpRetrievefile(http, "%s.asc" % path, signature_filename) 103 104 105def BuildIcuToolsAndData(data_filename): 106 # Keep track of the original cwd so we can go back to it at the end. 107 original_working_dir = os.getcwd() 108 109 # Create a directory to run 'make' from. 110 icu_working_dir = '%s/icu' % original_working_dir 111 os.mkdir(icu_working_dir) 112 os.chdir(icu_working_dir) 113 114 # Build the ICU tools. 115 print 'Configuring ICU tools...' 116 subprocess.check_call(['%s/runConfigureICU' % icu_dir, 'Linux']) 117 print 'Making ICU tools...' 118 subprocess.check_call(['make', '-j6']) 119 120 # Run the ICU tools. 121 os.chdir('tools/tzcode') 122 shutil.copyfile('%s/%s' % (original_working_dir, data_filename), data_filename) 123 print 'Making ICU data...' 124 subprocess.check_call(['make']) 125 126 # Copy the output files to their ultimate destination. 127 icu_txt_data_dir = '%s/data/misc' % icu_dir 128 print 'Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir 129 shutil.copy('zoneinfo64.txt', icu_txt_data_dir) 130 131 os.chdir(icu_working_dir) 132 icu_dat_data_dir = '%s/stubdata' % icu_dir 133 for file in glob.glob('data/out/tmp/*.dat'): 134 print 'Copying %s to %s ...' % (file, icu_dat_data_dir) 135 shutil.copy(file, icu_dat_data_dir) 136 137 # Switch back to the original working cwd. 138 os.chdir(original_working_dir) 139 140 141def CheckSignature(data_filename): 142 signature_filename = '%s.asc' % data_filename 143 print 'Verifying signature...' 144 # If this fails for you, you probably need to import Paul Eggert's public key: 145 # gpg --recv-keys ED97E90E62AA7E34 146 subprocess.check_call(['gpg', '--trusted-key=ED97E90E62AA7E34', '--verify', 147 signature_filename, data_filename]) 148 149 150def BuildBionicToolsAndData(data_filename): 151 new_version = re.search('(tzdata.+)\\.tar\\.gz', data_filename).group(1) 152 153 print 'Extracting...' 154 os.mkdir('extracted') 155 tar = tarfile.open(data_filename, 'r') 156 tar.extractall('extracted') 157 158 print 'Calling zic(1)...' 159 os.mkdir('data') 160 for region in regions: 161 if region != 'backward': 162 subprocess.check_call(['zic', '-d', 'data', 'extracted/%s' % region]) 163 164 WriteSetupFile() 165 166 print 'Calling ZoneCompactor to update bionic to %s...' % new_version 167 libcore_src_dir = '%s/../libcore/luni/src/main/java/' % bionic_dir 168 subprocess.check_call(['javac', '-d', '.', 169 '%s/ZoneCompactor.java' % bionic_libc_tools_zoneinfo_dir, 170 '%s/libcore/util/ZoneInfo.java' % libcore_src_dir, 171 '%s/libcore/io/BufferIterator.java' % libcore_src_dir]) 172 subprocess.check_call(['java', 'ZoneCompactor', 173 'setup', 'data', 'extracted/zone.tab', 174 bionic_libc_zoneinfo_dir, new_version]) 175 176 177# Run with no arguments from any directory, with no special setup required. 178# See http://www.iana.org/time-zones/ for more about the source of this data. 179def main(): 180 print 'Looking for new tzdata...' 181 182 tzdata_filenames = [] 183 184 # The FTP server lets you download intermediate releases, and also lets you 185 # download the signatures for verification, so it's your best choice. 186 use_ftp = True 187 188 if use_ftp: 189 ftp = ftplib.FTP('ftp.iana.org') 190 ftp.login() 191 ftp.cwd('tz/releases') 192 for filename in ftp.nlst(): 193 if filename.startswith('tzdata20') and filename.endswith('.tar.gz'): 194 tzdata_filenames.append(filename) 195 tzdata_filenames.sort() 196 else: 197 http = httplib.HTTPConnection('www.iana.org') 198 http.request("GET", "/time-zones") 199 index_lines = http.getresponse().read().split('\n') 200 for line in index_lines: 201 m = re.compile('.*href="/time-zones/repository/releases/(tzdata20\d\d\c\.tar\.gz)".*').match(line) 202 if m: 203 tzdata_filenames.append(m.group(1)) 204 205 # If you're several releases behind, we'll walk you through the upgrades 206 # one by one. 207 current_version = GetCurrentTzDataVersion() 208 current_filename = '%s.tar.gz' % current_version 209 for filename in tzdata_filenames: 210 if filename > current_filename: 211 print 'Found new tzdata: %s' % filename 212 SwitchToNewTemporaryDirectory() 213 if use_ftp: 214 FtpRetrieveFileAndSignature(ftp, filename) 215 else: 216 HttpRetrieveFileAndSignature(http, filename) 217 218 CheckSignature(filename) 219 BuildIcuToolsAndData(filename) 220 BuildBionicToolsAndData(filename) 221 print 'Look in %s and %s for new data files' % (bionic_dir, icu_dir) 222 sys.exit(0) 223 224 print 'You already have the latest tzdata (%s)!' % current_version 225 sys.exit(0) 226 227 228if __name__ == '__main__': 229 main() 230