1d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller#!/usr/bin/python 2d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 3d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller"""Updates the timezone data held in bionic and ICU.""" 4d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 5d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport ftplib 6d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport glob 7d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport httplib 8d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport os 9d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport re 10d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport shutil 11d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport subprocess 12d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport sys 13d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerimport tarfile 146a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller 156a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fullerimport i18nutil 166a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fullerimport updateicudata 17d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 18d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerregions = ['africa', 'antarctica', 'asia', 'australasia', 19d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 'etcetera', 'europe', 'northamerica', 'southamerica', 20d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller # These two deliberately come last so they override what came 21d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller # before (and each other). 22d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 'backward', 'backzone' ] 23d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 246a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller# Find the bionic directory. 256a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fullerandroid_build_top = i18nutil.GetAndroidRootOrDie() 26d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerbionic_dir = os.path.realpath('%s/bionic' % android_build_top) 27d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerbionic_libc_zoneinfo_dir = '%s/libc/zoneinfo' % bionic_dir 286a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fulleri18nutil.CheckDirExists(bionic_libc_zoneinfo_dir, 'bionic/libc/zoneinfo') 29d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullertools_dir = '%s/external/icu/tools' % android_build_top 306a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fulleri18nutil.CheckDirExists(tools_dir, 'external/icu/tools') 31d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 32d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef GetCurrentTzDataVersion(): 33d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller return open('%s/tzdata' % bionic_libc_zoneinfo_dir).read().split('\x00', 1)[0] 34d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 35d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 36d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef WriteSetupFile(): 37d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller """Writes the list of zones that ZoneCompactor should process.""" 38d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller links = [] 39d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller zones = [] 40d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller for region in regions: 41d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller for line in open('extracted/%s' % region): 42d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller fields = line.split() 43d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller if fields: 44d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller if fields[0] == 'Link': 45d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller links.append('%s %s %s' % (fields[0], fields[1], fields[2])) 46d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller zones.append(fields[2]) 47d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller elif fields[0] == 'Zone': 48d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller zones.append(fields[1]) 49d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller zones.sort() 50d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 51d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller setup = open('setup', 'w') 52d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller for link in sorted(set(links)): 53d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller setup.write('%s\n' % link) 54d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller for zone in sorted(set(zones)): 55d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller setup.write('%s\n' % zone) 56d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller setup.close() 57d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 58d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 59d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef FtpRetrieveFile(ftp, filename): 60d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller ftp.retrbinary('RETR %s' % filename, open(filename, 'wb').write) 61d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 62d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 63d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef FtpRetrieveFileAndSignature(ftp, data_filename): 64d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller """Downloads and repackages the given data from the given FTP server.""" 65d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Downloading data...' 66d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller FtpRetrieveFile(ftp, data_filename) 67d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 68d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Downloading signature...' 69d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller signature_filename = '%s.asc' % data_filename 70d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller FtpRetrieveFile(ftp, signature_filename) 71d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 72d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 73d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef HttpRetrieveFile(http, path, output_filename): 74d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller http.request("GET", path) 75d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller f = open(output_filename, 'wb') 76d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller f.write(http.getresponse().read()) 77d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller f.close() 78d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 79d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 80d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef HttpRetrieveFileAndSignature(http, data_filename): 81d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller """Downloads and repackages the given data from the given HTTP server.""" 82d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller path = "/time-zones/repository/releases/%s" % data_filename 83d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 84d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Downloading data...' 85d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller HttpRetrieveFile(http, path, data_filename) 86d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 87d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Downloading signature...' 88d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller signature_filename = '%s.asc' % data_filename 89d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller HttpRetrievefile(http, "%s.asc" % path, signature_filename) 90d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 91d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef BuildIcuToolsAndData(data_filename): 926a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller icu_build_dir = '%s/icu' % os.getcwd() 93d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 946a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller updateicudata.PrepareIcuBuild(icu_build_dir) 956a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller updateicudata.MakeTzDataFiles(icu_build_dir, data_filename) 966a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller updateicudata.MakeAndCopyIcuDataFiles(icu_build_dir) 97d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 98d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef CheckSignature(data_filename): 99d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller signature_filename = '%s.asc' % data_filename 100d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Verifying signature...' 101d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller # If this fails for you, you probably need to import Paul Eggert's public key: 102d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller # gpg --recv-keys ED97E90E62AA7E34 103d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller subprocess.check_call(['gpg', '--trusted-key=ED97E90E62AA7E34', '--verify', 104d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller signature_filename, data_filename]) 105d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 106d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 107d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef BuildBionicToolsAndData(data_filename): 108d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller new_version = re.search('(tzdata.+)\\.tar\\.gz', data_filename).group(1) 109d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 110d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Extracting...' 111d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller os.mkdir('extracted') 112d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller tar = tarfile.open(data_filename, 'r') 113d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller tar.extractall('extracted') 114d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 115d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Calling zic(1)...' 116d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller os.mkdir('data') 117d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller zic_inputs = [ 'extracted/%s' % x for x in regions ] 118d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller zic_cmd = ['zic', '-d', 'data' ] 119d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller zic_cmd.extend(zic_inputs) 120d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller subprocess.check_call(zic_cmd) 121d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 122d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller WriteSetupFile() 123d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 124d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Calling ZoneCompactor to update bionic to %s...' % new_version 125d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller subprocess.check_call(['javac', '-d', '.', 126d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller '%s/ZoneCompactor.java' % tools_dir]) 127d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller subprocess.check_call(['java', 'ZoneCompactor', 128d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 'setup', 'data', 'extracted/zone.tab', 129d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller bionic_libc_zoneinfo_dir, new_version]) 130d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 131d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 132d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller# Run with no arguments from any directory, with no special setup required. 133d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller# See http://www.iana.org/time-zones/ for more about the source of this data. 134d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerdef main(): 1356a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller print 'Found bionic in %s ...' % bionic_dir 1366a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller print 'Found icu in %s ...' % updateicudata.icuDir() 1376a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller 138d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Looking for new tzdata...' 139d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 140d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller tzdata_filenames = [] 141d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 142d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller # The FTP server lets you download intermediate releases, and also lets you 143d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller # download the signatures for verification, so it's your best choice. 144d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller use_ftp = True 145d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 146d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller if use_ftp: 147d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller ftp = ftplib.FTP('ftp.iana.org') 148d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller ftp.login() 149d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller ftp.cwd('tz/releases') 150d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller for filename in ftp.nlst(): 151d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller if filename.startswith('tzdata20') and filename.endswith('.tar.gz'): 152d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller tzdata_filenames.append(filename) 153d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller tzdata_filenames.sort() 154d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller else: 155d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller http = httplib.HTTPConnection('www.iana.org') 156d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller http.request("GET", "/time-zones") 157d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller index_lines = http.getresponse().read().split('\n') 158d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller for line in index_lines: 159d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller m = re.compile('.*href="/time-zones/repository/releases/(tzdata20\d\d\c\.tar\.gz)".*').match(line) 160d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller if m: 161d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller tzdata_filenames.append(m.group(1)) 162d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 163d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller # If you're several releases behind, we'll walk you through the upgrades 164d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller # one by one. 165d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller current_version = GetCurrentTzDataVersion() 166d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller current_filename = '%s.tar.gz' % current_version 167d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller for filename in tzdata_filenames: 168d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller if filename > current_filename: 169d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller print 'Found new tzdata: %s' % filename 1706a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller i18nutil.SwitchToNewTemporaryDirectory() 171d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller if use_ftp: 172d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller FtpRetrieveFileAndSignature(ftp, filename) 173d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller else: 174d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller HttpRetrieveFileAndSignature(http, filename) 175d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 176d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller CheckSignature(filename) 177d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller BuildIcuToolsAndData(filename) 178d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller BuildBionicToolsAndData(filename) 1796a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller print 'Look in %s and %s for new data files' % (bionic_dir, updateicudata.icuDir()) 180d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller sys.exit(0) 181d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 1826a1d94f8bd51a1f45fdb8f0b3efc5337c0d408dfNeil Fuller print 'You already have the latest tzdata in bionic (%s)!' % current_version 183d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller sys.exit(0) 184d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 185d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller 186d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fullerif __name__ == '__main__': 187d23839d4d460aa5f81eeb4986c05c8745e0633b7Neil Fuller main() 188