common.py revision 1684d9c0cf9f31a360e0e73dd6ed1e335d97f405
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import re 22import sha 23import shutil 24import subprocess 25import sys 26import tempfile 27import threading 28import time 29import zipfile 30 31# missing in Python 2.4 and before 32if not hasattr(os, "SEEK_SET"): 33 os.SEEK_SET = 0 34 35class Options(object): pass 36OPTIONS = Options() 37OPTIONS.search_path = "out/host/linux-x86" 38OPTIONS.verbose = False 39OPTIONS.tempfiles = [] 40OPTIONS.device_specific = None 41OPTIONS.extras = {} 42OPTIONS.info_dict = None 43 44 45# Values for "certificate" in apkcerts that mean special things. 46SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 47 48 49class ExternalError(RuntimeError): pass 50 51 52def Run(args, **kwargs): 53 """Create and return a subprocess.Popen object, printing the command 54 line on the terminal if -v was specified.""" 55 if OPTIONS.verbose: 56 print " running: ", " ".join(args) 57 return subprocess.Popen(args, **kwargs) 58 59 60def LoadInfoDict(zip): 61 """Read and parse the META/misc_info.txt key/value pairs from the 62 input target files and return a dict.""" 63 64 d = {} 65 try: 66 for line in zip.read("META/misc_info.txt").split("\n"): 67 line = line.strip() 68 if not line or line.startswith("#"): continue 69 k, v = line.split("=", 1) 70 d[k] = v 71 except KeyError: 72 # ok if misc_info.txt doesn't exist 73 pass 74 75 if "fs_type" not in d: d["fs_type"] = "yaffs2" 76 if "partition_type" not in d: d["partition_type"] = "MTD" 77 78 # backwards compatibility: These values used to be in their own 79 # files. Look for them, in case we're processing an old 80 # target_files zip. 81 82 if "mkyaffs2_extra_flags" not in d: 83 try: 84 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 85 except KeyError: 86 # ok if flags don't exist 87 pass 88 89 if "recovery_api_version" not in d: 90 try: 91 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 92 except KeyError: 93 raise ValueError("can't find recovery API version in input target-files") 94 95 if "tool_extensions" not in d: 96 try: 97 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 98 except KeyError: 99 # ok if extensions don't exist 100 pass 101 102 try: 103 data = zip.read("META/imagesizes.txt") 104 for line in data.split("\n"): 105 if not line: continue 106 name, value = line.split(" ", 1) 107 if not value: continue 108 if name == "blocksize": 109 d[name] = value 110 else: 111 d[name + "_size"] = value 112 except KeyError: 113 pass 114 115 def makeint(key): 116 if key in d: 117 d[key] = int(d[key], 0) 118 119 makeint("recovery_api_version") 120 makeint("blocksize") 121 makeint("system_size") 122 makeint("userdata_size") 123 makeint("recovery_size") 124 makeint("boot_size") 125 126 return d 127 128def DumpInfoDict(d): 129 for k, v in sorted(d.items()): 130 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 131 132def BuildAndAddBootableImage(sourcedir, targetname, output_zip, info_dict): 133 """Take a kernel, cmdline, and ramdisk directory from the input (in 134 'sourcedir'), and turn them into a boot image. Put the boot image 135 into the output zip file under the name 'targetname'. Returns 136 targetname on success or None on failure (if sourcedir does not 137 appear to contain files for the requested image).""" 138 139 print "creating %s..." % (targetname,) 140 141 img = BuildBootableImage(sourcedir) 142 if img is None: 143 return None 144 145 CheckSize(img, targetname, info_dict) 146 ZipWriteStr(output_zip, targetname, img) 147 return targetname 148 149def BuildBootableImage(sourcedir): 150 """Take a kernel, cmdline, and ramdisk directory from the input (in 151 'sourcedir'), and turn them into a boot image. Return the image 152 data, or None if sourcedir does not appear to contains files for 153 building the requested image.""" 154 155 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 156 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 157 return None 158 159 ramdisk_img = tempfile.NamedTemporaryFile() 160 img = tempfile.NamedTemporaryFile() 161 162 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 163 stdout=subprocess.PIPE) 164 p2 = Run(["minigzip"], 165 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 166 167 p2.wait() 168 p1.wait() 169 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 170 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 171 172 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 173 174 fn = os.path.join(sourcedir, "cmdline") 175 if os.access(fn, os.F_OK): 176 cmd.append("--cmdline") 177 cmd.append(open(fn).read().rstrip("\n")) 178 179 fn = os.path.join(sourcedir, "base") 180 if os.access(fn, os.F_OK): 181 cmd.append("--base") 182 cmd.append(open(fn).read().rstrip("\n")) 183 184 fn = os.path.join(sourcedir, "pagesize") 185 if os.access(fn, os.F_OK): 186 cmd.append("--pagesize") 187 cmd.append(open(fn).read().rstrip("\n")) 188 189 cmd.extend(["--ramdisk", ramdisk_img.name, 190 "--output", img.name]) 191 192 p = Run(cmd, stdout=subprocess.PIPE) 193 p.communicate() 194 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 195 os.path.basename(sourcedir),) 196 197 img.seek(os.SEEK_SET, 0) 198 data = img.read() 199 200 ramdisk_img.close() 201 img.close() 202 203 return data 204 205 206def AddRecovery(output_zip, info_dict): 207 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 208 "recovery.img", output_zip, info_dict) 209 210def AddBoot(output_zip, info_dict): 211 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 212 "boot.img", output_zip, info_dict) 213 214def UnzipTemp(filename, pattern=None): 215 """Unzip the given archive into a temporary directory and return the name.""" 216 217 tmp = tempfile.mkdtemp(prefix="targetfiles-") 218 OPTIONS.tempfiles.append(tmp) 219 cmd = ["unzip", "-o", "-q", filename, "-d", tmp] 220 if pattern is not None: 221 cmd.append(pattern) 222 p = Run(cmd, stdout=subprocess.PIPE) 223 p.communicate() 224 if p.returncode != 0: 225 raise ExternalError("failed to unzip input target-files \"%s\"" % 226 (filename,)) 227 return tmp 228 229 230def GetKeyPasswords(keylist): 231 """Given a list of keys, prompt the user to enter passwords for 232 those which require them. Return a {key: password} dict. password 233 will be None if the key has no password.""" 234 235 no_passwords = [] 236 need_passwords = [] 237 devnull = open("/dev/null", "w+b") 238 for k in sorted(keylist): 239 # We don't need a password for things that aren't really keys. 240 if k in SPECIAL_CERT_STRINGS: 241 no_passwords.append(k) 242 continue 243 244 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 245 "-inform", "DER", "-nocrypt"], 246 stdin=devnull.fileno(), 247 stdout=devnull.fileno(), 248 stderr=subprocess.STDOUT) 249 p.communicate() 250 if p.returncode == 0: 251 no_passwords.append(k) 252 else: 253 need_passwords.append(k) 254 devnull.close() 255 256 key_passwords = PasswordManager().GetPasswords(need_passwords) 257 key_passwords.update(dict.fromkeys(no_passwords, None)) 258 return key_passwords 259 260 261def SignFile(input_name, output_name, key, password, align=None, 262 whole_file=False): 263 """Sign the input_name zip/jar/apk, producing output_name. Use the 264 given key and password (the latter may be None if the key does not 265 have a password. 266 267 If align is an integer > 1, zipalign is run to align stored files in 268 the output zip on 'align'-byte boundaries. 269 270 If whole_file is true, use the "-w" option to SignApk to embed a 271 signature that covers the whole file in the archive comment of the 272 zip file. 273 """ 274 275 if align == 0 or align == 1: 276 align = None 277 278 if align: 279 temp = tempfile.NamedTemporaryFile() 280 sign_name = temp.name 281 else: 282 sign_name = output_name 283 284 cmd = ["java", "-Xmx512m", "-jar", 285 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 286 if whole_file: 287 cmd.append("-w") 288 cmd.extend([key + ".x509.pem", key + ".pk8", 289 input_name, sign_name]) 290 291 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 292 if password is not None: 293 password += "\n" 294 p.communicate(password) 295 if p.returncode != 0: 296 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 297 298 if align: 299 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 300 p.communicate() 301 if p.returncode != 0: 302 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 303 temp.close() 304 305 306def CheckSize(data, target, info_dict): 307 """Check the data string passed against the max size limit, if 308 any, for the given target. Raise exception if the data is too big. 309 Print a warning if the data is nearing the maximum size.""" 310 311 fs_type = info_dict.get("fs_type", None) 312 if not fs_type: return 313 314 if target.endswith(".img"): target = target[:-4] 315 limit = info_dict.get(target + "_size", None) 316 if limit is None: return 317 318 if fs_type == "yaffs2": 319 # image size should be increased by 1/64th to account for the 320 # spare area (64 bytes per 2k page) 321 limit = limit / 2048 * (2048+64) 322 323 size = len(data) 324 pct = float(size) * 100.0 / limit 325 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 326 if pct >= 99.0: 327 raise ExternalError(msg) 328 elif pct >= 95.0: 329 print 330 print " WARNING: ", msg 331 print 332 elif OPTIONS.verbose: 333 print " ", msg 334 335 336def ReadApkCerts(tf_zip): 337 """Given a target_files ZipFile, parse the META/apkcerts.txt file 338 and return a {package: cert} dict.""" 339 certmap = {} 340 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 341 line = line.strip() 342 if not line: continue 343 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 344 r'private_key="(.*)"$', line) 345 if m: 346 name, cert, privkey = m.groups() 347 if cert in SPECIAL_CERT_STRINGS and not privkey: 348 certmap[name] = cert 349 elif (cert.endswith(".x509.pem") and 350 privkey.endswith(".pk8") and 351 cert[:-9] == privkey[:-4]): 352 certmap[name] = cert[:-9] 353 else: 354 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 355 return certmap 356 357 358COMMON_DOCSTRING = """ 359 -p (--path) <dir> 360 Prepend <dir>/bin to the list of places to search for binaries 361 run by this script, and expect to find jars in <dir>/framework. 362 363 -s (--device_specific) <file> 364 Path to the python module containing device-specific 365 releasetools code. 366 367 -x (--extra) <key=value> 368 Add a key/value pair to the 'extras' dict, which device-specific 369 extension code may look at. 370 371 -v (--verbose) 372 Show command lines being executed. 373 374 -h (--help) 375 Display this usage message and exit. 376""" 377 378def Usage(docstring): 379 print docstring.rstrip("\n") 380 print COMMON_DOCSTRING 381 382 383def ParseOptions(argv, 384 docstring, 385 extra_opts="", extra_long_opts=(), 386 extra_option_handler=None): 387 """Parse the options in argv and return any arguments that aren't 388 flags. docstring is the calling module's docstring, to be displayed 389 for errors and -h. extra_opts and extra_long_opts are for flags 390 defined by the caller, which are processed by passing them to 391 extra_option_handler.""" 392 393 try: 394 opts, args = getopt.getopt( 395 argv, "hvp:s:x:" + extra_opts, 396 ["help", "verbose", "path=", "device_specific=", "extra="] + 397 list(extra_long_opts)) 398 except getopt.GetoptError, err: 399 Usage(docstring) 400 print "**", str(err), "**" 401 sys.exit(2) 402 403 path_specified = False 404 405 for o, a in opts: 406 if o in ("-h", "--help"): 407 Usage(docstring) 408 sys.exit() 409 elif o in ("-v", "--verbose"): 410 OPTIONS.verbose = True 411 elif o in ("-p", "--path"): 412 OPTIONS.search_path = a 413 elif o in ("-s", "--device_specific"): 414 OPTIONS.device_specific = a 415 elif o in ("-x", "--extra"): 416 key, value = a.split("=", 1) 417 OPTIONS.extras[key] = value 418 else: 419 if extra_option_handler is None or not extra_option_handler(o, a): 420 assert False, "unknown option \"%s\"" % (o,) 421 422 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 423 os.pathsep + os.environ["PATH"]) 424 425 return args 426 427 428def Cleanup(): 429 for i in OPTIONS.tempfiles: 430 if os.path.isdir(i): 431 shutil.rmtree(i) 432 else: 433 os.remove(i) 434 435 436class PasswordManager(object): 437 def __init__(self): 438 self.editor = os.getenv("EDITOR", None) 439 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 440 441 def GetPasswords(self, items): 442 """Get passwords corresponding to each string in 'items', 443 returning a dict. (The dict may have keys in addition to the 444 values in 'items'.) 445 446 Uses the passwords in $ANDROID_PW_FILE if available, letting the 447 user edit that file to add more needed passwords. If no editor is 448 available, or $ANDROID_PW_FILE isn't define, prompts the user 449 interactively in the ordinary way. 450 """ 451 452 current = self.ReadFile() 453 454 first = True 455 while True: 456 missing = [] 457 for i in items: 458 if i not in current or not current[i]: 459 missing.append(i) 460 # Are all the passwords already in the file? 461 if not missing: return current 462 463 for i in missing: 464 current[i] = "" 465 466 if not first: 467 print "key file %s still missing some passwords." % (self.pwfile,) 468 answer = raw_input("try to edit again? [y]> ").strip() 469 if answer and answer[0] not in 'yY': 470 raise RuntimeError("key passwords unavailable") 471 first = False 472 473 current = self.UpdateAndReadFile(current) 474 475 def PromptResult(self, current): 476 """Prompt the user to enter a value (password) for each key in 477 'current' whose value is fales. Returns a new dict with all the 478 values. 479 """ 480 result = {} 481 for k, v in sorted(current.iteritems()): 482 if v: 483 result[k] = v 484 else: 485 while True: 486 result[k] = getpass.getpass("Enter password for %s key> " 487 % (k,)).strip() 488 if result[k]: break 489 return result 490 491 def UpdateAndReadFile(self, current): 492 if not self.editor or not self.pwfile: 493 return self.PromptResult(current) 494 495 f = open(self.pwfile, "w") 496 os.chmod(self.pwfile, 0600) 497 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 498 f.write("# (Additional spaces are harmless.)\n\n") 499 500 first_line = None 501 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 502 sorted.sort() 503 for i, (_, k, v) in enumerate(sorted): 504 f.write("[[[ %s ]]] %s\n" % (v, k)) 505 if not v and first_line is None: 506 # position cursor on first line with no password. 507 first_line = i + 4 508 f.close() 509 510 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 511 _, _ = p.communicate() 512 513 return self.ReadFile() 514 515 def ReadFile(self): 516 result = {} 517 if self.pwfile is None: return result 518 try: 519 f = open(self.pwfile, "r") 520 for line in f: 521 line = line.strip() 522 if not line or line[0] == '#': continue 523 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 524 if not m: 525 print "failed to parse password file: ", line 526 else: 527 result[m.group(2)] = m.group(1) 528 f.close() 529 except IOError, e: 530 if e.errno != errno.ENOENT: 531 print "error reading password file: ", str(e) 532 return result 533 534 535def ZipWriteStr(zip, filename, data, perms=0644): 536 # use a fixed timestamp so the output is repeatable. 537 zinfo = zipfile.ZipInfo(filename=filename, 538 date_time=(2009, 1, 1, 0, 0, 0)) 539 zinfo.compress_type = zip.compression 540 zinfo.external_attr = perms << 16 541 zip.writestr(zinfo, data) 542 543 544class DeviceSpecificParams(object): 545 module = None 546 def __init__(self, **kwargs): 547 """Keyword arguments to the constructor become attributes of this 548 object, which is passed to all functions in the device-specific 549 module.""" 550 for k, v in kwargs.iteritems(): 551 setattr(self, k, v) 552 self.extras = OPTIONS.extras 553 554 if self.module is None: 555 path = OPTIONS.device_specific 556 if not path: return 557 try: 558 if os.path.isdir(path): 559 info = imp.find_module("releasetools", [path]) 560 else: 561 d, f = os.path.split(path) 562 b, x = os.path.splitext(f) 563 if x == ".py": 564 f = b 565 info = imp.find_module(f, [d]) 566 self.module = imp.load_module("device_specific", *info) 567 except ImportError: 568 print "unable to load device-specific module; assuming none" 569 570 def _DoCall(self, function_name, *args, **kwargs): 571 """Call the named function in the device-specific module, passing 572 the given args and kwargs. The first argument to the call will be 573 the DeviceSpecific object itself. If there is no module, or the 574 module does not define the function, return the value of the 575 'default' kwarg (which itself defaults to None).""" 576 if self.module is None or not hasattr(self.module, function_name): 577 return kwargs.get("default", None) 578 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 579 580 def FullOTA_Assertions(self): 581 """Called after emitting the block of assertions at the top of a 582 full OTA package. Implementations can add whatever additional 583 assertions they like.""" 584 return self._DoCall("FullOTA_Assertions") 585 586 def FullOTA_InstallEnd(self): 587 """Called at the end of full OTA installation; typically this is 588 used to install the image for the device's baseband processor.""" 589 return self._DoCall("FullOTA_InstallEnd") 590 591 def IncrementalOTA_Assertions(self): 592 """Called after emitting the block of assertions at the top of an 593 incremental OTA package. Implementations can add whatever 594 additional assertions they like.""" 595 return self._DoCall("IncrementalOTA_Assertions") 596 597 def IncrementalOTA_VerifyEnd(self): 598 """Called at the end of the verification phase of incremental OTA 599 installation; additional checks can be placed here to abort the 600 script before any changes are made.""" 601 return self._DoCall("IncrementalOTA_VerifyEnd") 602 603 def IncrementalOTA_InstallEnd(self): 604 """Called at the end of incremental OTA installation; typically 605 this is used to install the image for the device's baseband 606 processor.""" 607 return self._DoCall("IncrementalOTA_InstallEnd") 608 609class File(object): 610 def __init__(self, name, data): 611 self.name = name 612 self.data = data 613 self.size = len(data) 614 self.sha1 = sha.sha(data).hexdigest() 615 616 def WriteToTemp(self): 617 t = tempfile.NamedTemporaryFile() 618 t.write(self.data) 619 t.flush() 620 return t 621 622 def AddToZip(self, z): 623 ZipWriteStr(z, self.name, self.data) 624 625DIFF_PROGRAM_BY_EXT = { 626 ".gz" : "imgdiff", 627 ".zip" : ["imgdiff", "-z"], 628 ".jar" : ["imgdiff", "-z"], 629 ".apk" : ["imgdiff", "-z"], 630 ".img" : "imgdiff", 631 } 632 633class Difference(object): 634 def __init__(self, tf, sf): 635 self.tf = tf 636 self.sf = sf 637 self.patch = None 638 639 def ComputePatch(self): 640 """Compute the patch (as a string of data) needed to turn sf into 641 tf. Returns the same tuple as GetPatch().""" 642 643 tf = self.tf 644 sf = self.sf 645 646 ext = os.path.splitext(tf.name)[1] 647 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 648 649 ttemp = tf.WriteToTemp() 650 stemp = sf.WriteToTemp() 651 652 ext = os.path.splitext(tf.name)[1] 653 654 try: 655 ptemp = tempfile.NamedTemporaryFile() 656 if isinstance(diff_program, list): 657 cmd = copy.copy(diff_program) 658 else: 659 cmd = [diff_program] 660 cmd.append(stemp.name) 661 cmd.append(ttemp.name) 662 cmd.append(ptemp.name) 663 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 664 _, err = p.communicate() 665 if err or p.returncode != 0: 666 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 667 return None 668 diff = ptemp.read() 669 finally: 670 ptemp.close() 671 stemp.close() 672 ttemp.close() 673 674 self.patch = diff 675 return self.tf, self.sf, self.patch 676 677 678 def GetPatch(self): 679 """Return a tuple (target_file, source_file, patch_data). 680 patch_data may be None if ComputePatch hasn't been called, or if 681 computing the patch failed.""" 682 return self.tf, self.sf, self.patch 683 684 685def ComputeDifferences(diffs): 686 """Call ComputePatch on all the Difference objects in 'diffs'.""" 687 print len(diffs), "diffs to compute" 688 689 # Do the largest files first, to try and reduce the long-pole effect. 690 by_size = [(i.tf.size, i) for i in diffs] 691 by_size.sort(reverse=True) 692 by_size = [i[1] for i in by_size] 693 694 lock = threading.Lock() 695 diff_iter = iter(by_size) # accessed under lock 696 697 def worker(): 698 try: 699 lock.acquire() 700 for d in diff_iter: 701 lock.release() 702 start = time.time() 703 d.ComputePatch() 704 dur = time.time() - start 705 lock.acquire() 706 707 tf, sf, patch = d.GetPatch() 708 if sf.name == tf.name: 709 name = tf.name 710 else: 711 name = "%s (%s)" % (tf.name, sf.name) 712 if patch is None: 713 print "patching failed! %s" % (name,) 714 else: 715 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 716 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 717 lock.release() 718 except Exception, e: 719 print e 720 raise 721 722 # start worker threads; wait for them all to finish. 723 threads = [threading.Thread(target=worker) 724 for i in range(OPTIONS.worker_threads)] 725 for th in threads: 726 th.start() 727 while threads: 728 threads.pop().join() 729