common.py revision 37974731fcb4e32b1de5f213d34bd832ca889869
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import re 22import sha 23import shutil 24import subprocess 25import sys 26import tempfile 27import threading 28import time 29import zipfile 30 31# missing in Python 2.4 and before 32if not hasattr(os, "SEEK_SET"): 33 os.SEEK_SET = 0 34 35class Options(object): pass 36OPTIONS = Options() 37OPTIONS.search_path = "out/host/linux-x86" 38OPTIONS.max_image_size = {} 39OPTIONS.verbose = False 40OPTIONS.tempfiles = [] 41OPTIONS.device_specific = None 42OPTIONS.extras = {} 43OPTIONS.info_dict = None 44 45 46# Values for "certificate" in apkcerts that mean special things. 47SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 48 49 50class ExternalError(RuntimeError): pass 51 52 53def Run(args, **kwargs): 54 """Create and return a subprocess.Popen object, printing the command 55 line on the terminal if -v was specified.""" 56 if OPTIONS.verbose: 57 print " running: ", " ".join(args) 58 return subprocess.Popen(args, **kwargs) 59 60 61def LoadInfoDict(zip): 62 """Read and parse the META/misc_info.txt key/value pairs from the 63 input target files and return a dict.""" 64 65 d = {} 66 try: 67 for line in zip.read("META/misc_info.txt").split("\n"): 68 line = line.strip() 69 if not line or line.startswith("#"): continue 70 k, v = line.split("=", 1) 71 d[k] = v 72 except KeyError: 73 # ok if misc_info.txt doesn't exist 74 pass 75 76 if "fs_type" not in d: d["fs_type"] = "yaffs2" 77 if "partition_type" not in d: d["partition_type"] = "MTD" 78 79 # backwards compatibility: These values used to be in their own 80 # files. Look for them, in case we're processing an old 81 # target_files zip. 82 83 if "mkyaffs2_extra_flags" not in d: 84 try: 85 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 86 except KeyError: 87 # ok if flags don't exist 88 pass 89 90 if "recovery_api_version" not in d: 91 try: 92 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 93 except KeyError: 94 raise ValueError("can't find recovery API version in input target-files") 95 96 if "tool_extensions" not in d: 97 try: 98 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 99 except KeyError: 100 # ok if extensions don't exist 101 pass 102 103 try: 104 data = zip.read("META/imagesizes.txt") 105 for line in data.split("\n"): 106 if not line: continue 107 name, value = line.strip().split(None, 1) 108 if name == "blocksize": 109 d[name] = value 110 else: 111 d[name + "_size"] = value 112 except KeyError: 113 pass 114 115 def makeint(key): 116 if key in d: 117 d[key] = int(d[key], 0) 118 119 makeint("recovery_api_version") 120 makeint("blocksize") 121 makeint("system_size") 122 makeint("userdata_size") 123 makeint("recovery_size") 124 makeint("boot_size") 125 126 return d 127 128def DumpInfoDict(d): 129 for k, v in sorted(d.items()): 130 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 131 132def BuildAndAddBootableImage(sourcedir, targetname, output_zip, info_dict): 133 """Take a kernel, cmdline, and ramdisk directory from the input (in 134 'sourcedir'), and turn them into a boot image. Put the boot image 135 into the output zip file under the name 'targetname'. Returns 136 targetname on success or None on failure (if sourcedir does not 137 appear to contain files for the requested image).""" 138 139 print "creating %s..." % (targetname,) 140 141 img = BuildBootableImage(sourcedir) 142 if img is None: 143 return None 144 145 CheckSize(img, targetname, info_dict) 146 ZipWriteStr(output_zip, targetname, img) 147 return targetname 148 149def BuildBootableImage(sourcedir): 150 """Take a kernel, cmdline, and ramdisk directory from the input (in 151 'sourcedir'), and turn them into a boot image. Return the image 152 data, or None if sourcedir does not appear to contains files for 153 building the requested image.""" 154 155 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 156 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 157 return None 158 159 ramdisk_img = tempfile.NamedTemporaryFile() 160 img = tempfile.NamedTemporaryFile() 161 162 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 163 stdout=subprocess.PIPE) 164 p2 = Run(["minigzip"], 165 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 166 167 p2.wait() 168 p1.wait() 169 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 170 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 171 172 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 173 174 fn = os.path.join(sourcedir, "cmdline") 175 if os.access(fn, os.F_OK): 176 cmd.append("--cmdline") 177 cmd.append(open(fn).read().rstrip("\n")) 178 179 fn = os.path.join(sourcedir, "base") 180 if os.access(fn, os.F_OK): 181 cmd.append("--base") 182 cmd.append(open(fn).read().rstrip("\n")) 183 184 fn = os.path.join(sourcedir, "pagesize") 185 if os.access(fn, os.F_OK): 186 cmd.append("--pagesize") 187 cmd.append(open(fn).read().rstrip("\n")) 188 189 cmd.extend(["--ramdisk", ramdisk_img.name, 190 "--output", img.name]) 191 192 p = Run(cmd, stdout=subprocess.PIPE) 193 p.communicate() 194 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 195 os.path.basename(sourcedir),) 196 197 img.seek(os.SEEK_SET, 0) 198 data = img.read() 199 200 ramdisk_img.close() 201 img.close() 202 203 return data 204 205 206def AddRecovery(output_zip, info_dict): 207 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"), 208 "recovery.img", output_zip, info_dict) 209 210def AddBoot(output_zip, info_dict): 211 BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "BOOT"), 212 "boot.img", output_zip, info_dict) 213 214def UnzipTemp(filename, pattern=None): 215 """Unzip the given archive into a temporary directory and return the name.""" 216 217 tmp = tempfile.mkdtemp(prefix="targetfiles-") 218 OPTIONS.tempfiles.append(tmp) 219 cmd = ["unzip", "-o", "-q", filename, "-d", tmp] 220 if pattern is not None: 221 cmd.append(pattern) 222 p = Run(cmd, stdout=subprocess.PIPE) 223 p.communicate() 224 if p.returncode != 0: 225 raise ExternalError("failed to unzip input target-files \"%s\"" % 226 (filename,)) 227 return tmp 228 229 230def GetKeyPasswords(keylist): 231 """Given a list of keys, prompt the user to enter passwords for 232 those which require them. Return a {key: password} dict. password 233 will be None if the key has no password.""" 234 235 no_passwords = [] 236 need_passwords = [] 237 devnull = open("/dev/null", "w+b") 238 for k in sorted(keylist): 239 # We don't need a password for things that aren't really keys. 240 if k in SPECIAL_CERT_STRINGS: 241 no_passwords.append(k) 242 continue 243 244 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 245 "-inform", "DER", "-nocrypt"], 246 stdin=devnull.fileno(), 247 stdout=devnull.fileno(), 248 stderr=subprocess.STDOUT) 249 p.communicate() 250 if p.returncode == 0: 251 no_passwords.append(k) 252 else: 253 need_passwords.append(k) 254 devnull.close() 255 256 key_passwords = PasswordManager().GetPasswords(need_passwords) 257 key_passwords.update(dict.fromkeys(no_passwords, None)) 258 return key_passwords 259 260 261def SignFile(input_name, output_name, key, password, align=None, 262 whole_file=False): 263 """Sign the input_name zip/jar/apk, producing output_name. Use the 264 given key and password (the latter may be None if the key does not 265 have a password. 266 267 If align is an integer > 1, zipalign is run to align stored files in 268 the output zip on 'align'-byte boundaries. 269 270 If whole_file is true, use the "-w" option to SignApk to embed a 271 signature that covers the whole file in the archive comment of the 272 zip file. 273 """ 274 275 if align == 0 or align == 1: 276 align = None 277 278 if align: 279 temp = tempfile.NamedTemporaryFile() 280 sign_name = temp.name 281 else: 282 sign_name = output_name 283 284 cmd = ["java", "-Xmx512m", "-jar", 285 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 286 if whole_file: 287 cmd.append("-w") 288 cmd.extend([key + ".x509.pem", key + ".pk8", 289 input_name, sign_name]) 290 291 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 292 if password is not None: 293 password += "\n" 294 p.communicate(password) 295 if p.returncode != 0: 296 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 297 298 if align: 299 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 300 p.communicate() 301 if p.returncode != 0: 302 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 303 temp.close() 304 305 306def CheckSize(data, target, info_dict): 307 """Check the data string passed against the max size limit, if 308 any, for the given target. Raise exception if the data is too big. 309 Print a warning if the data is nearing the maximum size.""" 310 311 fs_type = info_dict.get("fs_type", None) 312 if not fs_type: return 313 314 limit = OPTIONS.max_image_size.get(target, None) 315 if limit is None: return 316 317 if fs_type == "yaffs2": 318 # image size should be increased by 1/64th to account for the 319 # spare area (64 bytes per 2k page) 320 limit = limit / 2048 * (2048+64) 321 322 size = len(data) 323 pct = float(size) * 100.0 / limit 324 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 325 if pct >= 99.0: 326 raise ExternalError(msg) 327 elif pct >= 95.0: 328 print 329 print " WARNING: ", msg 330 print 331 elif OPTIONS.verbose: 332 print " ", msg 333 334 335def ReadApkCerts(tf_zip): 336 """Given a target_files ZipFile, parse the META/apkcerts.txt file 337 and return a {package: cert} dict.""" 338 certmap = {} 339 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 340 line = line.strip() 341 if not line: continue 342 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 343 r'private_key="(.*)"$', line) 344 if m: 345 name, cert, privkey = m.groups() 346 if cert in SPECIAL_CERT_STRINGS and not privkey: 347 certmap[name] = cert 348 elif (cert.endswith(".x509.pem") and 349 privkey.endswith(".pk8") and 350 cert[:-9] == privkey[:-4]): 351 certmap[name] = cert[:-9] 352 else: 353 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 354 return certmap 355 356 357COMMON_DOCSTRING = """ 358 -p (--path) <dir> 359 Prepend <dir>/bin to the list of places to search for binaries 360 run by this script, and expect to find jars in <dir>/framework. 361 362 -s (--device_specific) <file> 363 Path to the python module containing device-specific 364 releasetools code. 365 366 -x (--extra) <key=value> 367 Add a key/value pair to the 'extras' dict, which device-specific 368 extension code may look at. 369 370 -v (--verbose) 371 Show command lines being executed. 372 373 -h (--help) 374 Display this usage message and exit. 375""" 376 377def Usage(docstring): 378 print docstring.rstrip("\n") 379 print COMMON_DOCSTRING 380 381 382def ParseOptions(argv, 383 docstring, 384 extra_opts="", extra_long_opts=(), 385 extra_option_handler=None): 386 """Parse the options in argv and return any arguments that aren't 387 flags. docstring is the calling module's docstring, to be displayed 388 for errors and -h. extra_opts and extra_long_opts are for flags 389 defined by the caller, which are processed by passing them to 390 extra_option_handler.""" 391 392 try: 393 opts, args = getopt.getopt( 394 argv, "hvp:s:x:" + extra_opts, 395 ["help", "verbose", "path=", "device_specific=", "extra="] + 396 list(extra_long_opts)) 397 except getopt.GetoptError, err: 398 Usage(docstring) 399 print "**", str(err), "**" 400 sys.exit(2) 401 402 path_specified = False 403 404 for o, a in opts: 405 if o in ("-h", "--help"): 406 Usage(docstring) 407 sys.exit() 408 elif o in ("-v", "--verbose"): 409 OPTIONS.verbose = True 410 elif o in ("-p", "--path"): 411 OPTIONS.search_path = a 412 elif o in ("-s", "--device_specific"): 413 OPTIONS.device_specific = a 414 elif o in ("-x", "--extra"): 415 key, value = a.split("=", 1) 416 OPTIONS.extras[key] = value 417 else: 418 if extra_option_handler is None or not extra_option_handler(o, a): 419 assert False, "unknown option \"%s\"" % (o,) 420 421 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 422 os.pathsep + os.environ["PATH"]) 423 424 return args 425 426 427def Cleanup(): 428 for i in OPTIONS.tempfiles: 429 if os.path.isdir(i): 430 shutil.rmtree(i) 431 else: 432 os.remove(i) 433 434 435class PasswordManager(object): 436 def __init__(self): 437 self.editor = os.getenv("EDITOR", None) 438 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 439 440 def GetPasswords(self, items): 441 """Get passwords corresponding to each string in 'items', 442 returning a dict. (The dict may have keys in addition to the 443 values in 'items'.) 444 445 Uses the passwords in $ANDROID_PW_FILE if available, letting the 446 user edit that file to add more needed passwords. If no editor is 447 available, or $ANDROID_PW_FILE isn't define, prompts the user 448 interactively in the ordinary way. 449 """ 450 451 current = self.ReadFile() 452 453 first = True 454 while True: 455 missing = [] 456 for i in items: 457 if i not in current or not current[i]: 458 missing.append(i) 459 # Are all the passwords already in the file? 460 if not missing: return current 461 462 for i in missing: 463 current[i] = "" 464 465 if not first: 466 print "key file %s still missing some passwords." % (self.pwfile,) 467 answer = raw_input("try to edit again? [y]> ").strip() 468 if answer and answer[0] not in 'yY': 469 raise RuntimeError("key passwords unavailable") 470 first = False 471 472 current = self.UpdateAndReadFile(current) 473 474 def PromptResult(self, current): 475 """Prompt the user to enter a value (password) for each key in 476 'current' whose value is fales. Returns a new dict with all the 477 values. 478 """ 479 result = {} 480 for k, v in sorted(current.iteritems()): 481 if v: 482 result[k] = v 483 else: 484 while True: 485 result[k] = getpass.getpass("Enter password for %s key> " 486 % (k,)).strip() 487 if result[k]: break 488 return result 489 490 def UpdateAndReadFile(self, current): 491 if not self.editor or not self.pwfile: 492 return self.PromptResult(current) 493 494 f = open(self.pwfile, "w") 495 os.chmod(self.pwfile, 0600) 496 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 497 f.write("# (Additional spaces are harmless.)\n\n") 498 499 first_line = None 500 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 501 sorted.sort() 502 for i, (_, k, v) in enumerate(sorted): 503 f.write("[[[ %s ]]] %s\n" % (v, k)) 504 if not v and first_line is None: 505 # position cursor on first line with no password. 506 first_line = i + 4 507 f.close() 508 509 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 510 _, _ = p.communicate() 511 512 return self.ReadFile() 513 514 def ReadFile(self): 515 result = {} 516 if self.pwfile is None: return result 517 try: 518 f = open(self.pwfile, "r") 519 for line in f: 520 line = line.strip() 521 if not line or line[0] == '#': continue 522 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 523 if not m: 524 print "failed to parse password file: ", line 525 else: 526 result[m.group(2)] = m.group(1) 527 f.close() 528 except IOError, e: 529 if e.errno != errno.ENOENT: 530 print "error reading password file: ", str(e) 531 return result 532 533 534def ZipWriteStr(zip, filename, data, perms=0644): 535 # use a fixed timestamp so the output is repeatable. 536 zinfo = zipfile.ZipInfo(filename=filename, 537 date_time=(2009, 1, 1, 0, 0, 0)) 538 zinfo.compress_type = zip.compression 539 zinfo.external_attr = perms << 16 540 zip.writestr(zinfo, data) 541 542 543class DeviceSpecificParams(object): 544 module = None 545 def __init__(self, **kwargs): 546 """Keyword arguments to the constructor become attributes of this 547 object, which is passed to all functions in the device-specific 548 module.""" 549 for k, v in kwargs.iteritems(): 550 setattr(self, k, v) 551 self.extras = OPTIONS.extras 552 553 if self.module is None: 554 path = OPTIONS.device_specific 555 if not path: return 556 try: 557 if os.path.isdir(path): 558 info = imp.find_module("releasetools", [path]) 559 else: 560 d, f = os.path.split(path) 561 b, x = os.path.splitext(f) 562 if x == ".py": 563 f = b 564 info = imp.find_module(f, [d]) 565 self.module = imp.load_module("device_specific", *info) 566 except ImportError: 567 print "unable to load device-specific module; assuming none" 568 569 def _DoCall(self, function_name, *args, **kwargs): 570 """Call the named function in the device-specific module, passing 571 the given args and kwargs. The first argument to the call will be 572 the DeviceSpecific object itself. If there is no module, or the 573 module does not define the function, return the value of the 574 'default' kwarg (which itself defaults to None).""" 575 if self.module is None or not hasattr(self.module, function_name): 576 return kwargs.get("default", None) 577 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 578 579 def FullOTA_Assertions(self): 580 """Called after emitting the block of assertions at the top of a 581 full OTA package. Implementations can add whatever additional 582 assertions they like.""" 583 return self._DoCall("FullOTA_Assertions") 584 585 def FullOTA_InstallEnd(self): 586 """Called at the end of full OTA installation; typically this is 587 used to install the image for the device's baseband processor.""" 588 return self._DoCall("FullOTA_InstallEnd") 589 590 def IncrementalOTA_Assertions(self): 591 """Called after emitting the block of assertions at the top of an 592 incremental OTA package. Implementations can add whatever 593 additional assertions they like.""" 594 return self._DoCall("IncrementalOTA_Assertions") 595 596 def IncrementalOTA_VerifyEnd(self): 597 """Called at the end of the verification phase of incremental OTA 598 installation; additional checks can be placed here to abort the 599 script before any changes are made.""" 600 return self._DoCall("IncrementalOTA_VerifyEnd") 601 602 def IncrementalOTA_InstallEnd(self): 603 """Called at the end of incremental OTA installation; typically 604 this is used to install the image for the device's baseband 605 processor.""" 606 return self._DoCall("IncrementalOTA_InstallEnd") 607 608class File(object): 609 def __init__(self, name, data): 610 self.name = name 611 self.data = data 612 self.size = len(data) 613 self.sha1 = sha.sha(data).hexdigest() 614 615 def WriteToTemp(self): 616 t = tempfile.NamedTemporaryFile() 617 t.write(self.data) 618 t.flush() 619 return t 620 621 def AddToZip(self, z): 622 ZipWriteStr(z, self.name, self.data) 623 624DIFF_PROGRAM_BY_EXT = { 625 ".gz" : "imgdiff", 626 ".zip" : ["imgdiff", "-z"], 627 ".jar" : ["imgdiff", "-z"], 628 ".apk" : ["imgdiff", "-z"], 629 ".img" : "imgdiff", 630 } 631 632class Difference(object): 633 def __init__(self, tf, sf): 634 self.tf = tf 635 self.sf = sf 636 self.patch = None 637 638 def ComputePatch(self): 639 """Compute the patch (as a string of data) needed to turn sf into 640 tf. Returns the same tuple as GetPatch().""" 641 642 tf = self.tf 643 sf = self.sf 644 645 ext = os.path.splitext(tf.name)[1] 646 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 647 648 ttemp = tf.WriteToTemp() 649 stemp = sf.WriteToTemp() 650 651 ext = os.path.splitext(tf.name)[1] 652 653 try: 654 ptemp = tempfile.NamedTemporaryFile() 655 if isinstance(diff_program, list): 656 cmd = copy.copy(diff_program) 657 else: 658 cmd = [diff_program] 659 cmd.append(stemp.name) 660 cmd.append(ttemp.name) 661 cmd.append(ptemp.name) 662 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 663 _, err = p.communicate() 664 if err or p.returncode != 0: 665 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 666 return None 667 diff = ptemp.read() 668 finally: 669 ptemp.close() 670 stemp.close() 671 ttemp.close() 672 673 self.patch = diff 674 return self.tf, self.sf, self.patch 675 676 677 def GetPatch(self): 678 """Return a tuple (target_file, source_file, patch_data). 679 patch_data may be None if ComputePatch hasn't been called, or if 680 computing the patch failed.""" 681 return self.tf, self.sf, self.patch 682 683 684def ComputeDifferences(diffs): 685 """Call ComputePatch on all the Difference objects in 'diffs'.""" 686 print len(diffs), "diffs to compute" 687 688 # Do the largest files first, to try and reduce the long-pole effect. 689 by_size = [(i.tf.size, i) for i in diffs] 690 by_size.sort(reverse=True) 691 by_size = [i[1] for i in by_size] 692 693 lock = threading.Lock() 694 diff_iter = iter(by_size) # accessed under lock 695 696 def worker(): 697 try: 698 lock.acquire() 699 for d in diff_iter: 700 lock.release() 701 start = time.time() 702 d.ComputePatch() 703 dur = time.time() - start 704 lock.acquire() 705 706 tf, sf, patch = d.GetPatch() 707 if sf.name == tf.name: 708 name = tf.name 709 else: 710 name = "%s (%s)" % (tf.name, sf.name) 711 if patch is None: 712 print "patching failed! %s" % (name,) 713 else: 714 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 715 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 716 lock.release() 717 except Exception, e: 718 print e 719 raise 720 721 # start worker threads; wait for them all to finish. 722 threads = [threading.Thread(target=worker) 723 for i in range(OPTIONS.worker_threads)] 724 for th in threads: 725 th.start() 726 while threads: 727 threads.pop().join() 728