common.py revision fb9076d9f202fa850e23e16cf2113f0a1e62951c
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(zip): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 d = {} 92 try: 93 for line in zip.read("META/misc_info.txt").split("\n"): 94 line = line.strip() 95 if not line or line.startswith("#"): continue 96 k, v = line.split("=", 1) 97 d[k] = v 98 except KeyError: 99 # ok if misc_info.txt doesn't exist 100 pass 101 102 # backwards compatibility: These values used to be in their own 103 # files. Look for them, in case we're processing an old 104 # target_files zip. 105 106 if "mkyaffs2_extra_flags" not in d: 107 try: 108 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 109 except KeyError: 110 # ok if flags don't exist 111 pass 112 113 if "recovery_api_version" not in d: 114 try: 115 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 116 except KeyError: 117 raise ValueError("can't find recovery API version in input target-files") 118 119 if "tool_extensions" not in d: 120 try: 121 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 122 except KeyError: 123 # ok if extensions don't exist 124 pass 125 126 if "fstab_version" not in d: 127 d["fstab_version"] = "1" 128 129 try: 130 data = zip.read("META/imagesizes.txt") 131 for line in data.split("\n"): 132 if not line: continue 133 name, value = line.split(" ", 1) 134 if not value: continue 135 if name == "blocksize": 136 d[name] = value 137 else: 138 d[name + "_size"] = value 139 except KeyError: 140 pass 141 142 def makeint(key): 143 if key in d: 144 d[key] = int(d[key], 0) 145 146 makeint("recovery_api_version") 147 makeint("blocksize") 148 makeint("system_size") 149 makeint("userdata_size") 150 makeint("cache_size") 151 makeint("recovery_size") 152 makeint("boot_size") 153 makeint("fstab_version") 154 155 d["fstab"] = LoadRecoveryFSTab(zip, d["fstab_version"]) 156 d["build.prop"] = LoadBuildProp(zip) 157 return d 158 159def LoadBuildProp(zip): 160 try: 161 data = zip.read("SYSTEM/build.prop") 162 except KeyError: 163 print "Warning: could not find SYSTEM/build.prop in %s" % zip 164 data = "" 165 166 d = {} 167 for line in data.split("\n"): 168 line = line.strip() 169 if not line or line.startswith("#"): continue 170 name, value = line.split("=", 1) 171 d[name] = value 172 return d 173 174def LoadRecoveryFSTab(zip, fstab_version): 175 class Partition(object): 176 pass 177 178 try: 179 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab") 180 except KeyError: 181 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip 182 data = "" 183 184 if fstab_version == 1: 185 d = {} 186 for line in data.split("\n"): 187 line = line.strip() 188 if not line or line.startswith("#"): continue 189 pieces = line.split() 190 if not (3 <= len(pieces) <= 4): 191 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 192 193 p = Partition() 194 p.mount_point = pieces[0] 195 p.fs_type = pieces[1] 196 p.device = pieces[2] 197 p.length = 0 198 options = None 199 if len(pieces) >= 4: 200 if pieces[3].startswith("/"): 201 p.device2 = pieces[3] 202 if len(pieces) >= 5: 203 options = pieces[4] 204 else: 205 p.device2 = None 206 options = pieces[3] 207 else: 208 p.device2 = None 209 210 if options: 211 options = options.split(",") 212 for i in options: 213 if i.startswith("length="): 214 p.length = int(i[7:]) 215 else: 216 print "%s: unknown option \"%s\"" % (p.mount_point, i) 217 218 d[p.mount_point] = p 219 220 elif fstab_version == 2: 221 d = {} 222 for line in data.split("\n"): 223 line = line.strip() 224 if not line or line.startswith("#"): continue 225 pieces = line.split() 226 if len(pieces) != 5: 227 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 228 229 # Ignore entries that are managed by vold 230 options = pieces[4] 231 if "voldmanaged=" in options: continue 232 233 # It's a good line, parse it 234 p = Partition() 235 p.device = pieces[0] 236 p.mount_point = pieces[1] 237 p.fs_type = pieces[2] 238 p.device2 = None 239 p.length = 0 240 241 options = options.split(",") 242 for i in options: 243 if i.startswith("length="): 244 p.length = int(i[7:]) 245 else: 246 # Ignore all unknown options in the unified fstab 247 continue 248 249 d[p.mount_point] = p 250 251 else: 252 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 253 254 return d 255 256 257def DumpInfoDict(d): 258 for k, v in sorted(d.items()): 259 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 260 261def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 262 """Take a kernel, cmdline, and ramdisk directory from the input (in 263 'sourcedir'), and turn them into a boot image. Return the image 264 data, or None if sourcedir does not appear to contains files for 265 building the requested image.""" 266 267 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 268 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 269 return None 270 271 if info_dict is None: 272 info_dict = OPTIONS.info_dict 273 274 ramdisk_img = tempfile.NamedTemporaryFile() 275 img = tempfile.NamedTemporaryFile() 276 277 if os.access(fs_config_file, os.F_OK): 278 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 279 else: 280 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 281 p1 = Run(cmd, stdout=subprocess.PIPE) 282 p2 = Run(["minigzip"], 283 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 284 285 p2.wait() 286 p1.wait() 287 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 288 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 289 290 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 291 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 292 293 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 294 295 fn = os.path.join(sourcedir, "second") 296 if os.access(fn, os.F_OK): 297 cmd.append("--second") 298 cmd.append(fn) 299 300 fn = os.path.join(sourcedir, "cmdline") 301 if os.access(fn, os.F_OK): 302 cmd.append("--cmdline") 303 cmd.append(open(fn).read().rstrip("\n")) 304 305 fn = os.path.join(sourcedir, "base") 306 if os.access(fn, os.F_OK): 307 cmd.append("--base") 308 cmd.append(open(fn).read().rstrip("\n")) 309 310 fn = os.path.join(sourcedir, "pagesize") 311 if os.access(fn, os.F_OK): 312 cmd.append("--pagesize") 313 cmd.append(open(fn).read().rstrip("\n")) 314 315 args = info_dict.get("mkbootimg_args", None) 316 if args and args.strip(): 317 cmd.extend(shlex.split(args)) 318 319 cmd.extend(["--ramdisk", ramdisk_img.name, 320 "--output", img.name]) 321 322 p = Run(cmd, stdout=subprocess.PIPE) 323 p.communicate() 324 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 325 os.path.basename(sourcedir),) 326 327 img.seek(os.SEEK_SET, 0) 328 data = img.read() 329 330 ramdisk_img.close() 331 img.close() 332 333 return data 334 335 336def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 337 info_dict=None): 338 """Return a File object (with name 'name') with the desired bootable 339 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 340 'prebuilt_name', otherwise construct it from the source files in 341 'unpack_dir'/'tree_subdir'.""" 342 343 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 344 if os.path.exists(prebuilt_path): 345 print "using prebuilt %s..." % (prebuilt_name,) 346 return File.FromLocalFile(name, prebuilt_path) 347 else: 348 print "building image from target_files %s..." % (tree_subdir,) 349 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 350 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 351 os.path.join(unpack_dir, fs_config), 352 info_dict)) 353 354 355def UnzipTemp(filename, pattern=None): 356 """Unzip the given archive into a temporary directory and return the name. 357 358 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 359 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 360 361 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 362 main file), open for reading. 363 """ 364 365 tmp = tempfile.mkdtemp(prefix="targetfiles-") 366 OPTIONS.tempfiles.append(tmp) 367 368 def unzip_to_dir(filename, dirname): 369 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 370 if pattern is not None: 371 cmd.append(pattern) 372 p = Run(cmd, stdout=subprocess.PIPE) 373 p.communicate() 374 if p.returncode != 0: 375 raise ExternalError("failed to unzip input target-files \"%s\"" % 376 (filename,)) 377 378 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 379 if m: 380 unzip_to_dir(m.group(1), tmp) 381 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 382 filename = m.group(1) 383 else: 384 unzip_to_dir(filename, tmp) 385 386 return tmp, zipfile.ZipFile(filename, "r") 387 388 389def GetKeyPasswords(keylist): 390 """Given a list of keys, prompt the user to enter passwords for 391 those which require them. Return a {key: password} dict. password 392 will be None if the key has no password.""" 393 394 no_passwords = [] 395 need_passwords = [] 396 key_passwords = {} 397 devnull = open("/dev/null", "w+b") 398 for k in sorted(keylist): 399 # We don't need a password for things that aren't really keys. 400 if k in SPECIAL_CERT_STRINGS: 401 no_passwords.append(k) 402 continue 403 404 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 405 "-inform", "DER", "-nocrypt"], 406 stdin=devnull.fileno(), 407 stdout=devnull.fileno(), 408 stderr=subprocess.STDOUT) 409 p.communicate() 410 if p.returncode == 0: 411 # Definitely an unencrypted key. 412 no_passwords.append(k) 413 else: 414 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 415 "-inform", "DER", "-passin", "pass:"], 416 stdin=devnull.fileno(), 417 stdout=devnull.fileno(), 418 stderr=subprocess.PIPE) 419 stdout, stderr = p.communicate() 420 if p.returncode == 0: 421 # Encrypted key with empty string as password. 422 key_passwords[k] = '' 423 elif stderr.startswith('Error decrypting key'): 424 # Definitely encrypted key. 425 # It would have said "Error reading key" if it didn't parse correctly. 426 need_passwords.append(k) 427 else: 428 # Potentially, a type of key that openssl doesn't understand. 429 # We'll let the routines in signapk.jar handle it. 430 no_passwords.append(k) 431 devnull.close() 432 433 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 434 key_passwords.update(dict.fromkeys(no_passwords, None)) 435 return key_passwords 436 437 438def SignFile(input_name, output_name, key, password, align=None, 439 whole_file=False): 440 """Sign the input_name zip/jar/apk, producing output_name. Use the 441 given key and password (the latter may be None if the key does not 442 have a password. 443 444 If align is an integer > 1, zipalign is run to align stored files in 445 the output zip on 'align'-byte boundaries. 446 447 If whole_file is true, use the "-w" option to SignApk to embed a 448 signature that covers the whole file in the archive comment of the 449 zip file. 450 """ 451 452 if align == 0 or align == 1: 453 align = None 454 455 if align: 456 temp = tempfile.NamedTemporaryFile() 457 sign_name = temp.name 458 else: 459 sign_name = output_name 460 461 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 462 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 463 cmd.extend(OPTIONS.extra_signapk_args) 464 if whole_file: 465 cmd.append("-w") 466 cmd.extend([key + OPTIONS.public_key_suffix, 467 key + OPTIONS.private_key_suffix, 468 input_name, sign_name]) 469 470 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 471 if password is not None: 472 password += "\n" 473 p.communicate(password) 474 if p.returncode != 0: 475 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 476 477 if align: 478 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 479 p.communicate() 480 if p.returncode != 0: 481 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 482 temp.close() 483 484 485def CheckSize(data, target, info_dict): 486 """Check the data string passed against the max size limit, if 487 any, for the given target. Raise exception if the data is too big. 488 Print a warning if the data is nearing the maximum size.""" 489 490 if target.endswith(".img"): target = target[:-4] 491 mount_point = "/" + target 492 493 if info_dict["fstab"]: 494 if mount_point == "/userdata": mount_point = "/data" 495 p = info_dict["fstab"][mount_point] 496 fs_type = p.fs_type 497 device = p.device 498 if "/" in device: 499 device = device[device.rfind("/")+1:] 500 limit = info_dict.get(device + "_size", None) 501 if not fs_type or not limit: return 502 503 if fs_type == "yaffs2": 504 # image size should be increased by 1/64th to account for the 505 # spare area (64 bytes per 2k page) 506 limit = limit / 2048 * (2048+64) 507 size = len(data) 508 pct = float(size) * 100.0 / limit 509 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 510 if pct >= 99.0: 511 raise ExternalError(msg) 512 elif pct >= 95.0: 513 print 514 print " WARNING: ", msg 515 print 516 elif OPTIONS.verbose: 517 print " ", msg 518 519 520def ReadApkCerts(tf_zip): 521 """Given a target_files ZipFile, parse the META/apkcerts.txt file 522 and return a {package: cert} dict.""" 523 certmap = {} 524 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 525 line = line.strip() 526 if not line: continue 527 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 528 r'private_key="(.*)"$', line) 529 if m: 530 name, cert, privkey = m.groups() 531 public_key_suffix_len = len(OPTIONS.public_key_suffix) 532 private_key_suffix_len = len(OPTIONS.private_key_suffix) 533 if cert in SPECIAL_CERT_STRINGS and not privkey: 534 certmap[name] = cert 535 elif (cert.endswith(OPTIONS.public_key_suffix) and 536 privkey.endswith(OPTIONS.private_key_suffix) and 537 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 538 certmap[name] = cert[:-public_key_suffix_len] 539 else: 540 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 541 return certmap 542 543 544COMMON_DOCSTRING = """ 545 -p (--path) <dir> 546 Prepend <dir>/bin to the list of places to search for binaries 547 run by this script, and expect to find jars in <dir>/framework. 548 549 -s (--device_specific) <file> 550 Path to the python module containing device-specific 551 releasetools code. 552 553 -x (--extra) <key=value> 554 Add a key/value pair to the 'extras' dict, which device-specific 555 extension code may look at. 556 557 -v (--verbose) 558 Show command lines being executed. 559 560 -h (--help) 561 Display this usage message and exit. 562""" 563 564def Usage(docstring): 565 print docstring.rstrip("\n") 566 print COMMON_DOCSTRING 567 568 569def ParseOptions(argv, 570 docstring, 571 extra_opts="", extra_long_opts=(), 572 extra_option_handler=None): 573 """Parse the options in argv and return any arguments that aren't 574 flags. docstring is the calling module's docstring, to be displayed 575 for errors and -h. extra_opts and extra_long_opts are for flags 576 defined by the caller, which are processed by passing them to 577 extra_option_handler.""" 578 579 try: 580 opts, args = getopt.getopt( 581 argv, "hvp:s:x:" + extra_opts, 582 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 583 "java_path=", "public_key_suffix=", "private_key_suffix=", 584 "device_specific=", "extra="] + 585 list(extra_long_opts)) 586 except getopt.GetoptError, err: 587 Usage(docstring) 588 print "**", str(err), "**" 589 sys.exit(2) 590 591 path_specified = False 592 593 for o, a in opts: 594 if o in ("-h", "--help"): 595 Usage(docstring) 596 sys.exit() 597 elif o in ("-v", "--verbose"): 598 OPTIONS.verbose = True 599 elif o in ("-p", "--path"): 600 OPTIONS.search_path = a 601 elif o in ("--signapk_path",): 602 OPTIONS.signapk_path = a 603 elif o in ("--extra_signapk_args",): 604 OPTIONS.extra_signapk_args = shlex.split(a) 605 elif o in ("--java_path",): 606 OPTIONS.java_path = a 607 elif o in ("--public_key_suffix",): 608 OPTIONS.public_key_suffix = a 609 elif o in ("--private_key_suffix",): 610 OPTIONS.private_key_suffix = a 611 elif o in ("-s", "--device_specific"): 612 OPTIONS.device_specific = a 613 elif o in ("-x", "--extra"): 614 key, value = a.split("=", 1) 615 OPTIONS.extras[key] = value 616 else: 617 if extra_option_handler is None or not extra_option_handler(o, a): 618 assert False, "unknown option \"%s\"" % (o,) 619 620 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 621 os.pathsep + os.environ["PATH"]) 622 623 return args 624 625 626def Cleanup(): 627 for i in OPTIONS.tempfiles: 628 if os.path.isdir(i): 629 shutil.rmtree(i) 630 else: 631 os.remove(i) 632 633 634class PasswordManager(object): 635 def __init__(self): 636 self.editor = os.getenv("EDITOR", None) 637 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 638 639 def GetPasswords(self, items): 640 """Get passwords corresponding to each string in 'items', 641 returning a dict. (The dict may have keys in addition to the 642 values in 'items'.) 643 644 Uses the passwords in $ANDROID_PW_FILE if available, letting the 645 user edit that file to add more needed passwords. If no editor is 646 available, or $ANDROID_PW_FILE isn't define, prompts the user 647 interactively in the ordinary way. 648 """ 649 650 current = self.ReadFile() 651 652 first = True 653 while True: 654 missing = [] 655 for i in items: 656 if i not in current or not current[i]: 657 missing.append(i) 658 # Are all the passwords already in the file? 659 if not missing: return current 660 661 for i in missing: 662 current[i] = "" 663 664 if not first: 665 print "key file %s still missing some passwords." % (self.pwfile,) 666 answer = raw_input("try to edit again? [y]> ").strip() 667 if answer and answer[0] not in 'yY': 668 raise RuntimeError("key passwords unavailable") 669 first = False 670 671 current = self.UpdateAndReadFile(current) 672 673 def PromptResult(self, current): 674 """Prompt the user to enter a value (password) for each key in 675 'current' whose value is fales. Returns a new dict with all the 676 values. 677 """ 678 result = {} 679 for k, v in sorted(current.iteritems()): 680 if v: 681 result[k] = v 682 else: 683 while True: 684 result[k] = getpass.getpass("Enter password for %s key> " 685 % (k,)).strip() 686 if result[k]: break 687 return result 688 689 def UpdateAndReadFile(self, current): 690 if not self.editor or not self.pwfile: 691 return self.PromptResult(current) 692 693 f = open(self.pwfile, "w") 694 os.chmod(self.pwfile, 0600) 695 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 696 f.write("# (Additional spaces are harmless.)\n\n") 697 698 first_line = None 699 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 700 sorted.sort() 701 for i, (_, k, v) in enumerate(sorted): 702 f.write("[[[ %s ]]] %s\n" % (v, k)) 703 if not v and first_line is None: 704 # position cursor on first line with no password. 705 first_line = i + 4 706 f.close() 707 708 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 709 _, _ = p.communicate() 710 711 return self.ReadFile() 712 713 def ReadFile(self): 714 result = {} 715 if self.pwfile is None: return result 716 try: 717 f = open(self.pwfile, "r") 718 for line in f: 719 line = line.strip() 720 if not line or line[0] == '#': continue 721 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 722 if not m: 723 print "failed to parse password file: ", line 724 else: 725 result[m.group(2)] = m.group(1) 726 f.close() 727 except IOError, e: 728 if e.errno != errno.ENOENT: 729 print "error reading password file: ", str(e) 730 return result 731 732 733def ZipWriteStr(zip, filename, data, perms=0644): 734 # use a fixed timestamp so the output is repeatable. 735 zinfo = zipfile.ZipInfo(filename=filename, 736 date_time=(2009, 1, 1, 0, 0, 0)) 737 zinfo.compress_type = zip.compression 738 zinfo.external_attr = perms << 16 739 zip.writestr(zinfo, data) 740 741 742class DeviceSpecificParams(object): 743 module = None 744 def __init__(self, **kwargs): 745 """Keyword arguments to the constructor become attributes of this 746 object, which is passed to all functions in the device-specific 747 module.""" 748 for k, v in kwargs.iteritems(): 749 setattr(self, k, v) 750 self.extras = OPTIONS.extras 751 752 if self.module is None: 753 path = OPTIONS.device_specific 754 if not path: return 755 try: 756 if os.path.isdir(path): 757 info = imp.find_module("releasetools", [path]) 758 else: 759 d, f = os.path.split(path) 760 b, x = os.path.splitext(f) 761 if x == ".py": 762 f = b 763 info = imp.find_module(f, [d]) 764 self.module = imp.load_module("device_specific", *info) 765 except ImportError: 766 print "unable to load device-specific module; assuming none" 767 768 def _DoCall(self, function_name, *args, **kwargs): 769 """Call the named function in the device-specific module, passing 770 the given args and kwargs. The first argument to the call will be 771 the DeviceSpecific object itself. If there is no module, or the 772 module does not define the function, return the value of the 773 'default' kwarg (which itself defaults to None).""" 774 if self.module is None or not hasattr(self.module, function_name): 775 return kwargs.get("default", None) 776 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 777 778 def FullOTA_Assertions(self): 779 """Called after emitting the block of assertions at the top of a 780 full OTA package. Implementations can add whatever additional 781 assertions they like.""" 782 return self._DoCall("FullOTA_Assertions") 783 784 def FullOTA_InstallBegin(self): 785 """Called at the start of full OTA installation.""" 786 return self._DoCall("FullOTA_InstallBegin") 787 788 def FullOTA_InstallEnd(self): 789 """Called at the end of full OTA installation; typically this is 790 used to install the image for the device's baseband processor.""" 791 return self._DoCall("FullOTA_InstallEnd") 792 793 def IncrementalOTA_Assertions(self): 794 """Called after emitting the block of assertions at the top of an 795 incremental OTA package. Implementations can add whatever 796 additional assertions they like.""" 797 return self._DoCall("IncrementalOTA_Assertions") 798 799 def IncrementalOTA_VerifyBegin(self): 800 """Called at the start of the verification phase of incremental 801 OTA installation; additional checks can be placed here to abort 802 the script before any changes are made.""" 803 return self._DoCall("IncrementalOTA_VerifyBegin") 804 805 def IncrementalOTA_VerifyEnd(self): 806 """Called at the end of the verification phase of incremental OTA 807 installation; additional checks can be placed here to abort the 808 script before any changes are made.""" 809 return self._DoCall("IncrementalOTA_VerifyEnd") 810 811 def IncrementalOTA_InstallBegin(self): 812 """Called at the start of incremental OTA installation (after 813 verification is complete).""" 814 return self._DoCall("IncrementalOTA_InstallBegin") 815 816 def IncrementalOTA_InstallEnd(self): 817 """Called at the end of incremental OTA installation; typically 818 this is used to install the image for the device's baseband 819 processor.""" 820 return self._DoCall("IncrementalOTA_InstallEnd") 821 822class File(object): 823 def __init__(self, name, data): 824 self.name = name 825 self.data = data 826 self.size = len(data) 827 self.sha1 = sha1(data).hexdigest() 828 829 @classmethod 830 def FromLocalFile(cls, name, diskname): 831 f = open(diskname, "rb") 832 data = f.read() 833 f.close() 834 return File(name, data) 835 836 def WriteToTemp(self): 837 t = tempfile.NamedTemporaryFile() 838 t.write(self.data) 839 t.flush() 840 return t 841 842 def AddToZip(self, z): 843 ZipWriteStr(z, self.name, self.data) 844 845DIFF_PROGRAM_BY_EXT = { 846 ".gz" : "imgdiff", 847 ".zip" : ["imgdiff", "-z"], 848 ".jar" : ["imgdiff", "-z"], 849 ".apk" : ["imgdiff", "-z"], 850 ".img" : "imgdiff", 851 } 852 853class Difference(object): 854 def __init__(self, tf, sf, diff_program=None): 855 self.tf = tf 856 self.sf = sf 857 self.patch = None 858 self.diff_program = diff_program 859 860 def ComputePatch(self): 861 """Compute the patch (as a string of data) needed to turn sf into 862 tf. Returns the same tuple as GetPatch().""" 863 864 tf = self.tf 865 sf = self.sf 866 867 if self.diff_program: 868 diff_program = self.diff_program 869 else: 870 ext = os.path.splitext(tf.name)[1] 871 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 872 873 ttemp = tf.WriteToTemp() 874 stemp = sf.WriteToTemp() 875 876 ext = os.path.splitext(tf.name)[1] 877 878 try: 879 ptemp = tempfile.NamedTemporaryFile() 880 if isinstance(diff_program, list): 881 cmd = copy.copy(diff_program) 882 else: 883 cmd = [diff_program] 884 cmd.append(stemp.name) 885 cmd.append(ttemp.name) 886 cmd.append(ptemp.name) 887 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 888 _, err = p.communicate() 889 if err or p.returncode != 0: 890 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 891 return None 892 diff = ptemp.read() 893 finally: 894 ptemp.close() 895 stemp.close() 896 ttemp.close() 897 898 self.patch = diff 899 return self.tf, self.sf, self.patch 900 901 902 def GetPatch(self): 903 """Return a tuple (target_file, source_file, patch_data). 904 patch_data may be None if ComputePatch hasn't been called, or if 905 computing the patch failed.""" 906 return self.tf, self.sf, self.patch 907 908 909def ComputeDifferences(diffs): 910 """Call ComputePatch on all the Difference objects in 'diffs'.""" 911 print len(diffs), "diffs to compute" 912 913 # Do the largest files first, to try and reduce the long-pole effect. 914 by_size = [(i.tf.size, i) for i in diffs] 915 by_size.sort(reverse=True) 916 by_size = [i[1] for i in by_size] 917 918 lock = threading.Lock() 919 diff_iter = iter(by_size) # accessed under lock 920 921 def worker(): 922 try: 923 lock.acquire() 924 for d in diff_iter: 925 lock.release() 926 start = time.time() 927 d.ComputePatch() 928 dur = time.time() - start 929 lock.acquire() 930 931 tf, sf, patch = d.GetPatch() 932 if sf.name == tf.name: 933 name = tf.name 934 else: 935 name = "%s (%s)" % (tf.name, sf.name) 936 if patch is None: 937 print "patching failed! %s" % (name,) 938 else: 939 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 940 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 941 lock.release() 942 except Exception, e: 943 print e 944 raise 945 946 # start worker threads; wait for them all to finish. 947 threads = [threading.Thread(target=worker) 948 for i in range(OPTIONS.worker_threads)] 949 for th in threads: 950 th.start() 951 while threads: 952 threads.pop().join() 953 954 955# map recovery.fstab's fs_types to mount/format "partition types" 956PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 957 "ext4": "EMMC", "emmc": "EMMC" } 958 959def GetTypeAndDevice(mount_point, info): 960 fstab = info["fstab"] 961 if fstab: 962 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 963 else: 964 return None 965 966 967def ParseCertificate(data): 968 """Parse a PEM-format certificate.""" 969 cert = [] 970 save = False 971 for line in data.split("\n"): 972 if "--END CERTIFICATE--" in line: 973 break 974 if save: 975 cert.append(line) 976 if "--BEGIN CERTIFICATE--" in line: 977 save = True 978 cert = "".join(cert).decode('base64') 979 return cert 980