common.py revision d89ffa8623b750f85cbb8be01aba9fa4a9448180
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 103 d = {} 104 try: 105 for line in read_helper("META/misc_info.txt").split("\n"): 106 line = line.strip() 107 if not line or line.startswith("#"): continue 108 k, v = line.split("=", 1) 109 d[k] = v 110 except KeyError: 111 # ok if misc_info.txt doesn't exist 112 pass 113 114 # backwards compatibility: These values used to be in their own 115 # files. Look for them, in case we're processing an old 116 # target_files zip. 117 118 if "mkyaffs2_extra_flags" not in d: 119 try: 120 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 121 except KeyError: 122 # ok if flags don't exist 123 pass 124 125 if "recovery_api_version" not in d: 126 try: 127 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 128 except KeyError: 129 raise ValueError("can't find recovery API version in input target-files") 130 131 if "tool_extensions" not in d: 132 try: 133 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 134 except KeyError: 135 # ok if extensions don't exist 136 pass 137 138 if "fstab_version" not in d: 139 d["fstab_version"] = "1" 140 141 try: 142 data = read_helper("META/imagesizes.txt") 143 for line in data.split("\n"): 144 if not line: continue 145 name, value = line.split(" ", 1) 146 if not value: continue 147 if name == "blocksize": 148 d[name] = value 149 else: 150 d[name + "_size"] = value 151 except KeyError: 152 pass 153 154 def makeint(key): 155 if key in d: 156 d[key] = int(d[key], 0) 157 158 makeint("recovery_api_version") 159 makeint("blocksize") 160 makeint("system_size") 161 makeint("userdata_size") 162 makeint("cache_size") 163 makeint("recovery_size") 164 makeint("boot_size") 165 makeint("fstab_version") 166 167 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 168 d["build.prop"] = LoadBuildProp(read_helper) 169 return d 170 171def LoadBuildProp(read_helper): 172 try: 173 data = read_helper("SYSTEM/build.prop") 174 except KeyError: 175 print "Warning: could not find SYSTEM/build.prop in %s" % zip 176 data = "" 177 178 d = {} 179 for line in data.split("\n"): 180 line = line.strip() 181 if not line or line.startswith("#"): continue 182 name, value = line.split("=", 1) 183 d[name] = value 184 return d 185 186def LoadRecoveryFSTab(read_helper, fstab_version): 187 class Partition(object): 188 pass 189 190 try: 191 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 192 except KeyError: 193 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 194 data = "" 195 196 if fstab_version == 1: 197 d = {} 198 for line in data.split("\n"): 199 line = line.strip() 200 if not line or line.startswith("#"): continue 201 pieces = line.split() 202 if not (3 <= len(pieces) <= 4): 203 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 204 205 p = Partition() 206 p.mount_point = pieces[0] 207 p.fs_type = pieces[1] 208 p.device = pieces[2] 209 p.length = 0 210 options = None 211 if len(pieces) >= 4: 212 if pieces[3].startswith("/"): 213 p.device2 = pieces[3] 214 if len(pieces) >= 5: 215 options = pieces[4] 216 else: 217 p.device2 = None 218 options = pieces[3] 219 else: 220 p.device2 = None 221 222 if options: 223 options = options.split(",") 224 for i in options: 225 if i.startswith("length="): 226 p.length = int(i[7:]) 227 else: 228 print "%s: unknown option \"%s\"" % (p.mount_point, i) 229 230 d[p.mount_point] = p 231 232 elif fstab_version == 2: 233 d = {} 234 for line in data.split("\n"): 235 line = line.strip() 236 if not line or line.startswith("#"): continue 237 pieces = line.split() 238 if len(pieces) != 5: 239 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 240 241 # Ignore entries that are managed by vold 242 options = pieces[4] 243 if "voldmanaged=" in options: continue 244 245 # It's a good line, parse it 246 p = Partition() 247 p.device = pieces[0] 248 p.mount_point = pieces[1] 249 p.fs_type = pieces[2] 250 p.device2 = None 251 p.length = 0 252 253 options = options.split(",") 254 for i in options: 255 if i.startswith("length="): 256 p.length = int(i[7:]) 257 else: 258 # Ignore all unknown options in the unified fstab 259 continue 260 261 d[p.mount_point] = p 262 263 else: 264 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 265 266 return d 267 268 269def DumpInfoDict(d): 270 for k, v in sorted(d.items()): 271 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 272 273def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 274 """Take a kernel, cmdline, and ramdisk directory from the input (in 275 'sourcedir'), and turn them into a boot image. Return the image 276 data, or None if sourcedir does not appear to contains files for 277 building the requested image.""" 278 279 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 280 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 281 return None 282 283 if info_dict is None: 284 info_dict = OPTIONS.info_dict 285 286 ramdisk_img = tempfile.NamedTemporaryFile() 287 img = tempfile.NamedTemporaryFile() 288 289 if os.access(fs_config_file, os.F_OK): 290 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 291 else: 292 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 293 p1 = Run(cmd, stdout=subprocess.PIPE) 294 p2 = Run(["minigzip"], 295 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 296 297 p2.wait() 298 p1.wait() 299 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 300 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 301 302 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 303 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 304 305 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 306 307 fn = os.path.join(sourcedir, "cmdline") 308 if os.access(fn, os.F_OK): 309 cmd.append("--cmdline") 310 cmd.append(open(fn).read().rstrip("\n")) 311 312 fn = os.path.join(sourcedir, "base") 313 if os.access(fn, os.F_OK): 314 cmd.append("--base") 315 cmd.append(open(fn).read().rstrip("\n")) 316 317 fn = os.path.join(sourcedir, "pagesize") 318 if os.access(fn, os.F_OK): 319 cmd.append("--pagesize") 320 cmd.append(open(fn).read().rstrip("\n")) 321 322 args = info_dict.get("mkbootimg_args", None) 323 if args and args.strip(): 324 cmd.extend(shlex.split(args)) 325 326 cmd.extend(["--ramdisk", ramdisk_img.name, 327 "--output", img.name]) 328 329 p = Run(cmd, stdout=subprocess.PIPE) 330 p.communicate() 331 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 332 os.path.basename(sourcedir),) 333 334 img.seek(os.SEEK_SET, 0) 335 data = img.read() 336 337 ramdisk_img.close() 338 img.close() 339 340 return data 341 342 343def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 344 info_dict=None): 345 """Return a File object (with name 'name') with the desired bootable 346 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 347 'prebuilt_name', otherwise construct it from the source files in 348 'unpack_dir'/'tree_subdir'.""" 349 350 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 351 if os.path.exists(prebuilt_path): 352 print "using prebuilt %s..." % (prebuilt_name,) 353 return File.FromLocalFile(name, prebuilt_path) 354 else: 355 print "building image from target_files %s..." % (tree_subdir,) 356 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 357 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 358 os.path.join(unpack_dir, fs_config), 359 info_dict)) 360 if data: 361 return File(name, data) 362 return None 363 364 365def UnzipTemp(filename, pattern=None): 366 """Unzip the given archive into a temporary directory and return the name. 367 368 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 369 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 370 371 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 372 main file), open for reading. 373 """ 374 375 tmp = tempfile.mkdtemp(prefix="targetfiles-") 376 OPTIONS.tempfiles.append(tmp) 377 378 def unzip_to_dir(filename, dirname): 379 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 380 if pattern is not None: 381 cmd.append(pattern) 382 p = Run(cmd, stdout=subprocess.PIPE) 383 p.communicate() 384 if p.returncode != 0: 385 raise ExternalError("failed to unzip input target-files \"%s\"" % 386 (filename,)) 387 388 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 389 if m: 390 unzip_to_dir(m.group(1), tmp) 391 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 392 filename = m.group(1) 393 else: 394 unzip_to_dir(filename, tmp) 395 396 return tmp, zipfile.ZipFile(filename, "r") 397 398 399def GetKeyPasswords(keylist): 400 """Given a list of keys, prompt the user to enter passwords for 401 those which require them. Return a {key: password} dict. password 402 will be None if the key has no password.""" 403 404 no_passwords = [] 405 need_passwords = [] 406 key_passwords = {} 407 devnull = open("/dev/null", "w+b") 408 for k in sorted(keylist): 409 # We don't need a password for things that aren't really keys. 410 if k in SPECIAL_CERT_STRINGS: 411 no_passwords.append(k) 412 continue 413 414 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 415 "-inform", "DER", "-nocrypt"], 416 stdin=devnull.fileno(), 417 stdout=devnull.fileno(), 418 stderr=subprocess.STDOUT) 419 p.communicate() 420 if p.returncode == 0: 421 # Definitely an unencrypted key. 422 no_passwords.append(k) 423 else: 424 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 425 "-inform", "DER", "-passin", "pass:"], 426 stdin=devnull.fileno(), 427 stdout=devnull.fileno(), 428 stderr=subprocess.PIPE) 429 stdout, stderr = p.communicate() 430 if p.returncode == 0: 431 # Encrypted key with empty string as password. 432 key_passwords[k] = '' 433 elif stderr.startswith('Error decrypting key'): 434 # Definitely encrypted key. 435 # It would have said "Error reading key" if it didn't parse correctly. 436 need_passwords.append(k) 437 else: 438 # Potentially, a type of key that openssl doesn't understand. 439 # We'll let the routines in signapk.jar handle it. 440 no_passwords.append(k) 441 devnull.close() 442 443 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 444 key_passwords.update(dict.fromkeys(no_passwords, None)) 445 return key_passwords 446 447 448def SignFile(input_name, output_name, key, password, align=None, 449 whole_file=False): 450 """Sign the input_name zip/jar/apk, producing output_name. Use the 451 given key and password (the latter may be None if the key does not 452 have a password. 453 454 If align is an integer > 1, zipalign is run to align stored files in 455 the output zip on 'align'-byte boundaries. 456 457 If whole_file is true, use the "-w" option to SignApk to embed a 458 signature that covers the whole file in the archive comment of the 459 zip file. 460 """ 461 462 if align == 0 or align == 1: 463 align = None 464 465 if align: 466 temp = tempfile.NamedTemporaryFile() 467 sign_name = temp.name 468 else: 469 sign_name = output_name 470 471 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 472 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 473 cmd.extend(OPTIONS.extra_signapk_args) 474 if whole_file: 475 cmd.append("-w") 476 cmd.extend([key + OPTIONS.public_key_suffix, 477 key + OPTIONS.private_key_suffix, 478 input_name, sign_name]) 479 480 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 481 if password is not None: 482 password += "\n" 483 p.communicate(password) 484 if p.returncode != 0: 485 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 486 487 if align: 488 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 489 p.communicate() 490 if p.returncode != 0: 491 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 492 temp.close() 493 494 495def CheckSize(data, target, info_dict): 496 """Check the data string passed against the max size limit, if 497 any, for the given target. Raise exception if the data is too big. 498 Print a warning if the data is nearing the maximum size.""" 499 500 if target.endswith(".img"): target = target[:-4] 501 mount_point = "/" + target 502 503 if info_dict["fstab"]: 504 if mount_point == "/userdata": mount_point = "/data" 505 p = info_dict["fstab"][mount_point] 506 fs_type = p.fs_type 507 device = p.device 508 if "/" in device: 509 device = device[device.rfind("/")+1:] 510 limit = info_dict.get(device + "_size", None) 511 if not fs_type or not limit: return 512 513 if fs_type == "yaffs2": 514 # image size should be increased by 1/64th to account for the 515 # spare area (64 bytes per 2k page) 516 limit = limit / 2048 * (2048+64) 517 size = len(data) 518 pct = float(size) * 100.0 / limit 519 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 520 if pct >= 99.0: 521 raise ExternalError(msg) 522 elif pct >= 95.0: 523 print 524 print " WARNING: ", msg 525 print 526 elif OPTIONS.verbose: 527 print " ", msg 528 529 530def ReadApkCerts(tf_zip): 531 """Given a target_files ZipFile, parse the META/apkcerts.txt file 532 and return a {package: cert} dict.""" 533 certmap = {} 534 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 535 line = line.strip() 536 if not line: continue 537 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 538 r'private_key="(.*)"$', line) 539 if m: 540 name, cert, privkey = m.groups() 541 public_key_suffix_len = len(OPTIONS.public_key_suffix) 542 private_key_suffix_len = len(OPTIONS.private_key_suffix) 543 if cert in SPECIAL_CERT_STRINGS and not privkey: 544 certmap[name] = cert 545 elif (cert.endswith(OPTIONS.public_key_suffix) and 546 privkey.endswith(OPTIONS.private_key_suffix) and 547 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 548 certmap[name] = cert[:-public_key_suffix_len] 549 else: 550 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 551 return certmap 552 553 554COMMON_DOCSTRING = """ 555 -p (--path) <dir> 556 Prepend <dir>/bin to the list of places to search for binaries 557 run by this script, and expect to find jars in <dir>/framework. 558 559 -s (--device_specific) <file> 560 Path to the python module containing device-specific 561 releasetools code. 562 563 -x (--extra) <key=value> 564 Add a key/value pair to the 'extras' dict, which device-specific 565 extension code may look at. 566 567 -v (--verbose) 568 Show command lines being executed. 569 570 -h (--help) 571 Display this usage message and exit. 572""" 573 574def Usage(docstring): 575 print docstring.rstrip("\n") 576 print COMMON_DOCSTRING 577 578 579def ParseOptions(argv, 580 docstring, 581 extra_opts="", extra_long_opts=(), 582 extra_option_handler=None): 583 """Parse the options in argv and return any arguments that aren't 584 flags. docstring is the calling module's docstring, to be displayed 585 for errors and -h. extra_opts and extra_long_opts are for flags 586 defined by the caller, which are processed by passing them to 587 extra_option_handler.""" 588 589 try: 590 opts, args = getopt.getopt( 591 argv, "hvp:s:x:" + extra_opts, 592 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 593 "java_path=", "public_key_suffix=", "private_key_suffix=", 594 "device_specific=", "extra="] + 595 list(extra_long_opts)) 596 except getopt.GetoptError, err: 597 Usage(docstring) 598 print "**", str(err), "**" 599 sys.exit(2) 600 601 path_specified = False 602 603 for o, a in opts: 604 if o in ("-h", "--help"): 605 Usage(docstring) 606 sys.exit() 607 elif o in ("-v", "--verbose"): 608 OPTIONS.verbose = True 609 elif o in ("-p", "--path"): 610 OPTIONS.search_path = a 611 elif o in ("--signapk_path",): 612 OPTIONS.signapk_path = a 613 elif o in ("--extra_signapk_args",): 614 OPTIONS.extra_signapk_args = shlex.split(a) 615 elif o in ("--java_path",): 616 OPTIONS.java_path = a 617 elif o in ("--public_key_suffix",): 618 OPTIONS.public_key_suffix = a 619 elif o in ("--private_key_suffix",): 620 OPTIONS.private_key_suffix = a 621 elif o in ("-s", "--device_specific"): 622 OPTIONS.device_specific = a 623 elif o in ("-x", "--extra"): 624 key, value = a.split("=", 1) 625 OPTIONS.extras[key] = value 626 else: 627 if extra_option_handler is None or not extra_option_handler(o, a): 628 assert False, "unknown option \"%s\"" % (o,) 629 630 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 631 os.pathsep + os.environ["PATH"]) 632 633 return args 634 635 636def Cleanup(): 637 for i in OPTIONS.tempfiles: 638 if os.path.isdir(i): 639 shutil.rmtree(i) 640 else: 641 os.remove(i) 642 643 644class PasswordManager(object): 645 def __init__(self): 646 self.editor = os.getenv("EDITOR", None) 647 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 648 649 def GetPasswords(self, items): 650 """Get passwords corresponding to each string in 'items', 651 returning a dict. (The dict may have keys in addition to the 652 values in 'items'.) 653 654 Uses the passwords in $ANDROID_PW_FILE if available, letting the 655 user edit that file to add more needed passwords. If no editor is 656 available, or $ANDROID_PW_FILE isn't define, prompts the user 657 interactively in the ordinary way. 658 """ 659 660 current = self.ReadFile() 661 662 first = True 663 while True: 664 missing = [] 665 for i in items: 666 if i not in current or not current[i]: 667 missing.append(i) 668 # Are all the passwords already in the file? 669 if not missing: return current 670 671 for i in missing: 672 current[i] = "" 673 674 if not first: 675 print "key file %s still missing some passwords." % (self.pwfile,) 676 answer = raw_input("try to edit again? [y]> ").strip() 677 if answer and answer[0] not in 'yY': 678 raise RuntimeError("key passwords unavailable") 679 first = False 680 681 current = self.UpdateAndReadFile(current) 682 683 def PromptResult(self, current): 684 """Prompt the user to enter a value (password) for each key in 685 'current' whose value is fales. Returns a new dict with all the 686 values. 687 """ 688 result = {} 689 for k, v in sorted(current.iteritems()): 690 if v: 691 result[k] = v 692 else: 693 while True: 694 result[k] = getpass.getpass("Enter password for %s key> " 695 % (k,)).strip() 696 if result[k]: break 697 return result 698 699 def UpdateAndReadFile(self, current): 700 if not self.editor or not self.pwfile: 701 return self.PromptResult(current) 702 703 f = open(self.pwfile, "w") 704 os.chmod(self.pwfile, 0600) 705 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 706 f.write("# (Additional spaces are harmless.)\n\n") 707 708 first_line = None 709 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 710 sorted.sort() 711 for i, (_, k, v) in enumerate(sorted): 712 f.write("[[[ %s ]]] %s\n" % (v, k)) 713 if not v and first_line is None: 714 # position cursor on first line with no password. 715 first_line = i + 4 716 f.close() 717 718 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 719 _, _ = p.communicate() 720 721 return self.ReadFile() 722 723 def ReadFile(self): 724 result = {} 725 if self.pwfile is None: return result 726 try: 727 f = open(self.pwfile, "r") 728 for line in f: 729 line = line.strip() 730 if not line or line[0] == '#': continue 731 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 732 if not m: 733 print "failed to parse password file: ", line 734 else: 735 result[m.group(2)] = m.group(1) 736 f.close() 737 except IOError, e: 738 if e.errno != errno.ENOENT: 739 print "error reading password file: ", str(e) 740 return result 741 742 743def ZipWriteStr(zip, filename, data, perms=0644): 744 # use a fixed timestamp so the output is repeatable. 745 zinfo = zipfile.ZipInfo(filename=filename, 746 date_time=(2009, 1, 1, 0, 0, 0)) 747 zinfo.compress_type = zip.compression 748 zinfo.external_attr = perms << 16 749 zip.writestr(zinfo, data) 750 751 752class DeviceSpecificParams(object): 753 module = None 754 def __init__(self, **kwargs): 755 """Keyword arguments to the constructor become attributes of this 756 object, which is passed to all functions in the device-specific 757 module.""" 758 for k, v in kwargs.iteritems(): 759 setattr(self, k, v) 760 self.extras = OPTIONS.extras 761 762 if self.module is None: 763 path = OPTIONS.device_specific 764 if not path: return 765 try: 766 if os.path.isdir(path): 767 info = imp.find_module("releasetools", [path]) 768 else: 769 d, f = os.path.split(path) 770 b, x = os.path.splitext(f) 771 if x == ".py": 772 f = b 773 info = imp.find_module(f, [d]) 774 print "loaded device-specific extensions from", path 775 self.module = imp.load_module("device_specific", *info) 776 except ImportError: 777 print "unable to load device-specific module; assuming none" 778 779 def _DoCall(self, function_name, *args, **kwargs): 780 """Call the named function in the device-specific module, passing 781 the given args and kwargs. The first argument to the call will be 782 the DeviceSpecific object itself. If there is no module, or the 783 module does not define the function, return the value of the 784 'default' kwarg (which itself defaults to None).""" 785 if self.module is None or not hasattr(self.module, function_name): 786 return kwargs.get("default", None) 787 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 788 789 def FullOTA_Assertions(self): 790 """Called after emitting the block of assertions at the top of a 791 full OTA package. Implementations can add whatever additional 792 assertions they like.""" 793 return self._DoCall("FullOTA_Assertions") 794 795 def FullOTA_InstallBegin(self): 796 """Called at the start of full OTA installation.""" 797 return self._DoCall("FullOTA_InstallBegin") 798 799 def FullOTA_InstallEnd(self): 800 """Called at the end of full OTA installation; typically this is 801 used to install the image for the device's baseband processor.""" 802 return self._DoCall("FullOTA_InstallEnd") 803 804 def IncrementalOTA_Assertions(self): 805 """Called after emitting the block of assertions at the top of an 806 incremental OTA package. Implementations can add whatever 807 additional assertions they like.""" 808 return self._DoCall("IncrementalOTA_Assertions") 809 810 def IncrementalOTA_VerifyBegin(self): 811 """Called at the start of the verification phase of incremental 812 OTA installation; additional checks can be placed here to abort 813 the script before any changes are made.""" 814 return self._DoCall("IncrementalOTA_VerifyBegin") 815 816 def IncrementalOTA_VerifyEnd(self): 817 """Called at the end of the verification phase of incremental OTA 818 installation; additional checks can be placed here to abort the 819 script before any changes are made.""" 820 return self._DoCall("IncrementalOTA_VerifyEnd") 821 822 def IncrementalOTA_InstallBegin(self): 823 """Called at the start of incremental OTA installation (after 824 verification is complete).""" 825 return self._DoCall("IncrementalOTA_InstallBegin") 826 827 def IncrementalOTA_InstallEnd(self): 828 """Called at the end of incremental OTA installation; typically 829 this is used to install the image for the device's baseband 830 processor.""" 831 return self._DoCall("IncrementalOTA_InstallEnd") 832 833class File(object): 834 def __init__(self, name, data): 835 self.name = name 836 self.data = data 837 self.size = len(data) 838 self.sha1 = sha1(data).hexdigest() 839 840 @classmethod 841 def FromLocalFile(cls, name, diskname): 842 f = open(diskname, "rb") 843 data = f.read() 844 f.close() 845 return File(name, data) 846 847 def WriteToTemp(self): 848 t = tempfile.NamedTemporaryFile() 849 t.write(self.data) 850 t.flush() 851 return t 852 853 def AddToZip(self, z): 854 ZipWriteStr(z, self.name, self.data) 855 856DIFF_PROGRAM_BY_EXT = { 857 ".gz" : "imgdiff", 858 ".zip" : ["imgdiff", "-z"], 859 ".jar" : ["imgdiff", "-z"], 860 ".apk" : ["imgdiff", "-z"], 861 ".img" : "imgdiff", 862 } 863 864class Difference(object): 865 def __init__(self, tf, sf, diff_program=None): 866 self.tf = tf 867 self.sf = sf 868 self.patch = None 869 self.diff_program = diff_program 870 871 def ComputePatch(self): 872 """Compute the patch (as a string of data) needed to turn sf into 873 tf. Returns the same tuple as GetPatch().""" 874 875 tf = self.tf 876 sf = self.sf 877 878 if self.diff_program: 879 diff_program = self.diff_program 880 else: 881 ext = os.path.splitext(tf.name)[1] 882 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 883 884 ttemp = tf.WriteToTemp() 885 stemp = sf.WriteToTemp() 886 887 ext = os.path.splitext(tf.name)[1] 888 889 try: 890 ptemp = tempfile.NamedTemporaryFile() 891 if isinstance(diff_program, list): 892 cmd = copy.copy(diff_program) 893 else: 894 cmd = [diff_program] 895 cmd.append(stemp.name) 896 cmd.append(ttemp.name) 897 cmd.append(ptemp.name) 898 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 899 _, err = p.communicate() 900 if err or p.returncode != 0: 901 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 902 return None 903 diff = ptemp.read() 904 finally: 905 ptemp.close() 906 stemp.close() 907 ttemp.close() 908 909 self.patch = diff 910 return self.tf, self.sf, self.patch 911 912 913 def GetPatch(self): 914 """Return a tuple (target_file, source_file, patch_data). 915 patch_data may be None if ComputePatch hasn't been called, or if 916 computing the patch failed.""" 917 return self.tf, self.sf, self.patch 918 919 920def ComputeDifferences(diffs): 921 """Call ComputePatch on all the Difference objects in 'diffs'.""" 922 print len(diffs), "diffs to compute" 923 924 # Do the largest files first, to try and reduce the long-pole effect. 925 by_size = [(i.tf.size, i) for i in diffs] 926 by_size.sort(reverse=True) 927 by_size = [i[1] for i in by_size] 928 929 lock = threading.Lock() 930 diff_iter = iter(by_size) # accessed under lock 931 932 def worker(): 933 try: 934 lock.acquire() 935 for d in diff_iter: 936 lock.release() 937 start = time.time() 938 d.ComputePatch() 939 dur = time.time() - start 940 lock.acquire() 941 942 tf, sf, patch = d.GetPatch() 943 if sf.name == tf.name: 944 name = tf.name 945 else: 946 name = "%s (%s)" % (tf.name, sf.name) 947 if patch is None: 948 print "patching failed! %s" % (name,) 949 else: 950 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 951 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 952 lock.release() 953 except Exception, e: 954 print e 955 raise 956 957 # start worker threads; wait for them all to finish. 958 threads = [threading.Thread(target=worker) 959 for i in range(OPTIONS.worker_threads)] 960 for th in threads: 961 th.start() 962 while threads: 963 threads.pop().join() 964 965 966# map recovery.fstab's fs_types to mount/format "partition types" 967PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 968 "ext4": "EMMC", "emmc": "EMMC" } 969 970def GetTypeAndDevice(mount_point, info): 971 fstab = info["fstab"] 972 if fstab: 973 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 974 else: 975 return None 976 977 978def ParseCertificate(data): 979 """Parse a PEM-format certificate.""" 980 cert = [] 981 save = False 982 for line in data.split("\n"): 983 if "--END CERTIFICATE--" in line: 984 break 985 if save: 986 cert.append(line) 987 if "--BEGIN CERTIFICATE--" in line: 988 save = True 989 cert = "".join(cert).decode('base64') 990 return cert 991 992 993def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img): 994 """Generate a binary patch that creates the recovery image starting 995 with the boot image. (Most of the space in these images is just the 996 kernel, which is identical for the two, so the resulting patch 997 should be efficient.) Add it to the output zip, along with a shell 998 script that is run from init.rc on first boot to actually do the 999 patching and install the new recovery image. 1000 1001 recovery_img and boot_img should be File objects for the 1002 corresponding images. info should be the dictionary returned by 1003 common.LoadInfoDict() on the input target_files. 1004 """ 1005 1006 diff_program = ["imgdiff"] 1007 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1008 if os.path.exists(path): 1009 diff_program.append("-b") 1010 diff_program.append(path) 1011 bonus_args = "-b /system/etc/recovery-resource.dat" 1012 else: 1013 bonus_args = "" 1014 1015 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1016 _, _, patch = d.ComputePatch() 1017 output_sink("recovery-from-boot.p", patch) 1018 1019 boot_type, boot_device = GetTypeAndDevice("/boot", OPTIONS.info_dict) 1020 recovery_type, recovery_device = GetTypeAndDevice("/recovery", OPTIONS.info_dict) 1021 1022 sh = """#!/system/bin/sh 1023if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1024 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1025else 1026 log -t recovery "Recovery image already installed" 1027fi 1028""" % { 'boot_size': boot_img.size, 1029 'boot_sha1': boot_img.sha1, 1030 'recovery_size': recovery_img.size, 1031 'recovery_sha1': recovery_img.sha1, 1032 'boot_type': boot_type, 1033 'boot_device': boot_device, 1034 'recovery_type': recovery_type, 1035 'recovery_device': recovery_device, 1036 'bonus_args': bonus_args, 1037 } 1038 1039 # The install script location moved from /system/etc to /system/bin 1040 # in the L release. Parse the init.rc file to find out where the 1041 # target-files expects it to be, and put it there. 1042 sh_location = "etc/install-recovery.sh" 1043 try: 1044 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1045 for line in f: 1046 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1047 if m: 1048 sh_location = m.group(1) 1049 print "putting script in", sh_location 1050 break 1051 except (OSError, IOError), e: 1052 print "failed to read init.rc: %s" % (e,) 1053 1054 output_sink(sh_location, sh) 1055