common.py revision 2347a3b7d9c8892c1556c08ba3e3bdf00e7f4509
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 d = {} 103 try: 104 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 105 except KeyError: 106 # ok if misc_info.txt doesn't exist 107 pass 108 109 # backwards compatibility: These values used to be in their own 110 # files. Look for them, in case we're processing an old 111 # target_files zip. 112 113 if "mkyaffs2_extra_flags" not in d: 114 try: 115 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 116 except KeyError: 117 # ok if flags don't exist 118 pass 119 120 if "recovery_api_version" not in d: 121 try: 122 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 123 except KeyError: 124 raise ValueError("can't find recovery API version in input target-files") 125 126 if "tool_extensions" not in d: 127 try: 128 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 129 except KeyError: 130 # ok if extensions don't exist 131 pass 132 133 if "fstab_version" not in d: 134 d["fstab_version"] = "1" 135 136 try: 137 data = read_helper("META/imagesizes.txt") 138 for line in data.split("\n"): 139 if not line: continue 140 name, value = line.split(" ", 1) 141 if not value: continue 142 if name == "blocksize": 143 d[name] = value 144 else: 145 d[name + "_size"] = value 146 except KeyError: 147 pass 148 149 def makeint(key): 150 if key in d: 151 d[key] = int(d[key], 0) 152 153 makeint("recovery_api_version") 154 makeint("blocksize") 155 makeint("system_size") 156 makeint("vendor_size") 157 makeint("userdata_size") 158 makeint("cache_size") 159 makeint("recovery_size") 160 makeint("boot_size") 161 makeint("fstab_version") 162 163 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 164 d["build.prop"] = LoadBuildProp(read_helper) 165 return d 166 167def LoadBuildProp(read_helper): 168 try: 169 data = read_helper("SYSTEM/build.prop") 170 except KeyError: 171 print "Warning: could not find SYSTEM/build.prop in %s" % zip 172 data = "" 173 return LoadDictionaryFromLines(data.split("\n")) 174 175def LoadDictionaryFromLines(lines): 176 d = {} 177 for line in lines: 178 line = line.strip() 179 if not line or line.startswith("#"): continue 180 if "=" in line: 181 name, value = line.split("=", 1) 182 d[name] = value 183 return d 184 185def LoadRecoveryFSTab(read_helper, fstab_version): 186 class Partition(object): 187 pass 188 189 try: 190 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 191 except KeyError: 192 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 193 data = "" 194 195 if fstab_version == 1: 196 d = {} 197 for line in data.split("\n"): 198 line = line.strip() 199 if not line or line.startswith("#"): continue 200 pieces = line.split() 201 if not (3 <= len(pieces) <= 4): 202 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 203 204 p = Partition() 205 p.mount_point = pieces[0] 206 p.fs_type = pieces[1] 207 p.device = pieces[2] 208 p.length = 0 209 options = None 210 if len(pieces) >= 4: 211 if pieces[3].startswith("/"): 212 p.device2 = pieces[3] 213 if len(pieces) >= 5: 214 options = pieces[4] 215 else: 216 p.device2 = None 217 options = pieces[3] 218 else: 219 p.device2 = None 220 221 if options: 222 options = options.split(",") 223 for i in options: 224 if i.startswith("length="): 225 p.length = int(i[7:]) 226 else: 227 print "%s: unknown option \"%s\"" % (p.mount_point, i) 228 229 d[p.mount_point] = p 230 231 elif fstab_version == 2: 232 d = {} 233 for line in data.split("\n"): 234 line = line.strip() 235 if not line or line.startswith("#"): continue 236 pieces = line.split() 237 if len(pieces) != 5: 238 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 239 240 # Ignore entries that are managed by vold 241 options = pieces[4] 242 if "voldmanaged=" in options: continue 243 244 # It's a good line, parse it 245 p = Partition() 246 p.device = pieces[0] 247 p.mount_point = pieces[1] 248 p.fs_type = pieces[2] 249 p.device2 = None 250 p.length = 0 251 252 options = options.split(",") 253 for i in options: 254 if i.startswith("length="): 255 p.length = int(i[7:]) 256 else: 257 # Ignore all unknown options in the unified fstab 258 continue 259 260 d[p.mount_point] = p 261 262 else: 263 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 264 265 return d 266 267 268def DumpInfoDict(d): 269 for k, v in sorted(d.items()): 270 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 271 272def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 273 """Take a kernel, cmdline, and ramdisk directory from the input (in 274 'sourcedir'), and turn them into a boot image. Return the image 275 data, or None if sourcedir does not appear to contains files for 276 building the requested image.""" 277 278 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 279 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 280 return None 281 282 if info_dict is None: 283 info_dict = OPTIONS.info_dict 284 285 ramdisk_img = tempfile.NamedTemporaryFile() 286 img = tempfile.NamedTemporaryFile() 287 288 if os.access(fs_config_file, os.F_OK): 289 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 290 else: 291 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 292 p1 = Run(cmd, stdout=subprocess.PIPE) 293 p2 = Run(["minigzip"], 294 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 295 296 p2.wait() 297 p1.wait() 298 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 299 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 300 301 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 302 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 303 304 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 305 306 fn = os.path.join(sourcedir, "second") 307 if os.access(fn, os.F_OK): 308 cmd.append("--second") 309 cmd.append(fn) 310 311 fn = os.path.join(sourcedir, "cmdline") 312 if os.access(fn, os.F_OK): 313 cmd.append("--cmdline") 314 cmd.append(open(fn).read().rstrip("\n")) 315 316 fn = os.path.join(sourcedir, "base") 317 if os.access(fn, os.F_OK): 318 cmd.append("--base") 319 cmd.append(open(fn).read().rstrip("\n")) 320 321 fn = os.path.join(sourcedir, "pagesize") 322 if os.access(fn, os.F_OK): 323 cmd.append("--pagesize") 324 cmd.append(open(fn).read().rstrip("\n")) 325 326 args = info_dict.get("mkbootimg_args", None) 327 if args and args.strip(): 328 cmd.extend(shlex.split(args)) 329 330 cmd.extend(["--ramdisk", ramdisk_img.name, 331 "--output", img.name]) 332 333 p = Run(cmd, stdout=subprocess.PIPE) 334 p.communicate() 335 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 336 os.path.basename(sourcedir),) 337 338 img.seek(os.SEEK_SET, 0) 339 data = img.read() 340 341 ramdisk_img.close() 342 img.close() 343 344 return data 345 346 347def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 348 info_dict=None): 349 """Return a File object (with name 'name') with the desired bootable 350 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 351 'prebuilt_name', otherwise construct it from the source files in 352 'unpack_dir'/'tree_subdir'.""" 353 354 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 355 if os.path.exists(prebuilt_path): 356 print "using prebuilt %s..." % (prebuilt_name,) 357 return File.FromLocalFile(name, prebuilt_path) 358 else: 359 print "building image from target_files %s..." % (tree_subdir,) 360 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 361 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 362 os.path.join(unpack_dir, fs_config), 363 info_dict) 364 if data: 365 return File(name, data) 366 return None 367 368 369def UnzipTemp(filename, pattern=None): 370 """Unzip the given archive into a temporary directory and return the name. 371 372 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 373 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 374 375 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 376 main file), open for reading. 377 """ 378 379 tmp = tempfile.mkdtemp(prefix="targetfiles-") 380 OPTIONS.tempfiles.append(tmp) 381 382 def unzip_to_dir(filename, dirname): 383 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 384 if pattern is not None: 385 cmd.append(pattern) 386 p = Run(cmd, stdout=subprocess.PIPE) 387 p.communicate() 388 if p.returncode != 0: 389 raise ExternalError("failed to unzip input target-files \"%s\"" % 390 (filename,)) 391 392 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 393 if m: 394 unzip_to_dir(m.group(1), tmp) 395 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 396 filename = m.group(1) 397 else: 398 unzip_to_dir(filename, tmp) 399 400 return tmp, zipfile.ZipFile(filename, "r") 401 402 403def GetKeyPasswords(keylist): 404 """Given a list of keys, prompt the user to enter passwords for 405 those which require them. Return a {key: password} dict. password 406 will be None if the key has no password.""" 407 408 no_passwords = [] 409 need_passwords = [] 410 key_passwords = {} 411 devnull = open("/dev/null", "w+b") 412 for k in sorted(keylist): 413 # We don't need a password for things that aren't really keys. 414 if k in SPECIAL_CERT_STRINGS: 415 no_passwords.append(k) 416 continue 417 418 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 419 "-inform", "DER", "-nocrypt"], 420 stdin=devnull.fileno(), 421 stdout=devnull.fileno(), 422 stderr=subprocess.STDOUT) 423 p.communicate() 424 if p.returncode == 0: 425 # Definitely an unencrypted key. 426 no_passwords.append(k) 427 else: 428 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 429 "-inform", "DER", "-passin", "pass:"], 430 stdin=devnull.fileno(), 431 stdout=devnull.fileno(), 432 stderr=subprocess.PIPE) 433 stdout, stderr = p.communicate() 434 if p.returncode == 0: 435 # Encrypted key with empty string as password. 436 key_passwords[k] = '' 437 elif stderr.startswith('Error decrypting key'): 438 # Definitely encrypted key. 439 # It would have said "Error reading key" if it didn't parse correctly. 440 need_passwords.append(k) 441 else: 442 # Potentially, a type of key that openssl doesn't understand. 443 # We'll let the routines in signapk.jar handle it. 444 no_passwords.append(k) 445 devnull.close() 446 447 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 448 key_passwords.update(dict.fromkeys(no_passwords, None)) 449 return key_passwords 450 451 452def SignFile(input_name, output_name, key, password, align=None, 453 whole_file=False): 454 """Sign the input_name zip/jar/apk, producing output_name. Use the 455 given key and password (the latter may be None if the key does not 456 have a password. 457 458 If align is an integer > 1, zipalign is run to align stored files in 459 the output zip on 'align'-byte boundaries. 460 461 If whole_file is true, use the "-w" option to SignApk to embed a 462 signature that covers the whole file in the archive comment of the 463 zip file. 464 """ 465 466 if align == 0 or align == 1: 467 align = None 468 469 if align: 470 temp = tempfile.NamedTemporaryFile() 471 sign_name = temp.name 472 else: 473 sign_name = output_name 474 475 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 476 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 477 cmd.extend(OPTIONS.extra_signapk_args) 478 if whole_file: 479 cmd.append("-w") 480 cmd.extend([key + OPTIONS.public_key_suffix, 481 key + OPTIONS.private_key_suffix, 482 input_name, sign_name]) 483 484 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 485 if password is not None: 486 password += "\n" 487 p.communicate(password) 488 if p.returncode != 0: 489 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 490 491 if align: 492 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 493 p.communicate() 494 if p.returncode != 0: 495 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 496 temp.close() 497 498 499def CheckSize(data, target, info_dict): 500 """Check the data string passed against the max size limit, if 501 any, for the given target. Raise exception if the data is too big. 502 Print a warning if the data is nearing the maximum size.""" 503 504 if target.endswith(".img"): target = target[:-4] 505 mount_point = "/" + target 506 507 fs_type = None 508 limit = None 509 if info_dict["fstab"]: 510 if mount_point == "/userdata": mount_point = "/data" 511 p = info_dict["fstab"][mount_point] 512 fs_type = p.fs_type 513 device = p.device 514 if "/" in device: 515 device = device[device.rfind("/")+1:] 516 limit = info_dict.get(device + "_size", None) 517 if not fs_type or not limit: return 518 519 if fs_type == "yaffs2": 520 # image size should be increased by 1/64th to account for the 521 # spare area (64 bytes per 2k page) 522 limit = limit / 2048 * (2048+64) 523 size = len(data) 524 pct = float(size) * 100.0 / limit 525 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 526 if pct >= 99.0: 527 raise ExternalError(msg) 528 elif pct >= 95.0: 529 print 530 print " WARNING: ", msg 531 print 532 elif OPTIONS.verbose: 533 print " ", msg 534 535 536def ReadApkCerts(tf_zip): 537 """Given a target_files ZipFile, parse the META/apkcerts.txt file 538 and return a {package: cert} dict.""" 539 certmap = {} 540 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 541 line = line.strip() 542 if not line: continue 543 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 544 r'private_key="(.*)"$', line) 545 if m: 546 name, cert, privkey = m.groups() 547 public_key_suffix_len = len(OPTIONS.public_key_suffix) 548 private_key_suffix_len = len(OPTIONS.private_key_suffix) 549 if cert in SPECIAL_CERT_STRINGS and not privkey: 550 certmap[name] = cert 551 elif (cert.endswith(OPTIONS.public_key_suffix) and 552 privkey.endswith(OPTIONS.private_key_suffix) and 553 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 554 certmap[name] = cert[:-public_key_suffix_len] 555 else: 556 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 557 return certmap 558 559 560COMMON_DOCSTRING = """ 561 -p (--path) <dir> 562 Prepend <dir>/bin to the list of places to search for binaries 563 run by this script, and expect to find jars in <dir>/framework. 564 565 -s (--device_specific) <file> 566 Path to the python module containing device-specific 567 releasetools code. 568 569 -x (--extra) <key=value> 570 Add a key/value pair to the 'extras' dict, which device-specific 571 extension code may look at. 572 573 -v (--verbose) 574 Show command lines being executed. 575 576 -h (--help) 577 Display this usage message and exit. 578""" 579 580def Usage(docstring): 581 print docstring.rstrip("\n") 582 print COMMON_DOCSTRING 583 584 585def ParseOptions(argv, 586 docstring, 587 extra_opts="", extra_long_opts=(), 588 extra_option_handler=None): 589 """Parse the options in argv and return any arguments that aren't 590 flags. docstring is the calling module's docstring, to be displayed 591 for errors and -h. extra_opts and extra_long_opts are for flags 592 defined by the caller, which are processed by passing them to 593 extra_option_handler.""" 594 595 try: 596 opts, args = getopt.getopt( 597 argv, "hvp:s:x:" + extra_opts, 598 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 599 "java_path=", "public_key_suffix=", "private_key_suffix=", 600 "device_specific=", "extra="] + 601 list(extra_long_opts)) 602 except getopt.GetoptError, err: 603 Usage(docstring) 604 print "**", str(err), "**" 605 sys.exit(2) 606 607 path_specified = False 608 609 for o, a in opts: 610 if o in ("-h", "--help"): 611 Usage(docstring) 612 sys.exit() 613 elif o in ("-v", "--verbose"): 614 OPTIONS.verbose = True 615 elif o in ("-p", "--path"): 616 OPTIONS.search_path = a 617 elif o in ("--signapk_path",): 618 OPTIONS.signapk_path = a 619 elif o in ("--extra_signapk_args",): 620 OPTIONS.extra_signapk_args = shlex.split(a) 621 elif o in ("--java_path",): 622 OPTIONS.java_path = a 623 elif o in ("--public_key_suffix",): 624 OPTIONS.public_key_suffix = a 625 elif o in ("--private_key_suffix",): 626 OPTIONS.private_key_suffix = a 627 elif o in ("-s", "--device_specific"): 628 OPTIONS.device_specific = a 629 elif o in ("-x", "--extra"): 630 key, value = a.split("=", 1) 631 OPTIONS.extras[key] = value 632 else: 633 if extra_option_handler is None or not extra_option_handler(o, a): 634 assert False, "unknown option \"%s\"" % (o,) 635 636 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 637 os.pathsep + os.environ["PATH"]) 638 639 return args 640 641 642def Cleanup(): 643 for i in OPTIONS.tempfiles: 644 if os.path.isdir(i): 645 shutil.rmtree(i) 646 else: 647 os.remove(i) 648 649 650class PasswordManager(object): 651 def __init__(self): 652 self.editor = os.getenv("EDITOR", None) 653 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 654 655 def GetPasswords(self, items): 656 """Get passwords corresponding to each string in 'items', 657 returning a dict. (The dict may have keys in addition to the 658 values in 'items'.) 659 660 Uses the passwords in $ANDROID_PW_FILE if available, letting the 661 user edit that file to add more needed passwords. If no editor is 662 available, or $ANDROID_PW_FILE isn't define, prompts the user 663 interactively in the ordinary way. 664 """ 665 666 current = self.ReadFile() 667 668 first = True 669 while True: 670 missing = [] 671 for i in items: 672 if i not in current or not current[i]: 673 missing.append(i) 674 # Are all the passwords already in the file? 675 if not missing: return current 676 677 for i in missing: 678 current[i] = "" 679 680 if not first: 681 print "key file %s still missing some passwords." % (self.pwfile,) 682 answer = raw_input("try to edit again? [y]> ").strip() 683 if answer and answer[0] not in 'yY': 684 raise RuntimeError("key passwords unavailable") 685 first = False 686 687 current = self.UpdateAndReadFile(current) 688 689 def PromptResult(self, current): 690 """Prompt the user to enter a value (password) for each key in 691 'current' whose value is fales. Returns a new dict with all the 692 values. 693 """ 694 result = {} 695 for k, v in sorted(current.iteritems()): 696 if v: 697 result[k] = v 698 else: 699 while True: 700 result[k] = getpass.getpass("Enter password for %s key> " 701 % (k,)).strip() 702 if result[k]: break 703 return result 704 705 def UpdateAndReadFile(self, current): 706 if not self.editor or not self.pwfile: 707 return self.PromptResult(current) 708 709 f = open(self.pwfile, "w") 710 os.chmod(self.pwfile, 0600) 711 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 712 f.write("# (Additional spaces are harmless.)\n\n") 713 714 first_line = None 715 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 716 sorted.sort() 717 for i, (_, k, v) in enumerate(sorted): 718 f.write("[[[ %s ]]] %s\n" % (v, k)) 719 if not v and first_line is None: 720 # position cursor on first line with no password. 721 first_line = i + 4 722 f.close() 723 724 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 725 _, _ = p.communicate() 726 727 return self.ReadFile() 728 729 def ReadFile(self): 730 result = {} 731 if self.pwfile is None: return result 732 try: 733 f = open(self.pwfile, "r") 734 for line in f: 735 line = line.strip() 736 if not line or line[0] == '#': continue 737 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 738 if not m: 739 print "failed to parse password file: ", line 740 else: 741 result[m.group(2)] = m.group(1) 742 f.close() 743 except IOError, e: 744 if e.errno != errno.ENOENT: 745 print "error reading password file: ", str(e) 746 return result 747 748 749def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 750 # use a fixed timestamp so the output is repeatable. 751 zinfo = zipfile.ZipInfo(filename=filename, 752 date_time=(2009, 1, 1, 0, 0, 0)) 753 if compression is None: 754 zinfo.compress_type = zip.compression 755 else: 756 zinfo.compress_type = compression 757 zinfo.external_attr = perms << 16 758 zip.writestr(zinfo, data) 759 760 761class DeviceSpecificParams(object): 762 module = None 763 def __init__(self, **kwargs): 764 """Keyword arguments to the constructor become attributes of this 765 object, which is passed to all functions in the device-specific 766 module.""" 767 for k, v in kwargs.iteritems(): 768 setattr(self, k, v) 769 self.extras = OPTIONS.extras 770 771 if self.module is None: 772 path = OPTIONS.device_specific 773 if not path: return 774 try: 775 if os.path.isdir(path): 776 info = imp.find_module("releasetools", [path]) 777 else: 778 d, f = os.path.split(path) 779 b, x = os.path.splitext(f) 780 if x == ".py": 781 f = b 782 info = imp.find_module(f, [d]) 783 print "loaded device-specific extensions from", path 784 self.module = imp.load_module("device_specific", *info) 785 except ImportError: 786 print "unable to load device-specific module; assuming none" 787 788 def _DoCall(self, function_name, *args, **kwargs): 789 """Call the named function in the device-specific module, passing 790 the given args and kwargs. The first argument to the call will be 791 the DeviceSpecific object itself. If there is no module, or the 792 module does not define the function, return the value of the 793 'default' kwarg (which itself defaults to None).""" 794 if self.module is None or not hasattr(self.module, function_name): 795 return kwargs.get("default", None) 796 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 797 798 def FullOTA_Assertions(self): 799 """Called after emitting the block of assertions at the top of a 800 full OTA package. Implementations can add whatever additional 801 assertions they like.""" 802 return self._DoCall("FullOTA_Assertions") 803 804 def FullOTA_InstallBegin(self): 805 """Called at the start of full OTA installation.""" 806 return self._DoCall("FullOTA_InstallBegin") 807 808 def FullOTA_InstallEnd(self): 809 """Called at the end of full OTA installation; typically this is 810 used to install the image for the device's baseband processor.""" 811 return self._DoCall("FullOTA_InstallEnd") 812 813 def IncrementalOTA_Assertions(self): 814 """Called after emitting the block of assertions at the top of an 815 incremental OTA package. Implementations can add whatever 816 additional assertions they like.""" 817 return self._DoCall("IncrementalOTA_Assertions") 818 819 def IncrementalOTA_VerifyBegin(self): 820 """Called at the start of the verification phase of incremental 821 OTA installation; additional checks can be placed here to abort 822 the script before any changes are made.""" 823 return self._DoCall("IncrementalOTA_VerifyBegin") 824 825 def IncrementalOTA_VerifyEnd(self): 826 """Called at the end of the verification phase of incremental OTA 827 installation; additional checks can be placed here to abort the 828 script before any changes are made.""" 829 return self._DoCall("IncrementalOTA_VerifyEnd") 830 831 def IncrementalOTA_InstallBegin(self): 832 """Called at the start of incremental OTA installation (after 833 verification is complete).""" 834 return self._DoCall("IncrementalOTA_InstallBegin") 835 836 def IncrementalOTA_InstallEnd(self): 837 """Called at the end of incremental OTA installation; typically 838 this is used to install the image for the device's baseband 839 processor.""" 840 return self._DoCall("IncrementalOTA_InstallEnd") 841 842class File(object): 843 def __init__(self, name, data): 844 self.name = name 845 self.data = data 846 self.size = len(data) 847 self.sha1 = sha1(data).hexdigest() 848 849 @classmethod 850 def FromLocalFile(cls, name, diskname): 851 f = open(diskname, "rb") 852 data = f.read() 853 f.close() 854 return File(name, data) 855 856 def WriteToTemp(self): 857 t = tempfile.NamedTemporaryFile() 858 t.write(self.data) 859 t.flush() 860 return t 861 862 def AddToZip(self, z, compression=None): 863 ZipWriteStr(z, self.name, self.data, compression=compression) 864 865DIFF_PROGRAM_BY_EXT = { 866 ".gz" : "imgdiff", 867 ".zip" : ["imgdiff", "-z"], 868 ".jar" : ["imgdiff", "-z"], 869 ".apk" : ["imgdiff", "-z"], 870 ".img" : "imgdiff", 871 } 872 873class Difference(object): 874 def __init__(self, tf, sf, diff_program=None): 875 self.tf = tf 876 self.sf = sf 877 self.patch = None 878 self.diff_program = diff_program 879 880 def ComputePatch(self): 881 """Compute the patch (as a string of data) needed to turn sf into 882 tf. Returns the same tuple as GetPatch().""" 883 884 tf = self.tf 885 sf = self.sf 886 887 if self.diff_program: 888 diff_program = self.diff_program 889 else: 890 ext = os.path.splitext(tf.name)[1] 891 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 892 893 ttemp = tf.WriteToTemp() 894 stemp = sf.WriteToTemp() 895 896 ext = os.path.splitext(tf.name)[1] 897 898 try: 899 ptemp = tempfile.NamedTemporaryFile() 900 if isinstance(diff_program, list): 901 cmd = copy.copy(diff_program) 902 else: 903 cmd = [diff_program] 904 cmd.append(stemp.name) 905 cmd.append(ttemp.name) 906 cmd.append(ptemp.name) 907 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 908 err = [] 909 def run(): 910 _, e = p.communicate() 911 if e: err.append(e) 912 th = threading.Thread(target=run) 913 th.start() 914 th.join(timeout=300) # 5 mins 915 if th.is_alive(): 916 print "WARNING: diff command timed out" 917 p.terminate() 918 th.join(5) 919 if th.is_alive(): 920 p.kill() 921 th.join() 922 923 if err or p.returncode != 0: 924 print "WARNING: failure running %s:\n%s\n" % ( 925 diff_program, "".join(err)) 926 self.patch = None 927 return None, None, None 928 diff = ptemp.read() 929 finally: 930 ptemp.close() 931 stemp.close() 932 ttemp.close() 933 934 self.patch = diff 935 return self.tf, self.sf, self.patch 936 937 938 def GetPatch(self): 939 """Return a tuple (target_file, source_file, patch_data). 940 patch_data may be None if ComputePatch hasn't been called, or if 941 computing the patch failed.""" 942 return self.tf, self.sf, self.patch 943 944 945def ComputeDifferences(diffs): 946 """Call ComputePatch on all the Difference objects in 'diffs'.""" 947 print len(diffs), "diffs to compute" 948 949 # Do the largest files first, to try and reduce the long-pole effect. 950 by_size = [(i.tf.size, i) for i in diffs] 951 by_size.sort(reverse=True) 952 by_size = [i[1] for i in by_size] 953 954 lock = threading.Lock() 955 diff_iter = iter(by_size) # accessed under lock 956 957 def worker(): 958 try: 959 lock.acquire() 960 for d in diff_iter: 961 lock.release() 962 start = time.time() 963 d.ComputePatch() 964 dur = time.time() - start 965 lock.acquire() 966 967 tf, sf, patch = d.GetPatch() 968 if sf.name == tf.name: 969 name = tf.name 970 else: 971 name = "%s (%s)" % (tf.name, sf.name) 972 if patch is None: 973 print "patching failed! %s" % (name,) 974 else: 975 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 976 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 977 lock.release() 978 except Exception, e: 979 print e 980 raise 981 982 # start worker threads; wait for them all to finish. 983 threads = [threading.Thread(target=worker) 984 for i in range(OPTIONS.worker_threads)] 985 for th in threads: 986 th.start() 987 while threads: 988 threads.pop().join() 989 990 991# map recovery.fstab's fs_types to mount/format "partition types" 992PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 993 "ext4": "EMMC", "emmc": "EMMC", 994 "f2fs": "EMMC" } 995 996def GetTypeAndDevice(mount_point, info): 997 fstab = info["fstab"] 998 if fstab: 999 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 1000 else: 1001 return None 1002 1003 1004def ParseCertificate(data): 1005 """Parse a PEM-format certificate.""" 1006 cert = [] 1007 save = False 1008 for line in data.split("\n"): 1009 if "--END CERTIFICATE--" in line: 1010 break 1011 if save: 1012 cert.append(line) 1013 if "--BEGIN CERTIFICATE--" in line: 1014 save = True 1015 cert = "".join(cert).decode('base64') 1016 return cert 1017 1018def XDelta3(source_path, target_path, output_path): 1019 diff_program = ["xdelta3", "-0", "-B", str(64<<20), "-e", "-f", "-s"] 1020 diff_program.append(source_path) 1021 diff_program.append(target_path) 1022 diff_program.append(output_path) 1023 p = Run(diff_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1024 p.communicate() 1025 assert p.returncode == 0, "Couldn't produce patch" 1026 1027def XZ(path): 1028 compress_program = ["xz", "-zk", "-9", "--check=crc32"] 1029 compress_program.append(path) 1030 p = Run(compress_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1031 p.communicate() 1032 assert p.returncode == 0, "Couldn't compress patch" 1033 1034def MakePartitionPatch(source_file, target_file, partition): 1035 with tempfile.NamedTemporaryFile() as output_file: 1036 XDelta3(source_file.name, target_file.name, output_file.name) 1037 XZ(output_file.name) 1038 with open(output_file.name + ".xz") as patch_file: 1039 patch_data = patch_file.read() 1040 os.unlink(patch_file.name) 1041 return File(partition + ".muimg.p", patch_data) 1042 1043def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1044 info_dict=None): 1045 """Generate a binary patch that creates the recovery image starting 1046 with the boot image. (Most of the space in these images is just the 1047 kernel, which is identical for the two, so the resulting patch 1048 should be efficient.) Add it to the output zip, along with a shell 1049 script that is run from init.rc on first boot to actually do the 1050 patching and install the new recovery image. 1051 1052 recovery_img and boot_img should be File objects for the 1053 corresponding images. info should be the dictionary returned by 1054 common.LoadInfoDict() on the input target_files. 1055 """ 1056 1057 if info_dict is None: 1058 info_dict = OPTIONS.info_dict 1059 1060 diff_program = ["imgdiff"] 1061 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1062 if os.path.exists(path): 1063 diff_program.append("-b") 1064 diff_program.append(path) 1065 bonus_args = "-b /system/etc/recovery-resource.dat" 1066 else: 1067 bonus_args = "" 1068 1069 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1070 _, _, patch = d.ComputePatch() 1071 output_sink("recovery-from-boot.p", patch) 1072 1073 td_pair = GetTypeAndDevice("/boot", info_dict) 1074 if not td_pair: 1075 return 1076 boot_type, boot_device = td_pair 1077 td_pair = GetTypeAndDevice("/recovery", info_dict) 1078 if not td_pair: 1079 return 1080 recovery_type, recovery_device = td_pair 1081 1082 sh = """#!/system/bin/sh 1083if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1084 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1085else 1086 log -t recovery "Recovery image already installed" 1087fi 1088""" % { 'boot_size': boot_img.size, 1089 'boot_sha1': boot_img.sha1, 1090 'recovery_size': recovery_img.size, 1091 'recovery_sha1': recovery_img.sha1, 1092 'boot_type': boot_type, 1093 'boot_device': boot_device, 1094 'recovery_type': recovery_type, 1095 'recovery_device': recovery_device, 1096 'bonus_args': bonus_args, 1097 } 1098 1099 # The install script location moved from /system/etc to /system/bin 1100 # in the L release. Parse the init.rc file to find out where the 1101 # target-files expects it to be, and put it there. 1102 sh_location = "etc/install-recovery.sh" 1103 try: 1104 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1105 for line in f: 1106 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1107 if m: 1108 sh_location = m.group(1) 1109 print "putting script in", sh_location 1110 break 1111 except (OSError, IOError), e: 1112 print "failed to read init.rc: %s" % (e,) 1113 1114 output_sink(sh_location, sh) 1115