common.py revision bdc2e3186fc4ab22b9249d8e1b317a024e295c7f
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32import blockimgdiff 33 34try: 35 from hashlib import sha1 as sha1 36except ImportError: 37 from sha import sha as sha1 38 39# missing in Python 2.4 and before 40if not hasattr(os, "SEEK_SET"): 41 os.SEEK_SET = 0 42 43class Options(object): pass 44OPTIONS = Options() 45OPTIONS.search_path = "out/host/linux-x86" 46OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 47OPTIONS.extra_signapk_args = [] 48OPTIONS.java_path = "java" # Use the one on the path by default. 49OPTIONS.java_args = "-Xmx2048m" # JVM Args 50OPTIONS.public_key_suffix = ".x509.pem" 51OPTIONS.private_key_suffix = ".pk8" 52OPTIONS.verbose = False 53OPTIONS.tempfiles = [] 54OPTIONS.device_specific = None 55OPTIONS.extras = {} 56OPTIONS.info_dict = None 57 58 59# Values for "certificate" in apkcerts that mean special things. 60SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 61 62 63class ExternalError(RuntimeError): pass 64 65 66def Run(args, **kwargs): 67 """Create and return a subprocess.Popen object, printing the command 68 line on the terminal if -v was specified.""" 69 if OPTIONS.verbose: 70 print " running: ", " ".join(args) 71 return subprocess.Popen(args, **kwargs) 72 73 74def CloseInheritedPipes(): 75 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 76 before doing other work.""" 77 if platform.system() != "Darwin": 78 return 79 for d in range(3, 1025): 80 try: 81 stat = os.fstat(d) 82 if stat is not None: 83 pipebit = stat[0] & 0x1000 84 if pipebit != 0: 85 os.close(d) 86 except OSError: 87 pass 88 89 90def LoadInfoDict(input): 91 """Read and parse the META/misc_info.txt key/value pairs from the 92 input target files and return a dict.""" 93 94 def read_helper(fn): 95 if isinstance(input, zipfile.ZipFile): 96 return input.read(fn) 97 else: 98 path = os.path.join(input, *fn.split("/")) 99 try: 100 with open(path) as f: 101 return f.read() 102 except IOError, e: 103 if e.errno == errno.ENOENT: 104 raise KeyError(fn) 105 d = {} 106 try: 107 d = LoadDictionaryFromLines(read_helper("META/misc_info.txt").split("\n")) 108 except KeyError: 109 # ok if misc_info.txt doesn't exist 110 pass 111 112 # backwards compatibility: These values used to be in their own 113 # files. Look for them, in case we're processing an old 114 # target_files zip. 115 116 if "mkyaffs2_extra_flags" not in d: 117 try: 118 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 119 except KeyError: 120 # ok if flags don't exist 121 pass 122 123 if "recovery_api_version" not in d: 124 try: 125 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 126 except KeyError: 127 raise ValueError("can't find recovery API version in input target-files") 128 129 if "tool_extensions" not in d: 130 try: 131 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 132 except KeyError: 133 # ok if extensions don't exist 134 pass 135 136 if "fstab_version" not in d: 137 d["fstab_version"] = "1" 138 139 try: 140 data = read_helper("META/imagesizes.txt") 141 for line in data.split("\n"): 142 if not line: continue 143 name, value = line.split(" ", 1) 144 if not value: continue 145 if name == "blocksize": 146 d[name] = value 147 else: 148 d[name + "_size"] = value 149 except KeyError: 150 pass 151 152 def makeint(key): 153 if key in d: 154 d[key] = int(d[key], 0) 155 156 makeint("recovery_api_version") 157 makeint("blocksize") 158 makeint("system_size") 159 makeint("vendor_size") 160 makeint("userdata_size") 161 makeint("cache_size") 162 makeint("recovery_size") 163 makeint("boot_size") 164 makeint("fstab_version") 165 166 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 167 d["build.prop"] = LoadBuildProp(read_helper) 168 return d 169 170def LoadBuildProp(read_helper): 171 try: 172 data = read_helper("SYSTEM/build.prop") 173 except KeyError: 174 print "Warning: could not find SYSTEM/build.prop in %s" % zip 175 data = "" 176 return LoadDictionaryFromLines(data.split("\n")) 177 178def LoadDictionaryFromLines(lines): 179 d = {} 180 for line in lines: 181 line = line.strip() 182 if not line or line.startswith("#"): continue 183 if "=" in line: 184 name, value = line.split("=", 1) 185 d[name] = value 186 return d 187 188def LoadRecoveryFSTab(read_helper, fstab_version): 189 class Partition(object): 190 pass 191 192 try: 193 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 194 except KeyError: 195 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 196 data = "" 197 198 if fstab_version == 1: 199 d = {} 200 for line in data.split("\n"): 201 line = line.strip() 202 if not line or line.startswith("#"): continue 203 pieces = line.split() 204 if not (3 <= len(pieces) <= 4): 205 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 206 207 p = Partition() 208 p.mount_point = pieces[0] 209 p.fs_type = pieces[1] 210 p.device = pieces[2] 211 p.length = 0 212 options = None 213 if len(pieces) >= 4: 214 if pieces[3].startswith("/"): 215 p.device2 = pieces[3] 216 if len(pieces) >= 5: 217 options = pieces[4] 218 else: 219 p.device2 = None 220 options = pieces[3] 221 else: 222 p.device2 = None 223 224 if options: 225 options = options.split(",") 226 for i in options: 227 if i.startswith("length="): 228 p.length = int(i[7:]) 229 else: 230 print "%s: unknown option \"%s\"" % (p.mount_point, i) 231 232 d[p.mount_point] = p 233 234 elif fstab_version == 2: 235 d = {} 236 for line in data.split("\n"): 237 line = line.strip() 238 if not line or line.startswith("#"): continue 239 pieces = line.split() 240 if len(pieces) != 5: 241 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 242 243 # Ignore entries that are managed by vold 244 options = pieces[4] 245 if "voldmanaged=" in options: continue 246 247 # It's a good line, parse it 248 p = Partition() 249 p.device = pieces[0] 250 p.mount_point = pieces[1] 251 p.fs_type = pieces[2] 252 p.device2 = None 253 p.length = 0 254 255 options = options.split(",") 256 for i in options: 257 if i.startswith("length="): 258 p.length = int(i[7:]) 259 else: 260 # Ignore all unknown options in the unified fstab 261 continue 262 263 d[p.mount_point] = p 264 265 else: 266 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 267 268 return d 269 270 271def DumpInfoDict(d): 272 for k, v in sorted(d.items()): 273 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 274 275def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 276 """Take a kernel, cmdline, and ramdisk directory from the input (in 277 'sourcedir'), and turn them into a boot image. Return the image 278 data, or None if sourcedir does not appear to contains files for 279 building the requested image.""" 280 281 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 282 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 283 return None 284 285 if info_dict is None: 286 info_dict = OPTIONS.info_dict 287 288 ramdisk_img = tempfile.NamedTemporaryFile() 289 img = tempfile.NamedTemporaryFile() 290 291 if os.access(fs_config_file, os.F_OK): 292 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 293 else: 294 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 295 p1 = Run(cmd, stdout=subprocess.PIPE) 296 p2 = Run(["minigzip"], 297 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 298 299 p2.wait() 300 p1.wait() 301 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 302 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 303 304 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 305 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 306 307 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 308 309 fn = os.path.join(sourcedir, "second") 310 if os.access(fn, os.F_OK): 311 cmd.append("--second") 312 cmd.append(fn) 313 314 fn = os.path.join(sourcedir, "cmdline") 315 if os.access(fn, os.F_OK): 316 cmd.append("--cmdline") 317 cmd.append(open(fn).read().rstrip("\n")) 318 319 fn = os.path.join(sourcedir, "base") 320 if os.access(fn, os.F_OK): 321 cmd.append("--base") 322 cmd.append(open(fn).read().rstrip("\n")) 323 324 fn = os.path.join(sourcedir, "pagesize") 325 if os.access(fn, os.F_OK): 326 cmd.append("--pagesize") 327 cmd.append(open(fn).read().rstrip("\n")) 328 329 args = info_dict.get("mkbootimg_args", None) 330 if args and args.strip(): 331 cmd.extend(shlex.split(args)) 332 333 cmd.extend(["--ramdisk", ramdisk_img.name, 334 "--output", img.name]) 335 336 p = Run(cmd, stdout=subprocess.PIPE) 337 p.communicate() 338 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 339 os.path.basename(sourcedir),) 340 341 if info_dict.get("verity_key", None): 342 path = "/" + os.path.basename(sourcedir).lower() 343 cmd = ["boot_signer", path, img.name, info_dict["verity_key"], img.name] 344 p = Run(cmd, stdout=subprocess.PIPE) 345 p.communicate() 346 assert p.returncode == 0, "boot_signer of %s image failed" % path 347 348 img.seek(os.SEEK_SET, 0) 349 data = img.read() 350 351 ramdisk_img.close() 352 img.close() 353 354 return data 355 356 357def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 358 info_dict=None): 359 """Return a File object (with name 'name') with the desired bootable 360 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 361 'prebuilt_name', otherwise look for it under 'unpack_dir'/IMAGES, 362 otherwise construct it from the source files in 363 'unpack_dir'/'tree_subdir'.""" 364 365 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 366 if os.path.exists(prebuilt_path): 367 print "using prebuilt %s from BOOTABLE_IMAGES..." % (prebuilt_name,) 368 return File.FromLocalFile(name, prebuilt_path) 369 370 prebuilt_path = os.path.join(unpack_dir, "IMAGES", prebuilt_name) 371 if os.path.exists(prebuilt_path): 372 print "using prebuilt %s from IMAGES..." % (prebuilt_name,) 373 return File.FromLocalFile(name, prebuilt_path) 374 375 print "building image from target_files %s..." % (tree_subdir,) 376 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 377 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 378 os.path.join(unpack_dir, fs_config), 379 info_dict) 380 if data: 381 return File(name, data) 382 return None 383 384 385def UnzipTemp(filename, pattern=None): 386 """Unzip the given archive into a temporary directory and return the name. 387 388 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 389 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 390 391 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 392 main file), open for reading. 393 """ 394 395 tmp = tempfile.mkdtemp(prefix="targetfiles-") 396 OPTIONS.tempfiles.append(tmp) 397 398 def unzip_to_dir(filename, dirname): 399 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 400 if pattern is not None: 401 cmd.append(pattern) 402 p = Run(cmd, stdout=subprocess.PIPE) 403 p.communicate() 404 if p.returncode != 0: 405 raise ExternalError("failed to unzip input target-files \"%s\"" % 406 (filename,)) 407 408 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 409 if m: 410 unzip_to_dir(m.group(1), tmp) 411 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 412 filename = m.group(1) 413 else: 414 unzip_to_dir(filename, tmp) 415 416 return tmp, zipfile.ZipFile(filename, "r") 417 418 419def GetKeyPasswords(keylist): 420 """Given a list of keys, prompt the user to enter passwords for 421 those which require them. Return a {key: password} dict. password 422 will be None if the key has no password.""" 423 424 no_passwords = [] 425 need_passwords = [] 426 key_passwords = {} 427 devnull = open("/dev/null", "w+b") 428 for k in sorted(keylist): 429 # We don't need a password for things that aren't really keys. 430 if k in SPECIAL_CERT_STRINGS: 431 no_passwords.append(k) 432 continue 433 434 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 435 "-inform", "DER", "-nocrypt"], 436 stdin=devnull.fileno(), 437 stdout=devnull.fileno(), 438 stderr=subprocess.STDOUT) 439 p.communicate() 440 if p.returncode == 0: 441 # Definitely an unencrypted key. 442 no_passwords.append(k) 443 else: 444 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 445 "-inform", "DER", "-passin", "pass:"], 446 stdin=devnull.fileno(), 447 stdout=devnull.fileno(), 448 stderr=subprocess.PIPE) 449 stdout, stderr = p.communicate() 450 if p.returncode == 0: 451 # Encrypted key with empty string as password. 452 key_passwords[k] = '' 453 elif stderr.startswith('Error decrypting key'): 454 # Definitely encrypted key. 455 # It would have said "Error reading key" if it didn't parse correctly. 456 need_passwords.append(k) 457 else: 458 # Potentially, a type of key that openssl doesn't understand. 459 # We'll let the routines in signapk.jar handle it. 460 no_passwords.append(k) 461 devnull.close() 462 463 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 464 key_passwords.update(dict.fromkeys(no_passwords, None)) 465 return key_passwords 466 467 468def SignFile(input_name, output_name, key, password, align=None, 469 whole_file=False): 470 """Sign the input_name zip/jar/apk, producing output_name. Use the 471 given key and password (the latter may be None if the key does not 472 have a password. 473 474 If align is an integer > 1, zipalign is run to align stored files in 475 the output zip on 'align'-byte boundaries. 476 477 If whole_file is true, use the "-w" option to SignApk to embed a 478 signature that covers the whole file in the archive comment of the 479 zip file. 480 """ 481 482 if align == 0 or align == 1: 483 align = None 484 485 if align: 486 temp = tempfile.NamedTemporaryFile() 487 sign_name = temp.name 488 else: 489 sign_name = output_name 490 491 cmd = [OPTIONS.java_path, OPTIONS.java_args, "-jar", 492 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 493 cmd.extend(OPTIONS.extra_signapk_args) 494 if whole_file: 495 cmd.append("-w") 496 cmd.extend([key + OPTIONS.public_key_suffix, 497 key + OPTIONS.private_key_suffix, 498 input_name, sign_name]) 499 500 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 501 if password is not None: 502 password += "\n" 503 p.communicate(password) 504 if p.returncode != 0: 505 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 506 507 if align: 508 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 509 p.communicate() 510 if p.returncode != 0: 511 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 512 temp.close() 513 514 515def CheckSize(data, target, info_dict): 516 """Check the data string passed against the max size limit, if 517 any, for the given target. Raise exception if the data is too big. 518 Print a warning if the data is nearing the maximum size.""" 519 520 if target.endswith(".img"): target = target[:-4] 521 mount_point = "/" + target 522 523 fs_type = None 524 limit = None 525 if info_dict["fstab"]: 526 if mount_point == "/userdata": mount_point = "/data" 527 p = info_dict["fstab"][mount_point] 528 fs_type = p.fs_type 529 device = p.device 530 if "/" in device: 531 device = device[device.rfind("/")+1:] 532 limit = info_dict.get(device + "_size", None) 533 if not fs_type or not limit: return 534 535 if fs_type == "yaffs2": 536 # image size should be increased by 1/64th to account for the 537 # spare area (64 bytes per 2k page) 538 limit = limit / 2048 * (2048+64) 539 size = len(data) 540 pct = float(size) * 100.0 / limit 541 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 542 if pct >= 99.0: 543 raise ExternalError(msg) 544 elif pct >= 95.0: 545 print 546 print " WARNING: ", msg 547 print 548 elif OPTIONS.verbose: 549 print " ", msg 550 551 552def ReadApkCerts(tf_zip): 553 """Given a target_files ZipFile, parse the META/apkcerts.txt file 554 and return a {package: cert} dict.""" 555 certmap = {} 556 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 557 line = line.strip() 558 if not line: continue 559 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 560 r'private_key="(.*)"$', line) 561 if m: 562 name, cert, privkey = m.groups() 563 public_key_suffix_len = len(OPTIONS.public_key_suffix) 564 private_key_suffix_len = len(OPTIONS.private_key_suffix) 565 if cert in SPECIAL_CERT_STRINGS and not privkey: 566 certmap[name] = cert 567 elif (cert.endswith(OPTIONS.public_key_suffix) and 568 privkey.endswith(OPTIONS.private_key_suffix) and 569 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 570 certmap[name] = cert[:-public_key_suffix_len] 571 else: 572 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 573 return certmap 574 575 576COMMON_DOCSTRING = """ 577 -p (--path) <dir> 578 Prepend <dir>/bin to the list of places to search for binaries 579 run by this script, and expect to find jars in <dir>/framework. 580 581 -s (--device_specific) <file> 582 Path to the python module containing device-specific 583 releasetools code. 584 585 -x (--extra) <key=value> 586 Add a key/value pair to the 'extras' dict, which device-specific 587 extension code may look at. 588 589 -v (--verbose) 590 Show command lines being executed. 591 592 -h (--help) 593 Display this usage message and exit. 594""" 595 596def Usage(docstring): 597 print docstring.rstrip("\n") 598 print COMMON_DOCSTRING 599 600 601def ParseOptions(argv, 602 docstring, 603 extra_opts="", extra_long_opts=(), 604 extra_option_handler=None): 605 """Parse the options in argv and return any arguments that aren't 606 flags. docstring is the calling module's docstring, to be displayed 607 for errors and -h. extra_opts and extra_long_opts are for flags 608 defined by the caller, which are processed by passing them to 609 extra_option_handler.""" 610 611 try: 612 opts, args = getopt.getopt( 613 argv, "hvp:s:x:" + extra_opts, 614 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 615 "java_path=", "java_args=", "public_key_suffix=", 616 "private_key_suffix=", "device_specific=", "extra="] + 617 list(extra_long_opts)) 618 except getopt.GetoptError, err: 619 Usage(docstring) 620 print "**", str(err), "**" 621 sys.exit(2) 622 623 path_specified = False 624 625 for o, a in opts: 626 if o in ("-h", "--help"): 627 Usage(docstring) 628 sys.exit() 629 elif o in ("-v", "--verbose"): 630 OPTIONS.verbose = True 631 elif o in ("-p", "--path"): 632 OPTIONS.search_path = a 633 elif o in ("--signapk_path",): 634 OPTIONS.signapk_path = a 635 elif o in ("--extra_signapk_args",): 636 OPTIONS.extra_signapk_args = shlex.split(a) 637 elif o in ("--java_path",): 638 OPTIONS.java_path = a 639 elif o in ("--java_args",): 640 OPTIONS.java_args = a 641 elif o in ("--public_key_suffix",): 642 OPTIONS.public_key_suffix = a 643 elif o in ("--private_key_suffix",): 644 OPTIONS.private_key_suffix = a 645 elif o in ("-s", "--device_specific"): 646 OPTIONS.device_specific = a 647 elif o in ("-x", "--extra"): 648 key, value = a.split("=", 1) 649 OPTIONS.extras[key] = value 650 else: 651 if extra_option_handler is None or not extra_option_handler(o, a): 652 assert False, "unknown option \"%s\"" % (o,) 653 654 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 655 os.pathsep + os.environ["PATH"]) 656 657 return args 658 659 660def MakeTempFile(prefix=None, suffix=None): 661 """Make a temp file and add it to the list of things to be deleted 662 when Cleanup() is called. Return the filename.""" 663 fd, fn = tempfile.mkstemp(prefix=prefix, suffix=suffix) 664 os.close(fd) 665 OPTIONS.tempfiles.append(fn) 666 return fn 667 668 669def Cleanup(): 670 for i in OPTIONS.tempfiles: 671 if os.path.isdir(i): 672 shutil.rmtree(i) 673 else: 674 os.remove(i) 675 676 677class PasswordManager(object): 678 def __init__(self): 679 self.editor = os.getenv("EDITOR", None) 680 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 681 682 def GetPasswords(self, items): 683 """Get passwords corresponding to each string in 'items', 684 returning a dict. (The dict may have keys in addition to the 685 values in 'items'.) 686 687 Uses the passwords in $ANDROID_PW_FILE if available, letting the 688 user edit that file to add more needed passwords. If no editor is 689 available, or $ANDROID_PW_FILE isn't define, prompts the user 690 interactively in the ordinary way. 691 """ 692 693 current = self.ReadFile() 694 695 first = True 696 while True: 697 missing = [] 698 for i in items: 699 if i not in current or not current[i]: 700 missing.append(i) 701 # Are all the passwords already in the file? 702 if not missing: return current 703 704 for i in missing: 705 current[i] = "" 706 707 if not first: 708 print "key file %s still missing some passwords." % (self.pwfile,) 709 answer = raw_input("try to edit again? [y]> ").strip() 710 if answer and answer[0] not in 'yY': 711 raise RuntimeError("key passwords unavailable") 712 first = False 713 714 current = self.UpdateAndReadFile(current) 715 716 def PromptResult(self, current): 717 """Prompt the user to enter a value (password) for each key in 718 'current' whose value is fales. Returns a new dict with all the 719 values. 720 """ 721 result = {} 722 for k, v in sorted(current.iteritems()): 723 if v: 724 result[k] = v 725 else: 726 while True: 727 result[k] = getpass.getpass("Enter password for %s key> " 728 % (k,)).strip() 729 if result[k]: break 730 return result 731 732 def UpdateAndReadFile(self, current): 733 if not self.editor or not self.pwfile: 734 return self.PromptResult(current) 735 736 f = open(self.pwfile, "w") 737 os.chmod(self.pwfile, 0600) 738 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 739 f.write("# (Additional spaces are harmless.)\n\n") 740 741 first_line = None 742 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 743 sorted.sort() 744 for i, (_, k, v) in enumerate(sorted): 745 f.write("[[[ %s ]]] %s\n" % (v, k)) 746 if not v and first_line is None: 747 # position cursor on first line with no password. 748 first_line = i + 4 749 f.close() 750 751 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 752 _, _ = p.communicate() 753 754 return self.ReadFile() 755 756 def ReadFile(self): 757 result = {} 758 if self.pwfile is None: return result 759 try: 760 f = open(self.pwfile, "r") 761 for line in f: 762 line = line.strip() 763 if not line or line[0] == '#': continue 764 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 765 if not m: 766 print "failed to parse password file: ", line 767 else: 768 result[m.group(2)] = m.group(1) 769 f.close() 770 except IOError, e: 771 if e.errno != errno.ENOENT: 772 print "error reading password file: ", str(e) 773 return result 774 775 776def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 777 # use a fixed timestamp so the output is repeatable. 778 zinfo = zipfile.ZipInfo(filename=filename, 779 date_time=(2009, 1, 1, 0, 0, 0)) 780 if compression is None: 781 zinfo.compress_type = zip.compression 782 else: 783 zinfo.compress_type = compression 784 zinfo.external_attr = perms << 16 785 zip.writestr(zinfo, data) 786 787 788class DeviceSpecificParams(object): 789 module = None 790 def __init__(self, **kwargs): 791 """Keyword arguments to the constructor become attributes of this 792 object, which is passed to all functions in the device-specific 793 module.""" 794 for k, v in kwargs.iteritems(): 795 setattr(self, k, v) 796 self.extras = OPTIONS.extras 797 798 if self.module is None: 799 path = OPTIONS.device_specific 800 if not path: return 801 try: 802 if os.path.isdir(path): 803 info = imp.find_module("releasetools", [path]) 804 else: 805 d, f = os.path.split(path) 806 b, x = os.path.splitext(f) 807 if x == ".py": 808 f = b 809 info = imp.find_module(f, [d]) 810 print "loaded device-specific extensions from", path 811 self.module = imp.load_module("device_specific", *info) 812 except ImportError: 813 print "unable to load device-specific module; assuming none" 814 815 def _DoCall(self, function_name, *args, **kwargs): 816 """Call the named function in the device-specific module, passing 817 the given args and kwargs. The first argument to the call will be 818 the DeviceSpecific object itself. If there is no module, or the 819 module does not define the function, return the value of the 820 'default' kwarg (which itself defaults to None).""" 821 if self.module is None or not hasattr(self.module, function_name): 822 return kwargs.get("default", None) 823 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 824 825 def FullOTA_Assertions(self): 826 """Called after emitting the block of assertions at the top of a 827 full OTA package. Implementations can add whatever additional 828 assertions they like.""" 829 return self._DoCall("FullOTA_Assertions") 830 831 def FullOTA_InstallBegin(self): 832 """Called at the start of full OTA installation.""" 833 return self._DoCall("FullOTA_InstallBegin") 834 835 def FullOTA_InstallEnd(self): 836 """Called at the end of full OTA installation; typically this is 837 used to install the image for the device's baseband processor.""" 838 return self._DoCall("FullOTA_InstallEnd") 839 840 def IncrementalOTA_Assertions(self): 841 """Called after emitting the block of assertions at the top of an 842 incremental OTA package. Implementations can add whatever 843 additional assertions they like.""" 844 return self._DoCall("IncrementalOTA_Assertions") 845 846 def IncrementalOTA_VerifyBegin(self): 847 """Called at the start of the verification phase of incremental 848 OTA installation; additional checks can be placed here to abort 849 the script before any changes are made.""" 850 return self._DoCall("IncrementalOTA_VerifyBegin") 851 852 def IncrementalOTA_VerifyEnd(self): 853 """Called at the end of the verification phase of incremental OTA 854 installation; additional checks can be placed here to abort the 855 script before any changes are made.""" 856 return self._DoCall("IncrementalOTA_VerifyEnd") 857 858 def IncrementalOTA_InstallBegin(self): 859 """Called at the start of incremental OTA installation (after 860 verification is complete).""" 861 return self._DoCall("IncrementalOTA_InstallBegin") 862 863 def IncrementalOTA_InstallEnd(self): 864 """Called at the end of incremental OTA installation; typically 865 this is used to install the image for the device's baseband 866 processor.""" 867 return self._DoCall("IncrementalOTA_InstallEnd") 868 869class File(object): 870 def __init__(self, name, data): 871 self.name = name 872 self.data = data 873 self.size = len(data) 874 self.sha1 = sha1(data).hexdigest() 875 876 @classmethod 877 def FromLocalFile(cls, name, diskname): 878 f = open(diskname, "rb") 879 data = f.read() 880 f.close() 881 return File(name, data) 882 883 def WriteToTemp(self): 884 t = tempfile.NamedTemporaryFile() 885 t.write(self.data) 886 t.flush() 887 return t 888 889 def AddToZip(self, z, compression=None): 890 ZipWriteStr(z, self.name, self.data, compression=compression) 891 892DIFF_PROGRAM_BY_EXT = { 893 ".gz" : "imgdiff", 894 ".zip" : ["imgdiff", "-z"], 895 ".jar" : ["imgdiff", "-z"], 896 ".apk" : ["imgdiff", "-z"], 897 ".img" : "imgdiff", 898 } 899 900class Difference(object): 901 def __init__(self, tf, sf, diff_program=None): 902 self.tf = tf 903 self.sf = sf 904 self.patch = None 905 self.diff_program = diff_program 906 907 def ComputePatch(self): 908 """Compute the patch (as a string of data) needed to turn sf into 909 tf. Returns the same tuple as GetPatch().""" 910 911 tf = self.tf 912 sf = self.sf 913 914 if self.diff_program: 915 diff_program = self.diff_program 916 else: 917 ext = os.path.splitext(tf.name)[1] 918 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 919 920 ttemp = tf.WriteToTemp() 921 stemp = sf.WriteToTemp() 922 923 ext = os.path.splitext(tf.name)[1] 924 925 try: 926 ptemp = tempfile.NamedTemporaryFile() 927 if isinstance(diff_program, list): 928 cmd = copy.copy(diff_program) 929 else: 930 cmd = [diff_program] 931 cmd.append(stemp.name) 932 cmd.append(ttemp.name) 933 cmd.append(ptemp.name) 934 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 935 err = [] 936 def run(): 937 _, e = p.communicate() 938 if e: err.append(e) 939 th = threading.Thread(target=run) 940 th.start() 941 th.join(timeout=300) # 5 mins 942 if th.is_alive(): 943 print "WARNING: diff command timed out" 944 p.terminate() 945 th.join(5) 946 if th.is_alive(): 947 p.kill() 948 th.join() 949 950 if err or p.returncode != 0: 951 print "WARNING: failure running %s:\n%s\n" % ( 952 diff_program, "".join(err)) 953 self.patch = None 954 return None, None, None 955 diff = ptemp.read() 956 finally: 957 ptemp.close() 958 stemp.close() 959 ttemp.close() 960 961 self.patch = diff 962 return self.tf, self.sf, self.patch 963 964 965 def GetPatch(self): 966 """Return a tuple (target_file, source_file, patch_data). 967 patch_data may be None if ComputePatch hasn't been called, or if 968 computing the patch failed.""" 969 return self.tf, self.sf, self.patch 970 971 972def ComputeDifferences(diffs): 973 """Call ComputePatch on all the Difference objects in 'diffs'.""" 974 print len(diffs), "diffs to compute" 975 976 # Do the largest files first, to try and reduce the long-pole effect. 977 by_size = [(i.tf.size, i) for i in diffs] 978 by_size.sort(reverse=True) 979 by_size = [i[1] for i in by_size] 980 981 lock = threading.Lock() 982 diff_iter = iter(by_size) # accessed under lock 983 984 def worker(): 985 try: 986 lock.acquire() 987 for d in diff_iter: 988 lock.release() 989 start = time.time() 990 d.ComputePatch() 991 dur = time.time() - start 992 lock.acquire() 993 994 tf, sf, patch = d.GetPatch() 995 if sf.name == tf.name: 996 name = tf.name 997 else: 998 name = "%s (%s)" % (tf.name, sf.name) 999 if patch is None: 1000 print "patching failed! %s" % (name,) 1001 else: 1002 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 1003 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 1004 lock.release() 1005 except Exception, e: 1006 print e 1007 raise 1008 1009 # start worker threads; wait for them all to finish. 1010 threads = [threading.Thread(target=worker) 1011 for i in range(OPTIONS.worker_threads)] 1012 for th in threads: 1013 th.start() 1014 while threads: 1015 threads.pop().join() 1016 1017 1018class BlockDifference: 1019 def __init__(self, partition, tgt, src=None): 1020 self.tgt = tgt 1021 self.src = src 1022 self.partition = partition 1023 1024 b = blockimgdiff.BlockImageDiff(tgt, src, threads=OPTIONS.worker_threads) 1025 tmpdir = tempfile.mkdtemp() 1026 OPTIONS.tempfiles.append(tmpdir) 1027 self.path = os.path.join(tmpdir, partition) 1028 b.Compute(self.path) 1029 1030 _, self.device = GetTypeAndDevice("/" + partition, OPTIONS.info_dict) 1031 1032 def WriteScript(self, script, output_zip, progress=None): 1033 if not self.src: 1034 # write the output unconditionally 1035 if progress: script.ShowProgress(progress, 0) 1036 self._WriteUpdate(script, output_zip) 1037 1038 else: 1039 script.AppendExtra('if range_sha1("%s", "%s") == "%s" then' % 1040 (self.device, self.src.care_map.to_string_raw(), 1041 self.src.TotalSha1())) 1042 script.Print("Patching %s image..." % (self.partition,)) 1043 if progress: script.ShowProgress(progress, 0) 1044 self._WriteUpdate(script, output_zip) 1045 script.AppendExtra(('else\n' 1046 ' (range_sha1("%s", "%s") == "%s") ||\n' 1047 ' abort("%s partition has unexpected contents");\n' 1048 'endif;') % 1049 (self.device, self.tgt.care_map.to_string_raw(), 1050 self.tgt.TotalSha1(), self.partition)) 1051 1052 def _WriteUpdate(self, script, output_zip): 1053 partition = self.partition 1054 with open(self.path + ".transfer.list", "rb") as f: 1055 ZipWriteStr(output_zip, partition + ".transfer.list", f.read()) 1056 with open(self.path + ".new.dat", "rb") as f: 1057 ZipWriteStr(output_zip, partition + ".new.dat", f.read()) 1058 with open(self.path + ".patch.dat", "rb") as f: 1059 ZipWriteStr(output_zip, partition + ".patch.dat", f.read(), 1060 compression=zipfile.ZIP_STORED) 1061 1062 call = (('block_image_update("%s", ' 1063 'package_extract_file("%s.transfer.list"), ' 1064 '"%s.new.dat", "%s.patch.dat");\n') % 1065 (self.device, partition, partition, partition)) 1066 script.AppendExtra(script._WordWrap(call)) 1067 1068 1069DataImage = blockimgdiff.DataImage 1070 1071 1072# map recovery.fstab's fs_types to mount/format "partition types" 1073PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 1074 "ext4": "EMMC", "emmc": "EMMC", 1075 "f2fs": "EMMC" } 1076 1077def GetTypeAndDevice(mount_point, info): 1078 fstab = info["fstab"] 1079 if fstab: 1080 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 1081 else: 1082 return None 1083 1084 1085def ParseCertificate(data): 1086 """Parse a PEM-format certificate.""" 1087 cert = [] 1088 save = False 1089 for line in data.split("\n"): 1090 if "--END CERTIFICATE--" in line: 1091 break 1092 if save: 1093 cert.append(line) 1094 if "--BEGIN CERTIFICATE--" in line: 1095 save = True 1096 cert = "".join(cert).decode('base64') 1097 return cert 1098 1099def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, 1100 info_dict=None): 1101 """Generate a binary patch that creates the recovery image starting 1102 with the boot image. (Most of the space in these images is just the 1103 kernel, which is identical for the two, so the resulting patch 1104 should be efficient.) Add it to the output zip, along with a shell 1105 script that is run from init.rc on first boot to actually do the 1106 patching and install the new recovery image. 1107 1108 recovery_img and boot_img should be File objects for the 1109 corresponding images. info should be the dictionary returned by 1110 common.LoadInfoDict() on the input target_files. 1111 """ 1112 1113 if info_dict is None: 1114 info_dict = OPTIONS.info_dict 1115 1116 diff_program = ["imgdiff"] 1117 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1118 if os.path.exists(path): 1119 diff_program.append("-b") 1120 diff_program.append(path) 1121 bonus_args = "-b /system/etc/recovery-resource.dat" 1122 else: 1123 bonus_args = "" 1124 1125 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1126 _, _, patch = d.ComputePatch() 1127 output_sink("recovery-from-boot.p", patch) 1128 1129 td_pair = GetTypeAndDevice("/boot", info_dict) 1130 if not td_pair: 1131 return 1132 boot_type, boot_device = td_pair 1133 td_pair = GetTypeAndDevice("/recovery", info_dict) 1134 if not td_pair: 1135 return 1136 recovery_type, recovery_device = td_pair 1137 1138 sh = """#!/system/bin/sh 1139if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1140 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1141else 1142 log -t recovery "Recovery image already installed" 1143fi 1144""" % { 'boot_size': boot_img.size, 1145 'boot_sha1': boot_img.sha1, 1146 'recovery_size': recovery_img.size, 1147 'recovery_sha1': recovery_img.sha1, 1148 'boot_type': boot_type, 1149 'boot_device': boot_device, 1150 'recovery_type': recovery_type, 1151 'recovery_device': recovery_device, 1152 'bonus_args': bonus_args, 1153 } 1154 1155 # The install script location moved from /system/etc to /system/bin 1156 # in the L release. Parse the init.rc file to find out where the 1157 # target-files expects it to be, and put it there. 1158 sh_location = "etc/install-recovery.sh" 1159 try: 1160 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1161 for line in f: 1162 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1163 if m: 1164 sh_location = m.group(1) 1165 print "putting script in", sh_location 1166 break 1167 except (OSError, IOError), e: 1168 print "failed to read init.rc: %s" % (e,) 1169 1170 output_sink(sh_location, sh) 1171