common.py revision 36bd365625e6beba77698a93795a603180a5b476
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shlex 24import shutil 25import subprocess 26import sys 27import tempfile 28import threading 29import time 30import zipfile 31 32try: 33 from hashlib import sha1 as sha1 34except ImportError: 35 from sha import sha as sha1 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.signapk_path = "framework/signapk.jar" # Relative to search_path 45OPTIONS.extra_signapk_args = [] 46OPTIONS.java_path = "java" # Use the one on the path by default. 47OPTIONS.public_key_suffix = ".x509.pem" 48OPTIONS.private_key_suffix = ".pk8" 49OPTIONS.verbose = False 50OPTIONS.tempfiles = [] 51OPTIONS.device_specific = None 52OPTIONS.extras = {} 53OPTIONS.info_dict = None 54 55 56# Values for "certificate" in apkcerts that mean special things. 57SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 58 59 60class ExternalError(RuntimeError): pass 61 62 63def Run(args, **kwargs): 64 """Create and return a subprocess.Popen object, printing the command 65 line on the terminal if -v was specified.""" 66 if OPTIONS.verbose: 67 print " running: ", " ".join(args) 68 return subprocess.Popen(args, **kwargs) 69 70 71def CloseInheritedPipes(): 72 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 73 before doing other work.""" 74 if platform.system() != "Darwin": 75 return 76 for d in range(3, 1025): 77 try: 78 stat = os.fstat(d) 79 if stat is not None: 80 pipebit = stat[0] & 0x1000 81 if pipebit != 0: 82 os.close(d) 83 except OSError: 84 pass 85 86 87def LoadInfoDict(input): 88 """Read and parse the META/misc_info.txt key/value pairs from the 89 input target files and return a dict.""" 90 91 def read_helper(fn): 92 if isinstance(input, zipfile.ZipFile): 93 return input.read(fn) 94 else: 95 path = os.path.join(input, *fn.split("/")) 96 try: 97 with open(path) as f: 98 return f.read() 99 except IOError, e: 100 if e.errno == errno.ENOENT: 101 raise KeyError(fn) 102 103 d = {} 104 try: 105 for line in read_helper("META/misc_info.txt").split("\n"): 106 line = line.strip() 107 if not line or line.startswith("#"): continue 108 k, v = line.split("=", 1) 109 d[k] = v 110 except KeyError: 111 # ok if misc_info.txt doesn't exist 112 pass 113 114 # backwards compatibility: These values used to be in their own 115 # files. Look for them, in case we're processing an old 116 # target_files zip. 117 118 if "mkyaffs2_extra_flags" not in d: 119 try: 120 d["mkyaffs2_extra_flags"] = read_helper("META/mkyaffs2-extra-flags.txt").strip() 121 except KeyError: 122 # ok if flags don't exist 123 pass 124 125 if "recovery_api_version" not in d: 126 try: 127 d["recovery_api_version"] = read_helper("META/recovery-api-version.txt").strip() 128 except KeyError: 129 raise ValueError("can't find recovery API version in input target-files") 130 131 if "tool_extensions" not in d: 132 try: 133 d["tool_extensions"] = read_helper("META/tool-extensions.txt").strip() 134 except KeyError: 135 # ok if extensions don't exist 136 pass 137 138 if "fstab_version" not in d: 139 d["fstab_version"] = "1" 140 141 try: 142 data = read_helper("META/imagesizes.txt") 143 for line in data.split("\n"): 144 if not line: continue 145 name, value = line.split(" ", 1) 146 if not value: continue 147 if name == "blocksize": 148 d[name] = value 149 else: 150 d[name + "_size"] = value 151 except KeyError: 152 pass 153 154 def makeint(key): 155 if key in d: 156 d[key] = int(d[key], 0) 157 158 makeint("recovery_api_version") 159 makeint("blocksize") 160 makeint("system_size") 161 makeint("userdata_size") 162 makeint("cache_size") 163 makeint("recovery_size") 164 makeint("boot_size") 165 makeint("fstab_version") 166 167 d["fstab"] = LoadRecoveryFSTab(read_helper, d["fstab_version"]) 168 d["build.prop"] = LoadBuildProp(read_helper) 169 return d 170 171def LoadBuildProp(read_helper): 172 try: 173 data = read_helper("SYSTEM/build.prop") 174 except KeyError: 175 print "Warning: could not find SYSTEM/build.prop in %s" % zip 176 data = "" 177 178 d = {} 179 for line in data.split("\n"): 180 line = line.strip() 181 if not line or line.startswith("#"): continue 182 name, value = line.split("=", 1) 183 d[name] = value 184 return d 185 186def LoadRecoveryFSTab(read_helper, fstab_version): 187 class Partition(object): 188 pass 189 190 try: 191 data = read_helper("RECOVERY/RAMDISK/etc/recovery.fstab") 192 except KeyError: 193 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab" 194 data = "" 195 196 if fstab_version == 1: 197 d = {} 198 for line in data.split("\n"): 199 line = line.strip() 200 if not line or line.startswith("#"): continue 201 pieces = line.split() 202 if not (3 <= len(pieces) <= 4): 203 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 204 205 p = Partition() 206 p.mount_point = pieces[0] 207 p.fs_type = pieces[1] 208 p.device = pieces[2] 209 p.length = 0 210 options = None 211 if len(pieces) >= 4: 212 if pieces[3].startswith("/"): 213 p.device2 = pieces[3] 214 if len(pieces) >= 5: 215 options = pieces[4] 216 else: 217 p.device2 = None 218 options = pieces[3] 219 else: 220 p.device2 = None 221 222 if options: 223 options = options.split(",") 224 for i in options: 225 if i.startswith("length="): 226 p.length = int(i[7:]) 227 else: 228 print "%s: unknown option \"%s\"" % (p.mount_point, i) 229 230 d[p.mount_point] = p 231 232 elif fstab_version == 2: 233 d = {} 234 for line in data.split("\n"): 235 line = line.strip() 236 if not line or line.startswith("#"): continue 237 pieces = line.split() 238 if len(pieces) != 5: 239 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 240 241 # Ignore entries that are managed by vold 242 options = pieces[4] 243 if "voldmanaged=" in options: continue 244 245 # It's a good line, parse it 246 p = Partition() 247 p.device = pieces[0] 248 p.mount_point = pieces[1] 249 p.fs_type = pieces[2] 250 p.device2 = None 251 p.length = 0 252 253 options = options.split(",") 254 for i in options: 255 if i.startswith("length="): 256 p.length = int(i[7:]) 257 else: 258 # Ignore all unknown options in the unified fstab 259 continue 260 261 d[p.mount_point] = p 262 263 else: 264 raise ValueError("Unknown fstab_version: \"%d\"" % (fstab_version,)) 265 266 return d 267 268 269def DumpInfoDict(d): 270 for k, v in sorted(d.items()): 271 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 272 273def BuildBootableImage(sourcedir, fs_config_file, info_dict=None): 274 """Take a kernel, cmdline, and ramdisk directory from the input (in 275 'sourcedir'), and turn them into a boot image. Return the image 276 data, or None if sourcedir does not appear to contains files for 277 building the requested image.""" 278 279 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 280 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 281 return None 282 283 if info_dict is None: 284 info_dict = OPTIONS.info_dict 285 286 ramdisk_img = tempfile.NamedTemporaryFile() 287 img = tempfile.NamedTemporaryFile() 288 289 if os.access(fs_config_file, os.F_OK): 290 cmd = ["mkbootfs", "-f", fs_config_file, os.path.join(sourcedir, "RAMDISK")] 291 else: 292 cmd = ["mkbootfs", os.path.join(sourcedir, "RAMDISK")] 293 p1 = Run(cmd, stdout=subprocess.PIPE) 294 p2 = Run(["minigzip"], 295 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 296 297 p2.wait() 298 p1.wait() 299 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 300 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 301 302 # use MKBOOTIMG from environ, or "mkbootimg" if empty or not set 303 mkbootimg = os.getenv('MKBOOTIMG') or "mkbootimg" 304 305 cmd = [mkbootimg, "--kernel", os.path.join(sourcedir, "kernel")] 306 307 fn = os.path.join(sourcedir, "cmdline") 308 if os.access(fn, os.F_OK): 309 cmd.append("--cmdline") 310 cmd.append(open(fn).read().rstrip("\n")) 311 312 fn = os.path.join(sourcedir, "base") 313 if os.access(fn, os.F_OK): 314 cmd.append("--base") 315 cmd.append(open(fn).read().rstrip("\n")) 316 317 fn = os.path.join(sourcedir, "pagesize") 318 if os.access(fn, os.F_OK): 319 cmd.append("--pagesize") 320 cmd.append(open(fn).read().rstrip("\n")) 321 322 args = info_dict.get("mkbootimg_args", None) 323 if args and args.strip(): 324 cmd.extend(shlex.split(args)) 325 326 cmd.extend(["--ramdisk", ramdisk_img.name, 327 "--output", img.name]) 328 329 p = Run(cmd, stdout=subprocess.PIPE) 330 p.communicate() 331 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 332 os.path.basename(sourcedir),) 333 334 img.seek(os.SEEK_SET, 0) 335 data = img.read() 336 337 ramdisk_img.close() 338 img.close() 339 340 return data 341 342 343def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir, 344 info_dict=None): 345 """Return a File object (with name 'name') with the desired bootable 346 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 347 'prebuilt_name', otherwise construct it from the source files in 348 'unpack_dir'/'tree_subdir'.""" 349 350 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 351 if os.path.exists(prebuilt_path): 352 print "using prebuilt %s..." % (prebuilt_name,) 353 return File.FromLocalFile(name, prebuilt_path) 354 else: 355 print "building image from target_files %s..." % (tree_subdir,) 356 fs_config = "META/" + tree_subdir.lower() + "_filesystem_config.txt" 357 data = BuildBootableImage(os.path.join(unpack_dir, tree_subdir), 358 os.path.join(unpack_dir, fs_config), 359 info_dict) 360 if data: 361 return File(name, data) 362 return None 363 364 365def UnzipTemp(filename, pattern=None): 366 """Unzip the given archive into a temporary directory and return the name. 367 368 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 369 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 370 371 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 372 main file), open for reading. 373 """ 374 375 tmp = tempfile.mkdtemp(prefix="targetfiles-") 376 OPTIONS.tempfiles.append(tmp) 377 378 def unzip_to_dir(filename, dirname): 379 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 380 if pattern is not None: 381 cmd.append(pattern) 382 p = Run(cmd, stdout=subprocess.PIPE) 383 p.communicate() 384 if p.returncode != 0: 385 raise ExternalError("failed to unzip input target-files \"%s\"" % 386 (filename,)) 387 388 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 389 if m: 390 unzip_to_dir(m.group(1), tmp) 391 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 392 filename = m.group(1) 393 else: 394 unzip_to_dir(filename, tmp) 395 396 return tmp, zipfile.ZipFile(filename, "r") 397 398 399def GetKeyPasswords(keylist): 400 """Given a list of keys, prompt the user to enter passwords for 401 those which require them. Return a {key: password} dict. password 402 will be None if the key has no password.""" 403 404 no_passwords = [] 405 need_passwords = [] 406 key_passwords = {} 407 devnull = open("/dev/null", "w+b") 408 for k in sorted(keylist): 409 # We don't need a password for things that aren't really keys. 410 if k in SPECIAL_CERT_STRINGS: 411 no_passwords.append(k) 412 continue 413 414 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 415 "-inform", "DER", "-nocrypt"], 416 stdin=devnull.fileno(), 417 stdout=devnull.fileno(), 418 stderr=subprocess.STDOUT) 419 p.communicate() 420 if p.returncode == 0: 421 # Definitely an unencrypted key. 422 no_passwords.append(k) 423 else: 424 p = Run(["openssl", "pkcs8", "-in", k+OPTIONS.private_key_suffix, 425 "-inform", "DER", "-passin", "pass:"], 426 stdin=devnull.fileno(), 427 stdout=devnull.fileno(), 428 stderr=subprocess.PIPE) 429 stdout, stderr = p.communicate() 430 if p.returncode == 0: 431 # Encrypted key with empty string as password. 432 key_passwords[k] = '' 433 elif stderr.startswith('Error decrypting key'): 434 # Definitely encrypted key. 435 # It would have said "Error reading key" if it didn't parse correctly. 436 need_passwords.append(k) 437 else: 438 # Potentially, a type of key that openssl doesn't understand. 439 # We'll let the routines in signapk.jar handle it. 440 no_passwords.append(k) 441 devnull.close() 442 443 key_passwords.update(PasswordManager().GetPasswords(need_passwords)) 444 key_passwords.update(dict.fromkeys(no_passwords, None)) 445 return key_passwords 446 447 448def SignFile(input_name, output_name, key, password, align=None, 449 whole_file=False): 450 """Sign the input_name zip/jar/apk, producing output_name. Use the 451 given key and password (the latter may be None if the key does not 452 have a password. 453 454 If align is an integer > 1, zipalign is run to align stored files in 455 the output zip on 'align'-byte boundaries. 456 457 If whole_file is true, use the "-w" option to SignApk to embed a 458 signature that covers the whole file in the archive comment of the 459 zip file. 460 """ 461 462 if align == 0 or align == 1: 463 align = None 464 465 if align: 466 temp = tempfile.NamedTemporaryFile() 467 sign_name = temp.name 468 else: 469 sign_name = output_name 470 471 cmd = [OPTIONS.java_path, "-Xmx2048m", "-jar", 472 os.path.join(OPTIONS.search_path, OPTIONS.signapk_path)] 473 cmd.extend(OPTIONS.extra_signapk_args) 474 if whole_file: 475 cmd.append("-w") 476 cmd.extend([key + OPTIONS.public_key_suffix, 477 key + OPTIONS.private_key_suffix, 478 input_name, sign_name]) 479 480 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 481 if password is not None: 482 password += "\n" 483 p.communicate(password) 484 if p.returncode != 0: 485 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 486 487 if align: 488 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 489 p.communicate() 490 if p.returncode != 0: 491 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 492 temp.close() 493 494 495def CheckSize(data, target, info_dict): 496 """Check the data string passed against the max size limit, if 497 any, for the given target. Raise exception if the data is too big. 498 Print a warning if the data is nearing the maximum size.""" 499 500 if target.endswith(".img"): target = target[:-4] 501 mount_point = "/" + target 502 503 if info_dict["fstab"]: 504 if mount_point == "/userdata": mount_point = "/data" 505 p = info_dict["fstab"][mount_point] 506 fs_type = p.fs_type 507 device = p.device 508 if "/" in device: 509 device = device[device.rfind("/")+1:] 510 limit = info_dict.get(device + "_size", None) 511 if not fs_type or not limit: return 512 513 if fs_type == "yaffs2": 514 # image size should be increased by 1/64th to account for the 515 # spare area (64 bytes per 2k page) 516 limit = limit / 2048 * (2048+64) 517 size = len(data) 518 pct = float(size) * 100.0 / limit 519 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 520 if pct >= 99.0: 521 raise ExternalError(msg) 522 elif pct >= 95.0: 523 print 524 print " WARNING: ", msg 525 print 526 elif OPTIONS.verbose: 527 print " ", msg 528 529 530def ReadApkCerts(tf_zip): 531 """Given a target_files ZipFile, parse the META/apkcerts.txt file 532 and return a {package: cert} dict.""" 533 certmap = {} 534 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 535 line = line.strip() 536 if not line: continue 537 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 538 r'private_key="(.*)"$', line) 539 if m: 540 name, cert, privkey = m.groups() 541 public_key_suffix_len = len(OPTIONS.public_key_suffix) 542 private_key_suffix_len = len(OPTIONS.private_key_suffix) 543 if cert in SPECIAL_CERT_STRINGS and not privkey: 544 certmap[name] = cert 545 elif (cert.endswith(OPTIONS.public_key_suffix) and 546 privkey.endswith(OPTIONS.private_key_suffix) and 547 cert[:-public_key_suffix_len] == privkey[:-private_key_suffix_len]): 548 certmap[name] = cert[:-public_key_suffix_len] 549 else: 550 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 551 return certmap 552 553 554COMMON_DOCSTRING = """ 555 -p (--path) <dir> 556 Prepend <dir>/bin to the list of places to search for binaries 557 run by this script, and expect to find jars in <dir>/framework. 558 559 -s (--device_specific) <file> 560 Path to the python module containing device-specific 561 releasetools code. 562 563 -x (--extra) <key=value> 564 Add a key/value pair to the 'extras' dict, which device-specific 565 extension code may look at. 566 567 -v (--verbose) 568 Show command lines being executed. 569 570 -h (--help) 571 Display this usage message and exit. 572""" 573 574def Usage(docstring): 575 print docstring.rstrip("\n") 576 print COMMON_DOCSTRING 577 578 579def ParseOptions(argv, 580 docstring, 581 extra_opts="", extra_long_opts=(), 582 extra_option_handler=None): 583 """Parse the options in argv and return any arguments that aren't 584 flags. docstring is the calling module's docstring, to be displayed 585 for errors and -h. extra_opts and extra_long_opts are for flags 586 defined by the caller, which are processed by passing them to 587 extra_option_handler.""" 588 589 try: 590 opts, args = getopt.getopt( 591 argv, "hvp:s:x:" + extra_opts, 592 ["help", "verbose", "path=", "signapk_path=", "extra_signapk_args=", 593 "java_path=", "public_key_suffix=", "private_key_suffix=", 594 "device_specific=", "extra="] + 595 list(extra_long_opts)) 596 except getopt.GetoptError, err: 597 Usage(docstring) 598 print "**", str(err), "**" 599 sys.exit(2) 600 601 path_specified = False 602 603 for o, a in opts: 604 if o in ("-h", "--help"): 605 Usage(docstring) 606 sys.exit() 607 elif o in ("-v", "--verbose"): 608 OPTIONS.verbose = True 609 elif o in ("-p", "--path"): 610 OPTIONS.search_path = a 611 elif o in ("--signapk_path",): 612 OPTIONS.signapk_path = a 613 elif o in ("--extra_signapk_args",): 614 OPTIONS.extra_signapk_args = shlex.split(a) 615 elif o in ("--java_path",): 616 OPTIONS.java_path = a 617 elif o in ("--public_key_suffix",): 618 OPTIONS.public_key_suffix = a 619 elif o in ("--private_key_suffix",): 620 OPTIONS.private_key_suffix = a 621 elif o in ("-s", "--device_specific"): 622 OPTIONS.device_specific = a 623 elif o in ("-x", "--extra"): 624 key, value = a.split("=", 1) 625 OPTIONS.extras[key] = value 626 else: 627 if extra_option_handler is None or not extra_option_handler(o, a): 628 assert False, "unknown option \"%s\"" % (o,) 629 630 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 631 os.pathsep + os.environ["PATH"]) 632 633 return args 634 635 636def Cleanup(): 637 for i in OPTIONS.tempfiles: 638 if os.path.isdir(i): 639 shutil.rmtree(i) 640 else: 641 os.remove(i) 642 643 644class PasswordManager(object): 645 def __init__(self): 646 self.editor = os.getenv("EDITOR", None) 647 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 648 649 def GetPasswords(self, items): 650 """Get passwords corresponding to each string in 'items', 651 returning a dict. (The dict may have keys in addition to the 652 values in 'items'.) 653 654 Uses the passwords in $ANDROID_PW_FILE if available, letting the 655 user edit that file to add more needed passwords. If no editor is 656 available, or $ANDROID_PW_FILE isn't define, prompts the user 657 interactively in the ordinary way. 658 """ 659 660 current = self.ReadFile() 661 662 first = True 663 while True: 664 missing = [] 665 for i in items: 666 if i not in current or not current[i]: 667 missing.append(i) 668 # Are all the passwords already in the file? 669 if not missing: return current 670 671 for i in missing: 672 current[i] = "" 673 674 if not first: 675 print "key file %s still missing some passwords." % (self.pwfile,) 676 answer = raw_input("try to edit again? [y]> ").strip() 677 if answer and answer[0] not in 'yY': 678 raise RuntimeError("key passwords unavailable") 679 first = False 680 681 current = self.UpdateAndReadFile(current) 682 683 def PromptResult(self, current): 684 """Prompt the user to enter a value (password) for each key in 685 'current' whose value is fales. Returns a new dict with all the 686 values. 687 """ 688 result = {} 689 for k, v in sorted(current.iteritems()): 690 if v: 691 result[k] = v 692 else: 693 while True: 694 result[k] = getpass.getpass("Enter password for %s key> " 695 % (k,)).strip() 696 if result[k]: break 697 return result 698 699 def UpdateAndReadFile(self, current): 700 if not self.editor or not self.pwfile: 701 return self.PromptResult(current) 702 703 f = open(self.pwfile, "w") 704 os.chmod(self.pwfile, 0600) 705 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 706 f.write("# (Additional spaces are harmless.)\n\n") 707 708 first_line = None 709 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 710 sorted.sort() 711 for i, (_, k, v) in enumerate(sorted): 712 f.write("[[[ %s ]]] %s\n" % (v, k)) 713 if not v and first_line is None: 714 # position cursor on first line with no password. 715 first_line = i + 4 716 f.close() 717 718 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 719 _, _ = p.communicate() 720 721 return self.ReadFile() 722 723 def ReadFile(self): 724 result = {} 725 if self.pwfile is None: return result 726 try: 727 f = open(self.pwfile, "r") 728 for line in f: 729 line = line.strip() 730 if not line or line[0] == '#': continue 731 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 732 if not m: 733 print "failed to parse password file: ", line 734 else: 735 result[m.group(2)] = m.group(1) 736 f.close() 737 except IOError, e: 738 if e.errno != errno.ENOENT: 739 print "error reading password file: ", str(e) 740 return result 741 742 743def ZipWriteStr(zip, filename, data, perms=0644, compression=None): 744 # use a fixed timestamp so the output is repeatable. 745 zinfo = zipfile.ZipInfo(filename=filename, 746 date_time=(2009, 1, 1, 0, 0, 0)) 747 if compression is None: 748 zinfo.compress_type = zip.compression 749 else: 750 zinfo.compress_type = compression 751 zinfo.external_attr = perms << 16 752 zip.writestr(zinfo, data) 753 754 755class DeviceSpecificParams(object): 756 module = None 757 def __init__(self, **kwargs): 758 """Keyword arguments to the constructor become attributes of this 759 object, which is passed to all functions in the device-specific 760 module.""" 761 for k, v in kwargs.iteritems(): 762 setattr(self, k, v) 763 self.extras = OPTIONS.extras 764 765 if self.module is None: 766 path = OPTIONS.device_specific 767 if not path: return 768 try: 769 if os.path.isdir(path): 770 info = imp.find_module("releasetools", [path]) 771 else: 772 d, f = os.path.split(path) 773 b, x = os.path.splitext(f) 774 if x == ".py": 775 f = b 776 info = imp.find_module(f, [d]) 777 print "loaded device-specific extensions from", path 778 self.module = imp.load_module("device_specific", *info) 779 except ImportError: 780 print "unable to load device-specific module; assuming none" 781 782 def _DoCall(self, function_name, *args, **kwargs): 783 """Call the named function in the device-specific module, passing 784 the given args and kwargs. The first argument to the call will be 785 the DeviceSpecific object itself. If there is no module, or the 786 module does not define the function, return the value of the 787 'default' kwarg (which itself defaults to None).""" 788 if self.module is None or not hasattr(self.module, function_name): 789 return kwargs.get("default", None) 790 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 791 792 def FullOTA_Assertions(self): 793 """Called after emitting the block of assertions at the top of a 794 full OTA package. Implementations can add whatever additional 795 assertions they like.""" 796 return self._DoCall("FullOTA_Assertions") 797 798 def FullOTA_InstallBegin(self): 799 """Called at the start of full OTA installation.""" 800 return self._DoCall("FullOTA_InstallBegin") 801 802 def FullOTA_InstallEnd(self): 803 """Called at the end of full OTA installation; typically this is 804 used to install the image for the device's baseband processor.""" 805 return self._DoCall("FullOTA_InstallEnd") 806 807 def IncrementalOTA_Assertions(self): 808 """Called after emitting the block of assertions at the top of an 809 incremental OTA package. Implementations can add whatever 810 additional assertions they like.""" 811 return self._DoCall("IncrementalOTA_Assertions") 812 813 def IncrementalOTA_VerifyBegin(self): 814 """Called at the start of the verification phase of incremental 815 OTA installation; additional checks can be placed here to abort 816 the script before any changes are made.""" 817 return self._DoCall("IncrementalOTA_VerifyBegin") 818 819 def IncrementalOTA_VerifyEnd(self): 820 """Called at the end of the verification phase of incremental OTA 821 installation; additional checks can be placed here to abort the 822 script before any changes are made.""" 823 return self._DoCall("IncrementalOTA_VerifyEnd") 824 825 def IncrementalOTA_InstallBegin(self): 826 """Called at the start of incremental OTA installation (after 827 verification is complete).""" 828 return self._DoCall("IncrementalOTA_InstallBegin") 829 830 def IncrementalOTA_InstallEnd(self): 831 """Called at the end of incremental OTA installation; typically 832 this is used to install the image for the device's baseband 833 processor.""" 834 return self._DoCall("IncrementalOTA_InstallEnd") 835 836class File(object): 837 def __init__(self, name, data): 838 self.name = name 839 self.data = data 840 self.size = len(data) 841 self.sha1 = sha1(data).hexdigest() 842 843 @classmethod 844 def FromLocalFile(cls, name, diskname): 845 f = open(diskname, "rb") 846 data = f.read() 847 f.close() 848 return File(name, data) 849 850 def WriteToTemp(self): 851 t = tempfile.NamedTemporaryFile() 852 t.write(self.data) 853 t.flush() 854 return t 855 856 def AddToZip(self, z, compression=None): 857 ZipWriteStr(z, self.name, self.data, compression=compression) 858 859DIFF_PROGRAM_BY_EXT = { 860 ".gz" : "imgdiff", 861 ".zip" : ["imgdiff", "-z"], 862 ".jar" : ["imgdiff", "-z"], 863 ".apk" : ["imgdiff", "-z"], 864 ".img" : "imgdiff", 865 } 866 867class Difference(object): 868 def __init__(self, tf, sf, diff_program=None): 869 self.tf = tf 870 self.sf = sf 871 self.patch = None 872 self.diff_program = diff_program 873 874 def ComputePatch(self): 875 """Compute the patch (as a string of data) needed to turn sf into 876 tf. Returns the same tuple as GetPatch().""" 877 878 tf = self.tf 879 sf = self.sf 880 881 if self.diff_program: 882 diff_program = self.diff_program 883 else: 884 ext = os.path.splitext(tf.name)[1] 885 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 886 887 ttemp = tf.WriteToTemp() 888 stemp = sf.WriteToTemp() 889 890 ext = os.path.splitext(tf.name)[1] 891 892 try: 893 ptemp = tempfile.NamedTemporaryFile() 894 if isinstance(diff_program, list): 895 cmd = copy.copy(diff_program) 896 else: 897 cmd = [diff_program] 898 cmd.append(stemp.name) 899 cmd.append(ttemp.name) 900 cmd.append(ptemp.name) 901 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 902 _, err = p.communicate() 903 if err or p.returncode != 0: 904 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 905 return None 906 diff = ptemp.read() 907 finally: 908 ptemp.close() 909 stemp.close() 910 ttemp.close() 911 912 self.patch = diff 913 return self.tf, self.sf, self.patch 914 915 916 def GetPatch(self): 917 """Return a tuple (target_file, source_file, patch_data). 918 patch_data may be None if ComputePatch hasn't been called, or if 919 computing the patch failed.""" 920 return self.tf, self.sf, self.patch 921 922 923def ComputeDifferences(diffs): 924 """Call ComputePatch on all the Difference objects in 'diffs'.""" 925 print len(diffs), "diffs to compute" 926 927 # Do the largest files first, to try and reduce the long-pole effect. 928 by_size = [(i.tf.size, i) for i in diffs] 929 by_size.sort(reverse=True) 930 by_size = [i[1] for i in by_size] 931 932 lock = threading.Lock() 933 diff_iter = iter(by_size) # accessed under lock 934 935 def worker(): 936 try: 937 lock.acquire() 938 for d in diff_iter: 939 lock.release() 940 start = time.time() 941 d.ComputePatch() 942 dur = time.time() - start 943 lock.acquire() 944 945 tf, sf, patch = d.GetPatch() 946 if sf.name == tf.name: 947 name = tf.name 948 else: 949 name = "%s (%s)" % (tf.name, sf.name) 950 if patch is None: 951 print "patching failed! %s" % (name,) 952 else: 953 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 954 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 955 lock.release() 956 except Exception, e: 957 print e 958 raise 959 960 # start worker threads; wait for them all to finish. 961 threads = [threading.Thread(target=worker) 962 for i in range(OPTIONS.worker_threads)] 963 for th in threads: 964 th.start() 965 while threads: 966 threads.pop().join() 967 968 969# map recovery.fstab's fs_types to mount/format "partition types" 970PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 971 "ext4": "EMMC", "emmc": "EMMC" } 972 973def GetTypeAndDevice(mount_point, info): 974 fstab = info["fstab"] 975 if fstab: 976 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 977 else: 978 return None 979 980 981def ParseCertificate(data): 982 """Parse a PEM-format certificate.""" 983 cert = [] 984 save = False 985 for line in data.split("\n"): 986 if "--END CERTIFICATE--" in line: 987 break 988 if save: 989 cert.append(line) 990 if "--BEGIN CERTIFICATE--" in line: 991 save = True 992 cert = "".join(cert).decode('base64') 993 return cert 994 995def XDelta3(source_path, target_path, output_path): 996 diff_program = ["xdelta3", "-0", "-B", str(64<<20), "-e", "-f", "-s"] 997 diff_program.append(source_path) 998 diff_program.append(target_path) 999 diff_program.append(output_path) 1000 p = Run(diff_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1001 p.communicate() 1002 assert p.returncode == 0, "Couldn't produce patch" 1003 1004def XZ(path): 1005 compress_program = ["xz", "-zk", "-9", "--check=crc32"] 1006 compress_program.append(path) 1007 p = Run(compress_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 1008 p.communicate() 1009 assert p.returncode == 0, "Couldn't compress patch" 1010 1011def MakeSystemPatch(source_file, target_file): 1012 with tempfile.NamedTemporaryFile() as output_file: 1013 XDelta3(source_file.name, target_file.name, output_file.name) 1014 XZ(output_file.name) 1015 with open(output_file.name + ".xz") as patch_file: 1016 patch_data = patch_file.read() 1017 os.unlink(patch_file.name) 1018 return File("system.img.p", patch_data) 1019 1020def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img): 1021 """Generate a binary patch that creates the recovery image starting 1022 with the boot image. (Most of the space in these images is just the 1023 kernel, which is identical for the two, so the resulting patch 1024 should be efficient.) Add it to the output zip, along with a shell 1025 script that is run from init.rc on first boot to actually do the 1026 patching and install the new recovery image. 1027 1028 recovery_img and boot_img should be File objects for the 1029 corresponding images. info should be the dictionary returned by 1030 common.LoadInfoDict() on the input target_files. 1031 """ 1032 1033 diff_program = ["imgdiff"] 1034 path = os.path.join(input_dir, "SYSTEM", "etc", "recovery-resource.dat") 1035 if os.path.exists(path): 1036 diff_program.append("-b") 1037 diff_program.append(path) 1038 bonus_args = "-b /system/etc/recovery-resource.dat" 1039 else: 1040 bonus_args = "" 1041 1042 d = Difference(recovery_img, boot_img, diff_program=diff_program) 1043 _, _, patch = d.ComputePatch() 1044 output_sink("recovery-from-boot.p", patch) 1045 1046 boot_type, boot_device = GetTypeAndDevice("/boot", OPTIONS.info_dict) 1047 recovery_type, recovery_device = GetTypeAndDevice("/recovery", OPTIONS.info_dict) 1048 1049 sh = """#!/system/bin/sh 1050if ! applypatch -c %(recovery_type)s:%(recovery_device)s:%(recovery_size)d:%(recovery_sha1)s; then 1051 applypatch %(bonus_args)s %(boot_type)s:%(boot_device)s:%(boot_size)d:%(boot_sha1)s %(recovery_type)s:%(recovery_device)s %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p && log -t recovery "Installing new recovery image: succeeded" || log -t recovery "Installing new recovery image: failed" 1052else 1053 log -t recovery "Recovery image already installed" 1054fi 1055""" % { 'boot_size': boot_img.size, 1056 'boot_sha1': boot_img.sha1, 1057 'recovery_size': recovery_img.size, 1058 'recovery_sha1': recovery_img.sha1, 1059 'boot_type': boot_type, 1060 'boot_device': boot_device, 1061 'recovery_type': recovery_type, 1062 'recovery_device': recovery_device, 1063 'bonus_args': bonus_args, 1064 } 1065 1066 # The install script location moved from /system/etc to /system/bin 1067 # in the L release. Parse the init.rc file to find out where the 1068 # target-files expects it to be, and put it there. 1069 sh_location = "etc/install-recovery.sh" 1070 try: 1071 with open(os.path.join(input_dir, "BOOT", "RAMDISK", "init.rc")) as f: 1072 for line in f: 1073 m = re.match("^service flash_recovery /system/(\S+)\s*$", line) 1074 if m: 1075 sh_location = m.group(1) 1076 print "putting script in", sh_location 1077 break 1078 except (OSError, IOError), e: 1079 print "failed to read init.rc: %s" % (e,) 1080 1081 output_sink(sh_location, sh) 1082