common.py revision 560d535cffcb38d475ba6af6ac516e61ba8fec56
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import threading 27import time 28import zipfile 29 30try: 31 import hashlib 32 sha1 = hashlib.sha1 33except ImportError: 34 import sha 35 sha1 = sha.sha 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.verbose = False 45OPTIONS.tempfiles = [] 46OPTIONS.device_specific = None 47OPTIONS.extras = {} 48OPTIONS.info_dict = None 49 50 51# Values for "certificate" in apkcerts that mean special things. 52SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 53 54 55class ExternalError(RuntimeError): pass 56 57 58def Run(args, **kwargs): 59 """Create and return a subprocess.Popen object, printing the command 60 line on the terminal if -v was specified.""" 61 if OPTIONS.verbose: 62 print " running: ", " ".join(args) 63 return subprocess.Popen(args, **kwargs) 64 65 66def LoadInfoDict(zip): 67 """Read and parse the META/misc_info.txt key/value pairs from the 68 input target files and return a dict.""" 69 70 d = {} 71 try: 72 for line in zip.read("META/misc_info.txt").split("\n"): 73 line = line.strip() 74 if not line or line.startswith("#"): continue 75 k, v = line.split("=", 1) 76 d[k] = v 77 except KeyError: 78 # ok if misc_info.txt doesn't exist 79 pass 80 81 # backwards compatibility: These values used to be in their own 82 # files. Look for them, in case we're processing an old 83 # target_files zip. 84 85 if "mkyaffs2_extra_flags" not in d: 86 try: 87 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 88 except KeyError: 89 # ok if flags don't exist 90 pass 91 92 if "recovery_api_version" not in d: 93 try: 94 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 95 except KeyError: 96 raise ValueError("can't find recovery API version in input target-files") 97 98 if "tool_extensions" not in d: 99 try: 100 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 101 except KeyError: 102 # ok if extensions don't exist 103 pass 104 105 try: 106 data = zip.read("META/imagesizes.txt") 107 for line in data.split("\n"): 108 if not line: continue 109 name, value = line.split(" ", 1) 110 if not value: continue 111 if name == "blocksize": 112 d[name] = value 113 else: 114 d[name + "_size"] = value 115 except KeyError: 116 pass 117 118 def makeint(key): 119 if key in d: 120 d[key] = int(d[key], 0) 121 122 makeint("recovery_api_version") 123 makeint("blocksize") 124 makeint("system_size") 125 makeint("userdata_size") 126 makeint("recovery_size") 127 makeint("boot_size") 128 129 d["fstab"] = LoadRecoveryFSTab(zip) 130 if not d["fstab"]: 131 if "fs_type" not in d: d["fs_type"] = "yaffs2" 132 if "partition_type" not in d: d["partition_type"] = "MTD" 133 134 return d 135 136def LoadRecoveryFSTab(zip): 137 class Partition(object): 138 pass 139 140 try: 141 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab") 142 except KeyError: 143 # older target-files that doesn't have a recovery.fstab; fall back 144 # to the fs_type and partition_type keys. 145 return 146 147 d = {} 148 for line in data.split("\n"): 149 line = line.strip() 150 if not line or line.startswith("#"): continue 151 pieces = line.split() 152 if not (3 <= len(pieces) <= 4): 153 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 154 155 p = Partition() 156 p.mount_point = pieces[0] 157 p.fs_type = pieces[1] 158 p.device = pieces[2] 159 p.length = 0 160 options = None 161 if len(pieces) >= 4: 162 if pieces[3].startswith("/"): 163 p.device2 = pieces[3] 164 if len(pieces) >= 5: 165 options = pieces[4] 166 else: 167 p.device2 = None 168 options = pieces[3] 169 else: 170 p.device2 = None 171 172 if options: 173 options = options.split(",") 174 for i in options: 175 if i.startswith("length="): 176 p.length = int(i[7:]) 177 else: 178 print "%s: unknown option \"%s\"" % (p.mount_point, i) 179 180 d[p.mount_point] = p 181 return d 182 183 184def DumpInfoDict(d): 185 for k, v in sorted(d.items()): 186 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 187 188def BuildBootableImage(sourcedir): 189 """Take a kernel, cmdline, and ramdisk directory from the input (in 190 'sourcedir'), and turn them into a boot image. Return the image 191 data, or None if sourcedir does not appear to contains files for 192 building the requested image.""" 193 194 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 195 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 196 return None 197 198 ramdisk_img = tempfile.NamedTemporaryFile() 199 img = tempfile.NamedTemporaryFile() 200 201 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 202 stdout=subprocess.PIPE) 203 p2 = Run(["minigzip"], 204 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 205 206 p2.wait() 207 p1.wait() 208 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 209 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 210 211 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 212 213 fn = os.path.join(sourcedir, "cmdline") 214 if os.access(fn, os.F_OK): 215 cmd.append("--cmdline") 216 cmd.append(open(fn).read().rstrip("\n")) 217 218 fn = os.path.join(sourcedir, "base") 219 if os.access(fn, os.F_OK): 220 cmd.append("--base") 221 cmd.append(open(fn).read().rstrip("\n")) 222 223 fn = os.path.join(sourcedir, "pagesize") 224 if os.access(fn, os.F_OK): 225 cmd.append("--pagesize") 226 cmd.append(open(fn).read().rstrip("\n")) 227 228 cmd.extend(["--ramdisk", ramdisk_img.name, 229 "--output", img.name]) 230 231 p = Run(cmd, stdout=subprocess.PIPE) 232 p.communicate() 233 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 234 os.path.basename(sourcedir),) 235 236 img.seek(os.SEEK_SET, 0) 237 data = img.read() 238 239 ramdisk_img.close() 240 img.close() 241 242 return data 243 244 245def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir): 246 """Return a File object (with name 'name') with the desired bootable 247 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 248 'prebuilt_name', otherwise construct it from the source files in 249 'unpack_dir'/'tree_subdir'.""" 250 251 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 252 if os.path.exists(prebuilt_path): 253 print "using prebuilt %s..." % (prebuilt_name,) 254 return File.FromLocalFile(name, prebuilt_path) 255 else: 256 print "building image from target_files %s..." % (tree_subdir,) 257 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir))) 258 259 260def UnzipTemp(filename, pattern=None): 261 """Unzip the given archive into a temporary directory and return the name. 262 263 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 264 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 265 266 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 267 main file), open for reading. 268 """ 269 270 tmp = tempfile.mkdtemp(prefix="targetfiles-") 271 OPTIONS.tempfiles.append(tmp) 272 273 def unzip_to_dir(filename, dirname): 274 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 275 if pattern is not None: 276 cmd.append(pattern) 277 p = Run(cmd, stdout=subprocess.PIPE) 278 p.communicate() 279 if p.returncode != 0: 280 raise ExternalError("failed to unzip input target-files \"%s\"" % 281 (filename,)) 282 283 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 284 if m: 285 unzip_to_dir(m.group(1), tmp) 286 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 287 filename = m.group(1) 288 else: 289 unzip_to_dir(filename, tmp) 290 291 return tmp, zipfile.ZipFile(filename, "r") 292 293 294def GetKeyPasswords(keylist): 295 """Given a list of keys, prompt the user to enter passwords for 296 those which require them. Return a {key: password} dict. password 297 will be None if the key has no password.""" 298 299 no_passwords = [] 300 need_passwords = [] 301 devnull = open("/dev/null", "w+b") 302 for k in sorted(keylist): 303 # We don't need a password for things that aren't really keys. 304 if k in SPECIAL_CERT_STRINGS: 305 no_passwords.append(k) 306 continue 307 308 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 309 "-inform", "DER", "-nocrypt"], 310 stdin=devnull.fileno(), 311 stdout=devnull.fileno(), 312 stderr=subprocess.STDOUT) 313 p.communicate() 314 if p.returncode == 0: 315 no_passwords.append(k) 316 else: 317 need_passwords.append(k) 318 devnull.close() 319 320 key_passwords = PasswordManager().GetPasswords(need_passwords) 321 key_passwords.update(dict.fromkeys(no_passwords, None)) 322 return key_passwords 323 324 325def SignFile(input_name, output_name, key, password, align=None, 326 whole_file=False): 327 """Sign the input_name zip/jar/apk, producing output_name. Use the 328 given key and password (the latter may be None if the key does not 329 have a password. 330 331 If align is an integer > 1, zipalign is run to align stored files in 332 the output zip on 'align'-byte boundaries. 333 334 If whole_file is true, use the "-w" option to SignApk to embed a 335 signature that covers the whole file in the archive comment of the 336 zip file. 337 """ 338 339 if align == 0 or align == 1: 340 align = None 341 342 if align: 343 temp = tempfile.NamedTemporaryFile() 344 sign_name = temp.name 345 else: 346 sign_name = output_name 347 348 cmd = ["java", "-Xmx512m", "-jar", 349 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 350 if whole_file: 351 cmd.append("-w") 352 cmd.extend([key + ".x509.pem", key + ".pk8", 353 input_name, sign_name]) 354 355 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 356 if password is not None: 357 password += "\n" 358 p.communicate(password) 359 if p.returncode != 0: 360 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 361 362 if align: 363 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 364 p.communicate() 365 if p.returncode != 0: 366 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 367 temp.close() 368 369 370def CheckSize(data, target, info_dict): 371 """Check the data string passed against the max size limit, if 372 any, for the given target. Raise exception if the data is too big. 373 Print a warning if the data is nearing the maximum size.""" 374 375 if target.endswith(".img"): target = target[:-4] 376 mount_point = "/" + target 377 378 if info_dict["fstab"]: 379 if mount_point == "/userdata": mount_point = "/data" 380 p = info_dict["fstab"][mount_point] 381 fs_type = p.fs_type 382 limit = info_dict.get(p.device + "_size", None) 383 else: 384 fs_type = info_dict.get("fs_type", None) 385 limit = info_dict.get(target + "_size", None) 386 if not fs_type or not limit: return 387 388 if fs_type == "yaffs2": 389 # image size should be increased by 1/64th to account for the 390 # spare area (64 bytes per 2k page) 391 limit = limit / 2048 * (2048+64) 392 size = len(data) 393 pct = float(size) * 100.0 / limit 394 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 395 if pct >= 99.0: 396 raise ExternalError(msg) 397 elif pct >= 95.0: 398 print 399 print " WARNING: ", msg 400 print 401 elif OPTIONS.verbose: 402 print " ", msg 403 404 405def ReadApkCerts(tf_zip): 406 """Given a target_files ZipFile, parse the META/apkcerts.txt file 407 and return a {package: cert} dict.""" 408 certmap = {} 409 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 410 line = line.strip() 411 if not line: continue 412 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 413 r'private_key="(.*)"$', line) 414 if m: 415 name, cert, privkey = m.groups() 416 if cert in SPECIAL_CERT_STRINGS and not privkey: 417 certmap[name] = cert 418 elif (cert.endswith(".x509.pem") and 419 privkey.endswith(".pk8") and 420 cert[:-9] == privkey[:-4]): 421 certmap[name] = cert[:-9] 422 else: 423 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 424 return certmap 425 426 427COMMON_DOCSTRING = """ 428 -p (--path) <dir> 429 Prepend <dir>/bin to the list of places to search for binaries 430 run by this script, and expect to find jars in <dir>/framework. 431 432 -s (--device_specific) <file> 433 Path to the python module containing device-specific 434 releasetools code. 435 436 -x (--extra) <key=value> 437 Add a key/value pair to the 'extras' dict, which device-specific 438 extension code may look at. 439 440 -v (--verbose) 441 Show command lines being executed. 442 443 -h (--help) 444 Display this usage message and exit. 445""" 446 447def Usage(docstring): 448 print docstring.rstrip("\n") 449 print COMMON_DOCSTRING 450 451 452def ParseOptions(argv, 453 docstring, 454 extra_opts="", extra_long_opts=(), 455 extra_option_handler=None): 456 """Parse the options in argv and return any arguments that aren't 457 flags. docstring is the calling module's docstring, to be displayed 458 for errors and -h. extra_opts and extra_long_opts are for flags 459 defined by the caller, which are processed by passing them to 460 extra_option_handler.""" 461 462 try: 463 opts, args = getopt.getopt( 464 argv, "hvp:s:x:" + extra_opts, 465 ["help", "verbose", "path=", "device_specific=", "extra="] + 466 list(extra_long_opts)) 467 except getopt.GetoptError, err: 468 Usage(docstring) 469 print "**", str(err), "**" 470 sys.exit(2) 471 472 path_specified = False 473 474 for o, a in opts: 475 if o in ("-h", "--help"): 476 Usage(docstring) 477 sys.exit() 478 elif o in ("-v", "--verbose"): 479 OPTIONS.verbose = True 480 elif o in ("-p", "--path"): 481 OPTIONS.search_path = a 482 elif o in ("-s", "--device_specific"): 483 OPTIONS.device_specific = a 484 elif o in ("-x", "--extra"): 485 key, value = a.split("=", 1) 486 OPTIONS.extras[key] = value 487 else: 488 if extra_option_handler is None or not extra_option_handler(o, a): 489 assert False, "unknown option \"%s\"" % (o,) 490 491 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 492 os.pathsep + os.environ["PATH"]) 493 494 return args 495 496 497def Cleanup(): 498 for i in OPTIONS.tempfiles: 499 if os.path.isdir(i): 500 shutil.rmtree(i) 501 else: 502 os.remove(i) 503 504 505class PasswordManager(object): 506 def __init__(self): 507 self.editor = os.getenv("EDITOR", None) 508 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 509 510 def GetPasswords(self, items): 511 """Get passwords corresponding to each string in 'items', 512 returning a dict. (The dict may have keys in addition to the 513 values in 'items'.) 514 515 Uses the passwords in $ANDROID_PW_FILE if available, letting the 516 user edit that file to add more needed passwords. If no editor is 517 available, or $ANDROID_PW_FILE isn't define, prompts the user 518 interactively in the ordinary way. 519 """ 520 521 current = self.ReadFile() 522 523 first = True 524 while True: 525 missing = [] 526 for i in items: 527 if i not in current or not current[i]: 528 missing.append(i) 529 # Are all the passwords already in the file? 530 if not missing: return current 531 532 for i in missing: 533 current[i] = "" 534 535 if not first: 536 print "key file %s still missing some passwords." % (self.pwfile,) 537 answer = raw_input("try to edit again? [y]> ").strip() 538 if answer and answer[0] not in 'yY': 539 raise RuntimeError("key passwords unavailable") 540 first = False 541 542 current = self.UpdateAndReadFile(current) 543 544 def PromptResult(self, current): 545 """Prompt the user to enter a value (password) for each key in 546 'current' whose value is fales. Returns a new dict with all the 547 values. 548 """ 549 result = {} 550 for k, v in sorted(current.iteritems()): 551 if v: 552 result[k] = v 553 else: 554 while True: 555 result[k] = getpass.getpass("Enter password for %s key> " 556 % (k,)).strip() 557 if result[k]: break 558 return result 559 560 def UpdateAndReadFile(self, current): 561 if not self.editor or not self.pwfile: 562 return self.PromptResult(current) 563 564 f = open(self.pwfile, "w") 565 os.chmod(self.pwfile, 0600) 566 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 567 f.write("# (Additional spaces are harmless.)\n\n") 568 569 first_line = None 570 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 571 sorted.sort() 572 for i, (_, k, v) in enumerate(sorted): 573 f.write("[[[ %s ]]] %s\n" % (v, k)) 574 if not v and first_line is None: 575 # position cursor on first line with no password. 576 first_line = i + 4 577 f.close() 578 579 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 580 _, _ = p.communicate() 581 582 return self.ReadFile() 583 584 def ReadFile(self): 585 result = {} 586 if self.pwfile is None: return result 587 try: 588 f = open(self.pwfile, "r") 589 for line in f: 590 line = line.strip() 591 if not line or line[0] == '#': continue 592 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 593 if not m: 594 print "failed to parse password file: ", line 595 else: 596 result[m.group(2)] = m.group(1) 597 f.close() 598 except IOError, e: 599 if e.errno != errno.ENOENT: 600 print "error reading password file: ", str(e) 601 return result 602 603 604def ZipWriteStr(zip, filename, data, perms=0644): 605 # use a fixed timestamp so the output is repeatable. 606 zinfo = zipfile.ZipInfo(filename=filename, 607 date_time=(2009, 1, 1, 0, 0, 0)) 608 zinfo.compress_type = zip.compression 609 zinfo.external_attr = perms << 16 610 zip.writestr(zinfo, data) 611 612 613class DeviceSpecificParams(object): 614 module = None 615 def __init__(self, **kwargs): 616 """Keyword arguments to the constructor become attributes of this 617 object, which is passed to all functions in the device-specific 618 module.""" 619 for k, v in kwargs.iteritems(): 620 setattr(self, k, v) 621 self.extras = OPTIONS.extras 622 623 if self.module is None: 624 path = OPTIONS.device_specific 625 if not path: return 626 try: 627 if os.path.isdir(path): 628 info = imp.find_module("releasetools", [path]) 629 else: 630 d, f = os.path.split(path) 631 b, x = os.path.splitext(f) 632 if x == ".py": 633 f = b 634 info = imp.find_module(f, [d]) 635 self.module = imp.load_module("device_specific", *info) 636 except ImportError: 637 print "unable to load device-specific module; assuming none" 638 639 def _DoCall(self, function_name, *args, **kwargs): 640 """Call the named function in the device-specific module, passing 641 the given args and kwargs. The first argument to the call will be 642 the DeviceSpecific object itself. If there is no module, or the 643 module does not define the function, return the value of the 644 'default' kwarg (which itself defaults to None).""" 645 if self.module is None or not hasattr(self.module, function_name): 646 return kwargs.get("default", None) 647 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 648 649 def FullOTA_Assertions(self): 650 """Called after emitting the block of assertions at the top of a 651 full OTA package. Implementations can add whatever additional 652 assertions they like.""" 653 return self._DoCall("FullOTA_Assertions") 654 655 def FullOTA_InstallEnd(self): 656 """Called at the end of full OTA installation; typically this is 657 used to install the image for the device's baseband processor.""" 658 return self._DoCall("FullOTA_InstallEnd") 659 660 def IncrementalOTA_Assertions(self): 661 """Called after emitting the block of assertions at the top of an 662 incremental OTA package. Implementations can add whatever 663 additional assertions they like.""" 664 return self._DoCall("IncrementalOTA_Assertions") 665 666 def IncrementalOTA_VerifyEnd(self): 667 """Called at the end of the verification phase of incremental OTA 668 installation; additional checks can be placed here to abort the 669 script before any changes are made.""" 670 return self._DoCall("IncrementalOTA_VerifyEnd") 671 672 def IncrementalOTA_InstallEnd(self): 673 """Called at the end of incremental OTA installation; typically 674 this is used to install the image for the device's baseband 675 processor.""" 676 return self._DoCall("IncrementalOTA_InstallEnd") 677 678class File(object): 679 def __init__(self, name, data): 680 self.name = name 681 self.data = data 682 self.size = len(data) 683 self.sha1 = sha1(data).hexdigest() 684 685 @classmethod 686 def FromLocalFile(cls, name, diskname): 687 f = open(diskname, "rb") 688 data = f.read() 689 f.close() 690 return File(name, data) 691 692 def WriteToTemp(self): 693 t = tempfile.NamedTemporaryFile() 694 t.write(self.data) 695 t.flush() 696 return t 697 698 def AddToZip(self, z): 699 ZipWriteStr(z, self.name, self.data) 700 701DIFF_PROGRAM_BY_EXT = { 702 ".gz" : "imgdiff", 703 ".zip" : ["imgdiff", "-z"], 704 ".jar" : ["imgdiff", "-z"], 705 ".apk" : ["imgdiff", "-z"], 706 ".img" : "imgdiff", 707 } 708 709class Difference(object): 710 def __init__(self, tf, sf): 711 self.tf = tf 712 self.sf = sf 713 self.patch = None 714 715 def ComputePatch(self): 716 """Compute the patch (as a string of data) needed to turn sf into 717 tf. Returns the same tuple as GetPatch().""" 718 719 tf = self.tf 720 sf = self.sf 721 722 ext = os.path.splitext(tf.name)[1] 723 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 724 725 ttemp = tf.WriteToTemp() 726 stemp = sf.WriteToTemp() 727 728 ext = os.path.splitext(tf.name)[1] 729 730 try: 731 ptemp = tempfile.NamedTemporaryFile() 732 if isinstance(diff_program, list): 733 cmd = copy.copy(diff_program) 734 else: 735 cmd = [diff_program] 736 cmd.append(stemp.name) 737 cmd.append(ttemp.name) 738 cmd.append(ptemp.name) 739 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 740 _, err = p.communicate() 741 if err or p.returncode != 0: 742 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 743 return None 744 diff = ptemp.read() 745 finally: 746 ptemp.close() 747 stemp.close() 748 ttemp.close() 749 750 self.patch = diff 751 return self.tf, self.sf, self.patch 752 753 754 def GetPatch(self): 755 """Return a tuple (target_file, source_file, patch_data). 756 patch_data may be None if ComputePatch hasn't been called, or if 757 computing the patch failed.""" 758 return self.tf, self.sf, self.patch 759 760 761def ComputeDifferences(diffs): 762 """Call ComputePatch on all the Difference objects in 'diffs'.""" 763 print len(diffs), "diffs to compute" 764 765 # Do the largest files first, to try and reduce the long-pole effect. 766 by_size = [(i.tf.size, i) for i in diffs] 767 by_size.sort(reverse=True) 768 by_size = [i[1] for i in by_size] 769 770 lock = threading.Lock() 771 diff_iter = iter(by_size) # accessed under lock 772 773 def worker(): 774 try: 775 lock.acquire() 776 for d in diff_iter: 777 lock.release() 778 start = time.time() 779 d.ComputePatch() 780 dur = time.time() - start 781 lock.acquire() 782 783 tf, sf, patch = d.GetPatch() 784 if sf.name == tf.name: 785 name = tf.name 786 else: 787 name = "%s (%s)" % (tf.name, sf.name) 788 if patch is None: 789 print "patching failed! %s" % (name,) 790 else: 791 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 792 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 793 lock.release() 794 except Exception, e: 795 print e 796 raise 797 798 # start worker threads; wait for them all to finish. 799 threads = [threading.Thread(target=worker) 800 for i in range(OPTIONS.worker_threads)] 801 for th in threads: 802 th.start() 803 while threads: 804 threads.pop().join() 805 806 807# map recovery.fstab's fs_types to mount/format "partition types" 808PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 809 "ext4": "EMMC", "emmc": "EMMC" } 810 811def GetTypeAndDevice(mount_point, info): 812 fstab = info["fstab"] 813 if fstab: 814 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 815 else: 816 devices = {"/boot": "boot", 817 "/recovery": "recovery", 818 "/radio": "radio", 819 "/data": "userdata", 820 "/cache": "cache"} 821 return info["partition_type"], info.get("partition_path", "") + devices[mount_point] 822