common.py revision 55d932840f1a5b412f2961f79368ecec2d28f647
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import re 22import shutil 23import subprocess 24import sys 25import tempfile 26import threading 27import time 28import zipfile 29 30try: 31 import hashlib 32 sha1 = hashlib.sha1 33except ImportError: 34 import sha 35 sha1 = sha.sha 36 37# missing in Python 2.4 and before 38if not hasattr(os, "SEEK_SET"): 39 os.SEEK_SET = 0 40 41class Options(object): pass 42OPTIONS = Options() 43OPTIONS.search_path = "out/host/linux-x86" 44OPTIONS.verbose = False 45OPTIONS.tempfiles = [] 46OPTIONS.device_specific = None 47OPTIONS.extras = {} 48OPTIONS.info_dict = None 49 50 51# Values for "certificate" in apkcerts that mean special things. 52SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 53 54 55class ExternalError(RuntimeError): pass 56 57 58def Run(args, **kwargs): 59 """Create and return a subprocess.Popen object, printing the command 60 line on the terminal if -v was specified.""" 61 if OPTIONS.verbose: 62 print " running: ", " ".join(args) 63 return subprocess.Popen(args, **kwargs) 64 65 66def LoadInfoDict(zip): 67 """Read and parse the META/misc_info.txt key/value pairs from the 68 input target files and return a dict.""" 69 70 d = {} 71 try: 72 for line in zip.read("META/misc_info.txt").split("\n"): 73 line = line.strip() 74 if not line or line.startswith("#"): continue 75 k, v = line.split("=", 1) 76 d[k] = v 77 except KeyError: 78 # ok if misc_info.txt doesn't exist 79 pass 80 81 # backwards compatibility: These values used to be in their own 82 # files. Look for them, in case we're processing an old 83 # target_files zip. 84 85 if "mkyaffs2_extra_flags" not in d: 86 try: 87 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 88 except KeyError: 89 # ok if flags don't exist 90 pass 91 92 if "recovery_api_version" not in d: 93 try: 94 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 95 except KeyError: 96 raise ValueError("can't find recovery API version in input target-files") 97 98 if "tool_extensions" not in d: 99 try: 100 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 101 except KeyError: 102 # ok if extensions don't exist 103 pass 104 105 try: 106 data = zip.read("META/imagesizes.txt") 107 for line in data.split("\n"): 108 if not line: continue 109 name, value = line.split(" ", 1) 110 if not value: continue 111 if name == "blocksize": 112 d[name] = value 113 else: 114 d[name + "_size"] = value 115 except KeyError: 116 pass 117 118 def makeint(key): 119 if key in d: 120 d[key] = int(d[key], 0) 121 122 makeint("recovery_api_version") 123 makeint("blocksize") 124 makeint("system_size") 125 makeint("userdata_size") 126 makeint("recovery_size") 127 makeint("boot_size") 128 129 d["fstab"] = LoadRecoveryFSTab(zip) 130 if not d["fstab"]: 131 if "fs_type" not in d: d["fs_type"] = "yaffs2" 132 if "partition_type" not in d: d["partition_type"] = "MTD" 133 134 return d 135 136def LoadRecoveryFSTab(zip): 137 class Partition(object): 138 pass 139 140 try: 141 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab") 142 except KeyError: 143 # older target-files that doesn't have a recovery.fstab; fall back 144 # to the fs_type and partition_type keys. 145 return 146 147 d = {} 148 for line in data.split("\n"): 149 line = line.strip() 150 if not line or line.startswith("#"): continue 151 pieces = line.split() 152 if not (3 <= len(pieces) <= 4): 153 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 154 155 p = Partition() 156 p.mount_point = pieces[0] 157 p.fs_type = pieces[1] 158 p.device = pieces[2] 159 if len(pieces) == 4: 160 p.device2 = pieces[3] 161 else: 162 p.device2 = None 163 164 d[p.mount_point] = p 165 return d 166 167 168def DumpInfoDict(d): 169 for k, v in sorted(d.items()): 170 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 171 172def BuildBootableImage(sourcedir): 173 """Take a kernel, cmdline, and ramdisk directory from the input (in 174 'sourcedir'), and turn them into a boot image. Return the image 175 data, or None if sourcedir does not appear to contains files for 176 building the requested image.""" 177 178 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 179 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 180 return None 181 182 ramdisk_img = tempfile.NamedTemporaryFile() 183 img = tempfile.NamedTemporaryFile() 184 185 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 186 stdout=subprocess.PIPE) 187 p2 = Run(["minigzip"], 188 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 189 190 p2.wait() 191 p1.wait() 192 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 193 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 194 195 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 196 197 fn = os.path.join(sourcedir, "cmdline") 198 if os.access(fn, os.F_OK): 199 cmd.append("--cmdline") 200 cmd.append(open(fn).read().rstrip("\n")) 201 202 fn = os.path.join(sourcedir, "base") 203 if os.access(fn, os.F_OK): 204 cmd.append("--base") 205 cmd.append(open(fn).read().rstrip("\n")) 206 207 fn = os.path.join(sourcedir, "pagesize") 208 if os.access(fn, os.F_OK): 209 cmd.append("--pagesize") 210 cmd.append(open(fn).read().rstrip("\n")) 211 212 cmd.extend(["--ramdisk", ramdisk_img.name, 213 "--output", img.name]) 214 215 p = Run(cmd, stdout=subprocess.PIPE) 216 p.communicate() 217 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 218 os.path.basename(sourcedir),) 219 220 img.seek(os.SEEK_SET, 0) 221 data = img.read() 222 223 ramdisk_img.close() 224 img.close() 225 226 return data 227 228 229def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir): 230 """Return a File object (with name 'name') with the desired bootable 231 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 232 'prebuilt_name', otherwise construct it from the source files in 233 'unpack_dir'/'tree_subdir'.""" 234 235 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 236 if os.path.exists(prebuilt_path): 237 print "using prebuilt %s..." % (prebuilt_name,) 238 return File.FromLocalFile(name, prebuilt_path) 239 else: 240 print "building image from target_files %s..." % (tree_subdir,) 241 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir))) 242 243 244def UnzipTemp(filename, pattern=None): 245 """Unzip the given archive into a temporary directory and return the name. 246 247 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 248 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 249 250 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 251 main file), open for reading. 252 """ 253 254 tmp = tempfile.mkdtemp(prefix="targetfiles-") 255 OPTIONS.tempfiles.append(tmp) 256 257 def unzip_to_dir(filename, dirname): 258 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 259 if pattern is not None: 260 cmd.append(pattern) 261 p = Run(cmd, stdout=subprocess.PIPE) 262 p.communicate() 263 if p.returncode != 0: 264 raise ExternalError("failed to unzip input target-files \"%s\"" % 265 (filename,)) 266 267 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 268 if m: 269 unzip_to_dir(m.group(1), tmp) 270 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 271 filename = m.group(1) 272 else: 273 unzip_to_dir(filename, tmp) 274 275 return tmp, zipfile.ZipFile(filename, "r") 276 277 278def GetKeyPasswords(keylist): 279 """Given a list of keys, prompt the user to enter passwords for 280 those which require them. Return a {key: password} dict. password 281 will be None if the key has no password.""" 282 283 no_passwords = [] 284 need_passwords = [] 285 devnull = open("/dev/null", "w+b") 286 for k in sorted(keylist): 287 # We don't need a password for things that aren't really keys. 288 if k in SPECIAL_CERT_STRINGS: 289 no_passwords.append(k) 290 continue 291 292 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 293 "-inform", "DER", "-nocrypt"], 294 stdin=devnull.fileno(), 295 stdout=devnull.fileno(), 296 stderr=subprocess.STDOUT) 297 p.communicate() 298 if p.returncode == 0: 299 no_passwords.append(k) 300 else: 301 need_passwords.append(k) 302 devnull.close() 303 304 key_passwords = PasswordManager().GetPasswords(need_passwords) 305 key_passwords.update(dict.fromkeys(no_passwords, None)) 306 return key_passwords 307 308 309def SignFile(input_name, output_name, key, password, align=None, 310 whole_file=False): 311 """Sign the input_name zip/jar/apk, producing output_name. Use the 312 given key and password (the latter may be None if the key does not 313 have a password. 314 315 If align is an integer > 1, zipalign is run to align stored files in 316 the output zip on 'align'-byte boundaries. 317 318 If whole_file is true, use the "-w" option to SignApk to embed a 319 signature that covers the whole file in the archive comment of the 320 zip file. 321 """ 322 323 if align == 0 or align == 1: 324 align = None 325 326 if align: 327 temp = tempfile.NamedTemporaryFile() 328 sign_name = temp.name 329 else: 330 sign_name = output_name 331 332 cmd = ["java", "-Xmx512m", "-jar", 333 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 334 if whole_file: 335 cmd.append("-w") 336 cmd.extend([key + ".x509.pem", key + ".pk8", 337 input_name, sign_name]) 338 339 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 340 if password is not None: 341 password += "\n" 342 p.communicate(password) 343 if p.returncode != 0: 344 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 345 346 if align: 347 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 348 p.communicate() 349 if p.returncode != 0: 350 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 351 temp.close() 352 353 354def CheckSize(data, target, info_dict): 355 """Check the data string passed against the max size limit, if 356 any, for the given target. Raise exception if the data is too big. 357 Print a warning if the data is nearing the maximum size.""" 358 359 if target.endswith(".img"): target = target[:-4] 360 mount_point = "/" + target 361 362 if info_dict["fstab"]: 363 if mount_point == "/userdata": mount_point = "/data" 364 p = info_dict["fstab"][mount_point] 365 fs_type = p.fs_type 366 limit = info_dict.get(p.device + "_size", None) 367 else: 368 fs_type = info_dict.get("fs_type", None) 369 limit = info_dict.get(target + "_size", None) 370 if not fs_type or not limit: return 371 372 if fs_type == "yaffs2": 373 # image size should be increased by 1/64th to account for the 374 # spare area (64 bytes per 2k page) 375 limit = limit / 2048 * (2048+64) 376 size = len(data) 377 pct = float(size) * 100.0 / limit 378 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 379 if pct >= 99.0: 380 raise ExternalError(msg) 381 elif pct >= 95.0: 382 print 383 print " WARNING: ", msg 384 print 385 elif OPTIONS.verbose: 386 print " ", msg 387 388 389def ReadApkCerts(tf_zip): 390 """Given a target_files ZipFile, parse the META/apkcerts.txt file 391 and return a {package: cert} dict.""" 392 certmap = {} 393 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 394 line = line.strip() 395 if not line: continue 396 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 397 r'private_key="(.*)"$', line) 398 if m: 399 name, cert, privkey = m.groups() 400 if cert in SPECIAL_CERT_STRINGS and not privkey: 401 certmap[name] = cert 402 elif (cert.endswith(".x509.pem") and 403 privkey.endswith(".pk8") and 404 cert[:-9] == privkey[:-4]): 405 certmap[name] = cert[:-9] 406 else: 407 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 408 return certmap 409 410 411COMMON_DOCSTRING = """ 412 -p (--path) <dir> 413 Prepend <dir>/bin to the list of places to search for binaries 414 run by this script, and expect to find jars in <dir>/framework. 415 416 -s (--device_specific) <file> 417 Path to the python module containing device-specific 418 releasetools code. 419 420 -x (--extra) <key=value> 421 Add a key/value pair to the 'extras' dict, which device-specific 422 extension code may look at. 423 424 -v (--verbose) 425 Show command lines being executed. 426 427 -h (--help) 428 Display this usage message and exit. 429""" 430 431def Usage(docstring): 432 print docstring.rstrip("\n") 433 print COMMON_DOCSTRING 434 435 436def ParseOptions(argv, 437 docstring, 438 extra_opts="", extra_long_opts=(), 439 extra_option_handler=None): 440 """Parse the options in argv and return any arguments that aren't 441 flags. docstring is the calling module's docstring, to be displayed 442 for errors and -h. extra_opts and extra_long_opts are for flags 443 defined by the caller, which are processed by passing them to 444 extra_option_handler.""" 445 446 try: 447 opts, args = getopt.getopt( 448 argv, "hvp:s:x:" + extra_opts, 449 ["help", "verbose", "path=", "device_specific=", "extra="] + 450 list(extra_long_opts)) 451 except getopt.GetoptError, err: 452 Usage(docstring) 453 print "**", str(err), "**" 454 sys.exit(2) 455 456 path_specified = False 457 458 for o, a in opts: 459 if o in ("-h", "--help"): 460 Usage(docstring) 461 sys.exit() 462 elif o in ("-v", "--verbose"): 463 OPTIONS.verbose = True 464 elif o in ("-p", "--path"): 465 OPTIONS.search_path = a 466 elif o in ("-s", "--device_specific"): 467 OPTIONS.device_specific = a 468 elif o in ("-x", "--extra"): 469 key, value = a.split("=", 1) 470 OPTIONS.extras[key] = value 471 else: 472 if extra_option_handler is None or not extra_option_handler(o, a): 473 assert False, "unknown option \"%s\"" % (o,) 474 475 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 476 os.pathsep + os.environ["PATH"]) 477 478 return args 479 480 481def Cleanup(): 482 for i in OPTIONS.tempfiles: 483 if os.path.isdir(i): 484 shutil.rmtree(i) 485 else: 486 os.remove(i) 487 488 489class PasswordManager(object): 490 def __init__(self): 491 self.editor = os.getenv("EDITOR", None) 492 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 493 494 def GetPasswords(self, items): 495 """Get passwords corresponding to each string in 'items', 496 returning a dict. (The dict may have keys in addition to the 497 values in 'items'.) 498 499 Uses the passwords in $ANDROID_PW_FILE if available, letting the 500 user edit that file to add more needed passwords. If no editor is 501 available, or $ANDROID_PW_FILE isn't define, prompts the user 502 interactively in the ordinary way. 503 """ 504 505 current = self.ReadFile() 506 507 first = True 508 while True: 509 missing = [] 510 for i in items: 511 if i not in current or not current[i]: 512 missing.append(i) 513 # Are all the passwords already in the file? 514 if not missing: return current 515 516 for i in missing: 517 current[i] = "" 518 519 if not first: 520 print "key file %s still missing some passwords." % (self.pwfile,) 521 answer = raw_input("try to edit again? [y]> ").strip() 522 if answer and answer[0] not in 'yY': 523 raise RuntimeError("key passwords unavailable") 524 first = False 525 526 current = self.UpdateAndReadFile(current) 527 528 def PromptResult(self, current): 529 """Prompt the user to enter a value (password) for each key in 530 'current' whose value is fales. Returns a new dict with all the 531 values. 532 """ 533 result = {} 534 for k, v in sorted(current.iteritems()): 535 if v: 536 result[k] = v 537 else: 538 while True: 539 result[k] = getpass.getpass("Enter password for %s key> " 540 % (k,)).strip() 541 if result[k]: break 542 return result 543 544 def UpdateAndReadFile(self, current): 545 if not self.editor or not self.pwfile: 546 return self.PromptResult(current) 547 548 f = open(self.pwfile, "w") 549 os.chmod(self.pwfile, 0600) 550 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 551 f.write("# (Additional spaces are harmless.)\n\n") 552 553 first_line = None 554 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 555 sorted.sort() 556 for i, (_, k, v) in enumerate(sorted): 557 f.write("[[[ %s ]]] %s\n" % (v, k)) 558 if not v and first_line is None: 559 # position cursor on first line with no password. 560 first_line = i + 4 561 f.close() 562 563 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 564 _, _ = p.communicate() 565 566 return self.ReadFile() 567 568 def ReadFile(self): 569 result = {} 570 if self.pwfile is None: return result 571 try: 572 f = open(self.pwfile, "r") 573 for line in f: 574 line = line.strip() 575 if not line or line[0] == '#': continue 576 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 577 if not m: 578 print "failed to parse password file: ", line 579 else: 580 result[m.group(2)] = m.group(1) 581 f.close() 582 except IOError, e: 583 if e.errno != errno.ENOENT: 584 print "error reading password file: ", str(e) 585 return result 586 587 588def ZipWriteStr(zip, filename, data, perms=0644): 589 # use a fixed timestamp so the output is repeatable. 590 zinfo = zipfile.ZipInfo(filename=filename, 591 date_time=(2009, 1, 1, 0, 0, 0)) 592 zinfo.compress_type = zip.compression 593 zinfo.external_attr = perms << 16 594 zip.writestr(zinfo, data) 595 596 597class DeviceSpecificParams(object): 598 module = None 599 def __init__(self, **kwargs): 600 """Keyword arguments to the constructor become attributes of this 601 object, which is passed to all functions in the device-specific 602 module.""" 603 for k, v in kwargs.iteritems(): 604 setattr(self, k, v) 605 self.extras = OPTIONS.extras 606 607 if self.module is None: 608 path = OPTIONS.device_specific 609 if not path: return 610 try: 611 if os.path.isdir(path): 612 info = imp.find_module("releasetools", [path]) 613 else: 614 d, f = os.path.split(path) 615 b, x = os.path.splitext(f) 616 if x == ".py": 617 f = b 618 info = imp.find_module(f, [d]) 619 self.module = imp.load_module("device_specific", *info) 620 except ImportError: 621 print "unable to load device-specific module; assuming none" 622 623 def _DoCall(self, function_name, *args, **kwargs): 624 """Call the named function in the device-specific module, passing 625 the given args and kwargs. The first argument to the call will be 626 the DeviceSpecific object itself. If there is no module, or the 627 module does not define the function, return the value of the 628 'default' kwarg (which itself defaults to None).""" 629 if self.module is None or not hasattr(self.module, function_name): 630 return kwargs.get("default", None) 631 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 632 633 def FullOTA_Assertions(self): 634 """Called after emitting the block of assertions at the top of a 635 full OTA package. Implementations can add whatever additional 636 assertions they like.""" 637 return self._DoCall("FullOTA_Assertions") 638 639 def FullOTA_InstallEnd(self): 640 """Called at the end of full OTA installation; typically this is 641 used to install the image for the device's baseband processor.""" 642 return self._DoCall("FullOTA_InstallEnd") 643 644 def IncrementalOTA_Assertions(self): 645 """Called after emitting the block of assertions at the top of an 646 incremental OTA package. Implementations can add whatever 647 additional assertions they like.""" 648 return self._DoCall("IncrementalOTA_Assertions") 649 650 def IncrementalOTA_VerifyEnd(self): 651 """Called at the end of the verification phase of incremental OTA 652 installation; additional checks can be placed here to abort the 653 script before any changes are made.""" 654 return self._DoCall("IncrementalOTA_VerifyEnd") 655 656 def IncrementalOTA_InstallEnd(self): 657 """Called at the end of incremental OTA installation; typically 658 this is used to install the image for the device's baseband 659 processor.""" 660 return self._DoCall("IncrementalOTA_InstallEnd") 661 662class File(object): 663 def __init__(self, name, data): 664 self.name = name 665 self.data = data 666 self.size = len(data) 667 self.sha1 = sha1(data).hexdigest() 668 669 @classmethod 670 def FromLocalFile(cls, name, diskname): 671 f = open(diskname, "rb") 672 data = f.read() 673 f.close() 674 return File(name, data) 675 676 def WriteToTemp(self): 677 t = tempfile.NamedTemporaryFile() 678 t.write(self.data) 679 t.flush() 680 return t 681 682 def AddToZip(self, z): 683 ZipWriteStr(z, self.name, self.data) 684 685DIFF_PROGRAM_BY_EXT = { 686 ".gz" : "imgdiff", 687 ".zip" : ["imgdiff", "-z"], 688 ".jar" : ["imgdiff", "-z"], 689 ".apk" : ["imgdiff", "-z"], 690 ".img" : "imgdiff", 691 } 692 693class Difference(object): 694 def __init__(self, tf, sf): 695 self.tf = tf 696 self.sf = sf 697 self.patch = None 698 699 def ComputePatch(self): 700 """Compute the patch (as a string of data) needed to turn sf into 701 tf. Returns the same tuple as GetPatch().""" 702 703 tf = self.tf 704 sf = self.sf 705 706 ext = os.path.splitext(tf.name)[1] 707 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 708 709 ttemp = tf.WriteToTemp() 710 stemp = sf.WriteToTemp() 711 712 ext = os.path.splitext(tf.name)[1] 713 714 try: 715 ptemp = tempfile.NamedTemporaryFile() 716 if isinstance(diff_program, list): 717 cmd = copy.copy(diff_program) 718 else: 719 cmd = [diff_program] 720 cmd.append(stemp.name) 721 cmd.append(ttemp.name) 722 cmd.append(ptemp.name) 723 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 724 _, err = p.communicate() 725 if err or p.returncode != 0: 726 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 727 return None 728 diff = ptemp.read() 729 finally: 730 ptemp.close() 731 stemp.close() 732 ttemp.close() 733 734 self.patch = diff 735 return self.tf, self.sf, self.patch 736 737 738 def GetPatch(self): 739 """Return a tuple (target_file, source_file, patch_data). 740 patch_data may be None if ComputePatch hasn't been called, or if 741 computing the patch failed.""" 742 return self.tf, self.sf, self.patch 743 744 745def ComputeDifferences(diffs): 746 """Call ComputePatch on all the Difference objects in 'diffs'.""" 747 print len(diffs), "diffs to compute" 748 749 # Do the largest files first, to try and reduce the long-pole effect. 750 by_size = [(i.tf.size, i) for i in diffs] 751 by_size.sort(reverse=True) 752 by_size = [i[1] for i in by_size] 753 754 lock = threading.Lock() 755 diff_iter = iter(by_size) # accessed under lock 756 757 def worker(): 758 try: 759 lock.acquire() 760 for d in diff_iter: 761 lock.release() 762 start = time.time() 763 d.ComputePatch() 764 dur = time.time() - start 765 lock.acquire() 766 767 tf, sf, patch = d.GetPatch() 768 if sf.name == tf.name: 769 name = tf.name 770 else: 771 name = "%s (%s)" % (tf.name, sf.name) 772 if patch is None: 773 print "patching failed! %s" % (name,) 774 else: 775 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 776 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 777 lock.release() 778 except Exception, e: 779 print e 780 raise 781 782 # start worker threads; wait for them all to finish. 783 threads = [threading.Thread(target=worker) 784 for i in range(OPTIONS.worker_threads)] 785 for th in threads: 786 th.start() 787 while threads: 788 threads.pop().join() 789 790 791# map recovery.fstab's fs_types to mount/format "partition types" 792PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 793 "ext4": "EMMC", "emmc": "EMMC" } 794 795def GetTypeAndDevice(mount_point, info): 796 fstab = info["fstab"] 797 if fstab: 798 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 799 else: 800 devices = {"/boot": "boot", 801 "/recovery": "recovery", 802 "/radio": "radio", 803 "/data": "userdata", 804 "/cache": "cache"} 805 return info["partition_type"], info.get("partition_path", "") + devices[mount_point] 806