common.py revision e5ff5907bef42fa9f3eff2e90ad941423dcc91dc
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shutil 24import subprocess 25import sys 26import tempfile 27import threading 28import time 29import zipfile 30 31try: 32 from hashlib import sha1 as sha1 33except ImportError: 34 from sha import sha as sha1 35 36# missing in Python 2.4 and before 37if not hasattr(os, "SEEK_SET"): 38 os.SEEK_SET = 0 39 40class Options(object): pass 41OPTIONS = Options() 42OPTIONS.search_path = "out/host/linux-x86" 43OPTIONS.verbose = False 44OPTIONS.tempfiles = [] 45OPTIONS.device_specific = None 46OPTIONS.extras = {} 47OPTIONS.info_dict = None 48 49 50# Values for "certificate" in apkcerts that mean special things. 51SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 52 53 54class ExternalError(RuntimeError): pass 55 56 57def Run(args, **kwargs): 58 """Create and return a subprocess.Popen object, printing the command 59 line on the terminal if -v was specified.""" 60 if OPTIONS.verbose: 61 print " running: ", " ".join(args) 62 return subprocess.Popen(args, **kwargs) 63 64 65def CloseInheritedPipes(): 66 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 67 before doing other work.""" 68 if platform.system() != "Darwin": 69 return 70 for d in range(3, 1025): 71 try: 72 stat = os.fstat(d) 73 if stat is not None: 74 pipebit = stat[0] & 0x1000 75 if pipebit != 0: 76 os.close(d) 77 except OSError: 78 pass 79 80 81def LoadInfoDict(zip): 82 """Read and parse the META/misc_info.txt key/value pairs from the 83 input target files and return a dict.""" 84 85 d = {} 86 try: 87 for line in zip.read("META/misc_info.txt").split("\n"): 88 line = line.strip() 89 if not line or line.startswith("#"): continue 90 k, v = line.split("=", 1) 91 d[k] = v 92 except KeyError: 93 # ok if misc_info.txt doesn't exist 94 pass 95 96 # backwards compatibility: These values used to be in their own 97 # files. Look for them, in case we're processing an old 98 # target_files zip. 99 100 if "mkyaffs2_extra_flags" not in d: 101 try: 102 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 103 except KeyError: 104 # ok if flags don't exist 105 pass 106 107 if "recovery_api_version" not in d: 108 try: 109 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 110 except KeyError: 111 raise ValueError("can't find recovery API version in input target-files") 112 113 if "tool_extensions" not in d: 114 try: 115 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 116 except KeyError: 117 # ok if extensions don't exist 118 pass 119 120 try: 121 data = zip.read("META/imagesizes.txt") 122 for line in data.split("\n"): 123 if not line: continue 124 name, value = line.split(" ", 1) 125 if not value: continue 126 if name == "blocksize": 127 d[name] = value 128 else: 129 d[name + "_size"] = value 130 except KeyError: 131 pass 132 133 def makeint(key): 134 if key in d: 135 d[key] = int(d[key], 0) 136 137 makeint("recovery_api_version") 138 makeint("blocksize") 139 makeint("system_size") 140 makeint("userdata_size") 141 makeint("cache_size") 142 makeint("recovery_size") 143 makeint("boot_size") 144 145 d["fstab"] = LoadRecoveryFSTab(zip) 146 return d 147 148def LoadRecoveryFSTab(zip): 149 class Partition(object): 150 pass 151 152 try: 153 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab") 154 except KeyError: 155 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip 156 data = "" 157 158 d = {} 159 for line in data.split("\n"): 160 line = line.strip() 161 if not line or line.startswith("#"): continue 162 pieces = line.split() 163 if not (3 <= len(pieces) <= 4): 164 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 165 166 p = Partition() 167 p.mount_point = pieces[0] 168 p.fs_type = pieces[1] 169 p.device = pieces[2] 170 p.length = 0 171 options = None 172 if len(pieces) >= 4: 173 if pieces[3].startswith("/"): 174 p.device2 = pieces[3] 175 if len(pieces) >= 5: 176 options = pieces[4] 177 else: 178 p.device2 = None 179 options = pieces[3] 180 else: 181 p.device2 = None 182 183 if options: 184 options = options.split(",") 185 for i in options: 186 if i.startswith("length="): 187 p.length = int(i[7:]) 188 else: 189 print "%s: unknown option \"%s\"" % (p.mount_point, i) 190 191 d[p.mount_point] = p 192 return d 193 194 195def DumpInfoDict(d): 196 for k, v in sorted(d.items()): 197 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 198 199def BuildBootableImage(sourcedir): 200 """Take a kernel, cmdline, and ramdisk directory from the input (in 201 'sourcedir'), and turn them into a boot image. Return the image 202 data, or None if sourcedir does not appear to contains files for 203 building the requested image.""" 204 205 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 206 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 207 return None 208 209 ramdisk_img = tempfile.NamedTemporaryFile() 210 img = tempfile.NamedTemporaryFile() 211 212 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 213 stdout=subprocess.PIPE) 214 p2 = Run(["minigzip"], 215 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 216 217 p2.wait() 218 p1.wait() 219 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 220 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 221 222 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 223 224 fn = os.path.join(sourcedir, "cmdline") 225 if os.access(fn, os.F_OK): 226 cmd.append("--cmdline") 227 cmd.append(open(fn).read().rstrip("\n")) 228 229 fn = os.path.join(sourcedir, "base") 230 if os.access(fn, os.F_OK): 231 cmd.append("--base") 232 cmd.append(open(fn).read().rstrip("\n")) 233 234 fn = os.path.join(sourcedir, "pagesize") 235 if os.access(fn, os.F_OK): 236 cmd.append("--pagesize") 237 cmd.append(open(fn).read().rstrip("\n")) 238 239 cmd.extend(["--ramdisk", ramdisk_img.name, 240 "--output", img.name]) 241 242 p = Run(cmd, stdout=subprocess.PIPE) 243 p.communicate() 244 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 245 os.path.basename(sourcedir),) 246 247 img.seek(os.SEEK_SET, 0) 248 data = img.read() 249 250 ramdisk_img.close() 251 img.close() 252 253 return data 254 255 256def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir): 257 """Return a File object (with name 'name') with the desired bootable 258 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 259 'prebuilt_name', otherwise construct it from the source files in 260 'unpack_dir'/'tree_subdir'.""" 261 262 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 263 if os.path.exists(prebuilt_path): 264 print "using prebuilt %s..." % (prebuilt_name,) 265 return File.FromLocalFile(name, prebuilt_path) 266 else: 267 print "building image from target_files %s..." % (tree_subdir,) 268 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir))) 269 270 271def UnzipTemp(filename, pattern=None): 272 """Unzip the given archive into a temporary directory and return the name. 273 274 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 275 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 276 277 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 278 main file), open for reading. 279 """ 280 281 tmp = tempfile.mkdtemp(prefix="targetfiles-") 282 OPTIONS.tempfiles.append(tmp) 283 284 def unzip_to_dir(filename, dirname): 285 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 286 if pattern is not None: 287 cmd.append(pattern) 288 p = Run(cmd, stdout=subprocess.PIPE) 289 p.communicate() 290 if p.returncode != 0: 291 raise ExternalError("failed to unzip input target-files \"%s\"" % 292 (filename,)) 293 294 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 295 if m: 296 unzip_to_dir(m.group(1), tmp) 297 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 298 filename = m.group(1) 299 else: 300 unzip_to_dir(filename, tmp) 301 302 return tmp, zipfile.ZipFile(filename, "r") 303 304 305def GetKeyPasswords(keylist): 306 """Given a list of keys, prompt the user to enter passwords for 307 those which require them. Return a {key: password} dict. password 308 will be None if the key has no password.""" 309 310 no_passwords = [] 311 need_passwords = [] 312 devnull = open("/dev/null", "w+b") 313 for k in sorted(keylist): 314 # We don't need a password for things that aren't really keys. 315 if k in SPECIAL_CERT_STRINGS: 316 no_passwords.append(k) 317 continue 318 319 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 320 "-inform", "DER", "-nocrypt"], 321 stdin=devnull.fileno(), 322 stdout=devnull.fileno(), 323 stderr=subprocess.STDOUT) 324 p.communicate() 325 if p.returncode == 0: 326 no_passwords.append(k) 327 else: 328 need_passwords.append(k) 329 devnull.close() 330 331 key_passwords = PasswordManager().GetPasswords(need_passwords) 332 key_passwords.update(dict.fromkeys(no_passwords, None)) 333 return key_passwords 334 335 336def SignFile(input_name, output_name, key, password, align=None, 337 whole_file=False): 338 """Sign the input_name zip/jar/apk, producing output_name. Use the 339 given key and password (the latter may be None if the key does not 340 have a password. 341 342 If align is an integer > 1, zipalign is run to align stored files in 343 the output zip on 'align'-byte boundaries. 344 345 If whole_file is true, use the "-w" option to SignApk to embed a 346 signature that covers the whole file in the archive comment of the 347 zip file. 348 """ 349 350 if align == 0 or align == 1: 351 align = None 352 353 if align: 354 temp = tempfile.NamedTemporaryFile() 355 sign_name = temp.name 356 else: 357 sign_name = output_name 358 359 cmd = ["java", "-Xmx2048m", "-jar", 360 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 361 if whole_file: 362 cmd.append("-w") 363 cmd.extend([key + ".x509.pem", key + ".pk8", 364 input_name, sign_name]) 365 366 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 367 if password is not None: 368 password += "\n" 369 p.communicate(password) 370 if p.returncode != 0: 371 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 372 373 if align: 374 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 375 p.communicate() 376 if p.returncode != 0: 377 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 378 temp.close() 379 380 381def CheckSize(data, target, info_dict): 382 """Check the data string passed against the max size limit, if 383 any, for the given target. Raise exception if the data is too big. 384 Print a warning if the data is nearing the maximum size.""" 385 386 if target.endswith(".img"): target = target[:-4] 387 mount_point = "/" + target 388 389 if info_dict["fstab"]: 390 if mount_point == "/userdata": mount_point = "/data" 391 p = info_dict["fstab"][mount_point] 392 fs_type = p.fs_type 393 limit = info_dict.get(p.device + "_size", None) 394 if not fs_type or not limit: return 395 396 if fs_type == "yaffs2": 397 # image size should be increased by 1/64th to account for the 398 # spare area (64 bytes per 2k page) 399 limit = limit / 2048 * (2048+64) 400 size = len(data) 401 pct = float(size) * 100.0 / limit 402 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 403 if pct >= 99.0: 404 raise ExternalError(msg) 405 elif pct >= 95.0: 406 print 407 print " WARNING: ", msg 408 print 409 elif OPTIONS.verbose: 410 print " ", msg 411 412 413def ReadApkCerts(tf_zip): 414 """Given a target_files ZipFile, parse the META/apkcerts.txt file 415 and return a {package: cert} dict.""" 416 certmap = {} 417 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 418 line = line.strip() 419 if not line: continue 420 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 421 r'private_key="(.*)"$', line) 422 if m: 423 name, cert, privkey = m.groups() 424 if cert in SPECIAL_CERT_STRINGS and not privkey: 425 certmap[name] = cert 426 elif (cert.endswith(".x509.pem") and 427 privkey.endswith(".pk8") and 428 cert[:-9] == privkey[:-4]): 429 certmap[name] = cert[:-9] 430 else: 431 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 432 return certmap 433 434 435COMMON_DOCSTRING = """ 436 -p (--path) <dir> 437 Prepend <dir>/bin to the list of places to search for binaries 438 run by this script, and expect to find jars in <dir>/framework. 439 440 -s (--device_specific) <file> 441 Path to the python module containing device-specific 442 releasetools code. 443 444 -x (--extra) <key=value> 445 Add a key/value pair to the 'extras' dict, which device-specific 446 extension code may look at. 447 448 -v (--verbose) 449 Show command lines being executed. 450 451 -h (--help) 452 Display this usage message and exit. 453""" 454 455def Usage(docstring): 456 print docstring.rstrip("\n") 457 print COMMON_DOCSTRING 458 459 460def ParseOptions(argv, 461 docstring, 462 extra_opts="", extra_long_opts=(), 463 extra_option_handler=None): 464 """Parse the options in argv and return any arguments that aren't 465 flags. docstring is the calling module's docstring, to be displayed 466 for errors and -h. extra_opts and extra_long_opts are for flags 467 defined by the caller, which are processed by passing them to 468 extra_option_handler.""" 469 470 try: 471 opts, args = getopt.getopt( 472 argv, "hvp:s:x:" + extra_opts, 473 ["help", "verbose", "path=", "device_specific=", "extra="] + 474 list(extra_long_opts)) 475 except getopt.GetoptError, err: 476 Usage(docstring) 477 print "**", str(err), "**" 478 sys.exit(2) 479 480 path_specified = False 481 482 for o, a in opts: 483 if o in ("-h", "--help"): 484 Usage(docstring) 485 sys.exit() 486 elif o in ("-v", "--verbose"): 487 OPTIONS.verbose = True 488 elif o in ("-p", "--path"): 489 OPTIONS.search_path = a 490 elif o in ("-s", "--device_specific"): 491 OPTIONS.device_specific = a 492 elif o in ("-x", "--extra"): 493 key, value = a.split("=", 1) 494 OPTIONS.extras[key] = value 495 else: 496 if extra_option_handler is None or not extra_option_handler(o, a): 497 assert False, "unknown option \"%s\"" % (o,) 498 499 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 500 os.pathsep + os.environ["PATH"]) 501 502 return args 503 504 505def Cleanup(): 506 for i in OPTIONS.tempfiles: 507 if os.path.isdir(i): 508 shutil.rmtree(i) 509 else: 510 os.remove(i) 511 512 513class PasswordManager(object): 514 def __init__(self): 515 self.editor = os.getenv("EDITOR", None) 516 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 517 518 def GetPasswords(self, items): 519 """Get passwords corresponding to each string in 'items', 520 returning a dict. (The dict may have keys in addition to the 521 values in 'items'.) 522 523 Uses the passwords in $ANDROID_PW_FILE if available, letting the 524 user edit that file to add more needed passwords. If no editor is 525 available, or $ANDROID_PW_FILE isn't define, prompts the user 526 interactively in the ordinary way. 527 """ 528 529 current = self.ReadFile() 530 531 first = True 532 while True: 533 missing = [] 534 for i in items: 535 if i not in current or not current[i]: 536 missing.append(i) 537 # Are all the passwords already in the file? 538 if not missing: return current 539 540 for i in missing: 541 current[i] = "" 542 543 if not first: 544 print "key file %s still missing some passwords." % (self.pwfile,) 545 answer = raw_input("try to edit again? [y]> ").strip() 546 if answer and answer[0] not in 'yY': 547 raise RuntimeError("key passwords unavailable") 548 first = False 549 550 current = self.UpdateAndReadFile(current) 551 552 def PromptResult(self, current): 553 """Prompt the user to enter a value (password) for each key in 554 'current' whose value is fales. Returns a new dict with all the 555 values. 556 """ 557 result = {} 558 for k, v in sorted(current.iteritems()): 559 if v: 560 result[k] = v 561 else: 562 while True: 563 result[k] = getpass.getpass("Enter password for %s key> " 564 % (k,)).strip() 565 if result[k]: break 566 return result 567 568 def UpdateAndReadFile(self, current): 569 if not self.editor or not self.pwfile: 570 return self.PromptResult(current) 571 572 f = open(self.pwfile, "w") 573 os.chmod(self.pwfile, 0600) 574 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 575 f.write("# (Additional spaces are harmless.)\n\n") 576 577 first_line = None 578 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 579 sorted.sort() 580 for i, (_, k, v) in enumerate(sorted): 581 f.write("[[[ %s ]]] %s\n" % (v, k)) 582 if not v and first_line is None: 583 # position cursor on first line with no password. 584 first_line = i + 4 585 f.close() 586 587 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 588 _, _ = p.communicate() 589 590 return self.ReadFile() 591 592 def ReadFile(self): 593 result = {} 594 if self.pwfile is None: return result 595 try: 596 f = open(self.pwfile, "r") 597 for line in f: 598 line = line.strip() 599 if not line or line[0] == '#': continue 600 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 601 if not m: 602 print "failed to parse password file: ", line 603 else: 604 result[m.group(2)] = m.group(1) 605 f.close() 606 except IOError, e: 607 if e.errno != errno.ENOENT: 608 print "error reading password file: ", str(e) 609 return result 610 611 612def ZipWriteStr(zip, filename, data, perms=0644): 613 # use a fixed timestamp so the output is repeatable. 614 zinfo = zipfile.ZipInfo(filename=filename, 615 date_time=(2009, 1, 1, 0, 0, 0)) 616 zinfo.compress_type = zip.compression 617 zinfo.external_attr = perms << 16 618 zip.writestr(zinfo, data) 619 620 621class DeviceSpecificParams(object): 622 module = None 623 def __init__(self, **kwargs): 624 """Keyword arguments to the constructor become attributes of this 625 object, which is passed to all functions in the device-specific 626 module.""" 627 for k, v in kwargs.iteritems(): 628 setattr(self, k, v) 629 self.extras = OPTIONS.extras 630 631 if self.module is None: 632 path = OPTIONS.device_specific 633 if not path: return 634 try: 635 if os.path.isdir(path): 636 info = imp.find_module("releasetools", [path]) 637 else: 638 d, f = os.path.split(path) 639 b, x = os.path.splitext(f) 640 if x == ".py": 641 f = b 642 info = imp.find_module(f, [d]) 643 self.module = imp.load_module("device_specific", *info) 644 except ImportError: 645 print "unable to load device-specific module; assuming none" 646 647 def _DoCall(self, function_name, *args, **kwargs): 648 """Call the named function in the device-specific module, passing 649 the given args and kwargs. The first argument to the call will be 650 the DeviceSpecific object itself. If there is no module, or the 651 module does not define the function, return the value of the 652 'default' kwarg (which itself defaults to None).""" 653 if self.module is None or not hasattr(self.module, function_name): 654 return kwargs.get("default", None) 655 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 656 657 def FullOTA_Assertions(self): 658 """Called after emitting the block of assertions at the top of a 659 full OTA package. Implementations can add whatever additional 660 assertions they like.""" 661 return self._DoCall("FullOTA_Assertions") 662 663 def FullOTA_InstallBegin(self): 664 """Called at the start of full OTA installation.""" 665 return self._DoCall("FullOTA_InstallBegin") 666 667 def FullOTA_InstallEnd(self): 668 """Called at the end of full OTA installation; typically this is 669 used to install the image for the device's baseband processor.""" 670 return self._DoCall("FullOTA_InstallEnd") 671 672 def IncrementalOTA_Assertions(self): 673 """Called after emitting the block of assertions at the top of an 674 incremental OTA package. Implementations can add whatever 675 additional assertions they like.""" 676 return self._DoCall("IncrementalOTA_Assertions") 677 678 def IncrementalOTA_VerifyBegin(self): 679 """Called at the start of the verification phase of incremental 680 OTA installation; additional checks can be placed here to abort 681 the script before any changes are made.""" 682 return self._DoCall("IncrementalOTA_VerifyBegin") 683 684 def IncrementalOTA_VerifyEnd(self): 685 """Called at the end of the verification phase of incremental OTA 686 installation; additional checks can be placed here to abort the 687 script before any changes are made.""" 688 return self._DoCall("IncrementalOTA_VerifyEnd") 689 690 def IncrementalOTA_InstallBegin(self): 691 """Called at the start of incremental OTA installation (after 692 verification is complete).""" 693 return self._DoCall("IncrementalOTA_InstallBegin") 694 695 def IncrementalOTA_InstallEnd(self): 696 """Called at the end of incremental OTA installation; typically 697 this is used to install the image for the device's baseband 698 processor.""" 699 return self._DoCall("IncrementalOTA_InstallEnd") 700 701class File(object): 702 def __init__(self, name, data): 703 self.name = name 704 self.data = data 705 self.size = len(data) 706 self.sha1 = sha1(data).hexdigest() 707 708 @classmethod 709 def FromLocalFile(cls, name, diskname): 710 f = open(diskname, "rb") 711 data = f.read() 712 f.close() 713 return File(name, data) 714 715 def WriteToTemp(self): 716 t = tempfile.NamedTemporaryFile() 717 t.write(self.data) 718 t.flush() 719 return t 720 721 def AddToZip(self, z): 722 ZipWriteStr(z, self.name, self.data) 723 724DIFF_PROGRAM_BY_EXT = { 725 ".gz" : "imgdiff", 726 ".zip" : ["imgdiff", "-z"], 727 ".jar" : ["imgdiff", "-z"], 728 ".apk" : ["imgdiff", "-z"], 729 ".img" : "imgdiff", 730 } 731 732class Difference(object): 733 def __init__(self, tf, sf): 734 self.tf = tf 735 self.sf = sf 736 self.patch = None 737 738 def ComputePatch(self): 739 """Compute the patch (as a string of data) needed to turn sf into 740 tf. Returns the same tuple as GetPatch().""" 741 742 tf = self.tf 743 sf = self.sf 744 745 ext = os.path.splitext(tf.name)[1] 746 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 747 748 ttemp = tf.WriteToTemp() 749 stemp = sf.WriteToTemp() 750 751 ext = os.path.splitext(tf.name)[1] 752 753 try: 754 ptemp = tempfile.NamedTemporaryFile() 755 if isinstance(diff_program, list): 756 cmd = copy.copy(diff_program) 757 else: 758 cmd = [diff_program] 759 cmd.append(stemp.name) 760 cmd.append(ttemp.name) 761 cmd.append(ptemp.name) 762 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 763 _, err = p.communicate() 764 if err or p.returncode != 0: 765 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 766 return None 767 diff = ptemp.read() 768 finally: 769 ptemp.close() 770 stemp.close() 771 ttemp.close() 772 773 self.patch = diff 774 return self.tf, self.sf, self.patch 775 776 777 def GetPatch(self): 778 """Return a tuple (target_file, source_file, patch_data). 779 patch_data may be None if ComputePatch hasn't been called, or if 780 computing the patch failed.""" 781 return self.tf, self.sf, self.patch 782 783 784def ComputeDifferences(diffs): 785 """Call ComputePatch on all the Difference objects in 'diffs'.""" 786 print len(diffs), "diffs to compute" 787 788 # Do the largest files first, to try and reduce the long-pole effect. 789 by_size = [(i.tf.size, i) for i in diffs] 790 by_size.sort(reverse=True) 791 by_size = [i[1] for i in by_size] 792 793 lock = threading.Lock() 794 diff_iter = iter(by_size) # accessed under lock 795 796 def worker(): 797 try: 798 lock.acquire() 799 for d in diff_iter: 800 lock.release() 801 start = time.time() 802 d.ComputePatch() 803 dur = time.time() - start 804 lock.acquire() 805 806 tf, sf, patch = d.GetPatch() 807 if sf.name == tf.name: 808 name = tf.name 809 else: 810 name = "%s (%s)" % (tf.name, sf.name) 811 if patch is None: 812 print "patching failed! %s" % (name,) 813 else: 814 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 815 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 816 lock.release() 817 except Exception, e: 818 print e 819 raise 820 821 # start worker threads; wait for them all to finish. 822 threads = [threading.Thread(target=worker) 823 for i in range(OPTIONS.worker_threads)] 824 for th in threads: 825 th.start() 826 while threads: 827 threads.pop().join() 828 829 830# map recovery.fstab's fs_types to mount/format "partition types" 831PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 832 "ext4": "EMMC", "emmc": "EMMC" } 833 834def GetTypeAndDevice(mount_point, info): 835 fstab = info["fstab"] 836 if fstab: 837 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 838 else: 839 return None 840