common.py revision 0f9aec837f98cadb2d959b235e7eee8db2bd299e
1# Copyright (C) 2008 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import copy 16import errno 17import getopt 18import getpass 19import imp 20import os 21import platform 22import re 23import shutil 24import subprocess 25import sys 26import tempfile 27import threading 28import time 29import zipfile 30 31try: 32 from hashlib import sha1 as sha1 33except ImportError: 34 from sha import sha as sha1 35 36# missing in Python 2.4 and before 37if not hasattr(os, "SEEK_SET"): 38 os.SEEK_SET = 0 39 40class Options(object): pass 41OPTIONS = Options() 42OPTIONS.search_path = "out/host/linux-x86" 43OPTIONS.verbose = False 44OPTIONS.tempfiles = [] 45OPTIONS.device_specific = None 46OPTIONS.extras = {} 47OPTIONS.info_dict = None 48 49 50# Values for "certificate" in apkcerts that mean special things. 51SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL") 52 53 54class ExternalError(RuntimeError): pass 55 56 57def Run(args, **kwargs): 58 """Create and return a subprocess.Popen object, printing the command 59 line on the terminal if -v was specified.""" 60 if OPTIONS.verbose: 61 print " running: ", " ".join(args) 62 return subprocess.Popen(args, **kwargs) 63 64 65def CloseInheritedPipes(): 66 """ Gmake in MAC OS has file descriptor (PIPE) leak. We close those fds 67 before doing other work.""" 68 if platform.system() != "Darwin": 69 return 70 for d in range(3, 1025): 71 try: 72 stat = os.fstat(d) 73 if stat is not None: 74 pipebit = stat[0] & 0x1000 75 if pipebit != 0: 76 os.close(d) 77 except OSError: 78 pass 79 80 81def LoadInfoDict(zip): 82 """Read and parse the META/misc_info.txt key/value pairs from the 83 input target files and return a dict.""" 84 85 d = {} 86 try: 87 for line in zip.read("META/misc_info.txt").split("\n"): 88 line = line.strip() 89 if not line or line.startswith("#"): continue 90 k, v = line.split("=", 1) 91 d[k] = v 92 except KeyError: 93 # ok if misc_info.txt doesn't exist 94 pass 95 96 # backwards compatibility: These values used to be in their own 97 # files. Look for them, in case we're processing an old 98 # target_files zip. 99 100 if "mkyaffs2_extra_flags" not in d: 101 try: 102 d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip() 103 except KeyError: 104 # ok if flags don't exist 105 pass 106 107 if "recovery_api_version" not in d: 108 try: 109 d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip() 110 except KeyError: 111 raise ValueError("can't find recovery API version in input target-files") 112 113 if "tool_extensions" not in d: 114 try: 115 d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip() 116 except KeyError: 117 # ok if extensions don't exist 118 pass 119 120 try: 121 data = zip.read("META/imagesizes.txt") 122 for line in data.split("\n"): 123 if not line: continue 124 name, value = line.split(" ", 1) 125 if not value: continue 126 if name == "blocksize": 127 d[name] = value 128 else: 129 d[name + "_size"] = value 130 except KeyError: 131 pass 132 133 def makeint(key): 134 if key in d: 135 d[key] = int(d[key], 0) 136 137 makeint("recovery_api_version") 138 makeint("blocksize") 139 makeint("system_size") 140 makeint("userdata_size") 141 makeint("cache_size") 142 makeint("recovery_size") 143 makeint("boot_size") 144 145 d["fstab"] = LoadRecoveryFSTab(zip) 146 return d 147 148def LoadRecoveryFSTab(zip): 149 class Partition(object): 150 pass 151 152 try: 153 data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab") 154 except KeyError: 155 print "Warning: could not find RECOVERY/RAMDISK/etc/recovery.fstab in %s." % zip 156 data = "" 157 158 d = {} 159 for line in data.split("\n"): 160 line = line.strip() 161 if not line or line.startswith("#"): continue 162 pieces = line.split() 163 if not (3 <= len(pieces) <= 4): 164 raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,)) 165 166 p = Partition() 167 p.mount_point = pieces[0] 168 p.fs_type = pieces[1] 169 p.device = pieces[2] 170 p.length = 0 171 options = None 172 if len(pieces) >= 4: 173 if pieces[3].startswith("/"): 174 p.device2 = pieces[3] 175 if len(pieces) >= 5: 176 options = pieces[4] 177 else: 178 p.device2 = None 179 options = pieces[3] 180 else: 181 p.device2 = None 182 183 if options: 184 options = options.split(",") 185 for i in options: 186 if i.startswith("length="): 187 p.length = int(i[7:]) 188 else: 189 print "%s: unknown option \"%s\"" % (p.mount_point, i) 190 191 d[p.mount_point] = p 192 return d 193 194 195def DumpInfoDict(d): 196 for k, v in sorted(d.items()): 197 print "%-25s = (%s) %s" % (k, type(v).__name__, v) 198 199def BuildBootableImage(sourcedir): 200 """Take a kernel, cmdline, and ramdisk directory from the input (in 201 'sourcedir'), and turn them into a boot image. Return the image 202 data, or None if sourcedir does not appear to contains files for 203 building the requested image.""" 204 205 if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or 206 not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)): 207 return None 208 209 ramdisk_img = tempfile.NamedTemporaryFile() 210 img = tempfile.NamedTemporaryFile() 211 212 p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")], 213 stdout=subprocess.PIPE) 214 p2 = Run(["minigzip"], 215 stdin=p1.stdout, stdout=ramdisk_img.file.fileno()) 216 217 p2.wait() 218 p1.wait() 219 assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,) 220 assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,) 221 222 cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")] 223 224 fn = os.path.join(sourcedir, "cmdline") 225 if os.access(fn, os.F_OK): 226 cmd.append("--cmdline") 227 cmd.append(open(fn).read().rstrip("\n")) 228 229 fn = os.path.join(sourcedir, "base") 230 if os.access(fn, os.F_OK): 231 cmd.append("--base") 232 cmd.append(open(fn).read().rstrip("\n")) 233 234 fn = os.path.join(sourcedir, "pagesize") 235 if os.access(fn, os.F_OK): 236 cmd.append("--pagesize") 237 cmd.append(open(fn).read().rstrip("\n")) 238 239 cmd.extend(["--ramdisk", ramdisk_img.name, 240 "--output", img.name]) 241 242 p = Run(cmd, stdout=subprocess.PIPE) 243 p.communicate() 244 assert p.returncode == 0, "mkbootimg of %s image failed" % ( 245 os.path.basename(sourcedir),) 246 247 img.seek(os.SEEK_SET, 0) 248 data = img.read() 249 250 ramdisk_img.close() 251 img.close() 252 253 return data 254 255 256def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir): 257 """Return a File object (with name 'name') with the desired bootable 258 image. Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name 259 'prebuilt_name', otherwise construct it from the source files in 260 'unpack_dir'/'tree_subdir'.""" 261 262 prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name) 263 if os.path.exists(prebuilt_path): 264 print "using prebuilt %s..." % (prebuilt_name,) 265 return File.FromLocalFile(name, prebuilt_path) 266 else: 267 print "building image from target_files %s..." % (tree_subdir,) 268 return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir))) 269 270 271def UnzipTemp(filename, pattern=None): 272 """Unzip the given archive into a temporary directory and return the name. 273 274 If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a 275 temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES. 276 277 Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the 278 main file), open for reading. 279 """ 280 281 tmp = tempfile.mkdtemp(prefix="targetfiles-") 282 OPTIONS.tempfiles.append(tmp) 283 284 def unzip_to_dir(filename, dirname): 285 cmd = ["unzip", "-o", "-q", filename, "-d", dirname] 286 if pattern is not None: 287 cmd.append(pattern) 288 p = Run(cmd, stdout=subprocess.PIPE) 289 p.communicate() 290 if p.returncode != 0: 291 raise ExternalError("failed to unzip input target-files \"%s\"" % 292 (filename,)) 293 294 m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE) 295 if m: 296 unzip_to_dir(m.group(1), tmp) 297 unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES")) 298 filename = m.group(1) 299 else: 300 unzip_to_dir(filename, tmp) 301 302 return tmp, zipfile.ZipFile(filename, "r") 303 304 305def GetKeyPasswords(keylist): 306 """Given a list of keys, prompt the user to enter passwords for 307 those which require them. Return a {key: password} dict. password 308 will be None if the key has no password.""" 309 310 no_passwords = [] 311 need_passwords = [] 312 devnull = open("/dev/null", "w+b") 313 for k in sorted(keylist): 314 # We don't need a password for things that aren't really keys. 315 if k in SPECIAL_CERT_STRINGS: 316 no_passwords.append(k) 317 continue 318 319 p = Run(["openssl", "pkcs8", "-in", k+".pk8", 320 "-inform", "DER", "-nocrypt"], 321 stdin=devnull.fileno(), 322 stdout=devnull.fileno(), 323 stderr=subprocess.STDOUT) 324 p.communicate() 325 if p.returncode == 0: 326 no_passwords.append(k) 327 else: 328 need_passwords.append(k) 329 devnull.close() 330 331 key_passwords = PasswordManager().GetPasswords(need_passwords) 332 key_passwords.update(dict.fromkeys(no_passwords, None)) 333 return key_passwords 334 335 336def SignFile(input_name, output_name, key, password, align=None, 337 whole_file=False): 338 """Sign the input_name zip/jar/apk, producing output_name. Use the 339 given key and password (the latter may be None if the key does not 340 have a password. 341 342 If align is an integer > 1, zipalign is run to align stored files in 343 the output zip on 'align'-byte boundaries. 344 345 If whole_file is true, use the "-w" option to SignApk to embed a 346 signature that covers the whole file in the archive comment of the 347 zip file. 348 """ 349 350 if align == 0 or align == 1: 351 align = None 352 353 if align: 354 temp = tempfile.NamedTemporaryFile() 355 sign_name = temp.name 356 else: 357 sign_name = output_name 358 359 cmd = ["java", "-Xmx2048m", "-jar", 360 os.path.join(OPTIONS.search_path, "framework", "signapk.jar")] 361 if whole_file: 362 cmd.append("-w") 363 cmd.extend([key + ".x509.pem", key + ".pk8", 364 input_name, sign_name]) 365 366 p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) 367 if password is not None: 368 password += "\n" 369 p.communicate(password) 370 if p.returncode != 0: 371 raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,)) 372 373 if align: 374 p = Run(["zipalign", "-f", str(align), sign_name, output_name]) 375 p.communicate() 376 if p.returncode != 0: 377 raise ExternalError("zipalign failed: return code %s" % (p.returncode,)) 378 temp.close() 379 380 381def CheckSize(data, target, info_dict): 382 """Check the data string passed against the max size limit, if 383 any, for the given target. Raise exception if the data is too big. 384 Print a warning if the data is nearing the maximum size.""" 385 386 if target.endswith(".img"): target = target[:-4] 387 mount_point = "/" + target 388 389 if info_dict["fstab"]: 390 if mount_point == "/userdata": mount_point = "/data" 391 p = info_dict["fstab"][mount_point] 392 fs_type = p.fs_type 393 device = p.device 394 if "/" in device: 395 device = device[device.rfind("/")+1:] 396 limit = info_dict.get(device + "_size", None) 397 if not fs_type or not limit: return 398 399 if fs_type == "yaffs2": 400 # image size should be increased by 1/64th to account for the 401 # spare area (64 bytes per 2k page) 402 limit = limit / 2048 * (2048+64) 403 size = len(data) 404 pct = float(size) * 100.0 / limit 405 msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit) 406 if pct >= 99.0: 407 raise ExternalError(msg) 408 elif pct >= 95.0: 409 print 410 print " WARNING: ", msg 411 print 412 elif OPTIONS.verbose: 413 print " ", msg 414 415 416def ReadApkCerts(tf_zip): 417 """Given a target_files ZipFile, parse the META/apkcerts.txt file 418 and return a {package: cert} dict.""" 419 certmap = {} 420 for line in tf_zip.read("META/apkcerts.txt").split("\n"): 421 line = line.strip() 422 if not line: continue 423 m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+' 424 r'private_key="(.*)"$', line) 425 if m: 426 name, cert, privkey = m.groups() 427 if cert in SPECIAL_CERT_STRINGS and not privkey: 428 certmap[name] = cert 429 elif (cert.endswith(".x509.pem") and 430 privkey.endswith(".pk8") and 431 cert[:-9] == privkey[:-4]): 432 certmap[name] = cert[:-9] 433 else: 434 raise ValueError("failed to parse line from apkcerts.txt:\n" + line) 435 return certmap 436 437 438COMMON_DOCSTRING = """ 439 -p (--path) <dir> 440 Prepend <dir>/bin to the list of places to search for binaries 441 run by this script, and expect to find jars in <dir>/framework. 442 443 -s (--device_specific) <file> 444 Path to the python module containing device-specific 445 releasetools code. 446 447 -x (--extra) <key=value> 448 Add a key/value pair to the 'extras' dict, which device-specific 449 extension code may look at. 450 451 -v (--verbose) 452 Show command lines being executed. 453 454 -h (--help) 455 Display this usage message and exit. 456""" 457 458def Usage(docstring): 459 print docstring.rstrip("\n") 460 print COMMON_DOCSTRING 461 462 463def ParseOptions(argv, 464 docstring, 465 extra_opts="", extra_long_opts=(), 466 extra_option_handler=None): 467 """Parse the options in argv and return any arguments that aren't 468 flags. docstring is the calling module's docstring, to be displayed 469 for errors and -h. extra_opts and extra_long_opts are for flags 470 defined by the caller, which are processed by passing them to 471 extra_option_handler.""" 472 473 try: 474 opts, args = getopt.getopt( 475 argv, "hvp:s:x:" + extra_opts, 476 ["help", "verbose", "path=", "device_specific=", "extra="] + 477 list(extra_long_opts)) 478 except getopt.GetoptError, err: 479 Usage(docstring) 480 print "**", str(err), "**" 481 sys.exit(2) 482 483 path_specified = False 484 485 for o, a in opts: 486 if o in ("-h", "--help"): 487 Usage(docstring) 488 sys.exit() 489 elif o in ("-v", "--verbose"): 490 OPTIONS.verbose = True 491 elif o in ("-p", "--path"): 492 OPTIONS.search_path = a 493 elif o in ("-s", "--device_specific"): 494 OPTIONS.device_specific = a 495 elif o in ("-x", "--extra"): 496 key, value = a.split("=", 1) 497 OPTIONS.extras[key] = value 498 else: 499 if extra_option_handler is None or not extra_option_handler(o, a): 500 assert False, "unknown option \"%s\"" % (o,) 501 502 os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") + 503 os.pathsep + os.environ["PATH"]) 504 505 return args 506 507 508def Cleanup(): 509 for i in OPTIONS.tempfiles: 510 if os.path.isdir(i): 511 shutil.rmtree(i) 512 else: 513 os.remove(i) 514 515 516class PasswordManager(object): 517 def __init__(self): 518 self.editor = os.getenv("EDITOR", None) 519 self.pwfile = os.getenv("ANDROID_PW_FILE", None) 520 521 def GetPasswords(self, items): 522 """Get passwords corresponding to each string in 'items', 523 returning a dict. (The dict may have keys in addition to the 524 values in 'items'.) 525 526 Uses the passwords in $ANDROID_PW_FILE if available, letting the 527 user edit that file to add more needed passwords. If no editor is 528 available, or $ANDROID_PW_FILE isn't define, prompts the user 529 interactively in the ordinary way. 530 """ 531 532 current = self.ReadFile() 533 534 first = True 535 while True: 536 missing = [] 537 for i in items: 538 if i not in current or not current[i]: 539 missing.append(i) 540 # Are all the passwords already in the file? 541 if not missing: return current 542 543 for i in missing: 544 current[i] = "" 545 546 if not first: 547 print "key file %s still missing some passwords." % (self.pwfile,) 548 answer = raw_input("try to edit again? [y]> ").strip() 549 if answer and answer[0] not in 'yY': 550 raise RuntimeError("key passwords unavailable") 551 first = False 552 553 current = self.UpdateAndReadFile(current) 554 555 def PromptResult(self, current): 556 """Prompt the user to enter a value (password) for each key in 557 'current' whose value is fales. Returns a new dict with all the 558 values. 559 """ 560 result = {} 561 for k, v in sorted(current.iteritems()): 562 if v: 563 result[k] = v 564 else: 565 while True: 566 result[k] = getpass.getpass("Enter password for %s key> " 567 % (k,)).strip() 568 if result[k]: break 569 return result 570 571 def UpdateAndReadFile(self, current): 572 if not self.editor or not self.pwfile: 573 return self.PromptResult(current) 574 575 f = open(self.pwfile, "w") 576 os.chmod(self.pwfile, 0600) 577 f.write("# Enter key passwords between the [[[ ]]] brackets.\n") 578 f.write("# (Additional spaces are harmless.)\n\n") 579 580 first_line = None 581 sorted = [(not v, k, v) for (k, v) in current.iteritems()] 582 sorted.sort() 583 for i, (_, k, v) in enumerate(sorted): 584 f.write("[[[ %s ]]] %s\n" % (v, k)) 585 if not v and first_line is None: 586 # position cursor on first line with no password. 587 first_line = i + 4 588 f.close() 589 590 p = Run([self.editor, "+%d" % (first_line,), self.pwfile]) 591 _, _ = p.communicate() 592 593 return self.ReadFile() 594 595 def ReadFile(self): 596 result = {} 597 if self.pwfile is None: return result 598 try: 599 f = open(self.pwfile, "r") 600 for line in f: 601 line = line.strip() 602 if not line or line[0] == '#': continue 603 m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line) 604 if not m: 605 print "failed to parse password file: ", line 606 else: 607 result[m.group(2)] = m.group(1) 608 f.close() 609 except IOError, e: 610 if e.errno != errno.ENOENT: 611 print "error reading password file: ", str(e) 612 return result 613 614 615def ZipWriteStr(zip, filename, data, perms=0644): 616 # use a fixed timestamp so the output is repeatable. 617 zinfo = zipfile.ZipInfo(filename=filename, 618 date_time=(2009, 1, 1, 0, 0, 0)) 619 zinfo.compress_type = zip.compression 620 zinfo.external_attr = perms << 16 621 zip.writestr(zinfo, data) 622 623 624class DeviceSpecificParams(object): 625 module = None 626 def __init__(self, **kwargs): 627 """Keyword arguments to the constructor become attributes of this 628 object, which is passed to all functions in the device-specific 629 module.""" 630 for k, v in kwargs.iteritems(): 631 setattr(self, k, v) 632 self.extras = OPTIONS.extras 633 634 if self.module is None: 635 path = OPTIONS.device_specific 636 if not path: return 637 try: 638 if os.path.isdir(path): 639 info = imp.find_module("releasetools", [path]) 640 else: 641 d, f = os.path.split(path) 642 b, x = os.path.splitext(f) 643 if x == ".py": 644 f = b 645 info = imp.find_module(f, [d]) 646 self.module = imp.load_module("device_specific", *info) 647 except ImportError: 648 print "unable to load device-specific module; assuming none" 649 650 def _DoCall(self, function_name, *args, **kwargs): 651 """Call the named function in the device-specific module, passing 652 the given args and kwargs. The first argument to the call will be 653 the DeviceSpecific object itself. If there is no module, or the 654 module does not define the function, return the value of the 655 'default' kwarg (which itself defaults to None).""" 656 if self.module is None or not hasattr(self.module, function_name): 657 return kwargs.get("default", None) 658 return getattr(self.module, function_name)(*((self,) + args), **kwargs) 659 660 def FullOTA_Assertions(self): 661 """Called after emitting the block of assertions at the top of a 662 full OTA package. Implementations can add whatever additional 663 assertions they like.""" 664 return self._DoCall("FullOTA_Assertions") 665 666 def FullOTA_InstallBegin(self): 667 """Called at the start of full OTA installation.""" 668 return self._DoCall("FullOTA_InstallBegin") 669 670 def FullOTA_InstallEnd(self): 671 """Called at the end of full OTA installation; typically this is 672 used to install the image for the device's baseband processor.""" 673 return self._DoCall("FullOTA_InstallEnd") 674 675 def IncrementalOTA_Assertions(self): 676 """Called after emitting the block of assertions at the top of an 677 incremental OTA package. Implementations can add whatever 678 additional assertions they like.""" 679 return self._DoCall("IncrementalOTA_Assertions") 680 681 def IncrementalOTA_VerifyBegin(self): 682 """Called at the start of the verification phase of incremental 683 OTA installation; additional checks can be placed here to abort 684 the script before any changes are made.""" 685 return self._DoCall("IncrementalOTA_VerifyBegin") 686 687 def IncrementalOTA_VerifyEnd(self): 688 """Called at the end of the verification phase of incremental OTA 689 installation; additional checks can be placed here to abort the 690 script before any changes are made.""" 691 return self._DoCall("IncrementalOTA_VerifyEnd") 692 693 def IncrementalOTA_InstallBegin(self): 694 """Called at the start of incremental OTA installation (after 695 verification is complete).""" 696 return self._DoCall("IncrementalOTA_InstallBegin") 697 698 def IncrementalOTA_InstallEnd(self): 699 """Called at the end of incremental OTA installation; typically 700 this is used to install the image for the device's baseband 701 processor.""" 702 return self._DoCall("IncrementalOTA_InstallEnd") 703 704class File(object): 705 def __init__(self, name, data): 706 self.name = name 707 self.data = data 708 self.size = len(data) 709 self.sha1 = sha1(data).hexdigest() 710 711 @classmethod 712 def FromLocalFile(cls, name, diskname): 713 f = open(diskname, "rb") 714 data = f.read() 715 f.close() 716 return File(name, data) 717 718 def WriteToTemp(self): 719 t = tempfile.NamedTemporaryFile() 720 t.write(self.data) 721 t.flush() 722 return t 723 724 def AddToZip(self, z): 725 ZipWriteStr(z, self.name, self.data) 726 727DIFF_PROGRAM_BY_EXT = { 728 ".gz" : "imgdiff", 729 ".zip" : ["imgdiff", "-z"], 730 ".jar" : ["imgdiff", "-z"], 731 ".apk" : ["imgdiff", "-z"], 732 ".img" : "imgdiff", 733 } 734 735class Difference(object): 736 def __init__(self, tf, sf): 737 self.tf = tf 738 self.sf = sf 739 self.patch = None 740 741 def ComputePatch(self): 742 """Compute the patch (as a string of data) needed to turn sf into 743 tf. Returns the same tuple as GetPatch().""" 744 745 tf = self.tf 746 sf = self.sf 747 748 ext = os.path.splitext(tf.name)[1] 749 diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff") 750 751 ttemp = tf.WriteToTemp() 752 stemp = sf.WriteToTemp() 753 754 ext = os.path.splitext(tf.name)[1] 755 756 try: 757 ptemp = tempfile.NamedTemporaryFile() 758 if isinstance(diff_program, list): 759 cmd = copy.copy(diff_program) 760 else: 761 cmd = [diff_program] 762 cmd.append(stemp.name) 763 cmd.append(ttemp.name) 764 cmd.append(ptemp.name) 765 p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 766 _, err = p.communicate() 767 if err or p.returncode != 0: 768 print "WARNING: failure running %s:\n%s\n" % (diff_program, err) 769 return None 770 diff = ptemp.read() 771 finally: 772 ptemp.close() 773 stemp.close() 774 ttemp.close() 775 776 self.patch = diff 777 return self.tf, self.sf, self.patch 778 779 780 def GetPatch(self): 781 """Return a tuple (target_file, source_file, patch_data). 782 patch_data may be None if ComputePatch hasn't been called, or if 783 computing the patch failed.""" 784 return self.tf, self.sf, self.patch 785 786 787def ComputeDifferences(diffs): 788 """Call ComputePatch on all the Difference objects in 'diffs'.""" 789 print len(diffs), "diffs to compute" 790 791 # Do the largest files first, to try and reduce the long-pole effect. 792 by_size = [(i.tf.size, i) for i in diffs] 793 by_size.sort(reverse=True) 794 by_size = [i[1] for i in by_size] 795 796 lock = threading.Lock() 797 diff_iter = iter(by_size) # accessed under lock 798 799 def worker(): 800 try: 801 lock.acquire() 802 for d in diff_iter: 803 lock.release() 804 start = time.time() 805 d.ComputePatch() 806 dur = time.time() - start 807 lock.acquire() 808 809 tf, sf, patch = d.GetPatch() 810 if sf.name == tf.name: 811 name = tf.name 812 else: 813 name = "%s (%s)" % (tf.name, sf.name) 814 if patch is None: 815 print "patching failed! %s" % (name,) 816 else: 817 print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % ( 818 dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name) 819 lock.release() 820 except Exception, e: 821 print e 822 raise 823 824 # start worker threads; wait for them all to finish. 825 threads = [threading.Thread(target=worker) 826 for i in range(OPTIONS.worker_threads)] 827 for th in threads: 828 th.start() 829 while threads: 830 threads.pop().join() 831 832 833# map recovery.fstab's fs_types to mount/format "partition types" 834PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD", 835 "ext4": "EMMC", "emmc": "EMMC" } 836 837def GetTypeAndDevice(mount_point, info): 838 fstab = info["fstab"] 839 if fstab: 840 return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device 841 else: 842 return None 843