common.py revision 55d932840f1a5b412f2961f79368ecec2d28f647
1# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import copy
16import errno
17import getopt
18import getpass
19import imp
20import os
21import re
22import shutil
23import subprocess
24import sys
25import tempfile
26import threading
27import time
28import zipfile
29
30try:
31  import hashlib
32  sha1 = hashlib.sha1
33except ImportError:
34  import sha
35  sha1 = sha.sha
36
37# missing in Python 2.4 and before
38if not hasattr(os, "SEEK_SET"):
39  os.SEEK_SET = 0
40
41class Options(object): pass
42OPTIONS = Options()
43OPTIONS.search_path = "out/host/linux-x86"
44OPTIONS.verbose = False
45OPTIONS.tempfiles = []
46OPTIONS.device_specific = None
47OPTIONS.extras = {}
48OPTIONS.info_dict = None
49
50
51# Values for "certificate" in apkcerts that mean special things.
52SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
53
54
55class ExternalError(RuntimeError): pass
56
57
58def Run(args, **kwargs):
59  """Create and return a subprocess.Popen object, printing the command
60  line on the terminal if -v was specified."""
61  if OPTIONS.verbose:
62    print "  running: ", " ".join(args)
63  return subprocess.Popen(args, **kwargs)
64
65
66def LoadInfoDict(zip):
67  """Read and parse the META/misc_info.txt key/value pairs from the
68  input target files and return a dict."""
69
70  d = {}
71  try:
72    for line in zip.read("META/misc_info.txt").split("\n"):
73      line = line.strip()
74      if not line or line.startswith("#"): continue
75      k, v = line.split("=", 1)
76      d[k] = v
77  except KeyError:
78    # ok if misc_info.txt doesn't exist
79    pass
80
81  # backwards compatibility: These values used to be in their own
82  # files.  Look for them, in case we're processing an old
83  # target_files zip.
84
85  if "mkyaffs2_extra_flags" not in d:
86    try:
87      d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
88    except KeyError:
89      # ok if flags don't exist
90      pass
91
92  if "recovery_api_version" not in d:
93    try:
94      d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
95    except KeyError:
96      raise ValueError("can't find recovery API version in input target-files")
97
98  if "tool_extensions" not in d:
99    try:
100      d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
101    except KeyError:
102      # ok if extensions don't exist
103      pass
104
105  try:
106    data = zip.read("META/imagesizes.txt")
107    for line in data.split("\n"):
108      if not line: continue
109      name, value = line.split(" ", 1)
110      if not value: continue
111      if name == "blocksize":
112        d[name] = value
113      else:
114        d[name + "_size"] = value
115  except KeyError:
116    pass
117
118  def makeint(key):
119    if key in d:
120      d[key] = int(d[key], 0)
121
122  makeint("recovery_api_version")
123  makeint("blocksize")
124  makeint("system_size")
125  makeint("userdata_size")
126  makeint("recovery_size")
127  makeint("boot_size")
128
129  d["fstab"] = LoadRecoveryFSTab(zip)
130  if not d["fstab"]:
131    if "fs_type" not in d: d["fs_type"] = "yaffs2"
132    if "partition_type" not in d: d["partition_type"] = "MTD"
133
134  return d
135
136def LoadRecoveryFSTab(zip):
137  class Partition(object):
138    pass
139
140  try:
141    data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
142  except KeyError:
143    # older target-files that doesn't have a recovery.fstab; fall back
144    # to the fs_type and partition_type keys.
145    return
146
147  d = {}
148  for line in data.split("\n"):
149    line = line.strip()
150    if not line or line.startswith("#"): continue
151    pieces = line.split()
152    if not (3 <= len(pieces) <= 4):
153      raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
154
155    p = Partition()
156    p.mount_point = pieces[0]
157    p.fs_type = pieces[1]
158    p.device = pieces[2]
159    if len(pieces) == 4:
160      p.device2 = pieces[3]
161    else:
162      p.device2 = None
163
164    d[p.mount_point] = p
165  return d
166
167
168def DumpInfoDict(d):
169  for k, v in sorted(d.items()):
170    print "%-25s = (%s) %s" % (k, type(v).__name__, v)
171
172def BuildBootableImage(sourcedir):
173  """Take a kernel, cmdline, and ramdisk directory from the input (in
174  'sourcedir'), and turn them into a boot image.  Return the image
175  data, or None if sourcedir does not appear to contains files for
176  building the requested image."""
177
178  if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
179      not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
180    return None
181
182  ramdisk_img = tempfile.NamedTemporaryFile()
183  img = tempfile.NamedTemporaryFile()
184
185  p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
186           stdout=subprocess.PIPE)
187  p2 = Run(["minigzip"],
188           stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
189
190  p2.wait()
191  p1.wait()
192  assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
193  assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
194
195  cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
196
197  fn = os.path.join(sourcedir, "cmdline")
198  if os.access(fn, os.F_OK):
199    cmd.append("--cmdline")
200    cmd.append(open(fn).read().rstrip("\n"))
201
202  fn = os.path.join(sourcedir, "base")
203  if os.access(fn, os.F_OK):
204    cmd.append("--base")
205    cmd.append(open(fn).read().rstrip("\n"))
206
207  fn = os.path.join(sourcedir, "pagesize")
208  if os.access(fn, os.F_OK):
209    cmd.append("--pagesize")
210    cmd.append(open(fn).read().rstrip("\n"))
211
212  cmd.extend(["--ramdisk", ramdisk_img.name,
213              "--output", img.name])
214
215  p = Run(cmd, stdout=subprocess.PIPE)
216  p.communicate()
217  assert p.returncode == 0, "mkbootimg of %s image failed" % (
218      os.path.basename(sourcedir),)
219
220  img.seek(os.SEEK_SET, 0)
221  data = img.read()
222
223  ramdisk_img.close()
224  img.close()
225
226  return data
227
228
229def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir):
230  """Return a File object (with name 'name') with the desired bootable
231  image.  Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name
232  'prebuilt_name', otherwise construct it from the source files in
233  'unpack_dir'/'tree_subdir'."""
234
235  prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
236  if os.path.exists(prebuilt_path):
237    print "using prebuilt %s..." % (prebuilt_name,)
238    return File.FromLocalFile(name, prebuilt_path)
239  else:
240    print "building image from target_files %s..." % (tree_subdir,)
241    return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir)))
242
243
244def UnzipTemp(filename, pattern=None):
245  """Unzip the given archive into a temporary directory and return the name.
246
247  If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
248  temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
249
250  Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
251  main file), open for reading.
252  """
253
254  tmp = tempfile.mkdtemp(prefix="targetfiles-")
255  OPTIONS.tempfiles.append(tmp)
256
257  def unzip_to_dir(filename, dirname):
258    cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
259    if pattern is not None:
260      cmd.append(pattern)
261    p = Run(cmd, stdout=subprocess.PIPE)
262    p.communicate()
263    if p.returncode != 0:
264      raise ExternalError("failed to unzip input target-files \"%s\"" %
265                          (filename,))
266
267  m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
268  if m:
269    unzip_to_dir(m.group(1), tmp)
270    unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
271    filename = m.group(1)
272  else:
273    unzip_to_dir(filename, tmp)
274
275  return tmp, zipfile.ZipFile(filename, "r")
276
277
278def GetKeyPasswords(keylist):
279  """Given a list of keys, prompt the user to enter passwords for
280  those which require them.  Return a {key: password} dict.  password
281  will be None if the key has no password."""
282
283  no_passwords = []
284  need_passwords = []
285  devnull = open("/dev/null", "w+b")
286  for k in sorted(keylist):
287    # We don't need a password for things that aren't really keys.
288    if k in SPECIAL_CERT_STRINGS:
289      no_passwords.append(k)
290      continue
291
292    p = Run(["openssl", "pkcs8", "-in", k+".pk8",
293             "-inform", "DER", "-nocrypt"],
294            stdin=devnull.fileno(),
295            stdout=devnull.fileno(),
296            stderr=subprocess.STDOUT)
297    p.communicate()
298    if p.returncode == 0:
299      no_passwords.append(k)
300    else:
301      need_passwords.append(k)
302  devnull.close()
303
304  key_passwords = PasswordManager().GetPasswords(need_passwords)
305  key_passwords.update(dict.fromkeys(no_passwords, None))
306  return key_passwords
307
308
309def SignFile(input_name, output_name, key, password, align=None,
310             whole_file=False):
311  """Sign the input_name zip/jar/apk, producing output_name.  Use the
312  given key and password (the latter may be None if the key does not
313  have a password.
314
315  If align is an integer > 1, zipalign is run to align stored files in
316  the output zip on 'align'-byte boundaries.
317
318  If whole_file is true, use the "-w" option to SignApk to embed a
319  signature that covers the whole file in the archive comment of the
320  zip file.
321  """
322
323  if align == 0 or align == 1:
324    align = None
325
326  if align:
327    temp = tempfile.NamedTemporaryFile()
328    sign_name = temp.name
329  else:
330    sign_name = output_name
331
332  cmd = ["java", "-Xmx512m", "-jar",
333           os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
334  if whole_file:
335    cmd.append("-w")
336  cmd.extend([key + ".x509.pem", key + ".pk8",
337              input_name, sign_name])
338
339  p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
340  if password is not None:
341    password += "\n"
342  p.communicate(password)
343  if p.returncode != 0:
344    raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
345
346  if align:
347    p = Run(["zipalign", "-f", str(align), sign_name, output_name])
348    p.communicate()
349    if p.returncode != 0:
350      raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
351    temp.close()
352
353
354def CheckSize(data, target, info_dict):
355  """Check the data string passed against the max size limit, if
356  any, for the given target.  Raise exception if the data is too big.
357  Print a warning if the data is nearing the maximum size."""
358
359  if target.endswith(".img"): target = target[:-4]
360  mount_point = "/" + target
361
362  if info_dict["fstab"]:
363    if mount_point == "/userdata": mount_point = "/data"
364    p = info_dict["fstab"][mount_point]
365    fs_type = p.fs_type
366    limit = info_dict.get(p.device + "_size", None)
367  else:
368    fs_type = info_dict.get("fs_type", None)
369    limit = info_dict.get(target + "_size", None)
370  if not fs_type or not limit: return
371
372  if fs_type == "yaffs2":
373    # image size should be increased by 1/64th to account for the
374    # spare area (64 bytes per 2k page)
375    limit = limit / 2048 * (2048+64)
376    size = len(data)
377    pct = float(size) * 100.0 / limit
378    msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
379    if pct >= 99.0:
380      raise ExternalError(msg)
381    elif pct >= 95.0:
382      print
383      print "  WARNING: ", msg
384      print
385    elif OPTIONS.verbose:
386      print "  ", msg
387
388
389def ReadApkCerts(tf_zip):
390  """Given a target_files ZipFile, parse the META/apkcerts.txt file
391  and return a {package: cert} dict."""
392  certmap = {}
393  for line in tf_zip.read("META/apkcerts.txt").split("\n"):
394    line = line.strip()
395    if not line: continue
396    m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
397                 r'private_key="(.*)"$', line)
398    if m:
399      name, cert, privkey = m.groups()
400      if cert in SPECIAL_CERT_STRINGS and not privkey:
401        certmap[name] = cert
402      elif (cert.endswith(".x509.pem") and
403            privkey.endswith(".pk8") and
404            cert[:-9] == privkey[:-4]):
405        certmap[name] = cert[:-9]
406      else:
407        raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
408  return certmap
409
410
411COMMON_DOCSTRING = """
412  -p  (--path)  <dir>
413      Prepend <dir>/bin to the list of places to search for binaries
414      run by this script, and expect to find jars in <dir>/framework.
415
416  -s  (--device_specific) <file>
417      Path to the python module containing device-specific
418      releasetools code.
419
420  -x  (--extra)  <key=value>
421      Add a key/value pair to the 'extras' dict, which device-specific
422      extension code may look at.
423
424  -v  (--verbose)
425      Show command lines being executed.
426
427  -h  (--help)
428      Display this usage message and exit.
429"""
430
431def Usage(docstring):
432  print docstring.rstrip("\n")
433  print COMMON_DOCSTRING
434
435
436def ParseOptions(argv,
437                 docstring,
438                 extra_opts="", extra_long_opts=(),
439                 extra_option_handler=None):
440  """Parse the options in argv and return any arguments that aren't
441  flags.  docstring is the calling module's docstring, to be displayed
442  for errors and -h.  extra_opts and extra_long_opts are for flags
443  defined by the caller, which are processed by passing them to
444  extra_option_handler."""
445
446  try:
447    opts, args = getopt.getopt(
448        argv, "hvp:s:x:" + extra_opts,
449        ["help", "verbose", "path=", "device_specific=", "extra="] +
450          list(extra_long_opts))
451  except getopt.GetoptError, err:
452    Usage(docstring)
453    print "**", str(err), "**"
454    sys.exit(2)
455
456  path_specified = False
457
458  for o, a in opts:
459    if o in ("-h", "--help"):
460      Usage(docstring)
461      sys.exit()
462    elif o in ("-v", "--verbose"):
463      OPTIONS.verbose = True
464    elif o in ("-p", "--path"):
465      OPTIONS.search_path = a
466    elif o in ("-s", "--device_specific"):
467      OPTIONS.device_specific = a
468    elif o in ("-x", "--extra"):
469      key, value = a.split("=", 1)
470      OPTIONS.extras[key] = value
471    else:
472      if extra_option_handler is None or not extra_option_handler(o, a):
473        assert False, "unknown option \"%s\"" % (o,)
474
475  os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
476                        os.pathsep + os.environ["PATH"])
477
478  return args
479
480
481def Cleanup():
482  for i in OPTIONS.tempfiles:
483    if os.path.isdir(i):
484      shutil.rmtree(i)
485    else:
486      os.remove(i)
487
488
489class PasswordManager(object):
490  def __init__(self):
491    self.editor = os.getenv("EDITOR", None)
492    self.pwfile = os.getenv("ANDROID_PW_FILE", None)
493
494  def GetPasswords(self, items):
495    """Get passwords corresponding to each string in 'items',
496    returning a dict.  (The dict may have keys in addition to the
497    values in 'items'.)
498
499    Uses the passwords in $ANDROID_PW_FILE if available, letting the
500    user edit that file to add more needed passwords.  If no editor is
501    available, or $ANDROID_PW_FILE isn't define, prompts the user
502    interactively in the ordinary way.
503    """
504
505    current = self.ReadFile()
506
507    first = True
508    while True:
509      missing = []
510      for i in items:
511        if i not in current or not current[i]:
512          missing.append(i)
513      # Are all the passwords already in the file?
514      if not missing: return current
515
516      for i in missing:
517        current[i] = ""
518
519      if not first:
520        print "key file %s still missing some passwords." % (self.pwfile,)
521        answer = raw_input("try to edit again? [y]> ").strip()
522        if answer and answer[0] not in 'yY':
523          raise RuntimeError("key passwords unavailable")
524      first = False
525
526      current = self.UpdateAndReadFile(current)
527
528  def PromptResult(self, current):
529    """Prompt the user to enter a value (password) for each key in
530    'current' whose value is fales.  Returns a new dict with all the
531    values.
532    """
533    result = {}
534    for k, v in sorted(current.iteritems()):
535      if v:
536        result[k] = v
537      else:
538        while True:
539          result[k] = getpass.getpass("Enter password for %s key> "
540                                      % (k,)).strip()
541          if result[k]: break
542    return result
543
544  def UpdateAndReadFile(self, current):
545    if not self.editor or not self.pwfile:
546      return self.PromptResult(current)
547
548    f = open(self.pwfile, "w")
549    os.chmod(self.pwfile, 0600)
550    f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
551    f.write("# (Additional spaces are harmless.)\n\n")
552
553    first_line = None
554    sorted = [(not v, k, v) for (k, v) in current.iteritems()]
555    sorted.sort()
556    for i, (_, k, v) in enumerate(sorted):
557      f.write("[[[  %s  ]]] %s\n" % (v, k))
558      if not v and first_line is None:
559        # position cursor on first line with no password.
560        first_line = i + 4
561    f.close()
562
563    p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
564    _, _ = p.communicate()
565
566    return self.ReadFile()
567
568  def ReadFile(self):
569    result = {}
570    if self.pwfile is None: return result
571    try:
572      f = open(self.pwfile, "r")
573      for line in f:
574        line = line.strip()
575        if not line or line[0] == '#': continue
576        m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
577        if not m:
578          print "failed to parse password file: ", line
579        else:
580          result[m.group(2)] = m.group(1)
581      f.close()
582    except IOError, e:
583      if e.errno != errno.ENOENT:
584        print "error reading password file: ", str(e)
585    return result
586
587
588def ZipWriteStr(zip, filename, data, perms=0644):
589  # use a fixed timestamp so the output is repeatable.
590  zinfo = zipfile.ZipInfo(filename=filename,
591                          date_time=(2009, 1, 1, 0, 0, 0))
592  zinfo.compress_type = zip.compression
593  zinfo.external_attr = perms << 16
594  zip.writestr(zinfo, data)
595
596
597class DeviceSpecificParams(object):
598  module = None
599  def __init__(self, **kwargs):
600    """Keyword arguments to the constructor become attributes of this
601    object, which is passed to all functions in the device-specific
602    module."""
603    for k, v in kwargs.iteritems():
604      setattr(self, k, v)
605    self.extras = OPTIONS.extras
606
607    if self.module is None:
608      path = OPTIONS.device_specific
609      if not path: return
610      try:
611        if os.path.isdir(path):
612          info = imp.find_module("releasetools", [path])
613        else:
614          d, f = os.path.split(path)
615          b, x = os.path.splitext(f)
616          if x == ".py":
617            f = b
618          info = imp.find_module(f, [d])
619        self.module = imp.load_module("device_specific", *info)
620      except ImportError:
621        print "unable to load device-specific module; assuming none"
622
623  def _DoCall(self, function_name, *args, **kwargs):
624    """Call the named function in the device-specific module, passing
625    the given args and kwargs.  The first argument to the call will be
626    the DeviceSpecific object itself.  If there is no module, or the
627    module does not define the function, return the value of the
628    'default' kwarg (which itself defaults to None)."""
629    if self.module is None or not hasattr(self.module, function_name):
630      return kwargs.get("default", None)
631    return getattr(self.module, function_name)(*((self,) + args), **kwargs)
632
633  def FullOTA_Assertions(self):
634    """Called after emitting the block of assertions at the top of a
635    full OTA package.  Implementations can add whatever additional
636    assertions they like."""
637    return self._DoCall("FullOTA_Assertions")
638
639  def FullOTA_InstallEnd(self):
640    """Called at the end of full OTA installation; typically this is
641    used to install the image for the device's baseband processor."""
642    return self._DoCall("FullOTA_InstallEnd")
643
644  def IncrementalOTA_Assertions(self):
645    """Called after emitting the block of assertions at the top of an
646    incremental OTA package.  Implementations can add whatever
647    additional assertions they like."""
648    return self._DoCall("IncrementalOTA_Assertions")
649
650  def IncrementalOTA_VerifyEnd(self):
651    """Called at the end of the verification phase of incremental OTA
652    installation; additional checks can be placed here to abort the
653    script before any changes are made."""
654    return self._DoCall("IncrementalOTA_VerifyEnd")
655
656  def IncrementalOTA_InstallEnd(self):
657    """Called at the end of incremental OTA installation; typically
658    this is used to install the image for the device's baseband
659    processor."""
660    return self._DoCall("IncrementalOTA_InstallEnd")
661
662class File(object):
663  def __init__(self, name, data):
664    self.name = name
665    self.data = data
666    self.size = len(data)
667    self.sha1 = sha1(data).hexdigest()
668
669  @classmethod
670  def FromLocalFile(cls, name, diskname):
671    f = open(diskname, "rb")
672    data = f.read()
673    f.close()
674    return File(name, data)
675
676  def WriteToTemp(self):
677    t = tempfile.NamedTemporaryFile()
678    t.write(self.data)
679    t.flush()
680    return t
681
682  def AddToZip(self, z):
683    ZipWriteStr(z, self.name, self.data)
684
685DIFF_PROGRAM_BY_EXT = {
686    ".gz" : "imgdiff",
687    ".zip" : ["imgdiff", "-z"],
688    ".jar" : ["imgdiff", "-z"],
689    ".apk" : ["imgdiff", "-z"],
690    ".img" : "imgdiff",
691    }
692
693class Difference(object):
694  def __init__(self, tf, sf):
695    self.tf = tf
696    self.sf = sf
697    self.patch = None
698
699  def ComputePatch(self):
700    """Compute the patch (as a string of data) needed to turn sf into
701    tf.  Returns the same tuple as GetPatch()."""
702
703    tf = self.tf
704    sf = self.sf
705
706    ext = os.path.splitext(tf.name)[1]
707    diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
708
709    ttemp = tf.WriteToTemp()
710    stemp = sf.WriteToTemp()
711
712    ext = os.path.splitext(tf.name)[1]
713
714    try:
715      ptemp = tempfile.NamedTemporaryFile()
716      if isinstance(diff_program, list):
717        cmd = copy.copy(diff_program)
718      else:
719        cmd = [diff_program]
720      cmd.append(stemp.name)
721      cmd.append(ttemp.name)
722      cmd.append(ptemp.name)
723      p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
724      _, err = p.communicate()
725      if err or p.returncode != 0:
726        print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
727        return None
728      diff = ptemp.read()
729    finally:
730      ptemp.close()
731      stemp.close()
732      ttemp.close()
733
734    self.patch = diff
735    return self.tf, self.sf, self.patch
736
737
738  def GetPatch(self):
739    """Return a tuple (target_file, source_file, patch_data).
740    patch_data may be None if ComputePatch hasn't been called, or if
741    computing the patch failed."""
742    return self.tf, self.sf, self.patch
743
744
745def ComputeDifferences(diffs):
746  """Call ComputePatch on all the Difference objects in 'diffs'."""
747  print len(diffs), "diffs to compute"
748
749  # Do the largest files first, to try and reduce the long-pole effect.
750  by_size = [(i.tf.size, i) for i in diffs]
751  by_size.sort(reverse=True)
752  by_size = [i[1] for i in by_size]
753
754  lock = threading.Lock()
755  diff_iter = iter(by_size)   # accessed under lock
756
757  def worker():
758    try:
759      lock.acquire()
760      for d in diff_iter:
761        lock.release()
762        start = time.time()
763        d.ComputePatch()
764        dur = time.time() - start
765        lock.acquire()
766
767        tf, sf, patch = d.GetPatch()
768        if sf.name == tf.name:
769          name = tf.name
770        else:
771          name = "%s (%s)" % (tf.name, sf.name)
772        if patch is None:
773          print "patching failed!                                  %s" % (name,)
774        else:
775          print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
776              dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
777      lock.release()
778    except Exception, e:
779      print e
780      raise
781
782  # start worker threads; wait for them all to finish.
783  threads = [threading.Thread(target=worker)
784             for i in range(OPTIONS.worker_threads)]
785  for th in threads:
786    th.start()
787  while threads:
788    threads.pop().join()
789
790
791# map recovery.fstab's fs_types to mount/format "partition types"
792PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
793                    "ext4": "EMMC", "emmc": "EMMC" }
794
795def GetTypeAndDevice(mount_point, info):
796  fstab = info["fstab"]
797  if fstab:
798    return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device
799  else:
800    devices = {"/boot": "boot",
801               "/recovery": "recovery",
802               "/radio": "radio",
803               "/data": "userdata",
804               "/cache": "cache"}
805    return info["partition_type"], info.get("partition_path", "") + devices[mount_point]
806