common.py revision 560d535cffcb38d475ba6af6ac516e61ba8fec56
1# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import copy
16import errno
17import getopt
18import getpass
19import imp
20import os
21import re
22import shutil
23import subprocess
24import sys
25import tempfile
26import threading
27import time
28import zipfile
29
30try:
31  import hashlib
32  sha1 = hashlib.sha1
33except ImportError:
34  import sha
35  sha1 = sha.sha
36
37# missing in Python 2.4 and before
38if not hasattr(os, "SEEK_SET"):
39  os.SEEK_SET = 0
40
41class Options(object): pass
42OPTIONS = Options()
43OPTIONS.search_path = "out/host/linux-x86"
44OPTIONS.verbose = False
45OPTIONS.tempfiles = []
46OPTIONS.device_specific = None
47OPTIONS.extras = {}
48OPTIONS.info_dict = None
49
50
51# Values for "certificate" in apkcerts that mean special things.
52SPECIAL_CERT_STRINGS = ("PRESIGNED", "EXTERNAL")
53
54
55class ExternalError(RuntimeError): pass
56
57
58def Run(args, **kwargs):
59  """Create and return a subprocess.Popen object, printing the command
60  line on the terminal if -v was specified."""
61  if OPTIONS.verbose:
62    print "  running: ", " ".join(args)
63  return subprocess.Popen(args, **kwargs)
64
65
66def LoadInfoDict(zip):
67  """Read and parse the META/misc_info.txt key/value pairs from the
68  input target files and return a dict."""
69
70  d = {}
71  try:
72    for line in zip.read("META/misc_info.txt").split("\n"):
73      line = line.strip()
74      if not line or line.startswith("#"): continue
75      k, v = line.split("=", 1)
76      d[k] = v
77  except KeyError:
78    # ok if misc_info.txt doesn't exist
79    pass
80
81  # backwards compatibility: These values used to be in their own
82  # files.  Look for them, in case we're processing an old
83  # target_files zip.
84
85  if "mkyaffs2_extra_flags" not in d:
86    try:
87      d["mkyaffs2_extra_flags"] = zip.read("META/mkyaffs2-extra-flags.txt").strip()
88    except KeyError:
89      # ok if flags don't exist
90      pass
91
92  if "recovery_api_version" not in d:
93    try:
94      d["recovery_api_version"] = zip.read("META/recovery-api-version.txt").strip()
95    except KeyError:
96      raise ValueError("can't find recovery API version in input target-files")
97
98  if "tool_extensions" not in d:
99    try:
100      d["tool_extensions"] = zip.read("META/tool-extensions.txt").strip()
101    except KeyError:
102      # ok if extensions don't exist
103      pass
104
105  try:
106    data = zip.read("META/imagesizes.txt")
107    for line in data.split("\n"):
108      if not line: continue
109      name, value = line.split(" ", 1)
110      if not value: continue
111      if name == "blocksize":
112        d[name] = value
113      else:
114        d[name + "_size"] = value
115  except KeyError:
116    pass
117
118  def makeint(key):
119    if key in d:
120      d[key] = int(d[key], 0)
121
122  makeint("recovery_api_version")
123  makeint("blocksize")
124  makeint("system_size")
125  makeint("userdata_size")
126  makeint("recovery_size")
127  makeint("boot_size")
128
129  d["fstab"] = LoadRecoveryFSTab(zip)
130  if not d["fstab"]:
131    if "fs_type" not in d: d["fs_type"] = "yaffs2"
132    if "partition_type" not in d: d["partition_type"] = "MTD"
133
134  return d
135
136def LoadRecoveryFSTab(zip):
137  class Partition(object):
138    pass
139
140  try:
141    data = zip.read("RECOVERY/RAMDISK/etc/recovery.fstab")
142  except KeyError:
143    # older target-files that doesn't have a recovery.fstab; fall back
144    # to the fs_type and partition_type keys.
145    return
146
147  d = {}
148  for line in data.split("\n"):
149    line = line.strip()
150    if not line or line.startswith("#"): continue
151    pieces = line.split()
152    if not (3 <= len(pieces) <= 4):
153      raise ValueError("malformed recovery.fstab line: \"%s\"" % (line,))
154
155    p = Partition()
156    p.mount_point = pieces[0]
157    p.fs_type = pieces[1]
158    p.device = pieces[2]
159    p.length = 0
160    options = None
161    if len(pieces) >= 4:
162      if pieces[3].startswith("/"):
163        p.device2 = pieces[3]
164        if len(pieces) >= 5:
165          options = pieces[4]
166      else:
167        p.device2 = None
168        options = pieces[3]
169    else:
170      p.device2 = None
171
172    if options:
173      options = options.split(",")
174      for i in options:
175        if i.startswith("length="):
176          p.length = int(i[7:])
177        else:
178          print "%s: unknown option \"%s\"" % (p.mount_point, i)
179
180    d[p.mount_point] = p
181  return d
182
183
184def DumpInfoDict(d):
185  for k, v in sorted(d.items()):
186    print "%-25s = (%s) %s" % (k, type(v).__name__, v)
187
188def BuildBootableImage(sourcedir):
189  """Take a kernel, cmdline, and ramdisk directory from the input (in
190  'sourcedir'), and turn them into a boot image.  Return the image
191  data, or None if sourcedir does not appear to contains files for
192  building the requested image."""
193
194  if (not os.access(os.path.join(sourcedir, "RAMDISK"), os.F_OK) or
195      not os.access(os.path.join(sourcedir, "kernel"), os.F_OK)):
196    return None
197
198  ramdisk_img = tempfile.NamedTemporaryFile()
199  img = tempfile.NamedTemporaryFile()
200
201  p1 = Run(["mkbootfs", os.path.join(sourcedir, "RAMDISK")],
202           stdout=subprocess.PIPE)
203  p2 = Run(["minigzip"],
204           stdin=p1.stdout, stdout=ramdisk_img.file.fileno())
205
206  p2.wait()
207  p1.wait()
208  assert p1.returncode == 0, "mkbootfs of %s ramdisk failed" % (targetname,)
209  assert p2.returncode == 0, "minigzip of %s ramdisk failed" % (targetname,)
210
211  cmd = ["mkbootimg", "--kernel", os.path.join(sourcedir, "kernel")]
212
213  fn = os.path.join(sourcedir, "cmdline")
214  if os.access(fn, os.F_OK):
215    cmd.append("--cmdline")
216    cmd.append(open(fn).read().rstrip("\n"))
217
218  fn = os.path.join(sourcedir, "base")
219  if os.access(fn, os.F_OK):
220    cmd.append("--base")
221    cmd.append(open(fn).read().rstrip("\n"))
222
223  fn = os.path.join(sourcedir, "pagesize")
224  if os.access(fn, os.F_OK):
225    cmd.append("--pagesize")
226    cmd.append(open(fn).read().rstrip("\n"))
227
228  cmd.extend(["--ramdisk", ramdisk_img.name,
229              "--output", img.name])
230
231  p = Run(cmd, stdout=subprocess.PIPE)
232  p.communicate()
233  assert p.returncode == 0, "mkbootimg of %s image failed" % (
234      os.path.basename(sourcedir),)
235
236  img.seek(os.SEEK_SET, 0)
237  data = img.read()
238
239  ramdisk_img.close()
240  img.close()
241
242  return data
243
244
245def GetBootableImage(name, prebuilt_name, unpack_dir, tree_subdir):
246  """Return a File object (with name 'name') with the desired bootable
247  image.  Look for it in 'unpack_dir'/BOOTABLE_IMAGES under the name
248  'prebuilt_name', otherwise construct it from the source files in
249  'unpack_dir'/'tree_subdir'."""
250
251  prebuilt_path = os.path.join(unpack_dir, "BOOTABLE_IMAGES", prebuilt_name)
252  if os.path.exists(prebuilt_path):
253    print "using prebuilt %s..." % (prebuilt_name,)
254    return File.FromLocalFile(name, prebuilt_path)
255  else:
256    print "building image from target_files %s..." % (tree_subdir,)
257    return File(name, BuildBootableImage(os.path.join(unpack_dir, tree_subdir)))
258
259
260def UnzipTemp(filename, pattern=None):
261  """Unzip the given archive into a temporary directory and return the name.
262
263  If filename is of the form "foo.zip+bar.zip", unzip foo.zip into a
264  temp dir, then unzip bar.zip into that_dir/BOOTABLE_IMAGES.
265
266  Returns (tempdir, zipobj) where zipobj is a zipfile.ZipFile (of the
267  main file), open for reading.
268  """
269
270  tmp = tempfile.mkdtemp(prefix="targetfiles-")
271  OPTIONS.tempfiles.append(tmp)
272
273  def unzip_to_dir(filename, dirname):
274    cmd = ["unzip", "-o", "-q", filename, "-d", dirname]
275    if pattern is not None:
276      cmd.append(pattern)
277    p = Run(cmd, stdout=subprocess.PIPE)
278    p.communicate()
279    if p.returncode != 0:
280      raise ExternalError("failed to unzip input target-files \"%s\"" %
281                          (filename,))
282
283  m = re.match(r"^(.*[.]zip)\+(.*[.]zip)$", filename, re.IGNORECASE)
284  if m:
285    unzip_to_dir(m.group(1), tmp)
286    unzip_to_dir(m.group(2), os.path.join(tmp, "BOOTABLE_IMAGES"))
287    filename = m.group(1)
288  else:
289    unzip_to_dir(filename, tmp)
290
291  return tmp, zipfile.ZipFile(filename, "r")
292
293
294def GetKeyPasswords(keylist):
295  """Given a list of keys, prompt the user to enter passwords for
296  those which require them.  Return a {key: password} dict.  password
297  will be None if the key has no password."""
298
299  no_passwords = []
300  need_passwords = []
301  devnull = open("/dev/null", "w+b")
302  for k in sorted(keylist):
303    # We don't need a password for things that aren't really keys.
304    if k in SPECIAL_CERT_STRINGS:
305      no_passwords.append(k)
306      continue
307
308    p = Run(["openssl", "pkcs8", "-in", k+".pk8",
309             "-inform", "DER", "-nocrypt"],
310            stdin=devnull.fileno(),
311            stdout=devnull.fileno(),
312            stderr=subprocess.STDOUT)
313    p.communicate()
314    if p.returncode == 0:
315      no_passwords.append(k)
316    else:
317      need_passwords.append(k)
318  devnull.close()
319
320  key_passwords = PasswordManager().GetPasswords(need_passwords)
321  key_passwords.update(dict.fromkeys(no_passwords, None))
322  return key_passwords
323
324
325def SignFile(input_name, output_name, key, password, align=None,
326             whole_file=False):
327  """Sign the input_name zip/jar/apk, producing output_name.  Use the
328  given key and password (the latter may be None if the key does not
329  have a password.
330
331  If align is an integer > 1, zipalign is run to align stored files in
332  the output zip on 'align'-byte boundaries.
333
334  If whole_file is true, use the "-w" option to SignApk to embed a
335  signature that covers the whole file in the archive comment of the
336  zip file.
337  """
338
339  if align == 0 or align == 1:
340    align = None
341
342  if align:
343    temp = tempfile.NamedTemporaryFile()
344    sign_name = temp.name
345  else:
346    sign_name = output_name
347
348  cmd = ["java", "-Xmx512m", "-jar",
349           os.path.join(OPTIONS.search_path, "framework", "signapk.jar")]
350  if whole_file:
351    cmd.append("-w")
352  cmd.extend([key + ".x509.pem", key + ".pk8",
353              input_name, sign_name])
354
355  p = Run(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
356  if password is not None:
357    password += "\n"
358  p.communicate(password)
359  if p.returncode != 0:
360    raise ExternalError("signapk.jar failed: return code %s" % (p.returncode,))
361
362  if align:
363    p = Run(["zipalign", "-f", str(align), sign_name, output_name])
364    p.communicate()
365    if p.returncode != 0:
366      raise ExternalError("zipalign failed: return code %s" % (p.returncode,))
367    temp.close()
368
369
370def CheckSize(data, target, info_dict):
371  """Check the data string passed against the max size limit, if
372  any, for the given target.  Raise exception if the data is too big.
373  Print a warning if the data is nearing the maximum size."""
374
375  if target.endswith(".img"): target = target[:-4]
376  mount_point = "/" + target
377
378  if info_dict["fstab"]:
379    if mount_point == "/userdata": mount_point = "/data"
380    p = info_dict["fstab"][mount_point]
381    fs_type = p.fs_type
382    limit = info_dict.get(p.device + "_size", None)
383  else:
384    fs_type = info_dict.get("fs_type", None)
385    limit = info_dict.get(target + "_size", None)
386  if not fs_type or not limit: return
387
388  if fs_type == "yaffs2":
389    # image size should be increased by 1/64th to account for the
390    # spare area (64 bytes per 2k page)
391    limit = limit / 2048 * (2048+64)
392    size = len(data)
393    pct = float(size) * 100.0 / limit
394    msg = "%s size (%d) is %.2f%% of limit (%d)" % (target, size, pct, limit)
395    if pct >= 99.0:
396      raise ExternalError(msg)
397    elif pct >= 95.0:
398      print
399      print "  WARNING: ", msg
400      print
401    elif OPTIONS.verbose:
402      print "  ", msg
403
404
405def ReadApkCerts(tf_zip):
406  """Given a target_files ZipFile, parse the META/apkcerts.txt file
407  and return a {package: cert} dict."""
408  certmap = {}
409  for line in tf_zip.read("META/apkcerts.txt").split("\n"):
410    line = line.strip()
411    if not line: continue
412    m = re.match(r'^name="(.*)"\s+certificate="(.*)"\s+'
413                 r'private_key="(.*)"$', line)
414    if m:
415      name, cert, privkey = m.groups()
416      if cert in SPECIAL_CERT_STRINGS and not privkey:
417        certmap[name] = cert
418      elif (cert.endswith(".x509.pem") and
419            privkey.endswith(".pk8") and
420            cert[:-9] == privkey[:-4]):
421        certmap[name] = cert[:-9]
422      else:
423        raise ValueError("failed to parse line from apkcerts.txt:\n" + line)
424  return certmap
425
426
427COMMON_DOCSTRING = """
428  -p  (--path)  <dir>
429      Prepend <dir>/bin to the list of places to search for binaries
430      run by this script, and expect to find jars in <dir>/framework.
431
432  -s  (--device_specific) <file>
433      Path to the python module containing device-specific
434      releasetools code.
435
436  -x  (--extra)  <key=value>
437      Add a key/value pair to the 'extras' dict, which device-specific
438      extension code may look at.
439
440  -v  (--verbose)
441      Show command lines being executed.
442
443  -h  (--help)
444      Display this usage message and exit.
445"""
446
447def Usage(docstring):
448  print docstring.rstrip("\n")
449  print COMMON_DOCSTRING
450
451
452def ParseOptions(argv,
453                 docstring,
454                 extra_opts="", extra_long_opts=(),
455                 extra_option_handler=None):
456  """Parse the options in argv and return any arguments that aren't
457  flags.  docstring is the calling module's docstring, to be displayed
458  for errors and -h.  extra_opts and extra_long_opts are for flags
459  defined by the caller, which are processed by passing them to
460  extra_option_handler."""
461
462  try:
463    opts, args = getopt.getopt(
464        argv, "hvp:s:x:" + extra_opts,
465        ["help", "verbose", "path=", "device_specific=", "extra="] +
466          list(extra_long_opts))
467  except getopt.GetoptError, err:
468    Usage(docstring)
469    print "**", str(err), "**"
470    sys.exit(2)
471
472  path_specified = False
473
474  for o, a in opts:
475    if o in ("-h", "--help"):
476      Usage(docstring)
477      sys.exit()
478    elif o in ("-v", "--verbose"):
479      OPTIONS.verbose = True
480    elif o in ("-p", "--path"):
481      OPTIONS.search_path = a
482    elif o in ("-s", "--device_specific"):
483      OPTIONS.device_specific = a
484    elif o in ("-x", "--extra"):
485      key, value = a.split("=", 1)
486      OPTIONS.extras[key] = value
487    else:
488      if extra_option_handler is None or not extra_option_handler(o, a):
489        assert False, "unknown option \"%s\"" % (o,)
490
491  os.environ["PATH"] = (os.path.join(OPTIONS.search_path, "bin") +
492                        os.pathsep + os.environ["PATH"])
493
494  return args
495
496
497def Cleanup():
498  for i in OPTIONS.tempfiles:
499    if os.path.isdir(i):
500      shutil.rmtree(i)
501    else:
502      os.remove(i)
503
504
505class PasswordManager(object):
506  def __init__(self):
507    self.editor = os.getenv("EDITOR", None)
508    self.pwfile = os.getenv("ANDROID_PW_FILE", None)
509
510  def GetPasswords(self, items):
511    """Get passwords corresponding to each string in 'items',
512    returning a dict.  (The dict may have keys in addition to the
513    values in 'items'.)
514
515    Uses the passwords in $ANDROID_PW_FILE if available, letting the
516    user edit that file to add more needed passwords.  If no editor is
517    available, or $ANDROID_PW_FILE isn't define, prompts the user
518    interactively in the ordinary way.
519    """
520
521    current = self.ReadFile()
522
523    first = True
524    while True:
525      missing = []
526      for i in items:
527        if i not in current or not current[i]:
528          missing.append(i)
529      # Are all the passwords already in the file?
530      if not missing: return current
531
532      for i in missing:
533        current[i] = ""
534
535      if not first:
536        print "key file %s still missing some passwords." % (self.pwfile,)
537        answer = raw_input("try to edit again? [y]> ").strip()
538        if answer and answer[0] not in 'yY':
539          raise RuntimeError("key passwords unavailable")
540      first = False
541
542      current = self.UpdateAndReadFile(current)
543
544  def PromptResult(self, current):
545    """Prompt the user to enter a value (password) for each key in
546    'current' whose value is fales.  Returns a new dict with all the
547    values.
548    """
549    result = {}
550    for k, v in sorted(current.iteritems()):
551      if v:
552        result[k] = v
553      else:
554        while True:
555          result[k] = getpass.getpass("Enter password for %s key> "
556                                      % (k,)).strip()
557          if result[k]: break
558    return result
559
560  def UpdateAndReadFile(self, current):
561    if not self.editor or not self.pwfile:
562      return self.PromptResult(current)
563
564    f = open(self.pwfile, "w")
565    os.chmod(self.pwfile, 0600)
566    f.write("# Enter key passwords between the [[[ ]]] brackets.\n")
567    f.write("# (Additional spaces are harmless.)\n\n")
568
569    first_line = None
570    sorted = [(not v, k, v) for (k, v) in current.iteritems()]
571    sorted.sort()
572    for i, (_, k, v) in enumerate(sorted):
573      f.write("[[[  %s  ]]] %s\n" % (v, k))
574      if not v and first_line is None:
575        # position cursor on first line with no password.
576        first_line = i + 4
577    f.close()
578
579    p = Run([self.editor, "+%d" % (first_line,), self.pwfile])
580    _, _ = p.communicate()
581
582    return self.ReadFile()
583
584  def ReadFile(self):
585    result = {}
586    if self.pwfile is None: return result
587    try:
588      f = open(self.pwfile, "r")
589      for line in f:
590        line = line.strip()
591        if not line or line[0] == '#': continue
592        m = re.match(r"^\[\[\[\s*(.*?)\s*\]\]\]\s*(\S+)$", line)
593        if not m:
594          print "failed to parse password file: ", line
595        else:
596          result[m.group(2)] = m.group(1)
597      f.close()
598    except IOError, e:
599      if e.errno != errno.ENOENT:
600        print "error reading password file: ", str(e)
601    return result
602
603
604def ZipWriteStr(zip, filename, data, perms=0644):
605  # use a fixed timestamp so the output is repeatable.
606  zinfo = zipfile.ZipInfo(filename=filename,
607                          date_time=(2009, 1, 1, 0, 0, 0))
608  zinfo.compress_type = zip.compression
609  zinfo.external_attr = perms << 16
610  zip.writestr(zinfo, data)
611
612
613class DeviceSpecificParams(object):
614  module = None
615  def __init__(self, **kwargs):
616    """Keyword arguments to the constructor become attributes of this
617    object, which is passed to all functions in the device-specific
618    module."""
619    for k, v in kwargs.iteritems():
620      setattr(self, k, v)
621    self.extras = OPTIONS.extras
622
623    if self.module is None:
624      path = OPTIONS.device_specific
625      if not path: return
626      try:
627        if os.path.isdir(path):
628          info = imp.find_module("releasetools", [path])
629        else:
630          d, f = os.path.split(path)
631          b, x = os.path.splitext(f)
632          if x == ".py":
633            f = b
634          info = imp.find_module(f, [d])
635        self.module = imp.load_module("device_specific", *info)
636      except ImportError:
637        print "unable to load device-specific module; assuming none"
638
639  def _DoCall(self, function_name, *args, **kwargs):
640    """Call the named function in the device-specific module, passing
641    the given args and kwargs.  The first argument to the call will be
642    the DeviceSpecific object itself.  If there is no module, or the
643    module does not define the function, return the value of the
644    'default' kwarg (which itself defaults to None)."""
645    if self.module is None or not hasattr(self.module, function_name):
646      return kwargs.get("default", None)
647    return getattr(self.module, function_name)(*((self,) + args), **kwargs)
648
649  def FullOTA_Assertions(self):
650    """Called after emitting the block of assertions at the top of a
651    full OTA package.  Implementations can add whatever additional
652    assertions they like."""
653    return self._DoCall("FullOTA_Assertions")
654
655  def FullOTA_InstallEnd(self):
656    """Called at the end of full OTA installation; typically this is
657    used to install the image for the device's baseband processor."""
658    return self._DoCall("FullOTA_InstallEnd")
659
660  def IncrementalOTA_Assertions(self):
661    """Called after emitting the block of assertions at the top of an
662    incremental OTA package.  Implementations can add whatever
663    additional assertions they like."""
664    return self._DoCall("IncrementalOTA_Assertions")
665
666  def IncrementalOTA_VerifyEnd(self):
667    """Called at the end of the verification phase of incremental OTA
668    installation; additional checks can be placed here to abort the
669    script before any changes are made."""
670    return self._DoCall("IncrementalOTA_VerifyEnd")
671
672  def IncrementalOTA_InstallEnd(self):
673    """Called at the end of incremental OTA installation; typically
674    this is used to install the image for the device's baseband
675    processor."""
676    return self._DoCall("IncrementalOTA_InstallEnd")
677
678class File(object):
679  def __init__(self, name, data):
680    self.name = name
681    self.data = data
682    self.size = len(data)
683    self.sha1 = sha1(data).hexdigest()
684
685  @classmethod
686  def FromLocalFile(cls, name, diskname):
687    f = open(diskname, "rb")
688    data = f.read()
689    f.close()
690    return File(name, data)
691
692  def WriteToTemp(self):
693    t = tempfile.NamedTemporaryFile()
694    t.write(self.data)
695    t.flush()
696    return t
697
698  def AddToZip(self, z):
699    ZipWriteStr(z, self.name, self.data)
700
701DIFF_PROGRAM_BY_EXT = {
702    ".gz" : "imgdiff",
703    ".zip" : ["imgdiff", "-z"],
704    ".jar" : ["imgdiff", "-z"],
705    ".apk" : ["imgdiff", "-z"],
706    ".img" : "imgdiff",
707    }
708
709class Difference(object):
710  def __init__(self, tf, sf):
711    self.tf = tf
712    self.sf = sf
713    self.patch = None
714
715  def ComputePatch(self):
716    """Compute the patch (as a string of data) needed to turn sf into
717    tf.  Returns the same tuple as GetPatch()."""
718
719    tf = self.tf
720    sf = self.sf
721
722    ext = os.path.splitext(tf.name)[1]
723    diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
724
725    ttemp = tf.WriteToTemp()
726    stemp = sf.WriteToTemp()
727
728    ext = os.path.splitext(tf.name)[1]
729
730    try:
731      ptemp = tempfile.NamedTemporaryFile()
732      if isinstance(diff_program, list):
733        cmd = copy.copy(diff_program)
734      else:
735        cmd = [diff_program]
736      cmd.append(stemp.name)
737      cmd.append(ttemp.name)
738      cmd.append(ptemp.name)
739      p = Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
740      _, err = p.communicate()
741      if err or p.returncode != 0:
742        print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
743        return None
744      diff = ptemp.read()
745    finally:
746      ptemp.close()
747      stemp.close()
748      ttemp.close()
749
750    self.patch = diff
751    return self.tf, self.sf, self.patch
752
753
754  def GetPatch(self):
755    """Return a tuple (target_file, source_file, patch_data).
756    patch_data may be None if ComputePatch hasn't been called, or if
757    computing the patch failed."""
758    return self.tf, self.sf, self.patch
759
760
761def ComputeDifferences(diffs):
762  """Call ComputePatch on all the Difference objects in 'diffs'."""
763  print len(diffs), "diffs to compute"
764
765  # Do the largest files first, to try and reduce the long-pole effect.
766  by_size = [(i.tf.size, i) for i in diffs]
767  by_size.sort(reverse=True)
768  by_size = [i[1] for i in by_size]
769
770  lock = threading.Lock()
771  diff_iter = iter(by_size)   # accessed under lock
772
773  def worker():
774    try:
775      lock.acquire()
776      for d in diff_iter:
777        lock.release()
778        start = time.time()
779        d.ComputePatch()
780        dur = time.time() - start
781        lock.acquire()
782
783        tf, sf, patch = d.GetPatch()
784        if sf.name == tf.name:
785          name = tf.name
786        else:
787          name = "%s (%s)" % (tf.name, sf.name)
788        if patch is None:
789          print "patching failed!                                  %s" % (name,)
790        else:
791          print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
792              dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
793      lock.release()
794    except Exception, e:
795      print e
796      raise
797
798  # start worker threads; wait for them all to finish.
799  threads = [threading.Thread(target=worker)
800             for i in range(OPTIONS.worker_threads)]
801  for th in threads:
802    th.start()
803  while threads:
804    threads.pop().join()
805
806
807# map recovery.fstab's fs_types to mount/format "partition types"
808PARTITION_TYPES = { "yaffs2": "MTD", "mtd": "MTD",
809                    "ext4": "EMMC", "emmc": "EMMC" }
810
811def GetTypeAndDevice(mount_point, info):
812  fstab = info["fstab"]
813  if fstab:
814    return PARTITION_TYPES[fstab[mount_point].fs_type], fstab[mount_point].device
815  else:
816    devices = {"/boot": "boot",
817               "/recovery": "recovery",
818               "/radio": "radio",
819               "/data": "userdata",
820               "/cache": "cache"}
821    return info["partition_type"], info.get("partition_path", "") + devices[mount_point]
822