ota_from_target_files.py revision 283e2a1e1bae4e21824969a15da6420204633ddd 
1#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build.  An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage:  ota_from_target_files [flags] input_target_files output_ota_package
23
24  -b  (--board_config)  <file>
25      Deprecated.
26
27  -k  (--package_key)  <key>
28      Key to use to sign the package (default is
29      "build/target/product/security/testkey").
30
31  -i  (--incremental_from)  <file>
32      Generate an incremental OTA using the given target-files zip as
33      the starting build.
34
35  -w  (--wipe_user_data)
36      Generate an OTA package that will wipe the user data partition
37      when installed.
38
39  -n  (--no_prereq)
40      Omit the timestamp prereq check normally included at the top of
41      the build scripts (used for developer OTA packages which
42      legitimately need to go back and forth).
43
44  -e  (--extra_script)  <file>
45      Insert the contents of file at the end of the update script.
46
47  -m  (--script_mode)  <mode>
48      Specify 'amend' or 'edify' scripts, or 'auto' to pick
49      automatically (this is the default).
50
51"""
52
53import sys
54
55if sys.hexversion < 0x02040000:
56  print >> sys.stderr, "Python 2.4 or newer is required."
57  sys.exit(1)
58
59import copy
60import errno
61import os
62import re
63import sha
64import subprocess
65import tempfile
66import threading
67import time
68import zipfile
69
70import common
71import amend_generator
72import edify_generator
73import both_generator
74
75OPTIONS = common.OPTIONS
76OPTIONS.package_key = "build/target/product/security/testkey"
77OPTIONS.incremental_source = None
78OPTIONS.require_verbatim = set()
79OPTIONS.prohibit_verbatim = set(("system/build.prop",))
80OPTIONS.patch_threshold = 0.95
81OPTIONS.wipe_user_data = False
82OPTIONS.omit_prereq = False
83OPTIONS.extra_script = None
84OPTIONS.script_mode = 'auto'
85OPTIONS.worker_threads = 3
86
87def MostPopularKey(d, default):
88  """Given a dict, return the key corresponding to the largest
89  value.  Returns 'default' if the dict is empty."""
90  x = [(v, k) for (k, v) in d.iteritems()]
91  if not x: return default
92  x.sort()
93  return x[-1][1]
94
95
96def IsSymlink(info):
97  """Return true if the zipfile.ZipInfo object passed in represents a
98  symlink."""
99  return (info.external_attr >> 16) == 0120777
100
101
102
103class Item:
104  """Items represent the metadata (user, group, mode) of files and
105  directories in the system image."""
106  ITEMS = {}
107  def __init__(self, name, dir=False):
108    self.name = name
109    self.uid = None
110    self.gid = None
111    self.mode = None
112    self.dir = dir
113
114    if name:
115      self.parent = Item.Get(os.path.dirname(name), dir=True)
116      self.parent.children.append(self)
117    else:
118      self.parent = None
119    if dir:
120      self.children = []
121
122  def Dump(self, indent=0):
123    if self.uid is not None:
124      print "%s%s %d %d %o" % ("  "*indent, self.name, self.uid, self.gid, self.mode)
125    else:
126      print "%s%s %s %s %s" % ("  "*indent, self.name, self.uid, self.gid, self.mode)
127    if self.dir:
128      print "%s%s" % ("  "*indent, self.descendants)
129      print "%s%s" % ("  "*indent, self.best_subtree)
130      for i in self.children:
131        i.Dump(indent=indent+1)
132
133  @classmethod
134  def Get(cls, name, dir=False):
135    if name not in cls.ITEMS:
136      cls.ITEMS[name] = Item(name, dir=dir)
137    return cls.ITEMS[name]
138
139  @classmethod
140  def GetMetadata(cls, input_zip):
141
142    try:
143      # See if the target_files contains a record of what the uid,
144      # gid, and mode is supposed to be.
145      output = input_zip.read("META/filesystem_config.txt")
146    except KeyError:
147      # Run the external 'fs_config' program to determine the desired
148      # uid, gid, and mode for every Item object.  Note this uses the
149      # one in the client now, which might not be the same as the one
150      # used when this target_files was built.
151      p = common.Run(["fs_config"], stdin=subprocess.PIPE,
152                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
153      suffix = { False: "", True: "/" }
154      input = "".join(["%s%s\n" % (i.name, suffix[i.dir])
155                       for i in cls.ITEMS.itervalues() if i.name])
156      output2, error = p.communicate(input)
157      assert not error
158
159    for line in output.split("\n"):
160      if not line: continue
161      name, uid, gid, mode = line.split()
162      i = cls.ITEMS.get(name, None)
163      if i is not None:
164        i.uid = int(uid)
165        i.gid = int(gid)
166        i.mode = int(mode, 8)
167        if i.dir:
168          i.children.sort(key=lambda i: i.name)
169
170    # set metadata for the files generated by this script.
171    i = cls.ITEMS.get("system/recovery-from-boot.p", None)
172    if i: i.uid, i.gid, i.mode = 0, 0, 0644
173    i = cls.ITEMS.get("system/etc/install-recovery.sh", None)
174    if i: i.uid, i.gid, i.mode = 0, 0, 0544
175
176  def CountChildMetadata(self):
177    """Count up the (uid, gid, mode) tuples for all children and
178    determine the best strategy for using set_perm_recursive and
179    set_perm to correctly chown/chmod all the files to their desired
180    values.  Recursively calls itself for all descendants.
181
182    Returns a dict of {(uid, gid, dmode, fmode): count} counting up
183    all descendants of this node.  (dmode or fmode may be None.)  Also
184    sets the best_subtree of each directory Item to the (uid, gid,
185    dmode, fmode) tuple that will match the most descendants of that
186    Item.
187    """
188
189    assert self.dir
190    d = self.descendants = {(self.uid, self.gid, self.mode, None): 1}
191    for i in self.children:
192      if i.dir:
193        for k, v in i.CountChildMetadata().iteritems():
194          d[k] = d.get(k, 0) + v
195      else:
196        k = (i.uid, i.gid, None, i.mode)
197        d[k] = d.get(k, 0) + 1
198
199    # Find the (uid, gid, dmode, fmode) tuple that matches the most
200    # descendants.
201
202    # First, find the (uid, gid) pair that matches the most
203    # descendants.
204    ug = {}
205    for (uid, gid, _, _), count in d.iteritems():
206      ug[(uid, gid)] = ug.get((uid, gid), 0) + count
207    ug = MostPopularKey(ug, (0, 0))
208
209    # Now find the dmode and fmode that match the most descendants
210    # with that (uid, gid), and choose those.
211    best_dmode = (0, 0755)
212    best_fmode = (0, 0644)
213    for k, count in d.iteritems():
214      if k[:2] != ug: continue
215      if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
216      if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
217    self.best_subtree = ug + (best_dmode[1], best_fmode[1])
218
219    return d
220
221  def SetPermissions(self, script):
222    """Append set_perm/set_perm_recursive commands to 'script' to
223    set all permissions, users, and groups for the tree of files
224    rooted at 'self'."""
225
226    self.CountChildMetadata()
227
228    def recurse(item, current):
229      # current is the (uid, gid, dmode, fmode) tuple that the current
230      # item (and all its children) have already been set to.  We only
231      # need to issue set_perm/set_perm_recursive commands if we're
232      # supposed to be something different.
233      if item.dir:
234        if current != item.best_subtree:
235          script.SetPermissionsRecursive("/"+item.name, *item.best_subtree)
236          current = item.best_subtree
237
238        if item.uid != current[0] or item.gid != current[1] or \
239           item.mode != current[2]:
240          script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
241
242        for i in item.children:
243          recurse(i, current)
244      else:
245        if item.uid != current[0] or item.gid != current[1] or \
246               item.mode != current[3]:
247          script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
248
249    recurse(self, (-1, -1, -1, -1))
250
251
252def CopySystemFiles(input_zip, output_zip=None,
253                    substitute=None):
254  """Copies files underneath system/ in the input zip to the output
255  zip.  Populates the Item class with their metadata, and returns a
256  list of symlinks.  output_zip may be None, in which case the copy is
257  skipped (but the other side effects still happen).  substitute is an
258  optional dict of {output filename: contents} to be output instead of
259  certain input files.
260  """
261
262  symlinks = []
263
264  for info in input_zip.infolist():
265    if info.filename.startswith("SYSTEM/"):
266      basefilename = info.filename[7:]
267      if IsSymlink(info):
268        symlinks.append((input_zip.read(info.filename),
269                         "/system/" + basefilename))
270      else:
271        info2 = copy.copy(info)
272        fn = info2.filename = "system/" + basefilename
273        if substitute and fn in substitute and substitute[fn] is None:
274          continue
275        if output_zip is not None:
276          if substitute and fn in substitute:
277            data = substitute[fn]
278          else:
279            data = input_zip.read(info.filename)
280          output_zip.writestr(info2, data)
281        if fn.endswith("/"):
282          Item.Get(fn[:-1], dir=True)
283        else:
284          Item.Get(fn, dir=False)
285
286  symlinks.sort()
287  return symlinks
288
289
290def SignOutput(temp_zip_name, output_zip_name):
291  key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
292  pw = key_passwords[OPTIONS.package_key]
293
294  common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
295                  whole_file=True)
296
297
298def AppendAssertions(script, input_zip):
299  device = GetBuildProp("ro.product.device", input_zip)
300  script.AssertDevice(device)
301
302
303def MakeRecoveryPatch(output_zip, recovery_img, boot_img):
304  """Generate a binary patch that creates the recovery image starting
305  with the boot image.  (Most of the space in these images is just the
306  kernel, which is identical for the two, so the resulting patch
307  should be efficient.)  Add it to the output zip, along with a shell
308  script that is run from init.rc on first boot to actually do the
309  patching and install the new recovery image.
310
311  recovery_img and boot_img should be File objects for the
312  corresponding images.
313
314  Returns an Item for the shell script, which must be made
315  executable.
316  """
317
318  d = Difference(recovery_img, boot_img)
319  _, _, patch = d.ComputePatch()
320  common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch)
321  Item.Get("system/recovery-from-boot.p", dir=False)
322
323  # Images with different content will have a different first page, so
324  # we check to see if this recovery has already been installed by
325  # testing just the first 2k.
326  HEADER_SIZE = 2048
327  header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest()
328  sh = """#!/system/bin/sh
329if ! applypatch -c MTD:recovery:%(header_size)d:%(header_sha1)s; then
330  log -t recovery "Installing new recovery image"
331  applypatch MTD:boot:%(boot_size)d:%(boot_sha1)s MTD:recovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p
332else
333  log -t recovery "Recovery image already installed"
334fi
335""" % { 'boot_size': boot_img.size,
336        'boot_sha1': boot_img.sha1,
337        'header_size': HEADER_SIZE,
338        'header_sha1': header_sha1,
339        'recovery_size': recovery_img.size,
340        'recovery_sha1': recovery_img.sha1 }
341  common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh)
342  return Item.Get("system/etc/install-recovery.sh", dir=False)
343
344
345def WriteFullOTAPackage(input_zip, output_zip):
346  if OPTIONS.script_mode == "auto":
347    script = both_generator.BothGenerator(2)
348  elif OPTIONS.script_mode == "amend":
349    script = amend_generator.AmendGenerator()
350  else:
351    # TODO: how to determine this?  We don't know what version it will
352    # be installed on top of.  For now, we expect the API just won't
353    # change very often.
354    script = edify_generator.EdifyGenerator(2)
355
356  device_specific = common.DeviceSpecificParams(
357      input_zip=input_zip,
358      input_version=GetRecoveryAPIVersion(input_zip),
359      output_zip=output_zip,
360      script=script,
361      input_tmp=OPTIONS.input_tmp)
362
363  if not OPTIONS.omit_prereq:
364    ts = GetBuildProp("ro.build.date.utc", input_zip)
365    script.AssertOlderBuild(ts)
366
367  AppendAssertions(script, input_zip)
368  device_specific.FullOTA_Assertions()
369
370  script.ShowProgress(0.5, 0)
371
372  if OPTIONS.wipe_user_data:
373    script.FormatPartition("userdata")
374
375  script.FormatPartition("system")
376  script.Mount("MTD", "system", "/system")
377  script.UnpackPackageDir("recovery", "/system")
378  script.UnpackPackageDir("system", "/system")
379
380  symlinks = CopySystemFiles(input_zip, output_zip)
381  script.MakeSymlinks(symlinks)
382
383  boot_img = File("boot.img", common.BuildBootableImage(
384      os.path.join(OPTIONS.input_tmp, "BOOT")))
385  recovery_img = File("recovery.img", common.BuildBootableImage(
386      os.path.join(OPTIONS.input_tmp, "RECOVERY")))
387  MakeRecoveryPatch(output_zip, recovery_img, boot_img)
388
389  Item.GetMetadata(input_zip)
390  Item.Get("system").SetPermissions(script)
391
392  common.CheckSize(boot_img.data, "boot.img")
393  common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
394  script.ShowProgress(0.2, 0)
395
396  script.ShowProgress(0.2, 10)
397  script.WriteRawImage("boot", "boot.img")
398
399  script.ShowProgress(0.1, 0)
400  device_specific.FullOTA_InstallEnd()
401
402  if OPTIONS.extra_script is not None:
403    script.AppendExtra(OPTIONS.extra_script)
404
405  script.UnmountAll()
406  script.AddToZip(input_zip, output_zip)
407
408
409class File(object):
410  def __init__(self, name, data):
411    self.name = name
412    self.data = data
413    self.size = len(data)
414    self.sha1 = sha.sha(data).hexdigest()
415
416  def WriteToTemp(self):
417    t = tempfile.NamedTemporaryFile()
418    t.write(self.data)
419    t.flush()
420    return t
421
422  def AddToZip(self, z):
423    common.ZipWriteStr(z, self.name, self.data)
424
425
426def LoadSystemFiles(z):
427  """Load all the files from SYSTEM/... in a given target-files
428  ZipFile, and return a dict of {filename: File object}."""
429  out = {}
430  for info in z.infolist():
431    if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
432      fn = "system/" + info.filename[7:]
433      data = z.read(info.filename)
434      out[fn] = File(fn, data)
435  return out
436
437
438DIFF_PROGRAM_BY_EXT = {
439    ".gz" : "imgdiff",
440    ".zip" : ["imgdiff", "-z"],
441    ".jar" : ["imgdiff", "-z"],
442    ".apk" : ["imgdiff", "-z"],
443    ".img" : "imgdiff",
444    }
445
446
447class Difference(object):
448  def __init__(self, tf, sf):
449    self.tf = tf
450    self.sf = sf
451    self.patch = None
452
453  def ComputePatch(self):
454    """Compute the patch (as a string of data) needed to turn sf into
455    tf.  Returns the same tuple as GetPatch()."""
456
457    tf = self.tf
458    sf = self.sf
459
460    ext = os.path.splitext(tf.name)[1]
461    diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
462
463    ttemp = tf.WriteToTemp()
464    stemp = sf.WriteToTemp()
465
466    ext = os.path.splitext(tf.name)[1]
467
468    try:
469      ptemp = tempfile.NamedTemporaryFile()
470      if isinstance(diff_program, list):
471        cmd = copy.copy(diff_program)
472      else:
473        cmd = [diff_program]
474      cmd.append(stemp.name)
475      cmd.append(ttemp.name)
476      cmd.append(ptemp.name)
477      p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
478      _, err = p.communicate()
479      if err or p.returncode != 0:
480        print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
481        return None
482      diff = ptemp.read()
483    finally:
484      ptemp.close()
485      stemp.close()
486      ttemp.close()
487
488    self.patch = diff
489    return self.tf, self.sf, self.patch
490
491
492  def GetPatch(self):
493    """Return a tuple (target_file, source_file, patch_data).
494    patch_data may be None if ComputePatch hasn't been called, or if
495    computing the patch failed."""
496    return self.tf, self.sf, self.patch
497
498
499def ComputeDifferences(diffs):
500  """Call ComputePatch on all the Difference objects in 'diffs'."""
501  print len(diffs), "diffs to compute"
502
503  # Do the largest files first, to try and reduce the long-pole effect.
504  by_size = [(i.tf.size, i) for i in diffs]
505  by_size.sort(reverse=True)
506  by_size = [i[1] for i in by_size]
507
508  lock = threading.Lock()
509  diff_iter = iter(by_size)   # accessed under lock
510
511  def worker():
512    try:
513      lock.acquire()
514      for d in diff_iter:
515        lock.release()
516        start = time.time()
517        d.ComputePatch()
518        dur = time.time() - start
519        lock.acquire()
520
521        tf, sf, patch = d.GetPatch()
522        if sf.name == tf.name:
523          name = tf.name
524        else:
525          name = "%s (%s)" % (tf.name, sf.name)
526        if patch is None:
527          print "patching failed!                                  %s" % (name,)
528        else:
529          print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
530              dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
531      lock.release()
532    except Exception, e:
533      print e
534      raise
535
536  # start worker threads; wait for them all to finish.
537  threads = [threading.Thread(target=worker)
538             for i in range(OPTIONS.worker_threads)]
539  for th in threads:
540    th.start()
541  while threads:
542    threads.pop().join()
543
544
545def GetBuildProp(property, z):
546  """Return the fingerprint of the build of a given target-files
547  ZipFile object."""
548  bp = z.read("SYSTEM/build.prop")
549  if not property:
550    return bp
551  m = re.search(re.escape(property) + r"=(.*)\n", bp)
552  if not m:
553    raise common.ExternalError("couldn't find %s in build.prop" % (property,))
554  return m.group(1).strip()
555
556
557def GetRecoveryAPIVersion(zip):
558  """Returns the version of the recovery API.  Version 0 is the older
559  amend code (no separate binary)."""
560  try:
561    version = zip.read("META/recovery-api-version.txt")
562    return int(version)
563  except KeyError:
564    try:
565      # version one didn't have the recovery-api-version.txt file, but
566      # it did include an updater binary.
567      zip.getinfo("OTA/bin/updater")
568      return 1
569    except KeyError:
570      return 0
571
572
573def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
574  source_version = GetRecoveryAPIVersion(source_zip)
575  target_version = GetRecoveryAPIVersion(target_zip)
576
577  if OPTIONS.script_mode == 'amend':
578    script = amend_generator.AmendGenerator()
579  elif OPTIONS.script_mode == 'edify':
580    if source_version == 0:
581      print ("WARNING: generating edify script for a source that "
582             "can't install it.")
583    script = edify_generator.EdifyGenerator(source_version)
584  elif OPTIONS.script_mode == 'auto':
585    if source_version > 0:
586      script = edify_generator.EdifyGenerator(source_version)
587    else:
588      script = amend_generator.AmendGenerator()
589  else:
590    raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,))
591
592  device_specific = common.DeviceSpecificParams(
593      source_zip=source_zip,
594      source_version=source_version,
595      target_zip=target_zip,
596      target_version=target_version,
597      output_zip=output_zip,
598      script=script)
599
600  print "Loading target..."
601  target_data = LoadSystemFiles(target_zip)
602  print "Loading source..."
603  source_data = LoadSystemFiles(source_zip)
604
605  verbatim_targets = []
606  patch_list = []
607  diffs = []
608  largest_source_size = 0
609  for fn in sorted(target_data.keys()):
610    tf = target_data[fn]
611    assert fn == tf.name
612    sf = source_data.get(fn, None)
613
614    if sf is None or fn in OPTIONS.require_verbatim:
615      # This file should be included verbatim
616      if fn in OPTIONS.prohibit_verbatim:
617        raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,))
618      print "send", fn, "verbatim"
619      tf.AddToZip(output_zip)
620      verbatim_targets.append((fn, tf.size))
621    elif tf.sha1 != sf.sha1:
622      # File is different; consider sending as a patch
623      diffs.append(Difference(tf, sf))
624    else:
625      # Target file identical to source.
626      pass
627
628  ComputeDifferences(diffs)
629
630  for diff in diffs:
631    tf, sf, d = diff.GetPatch()
632    if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
633      # patch is almost as big as the file; don't bother patching
634      tf.AddToZip(output_zip)
635      verbatim_targets.append((tf.name, tf.size))
636    else:
637      common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d)
638      patch_list.append((tf.name, tf, sf, tf.size, sha.sha(d).hexdigest()))
639      largest_source_size = max(largest_source_size, sf.size)
640
641  source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
642  target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
643
644  script.Mount("MTD", "system", "/system")
645  script.AssertSomeFingerprint(source_fp, target_fp)
646
647  source_boot = File("/tmp/boot.img",
648                     common.BuildBootableImage(
649      os.path.join(OPTIONS.source_tmp, "BOOT")))
650  target_boot = File("/tmp/boot.img",
651                     common.BuildBootableImage(
652      os.path.join(OPTIONS.target_tmp, "BOOT")))
653  updating_boot = (source_boot.data != target_boot.data)
654
655  source_recovery = File("system/recovery.img",
656                         common.BuildBootableImage(
657      os.path.join(OPTIONS.source_tmp, "RECOVERY")))
658  target_recovery = File("system/recovery.img",
659                         common.BuildBootableImage(
660      os.path.join(OPTIONS.target_tmp, "RECOVERY")))
661  updating_recovery = (source_recovery.data != target_recovery.data)
662
663  # Here's how we divide up the progress bar:
664  #  0.1 for verifying the start state (PatchCheck calls)
665  #  0.8 for applying patches (ApplyPatch calls)
666  #  0.1 for unpacking verbatim files, symlinking, and doing the
667  #      device-specific commands.
668
669  AppendAssertions(script, target_zip)
670  device_specific.IncrementalOTA_Assertions()
671
672  script.Print("Verifying current system...")
673
674  script.ShowProgress(0.1, 0)
675  total_verify_size = float(sum([i[2].size for i in patch_list]) + 1)
676  if updating_boot:
677    total_verify_size += source_boot.size
678  so_far = 0
679
680  for fn, tf, sf, size, patch_sha in patch_list:
681    script.PatchCheck("/"+fn, tf.sha1, sf.sha1)
682    so_far += sf.size
683    script.SetProgress(so_far / total_verify_size)
684
685  if updating_boot:
686    d = Difference(target_boot, source_boot)
687    _, _, d = d.ComputePatch()
688    print "boot      target: %d  source: %d  diff: %d" % (
689        target_boot.size, source_boot.size, len(d))
690
691    common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
692
693    script.PatchCheck("MTD:boot:%d:%s:%d:%s" %
694                      (source_boot.size, source_boot.sha1,
695                       target_boot.size, target_boot.sha1))
696    so_far += source_boot.size
697    script.SetProgress(so_far / total_verify_size)
698
699  if patch_list or updating_recovery or updating_boot:
700    script.CacheFreeSpaceCheck(largest_source_size)
701
702  device_specific.IncrementalOTA_VerifyEnd()
703
704  script.Comment("---- start making changes here ----")
705
706  if OPTIONS.wipe_user_data:
707    script.Print("Erasing user data...")
708    script.FormatPartition("userdata")
709
710  script.Print("Removing unneeded files...")
711  script.DeleteFiles(["/"+i[0] for i in verbatim_targets] +
712                     ["/"+i for i in sorted(source_data)
713                            if i not in target_data] +
714                     ["/system/recovery.img"])
715
716  script.ShowProgress(0.8, 0)
717  total_patch_size = float(sum([i[1].size for i in patch_list]) + 1)
718  if updating_boot:
719    total_patch_size += target_boot.size
720  so_far = 0
721
722  script.Print("Patching system files...")
723  for fn, tf, sf, size, _ in patch_list:
724    script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1, sf.sha1, "patch/"+fn+".p")
725    so_far += tf.size
726    script.SetProgress(so_far / total_patch_size)
727
728  if updating_boot:
729    # Produce the boot image by applying a patch to the current
730    # contents of the boot partition, and write it back to the
731    # partition.
732    script.Print("Patching boot image...")
733    script.ApplyPatch("MTD:boot:%d:%s:%d:%s"
734                      % (source_boot.size, source_boot.sha1,
735                         target_boot.size, target_boot.sha1),
736                      "-",
737                      target_boot.size, target_boot.sha1,
738                      source_boot.sha1, "patch/boot.img.p")
739    so_far += target_boot.size
740    script.SetProgress(so_far / total_patch_size)
741    print "boot image changed; including."
742  else:
743    print "boot image unchanged; skipping."
744
745  if updating_recovery:
746    # Is it better to generate recovery as a patch from the current
747    # boot image, or from the previous recovery image?  For large
748    # updates with significant kernel changes, probably the former.
749    # For small updates where the kernel hasn't changed, almost
750    # certainly the latter.  We pick the first option.  Future
751    # complicated schemes may let us effectively use both.
752    #
753    # A wacky possibility: as long as there is room in the boot
754    # partition, include the binaries and image files from recovery in
755    # the boot image (though not in the ramdisk) so they can be used
756    # as fodder for constructing the recovery image.
757    MakeRecoveryPatch(output_zip, target_recovery, target_boot)
758    script.DeleteFiles(["/system/recovery-from-boot.p",
759                        "/system/etc/install-recovery.sh"])
760    print "recovery image changed; including as patch from boot."
761  else:
762    print "recovery image unchanged; skipping."
763
764  script.ShowProgress(0.1, 10)
765
766  target_symlinks = CopySystemFiles(target_zip, None)
767
768  target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
769  temp_script = script.MakeTemporary()
770  Item.GetMetadata(target_zip)
771  Item.Get("system").SetPermissions(temp_script)
772
773  # Note that this call will mess up the tree of Items, so make sure
774  # we're done with it.
775  source_symlinks = CopySystemFiles(source_zip, None)
776  source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
777
778  # Delete all the symlinks in source that aren't in target.  This
779  # needs to happen before verbatim files are unpacked, in case a
780  # symlink in the source is replaced by a real file in the target.
781  to_delete = []
782  for dest, link in source_symlinks:
783    if link not in target_symlinks_d:
784      to_delete.append(link)
785  script.DeleteFiles(to_delete)
786
787  if verbatim_targets:
788    script.Print("Unpacking new files...")
789    script.UnpackPackageDir("system", "/system")
790
791  if updating_recovery:
792    script.Print("Unpacking new recovery...")
793    script.UnpackPackageDir("recovery", "/system")
794
795  script.Print("Symlinks and permissions...")
796
797  # Create all the symlinks that don't already exist, or point to
798  # somewhere different than what we want.  Delete each symlink before
799  # creating it, since the 'symlink' command won't overwrite.
800  to_create = []
801  for dest, link in target_symlinks:
802    if link in source_symlinks_d:
803      if dest != source_symlinks_d[link]:
804        to_create.append((dest, link))
805    else:
806      to_create.append((dest, link))
807  script.DeleteFiles([i[1] for i in to_create])
808  script.MakeSymlinks(to_create)
809
810  # Now that the symlinks are created, we can set all the
811  # permissions.
812  script.AppendScript(temp_script)
813
814  # Do device-specific installation (eg, write radio image).
815  device_specific.IncrementalOTA_InstallEnd()
816
817  if OPTIONS.extra_script is not None:
818    scirpt.AppendExtra(OPTIONS.extra_script)
819
820  script.AddToZip(target_zip, output_zip)
821
822
823def main(argv):
824
825  def option_handler(o, a):
826    if o in ("-b", "--board_config"):
827      pass   # deprecated
828    elif o in ("-k", "--package_key"):
829      OPTIONS.package_key = a
830    elif o in ("-i", "--incremental_from"):
831      OPTIONS.incremental_source = a
832    elif o in ("-w", "--wipe_user_data"):
833      OPTIONS.wipe_user_data = True
834    elif o in ("-n", "--no_prereq"):
835      OPTIONS.omit_prereq = True
836    elif o in ("-e", "--extra_script"):
837      OPTIONS.extra_script = a
838    elif o in ("-m", "--script_mode"):
839      OPTIONS.script_mode = a
840    elif o in ("--worker_threads"):
841      OPTIONS.worker_threads = int(a)
842    else:
843      return False
844    return True
845
846  args = common.ParseOptions(argv, __doc__,
847                             extra_opts="b:k:i:d:wne:m:",
848                             extra_long_opts=["board_config=",
849                                              "package_key=",
850                                              "incremental_from=",
851                                              "wipe_user_data",
852                                              "no_prereq",
853                                              "extra_script=",
854                                              "script_mode=",
855                                              "worker_threads="],
856                             extra_option_handler=option_handler)
857
858  if len(args) != 2:
859    common.Usage(__doc__)
860    sys.exit(1)
861
862  if OPTIONS.script_mode not in ("amend", "edify", "auto"):
863    raise ValueError('unknown script mode "%s"' % (OPTIONS.script_mode,))
864
865  if OPTIONS.extra_script is not None:
866    OPTIONS.extra_script = open(OPTIONS.extra_script).read()
867
868  print "unzipping target target-files..."
869  OPTIONS.input_tmp = common.UnzipTemp(args[0])
870
871  if OPTIONS.device_specific is None:
872    # look for the device-specific tools extension location in the input
873    try:
874      f = open(os.path.join(OPTIONS.input_tmp, "META", "tool-extensions.txt"))
875      ds = f.read().strip()
876      f.close()
877      if ds:
878        ds = os.path.normpath(ds)
879        print "using device-specific extensions in", ds
880        OPTIONS.device_specific = ds
881    except IOError, e:
882      if e.errno == errno.ENOENT:
883        # nothing specified in the file
884        pass
885      else:
886        raise
887
888  common.LoadMaxSizes()
889  if not OPTIONS.max_image_size:
890    print
891    print "  WARNING:  Failed to load max image sizes; will not enforce"
892    print "  image size limits."
893    print
894
895  OPTIONS.target_tmp = OPTIONS.input_tmp
896  input_zip = zipfile.ZipFile(args[0], "r")
897  if OPTIONS.package_key:
898    temp_zip_file = tempfile.NamedTemporaryFile()
899    output_zip = zipfile.ZipFile(temp_zip_file, "w",
900                                 compression=zipfile.ZIP_DEFLATED)
901  else:
902    output_zip = zipfile.ZipFile(args[1], "w",
903                                 compression=zipfile.ZIP_DEFLATED)
904
905  if OPTIONS.incremental_source is None:
906    WriteFullOTAPackage(input_zip, output_zip)
907  else:
908    print "unzipping source target-files..."
909    OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source)
910    source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r")
911    WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)
912
913  output_zip.close()
914  if OPTIONS.package_key:
915    SignOutput(temp_zip_file.name, args[1])
916    temp_zip_file.close()
917
918  common.Cleanup()
919
920  print "done."
921
922
923if __name__ == '__main__':
924  try:
925    main(sys.argv[1:])
926  except common.ExternalError, e:
927    print
928    print "   ERROR: %s" % (e,)
929    print
930    sys.exit(1)
931