ota_from_target_files.py revision dafb04275588fff8248b6a5360ca047cdffd14a5 
1#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build.  An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage:  ota_from_target_files [flags] input_target_files output_ota_package
23
24  -b  (--board_config)  <file>
25      Deprecated.
26
27  -k  (--package_key)  <key>
28      Key to use to sign the package (default is
29      "build/target/product/security/testkey").
30
31  -i  (--incremental_from)  <file>
32      Generate an incremental OTA using the given target-files zip as
33      the starting build.
34
35  -w  (--wipe_user_data)
36      Generate an OTA package that will wipe the user data partition
37      when installed.
38
39  -n  (--no_prereq)
40      Omit the timestamp prereq check normally included at the top of
41      the build scripts (used for developer OTA packages which
42      legitimately need to go back and forth).
43
44  -e  (--extra_script)  <file>
45      Insert the contents of file at the end of the update script.
46
47  -a  (--aslr_mode)  <on|off>
48      Specify whether to turn on ASLR for the package (on by default).
49"""
50
51import sys
52
53if sys.hexversion < 0x02040000:
54  print >> sys.stderr, "Python 2.4 or newer is required."
55  sys.exit(1)
56
57import copy
58import errno
59import os
60import re
61import sha
62import subprocess
63import tempfile
64import threading
65import time
66import zipfile
67
68import common
69import edify_generator
70
71OPTIONS = common.OPTIONS
72OPTIONS.package_key = "build/target/product/security/testkey"
73OPTIONS.incremental_source = None
74OPTIONS.require_verbatim = set()
75OPTIONS.prohibit_verbatim = set(("system/build.prop",))
76OPTIONS.patch_threshold = 0.95
77OPTIONS.wipe_user_data = False
78OPTIONS.omit_prereq = False
79OPTIONS.extra_script = None
80OPTIONS.aslr_mode = True
81OPTIONS.worker_threads = 3
82
83def MostPopularKey(d, default):
84  """Given a dict, return the key corresponding to the largest
85  value.  Returns 'default' if the dict is empty."""
86  x = [(v, k) for (k, v) in d.iteritems()]
87  if not x: return default
88  x.sort()
89  return x[-1][1]
90
91
92def IsSymlink(info):
93  """Return true if the zipfile.ZipInfo object passed in represents a
94  symlink."""
95  return (info.external_attr >> 16) == 0120777
96
97def IsRegular(info):
98  """Return true if the zipfile.ZipInfo object passed in represents a
99  symlink."""
100  return (info.external_attr >> 28) == 010
101
102
103class Item:
104  """Items represent the metadata (user, group, mode) of files and
105  directories in the system image."""
106  ITEMS = {}
107  def __init__(self, name, dir=False):
108    self.name = name
109    self.uid = None
110    self.gid = None
111    self.mode = None
112    self.dir = dir
113
114    if name:
115      self.parent = Item.Get(os.path.dirname(name), dir=True)
116      self.parent.children.append(self)
117    else:
118      self.parent = None
119    if dir:
120      self.children = []
121
122  def Dump(self, indent=0):
123    if self.uid is not None:
124      print "%s%s %d %d %o" % ("  "*indent, self.name, self.uid, self.gid, self.mode)
125    else:
126      print "%s%s %s %s %s" % ("  "*indent, self.name, self.uid, self.gid, self.mode)
127    if self.dir:
128      print "%s%s" % ("  "*indent, self.descendants)
129      print "%s%s" % ("  "*indent, self.best_subtree)
130      for i in self.children:
131        i.Dump(indent=indent+1)
132
133  @classmethod
134  def Get(cls, name, dir=False):
135    if name not in cls.ITEMS:
136      cls.ITEMS[name] = Item(name, dir=dir)
137    return cls.ITEMS[name]
138
139  @classmethod
140  def GetMetadata(cls, input_zip):
141
142    try:
143      # See if the target_files contains a record of what the uid,
144      # gid, and mode is supposed to be.
145      output = input_zip.read("META/filesystem_config.txt")
146    except KeyError:
147      # Run the external 'fs_config' program to determine the desired
148      # uid, gid, and mode for every Item object.  Note this uses the
149      # one in the client now, which might not be the same as the one
150      # used when this target_files was built.
151      p = common.Run(["fs_config"], stdin=subprocess.PIPE,
152                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
153      suffix = { False: "", True: "/" }
154      input = "".join(["%s%s\n" % (i.name, suffix[i.dir])
155                       for i in cls.ITEMS.itervalues() if i.name])
156      output, error = p.communicate(input)
157      assert not error
158
159    for line in output.split("\n"):
160      if not line: continue
161      name, uid, gid, mode = line.split()
162      i = cls.ITEMS.get(name, None)
163      if i is not None:
164        i.uid = int(uid)
165        i.gid = int(gid)
166        i.mode = int(mode, 8)
167        if i.dir:
168          i.children.sort(key=lambda i: i.name)
169
170    # set metadata for the files generated by this script.
171    i = cls.ITEMS.get("system/recovery-from-boot.p", None)
172    if i: i.uid, i.gid, i.mode = 0, 0, 0644
173    i = cls.ITEMS.get("system/etc/install-recovery.sh", None)
174    if i: i.uid, i.gid, i.mode = 0, 0, 0544
175
176  def CountChildMetadata(self):
177    """Count up the (uid, gid, mode) tuples for all children and
178    determine the best strategy for using set_perm_recursive and
179    set_perm to correctly chown/chmod all the files to their desired
180    values.  Recursively calls itself for all descendants.
181
182    Returns a dict of {(uid, gid, dmode, fmode): count} counting up
183    all descendants of this node.  (dmode or fmode may be None.)  Also
184    sets the best_subtree of each directory Item to the (uid, gid,
185    dmode, fmode) tuple that will match the most descendants of that
186    Item.
187    """
188
189    assert self.dir
190    d = self.descendants = {(self.uid, self.gid, self.mode, None): 1}
191    for i in self.children:
192      if i.dir:
193        for k, v in i.CountChildMetadata().iteritems():
194          d[k] = d.get(k, 0) + v
195      else:
196        k = (i.uid, i.gid, None, i.mode)
197        d[k] = d.get(k, 0) + 1
198
199    # Find the (uid, gid, dmode, fmode) tuple that matches the most
200    # descendants.
201
202    # First, find the (uid, gid) pair that matches the most
203    # descendants.
204    ug = {}
205    for (uid, gid, _, _), count in d.iteritems():
206      ug[(uid, gid)] = ug.get((uid, gid), 0) + count
207    ug = MostPopularKey(ug, (0, 0))
208
209    # Now find the dmode and fmode that match the most descendants
210    # with that (uid, gid), and choose those.
211    best_dmode = (0, 0755)
212    best_fmode = (0, 0644)
213    for k, count in d.iteritems():
214      if k[:2] != ug: continue
215      if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
216      if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
217    self.best_subtree = ug + (best_dmode[1], best_fmode[1])
218
219    return d
220
221  def SetPermissions(self, script):
222    """Append set_perm/set_perm_recursive commands to 'script' to
223    set all permissions, users, and groups for the tree of files
224    rooted at 'self'."""
225
226    self.CountChildMetadata()
227
228    def recurse(item, current):
229      # current is the (uid, gid, dmode, fmode) tuple that the current
230      # item (and all its children) have already been set to.  We only
231      # need to issue set_perm/set_perm_recursive commands if we're
232      # supposed to be something different.
233      if item.dir:
234        if current != item.best_subtree:
235          script.SetPermissionsRecursive("/"+item.name, *item.best_subtree)
236          current = item.best_subtree
237
238        if item.uid != current[0] or item.gid != current[1] or \
239           item.mode != current[2]:
240          script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
241
242        for i in item.children:
243          recurse(i, current)
244      else:
245        if item.uid != current[0] or item.gid != current[1] or \
246               item.mode != current[3]:
247          script.SetPermissions("/"+item.name, item.uid, item.gid, item.mode)
248
249    recurse(self, (-1, -1, -1, -1))
250
251
252def CopySystemFiles(input_zip, output_zip=None,
253                    substitute=None):
254  """Copies files underneath system/ in the input zip to the output
255  zip.  Populates the Item class with their metadata, and returns a
256  list of symlinks as well as a list of files that will be retouched.
257  output_zip may be None, in which case the copy is skipped (but the
258  other side effects still happen).  substitute is an optional dict
259  of {output filename: contents} to be output instead of certain input
260  files.
261  """
262
263  symlinks = []
264  retouch_files = []
265
266  for info in input_zip.infolist():
267    if info.filename.startswith("SYSTEM/"):
268      basefilename = info.filename[7:]
269      if IsSymlink(info):
270        symlinks.append((input_zip.read(info.filename),
271                         "/system/" + basefilename))
272      else:
273        info2 = copy.copy(info)
274        fn = info2.filename = "system/" + basefilename
275        if substitute and fn in substitute and substitute[fn] is None:
276          continue
277        if output_zip is not None:
278          if substitute and fn in substitute:
279            data = substitute[fn]
280          else:
281            data = input_zip.read(info.filename)
282          if info.filename.startswith("SYSTEM/lib/") and IsRegular(info):
283            retouch_files.append(("/system/" + basefilename,
284                                  sha.sha(data).hexdigest()))
285          output_zip.writestr(info2, data)
286        if fn.endswith("/"):
287          Item.Get(fn[:-1], dir=True)
288        else:
289          Item.Get(fn, dir=False)
290
291  symlinks.sort()
292  return (symlinks, retouch_files)
293
294
295def SignOutput(temp_zip_name, output_zip_name):
296  key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
297  pw = key_passwords[OPTIONS.package_key]
298
299  common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw,
300                  whole_file=True)
301
302
303def AppendAssertions(script, input_zip):
304  device = GetBuildProp("ro.product.device", input_zip)
305  script.AssertDevice(device)
306
307
308def MakeRecoveryPatch(output_zip, recovery_img, boot_img, info):
309  """Generate a binary patch that creates the recovery image starting
310  with the boot image.  (Most of the space in these images is just the
311  kernel, which is identical for the two, so the resulting patch
312  should be efficient.)  Add it to the output zip, along with a shell
313  script that is run from init.rc on first boot to actually do the
314  patching and install the new recovery image.
315
316  recovery_img and boot_img should be File objects for the
317  corresponding images.  info should be the dictionary returned by
318  common.LoadInfoDict() on the input target_files.
319
320  Returns an Item for the shell script, which must be made
321  executable.
322  """
323
324  d = Difference(recovery_img, boot_img)
325  _, _, patch = d.ComputePatch()
326  common.ZipWriteStr(output_zip, "recovery/recovery-from-boot.p", patch)
327  Item.Get("system/recovery-from-boot.p", dir=False)
328
329  # Images with different content will have a different first page, so
330  # we check to see if this recovery has already been installed by
331  # testing just the first 2k.
332  HEADER_SIZE = 2048
333  header_sha1 = sha.sha(recovery_img.data[:HEADER_SIZE]).hexdigest()
334  sh = """#!/system/bin/sh
335if ! applypatch -c %(partition_type)s:%(partition_path)srecovery:%(header_size)d:%(header_sha1)s; then
336  log -t recovery "Installing new recovery image"
337  applypatch %(partition_type)s:%(partition_path)sboot:%(boot_size)d:%(boot_sha1)s %(partition_type)s:%(partition_path)srecovery %(recovery_sha1)s %(recovery_size)d %(boot_sha1)s:/system/recovery-from-boot.p
338else
339  log -t recovery "Recovery image already installed"
340fi
341""" % { 'boot_size': boot_img.size,
342        'boot_sha1': boot_img.sha1,
343        'header_size': HEADER_SIZE,
344        'header_sha1': header_sha1,
345        'recovery_size': recovery_img.size,
346        'recovery_sha1': recovery_img.sha1,
347        'partition_type': info["partition_type"],
348        'partition_path': info.get("partition_path", ""),
349        }
350  common.ZipWriteStr(output_zip, "recovery/etc/install-recovery.sh", sh)
351  return Item.Get("system/etc/install-recovery.sh", dir=False)
352
353
354def WriteFullOTAPackage(input_zip, output_zip, info):
355  # TODO: how to determine this?  We don't know what version it will
356  # be installed on top of.  For now, we expect the API just won't
357  # change very often.
358  script = edify_generator.EdifyGenerator(3, info)
359
360  metadata = {"post-build": GetBuildProp("ro.build.fingerprint", input_zip),
361              "pre-device": GetBuildProp("ro.product.device", input_zip),
362              "post-timestamp": GetBuildProp("ro.build.date.utc", input_zip),
363              }
364
365  device_specific = common.DeviceSpecificParams(
366      input_zip=input_zip,
367      input_version=GetRecoveryAPIVersion(input_zip),
368      output_zip=output_zip,
369      script=script,
370      input_tmp=OPTIONS.input_tmp,
371      metadata=metadata)
372
373  if not OPTIONS.omit_prereq:
374    ts = GetBuildProp("ro.build.date.utc", input_zip)
375    script.AssertOlderBuild(ts)
376
377  AppendAssertions(script, input_zip)
378  device_specific.FullOTA_Assertions()
379
380  script.ShowProgress(0.5, 0)
381
382  if OPTIONS.wipe_user_data:
383    script.FormatPartition("userdata")
384
385  script.FormatPartition("system")
386  script.Mount("system", "/system")
387  script.UnpackPackageDir("recovery", "/system")
388  script.UnpackPackageDir("system", "/system")
389
390  (symlinks, retouch_files) = CopySystemFiles(input_zip, output_zip)
391  script.MakeSymlinks(symlinks)
392  if OPTIONS.aslr_mode:
393    script.RetouchBinaries(retouch_files)
394  else:
395    script.UndoRetouchBinaries(retouch_files)
396
397  boot_img = File("boot.img", common.BuildBootableImage(
398      os.path.join(OPTIONS.input_tmp, "BOOT")))
399  recovery_img = File("recovery.img", common.BuildBootableImage(
400      os.path.join(OPTIONS.input_tmp, "RECOVERY")))
401  MakeRecoveryPatch(output_zip, recovery_img, boot_img, info)
402
403  Item.GetMetadata(input_zip)
404  Item.Get("system").SetPermissions(script)
405
406  common.CheckSize(boot_img.data, "boot.img")
407  common.ZipWriteStr(output_zip, "boot.img", boot_img.data)
408  script.ShowProgress(0.2, 0)
409
410  script.ShowProgress(0.2, 10)
411  script.WriteRawImage("boot", "boot.img")
412
413  script.ShowProgress(0.1, 0)
414  device_specific.FullOTA_InstallEnd()
415
416  if OPTIONS.extra_script is not None:
417    script.AppendExtra(OPTIONS.extra_script)
418
419  script.UnmountAll()
420  script.AddToZip(input_zip, output_zip)
421  WriteMetadata(metadata, output_zip)
422
423
424def WriteMetadata(metadata, output_zip):
425  common.ZipWriteStr(output_zip, "META-INF/com/android/metadata",
426                     "".join(["%s=%s\n" % kv
427                              for kv in sorted(metadata.iteritems())]))
428
429
430class File(object):
431  def __init__(self, name, data):
432    self.name = name
433    self.data = data
434    self.size = len(data)
435    self.sha1 = sha.sha(data).hexdigest()
436
437  def WriteToTemp(self):
438    t = tempfile.NamedTemporaryFile()
439    t.write(self.data)
440    t.flush()
441    return t
442
443  def AddToZip(self, z):
444    common.ZipWriteStr(z, self.name, self.data)
445
446
447def LoadSystemFiles(z):
448  """Load all the files from SYSTEM/... in a given target-files
449  ZipFile, and return a dict of {filename: File object}."""
450  out = {}
451  retouch_files = []
452  for info in z.infolist():
453    if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
454      basefilename = info.filename[7:]
455      fn = "system/" + basefilename
456      data = z.read(info.filename)
457      out[fn] = File(fn, data)
458      if info.filename.startswith("SYSTEM/lib/") and IsRegular(info):
459        retouch_files.append(("/system/" + basefilename,
460                              out[fn].sha1))
461  return (out, retouch_files)
462
463
464DIFF_PROGRAM_BY_EXT = {
465    ".gz" : "imgdiff",
466    ".zip" : ["imgdiff", "-z"],
467    ".jar" : ["imgdiff", "-z"],
468    ".apk" : ["imgdiff", "-z"],
469    ".img" : "imgdiff",
470    }
471
472
473class Difference(object):
474  def __init__(self, tf, sf):
475    self.tf = tf
476    self.sf = sf
477    self.patch = None
478
479  def ComputePatch(self):
480    """Compute the patch (as a string of data) needed to turn sf into
481    tf.  Returns the same tuple as GetPatch()."""
482
483    tf = self.tf
484    sf = self.sf
485
486    ext = os.path.splitext(tf.name)[1]
487    diff_program = DIFF_PROGRAM_BY_EXT.get(ext, "bsdiff")
488
489    ttemp = tf.WriteToTemp()
490    stemp = sf.WriteToTemp()
491
492    ext = os.path.splitext(tf.name)[1]
493
494    try:
495      ptemp = tempfile.NamedTemporaryFile()
496      if isinstance(diff_program, list):
497        cmd = copy.copy(diff_program)
498      else:
499        cmd = [diff_program]
500      cmd.append(stemp.name)
501      cmd.append(ttemp.name)
502      cmd.append(ptemp.name)
503      p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
504      _, err = p.communicate()
505      if err or p.returncode != 0:
506        print "WARNING: failure running %s:\n%s\n" % (diff_program, err)
507        return None
508      diff = ptemp.read()
509    finally:
510      ptemp.close()
511      stemp.close()
512      ttemp.close()
513
514    self.patch = diff
515    return self.tf, self.sf, self.patch
516
517
518  def GetPatch(self):
519    """Return a tuple (target_file, source_file, patch_data).
520    patch_data may be None if ComputePatch hasn't been called, or if
521    computing the patch failed."""
522    return self.tf, self.sf, self.patch
523
524
525def ComputeDifferences(diffs):
526  """Call ComputePatch on all the Difference objects in 'diffs'."""
527  print len(diffs), "diffs to compute"
528
529  # Do the largest files first, to try and reduce the long-pole effect.
530  by_size = [(i.tf.size, i) for i in diffs]
531  by_size.sort(reverse=True)
532  by_size = [i[1] for i in by_size]
533
534  lock = threading.Lock()
535  diff_iter = iter(by_size)   # accessed under lock
536
537  def worker():
538    try:
539      lock.acquire()
540      for d in diff_iter:
541        lock.release()
542        start = time.time()
543        d.ComputePatch()
544        dur = time.time() - start
545        lock.acquire()
546
547        tf, sf, patch = d.GetPatch()
548        if sf.name == tf.name:
549          name = tf.name
550        else:
551          name = "%s (%s)" % (tf.name, sf.name)
552        if patch is None:
553          print "patching failed!                                  %s" % (name,)
554        else:
555          print "%8.2f sec %8d / %8d bytes (%6.2f%%) %s" % (
556              dur, len(patch), tf.size, 100.0 * len(patch) / tf.size, name)
557      lock.release()
558    except Exception, e:
559      print e
560      raise
561
562  # start worker threads; wait for them all to finish.
563  threads = [threading.Thread(target=worker)
564             for i in range(OPTIONS.worker_threads)]
565  for th in threads:
566    th.start()
567  while threads:
568    threads.pop().join()
569
570
571def GetBuildProp(property, z):
572  """Return the fingerprint of the build of a given target-files
573  ZipFile object."""
574  bp = z.read("SYSTEM/build.prop")
575  if not property:
576    return bp
577  m = re.search(re.escape(property) + r"=(.*)\n", bp)
578  if not m:
579    raise common.ExternalError("couldn't find %s in build.prop" % (property,))
580  return m.group(1).strip()
581
582
583def GetRecoveryAPIVersion(zip):
584  """Returns the version of the recovery API.  Version 0 is the older
585  amend code (no separate binary)."""
586  try:
587    version = zip.read("META/recovery-api-version.txt")
588    return int(version)
589  except KeyError:
590    try:
591      # version one didn't have the recovery-api-version.txt file, but
592      # it did include an updater binary.
593      zip.getinfo("OTA/bin/updater")
594      return 1
595    except KeyError:
596      return 0
597
598
599def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip, info):
600  source_version = GetRecoveryAPIVersion(source_zip)
601  target_version = GetRecoveryAPIVersion(target_zip)
602  partition_type = info["partition_type"]
603  partition_path = info.get("partition_path", "")
604
605  if source_version == 0:
606    print ("WARNING: generating edify script for a source that "
607           "can't install it.")
608  script = edify_generator.EdifyGenerator(source_version, info)
609
610  metadata = {"pre-device": GetBuildProp("ro.product.device", source_zip),
611              "post-timestamp": GetBuildProp("ro.build.date.utc", target_zip),
612              }
613
614  device_specific = common.DeviceSpecificParams(
615      source_zip=source_zip,
616      source_version=source_version,
617      target_zip=target_zip,
618      target_version=target_version,
619      output_zip=output_zip,
620      script=script,
621      metadata=metadata)
622
623  print "Loading target..."
624  (target_data, target_retouch_files) = LoadSystemFiles(target_zip)
625  print "Loading source..."
626  (source_data, source_retouch_files) = LoadSystemFiles(source_zip)
627
628  verbatim_targets = []
629  patch_list = []
630  diffs = []
631  largest_source_size = 0
632  for fn in sorted(target_data.keys()):
633    tf = target_data[fn]
634    assert fn == tf.name
635    sf = source_data.get(fn, None)
636
637    if sf is None or fn in OPTIONS.require_verbatim:
638      # This file should be included verbatim
639      if fn in OPTIONS.prohibit_verbatim:
640        raise common.ExternalError("\"%s\" must be sent verbatim" % (fn,))
641      print "send", fn, "verbatim"
642      tf.AddToZip(output_zip)
643      verbatim_targets.append((fn, tf.size))
644    elif tf.sha1 != sf.sha1:
645      # File is different; consider sending as a patch
646      diffs.append(Difference(tf, sf))
647    else:
648      # Target file identical to source.
649      pass
650
651  ComputeDifferences(diffs)
652
653  for diff in diffs:
654    tf, sf, d = diff.GetPatch()
655    if d is None or len(d) > tf.size * OPTIONS.patch_threshold:
656      # patch is almost as big as the file; don't bother patching
657      tf.AddToZip(output_zip)
658      verbatim_targets.append((tf.name, tf.size))
659    else:
660      common.ZipWriteStr(output_zip, "patch/" + tf.name + ".p", d)
661      patch_list.append((tf.name, tf, sf, tf.size, sha.sha(d).hexdigest()))
662      largest_source_size = max(largest_source_size, sf.size)
663
664  source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
665  target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
666  metadata["pre-build"] = source_fp
667  metadata["post-build"] = target_fp
668
669  script.Mount("system", "/system")
670  script.AssertSomeFingerprint(source_fp, target_fp)
671
672  source_boot = File("/tmp/boot.img",
673                     common.BuildBootableImage(
674      os.path.join(OPTIONS.source_tmp, "BOOT")))
675  target_boot = File("/tmp/boot.img",
676                     common.BuildBootableImage(
677      os.path.join(OPTIONS.target_tmp, "BOOT")))
678  updating_boot = (source_boot.data != target_boot.data)
679
680  source_recovery = File("system/recovery.img",
681                         common.BuildBootableImage(
682      os.path.join(OPTIONS.source_tmp, "RECOVERY")))
683  target_recovery = File("system/recovery.img",
684                         common.BuildBootableImage(
685      os.path.join(OPTIONS.target_tmp, "RECOVERY")))
686  updating_recovery = (source_recovery.data != target_recovery.data)
687
688  # Here's how we divide up the progress bar:
689  #  0.1 for verifying the start state (PatchCheck calls)
690  #  0.8 for applying patches (ApplyPatch calls)
691  #  0.1 for unpacking verbatim files, symlinking, and doing the
692  #      device-specific commands.
693
694  AppendAssertions(script, target_zip)
695  device_specific.IncrementalOTA_Assertions()
696
697  script.Print("Verifying current system...")
698
699  script.ShowProgress(0.1, 0)
700  total_verify_size = float(sum([i[2].size for i in patch_list]) + 1)
701  if updating_boot:
702    total_verify_size += source_boot.size
703  so_far = 0
704
705  for fn, tf, sf, size, patch_sha in patch_list:
706    script.PatchCheck("/"+fn, tf.sha1, sf.sha1)
707    so_far += sf.size
708    script.SetProgress(so_far / total_verify_size)
709
710  if updating_boot:
711    d = Difference(target_boot, source_boot)
712    _, _, d = d.ComputePatch()
713    print "boot      target: %d  source: %d  diff: %d" % (
714        target_boot.size, source_boot.size, len(d))
715
716    common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
717
718    script.PatchCheck("%s:%sboot:%d:%s:%d:%s" %
719                      (partition_type, partition_path,
720                       source_boot.size, source_boot.sha1,
721                       target_boot.size, target_boot.sha1))
722    so_far += source_boot.size
723    script.SetProgress(so_far / total_verify_size)
724
725  if patch_list or updating_recovery or updating_boot:
726    script.CacheFreeSpaceCheck(largest_source_size)
727
728  device_specific.IncrementalOTA_VerifyEnd()
729
730  script.Comment("---- start making changes here ----")
731
732  if OPTIONS.wipe_user_data:
733    script.Print("Erasing user data...")
734    script.FormatPartition("userdata")
735
736  script.Print("Removing unneeded files...")
737  script.DeleteFiles(["/"+i[0] for i in verbatim_targets] +
738                     ["/"+i for i in sorted(source_data)
739                            if i not in target_data] +
740                     ["/system/recovery.img"])
741
742  script.ShowProgress(0.8, 0)
743  total_patch_size = float(sum([i[1].size for i in patch_list]) + 1)
744  if updating_boot:
745    total_patch_size += target_boot.size
746  so_far = 0
747
748  script.Print("Patching system files...")
749  for fn, tf, sf, size, _ in patch_list:
750    script.ApplyPatch("/"+fn, "-", tf.size, tf.sha1, sf.sha1, "patch/"+fn+".p")
751    so_far += tf.size
752    script.SetProgress(so_far / total_patch_size)
753
754  if updating_boot:
755    # Produce the boot image by applying a patch to the current
756    # contents of the boot partition, and write it back to the
757    # partition.
758    script.Print("Patching boot image...")
759    script.ApplyPatch("%s:%sboot:%d:%s:%d:%s"
760                      % (partition_type, partition_path,
761                         source_boot.size, source_boot.sha1,
762                         target_boot.size, target_boot.sha1),
763                      "-",
764                      target_boot.size, target_boot.sha1,
765                      source_boot.sha1, "patch/boot.img.p")
766    so_far += target_boot.size
767    script.SetProgress(so_far / total_patch_size)
768    print "boot image changed; including."
769  else:
770    print "boot image unchanged; skipping."
771
772  if updating_recovery:
773    # Is it better to generate recovery as a patch from the current
774    # boot image, or from the previous recovery image?  For large
775    # updates with significant kernel changes, probably the former.
776    # For small updates where the kernel hasn't changed, almost
777    # certainly the latter.  We pick the first option.  Future
778    # complicated schemes may let us effectively use both.
779    #
780    # A wacky possibility: as long as there is room in the boot
781    # partition, include the binaries and image files from recovery in
782    # the boot image (though not in the ramdisk) so they can be used
783    # as fodder for constructing the recovery image.
784    MakeRecoveryPatch(output_zip, target_recovery, target_boot, info)
785    script.DeleteFiles(["/system/recovery-from-boot.p",
786                        "/system/etc/install-recovery.sh"])
787    print "recovery image changed; including as patch from boot."
788  else:
789    print "recovery image unchanged; skipping."
790
791  script.ShowProgress(0.1, 10)
792
793  (target_symlinks, target_retouch_dummies) = CopySystemFiles(target_zip, None)
794
795  target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
796  temp_script = script.MakeTemporary()
797  Item.GetMetadata(target_zip)
798  Item.Get("system").SetPermissions(temp_script)
799
800  # Note that this call will mess up the tree of Items, so make sure
801  # we're done with it.
802  (source_symlinks, source_retouch_dummies) = CopySystemFiles(source_zip, None)
803  source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
804
805  # Delete all the symlinks in source that aren't in target.  This
806  # needs to happen before verbatim files are unpacked, in case a
807  # symlink in the source is replaced by a real file in the target.
808  to_delete = []
809  for dest, link in source_symlinks:
810    if link not in target_symlinks_d:
811      to_delete.append(link)
812  script.DeleteFiles(to_delete)
813
814  if verbatim_targets:
815    script.Print("Unpacking new files...")
816    script.UnpackPackageDir("system", "/system")
817
818  if updating_recovery:
819    script.Print("Unpacking new recovery...")
820    script.UnpackPackageDir("recovery", "/system")
821
822  script.Print("Symlinks and permissions...")
823
824  # Create all the symlinks that don't already exist, or point to
825  # somewhere different than what we want.  Delete each symlink before
826  # creating it, since the 'symlink' command won't overwrite.
827  to_create = []
828  for dest, link in target_symlinks:
829    if link in source_symlinks_d:
830      if dest != source_symlinks_d[link]:
831        to_create.append((dest, link))
832    else:
833      to_create.append((dest, link))
834  script.DeleteFiles([i[1] for i in to_create])
835  script.MakeSymlinks(to_create)
836  if OPTIONS.aslr_mode:
837    script.RetouchBinaries(target_retouch_files)
838  else:
839    script.UndoRetouchBinaries(target_retouch_files)
840
841  # Now that the symlinks are created, we can set all the
842  # permissions.
843  script.AppendScript(temp_script)
844
845  # Do device-specific installation (eg, write radio image).
846  device_specific.IncrementalOTA_InstallEnd()
847
848  if OPTIONS.extra_script is not None:
849    script.AppendExtra(OPTIONS.extra_script)
850
851  script.AddToZip(target_zip, output_zip)
852  WriteMetadata(metadata, output_zip)
853
854
855def main(argv):
856
857  def option_handler(o, a):
858    if o in ("-b", "--board_config"):
859      pass   # deprecated
860    elif o in ("-k", "--package_key"):
861      OPTIONS.package_key = a
862    elif o in ("-i", "--incremental_from"):
863      OPTIONS.incremental_source = a
864    elif o in ("-w", "--wipe_user_data"):
865      OPTIONS.wipe_user_data = True
866    elif o in ("-n", "--no_prereq"):
867      OPTIONS.omit_prereq = True
868    elif o in ("-e", "--extra_script"):
869      OPTIONS.extra_script = a
870    elif o in ("-a", "--aslr_mode"):
871      if a in ("on", "On", "true", "True", "yes", "Yes"):
872        OPTIONS.aslr_mode = True
873      else:
874        OPTIONS.aslr_mode = False
875    elif o in ("--worker_threads"):
876      OPTIONS.worker_threads = int(a)
877    else:
878      return False
879    return True
880
881  args = common.ParseOptions(argv, __doc__,
882                             extra_opts="b:k:i:d:wne:a:",
883                             extra_long_opts=["board_config=",
884                                              "package_key=",
885                                              "incremental_from=",
886                                              "wipe_user_data",
887                                              "no_prereq",
888                                              "extra_script=",
889                                              "worker_threads=",
890                                              "aslr_mode="],
891                             extra_option_handler=option_handler)
892
893  if len(args) != 2:
894    common.Usage(__doc__)
895    sys.exit(1)
896
897  if OPTIONS.extra_script is not None:
898    OPTIONS.extra_script = open(OPTIONS.extra_script).read()
899
900  print "unzipping target target-files..."
901  OPTIONS.input_tmp = common.UnzipTemp(args[0])
902
903  if OPTIONS.device_specific is None:
904    # look for the device-specific tools extension location in the input
905    try:
906      f = open(os.path.join(OPTIONS.input_tmp, "META", "tool-extensions.txt"))
907      ds = f.read().strip()
908      f.close()
909      if ds:
910        ds = os.path.normpath(ds)
911        print "using device-specific extensions in", ds
912        OPTIONS.device_specific = ds
913    except IOError, e:
914      if e.errno == errno.ENOENT:
915        # nothing specified in the file
916        pass
917      else:
918        raise
919
920  info = common.LoadInfoDict()
921  common.LoadMaxSizes(info)
922  if not OPTIONS.max_image_size:
923    print
924    print "  WARNING:  Failed to load max image sizes; will not enforce"
925    print "  image size limits."
926    print
927
928  OPTIONS.target_tmp = OPTIONS.input_tmp
929  input_zip = zipfile.ZipFile(args[0], "r")
930  if OPTIONS.package_key:
931    temp_zip_file = tempfile.NamedTemporaryFile()
932    output_zip = zipfile.ZipFile(temp_zip_file, "w",
933                                 compression=zipfile.ZIP_DEFLATED)
934  else:
935    output_zip = zipfile.ZipFile(args[1], "w",
936                                 compression=zipfile.ZIP_DEFLATED)
937
938  if OPTIONS.incremental_source is None:
939    WriteFullOTAPackage(input_zip, output_zip, info)
940  else:
941    print "unzipping source target-files..."
942    OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source)
943    source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r")
944    WriteIncrementalOTAPackage(input_zip, source_zip, output_zip, info)
945
946  output_zip.close()
947  if OPTIONS.package_key:
948    SignOutput(temp_zip_file.name, args[1])
949    temp_zip_file.close()
950
951  common.Cleanup()
952
953  print "done."
954
955
956if __name__ == '__main__':
957  try:
958    main(sys.argv[1:])
959  except common.ExternalError, e:
960    print
961    print "   ERROR: %s" % (e,)
962    print
963    sys.exit(1)
964