ota_from_target_files.py revision eef3944eb3673329b5e89cf188ac592805a0b08d 
1#!/usr/bin/env python
2#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile, produces an OTA package that installs
19that build.  An incremental OTA is produced if -i is given, otherwise
20a full OTA is produced.
21
22Usage:  ota_from_target_files [flags] input_target_files output_ota_package
23
24  -b  (--board_config)  <file>
25      Specifies a BoardConfig.mk file containing image max sizes
26      against which the generated image files are checked.
27
28  -k  (--package_key)  <key>
29      Key to use to sign the package (default is
30      "build/target/product/security/testkey").
31
32  -i  (--incremental_from)  <file>
33      Generate an incremental OTA using the given target-files zip as
34      the starting build.
35
36"""
37
38import sys
39
40if sys.hexversion < 0x02040000:
41  print >> sys.stderr, "Python 2.4 or newer is required."
42  sys.exit(1)
43
44import copy
45import os
46import re
47import sha
48import subprocess
49import tempfile
50import time
51import zipfile
52
53import common
54
55OPTIONS = common.OPTIONS
56OPTIONS.package_key = "build/target/product/security/testkey"
57OPTIONS.incremental_source = None
58OPTIONS.require_verbatim = set()
59OPTIONS.prohibit_verbatim = set(("system/build.prop",))
60OPTIONS.patch_threshold = 0.95
61
62def MostPopularKey(d, default):
63  """Given a dict, return the key corresponding to the largest
64  value.  Returns 'default' if the dict is empty."""
65  x = [(v, k) for (k, v) in d.iteritems()]
66  if not x: return default
67  x.sort()
68  return x[-1][1]
69
70
71def IsSymlink(info):
72  """Return true if the zipfile.ZipInfo object passed in represents a
73  symlink."""
74  return (info.external_attr >> 16) == 0120777
75
76
77
78class Item:
79  """Items represent the metadata (user, group, mode) of files and
80  directories in the system image."""
81  ITEMS = {}
82  def __init__(self, name, dir=False):
83    self.name = name
84    self.uid = None
85    self.gid = None
86    self.mode = None
87    self.dir = dir
88
89    if name:
90      self.parent = Item.Get(os.path.dirname(name), dir=True)
91      self.parent.children.append(self)
92    else:
93      self.parent = None
94    if dir:
95      self.children = []
96
97  def Dump(self, indent=0):
98    if self.uid is not None:
99      print "%s%s %d %d %o" % ("  "*indent, self.name, self.uid, self.gid, self.mode)
100    else:
101      print "%s%s %s %s %s" % ("  "*indent, self.name, self.uid, self.gid, self.mode)
102    if self.dir:
103      print "%s%s" % ("  "*indent, self.descendants)
104      print "%s%s" % ("  "*indent, self.best_subtree)
105      for i in self.children:
106        i.Dump(indent=indent+1)
107
108  @classmethod
109  def Get(cls, name, dir=False):
110    if name not in cls.ITEMS:
111      cls.ITEMS[name] = Item(name, dir=dir)
112    return cls.ITEMS[name]
113
114  @classmethod
115  def GetMetadata(cls):
116    """Run the external 'fs_config' program to determine the desired
117    uid, gid, and mode for every Item object."""
118    p = common.Run(["fs_config"], stdin=subprocess.PIPE,
119                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
120    suffix = { False: "", True: "/" }
121    input = "".join(["%s%s\n" % (i.name, suffix[i.dir])
122                     for i in cls.ITEMS.itervalues() if i.name])
123    output, error = p.communicate(input)
124    assert not error
125
126    for line in output.split("\n"):
127      if not line: continue
128      name, uid, gid, mode = line.split()
129      i = cls.ITEMS[name]
130      i.uid = int(uid)
131      i.gid = int(gid)
132      i.mode = int(mode, 8)
133      if i.dir:
134        i.children.sort(key=lambda i: i.name)
135
136  def CountChildMetadata(self):
137    """Count up the (uid, gid, mode) tuples for all children and
138    determine the best strategy for using set_perm_recursive and
139    set_perm to correctly chown/chmod all the files to their desired
140    values.  Recursively calls itself for all descendants.
141
142    Returns a dict of {(uid, gid, dmode, fmode): count} counting up
143    all descendants of this node.  (dmode or fmode may be None.)  Also
144    sets the best_subtree of each directory Item to the (uid, gid,
145    dmode, fmode) tuple that will match the most descendants of that
146    Item.
147    """
148
149    assert self.dir
150    d = self.descendants = {(self.uid, self.gid, self.mode, None): 1}
151    for i in self.children:
152      if i.dir:
153        for k, v in i.CountChildMetadata().iteritems():
154          d[k] = d.get(k, 0) + v
155      else:
156        k = (i.uid, i.gid, None, i.mode)
157        d[k] = d.get(k, 0) + 1
158
159    # Find the (uid, gid, dmode, fmode) tuple that matches the most
160    # descendants.
161
162    # First, find the (uid, gid) pair that matches the most
163    # descendants.
164    ug = {}
165    for (uid, gid, _, _), count in d.iteritems():
166      ug[(uid, gid)] = ug.get((uid, gid), 0) + count
167    ug = MostPopularKey(ug, (0, 0))
168
169    # Now find the dmode and fmode that match the most descendants
170    # with that (uid, gid), and choose those.
171    best_dmode = (0, 0755)
172    best_fmode = (0, 0644)
173    for k, count in d.iteritems():
174      if k[:2] != ug: continue
175      if k[2] is not None and count >= best_dmode[0]: best_dmode = (count, k[2])
176      if k[3] is not None and count >= best_fmode[0]: best_fmode = (count, k[3])
177    self.best_subtree = ug + (best_dmode[1], best_fmode[1])
178
179    return d
180
181  def SetPermissions(self, script, renamer=lambda x: x):
182    """Append set_perm/set_perm_recursive commands to 'script' to
183    set all permissions, users, and groups for the tree of files
184    rooted at 'self'.  'renamer' turns the filenames stored in the
185    tree of Items into the strings used in the script."""
186
187    self.CountChildMetadata()
188
189    def recurse(item, current):
190      # current is the (uid, gid, dmode, fmode) tuple that the current
191      # item (and all its children) have already been set to.  We only
192      # need to issue set_perm/set_perm_recursive commands if we're
193      # supposed to be something different.
194      if item.dir:
195        if current != item.best_subtree:
196          script.append("set_perm_recursive %d %d 0%o 0%o %s" %
197                        (item.best_subtree + (renamer(item.name),)))
198          current = item.best_subtree
199
200        if item.uid != current[0] or item.gid != current[1] or \
201           item.mode != current[2]:
202          script.append("set_perm %d %d 0%o %s" %
203                        (item.uid, item.gid, item.mode, renamer(item.name)))
204
205        for i in item.children:
206          recurse(i, current)
207      else:
208        if item.uid != current[0] or item.gid != current[1] or \
209               item.mode != current[3]:
210          script.append("set_perm %d %d 0%o %s" %
211                        (item.uid, item.gid, item.mode, renamer(item.name)))
212
213    recurse(self, (-1, -1, -1, -1))
214
215
216def CopySystemFiles(input_zip, output_zip=None,
217                    substitute=None):
218  """Copies files underneath system/ in the input zip to the output
219  zip.  Populates the Item class with their metadata, and returns a
220  list of symlinks.  output_zip may be None, in which case the copy is
221  skipped (but the other side effects still happen).  substitute is an
222  optional dict of {output filename: contents} to be output instead of
223  certain input files.
224  """
225
226  symlinks = []
227
228  for info in input_zip.infolist():
229    if info.filename.startswith("SYSTEM/"):
230      basefilename = info.filename[7:]
231      if IsSymlink(info):
232        symlinks.append((input_zip.read(info.filename),
233                         "SYSTEM:" + basefilename))
234      else:
235        info2 = copy.copy(info)
236        fn = info2.filename = "system/" + basefilename
237        if substitute and fn in substitute and substitute[fn] is None:
238          continue
239        if output_zip is not None:
240          if substitute and fn in substitute:
241            data = substitute[fn]
242          else:
243            data = input_zip.read(info.filename)
244          output_zip.writestr(info2, data)
245        if fn.endswith("/"):
246          Item.Get(fn[:-1], dir=True)
247        else:
248          Item.Get(fn, dir=False)
249
250  symlinks.sort()
251  return symlinks
252
253
254def AddScript(script, output_zip):
255  now = time.localtime()
256  i = zipfile.ZipInfo("META-INF/com/google/android/update-script",
257                      (now.tm_year, now.tm_mon, now.tm_mday,
258                       now.tm_hour, now.tm_min, now.tm_sec))
259  output_zip.writestr(i, "\n".join(script) + "\n")
260
261
262def SignOutput(temp_zip_name, output_zip_name):
263  key_passwords = common.GetKeyPasswords([OPTIONS.package_key])
264  pw = key_passwords[OPTIONS.package_key]
265
266  common.SignFile(temp_zip_name, output_zip_name, OPTIONS.package_key, pw)
267
268
269def SubstituteRoot(s):
270  if s == "system": return "SYSTEM:"
271  assert s.startswith("system/")
272  return "SYSTEM:" + s[7:]
273
274def FixPermissions(script):
275  Item.GetMetadata()
276  root = Item.Get("system")
277  root.SetPermissions(script, renamer=SubstituteRoot)
278
279def DeleteFiles(script, to_delete):
280  line = []
281  t = 0
282  for i in to_delete:
283    line.append(i)
284    t += len(i) + 1
285    if t > 80:
286      script.append("delete " + " ".join(line))
287      line = []
288      t = 0
289  if line:
290    script.append("delete " + " ".join(line))
291
292def AppendAssertions(script, input_zip):
293  script.append('assert compatible_with("0.2") == "true"')
294
295  device = GetBuildProp("ro.product.device", input_zip)
296  script.append('assert getprop("ro.product.device") == "%s" || '
297                'getprop("ro.build.product") == "%s"' % (device, device))
298
299  info = input_zip.read("OTA/android-info.txt")
300  m = re.search(r"require\s+version-bootloader\s*=\s*(\S+)", info)
301  if not m:
302    raise ExternalError("failed to find required bootloaders in "
303                        "android-info.txt")
304  bootloaders = m.group(1).split("|")
305  script.append("assert " +
306                " || ".join(['getprop("ro.bootloader") == "%s"' % (b,)
307                             for b in bootloaders]))
308
309
310def IncludeBinary(name, input_zip, output_zip):
311  try:
312    data = input_zip.read(os.path.join("OTA/bin", name))
313    output_zip.writestr(name, data)
314  except IOError:
315    raise ExternalError('unable to include device binary "%s"' % (name,))
316
317
318def WriteFullOTAPackage(input_zip, output_zip):
319  script = []
320
321  ts = GetBuildProp("ro.build.date.utc", input_zip)
322  script.append("run_program PACKAGE:check_prereq %s" % (ts,))
323  IncludeBinary("check_prereq", input_zip, output_zip)
324
325  AppendAssertions(script, input_zip)
326
327  script.append("format BOOT:")
328  script.append("show_progress 0.1 0")
329
330  output_zip.writestr("radio.img", input_zip.read("RADIO/image"))
331  script.append("write_radio_image PACKAGE:radio.img")
332  script.append("show_progress 0.5 0")
333
334  script.append("format SYSTEM:")
335  script.append("copy_dir PACKAGE:system SYSTEM:")
336
337  symlinks = CopySystemFiles(input_zip, output_zip)
338  script.extend(["symlink %s %s" % s for s in symlinks])
339
340  common.BuildAndAddBootableImage(os.path.join(OPTIONS.input_tmp, "RECOVERY"),
341                                  "system/recovery.img", output_zip)
342  Item.Get("system/recovery.img", dir=False)
343
344  FixPermissions(script)
345
346  common.AddBoot(output_zip)
347  script.append("show_progress 0.2 0")
348  script.append("write_raw_image PACKAGE:boot.img BOOT:")
349  script.append("show_progress 0.2 10")
350
351  AddScript(script, output_zip)
352
353
354class File(object):
355  def __init__(self, name, data):
356    self.name = name
357    self.data = data
358    self.size = len(data)
359    self.sha1 = sha.sha(data).hexdigest()
360
361  def WriteToTemp(self):
362    t = tempfile.NamedTemporaryFile()
363    t.write(self.data)
364    t.flush()
365    return t
366
367  def AddToZip(self, z):
368    z.writestr(self.name, self.data)
369
370
371def LoadSystemFiles(z):
372  """Load all the files from SYSTEM/... in a given target-files
373  ZipFile, and return a dict of {filename: File object}."""
374  out = {}
375  for info in z.infolist():
376    if info.filename.startswith("SYSTEM/") and not IsSymlink(info):
377      fn = "system/" + info.filename[7:]
378      data = z.read(info.filename)
379      out[fn] = File(fn, data)
380  return out
381
382
383def Difference(tf, sf):
384  """Return the patch (as a string of data) needed to turn sf into tf."""
385
386  ttemp = tf.WriteToTemp()
387  stemp = sf.WriteToTemp()
388
389  ext = os.path.splitext(tf.name)[1]
390
391  try:
392    ptemp = tempfile.NamedTemporaryFile()
393    p = common.Run(["bsdiff", stemp.name, ttemp.name, ptemp.name])
394    _, err = p.communicate()
395    if err:
396      raise ExternalError("failure running bsdiff:\n%s\n" % (err,))
397    diff = ptemp.read()
398    ptemp.close()
399  finally:
400    stemp.close()
401    ttemp.close()
402
403  return diff
404
405
406def GetBuildProp(property, z):
407  """Return the fingerprint of the build of a given target-files
408  ZipFile object."""
409  bp = z.read("SYSTEM/build.prop")
410  if not property:
411    return bp
412  m = re.search(re.escape(property) + r"=(.*)\n", bp)
413  if not m:
414    raise ExternalException("couldn't find %s in build.prop" % (property,))
415  return m.group(1).strip()
416
417
418def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
419  script = []
420
421  print "Loading target..."
422  target_data = LoadSystemFiles(target_zip)
423  print "Loading source..."
424  source_data = LoadSystemFiles(source_zip)
425
426  verbatim_targets = []
427  patch_list = []
428  largest_source_size = 0
429  for fn in sorted(target_data.keys()):
430    tf = target_data[fn]
431    sf = source_data.get(fn, None)
432
433    if sf is None or fn in OPTIONS.require_verbatim:
434      # This file should be included verbatim
435      if fn in OPTIONS.prohibit_verbatim:
436        raise ExternalError("\"%s\" must be sent verbatim" % (fn,))
437      print "send", fn, "verbatim"
438      tf.AddToZip(output_zip)
439      verbatim_targets.append((fn, tf.size))
440    elif tf.sha1 != sf.sha1:
441      # File is different; consider sending as a patch
442      d = Difference(tf, sf)
443      print fn, tf.size, len(d), (float(len(d)) / tf.size)
444      if len(d) > tf.size * OPTIONS.patch_threshold:
445        # patch is almost as big as the file; don't bother patching
446        tf.AddToZip(output_zip)
447        verbatim_targets.append((fn, tf.size))
448      else:
449        output_zip.writestr("patch/" + fn + ".p", d)
450        patch_list.append((fn, tf, sf, tf.size))
451        largest_source_size = max(largest_source_size, sf.size)
452    else:
453      # Target file identical to source.
454      pass
455
456  total_verbatim_size = sum([i[1] for i in verbatim_targets])
457  total_patched_size = sum([i[3] for i in patch_list])
458
459  source_fp = GetBuildProp("ro.build.fingerprint", source_zip)
460  target_fp = GetBuildProp("ro.build.fingerprint", target_zip)
461
462  script.append(('assert file_contains("SYSTEM:build.prop", '
463                 '"ro.build.fingerprint=%s") == "true" || '
464                 'file_contains("SYSTEM:build.prop", '
465                 '"ro.build.fingerprint=%s") == "true"') %
466                (source_fp, target_fp))
467
468  source_boot = common.BuildBootableImage(
469      os.path.join(OPTIONS.source_tmp, "BOOT"))
470  target_boot = common.BuildBootableImage(
471      os.path.join(OPTIONS.target_tmp, "BOOT"))
472  updating_boot = (source_boot != target_boot)
473
474  source_recovery = common.BuildBootableImage(
475      os.path.join(OPTIONS.source_tmp, "RECOVERY"))
476  target_recovery = common.BuildBootableImage(
477      os.path.join(OPTIONS.target_tmp, "RECOVERY"))
478  updating_recovery = (source_recovery != target_recovery)
479
480  source_radio = source_zip.read("RADIO/image")
481  target_radio = target_zip.read("RADIO/image")
482  updating_radio = (source_radio != target_radio)
483
484  # The last 0.1 is reserved for creating symlinks, fixing
485  # permissions, and writing the boot image (if necessary).
486  progress_bar_total = 1.0
487  if updating_boot:
488    progress_bar_total -= 0.1
489  if updating_radio:
490    progress_bar_total -= 0.3
491
492  AppendAssertions(script, target_zip)
493
494  pb_verify = progress_bar_total * 0.3 * \
495              (total_patched_size /
496               float(total_patched_size+total_verbatim_size))
497
498  for i, (fn, tf, sf, size) in enumerate(patch_list):
499    if i % 5 == 0:
500      next_sizes = sum([i[3] for i in patch_list[i:i+5]])
501      script.append("show_progress %f 1" %
502                    (next_sizes * pb_verify / total_patched_size,))
503    script.append("run_program PACKAGE:applypatch -c /%s %s %s" %
504                  (fn, tf.sha1, sf.sha1))
505
506  if patch_list:
507    script.append("run_program PACKAGE:applypatch -s %d" %
508                  (largest_source_size,))
509    script.append("copy_dir PACKAGE:patch CACHE:../tmp/patchtmp")
510    IncludeBinary("applypatch", target_zip, output_zip)
511
512  script.append("\n# ---- start making changes here\n")
513
514  DeleteFiles(script, [SubstituteRoot(i[0]) for i in verbatim_targets])
515
516  if updating_boot:
517    script.append("format BOOT:")
518    output_zip.writestr("boot.img", target_boot)
519    print "boot image changed; including."
520  else:
521    print "boot image unchanged; skipping."
522
523  if updating_recovery:
524    output_zip.writestr("system/recovery.img", target_recovery)
525    print "recovery image changed; including."
526  else:
527    print "recovery image unchanged; skipping."
528
529  if updating_radio:
530    script.append("show_progress 0.3 10")
531    script.append("write_radio_image PACKAGE:radio.img")
532    output_zip.writestr("radio.img", target_radio)
533    print "radio image changed; including."
534  else:
535    print "radio image unchanged; skipping."
536
537  pb_apply = progress_bar_total * 0.7 * \
538             (total_patched_size /
539              float(total_patched_size+total_verbatim_size))
540  for i, (fn, tf, sf, size) in enumerate(patch_list):
541    if i % 5 == 0:
542      next_sizes = sum([i[3] for i in patch_list[i:i+5]])
543      script.append("show_progress %f 1" %
544                    (next_sizes * pb_apply / total_patched_size,))
545    script.append(("run_program PACKAGE:applypatch "
546                   "/%s %s %d %s:/tmp/patchtmp/%s.p") %
547                  (fn, tf.sha1, tf.size, sf.sha1, fn))
548
549  target_symlinks = CopySystemFiles(target_zip, None)
550
551  target_symlinks_d = dict([(i[1], i[0]) for i in target_symlinks])
552  temp_script = []
553  FixPermissions(temp_script)
554
555  # Note that this call will mess up the tree of Items, so make sure
556  # we're done with it.
557  source_symlinks = CopySystemFiles(source_zip, None)
558  source_symlinks_d = dict([(i[1], i[0]) for i in source_symlinks])
559
560  # Delete all the symlinks in source that aren't in target.  This
561  # needs to happen before verbatim files are unpacked, in case a
562  # symlink in the source is replaced by a real file in the target.
563  to_delete = []
564  for dest, link in source_symlinks:
565    if link not in target_symlinks_d:
566      to_delete.append(link)
567  DeleteFiles(script, to_delete)
568
569  if verbatim_targets:
570    pb_verbatim = progress_bar_total * \
571                  (total_verbatim_size /
572                   float(total_patched_size+total_verbatim_size))
573    script.append("show_progress %f 5" % (pb_verbatim,))
574    script.append("copy_dir PACKAGE:system SYSTEM:")
575
576  # Create all the symlinks that don't already exist, or point to
577  # somewhere different than what we want.  Delete each symlink before
578  # creating it, since the 'symlink' command won't overwrite.
579  to_create = []
580  for dest, link in target_symlinks:
581    if link in source_symlinks_d:
582      if dest != source_symlinks_d[link]:
583        to_create.append((dest, link))
584    else:
585      to_create.append((dest, link))
586  DeleteFiles(script, [i[1] for i in to_create])
587  script.extend(["symlink %s %s" % s for s in to_create])
588
589  # Now that the symlinks are created, we can set all the
590  # permissions.
591  script.extend(temp_script)
592
593  if updating_boot:
594    script.append("show_progress 0.1 5")
595    script.append("write_raw_image PACKAGE:boot.img BOOT:")
596
597  AddScript(script, output_zip)
598
599
600def main(argv):
601
602  def option_handler(o, a):
603    if o in ("-b", "--board_config"):
604      common.LoadBoardConfig(a)
605      return True
606    elif o in ("-k", "--package_key"):
607      OPTIONS.package_key = a
608      return True
609    elif o in ("-i", "--incremental_from"):
610      OPTIONS.incremental_source = a
611      return True
612    else:
613      return False
614
615  args = common.ParseOptions(argv, __doc__,
616                             extra_opts="b:k:i:d:",
617                             extra_long_opts=["board_config=",
618                                              "package_key=",
619                                              "incremental_from="],
620                             extra_option_handler=option_handler)
621
622  if len(args) != 2:
623    common.Usage(__doc__)
624    sys.exit(1)
625
626  if not OPTIONS.max_image_size:
627    print
628    print "  WARNING:  No board config specified; will not check image"
629    print "  sizes against limits.  Use -b to make sure the generated"
630    print "  images don't exceed partition sizes."
631    print
632
633  print "unzipping target target-files..."
634  OPTIONS.input_tmp = common.UnzipTemp(args[0])
635  OPTIONS.target_tmp = OPTIONS.input_tmp
636  input_zip = zipfile.ZipFile(args[0], "r")
637  if OPTIONS.package_key:
638    temp_zip_file = tempfile.NamedTemporaryFile()
639    output_zip = zipfile.ZipFile(temp_zip_file, "w",
640                                 compression=zipfile.ZIP_DEFLATED)
641  else:
642    output_zip = zipfile.ZipFile(args[1], "w",
643                                 compression=zipfile.ZIP_DEFLATED)
644
645  if OPTIONS.incremental_source is None:
646    WriteFullOTAPackage(input_zip, output_zip)
647  else:
648    print "unzipping source target-files..."
649    OPTIONS.source_tmp = common.UnzipTemp(OPTIONS.incremental_source)
650    source_zip = zipfile.ZipFile(OPTIONS.incremental_source, "r")
651    WriteIncrementalOTAPackage(input_zip, source_zip, output_zip)
652
653  output_zip.close()
654  if OPTIONS.package_key:
655    SignOutput(temp_zip_file.name, args[1])
656    temp_zip_file.close()
657
658  common.Cleanup()
659
660  print "done."
661
662
663if __name__ == '__main__':
664  try:
665    main(sys.argv[1:])
666  except common.ExternalError, e:
667    print
668    print "   ERROR: %s" % (e,)
669    print
670    sys.exit(1)
671