add_img_to_target_files.py revision 4014a9daa742c2fa45c93729fc9d75f488232e3c 
1#!/usr/bin/env python
2#
3# Copyright (C) 2014 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile that does not contain images (ie, does
19not have an IMAGES/ top-level subdirectory), produce the images and
20add them to the zipfile.
21
22Usage:  add_img_to_target_files [flag] target_files
23
24  -a  (--add_missing)
25      Build and add missing images to "IMAGES/". If this option is
26      not specified, this script will simply exit when "IMAGES/"
27      directory exists in the target file.
28
29  -r  (--rebuild_recovery)
30      Rebuild the recovery patch and write it to the system image. Only
31      meaningful when system image needs to be rebuilt.
32
33  --replace_verity_private_key
34      Replace the private key used for verity signing. (same as the option
35      in sign_target_files_apks)
36
37  --replace_verity_public_key
38       Replace the certificate (public key) used for verity verification. (same
39       as the option in sign_target_files_apks)
40
41  --is_signing
42      Skip building & adding the images for "userdata" and "cache" if we
43      are signing the target files.
44
45  --verity_signer_path
46      Specify the signer path to build verity metadata.
47"""
48
49import sys
50
51if sys.hexversion < 0x02070000:
52  print >> sys.stderr, "Python 2.7 or newer is required."
53  sys.exit(1)
54
55import datetime
56import errno
57import os
58import shlex
59import shutil
60import subprocess
61import tempfile
62import zipfile
63
64import build_image
65import common
66
67OPTIONS = common.OPTIONS
68
69OPTIONS.add_missing = False
70OPTIONS.rebuild_recovery = False
71OPTIONS.replace_verity_public_key = False
72OPTIONS.replace_verity_private_key = False
73OPTIONS.is_signing = False
74OPTIONS.verity_signer_path = None
75
76def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
77  """Turn the contents of SYSTEM into a system image and store it in
78  output_zip. Returns the name of the system image file."""
79
80  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img")
81  if os.path.exists(prebuilt_path):
82    print "system.img already exists in %s, no need to rebuild..." % (prefix,)
83    return prebuilt_path
84
85  def output_sink(fn, data):
86    ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w")
87    ofile.write(data)
88    ofile.close()
89
90  if OPTIONS.rebuild_recovery:
91    print "Building new recovery patch"
92    common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,
93                             boot_img, info_dict=OPTIONS.info_dict)
94
95  block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map")
96  imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict,
97                        block_list=block_list)
98
99  common.ZipWrite(output_zip, imgname, prefix + "system.img")
100  common.ZipWrite(output_zip, block_list, prefix + "system.map")
101  return imgname
102
103
104def BuildSystem(input_dir, info_dict, block_list=None):
105  """Build the (sparse) system image and return the name of a temp
106  file containing it."""
107  return CreateImage(input_dir, info_dict, "system", block_list=block_list)
108
109
110def AddVendor(output_zip, prefix="IMAGES/"):
111  """Turn the contents of VENDOR into a vendor image and store in it
112  output_zip."""
113
114  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img")
115  if os.path.exists(prebuilt_path):
116    print "vendor.img already exists in %s, no need to rebuild..." % (prefix,)
117    return
118
119  block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map")
120  imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict,
121                        block_list=block_list)
122  common.ZipWrite(output_zip, imgname, prefix + "vendor.img")
123  common.ZipWrite(output_zip, block_list, prefix + "vendor.map")
124
125
126def BuildVendor(input_dir, info_dict, block_list=None):
127  """Build the (sparse) vendor image and return the name of a temp
128  file containing it."""
129  return CreateImage(input_dir, info_dict, "vendor", block_list=block_list)
130
131
132def CreateImage(input_dir, info_dict, what, block_list=None):
133  print "creating " + what + ".img..."
134
135  img = common.MakeTempFile(prefix=what + "-", suffix=".img")
136
137  # The name of the directory it is making an image out of matters to
138  # mkyaffs2image.  It wants "system" but we have a directory named
139  # "SYSTEM", so create a symlink.
140  try:
141    os.symlink(os.path.join(input_dir, what.upper()),
142               os.path.join(input_dir, what))
143  except OSError as e:
144    # bogus error on my mac version?
145    #   File "./build/tools/releasetools/img_from_target_files"
146    #     os.path.join(OPTIONS.input_tmp, "system"))
147    # OSError: [Errno 17] File exists
148    if e.errno == errno.EEXIST:
149      pass
150
151  image_props = build_image.ImagePropFromGlobalDict(info_dict, what)
152  fstab = info_dict["fstab"]
153  if fstab:
154    image_props["fs_type"] = fstab["/" + what].fs_type
155
156  # Use a fixed timestamp (01/01/2009) when packaging the image.
157  # Bug: 24377993
158  epoch = datetime.datetime.fromtimestamp(0)
159  timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
160  image_props["timestamp"] = int(timestamp)
161
162  if what == "system":
163    fs_config_prefix = ""
164  else:
165    fs_config_prefix = what + "_"
166
167  fs_config = os.path.join(
168      input_dir, "META/" + fs_config_prefix + "filesystem_config.txt")
169  if not os.path.exists(fs_config):
170    fs_config = None
171
172  # Override values loaded from info_dict.
173  if fs_config:
174    image_props["fs_config"] = fs_config
175  if block_list:
176    image_props["block_list"] = block_list
177
178  succ = build_image.BuildImage(os.path.join(input_dir, what),
179                                image_props, img)
180  assert succ, "build " + what + ".img image failed"
181
182  return img
183
184
185def AddUserdata(output_zip, prefix="IMAGES/"):
186  """Create a userdata image and store it in output_zip.
187
188  In most case we just create and store an empty userdata.img;
189  But the invoker can also request to create userdata.img with real
190  data from the target files, by setting "userdata_img_with_data=true"
191  in OPTIONS.info_dict.
192  """
193
194  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img")
195  if os.path.exists(prebuilt_path):
196    print "userdata.img already exists in %s, no need to rebuild..." % (prefix,)
197    return
198
199  # Skip userdata.img if no size.
200  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "data")
201  if not image_props.get("partition_size"):
202    return
203
204  print "creating userdata.img..."
205
206  # Use a fixed timestamp (01/01/2009) when packaging the image.
207  # Bug: 24377993
208  epoch = datetime.datetime.fromtimestamp(0)
209  timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
210  image_props["timestamp"] = int(timestamp)
211
212  # The name of the directory it is making an image out of matters to
213  # mkyaffs2image.  So we create a temp dir, and within it we create an
214  # empty dir named "data", or a symlink to the DATA dir,
215  # and build the image from that.
216  temp_dir = tempfile.mkdtemp()
217  user_dir = os.path.join(temp_dir, "data")
218  empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true")
219  if empty:
220    # Create an empty dir.
221    os.mkdir(user_dir)
222  else:
223    # Symlink to the DATA dir.
224    os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"),
225               user_dir)
226
227  img = tempfile.NamedTemporaryFile()
228
229  fstab = OPTIONS.info_dict["fstab"]
230  if fstab:
231    image_props["fs_type"] = fstab["/data"].fs_type
232  succ = build_image.BuildImage(user_dir, image_props, img.name)
233  assert succ, "build userdata.img image failed"
234
235  common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
236  common.ZipWrite(output_zip, img.name, prefix + "userdata.img")
237  img.close()
238  shutil.rmtree(temp_dir)
239
240
241def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"):
242  """Create a VBMeta image and store it in output_zip."""
243  _, img_file_name = tempfile.mkstemp()
244  avbtool = os.getenv('AVBTOOL') or "avbtool"
245  cmd = [avbtool, "make_vbmeta_image",
246         "--output", img_file_name,
247         "--include_descriptors_from_image", boot_img_path,
248         "--include_descriptors_from_image", system_img_path,
249         "--generate_dm_verity_cmdline_from_hashtree", system_img_path]
250  common.AppendAVBSigningArgs(cmd)
251  args = OPTIONS.info_dict.get("board_avb_make_vbmeta_image_args", None)
252  if args and args.strip():
253    cmd.extend(shlex.split(args))
254  p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
255  p.communicate()
256  assert p.returncode == 0, "avbtool make_vbmeta_image failed"
257  common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img")
258
259
260def AddPartitionTable(output_zip, prefix="IMAGES/"):
261  """Create a partition table image and store it in output_zip."""
262
263  _, img_file_name = tempfile.mkstemp()
264  _, bpt_file_name = tempfile.mkstemp()
265
266  # use BPTTOOL from environ, or "bpttool" if empty or not set.
267  bpttool = os.getenv("BPTTOOL") or "bpttool"
268  cmd = [bpttool, "make_table", "--output_json", bpt_file_name,
269         "--output_gpt", img_file_name]
270  input_files_str = OPTIONS.info_dict["board_bpt_input_files"]
271  input_files = input_files_str.split(" ")
272  for i in input_files:
273    cmd.extend(["--input", i])
274  disk_size = OPTIONS.info_dict.get("board_bpt_disk_size")
275  if disk_size:
276    cmd.extend(["--disk_size", disk_size])
277  args = OPTIONS.info_dict.get("board_bpt_make_table_args")
278  if args:
279    cmd.extend(shlex.split(args))
280
281  p = common.Run(cmd, stdout=subprocess.PIPE)
282  p.communicate()
283  assert p.returncode == 0, "bpttool make_table failed"
284
285  common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img")
286  common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt")
287
288
289def AddCache(output_zip, prefix="IMAGES/"):
290  """Create an empty cache image and store it in output_zip."""
291
292  prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img")
293  if os.path.exists(prebuilt_path):
294    print "cache.img already exists in %s, no need to rebuild..." % (prefix,)
295    return
296
297  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "cache")
298  # The build system has to explicitly request for cache.img.
299  if "fs_type" not in image_props:
300    return
301
302  print "creating cache.img..."
303
304  # Use a fixed timestamp (01/01/2009) when packaging the image.
305  # Bug: 24377993
306  epoch = datetime.datetime.fromtimestamp(0)
307  timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
308  image_props["timestamp"] = int(timestamp)
309
310  # The name of the directory it is making an image out of matters to
311  # mkyaffs2image.  So we create a temp dir, and within it we create an
312  # empty dir named "cache", and build the image from that.
313  temp_dir = tempfile.mkdtemp()
314  user_dir = os.path.join(temp_dir, "cache")
315  os.mkdir(user_dir)
316  img = tempfile.NamedTemporaryFile()
317
318  fstab = OPTIONS.info_dict["fstab"]
319  if fstab:
320    image_props["fs_type"] = fstab["/cache"].fs_type
321  succ = build_image.BuildImage(user_dir, image_props, img.name)
322  assert succ, "build cache.img image failed"
323
324  common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)
325  common.ZipWrite(output_zip, img.name, prefix + "cache.img")
326  img.close()
327  os.rmdir(user_dir)
328  os.rmdir(temp_dir)
329
330
331def AddImagesToTargetFiles(filename):
332  OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)
333
334  if not OPTIONS.add_missing:
335    for n in input_zip.namelist():
336      if n.startswith("IMAGES/"):
337        print "target_files appears to already contain images."
338        sys.exit(1)
339
340  try:
341    input_zip.getinfo("VENDOR/")
342    has_vendor = True
343  except KeyError:
344    has_vendor = False
345
346  OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)
347
348  common.ZipClose(input_zip)
349  output_zip = zipfile.ZipFile(filename, "a",
350                               compression=zipfile.ZIP_DEFLATED,
351                               allowZip64=True)
352
353  has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true")
354  system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true")
355
356  def banner(s):
357    print "\n\n++++ " + s + " ++++\n\n"
358
359  prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "boot.img")
360  boot_image = None
361  if os.path.exists(prebuilt_path):
362    banner("boot")
363    print "boot.img already exists in IMAGES/, no need to rebuild..."
364    if OPTIONS.rebuild_recovery:
365      boot_image = common.GetBootableImage(
366          "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
367  else:
368    banner("boot")
369    boot_image = common.GetBootableImage(
370        "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
371    if boot_image:
372      boot_image.AddToZip(output_zip)
373
374  recovery_image = None
375  if has_recovery:
376    banner("recovery")
377    prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "recovery.img")
378    if os.path.exists(prebuilt_path):
379      print "recovery.img already exists in IMAGES/, no need to rebuild..."
380      if OPTIONS.rebuild_recovery:
381        recovery_image = common.GetBootableImage(
382            "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp,
383            "RECOVERY")
384    else:
385      recovery_image = common.GetBootableImage(
386          "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY")
387      if recovery_image:
388        recovery_image.AddToZip(output_zip)
389
390  banner("system")
391  system_img_path = AddSystem(
392    output_zip, recovery_img=recovery_image, boot_img=boot_image)
393  if has_vendor:
394    banner("vendor")
395    AddVendor(output_zip)
396  if not OPTIONS.is_signing:
397    banner("userdata")
398    AddUserdata(output_zip)
399    banner("cache")
400    AddCache(output_zip)
401  if OPTIONS.info_dict.get("board_bpt_enable", None) == "true":
402    banner("partition-table")
403    AddPartitionTable(output_zip)
404  if OPTIONS.info_dict.get("board_avb_enable", None) == "true":
405    banner("vbmeta")
406    boot_contents = boot_image.WriteToTemp()
407    AddVBMeta(output_zip, boot_contents.name, system_img_path)
408
409  # For devices using A/B update, copy over images from RADIO/ and/or
410  # VENDOR_IMAGES/ to IMAGES/ and make sure we have all the needed
411  # images ready under IMAGES/. All images should have '.img' as extension.
412  banner("radio")
413  ab_partitions = os.path.join(OPTIONS.input_tmp, "META", "ab_partitions.txt")
414  if os.path.exists(ab_partitions):
415    with open(ab_partitions, 'r') as f:
416      lines = f.readlines()
417    for line in lines:
418      img_name = line.strip() + ".img"
419      prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)
420      if os.path.exists(prebuilt_path):
421        print "%s already exists, no need to overwrite..." % (img_name,)
422        continue
423
424      img_radio_path = os.path.join(OPTIONS.input_tmp, "RADIO", img_name)
425      img_vendor_dir = os.path.join(
426        OPTIONS.input_tmp, "VENDOR_IMAGES")
427      if os.path.exists(img_radio_path):
428        common.ZipWrite(output_zip, img_radio_path,
429                        os.path.join("IMAGES", img_name))
430      else:
431        for root, _, files in os.walk(img_vendor_dir):
432          if img_name in files:
433            common.ZipWrite(output_zip, os.path.join(root, img_name),
434              os.path.join("IMAGES", img_name))
435            break
436
437      # Zip spec says: All slashes MUST be forward slashes.
438      img_path = 'IMAGES/' + img_name
439      assert img_path in output_zip.namelist(), "cannot find " + img_name
440
441  common.ZipClose(output_zip)
442
443def main(argv):
444  def option_handler(o, a):
445    if o in ("-a", "--add_missing"):
446      OPTIONS.add_missing = True
447    elif o in ("-r", "--rebuild_recovery",):
448      OPTIONS.rebuild_recovery = True
449    elif o == "--replace_verity_private_key":
450      OPTIONS.replace_verity_private_key = (True, a)
451    elif o == "--replace_verity_public_key":
452      OPTIONS.replace_verity_public_key = (True, a)
453    elif o == "--is_signing":
454      OPTIONS.is_signing = True
455    elif o == "--verity_signer_path":
456      OPTIONS.verity_signer_path = a
457    else:
458      return False
459    return True
460
461  args = common.ParseOptions(
462      argv, __doc__, extra_opts="ar",
463      extra_long_opts=["add_missing", "rebuild_recovery",
464                       "replace_verity_public_key=",
465                       "replace_verity_private_key=",
466                       "is_signing",
467                       "verity_signer_path="],
468      extra_option_handler=option_handler)
469
470
471  if len(args) != 1:
472    common.Usage(__doc__)
473    sys.exit(1)
474
475  AddImagesToTargetFiles(args[0])
476  print "done."
477
478if __name__ == '__main__':
479  try:
480    common.CloseInheritedPipes()
481    main(sys.argv[1:])
482  except common.ExternalError as e:
483    print
484    print "   ERROR: %s" % (e,)
485    print
486    sys.exit(1)
487  finally:
488    common.Cleanup()
489