1#!/usr/bin/env python
2#
3# Copyright (C) 2014 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Given a target-files zipfile that does not contain images (ie, does
19not have an IMAGES/ top-level subdirectory), produce the images and
20add them to the zipfile.
21
22Usage:  add_img_to_target_files [flag] target_files
23
24  -a  (--add_missing)
25      Build and add missing images to "IMAGES/". If this option is
26      not specified, this script will simply exit when "IMAGES/"
27      directory exists in the target file.
28
29  -r  (--rebuild_recovery)
30      Rebuild the recovery patch and write it to the system image. Only
31      meaningful when system image needs to be rebuilt.
32
33  --replace_verity_private_key
34      Replace the private key used for verity signing. (same as the option
35      in sign_target_files_apks)
36
37  --replace_verity_public_key
38       Replace the certificate (public key) used for verity verification. (same
39       as the option in sign_target_files_apks)
40
41  --is_signing
42      Skip building & adding the images for "userdata" and "cache" if we
43      are signing the target files.
44"""
45
46from __future__ import print_function
47
48import sys
49
50if sys.hexversion < 0x02070000:
51  print("Python 2.7 or newer is required.", file=sys.stderr)
52  sys.exit(1)
53
54import datetime
55import errno
56import os
57import shlex
58import shutil
59import subprocess
60import tempfile
61import zipfile
62
63import build_image
64import common
65import rangelib
66import sparse_img
67
68OPTIONS = common.OPTIONS
69
70OPTIONS.add_missing = False
71OPTIONS.rebuild_recovery = False
72OPTIONS.replace_verity_public_key = False
73OPTIONS.replace_verity_private_key = False
74OPTIONS.is_signing = False
75
76
77class OutputFile(object):
78  def __init__(self, output_zip, input_dir, prefix, name):
79    self._output_zip = output_zip
80    self.input_name = os.path.join(input_dir, prefix, name)
81
82    if self._output_zip:
83      self._zip_name = os.path.join(prefix, name)
84
85      root, suffix = os.path.splitext(name)
86      self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix)
87    else:
88      self.name = self.input_name
89
90  def Write(self):
91    if self._output_zip:
92      common.ZipWrite(self._output_zip, self.name, self._zip_name)
93
94
95def GetCareMap(which, imgname):
96  """Generate care_map of system (or vendor) partition"""
97
98  assert which in ("system", "vendor")
99
100  simg = sparse_img.SparseImage(imgname)
101  care_map_list = []
102  care_map_list.append(which)
103
104  care_map_ranges = simg.care_map
105  key = which + "_adjusted_partition_size"
106  adjusted_blocks = OPTIONS.info_dict.get(key)
107  if adjusted_blocks:
108    assert adjusted_blocks > 0, "blocks should be positive for " + which
109    care_map_ranges = care_map_ranges.intersect(rangelib.RangeSet(
110        "0-%d" % (adjusted_blocks,)))
111
112  care_map_list.append(care_map_ranges.to_string_raw())
113  return care_map_list
114
115
116def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None):
117  """Turn the contents of SYSTEM into a system image and store it in
118  output_zip. Returns the name of the system image file."""
119
120  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img")
121  if os.path.exists(img.input_name):
122    print("system.img already exists in %s, no need to rebuild..." % (prefix,))
123    return img.input_name
124
125  def output_sink(fn, data):
126    ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w")
127    ofile.write(data)
128    ofile.close()
129
130  if OPTIONS.rebuild_recovery:
131    print("Building new recovery patch")
132    common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img,
133                             boot_img, info_dict=OPTIONS.info_dict)
134
135  block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map")
136  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img,
137              block_list=block_list)
138
139  return img.name
140
141
142def AddSystemOther(output_zip, prefix="IMAGES/"):
143  """Turn the contents of SYSTEM_OTHER into a system_other image
144  and store it in output_zip."""
145
146  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img")
147  if os.path.exists(img.input_name):
148    print("system_other.img already exists in %s, no need to rebuild..." % (
149        prefix,))
150    return
151
152  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img)
153
154
155def AddVendor(output_zip, prefix="IMAGES/"):
156  """Turn the contents of VENDOR into a vendor image and store in it
157  output_zip."""
158
159  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img")
160  if os.path.exists(img.input_name):
161    print("vendor.img already exists in %s, no need to rebuild..." % (prefix,))
162    return img.input_name
163
164  block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map")
165  CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img,
166              block_list=block_list)
167  return img.name
168
169
170def CreateImage(input_dir, info_dict, what, output_file, block_list=None):
171  print("creating " + what + ".img...")
172
173  # The name of the directory it is making an image out of matters to
174  # mkyaffs2image.  It wants "system" but we have a directory named
175  # "SYSTEM", so create a symlink.
176  temp_dir = tempfile.mkdtemp()
177  OPTIONS.tempfiles.append(temp_dir)
178  try:
179    os.symlink(os.path.join(input_dir, what.upper()),
180               os.path.join(temp_dir, what))
181  except OSError as e:
182    # bogus error on my mac version?
183    #   File "./build/tools/releasetools/img_from_target_files"
184    #     os.path.join(OPTIONS.input_tmp, "system"))
185    # OSError: [Errno 17] File exists
186    if e.errno == errno.EEXIST:
187      pass
188
189  image_props = build_image.ImagePropFromGlobalDict(info_dict, what)
190  fstab = info_dict["fstab"]
191  mount_point = "/" + what
192  if fstab and mount_point in fstab:
193    image_props["fs_type"] = fstab[mount_point].fs_type
194
195  # Use a fixed timestamp (01/01/2009) when packaging the image.
196  # Bug: 24377993
197  epoch = datetime.datetime.fromtimestamp(0)
198  timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
199  image_props["timestamp"] = int(timestamp)
200
201  if what == "system":
202    fs_config_prefix = ""
203  else:
204    fs_config_prefix = what + "_"
205
206  fs_config = os.path.join(
207      input_dir, "META/" + fs_config_prefix + "filesystem_config.txt")
208  if not os.path.exists(fs_config):
209    fs_config = None
210
211  # Override values loaded from info_dict.
212  if fs_config:
213    image_props["fs_config"] = fs_config
214  if block_list:
215    image_props["block_list"] = block_list.name
216
217  succ = build_image.BuildImage(os.path.join(temp_dir, what),
218                                image_props, output_file.name)
219  assert succ, "build " + what + ".img image failed"
220
221  output_file.Write()
222  if block_list:
223    block_list.Write()
224
225  is_verity_partition = "verity_block_device" in image_props
226  verity_supported = image_props.get("verity") == "true"
227  if is_verity_partition and verity_supported:
228    adjusted_blocks_value = image_props.get("partition_size")
229    if adjusted_blocks_value:
230      adjusted_blocks_key = what + "_adjusted_partition_size"
231      info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1
232
233
234def AddUserdata(output_zip, prefix="IMAGES/"):
235  """Create a userdata image and store it in output_zip.
236
237  In most case we just create and store an empty userdata.img;
238  But the invoker can also request to create userdata.img with real
239  data from the target files, by setting "userdata_img_with_data=true"
240  in OPTIONS.info_dict.
241  """
242
243  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img")
244  if os.path.exists(img.input_name):
245    print("userdata.img already exists in %s, no need to rebuild..." % (
246        prefix,))
247    return
248
249  # Skip userdata.img if no size.
250  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "data")
251  if not image_props.get("partition_size"):
252    return
253
254  print("creating userdata.img...")
255
256  # Use a fixed timestamp (01/01/2009) when packaging the image.
257  # Bug: 24377993
258  epoch = datetime.datetime.fromtimestamp(0)
259  timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
260  image_props["timestamp"] = int(timestamp)
261
262  # The name of the directory it is making an image out of matters to
263  # mkyaffs2image.  So we create a temp dir, and within it we create an
264  # empty dir named "data", or a symlink to the DATA dir,
265  # and build the image from that.
266  temp_dir = tempfile.mkdtemp()
267  OPTIONS.tempfiles.append(temp_dir)
268  user_dir = os.path.join(temp_dir, "data")
269  empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true")
270  if empty:
271    # Create an empty dir.
272    os.mkdir(user_dir)
273  else:
274    # Symlink to the DATA dir.
275    os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"),
276               user_dir)
277
278  fstab = OPTIONS.info_dict["fstab"]
279  if fstab:
280    image_props["fs_type"] = fstab["/data"].fs_type
281  succ = build_image.BuildImage(user_dir, image_props, img.name)
282  assert succ, "build userdata.img image failed"
283
284  common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict)
285  img.Write()
286
287
288def AddVBMeta(output_zip, boot_img_path, system_img_path, vendor_img_path,
289              prefix="IMAGES/"):
290  """Create a VBMeta image and store it in output_zip."""
291  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img")
292  avbtool = os.getenv('AVBTOOL') or "avbtool"
293  cmd = [avbtool, "make_vbmeta_image",
294         "--output", img.name,
295         "--include_descriptors_from_image", boot_img_path,
296         "--include_descriptors_from_image", system_img_path]
297  if vendor_img_path is not None:
298    cmd.extend(["--include_descriptors_from_image", vendor_img_path])
299  if OPTIONS.info_dict.get("system_root_image", None) == "true":
300    cmd.extend(["--setup_rootfs_from_kernel", system_img_path])
301  common.AppendAVBSigningArgs(cmd)
302  args = OPTIONS.info_dict.get("board_avb_make_vbmeta_image_args", None)
303  if args and args.strip():
304    cmd.extend(shlex.split(args))
305  p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
306  p.communicate()
307  assert p.returncode == 0, "avbtool make_vbmeta_image failed"
308  img.Write()
309
310
311def AddPartitionTable(output_zip, prefix="IMAGES/"):
312  """Create a partition table image and store it in output_zip."""
313
314  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img")
315  bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt")
316
317  # use BPTTOOL from environ, or "bpttool" if empty or not set.
318  bpttool = os.getenv("BPTTOOL") or "bpttool"
319  cmd = [bpttool, "make_table", "--output_json", bpt.name,
320         "--output_gpt", img.name]
321  input_files_str = OPTIONS.info_dict["board_bpt_input_files"]
322  input_files = input_files_str.split(" ")
323  for i in input_files:
324    cmd.extend(["--input", i])
325  disk_size = OPTIONS.info_dict.get("board_bpt_disk_size")
326  if disk_size:
327    cmd.extend(["--disk_size", disk_size])
328  args = OPTIONS.info_dict.get("board_bpt_make_table_args")
329  if args:
330    cmd.extend(shlex.split(args))
331
332  p = common.Run(cmd, stdout=subprocess.PIPE)
333  p.communicate()
334  assert p.returncode == 0, "bpttool make_table failed"
335
336  img.Write()
337  bpt.Write()
338
339
340def AddCache(output_zip, prefix="IMAGES/"):
341  """Create an empty cache image and store it in output_zip."""
342
343  img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img")
344  if os.path.exists(img.input_name):
345    print("cache.img already exists in %s, no need to rebuild..." % (prefix,))
346    return
347
348  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "cache")
349  # The build system has to explicitly request for cache.img.
350  if "fs_type" not in image_props:
351    return
352
353  print("creating cache.img...")
354
355  # Use a fixed timestamp (01/01/2009) when packaging the image.
356  # Bug: 24377993
357  epoch = datetime.datetime.fromtimestamp(0)
358  timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds()
359  image_props["timestamp"] = int(timestamp)
360
361  # The name of the directory it is making an image out of matters to
362  # mkyaffs2image.  So we create a temp dir, and within it we create an
363  # empty dir named "cache", and build the image from that.
364  temp_dir = tempfile.mkdtemp()
365  OPTIONS.tempfiles.append(temp_dir)
366  user_dir = os.path.join(temp_dir, "cache")
367  os.mkdir(user_dir)
368
369  fstab = OPTIONS.info_dict["fstab"]
370  if fstab:
371    image_props["fs_type"] = fstab["/cache"].fs_type
372  succ = build_image.BuildImage(user_dir, image_props, img.name)
373  assert succ, "build cache.img image failed"
374
375  common.CheckSize(img.name, "cache.img", OPTIONS.info_dict)
376  img.Write()
377
378
379def AddImagesToTargetFiles(filename):
380  if os.path.isdir(filename):
381    OPTIONS.input_tmp = os.path.abspath(filename)
382    input_zip = None
383  else:
384    OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename)
385
386  if not OPTIONS.add_missing:
387    if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")):
388      print("target_files appears to already contain images.")
389      sys.exit(1)
390
391  has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR"))
392  has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp,
393                                                "SYSTEM_OTHER"))
394
395  if input_zip:
396    OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp)
397
398    common.ZipClose(input_zip)
399    output_zip = zipfile.ZipFile(filename, "a",
400                                 compression=zipfile.ZIP_DEFLATED,
401                                 allowZip64=True)
402  else:
403    OPTIONS.info_dict = common.LoadInfoDict(filename, filename)
404    output_zip = None
405    images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES")
406    if not os.path.isdir(images_dir):
407      os.makedirs(images_dir)
408    images_dir = None
409
410  has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true")
411
412  def banner(s):
413    print("\n\n++++ " + s + " ++++\n\n")
414
415  prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "boot.img")
416  boot_image = None
417  if os.path.exists(prebuilt_path):
418    banner("boot")
419    print("boot.img already exists in IMAGES/, no need to rebuild...")
420    if OPTIONS.rebuild_recovery:
421      boot_image = common.GetBootableImage(
422          "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
423  else:
424    banner("boot")
425    boot_image = common.GetBootableImage(
426        "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT")
427    if boot_image:
428      if output_zip:
429        boot_image.AddToZip(output_zip)
430      else:
431        boot_image.WriteToDir(OPTIONS.input_tmp)
432
433  recovery_image = None
434  if has_recovery:
435    banner("recovery")
436    prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "recovery.img")
437    if os.path.exists(prebuilt_path):
438      print("recovery.img already exists in IMAGES/, no need to rebuild...")
439      if OPTIONS.rebuild_recovery:
440        recovery_image = common.GetBootableImage(
441            "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp,
442            "RECOVERY")
443    else:
444      recovery_image = common.GetBootableImage(
445          "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY")
446      if recovery_image:
447        if output_zip:
448          recovery_image.AddToZip(output_zip)
449        else:
450          recovery_image.WriteToDir(OPTIONS.input_tmp)
451
452      banner("recovery (two-step image)")
453      # The special recovery.img for two-step package use.
454      recovery_two_step_image = common.GetBootableImage(
455          "IMAGES/recovery-two-step.img", "recovery-two-step.img",
456          OPTIONS.input_tmp, "RECOVERY", two_step_image=True)
457      if recovery_two_step_image:
458        if output_zip:
459          recovery_two_step_image.AddToZip(output_zip)
460        else:
461          recovery_two_step_image.WriteToDir(OPTIONS.input_tmp)
462
463  banner("system")
464  system_img_path = AddSystem(
465    output_zip, recovery_img=recovery_image, boot_img=boot_image)
466  vendor_img_path = None
467  if has_vendor:
468    banner("vendor")
469    vendor_img_path = AddVendor(output_zip)
470  if has_system_other:
471    banner("system_other")
472    AddSystemOther(output_zip)
473  if not OPTIONS.is_signing:
474    banner("userdata")
475    AddUserdata(output_zip)
476    banner("cache")
477    AddCache(output_zip)
478  if OPTIONS.info_dict.get("board_bpt_enable", None) == "true":
479    banner("partition-table")
480    AddPartitionTable(output_zip)
481  if OPTIONS.info_dict.get("board_avb_enable", None) == "true":
482    banner("vbmeta")
483    boot_contents = boot_image.WriteToTemp()
484    AddVBMeta(output_zip, boot_contents.name, system_img_path, vendor_img_path)
485
486  # For devices using A/B update, copy over images from RADIO/ and/or
487  # VENDOR_IMAGES/ to IMAGES/ and make sure we have all the needed
488  # images ready under IMAGES/. All images should have '.img' as extension.
489  banner("radio")
490  ab_partitions = os.path.join(OPTIONS.input_tmp, "META", "ab_partitions.txt")
491  if os.path.exists(ab_partitions):
492    with open(ab_partitions, 'r') as f:
493      lines = f.readlines()
494    # For devices using A/B update, generate care_map for system and vendor
495    # partitions (if present), then write this file to target_files package.
496    care_map_list = []
497    for line in lines:
498      if line.strip() == "system" and OPTIONS.info_dict.get(
499          "system_verity_block_device", None) is not None:
500        assert os.path.exists(system_img_path)
501        care_map_list += GetCareMap("system", system_img_path)
502      if line.strip() == "vendor" and OPTIONS.info_dict.get(
503          "vendor_verity_block_device", None) is not None:
504        assert os.path.exists(vendor_img_path)
505        care_map_list += GetCareMap("vendor", vendor_img_path)
506
507      img_name = line.strip() + ".img"
508      prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)
509      if os.path.exists(prebuilt_path):
510        print("%s already exists, no need to overwrite..." % (img_name,))
511        continue
512
513      img_radio_path = os.path.join(OPTIONS.input_tmp, "RADIO", img_name)
514      img_vendor_dir = os.path.join(
515        OPTIONS.input_tmp, "VENDOR_IMAGES")
516      if os.path.exists(img_radio_path):
517        if output_zip:
518          common.ZipWrite(output_zip, img_radio_path,
519                          os.path.join("IMAGES", img_name))
520        else:
521          shutil.copy(img_radio_path, prebuilt_path)
522      else:
523        for root, _, files in os.walk(img_vendor_dir):
524          if img_name in files:
525            if output_zip:
526              common.ZipWrite(output_zip, os.path.join(root, img_name),
527                os.path.join("IMAGES", img_name))
528            else:
529              shutil.copy(os.path.join(root, img_name), prebuilt_path)
530            break
531
532      if output_zip:
533        # Zip spec says: All slashes MUST be forward slashes.
534        img_path = 'IMAGES/' + img_name
535        assert img_path in output_zip.namelist(), "cannot find " + img_name
536      else:
537        img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name)
538        assert os.path.exists(img_path), "cannot find " + img_name
539
540    if care_map_list:
541      file_path = "META/care_map.txt"
542      if output_zip:
543        common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list))
544      else:
545        with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp:
546          fp.write('\n'.join(care_map_list))
547
548  if output_zip:
549    common.ZipClose(output_zip)
550
551def main(argv):
552  def option_handler(o, a):
553    if o in ("-a", "--add_missing"):
554      OPTIONS.add_missing = True
555    elif o in ("-r", "--rebuild_recovery",):
556      OPTIONS.rebuild_recovery = True
557    elif o == "--replace_verity_private_key":
558      OPTIONS.replace_verity_private_key = (True, a)
559    elif o == "--replace_verity_public_key":
560      OPTIONS.replace_verity_public_key = (True, a)
561    elif o == "--is_signing":
562      OPTIONS.is_signing = True
563    else:
564      return False
565    return True
566
567  args = common.ParseOptions(
568      argv, __doc__, extra_opts="ar",
569      extra_long_opts=["add_missing", "rebuild_recovery",
570                       "replace_verity_public_key=",
571                       "replace_verity_private_key=",
572                       "is_signing"],
573      extra_option_handler=option_handler)
574
575
576  if len(args) != 1:
577    common.Usage(__doc__)
578    sys.exit(1)
579
580  AddImagesToTargetFiles(args[0])
581  print("done.")
582
583if __name__ == '__main__':
584  try:
585    common.CloseInheritedPipes()
586    main(sys.argv[1:])
587  except common.ExternalError as e:
588    print("\n   ERROR: %s\n" % (e,))
589    sys.exit(1)
590  finally:
591    common.Cleanup()
592