add_img_to_target_files.py revision 8ee4a3db8c76656b0d8bfd6ce8490451e3c10620
1#!/usr/bin/env python 2# 3# Copyright (C) 2014 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile that does not contain images (ie, does 19not have an IMAGES/ top-level subdirectory), produce the images and 20add them to the zipfile. 21 22Usage: add_img_to_target_files [flag] target_files 23 24 -a (--add_missing) 25 Build and add missing images to "IMAGES/". If this option is 26 not specified, this script will simply exit when "IMAGES/" 27 directory exists in the target file. 28 29 -r (--rebuild_recovery) 30 Rebuild the recovery patch and write it to the system image. Only 31 meaningful when system image needs to be rebuilt. 32 33 --replace_verity_private_key 34 Replace the private key used for verity signing. (same as the option 35 in sign_target_files_apks) 36 37 --replace_verity_public_key 38 Replace the certificate (public key) used for verity verification. (same 39 as the option in sign_target_files_apks) 40 41 --is_signing 42 Skip building & adding the images for "userdata" and "cache" if we 43 are signing the target files. 44""" 45 46from __future__ import print_function 47 48import sys 49 50if sys.hexversion < 0x02070000: 51 print("Python 2.7 or newer is required.", file=sys.stderr) 52 sys.exit(1) 53 54import datetime 55import errno 56import os 57import shlex 58import shutil 59import subprocess 60import tempfile 61import zipfile 62 63import build_image 64import common 65import rangelib 66import sparse_img 67 68OPTIONS = common.OPTIONS 69 70OPTIONS.add_missing = False 71OPTIONS.rebuild_recovery = False 72OPTIONS.replace_verity_public_key = False 73OPTIONS.replace_verity_private_key = False 74OPTIONS.is_signing = False 75 76 77class OutputFile(object): 78 def __init__(self, output_zip, input_dir, prefix, name): 79 self._output_zip = output_zip 80 self.input_name = os.path.join(input_dir, prefix, name) 81 82 if self._output_zip: 83 self._zip_name = os.path.join(prefix, name) 84 85 root, suffix = os.path.splitext(name) 86 self.name = common.MakeTempFile(prefix=root + '-', suffix=suffix) 87 else: 88 self.name = self.input_name 89 90 def Write(self): 91 if self._output_zip: 92 common.ZipWrite(self._output_zip, self.name, self._zip_name) 93 94 95def GetCareMap(which, imgname): 96 """Generate care_map of system (or vendor) partition""" 97 98 assert which in ("system", "vendor") 99 100 simg = sparse_img.SparseImage(imgname) 101 care_map_list = [] 102 care_map_list.append(which) 103 104 care_map_ranges = simg.care_map 105 key = which + "_adjusted_partition_size" 106 adjusted_blocks = OPTIONS.info_dict.get(key) 107 if adjusted_blocks: 108 assert adjusted_blocks > 0, "blocks should be positive for " + which 109 care_map_ranges = care_map_ranges.intersect(rangelib.RangeSet( 110 "0-%d" % (adjusted_blocks,))) 111 112 care_map_list.append(care_map_ranges.to_string_raw()) 113 return care_map_list 114 115 116def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): 117 """Turn the contents of SYSTEM into a system image and store it in 118 output_zip. Returns the name of the system image file.""" 119 120 img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.img") 121 if os.path.exists(img.input_name): 122 print("system.img already exists in %s, no need to rebuild..." % (prefix,)) 123 return img.input_name 124 125 def output_sink(fn, data): 126 ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w") 127 ofile.write(data) 128 ofile.close() 129 130 if OPTIONS.rebuild_recovery: 131 print("Building new recovery patch") 132 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img, 133 boot_img, info_dict=OPTIONS.info_dict) 134 135 block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system.map") 136 CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system", img, 137 block_list=block_list) 138 139 return img.name 140 141 142def AddSystemOther(output_zip, prefix="IMAGES/"): 143 """Turn the contents of SYSTEM_OTHER into a system_other image 144 and store it in output_zip.""" 145 146 img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "system_other.img") 147 if os.path.exists(img.input_name): 148 print("system_other.img already exists in %s, no need to rebuild..." % ( 149 prefix,)) 150 return 151 152 CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "system_other", img) 153 154 155def AddVendor(output_zip, prefix="IMAGES/"): 156 """Turn the contents of VENDOR into a vendor image and store in it 157 output_zip.""" 158 159 img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.img") 160 if os.path.exists(img.input_name): 161 print("vendor.img already exists in %s, no need to rebuild..." % (prefix,)) 162 return img.input_name 163 164 block_list = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vendor.map") 165 CreateImage(OPTIONS.input_tmp, OPTIONS.info_dict, "vendor", img, 166 block_list=block_list) 167 return img.name 168 169 170def CreateImage(input_dir, info_dict, what, output_file, block_list=None): 171 print("creating " + what + ".img...") 172 173 # The name of the directory it is making an image out of matters to 174 # mkyaffs2image. It wants "system" but we have a directory named 175 # "SYSTEM", so create a symlink. 176 temp_dir = tempfile.mkdtemp() 177 OPTIONS.tempfiles.append(temp_dir) 178 try: 179 os.symlink(os.path.join(input_dir, what.upper()), 180 os.path.join(temp_dir, what)) 181 except OSError as e: 182 # bogus error on my mac version? 183 # File "./build/tools/releasetools/img_from_target_files" 184 # os.path.join(OPTIONS.input_tmp, "system")) 185 # OSError: [Errno 17] File exists 186 if e.errno == errno.EEXIST: 187 pass 188 189 image_props = build_image.ImagePropFromGlobalDict(info_dict, what) 190 fstab = info_dict["fstab"] 191 mount_point = "/" + what 192 if fstab and mount_point in fstab: 193 image_props["fs_type"] = fstab[mount_point].fs_type 194 195 # Use a fixed timestamp (01/01/2009) when packaging the image. 196 # Bug: 24377993 197 epoch = datetime.datetime.fromtimestamp(0) 198 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 199 image_props["timestamp"] = int(timestamp) 200 201 if what == "system": 202 fs_config_prefix = "" 203 else: 204 fs_config_prefix = what + "_" 205 206 fs_config = os.path.join( 207 input_dir, "META/" + fs_config_prefix + "filesystem_config.txt") 208 if not os.path.exists(fs_config): 209 fs_config = None 210 211 # Override values loaded from info_dict. 212 if fs_config: 213 image_props["fs_config"] = fs_config 214 if block_list: 215 image_props["block_list"] = block_list.name 216 217 succ = build_image.BuildImage(os.path.join(temp_dir, what), 218 image_props, output_file.name) 219 assert succ, "build " + what + ".img image failed" 220 221 output_file.Write() 222 if block_list: 223 block_list.Write() 224 225 is_verity_partition = "verity_block_device" in image_props 226 verity_supported = image_props.get("verity") == "true" 227 if is_verity_partition and verity_supported: 228 adjusted_blocks_value = image_props.get("partition_size") 229 if adjusted_blocks_value: 230 adjusted_blocks_key = what + "_adjusted_partition_size" 231 info_dict[adjusted_blocks_key] = int(adjusted_blocks_value)/4096 - 1 232 233 234def AddUserdata(output_zip, prefix="IMAGES/"): 235 """Create a userdata image and store it in output_zip. 236 237 In most case we just create and store an empty userdata.img; 238 But the invoker can also request to create userdata.img with real 239 data from the target files, by setting "userdata_img_with_data=true" 240 in OPTIONS.info_dict. 241 """ 242 243 img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "userdata.img") 244 if os.path.exists(img.input_name): 245 print("userdata.img already exists in %s, no need to rebuild..." % ( 246 prefix,)) 247 return 248 249 # Skip userdata.img if no size. 250 image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "data") 251 if not image_props.get("partition_size"): 252 return 253 254 print("creating userdata.img...") 255 256 # Use a fixed timestamp (01/01/2009) when packaging the image. 257 # Bug: 24377993 258 epoch = datetime.datetime.fromtimestamp(0) 259 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 260 image_props["timestamp"] = int(timestamp) 261 262 # The name of the directory it is making an image out of matters to 263 # mkyaffs2image. So we create a temp dir, and within it we create an 264 # empty dir named "data", or a symlink to the DATA dir, 265 # and build the image from that. 266 temp_dir = tempfile.mkdtemp() 267 OPTIONS.tempfiles.append(temp_dir) 268 user_dir = os.path.join(temp_dir, "data") 269 empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true") 270 if empty: 271 # Create an empty dir. 272 os.mkdir(user_dir) 273 else: 274 # Symlink to the DATA dir. 275 os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"), 276 user_dir) 277 278 fstab = OPTIONS.info_dict["fstab"] 279 if fstab: 280 image_props["fs_type"] = fstab["/data"].fs_type 281 succ = build_image.BuildImage(user_dir, image_props, img.name) 282 assert succ, "build userdata.img image failed" 283 284 common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict) 285 img.Write() 286 287 288def AddVBMeta(output_zip, boot_img_path, system_img_path, vendor_img_path, 289 prefix="IMAGES/"): 290 """Create a VBMeta image and store it in output_zip.""" 291 img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "vbmeta.img") 292 avbtool = os.getenv('AVBTOOL') or "avbtool" 293 cmd = [avbtool, "make_vbmeta_image", 294 "--output", img.name, 295 "--include_descriptors_from_image", boot_img_path, 296 "--include_descriptors_from_image", system_img_path, 297 "--generate_dm_verity_cmdline_from_hashtree", system_img_path] 298 if vendor_img_path is not None: 299 cmd.extend(["--include_descriptors_from_image", vendor_img_path]) 300 common.AppendAVBSigningArgs(cmd) 301 args = OPTIONS.info_dict.get("board_avb_make_vbmeta_image_args", None) 302 if args and args.strip(): 303 cmd.extend(shlex.split(args)) 304 p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 305 p.communicate() 306 assert p.returncode == 0, "avbtool make_vbmeta_image failed" 307 img.Write() 308 309 310def AddPartitionTable(output_zip, prefix="IMAGES/"): 311 """Create a partition table image and store it in output_zip.""" 312 313 img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.img") 314 bpt = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "partition-table.bpt") 315 316 # use BPTTOOL from environ, or "bpttool" if empty or not set. 317 bpttool = os.getenv("BPTTOOL") or "bpttool" 318 cmd = [bpttool, "make_table", "--output_json", bpt.name, 319 "--output_gpt", img.name] 320 input_files_str = OPTIONS.info_dict["board_bpt_input_files"] 321 input_files = input_files_str.split(" ") 322 for i in input_files: 323 cmd.extend(["--input", i]) 324 disk_size = OPTIONS.info_dict.get("board_bpt_disk_size") 325 if disk_size: 326 cmd.extend(["--disk_size", disk_size]) 327 args = OPTIONS.info_dict.get("board_bpt_make_table_args") 328 if args: 329 cmd.extend(shlex.split(args)) 330 331 p = common.Run(cmd, stdout=subprocess.PIPE) 332 p.communicate() 333 assert p.returncode == 0, "bpttool make_table failed" 334 335 img.Write() 336 bpt.Write() 337 338 339def AddCache(output_zip, prefix="IMAGES/"): 340 """Create an empty cache image and store it in output_zip.""" 341 342 img = OutputFile(output_zip, OPTIONS.input_tmp, prefix, "cache.img") 343 if os.path.exists(img.input_name): 344 print("cache.img already exists in %s, no need to rebuild..." % (prefix,)) 345 return 346 347 image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "cache") 348 # The build system has to explicitly request for cache.img. 349 if "fs_type" not in image_props: 350 return 351 352 print("creating cache.img...") 353 354 # Use a fixed timestamp (01/01/2009) when packaging the image. 355 # Bug: 24377993 356 epoch = datetime.datetime.fromtimestamp(0) 357 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 358 image_props["timestamp"] = int(timestamp) 359 360 # The name of the directory it is making an image out of matters to 361 # mkyaffs2image. So we create a temp dir, and within it we create an 362 # empty dir named "cache", and build the image from that. 363 temp_dir = tempfile.mkdtemp() 364 OPTIONS.tempfiles.append(temp_dir) 365 user_dir = os.path.join(temp_dir, "cache") 366 os.mkdir(user_dir) 367 368 fstab = OPTIONS.info_dict["fstab"] 369 if fstab: 370 image_props["fs_type"] = fstab["/cache"].fs_type 371 succ = build_image.BuildImage(user_dir, image_props, img.name) 372 assert succ, "build cache.img image failed" 373 374 common.CheckSize(img.name, "cache.img", OPTIONS.info_dict) 375 img.Write() 376 377 378def AddImagesToTargetFiles(filename): 379 if os.path.isdir(filename): 380 OPTIONS.input_tmp = os.path.abspath(filename) 381 input_zip = None 382 else: 383 OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) 384 385 if not OPTIONS.add_missing: 386 if os.path.isdir(os.path.join(OPTIONS.input_tmp, "IMAGES")): 387 print("target_files appears to already contain images.") 388 sys.exit(1) 389 390 has_vendor = os.path.isdir(os.path.join(OPTIONS.input_tmp, "VENDOR")) 391 has_system_other = os.path.isdir(os.path.join(OPTIONS.input_tmp, 392 "SYSTEM_OTHER")) 393 394 if input_zip: 395 OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) 396 397 common.ZipClose(input_zip) 398 output_zip = zipfile.ZipFile(filename, "a", 399 compression=zipfile.ZIP_DEFLATED, 400 allowZip64=True) 401 else: 402 OPTIONS.info_dict = common.LoadInfoDict(filename, filename) 403 output_zip = None 404 images_dir = os.path.join(OPTIONS.input_tmp, "IMAGES") 405 if not os.path.isdir(images_dir): 406 os.makedirs(images_dir) 407 images_dir = None 408 409 has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true") 410 411 def banner(s): 412 print("\n\n++++ " + s + " ++++\n\n") 413 414 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "boot.img") 415 boot_image = None 416 if os.path.exists(prebuilt_path): 417 banner("boot") 418 print("boot.img already exists in IMAGES/, no need to rebuild...") 419 if OPTIONS.rebuild_recovery: 420 boot_image = common.GetBootableImage( 421 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 422 else: 423 banner("boot") 424 boot_image = common.GetBootableImage( 425 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 426 if boot_image: 427 if output_zip: 428 boot_image.AddToZip(output_zip) 429 else: 430 boot_image.WriteToDir(OPTIONS.input_tmp) 431 432 recovery_image = None 433 if has_recovery: 434 banner("recovery") 435 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "recovery.img") 436 if os.path.exists(prebuilt_path): 437 print("recovery.img already exists in IMAGES/, no need to rebuild...") 438 if OPTIONS.rebuild_recovery: 439 recovery_image = common.GetBootableImage( 440 "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, 441 "RECOVERY") 442 else: 443 recovery_image = common.GetBootableImage( 444 "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") 445 if recovery_image: 446 if output_zip: 447 recovery_image.AddToZip(output_zip) 448 else: 449 recovery_image.WriteToDir(OPTIONS.input_tmp) 450 451 banner("recovery (two-step image)") 452 # The special recovery.img for two-step package use. 453 recovery_two_step_image = common.GetBootableImage( 454 "IMAGES/recovery-two-step.img", "recovery-two-step.img", 455 OPTIONS.input_tmp, "RECOVERY", two_step_image=True) 456 if recovery_two_step_image: 457 if output_zip: 458 recovery_two_step_image.AddToZip(output_zip) 459 else: 460 recovery_two_step_image.WriteToDir(OPTIONS.input_tmp) 461 462 banner("system") 463 system_img_path = AddSystem( 464 output_zip, recovery_img=recovery_image, boot_img=boot_image) 465 vendor_img_path = None 466 if has_vendor: 467 banner("vendor") 468 vendor_img_path = AddVendor(output_zip) 469 if has_system_other: 470 banner("system_other") 471 AddSystemOther(output_zip) 472 if not OPTIONS.is_signing: 473 banner("userdata") 474 AddUserdata(output_zip) 475 banner("cache") 476 AddCache(output_zip) 477 if OPTIONS.info_dict.get("board_bpt_enable", None) == "true": 478 banner("partition-table") 479 AddPartitionTable(output_zip) 480 if OPTIONS.info_dict.get("board_avb_enable", None) == "true": 481 banner("vbmeta") 482 boot_contents = boot_image.WriteToTemp() 483 AddVBMeta(output_zip, boot_contents.name, system_img_path, vendor_img_path) 484 485 # For devices using A/B update, copy over images from RADIO/ and/or 486 # VENDOR_IMAGES/ to IMAGES/ and make sure we have all the needed 487 # images ready under IMAGES/. All images should have '.img' as extension. 488 banner("radio") 489 ab_partitions = os.path.join(OPTIONS.input_tmp, "META", "ab_partitions.txt") 490 if os.path.exists(ab_partitions): 491 with open(ab_partitions, 'r') as f: 492 lines = f.readlines() 493 # For devices using A/B update, generate care_map for system and vendor 494 # partitions (if present), then write this file to target_files package. 495 care_map_list = [] 496 for line in lines: 497 if line.strip() == "system" and OPTIONS.info_dict.get( 498 "system_verity_block_device", None) is not None: 499 assert os.path.exists(system_img_path) 500 care_map_list += GetCareMap("system", system_img_path) 501 if line.strip() == "vendor" and OPTIONS.info_dict.get( 502 "vendor_verity_block_device", None) is not None: 503 assert os.path.exists(vendor_img_path) 504 care_map_list += GetCareMap("vendor", vendor_img_path) 505 506 img_name = line.strip() + ".img" 507 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name) 508 if os.path.exists(prebuilt_path): 509 print("%s already exists, no need to overwrite..." % (img_name,)) 510 continue 511 512 img_radio_path = os.path.join(OPTIONS.input_tmp, "RADIO", img_name) 513 img_vendor_dir = os.path.join( 514 OPTIONS.input_tmp, "VENDOR_IMAGES") 515 if os.path.exists(img_radio_path): 516 if output_zip: 517 common.ZipWrite(output_zip, img_radio_path, 518 os.path.join("IMAGES", img_name)) 519 else: 520 shutil.copy(img_radio_path, prebuilt_path) 521 else: 522 for root, _, files in os.walk(img_vendor_dir): 523 if img_name in files: 524 if output_zip: 525 common.ZipWrite(output_zip, os.path.join(root, img_name), 526 os.path.join("IMAGES", img_name)) 527 else: 528 shutil.copy(os.path.join(root, img_name), prebuilt_path) 529 break 530 531 if output_zip: 532 # Zip spec says: All slashes MUST be forward slashes. 533 img_path = 'IMAGES/' + img_name 534 assert img_path in output_zip.namelist(), "cannot find " + img_name 535 else: 536 img_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name) 537 assert os.path.exists(img_path), "cannot find " + img_name 538 539 if care_map_list: 540 file_path = "META/care_map.txt" 541 if output_zip: 542 common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) 543 else: 544 with open(os.path.join(OPTIONS.input_tmp, file_path), 'w') as fp: 545 fp.write('\n'.join(care_map_list)) 546 547 if output_zip: 548 common.ZipClose(output_zip) 549 550def main(argv): 551 def option_handler(o, a): 552 if o in ("-a", "--add_missing"): 553 OPTIONS.add_missing = True 554 elif o in ("-r", "--rebuild_recovery",): 555 OPTIONS.rebuild_recovery = True 556 elif o == "--replace_verity_private_key": 557 OPTIONS.replace_verity_private_key = (True, a) 558 elif o == "--replace_verity_public_key": 559 OPTIONS.replace_verity_public_key = (True, a) 560 elif o == "--is_signing": 561 OPTIONS.is_signing = True 562 else: 563 return False 564 return True 565 566 args = common.ParseOptions( 567 argv, __doc__, extra_opts="ar", 568 extra_long_opts=["add_missing", "rebuild_recovery", 569 "replace_verity_public_key=", 570 "replace_verity_private_key=", 571 "is_signing"], 572 extra_option_handler=option_handler) 573 574 575 if len(args) != 1: 576 common.Usage(__doc__) 577 sys.exit(1) 578 579 AddImagesToTargetFiles(args[0]) 580 print("done.") 581 582if __name__ == '__main__': 583 try: 584 common.CloseInheritedPipes() 585 main(sys.argv[1:]) 586 except common.ExternalError as e: 587 print("\n ERROR: %s\n" % (e,)) 588 sys.exit(1) 589 finally: 590 common.Cleanup() 591