add_img_to_target_files.py revision d07f9995b823157c7763c7a73b335bda99e603b0
1#!/usr/bin/env python 2# 3# Copyright (C) 2014 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Given a target-files zipfile that does not contain images (ie, does 19not have an IMAGES/ top-level subdirectory), produce the images and 20add them to the zipfile. 21 22Usage: add_img_to_target_files [flag] target_files 23 24 -a (--add_missing) 25 Build and add missing images to "IMAGES/". If this option is 26 not specified, this script will simply exit when "IMAGES/" 27 directory exists in the target file. 28 29 -r (--rebuild_recovery) 30 Rebuild the recovery patch and write it to the system image. Only 31 meaningful when system image needs to be rebuilt. 32 33 --replace_verity_private_key 34 Replace the private key used for verity signing. (same as the option 35 in sign_target_files_apks) 36 37 --replace_verity_public_key 38 Replace the certificate (public key) used for verity verification. (same 39 as the option in sign_target_files_apks) 40 41 --is_signing 42 Skip building & adding the images for "userdata" and "cache" if we 43 are signing the target files. 44""" 45 46import sys 47 48if sys.hexversion < 0x02070000: 49 print >> sys.stderr, "Python 2.7 or newer is required." 50 sys.exit(1) 51 52import datetime 53import errno 54import os 55import shlex 56import shutil 57import subprocess 58import tempfile 59import zipfile 60 61import build_image 62import common 63import sparse_img 64 65OPTIONS = common.OPTIONS 66 67OPTIONS.add_missing = False 68OPTIONS.rebuild_recovery = False 69OPTIONS.replace_verity_public_key = False 70OPTIONS.replace_verity_private_key = False 71OPTIONS.is_signing = False 72 73def GetCareMap(which, imgname): 74 """Generate care_map of system (or vendor) partition""" 75 76 assert which in ("system", "vendor") 77 _, blk_device = common.GetTypeAndDevice("/" + which, OPTIONS.info_dict) 78 79 simg = sparse_img.SparseImage(imgname) 80 care_map_list = [] 81 care_map_list.append(blk_device) 82 care_map_list.append(simg.care_map.to_string_raw()) 83 return care_map_list 84 85 86def AddSystem(output_zip, prefix="IMAGES/", recovery_img=None, boot_img=None): 87 """Turn the contents of SYSTEM into a system image and store it in 88 output_zip. Returns the name of the system image file.""" 89 90 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system.img") 91 if os.path.exists(prebuilt_path): 92 print "system.img already exists in %s, no need to rebuild..." % (prefix,) 93 return prebuilt_path 94 95 def output_sink(fn, data): 96 ofile = open(os.path.join(OPTIONS.input_tmp, "SYSTEM", fn), "w") 97 ofile.write(data) 98 ofile.close() 99 100 if OPTIONS.rebuild_recovery: 101 print "Building new recovery patch" 102 common.MakeRecoveryPatch(OPTIONS.input_tmp, output_sink, recovery_img, 103 boot_img, info_dict=OPTIONS.info_dict) 104 105 block_list = common.MakeTempFile(prefix="system-blocklist-", suffix=".map") 106 imgname = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict, 107 block_list=block_list) 108 109 common.ZipWrite(output_zip, imgname, prefix + "system.img") 110 common.ZipWrite(output_zip, block_list, prefix + "system.map") 111 return imgname 112 113 114def BuildSystem(input_dir, info_dict, block_list=None): 115 """Build the (sparse) system image and return the name of a temp 116 file containing it.""" 117 return CreateImage(input_dir, info_dict, "system", block_list=block_list) 118 119 120def AddSystemOther(output_zip, prefix="IMAGES/"): 121 """Turn the contents of SYSTEM_OTHER into a system_other image 122 and store it in output_zip.""" 123 124 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "system_other.img") 125 if os.path.exists(prebuilt_path): 126 print "system_other.img already exists in %s, no need to rebuild..." % (prefix,) 127 return 128 129 imgname = BuildSystemOther(OPTIONS.input_tmp, OPTIONS.info_dict) 130 common.ZipWrite(output_zip, imgname, prefix + "system_other.img") 131 132def BuildSystemOther(input_dir, info_dict): 133 """Build the (sparse) system_other image and return the name of a temp 134 file containing it.""" 135 return CreateImage(input_dir, info_dict, "system_other", block_list=None) 136 137 138def AddVendor(output_zip, prefix="IMAGES/"): 139 """Turn the contents of VENDOR into a vendor image and store in it 140 output_zip.""" 141 142 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "vendor.img") 143 if os.path.exists(prebuilt_path): 144 print "vendor.img already exists in %s, no need to rebuild..." % (prefix,) 145 return prebuilt_path 146 147 block_list = common.MakeTempFile(prefix="vendor-blocklist-", suffix=".map") 148 imgname = BuildVendor(OPTIONS.input_tmp, OPTIONS.info_dict, 149 block_list=block_list) 150 common.ZipWrite(output_zip, imgname, prefix + "vendor.img") 151 common.ZipWrite(output_zip, block_list, prefix + "vendor.map") 152 return imgname 153 154 155def BuildVendor(input_dir, info_dict, block_list=None): 156 """Build the (sparse) vendor image and return the name of a temp 157 file containing it.""" 158 return CreateImage(input_dir, info_dict, "vendor", block_list=block_list) 159 160 161def CreateImage(input_dir, info_dict, what, block_list=None): 162 print "creating " + what + ".img..." 163 164 img = common.MakeTempFile(prefix=what + "-", suffix=".img") 165 166 # The name of the directory it is making an image out of matters to 167 # mkyaffs2image. It wants "system" but we have a directory named 168 # "SYSTEM", so create a symlink. 169 try: 170 os.symlink(os.path.join(input_dir, what.upper()), 171 os.path.join(input_dir, what)) 172 except OSError as e: 173 # bogus error on my mac version? 174 # File "./build/tools/releasetools/img_from_target_files" 175 # os.path.join(OPTIONS.input_tmp, "system")) 176 # OSError: [Errno 17] File exists 177 if e.errno == errno.EEXIST: 178 pass 179 180 image_props = build_image.ImagePropFromGlobalDict(info_dict, what) 181 fstab = info_dict["fstab"] 182 mount_point = "/" + what 183 if fstab and mount_point in fstab: 184 image_props["fs_type"] = fstab[mount_point].fs_type 185 186 # Use a fixed timestamp (01/01/2009) when packaging the image. 187 # Bug: 24377993 188 epoch = datetime.datetime.fromtimestamp(0) 189 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 190 image_props["timestamp"] = int(timestamp) 191 192 if what == "system": 193 fs_config_prefix = "" 194 else: 195 fs_config_prefix = what + "_" 196 197 fs_config = os.path.join( 198 input_dir, "META/" + fs_config_prefix + "filesystem_config.txt") 199 if not os.path.exists(fs_config): 200 fs_config = None 201 202 # Override values loaded from info_dict. 203 if fs_config: 204 image_props["fs_config"] = fs_config 205 if block_list: 206 image_props["block_list"] = block_list 207 208 succ = build_image.BuildImage(os.path.join(input_dir, what), 209 image_props, img) 210 assert succ, "build " + what + ".img image failed" 211 212 return img 213 214 215def AddUserdata(output_zip, prefix="IMAGES/"): 216 """Create a userdata image and store it in output_zip. 217 218 In most case we just create and store an empty userdata.img; 219 But the invoker can also request to create userdata.img with real 220 data from the target files, by setting "userdata_img_with_data=true" 221 in OPTIONS.info_dict. 222 """ 223 224 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "userdata.img") 225 if os.path.exists(prebuilt_path): 226 print "userdata.img already exists in %s, no need to rebuild..." % (prefix,) 227 return 228 229 # Skip userdata.img if no size. 230 image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "data") 231 if not image_props.get("partition_size"): 232 return 233 234 print "creating userdata.img..." 235 236 # Use a fixed timestamp (01/01/2009) when packaging the image. 237 # Bug: 24377993 238 epoch = datetime.datetime.fromtimestamp(0) 239 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 240 image_props["timestamp"] = int(timestamp) 241 242 # The name of the directory it is making an image out of matters to 243 # mkyaffs2image. So we create a temp dir, and within it we create an 244 # empty dir named "data", or a symlink to the DATA dir, 245 # and build the image from that. 246 temp_dir = tempfile.mkdtemp() 247 user_dir = os.path.join(temp_dir, "data") 248 empty = (OPTIONS.info_dict.get("userdata_img_with_data") != "true") 249 if empty: 250 # Create an empty dir. 251 os.mkdir(user_dir) 252 else: 253 # Symlink to the DATA dir. 254 os.symlink(os.path.join(OPTIONS.input_tmp, "DATA"), 255 user_dir) 256 257 img = tempfile.NamedTemporaryFile() 258 259 fstab = OPTIONS.info_dict["fstab"] 260 if fstab: 261 image_props["fs_type"] = fstab["/data"].fs_type 262 succ = build_image.BuildImage(user_dir, image_props, img.name) 263 assert succ, "build userdata.img image failed" 264 265 common.CheckSize(img.name, "userdata.img", OPTIONS.info_dict) 266 common.ZipWrite(output_zip, img.name, prefix + "userdata.img") 267 img.close() 268 shutil.rmtree(temp_dir) 269 270 271def AddVBMeta(output_zip, boot_img_path, system_img_path, prefix="IMAGES/"): 272 """Create a VBMeta image and store it in output_zip.""" 273 _, img_file_name = tempfile.mkstemp() 274 avbtool = os.getenv('AVBTOOL') or "avbtool" 275 cmd = [avbtool, "make_vbmeta_image", 276 "--output", img_file_name, 277 "--include_descriptors_from_image", boot_img_path, 278 "--include_descriptors_from_image", system_img_path, 279 "--generate_dm_verity_cmdline_from_hashtree", system_img_path] 280 common.AppendAVBSigningArgs(cmd) 281 args = OPTIONS.info_dict.get("board_avb_make_vbmeta_image_args", None) 282 if args and args.strip(): 283 cmd.extend(shlex.split(args)) 284 p = common.Run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 285 p.communicate() 286 assert p.returncode == 0, "avbtool make_vbmeta_image failed" 287 common.ZipWrite(output_zip, img_file_name, prefix + "vbmeta.img") 288 289 290def AddPartitionTable(output_zip, prefix="IMAGES/"): 291 """Create a partition table image and store it in output_zip.""" 292 293 _, img_file_name = tempfile.mkstemp() 294 _, bpt_file_name = tempfile.mkstemp() 295 296 # use BPTTOOL from environ, or "bpttool" if empty or not set. 297 bpttool = os.getenv("BPTTOOL") or "bpttool" 298 cmd = [bpttool, "make_table", "--output_json", bpt_file_name, 299 "--output_gpt", img_file_name] 300 input_files_str = OPTIONS.info_dict["board_bpt_input_files"] 301 input_files = input_files_str.split(" ") 302 for i in input_files: 303 cmd.extend(["--input", i]) 304 disk_size = OPTIONS.info_dict.get("board_bpt_disk_size") 305 if disk_size: 306 cmd.extend(["--disk_size", disk_size]) 307 args = OPTIONS.info_dict.get("board_bpt_make_table_args") 308 if args: 309 cmd.extend(shlex.split(args)) 310 311 p = common.Run(cmd, stdout=subprocess.PIPE) 312 p.communicate() 313 assert p.returncode == 0, "bpttool make_table failed" 314 315 common.ZipWrite(output_zip, img_file_name, prefix + "partition-table.img") 316 common.ZipWrite(output_zip, bpt_file_name, prefix + "partition-table.bpt") 317 318 319def AddCache(output_zip, prefix="IMAGES/"): 320 """Create an empty cache image and store it in output_zip.""" 321 322 prebuilt_path = os.path.join(OPTIONS.input_tmp, prefix, "cache.img") 323 if os.path.exists(prebuilt_path): 324 print "cache.img already exists in %s, no need to rebuild..." % (prefix,) 325 return 326 327 image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict, "cache") 328 # The build system has to explicitly request for cache.img. 329 if "fs_type" not in image_props: 330 return 331 332 print "creating cache.img..." 333 334 # Use a fixed timestamp (01/01/2009) when packaging the image. 335 # Bug: 24377993 336 epoch = datetime.datetime.fromtimestamp(0) 337 timestamp = (datetime.datetime(2009, 1, 1) - epoch).total_seconds() 338 image_props["timestamp"] = int(timestamp) 339 340 # The name of the directory it is making an image out of matters to 341 # mkyaffs2image. So we create a temp dir, and within it we create an 342 # empty dir named "cache", and build the image from that. 343 temp_dir = tempfile.mkdtemp() 344 user_dir = os.path.join(temp_dir, "cache") 345 os.mkdir(user_dir) 346 img = tempfile.NamedTemporaryFile() 347 348 fstab = OPTIONS.info_dict["fstab"] 349 if fstab: 350 image_props["fs_type"] = fstab["/cache"].fs_type 351 succ = build_image.BuildImage(user_dir, image_props, img.name) 352 assert succ, "build cache.img image failed" 353 354 common.CheckSize(img.name, "cache.img", OPTIONS.info_dict) 355 common.ZipWrite(output_zip, img.name, prefix + "cache.img") 356 img.close() 357 os.rmdir(user_dir) 358 os.rmdir(temp_dir) 359 360 361def AddImagesToTargetFiles(filename): 362 OPTIONS.input_tmp, input_zip = common.UnzipTemp(filename) 363 364 if not OPTIONS.add_missing: 365 for n in input_zip.namelist(): 366 if n.startswith("IMAGES/"): 367 print "target_files appears to already contain images." 368 sys.exit(1) 369 370 try: 371 input_zip.getinfo("VENDOR/") 372 has_vendor = True 373 except KeyError: 374 has_vendor = False 375 376 has_system_other = "SYSTEM_OTHER/" in input_zip.namelist() 377 378 OPTIONS.info_dict = common.LoadInfoDict(input_zip, OPTIONS.input_tmp) 379 380 common.ZipClose(input_zip) 381 output_zip = zipfile.ZipFile(filename, "a", 382 compression=zipfile.ZIP_DEFLATED, 383 allowZip64=True) 384 385 has_recovery = (OPTIONS.info_dict.get("no_recovery") != "true") 386 system_root_image = (OPTIONS.info_dict.get("system_root_image", None) == "true") 387 388 def banner(s): 389 print "\n\n++++ " + s + " ++++\n\n" 390 391 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "boot.img") 392 boot_image = None 393 if os.path.exists(prebuilt_path): 394 banner("boot") 395 print "boot.img already exists in IMAGES/, no need to rebuild..." 396 if OPTIONS.rebuild_recovery: 397 boot_image = common.GetBootableImage( 398 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 399 else: 400 banner("boot") 401 boot_image = common.GetBootableImage( 402 "IMAGES/boot.img", "boot.img", OPTIONS.input_tmp, "BOOT") 403 if boot_image: 404 boot_image.AddToZip(output_zip) 405 406 recovery_image = None 407 if has_recovery: 408 banner("recovery") 409 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", "recovery.img") 410 if os.path.exists(prebuilt_path): 411 print "recovery.img already exists in IMAGES/, no need to rebuild..." 412 if OPTIONS.rebuild_recovery: 413 recovery_image = common.GetBootableImage( 414 "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, 415 "RECOVERY") 416 else: 417 recovery_image = common.GetBootableImage( 418 "IMAGES/recovery.img", "recovery.img", OPTIONS.input_tmp, "RECOVERY") 419 if recovery_image: 420 recovery_image.AddToZip(output_zip) 421 422 banner("recovery (two-step image)") 423 # The special recovery.img for two-step package use. 424 recovery_two_step_image = common.GetBootableImage( 425 "IMAGES/recovery-two-step.img", "recovery-two-step.img", 426 OPTIONS.input_tmp, "RECOVERY", two_step_image=True) 427 if recovery_two_step_image: 428 recovery_two_step_image.AddToZip(output_zip) 429 430 banner("system") 431 system_img_path = AddSystem( 432 output_zip, recovery_img=recovery_image, boot_img=boot_image) 433 vendor_img_path = None 434 if has_vendor: 435 banner("vendor") 436 vendor_img_path = AddVendor(output_zip) 437 if has_system_other: 438 banner("system_other") 439 AddSystemOther(output_zip) 440 if not OPTIONS.is_signing: 441 banner("userdata") 442 AddUserdata(output_zip) 443 banner("cache") 444 AddCache(output_zip) 445 if OPTIONS.info_dict.get("board_bpt_enable", None) == "true": 446 banner("partition-table") 447 AddPartitionTable(output_zip) 448 if OPTIONS.info_dict.get("board_avb_enable", None) == "true": 449 banner("vbmeta") 450 boot_contents = boot_image.WriteToTemp() 451 AddVBMeta(output_zip, boot_contents.name, system_img_path) 452 453 # For devices using A/B update, copy over images from RADIO/ and/or 454 # VENDOR_IMAGES/ to IMAGES/ and make sure we have all the needed 455 # images ready under IMAGES/. All images should have '.img' as extension. 456 banner("radio") 457 ab_partitions = os.path.join(OPTIONS.input_tmp, "META", "ab_partitions.txt") 458 if os.path.exists(ab_partitions): 459 with open(ab_partitions, 'r') as f: 460 lines = f.readlines() 461 # For devices using A/B update, generate care_map for system and vendor 462 # partitions (if present), then write this file to target_files package. 463 care_map_list = [] 464 for line in lines: 465 if line.strip() == "system" and OPTIONS.info_dict.get( 466 "system_verity_block_device", None) is not None: 467 assert os.path.exists(system_img_path) 468 care_map_list += GetCareMap("system", system_img_path) 469 if line.strip() == "vendor" and OPTIONS.info_dict.get( 470 "vendor_verity_block_device", None) is not None: 471 assert os.path.exists(vendor_img_path) 472 care_map_list += GetCareMap("vendor", vendor_img_path) 473 474 img_name = line.strip() + ".img" 475 prebuilt_path = os.path.join(OPTIONS.input_tmp, "IMAGES", img_name) 476 if os.path.exists(prebuilt_path): 477 print "%s already exists, no need to overwrite..." % (img_name,) 478 continue 479 480 img_radio_path = os.path.join(OPTIONS.input_tmp, "RADIO", img_name) 481 img_vendor_dir = os.path.join( 482 OPTIONS.input_tmp, "VENDOR_IMAGES") 483 if os.path.exists(img_radio_path): 484 common.ZipWrite(output_zip, img_radio_path, 485 os.path.join("IMAGES", img_name)) 486 else: 487 for root, _, files in os.walk(img_vendor_dir): 488 if img_name in files: 489 common.ZipWrite(output_zip, os.path.join(root, img_name), 490 os.path.join("IMAGES", img_name)) 491 break 492 493 # Zip spec says: All slashes MUST be forward slashes. 494 img_path = 'IMAGES/' + img_name 495 assert img_path in output_zip.namelist(), "cannot find " + img_name 496 497 if care_map_list: 498 file_path = "META/care_map.txt" 499 common.ZipWriteStr(output_zip, file_path, '\n'.join(care_map_list)) 500 501 common.ZipClose(output_zip) 502 503def main(argv): 504 def option_handler(o, a): 505 if o in ("-a", "--add_missing"): 506 OPTIONS.add_missing = True 507 elif o in ("-r", "--rebuild_recovery",): 508 OPTIONS.rebuild_recovery = True 509 elif o == "--replace_verity_private_key": 510 OPTIONS.replace_verity_private_key = (True, a) 511 elif o == "--replace_verity_public_key": 512 OPTIONS.replace_verity_public_key = (True, a) 513 elif o == "--is_signing": 514 OPTIONS.is_signing = True 515 else: 516 return False 517 return True 518 519 args = common.ParseOptions( 520 argv, __doc__, extra_opts="ar", 521 extra_long_opts=["add_missing", "rebuild_recovery", 522 "replace_verity_public_key=", 523 "replace_verity_private_key=", 524 "is_signing"], 525 extra_option_handler=option_handler) 526 527 528 if len(args) != 1: 529 common.Usage(__doc__) 530 sys.exit(1) 531 532 AddImagesToTargetFiles(args[0]) 533 print "done." 534 535if __name__ == '__main__': 536 try: 537 common.CloseInheritedPipes() 538 main(sys.argv[1:]) 539 except common.ExternalError as e: 540 print 541 print " ERROR: %s" % (e,) 542 print 543 sys.exit(1) 544 finally: 545 common.Cleanup() 546