delta_diff_generator.cc revision f1cbe1783dd025bd7243a8df12f20548196cc023
1// Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "update_engine/payload_generator/delta_diff_generator.h" 6 7#include <errno.h> 8#include <fcntl.h> 9#include <inttypes.h> 10#include <sys/stat.h> 11#include <sys/types.h> 12 13#include <algorithm> 14#include <map> 15#include <memory> 16#include <string> 17#include <utility> 18#include <vector> 19 20#include <base/files/file_path.h> 21#include <base/files/file_util.h> 22#include <base/logging.h> 23#include <base/strings/stringprintf.h> 24#include <base/strings/string_number_conversions.h> 25#include <base/strings/string_util.h> 26#include <bzlib.h> 27 28#include "update_engine/bzip.h" 29#include "update_engine/delta_performer.h" 30#include "update_engine/file_writer.h" 31#include "update_engine/omaha_hash_calculator.h" 32#include "update_engine/payload_constants.h" 33#include "update_engine/payload_generator/extent_mapper.h" 34#include "update_engine/payload_generator/filesystem_iterator.h" 35#include "update_engine/payload_generator/full_update_generator.h" 36#include "update_engine/payload_generator/graph_types.h" 37#include "update_engine/payload_generator/graph_utils.h" 38#include "update_engine/payload_generator/inplace_generator.h" 39#include "update_engine/payload_generator/metadata.h" 40#include "update_engine/payload_generator/payload_signer.h" 41#include "update_engine/payload_verifier.h" 42#include "update_engine/subprocess.h" 43#include "update_engine/update_metadata.pb.h" 44#include "update_engine/utils.h" 45 46using std::map; 47using std::max; 48using std::min; 49using std::set; 50using std::string; 51using std::unique_ptr; 52using std::vector; 53 54namespace { 55 56const uint64_t kMajorVersionNumber = 1; 57 58// The maximum destination size allowed for bsdiff. In general, bsdiff should 59// work for arbitrary big files, but the payload generation and payload 60// application requires a significant amount of RAM. We put a hard-limit of 61// 200 MiB that should not affect any released board, but will limit the 62// Chrome binary in ASan builders. 63const off_t kMaxBsdiffDestinationSize = 200 * 1024 * 1024; // bytes 64 65static const char* kInstallOperationTypes[] = { 66 "REPLACE", 67 "REPLACE_BZ", 68 "MOVE", 69 "BSDIFF", 70 "SOURCE_COPY", 71 "SOURCE_BSDIFF" 72}; 73 74} // namespace 75 76namespace chromeos_update_engine { 77 78typedef DeltaDiffGenerator::Block Block; 79typedef map<const DeltaArchiveManifest_InstallOperation*, 80 string> OperationNameMap; 81 82// bytes 83const size_t kRootFSPartitionSize = static_cast<size_t>(2) * 1024 * 1024 * 1024; 84const size_t kBlockSize = 4096; // bytes 85const char* const kEmptyPath = ""; 86const char* const kBsdiffPath = "bsdiff"; 87 88const uint32_t kInPlaceMinorPayloadVersion = 1; 89const uint32_t kSourceMinorPayloadVersion = 2; 90 91// Needed for testing purposes, in case we can't use actual filesystem objects. 92// TODO(garnold) (chromium:331965) Replace this hack with a properly injected 93// parameter in form of a mockable abstract class. 94bool (*get_extents_with_chunk_func)(const string&, off_t, off_t, 95 vector<Extent>*) = 96 extent_mapper::ExtentsForFileChunkFibmap; 97 98namespace { 99 100// Stores all the extents of |path| into |extents|. Returns true on success. 101bool GatherExtents(const string& path, 102 off_t chunk_offset, 103 off_t chunk_size, 104 vector<Extent>* extents) { 105 extents->clear(); 106 TEST_AND_RETURN_FALSE( 107 get_extents_with_chunk_func( 108 path, chunk_offset, chunk_size, extents)); 109 return true; 110} 111 112// For each regular file within new_root, creates a node in the graph, 113// determines the best way to compress it (REPLACE, REPLACE_BZ, COPY, BSDIFF), 114// and writes any necessary data to the end of data_fd. 115bool DeltaReadFiles(Graph* graph, 116 vector<Block>* blocks, 117 const string& old_root, 118 const string& new_root, 119 off_t chunk_size, 120 int data_fd, 121 off_t* data_file_size, 122 bool src_ops_allowed) { 123 set<ino_t> visited_inodes; 124 set<ino_t> visited_src_inodes; 125 for (FilesystemIterator fs_iter(new_root, 126 set<string>{"/lost+found"}); 127 !fs_iter.IsEnd(); fs_iter.Increment()) { 128 // We never diff symlinks (here, we check that dst file is not a symlink). 129 if (!S_ISREG(fs_iter.GetStat().st_mode)) 130 continue; 131 132 // Make sure we visit each inode only once. 133 if (utils::SetContainsKey(visited_inodes, fs_iter.GetStat().st_ino)) 134 continue; 135 visited_inodes.insert(fs_iter.GetStat().st_ino); 136 off_t dst_size = fs_iter.GetFileSize(); 137 if (dst_size == 0) 138 continue; 139 140 LOG(INFO) << "Encoding file " << fs_iter.GetPartialPath(); 141 142 // We can't visit each dst image inode more than once, as that would 143 // duplicate work. Here, we avoid visiting each source image inode 144 // more than once. Technically, we could have multiple operations 145 // that read the same blocks from the source image for diffing, but 146 // we choose not to avoid complexity. Eventually we will move away 147 // from using a graph/cycle detection/etc to generate diffs, and at that 148 // time, it will be easy (non-complex) to have many operations read 149 // from the same source blocks. At that time, this code can die. -adlr 150 bool should_diff_from_source = false; 151 string src_path = old_root + fs_iter.GetPartialPath(); 152 struct stat src_stbuf; 153 // We never diff symlinks (here, we check that src file is not a symlink). 154 if (0 == lstat(src_path.c_str(), &src_stbuf) && 155 S_ISREG(src_stbuf.st_mode)) { 156 should_diff_from_source = !utils::SetContainsKey(visited_src_inodes, 157 src_stbuf.st_ino); 158 visited_src_inodes.insert(src_stbuf.st_ino); 159 } 160 161 off_t size = chunk_size == -1 ? dst_size : chunk_size; 162 off_t step = size; 163 for (off_t offset = 0; offset < dst_size; offset += step) { 164 if (offset + size >= dst_size) { 165 size = -1; // Read through the end of the file. 166 } 167 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::DeltaReadFile( 168 graph, 169 Vertex::kInvalidIndex, 170 blocks, 171 (should_diff_from_source ? old_root : kEmptyPath), 172 new_root, 173 fs_iter.GetPartialPath(), 174 offset, 175 size, 176 data_fd, 177 data_file_size, 178 src_ops_allowed)); 179 } 180 } 181 return true; 182} 183 184// Reads blocks from image_path that are not yet marked as being written in the 185// blocks array. These blocks that remain are either unchanged files or 186// non-file-data blocks. We compare each of them to the old image, and compress 187// the ones that changed into a single REPLACE_BZ operation. This updates a 188// newly created node in the graph to write these blocks and writes the 189// appropriate blob to blobs_fd. Reads and updates blobs_length. 190bool ReadUnwrittenBlocks(const vector<Block>& blocks, 191 int blobs_fd, 192 off_t* blobs_length, 193 const string& old_image_path, 194 const string& new_image_path, 195 Vertex* vertex) { 196 vertex->file_name = "<rootfs-non-file-data>"; 197 198 DeltaArchiveManifest_InstallOperation* out_op = &vertex->op; 199 int new_image_fd = open(new_image_path.c_str(), O_RDONLY, 000); 200 TEST_AND_RETURN_FALSE_ERRNO(new_image_fd >= 0); 201 ScopedFdCloser new_image_fd_closer(&new_image_fd); 202 int old_image_fd = open(old_image_path.c_str(), O_RDONLY, 000); 203 TEST_AND_RETURN_FALSE_ERRNO(old_image_fd >= 0); 204 ScopedFdCloser old_image_fd_closer(&old_image_fd); 205 206 string temp_file_path; 207 TEST_AND_RETURN_FALSE(utils::MakeTempFile("CrAU_temp_data.XXXXXX", 208 &temp_file_path, 209 nullptr)); 210 211 FILE* file = fopen(temp_file_path.c_str(), "w"); 212 TEST_AND_RETURN_FALSE(file); 213 int err = BZ_OK; 214 215 BZFILE* bz_file = BZ2_bzWriteOpen(&err, 216 file, 217 9, // max compression 218 0, // verbosity 219 0); // default work factor 220 TEST_AND_RETURN_FALSE(err == BZ_OK); 221 222 vector<Extent> extents; 223 vector<Block>::size_type block_count = 0; 224 225 LOG(INFO) << "Appending unwritten blocks to extents"; 226 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { 227 if (blocks[i].writer != Vertex::kInvalidIndex) 228 continue; 229 graph_utils::AppendBlockToExtents(&extents, i); 230 block_count++; 231 } 232 233 // Code will handle buffers of any size that's a multiple of kBlockSize, 234 // so we arbitrarily set it to 1024 * kBlockSize. 235 chromeos::Blob new_buf(1024 * kBlockSize); 236 chromeos::Blob old_buf(1024 * kBlockSize); 237 238 LOG(INFO) << "Scanning " << block_count << " unwritten blocks"; 239 vector<Extent> changed_extents; 240 vector<Block>::size_type changed_block_count = 0; 241 vector<Block>::size_type blocks_copied_count = 0; 242 243 // For each extent in extents, write the unchanged blocks into BZ2_bzWrite, 244 // which sends it to an output file. We use the temporary buffers to hold the 245 // old and new data, which may be smaller than the extent, so in that case we 246 // have to loop to get the extent's data (that's the inner while loop). 247 for (const Extent& extent : extents) { 248 vector<Block>::size_type blocks_read = 0; 249 float printed_progress = -1; 250 while (blocks_read < extent.num_blocks()) { 251 const uint64_t copy_first_block = extent.start_block() + blocks_read; 252 const int copy_block_cnt = 253 min(new_buf.size() / kBlockSize, 254 static_cast<chromeos::Blob::size_type>( 255 extent.num_blocks() - blocks_read)); 256 const size_t count = copy_block_cnt * kBlockSize; 257 const off_t offset = copy_first_block * kBlockSize; 258 ssize_t rc = pread(new_image_fd, new_buf.data(), count, offset); 259 TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); 260 TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == count); 261 262 rc = pread(old_image_fd, old_buf.data(), count, offset); 263 TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); 264 TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == count); 265 266 // Compare each block in the buffer to its counterpart in the old image 267 // and only compress it if its content has changed. 268 int buf_offset = 0; 269 for (int i = 0; i < copy_block_cnt; ++i) { 270 int buf_end_offset = buf_offset + kBlockSize; 271 if (!std::equal(new_buf.begin() + buf_offset, 272 new_buf.begin() + buf_end_offset, 273 old_buf.begin() + buf_offset)) { 274 BZ2_bzWrite(&err, bz_file, &new_buf[buf_offset], kBlockSize); 275 TEST_AND_RETURN_FALSE(err == BZ_OK); 276 const uint64_t block_idx = copy_first_block + i; 277 if (blocks[block_idx].reader != Vertex::kInvalidIndex) { 278 graph_utils::AddReadBeforeDep(vertex, blocks[block_idx].reader, 279 block_idx); 280 } 281 graph_utils::AppendBlockToExtents(&changed_extents, block_idx); 282 changed_block_count++; 283 } 284 buf_offset = buf_end_offset; 285 } 286 287 blocks_read += copy_block_cnt; 288 blocks_copied_count += copy_block_cnt; 289 float current_progress = 290 static_cast<float>(blocks_copied_count) / block_count; 291 if (printed_progress + 0.1 < current_progress || 292 blocks_copied_count == block_count) { 293 LOG(INFO) << "progress: " << current_progress; 294 printed_progress = current_progress; 295 } 296 } 297 } 298 BZ2_bzWriteClose(&err, bz_file, 0, nullptr, nullptr); 299 TEST_AND_RETURN_FALSE(err == BZ_OK); 300 bz_file = nullptr; 301 TEST_AND_RETURN_FALSE_ERRNO(0 == fclose(file)); 302 file = nullptr; 303 304 LOG(INFO) << "Compressed " << changed_block_count << " blocks (" 305 << block_count - changed_block_count << " blocks unchanged)"; 306 chromeos::Blob compressed_data; 307 if (changed_block_count > 0) { 308 LOG(INFO) << "Reading compressed data off disk"; 309 TEST_AND_RETURN_FALSE(utils::ReadFile(temp_file_path, &compressed_data)); 310 } 311 TEST_AND_RETURN_FALSE(unlink(temp_file_path.c_str()) == 0); 312 313 // Add node to graph to write these blocks 314 out_op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); 315 out_op->set_data_offset(*blobs_length); 316 out_op->set_data_length(compressed_data.size()); 317 LOG(INFO) << "Rootfs non-data blocks compressed take up " 318 << compressed_data.size(); 319 *blobs_length += compressed_data.size(); 320 out_op->set_dst_length(kBlockSize * changed_block_count); 321 DeltaDiffGenerator::StoreExtents(changed_extents, 322 out_op->mutable_dst_extents()); 323 324 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, 325 compressed_data.data(), 326 compressed_data.size())); 327 LOG(INFO) << "Done processing unwritten blocks"; 328 return true; 329} 330 331// Writes the uint64_t passed in in host-endian to the file as big-endian. 332// Returns true on success. 333bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) { 334 uint64_t value_be = htobe64(value); 335 TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be))); 336 return true; 337} 338 339// Adds each operation from |graph| to |out_manifest| in the order specified by 340// |order| while building |out_op_name_map| with operation to name 341// mappings. Adds all |kernel_ops| to |out_manifest|. Filters out no-op 342// operations. 343void InstallOperationsToManifest( 344 const Graph& graph, 345 const vector<Vertex::Index>& order, 346 const vector<DeltaArchiveManifest_InstallOperation>& kernel_ops, 347 DeltaArchiveManifest* out_manifest, 348 OperationNameMap* out_op_name_map) { 349 for (Vertex::Index vertex_index : order) { 350 const Vertex& vertex = graph[vertex_index]; 351 const DeltaArchiveManifest_InstallOperation& add_op = vertex.op; 352 if (DeltaDiffGenerator::IsNoopOperation(add_op)) { 353 continue; 354 } 355 DeltaArchiveManifest_InstallOperation* op = 356 out_manifest->add_install_operations(); 357 *op = add_op; 358 string name = vertex.file_name; 359 if (vertex.chunk_offset || vertex.chunk_size != -1) { 360 string offset = base::Int64ToString(vertex.chunk_offset); 361 if (vertex.chunk_size != -1) { 362 name += " [" + offset + ", " + 363 base::Int64ToString(vertex.chunk_offset + vertex.chunk_size - 1) + 364 "]"; 365 } else { 366 name += " [" + offset + ", end]"; 367 } 368 } 369 (*out_op_name_map)[op] = name; 370 } 371 for (vector<DeltaArchiveManifest_InstallOperation>::const_iterator it = 372 kernel_ops.begin(); it != kernel_ops.end(); ++it) { 373 const DeltaArchiveManifest_InstallOperation& add_op = *it; 374 if (DeltaDiffGenerator::IsNoopOperation(add_op)) { 375 continue; 376 } 377 DeltaArchiveManifest_InstallOperation* op = 378 out_manifest->add_kernel_install_operations(); 379 *op = add_op; 380 } 381} 382 383// Delta compresses a kernel partition |new_kernel_part| with knowledge of the 384// old kernel partition |old_kernel_part|. If |old_kernel_part| is an empty 385// string, generates a full update of the partition. 386bool DeltaCompressKernelPartition( 387 const string& old_kernel_part, 388 const string& new_kernel_part, 389 vector<DeltaArchiveManifest_InstallOperation>* ops, 390 int blobs_fd, 391 off_t* blobs_length, 392 bool src_ops_allowed) { 393 LOG(INFO) << "Delta compressing kernel partition..."; 394 LOG_IF(INFO, old_kernel_part.empty()) << "Generating full kernel update..."; 395 396 DeltaArchiveManifest_InstallOperation op; 397 chromeos::Blob data; 398 TEST_AND_RETURN_FALSE( 399 DeltaDiffGenerator::ReadFileToDiff(old_kernel_part, 400 new_kernel_part, 401 0, // chunk_offset 402 -1, // chunk_size 403 true, // bsdiff_allowed 404 &data, 405 &op, 406 false, 407 src_ops_allowed)); 408 409 // Check if the operation writes nothing. 410 if (op.dst_extents_size() == 0) { 411 if (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 412 LOG(INFO) << "Empty MOVE operation, nothing to do."; 413 return true; 414 } else { 415 LOG(ERROR) << "Empty non-MOVE operation"; 416 return false; 417 } 418 } 419 420 // Write the data. 421 if (op.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE && 422 op.type() != DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) { 423 op.set_data_offset(*blobs_length); 424 op.set_data_length(data.size()); 425 } 426 427 // Add the new install operation. 428 ops->clear(); 429 ops->push_back(op); 430 431 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, data.data(), data.size())); 432 *blobs_length += data.size(); 433 434 LOG(INFO) << "Done delta compressing kernel partition: " 435 << kInstallOperationTypes[op.type()]; 436 return true; 437} 438 439struct DeltaObject { 440 DeltaObject(const string& in_name, const int in_type, const off_t in_size) 441 : name(in_name), 442 type(in_type), 443 size(in_size) {} 444 bool operator <(const DeltaObject& object) const { 445 return (size != object.size) ? (size < object.size) : (name < object.name); 446 } 447 string name; 448 int type; 449 off_t size; 450}; 451 452void ReportPayloadUsage(const DeltaArchiveManifest& manifest, 453 const int64_t manifest_metadata_size, 454 const OperationNameMap& op_name_map) { 455 vector<DeltaObject> objects; 456 off_t total_size = 0; 457 458 // Rootfs install operations. 459 for (int i = 0; i < manifest.install_operations_size(); ++i) { 460 const DeltaArchiveManifest_InstallOperation& op = 461 manifest.install_operations(i); 462 objects.push_back(DeltaObject(op_name_map.find(&op)->second, 463 op.type(), 464 op.data_length())); 465 total_size += op.data_length(); 466 } 467 468 // Kernel install operations. 469 for (int i = 0; i < manifest.kernel_install_operations_size(); ++i) { 470 const DeltaArchiveManifest_InstallOperation& op = 471 manifest.kernel_install_operations(i); 472 objects.push_back(DeltaObject(base::StringPrintf("<kernel-operation-%d>", 473 i), 474 op.type(), 475 op.data_length())); 476 total_size += op.data_length(); 477 } 478 479 objects.push_back(DeltaObject("<manifest-metadata>", 480 -1, 481 manifest_metadata_size)); 482 total_size += manifest_metadata_size; 483 484 std::sort(objects.begin(), objects.end()); 485 486 static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n"; 487 for (const DeltaObject& object : objects) { 488 fprintf(stderr, kFormatString, 489 object.size * 100.0 / total_size, 490 static_cast<intmax_t>(object.size), 491 object.type >= 0 ? kInstallOperationTypes[object.type] : "-", 492 object.name.c_str()); 493 } 494 fprintf(stderr, kFormatString, 495 100.0, static_cast<intmax_t>(total_size), "", "<total>"); 496} 497 498// Process a range of blocks from |range_start| to |range_end| in the extent at 499// position |*idx_p| of |extents|. If |do_remove| is true, this range will be 500// removed, which may cause the extent to be trimmed, split or removed entirely. 501// The value of |*idx_p| is updated to point to the next extent to be processed. 502// Returns true iff the next extent to process is a new or updated one. 503bool ProcessExtentBlockRange(vector<Extent>* extents, size_t* idx_p, 504 const bool do_remove, uint64_t range_start, 505 uint64_t range_end) { 506 size_t idx = *idx_p; 507 uint64_t start_block = (*extents)[idx].start_block(); 508 uint64_t num_blocks = (*extents)[idx].num_blocks(); 509 uint64_t range_size = range_end - range_start; 510 511 if (do_remove) { 512 if (range_size == num_blocks) { 513 // Remove the entire extent. 514 extents->erase(extents->begin() + idx); 515 } else if (range_end == num_blocks) { 516 // Trim the end of the extent. 517 (*extents)[idx].set_num_blocks(num_blocks - range_size); 518 idx++; 519 } else if (range_start == 0) { 520 // Trim the head of the extent. 521 (*extents)[idx].set_start_block(start_block + range_size); 522 (*extents)[idx].set_num_blocks(num_blocks - range_size); 523 } else { 524 // Trim the middle, splitting the remainder into two parts. 525 (*extents)[idx].set_num_blocks(range_start); 526 Extent e; 527 e.set_start_block(start_block + range_end); 528 e.set_num_blocks(num_blocks - range_end); 529 idx++; 530 extents->insert(extents->begin() + idx, e); 531 } 532 } else if (range_end == num_blocks) { 533 // Done with this extent. 534 idx++; 535 } else { 536 return false; 537 } 538 539 *idx_p = idx; 540 return true; 541} 542 543// Remove identical corresponding block ranges in |src_extents| and 544// |dst_extents|. Used for preventing moving of blocks onto themselves during 545// MOVE operations. The value of |total_bytes| indicates the actual length of 546// content; this may be slightly less than the total size of blocks, in which 547// case the last block is only partly occupied with data. Returns the total 548// number of bytes removed. 549size_t RemoveIdenticalBlockRanges(vector<Extent>* src_extents, 550 vector<Extent>* dst_extents, 551 const size_t total_bytes) { 552 size_t src_idx = 0; 553 size_t dst_idx = 0; 554 uint64_t src_offset = 0, dst_offset = 0; 555 bool new_src = true, new_dst = true; 556 size_t removed_bytes = 0, nonfull_block_bytes; 557 bool do_remove = false; 558 while (src_idx < src_extents->size() && dst_idx < dst_extents->size()) { 559 if (new_src) { 560 src_offset = 0; 561 new_src = false; 562 } 563 if (new_dst) { 564 dst_offset = 0; 565 new_dst = false; 566 } 567 568 do_remove = ((*src_extents)[src_idx].start_block() + src_offset == 569 (*dst_extents)[dst_idx].start_block() + dst_offset); 570 571 uint64_t src_num_blocks = (*src_extents)[src_idx].num_blocks(); 572 uint64_t dst_num_blocks = (*dst_extents)[dst_idx].num_blocks(); 573 uint64_t min_num_blocks = min(src_num_blocks - src_offset, 574 dst_num_blocks - dst_offset); 575 uint64_t prev_src_offset = src_offset; 576 uint64_t prev_dst_offset = dst_offset; 577 src_offset += min_num_blocks; 578 dst_offset += min_num_blocks; 579 580 new_src = ProcessExtentBlockRange(src_extents, &src_idx, do_remove, 581 prev_src_offset, src_offset); 582 new_dst = ProcessExtentBlockRange(dst_extents, &dst_idx, do_remove, 583 prev_dst_offset, dst_offset); 584 if (do_remove) 585 removed_bytes += min_num_blocks * kBlockSize; 586 } 587 588 // If we removed the last block and this block is only partly used by file 589 // content, deduct the unused portion from the total removed byte count. 590 if (do_remove && (nonfull_block_bytes = total_bytes % kBlockSize)) 591 removed_bytes -= kBlockSize - nonfull_block_bytes; 592 593 return removed_bytes; 594} 595 596} // namespace 597 598void DeltaDiffGenerator::CheckGraph(const Graph& graph) { 599 for (const Vertex& v : graph) { 600 CHECK(v.op.has_type()); 601 } 602} 603 604bool DeltaDiffGenerator::DeltaReadFile(Graph* graph, 605 Vertex::Index existing_vertex, 606 vector<Block>* blocks, 607 const string& old_root, 608 const string& new_root, 609 const string& path, // within new_root 610 off_t chunk_offset, 611 off_t chunk_size, 612 int data_fd, 613 off_t* data_file_size, 614 bool src_ops_allowed) { 615 chromeos::Blob data; 616 DeltaArchiveManifest_InstallOperation operation; 617 618 string old_path = (old_root == kEmptyPath) ? kEmptyPath : 619 old_root + path; 620 621 // If bsdiff breaks again, blacklist the problem file by using: 622 // bsdiff_allowed = (path != "/foo/bar") 623 // 624 // TODO(dgarrett): chromium-os:15274 connect this test to the command line. 625 bool bsdiff_allowed = true; 626 627 if (utils::FileSize(new_root + path) > kMaxBsdiffDestinationSize) 628 bsdiff_allowed = false; 629 630 if (!bsdiff_allowed) 631 LOG(INFO) << "bsdiff blacklisting: " << path; 632 633 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_path, 634 new_root + path, 635 chunk_offset, 636 chunk_size, 637 bsdiff_allowed, 638 &data, 639 &operation, 640 true, 641 src_ops_allowed)); 642 643 // Check if the operation writes nothing. 644 if (operation.dst_extents_size() == 0) { 645 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 646 LOG(INFO) << "Empty MOVE operation (" 647 << new_root + path << "), skipping"; 648 return true; 649 } else { 650 LOG(ERROR) << "Empty non-MOVE operation"; 651 return false; 652 } 653 } 654 655 // Write the data 656 if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE && 657 operation.type() != 658 DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) { 659 operation.set_data_offset(*data_file_size); 660 operation.set_data_length(data.size()); 661 } 662 663 TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, data.data(), data.size())); 664 *data_file_size += data.size(); 665 666 // Now, insert into graph and blocks vector 667 Vertex::Index vertex = existing_vertex; 668 if (vertex == Vertex::kInvalidIndex) { 669 graph->resize(graph->size() + 1); 670 vertex = graph->size() - 1; 671 } 672 (*graph)[vertex].op = operation; 673 CHECK((*graph)[vertex].op.has_type()); 674 (*graph)[vertex].file_name = path; 675 (*graph)[vertex].chunk_offset = chunk_offset; 676 (*graph)[vertex].chunk_size = chunk_size; 677 678 if (blocks) 679 TEST_AND_RETURN_FALSE(InplaceGenerator::AddInstallOpToBlocksVector( 680 (*graph)[vertex].op, 681 *graph, 682 vertex, 683 blocks)); 684 return true; 685} 686 687 688bool DeltaDiffGenerator::ReadFileToDiff( 689 const string& old_filename, 690 const string& new_filename, 691 off_t chunk_offset, 692 off_t chunk_size, 693 bool bsdiff_allowed, 694 chromeos::Blob* out_data, 695 DeltaArchiveManifest_InstallOperation* out_op, 696 bool gather_extents, 697 bool src_ops_allowed) { 698 // Read new data in 699 chromeos::Blob new_data; 700 TEST_AND_RETURN_FALSE( 701 utils::ReadFileChunk(new_filename, chunk_offset, chunk_size, &new_data)); 702 703 TEST_AND_RETURN_FALSE(!new_data.empty()); 704 TEST_AND_RETURN_FALSE(chunk_size == -1 || 705 static_cast<off_t>(new_data.size()) <= chunk_size); 706 707 chromeos::Blob new_data_bz; 708 TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz)); 709 CHECK(!new_data_bz.empty()); 710 711 chromeos::Blob data; // Data blob that will be written to delta file. 712 713 DeltaArchiveManifest_InstallOperation operation; 714 size_t current_best_size = 0; 715 if (new_data.size() <= new_data_bz.size()) { 716 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE); 717 current_best_size = new_data.size(); 718 data = new_data; 719 } else { 720 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); 721 current_best_size = new_data_bz.size(); 722 data = new_data_bz; 723 } 724 725 // Do we have an original file to consider? 726 off_t old_size = 0; 727 bool original = !old_filename.empty(); 728 if (original && (old_size = utils::FileSize(old_filename)) < 0) { 729 // If stat-ing the old file fails, it should be because it doesn't exist. 730 TEST_AND_RETURN_FALSE(!utils::FileExists(old_filename.c_str())); 731 original = false; 732 } 733 734 chromeos::Blob old_data; 735 if (original) { 736 // Read old data 737 TEST_AND_RETURN_FALSE( 738 utils::ReadFileChunk( 739 old_filename, chunk_offset, chunk_size, &old_data)); 740 if (old_data == new_data) { 741 // No change in data. 742 if (src_ops_allowed) { 743 operation.set_type( 744 DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY); 745 } else { 746 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE); 747 } 748 current_best_size = 0; 749 data.clear(); 750 } else if (!old_data.empty() && bsdiff_allowed) { 751 // If the source file is considered bsdiff safe (no bsdiff bugs 752 // triggered), see if BSDIFF encoding is smaller. 753 base::FilePath old_chunk; 754 TEST_AND_RETURN_FALSE(base::CreateTemporaryFile(&old_chunk)); 755 ScopedPathUnlinker old_unlinker(old_chunk.value()); 756 TEST_AND_RETURN_FALSE( 757 utils::WriteFile(old_chunk.value().c_str(), 758 old_data.data(), old_data.size())); 759 base::FilePath new_chunk; 760 TEST_AND_RETURN_FALSE(base::CreateTemporaryFile(&new_chunk)); 761 ScopedPathUnlinker new_unlinker(new_chunk.value()); 762 TEST_AND_RETURN_FALSE( 763 utils::WriteFile(new_chunk.value().c_str(), 764 new_data.data(), new_data.size())); 765 766 chromeos::Blob bsdiff_delta; 767 TEST_AND_RETURN_FALSE( 768 BsdiffFiles(old_chunk.value(), new_chunk.value(), &bsdiff_delta)); 769 CHECK_GT(bsdiff_delta.size(), static_cast<chromeos::Blob::size_type>(0)); 770 if (bsdiff_delta.size() < current_best_size) { 771 if (src_ops_allowed) { 772 operation.set_type( 773 DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF); 774 } else { 775 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF); 776 } 777 current_best_size = bsdiff_delta.size(); 778 data = bsdiff_delta; 779 } 780 } 781 } 782 783 // Set parameters of the operations 784 CHECK_EQ(data.size(), current_best_size); 785 786 vector<Extent> src_extents, dst_extents; 787 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE || 788 operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF || 789 operation.type() == 790 DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY || 791 operation.type() == 792 DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF) { 793 if (gather_extents) { 794 TEST_AND_RETURN_FALSE( 795 GatherExtents(old_filename, 796 chunk_offset, 797 chunk_size, 798 &src_extents)); 799 } else { 800 Extent* src_extent = operation.add_src_extents(); 801 src_extent->set_start_block(0); 802 src_extent->set_num_blocks((old_size + kBlockSize - 1) / kBlockSize); 803 } 804 operation.set_src_length(old_data.size()); 805 } 806 807 if (gather_extents) { 808 TEST_AND_RETURN_FALSE( 809 GatherExtents(new_filename, 810 chunk_offset, 811 chunk_size, 812 &dst_extents)); 813 } else { 814 Extent* dst_extent = operation.add_dst_extents(); 815 dst_extent->set_start_block(0); 816 dst_extent->set_num_blocks((new_data.size() + kBlockSize - 1) / kBlockSize); 817 } 818 operation.set_dst_length(new_data.size()); 819 820 if (gather_extents) { 821 // Remove identical src/dst block ranges in MOVE operations. 822 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 823 size_t removed_bytes = RemoveIdenticalBlockRanges( 824 &src_extents, &dst_extents, new_data.size()); 825 826 // Adjust the file length field accordingly. 827 if (removed_bytes) { 828 operation.set_src_length(old_data.size() - removed_bytes); 829 operation.set_dst_length(new_data.size() - removed_bytes); 830 } 831 } 832 833 // Embed extents in the operation. 834 DeltaDiffGenerator::StoreExtents(src_extents, 835 operation.mutable_src_extents()); 836 DeltaDiffGenerator::StoreExtents(dst_extents, 837 operation.mutable_dst_extents()); 838 } 839 840 out_data->swap(data); 841 *out_op = operation; 842 843 return true; 844} 845 846bool DeltaDiffGenerator::InitializePartitionInfo(bool is_kernel, 847 const string& partition, 848 PartitionInfo* info) { 849 int64_t size = 0; 850 if (is_kernel) { 851 size = utils::FileSize(partition); 852 } else { 853 int block_count = 0, block_size = 0; 854 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(partition, 855 &block_count, 856 &block_size)); 857 size = static_cast<int64_t>(block_count) * block_size; 858 } 859 TEST_AND_RETURN_FALSE(size > 0); 860 info->set_size(size); 861 OmahaHashCalculator hasher; 862 TEST_AND_RETURN_FALSE(hasher.UpdateFile(partition, size) == size); 863 TEST_AND_RETURN_FALSE(hasher.Finalize()); 864 const chromeos::Blob& hash = hasher.raw_hash(); 865 info->set_hash(hash.data(), hash.size()); 866 LOG(INFO) << partition << ": size=" << size << " hash=" << hasher.hash(); 867 return true; 868} 869 870bool InitializePartitionInfos(const PayloadGenerationConfig& config, 871 DeltaArchiveManifest* manifest) { 872 if (!config.source.kernel_part.empty()) { 873 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 874 true, 875 config.source.kernel_part, 876 manifest->mutable_old_kernel_info())); 877 } 878 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 879 true, 880 config.target.kernel_part, 881 manifest->mutable_new_kernel_info())); 882 if (!config.source.rootfs_part.empty()) { 883 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 884 false, 885 config.source.rootfs_part, 886 manifest->mutable_old_rootfs_info())); 887 } 888 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 889 false, 890 config.target.rootfs_part, 891 manifest->mutable_new_rootfs_info())); 892 return true; 893} 894 895// Stores all Extents in 'extents' into 'out'. 896void DeltaDiffGenerator::StoreExtents( 897 const vector<Extent>& extents, 898 google::protobuf::RepeatedPtrField<Extent>* out) { 899 for (const Extent& extent : extents) { 900 Extent* new_extent = out->Add(); 901 *new_extent = extent; 902 } 903} 904 905// Returns true if |op| is a no-op operation that doesn't do any useful work 906// (e.g., a move operation that copies blocks onto themselves). 907bool DeltaDiffGenerator::IsNoopOperation( 908 const DeltaArchiveManifest_InstallOperation& op) { 909 return (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE && 910 ExpandExtents(op.src_extents()) == ExpandExtents(op.dst_extents())); 911} 912 913bool DeltaDiffGenerator::ReorderDataBlobs( 914 DeltaArchiveManifest* manifest, 915 const string& data_blobs_path, 916 const string& new_data_blobs_path) { 917 int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0); 918 TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0); 919 ScopedFdCloser in_fd_closer(&in_fd); 920 921 DirectFileWriter writer; 922 TEST_AND_RETURN_FALSE( 923 writer.Open(new_data_blobs_path.c_str(), 924 O_WRONLY | O_TRUNC | O_CREAT, 925 0644) == 0); 926 ScopedFileWriterCloser writer_closer(&writer); 927 uint64_t out_file_size = 0; 928 929 for (int i = 0; i < (manifest->install_operations_size() + 930 manifest->kernel_install_operations_size()); i++) { 931 DeltaArchiveManifest_InstallOperation* op = nullptr; 932 if (i < manifest->install_operations_size()) { 933 op = manifest->mutable_install_operations(i); 934 } else { 935 op = manifest->mutable_kernel_install_operations( 936 i - manifest->install_operations_size()); 937 } 938 if (!op->has_data_offset()) 939 continue; 940 CHECK(op->has_data_length()); 941 chromeos::Blob buf(op->data_length()); 942 ssize_t rc = pread(in_fd, buf.data(), buf.size(), op->data_offset()); 943 TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size())); 944 945 // Add the hash of the data blobs for this operation 946 TEST_AND_RETURN_FALSE(AddOperationHash(op, buf)); 947 948 op->set_data_offset(out_file_size); 949 TEST_AND_RETURN_FALSE(writer.Write(buf.data(), buf.size())); 950 out_file_size += buf.size(); 951 } 952 return true; 953} 954 955bool DeltaDiffGenerator::AddOperationHash( 956 DeltaArchiveManifest_InstallOperation* op, 957 const chromeos::Blob& buf) { 958 OmahaHashCalculator hasher; 959 960 TEST_AND_RETURN_FALSE(hasher.Update(buf.data(), buf.size())); 961 TEST_AND_RETURN_FALSE(hasher.Finalize()); 962 963 const chromeos::Blob& hash = hasher.raw_hash(); 964 op->set_data_sha256_hash(hash.data(), hash.size()); 965 return true; 966} 967 968vector<Vertex::Index> DeltaDiffGenerator::OrderIndices(const Graph& graph) { 969 vector<Vertex::Index> order(graph.size()); 970 std::iota(order.begin(), order.end(), 0); 971 return order; 972} 973 974bool DeltaDiffGenerator::GenerateDeltaUpdateFile( 975 const PayloadGenerationConfig& config, 976 const string& output_path, 977 const string& private_key_path, 978 size_t rootfs_partition_size, 979 uint64_t* metadata_size) { 980 int old_image_block_count = config.source.rootfs_size / config.block_size; 981 int new_image_block_count = config.target.rootfs_size / config.block_size; 982 983 if (config.is_delta) { 984 LOG_IF(WARNING, old_image_block_count != new_image_block_count) 985 << "Old and new images have different block counts."; 986 // TODO(deymo): Our tools only support growing the filesystem size during 987 // an update. Remove this check when that's fixed. crbug.com/192136 988 LOG_IF(FATAL, old_image_block_count > new_image_block_count) 989 << "Shirking the rootfs size is not supported at the moment."; 990 } 991 992 // Sanity checks for the partition size. 993 TEST_AND_RETURN_FALSE(rootfs_partition_size % config.block_size == 0); 994 LOG(INFO) << "Rootfs partition size: " << rootfs_partition_size; 995 LOG(INFO) << "Actual filesystem size: " << config.target.rootfs_size; 996 TEST_AND_RETURN_FALSE(rootfs_partition_size >= config.target.rootfs_size); 997 998 vector<Block> blocks(max(old_image_block_count, new_image_block_count)); 999 LOG(INFO) << "Invalid block index: " << Vertex::kInvalidIndex; 1000 LOG(INFO) << "Block count: " << blocks.size(); 1001 for (const Block& block : blocks) { 1002 CHECK(block.reader == Vertex::kInvalidIndex); 1003 CHECK(block.writer == Vertex::kInvalidIndex); 1004 } 1005 Graph graph; 1006 CheckGraph(graph); 1007 1008 const string kTempFileTemplate("CrAU_temp_data.XXXXXX"); 1009 string temp_file_path; 1010 unique_ptr<ScopedPathUnlinker> temp_file_unlinker; 1011 off_t data_file_size = 0; 1012 1013 LOG(INFO) << "Reading files..."; 1014 1015 // Create empty protobuf Manifest object 1016 DeltaArchiveManifest manifest; 1017 1018 vector<DeltaArchiveManifest_InstallOperation> kernel_ops; 1019 1020 vector<Vertex::Index> final_order; 1021 Vertex::Index scratch_vertex = Vertex::kInvalidIndex; 1022 { 1023 int data_file_fd; 1024 TEST_AND_RETURN_FALSE( 1025 utils::MakeTempFile(kTempFileTemplate, &temp_file_path, &data_file_fd)); 1026 temp_file_unlinker.reset(new ScopedPathUnlinker(temp_file_path)); 1027 TEST_AND_RETURN_FALSE(data_file_fd >= 0); 1028 ScopedFdCloser data_file_fd_closer(&data_file_fd); 1029 if (config.is_delta) { 1030 // Delta update 1031 1032 // Set the minor version for this payload. 1033 LOG(INFO) << "Adding Delta Minor Version."; 1034 manifest.set_minor_version(config.minor_version); 1035 if (config.minor_version == kInPlaceMinorPayloadVersion) { 1036 TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph, 1037 &blocks, 1038 config.source.rootfs_mountpt, 1039 config.target.rootfs_mountpt, 1040 config.chunk_size, 1041 data_file_fd, 1042 &data_file_size, 1043 false)); // src_ops_allowed 1044 LOG(INFO) << "done reading normal files"; 1045 CheckGraph(graph); 1046 1047 LOG(INFO) << "Starting metadata processing"; 1048 TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata( 1049 &graph, 1050 &blocks, 1051 config.source.rootfs_part, 1052 config.target.rootfs_part, 1053 data_file_fd, 1054 &data_file_size)); 1055 LOG(INFO) << "Done metadata processing"; 1056 CheckGraph(graph); 1057 1058 graph.resize(graph.size() + 1); 1059 TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks, 1060 data_file_fd, 1061 &data_file_size, 1062 config.source.rootfs_part, 1063 config.target.rootfs_part, 1064 &graph.back())); 1065 if (graph.back().op.data_length() == 0) { 1066 LOG(INFO) << "No unwritten blocks to write, omitting operation"; 1067 graph.pop_back(); 1068 } 1069 1070 // Final scratch block (if there's space) 1071 if (blocks.size() < (rootfs_partition_size / kBlockSize)) { 1072 scratch_vertex = graph.size(); 1073 graph.resize(graph.size() + 1); 1074 InplaceGenerator::CreateScratchNode( 1075 blocks.size(), 1076 (rootfs_partition_size / kBlockSize) - blocks.size(), 1077 &graph.back()); 1078 } 1079 1080 // Read kernel partition 1081 TEST_AND_RETURN_FALSE( 1082 DeltaCompressKernelPartition(config.source.kernel_part, 1083 config.target.kernel_part, 1084 &kernel_ops, 1085 data_file_fd, 1086 &data_file_size, 1087 false)); // src_ops_allowed 1088 1089 LOG(INFO) << "done reading kernel"; 1090 CheckGraph(graph); 1091 1092 LOG(INFO) << "Creating edges..."; 1093 InplaceGenerator::CreateEdges(&graph, blocks); 1094 LOG(INFO) << "Done creating edges"; 1095 CheckGraph(graph); 1096 1097 TEST_AND_RETURN_FALSE(InplaceGenerator::ConvertGraphToDag( 1098 &graph, 1099 config.target.rootfs_mountpt, 1100 data_file_fd, 1101 &data_file_size, 1102 &final_order, 1103 scratch_vertex)); 1104 } else if (config.minor_version == kSourceMinorPayloadVersion) { 1105 TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph, 1106 &blocks, 1107 config.source.rootfs_mountpt, 1108 config.target.rootfs_mountpt, 1109 config.chunk_size, 1110 data_file_fd, 1111 &data_file_size, 1112 true)); // src_ops_allowed 1113 1114 LOG(INFO) << "done reading normal files"; 1115 CheckGraph(graph); 1116 1117 // Read kernel partition 1118 TEST_AND_RETURN_FALSE( 1119 DeltaCompressKernelPartition(config.source.kernel_part, 1120 config.target.kernel_part, 1121 &kernel_ops, 1122 data_file_fd, 1123 &data_file_size, 1124 true)); // src_ops_allowed 1125 LOG(INFO) << "done reading kernel"; 1126 CheckGraph(graph); 1127 1128 graph.resize(graph.size() + 1); 1129 TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks, 1130 data_file_fd, 1131 &data_file_size, 1132 config.source.rootfs_part, 1133 config.target.rootfs_part, 1134 &graph.back())); 1135 if (graph.back().op.data_length() == 0) { 1136 LOG(INFO) << "No unwritten blocks to write, omitting operation"; 1137 graph.pop_back(); 1138 } 1139 1140 final_order = OrderIndices(graph); 1141 } else { 1142 LOG(ERROR) << "Unsupported minor version given for delta payload: " 1143 << config.minor_version; 1144 return false; 1145 } 1146 } else { 1147 TEST_AND_RETURN_FALSE(FullUpdateGenerator::Run(config, 1148 data_file_fd, 1149 &data_file_size, 1150 &graph, 1151 &kernel_ops, 1152 &final_order)); 1153 1154 // Set the minor version for this payload. 1155 LOG(INFO) << "Adding Full Minor Version."; 1156 manifest.set_minor_version(DeltaPerformer::kFullPayloadMinorVersion); 1157 } 1158 } 1159 1160 if (!config.source.ImageInfoIsEmpty()) 1161 *(manifest.mutable_old_image_info()) = config.source.image_info; 1162 1163 if (!config.target.ImageInfoIsEmpty()) 1164 *(manifest.mutable_new_image_info()) = config.target.image_info; 1165 1166 OperationNameMap op_name_map; 1167 CheckGraph(graph); 1168 InstallOperationsToManifest(graph, 1169 final_order, 1170 kernel_ops, 1171 &manifest, 1172 &op_name_map); 1173 CheckGraph(graph); 1174 manifest.set_block_size(kBlockSize); 1175 1176 // Reorder the data blobs with the newly ordered manifest 1177 string ordered_blobs_path; 1178 TEST_AND_RETURN_FALSE(utils::MakeTempFile( 1179 "CrAU_temp_data.ordered.XXXXXX", 1180 &ordered_blobs_path, 1181 nullptr)); 1182 ScopedPathUnlinker ordered_blobs_unlinker(ordered_blobs_path); 1183 TEST_AND_RETURN_FALSE(ReorderDataBlobs(&manifest, 1184 temp_file_path, 1185 ordered_blobs_path)); 1186 temp_file_unlinker.reset(); 1187 1188 // Check that install op blobs are in order. 1189 uint64_t next_blob_offset = 0; 1190 { 1191 for (int i = 0; i < (manifest.install_operations_size() + 1192 manifest.kernel_install_operations_size()); i++) { 1193 DeltaArchiveManifest_InstallOperation* op = 1194 i < manifest.install_operations_size() ? 1195 manifest.mutable_install_operations(i) : 1196 manifest.mutable_kernel_install_operations( 1197 i - manifest.install_operations_size()); 1198 if (op->has_data_offset()) { 1199 if (op->data_offset() != next_blob_offset) { 1200 LOG(FATAL) << "bad blob offset! " << op->data_offset() << " != " 1201 << next_blob_offset; 1202 } 1203 next_blob_offset += op->data_length(); 1204 } 1205 } 1206 } 1207 1208 // Signatures appear at the end of the blobs. Note the offset in the 1209 // manifest 1210 if (!private_key_path.empty()) { 1211 uint64_t signature_blob_length = 0; 1212 TEST_AND_RETURN_FALSE( 1213 PayloadSigner::SignatureBlobLength(vector<string>(1, private_key_path), 1214 &signature_blob_length)); 1215 AddSignatureOp(next_blob_offset, signature_blob_length, &manifest); 1216 } 1217 1218 TEST_AND_RETURN_FALSE(InitializePartitionInfos(config, &manifest)); 1219 1220 // Serialize protobuf 1221 string serialized_manifest; 1222 1223 CheckGraph(graph); 1224 TEST_AND_RETURN_FALSE(manifest.AppendToString(&serialized_manifest)); 1225 CheckGraph(graph); 1226 1227 LOG(INFO) << "Writing final delta file header..."; 1228 DirectFileWriter writer; 1229 TEST_AND_RETURN_FALSE_ERRNO(writer.Open(output_path.c_str(), 1230 O_WRONLY | O_CREAT | O_TRUNC, 1231 0644) == 0); 1232 ScopedFileWriterCloser writer_closer(&writer); 1233 1234 // Write header 1235 TEST_AND_RETURN_FALSE(writer.Write(kDeltaMagic, strlen(kDeltaMagic))); 1236 1237 // Write major version number 1238 TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, kMajorVersionNumber)); 1239 1240 // Write protobuf length 1241 TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, 1242 serialized_manifest.size())); 1243 1244 // Write protobuf 1245 LOG(INFO) << "Writing final delta file protobuf... " 1246 << serialized_manifest.size(); 1247 TEST_AND_RETURN_FALSE(writer.Write(serialized_manifest.data(), 1248 serialized_manifest.size())); 1249 1250 // Append the data blobs 1251 LOG(INFO) << "Writing final delta file data blobs..."; 1252 int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0); 1253 ScopedFdCloser blobs_fd_closer(&blobs_fd); 1254 TEST_AND_RETURN_FALSE(blobs_fd >= 0); 1255 for (;;) { 1256 char buf[kBlockSize]; 1257 ssize_t rc = read(blobs_fd, buf, sizeof(buf)); 1258 if (0 == rc) { 1259 // EOF 1260 break; 1261 } 1262 TEST_AND_RETURN_FALSE_ERRNO(rc > 0); 1263 TEST_AND_RETURN_FALSE(writer.Write(buf, rc)); 1264 } 1265 1266 // Write signature blob. 1267 if (!private_key_path.empty()) { 1268 LOG(INFO) << "Signing the update..."; 1269 chromeos::Blob signature_blob; 1270 TEST_AND_RETURN_FALSE(PayloadSigner::SignPayload( 1271 output_path, 1272 vector<string>(1, private_key_path), 1273 &signature_blob)); 1274 TEST_AND_RETURN_FALSE(writer.Write(signature_blob.data(), 1275 signature_blob.size())); 1276 } 1277 1278 *metadata_size = 1279 strlen(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size(); 1280 ReportPayloadUsage(manifest, *metadata_size, op_name_map); 1281 1282 LOG(INFO) << "All done. Successfully created delta file with " 1283 << "metadata size = " << *metadata_size; 1284 return true; 1285} 1286 1287// Runs the bsdiff tool on two files and returns the resulting delta in 1288// 'out'. Returns true on success. 1289bool DeltaDiffGenerator::BsdiffFiles(const string& old_file, 1290 const string& new_file, 1291 chromeos::Blob* out) { 1292 const string kPatchFile = "delta.patchXXXXXX"; 1293 string patch_file_path; 1294 1295 TEST_AND_RETURN_FALSE( 1296 utils::MakeTempFile(kPatchFile, &patch_file_path, nullptr)); 1297 1298 vector<string> cmd; 1299 cmd.push_back(kBsdiffPath); 1300 cmd.push_back(old_file); 1301 cmd.push_back(new_file); 1302 cmd.push_back(patch_file_path); 1303 1304 int rc = 1; 1305 chromeos::Blob patch_file; 1306 TEST_AND_RETURN_FALSE(Subprocess::SynchronousExec(cmd, &rc, nullptr)); 1307 TEST_AND_RETURN_FALSE(rc == 0); 1308 TEST_AND_RETURN_FALSE(utils::ReadFile(patch_file_path, out)); 1309 unlink(patch_file_path.c_str()); 1310 return true; 1311} 1312 1313void DeltaDiffGenerator::AddSignatureOp(uint64_t signature_blob_offset, 1314 uint64_t signature_blob_length, 1315 DeltaArchiveManifest* manifest) { 1316 LOG(INFO) << "Making room for signature in file"; 1317 manifest->set_signatures_offset(signature_blob_offset); 1318 LOG(INFO) << "set? " << manifest->has_signatures_offset(); 1319 // Add a dummy op at the end to appease older clients 1320 DeltaArchiveManifest_InstallOperation* dummy_op = 1321 manifest->add_kernel_install_operations(); 1322 dummy_op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE); 1323 dummy_op->set_data_offset(signature_blob_offset); 1324 manifest->set_signatures_offset(signature_blob_offset); 1325 dummy_op->set_data_length(signature_blob_length); 1326 manifest->set_signatures_size(signature_blob_length); 1327 Extent* dummy_extent = dummy_op->add_dst_extents(); 1328 // Tell the dummy op to write this data to a big sparse hole 1329 dummy_extent->set_start_block(kSparseHole); 1330 dummy_extent->set_num_blocks((signature_blob_length + kBlockSize - 1) / 1331 kBlockSize); 1332} 1333 1334}; // namespace chromeos_update_engine 1335