delta_diff_generator.cc revision cd514b531a38da1497630fa68b6e3ef45871893d
1// Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "update_engine/payload_generator/delta_diff_generator.h" 6 7#include <errno.h> 8#include <fcntl.h> 9#include <inttypes.h> 10#include <sys/stat.h> 11#include <sys/types.h> 12 13#include <algorithm> 14#include <map> 15#include <memory> 16#include <string> 17#include <utility> 18#include <vector> 19 20#include <base/files/file_path.h> 21#include <base/files/file_util.h> 22#include <base/logging.h> 23#include <base/strings/stringprintf.h> 24#include <base/strings/string_number_conversions.h> 25#include <base/strings/string_util.h> 26#include <bzlib.h> 27 28#include "update_engine/bzip.h" 29#include "update_engine/delta_performer.h" 30#include "update_engine/file_writer.h" 31#include "update_engine/omaha_hash_calculator.h" 32#include "update_engine/payload_constants.h" 33#include "update_engine/payload_generator/extent_mapper.h" 34#include "update_engine/payload_generator/filesystem_iterator.h" 35#include "update_engine/payload_generator/full_update_generator.h" 36#include "update_engine/payload_generator/graph_types.h" 37#include "update_engine/payload_generator/graph_utils.h" 38#include "update_engine/payload_generator/inplace_generator.h" 39#include "update_engine/payload_generator/metadata.h" 40#include "update_engine/payload_generator/payload_signer.h" 41#include "update_engine/payload_verifier.h" 42#include "update_engine/subprocess.h" 43#include "update_engine/update_metadata.pb.h" 44#include "update_engine/utils.h" 45 46using std::map; 47using std::max; 48using std::min; 49using std::set; 50using std::string; 51using std::unique_ptr; 52using std::vector; 53 54namespace { 55 56const uint64_t kVersionNumber = 1; 57const uint64_t kFullUpdateChunkSize = 1024 * 1024; // bytes 58 59// The maximum destination size allowed for bsdiff. In general, bsdiff should 60// work for arbitrary big files, but the payload generation and payload 61// application requires a significant amount of RAM. We put a hard-limit of 62// 200 MiB that should not affect any released board, but will limit the 63// Chrome binary in ASan builders. 64const off_t kMaxBsdiffDestinationSize = 200 * 1024 * 1024; // bytes 65 66static const char* kInstallOperationTypes[] = { 67 "REPLACE", 68 "REPLACE_BZ", 69 "MOVE", 70 "BSDIFF", 71 "SOURCE_COPY", 72 "SOURCE_BSDIFF" 73}; 74 75} // namespace 76 77namespace chromeos_update_engine { 78 79typedef DeltaDiffGenerator::Block Block; 80typedef map<const DeltaArchiveManifest_InstallOperation*, 81 string> OperationNameMap; 82 83// bytes 84const size_t kRootFSPartitionSize = static_cast<size_t>(2) * 1024 * 1024 * 1024; 85const size_t kBlockSize = 4096; // bytes 86const char* const kEmptyPath = ""; 87const char* const kBsdiffPath = "bsdiff"; 88 89// Needed for testing purposes, in case we can't use actual filesystem objects. 90// TODO(garnold) (chromium:331965) Replace this hack with a properly injected 91// parameter in form of a mockable abstract class. 92bool (*get_extents_with_chunk_func)(const string&, off_t, off_t, 93 vector<Extent>*) = 94 extent_mapper::ExtentsForFileChunkFibmap; 95 96namespace { 97 98// Stores all the extents of |path| into |extents|. Returns true on success. 99bool GatherExtents(const string& path, 100 off_t chunk_offset, 101 off_t chunk_size, 102 vector<Extent>* extents) { 103 extents->clear(); 104 TEST_AND_RETURN_FALSE( 105 get_extents_with_chunk_func( 106 path, chunk_offset, chunk_size, extents)); 107 return true; 108} 109 110// For each regular file within new_root, creates a node in the graph, 111// determines the best way to compress it (REPLACE, REPLACE_BZ, COPY, BSDIFF), 112// and writes any necessary data to the end of data_fd. 113bool DeltaReadFiles(Graph* graph, 114 vector<Block>* blocks, 115 const string& old_root, 116 const string& new_root, 117 off_t chunk_size, 118 int data_fd, 119 off_t* data_file_size) { 120 set<ino_t> visited_inodes; 121 set<ino_t> visited_src_inodes; 122 for (FilesystemIterator fs_iter(new_root, 123 set<string>{"/lost+found"}); 124 !fs_iter.IsEnd(); fs_iter.Increment()) { 125 // We never diff symlinks (here, we check that dst file is not a symlink). 126 if (!S_ISREG(fs_iter.GetStat().st_mode)) 127 continue; 128 129 // Make sure we visit each inode only once. 130 if (utils::SetContainsKey(visited_inodes, fs_iter.GetStat().st_ino)) 131 continue; 132 visited_inodes.insert(fs_iter.GetStat().st_ino); 133 off_t dst_size = fs_iter.GetFileSize(); 134 if (dst_size == 0) 135 continue; 136 137 LOG(INFO) << "Encoding file " << fs_iter.GetPartialPath(); 138 139 // We can't visit each dst image inode more than once, as that would 140 // duplicate work. Here, we avoid visiting each source image inode 141 // more than once. Technically, we could have multiple operations 142 // that read the same blocks from the source image for diffing, but 143 // we choose not to avoid complexity. Eventually we will move away 144 // from using a graph/cycle detection/etc to generate diffs, and at that 145 // time, it will be easy (non-complex) to have many operations read 146 // from the same source blocks. At that time, this code can die. -adlr 147 bool should_diff_from_source = false; 148 string src_path = old_root + fs_iter.GetPartialPath(); 149 struct stat src_stbuf; 150 // We never diff symlinks (here, we check that src file is not a symlink). 151 if (0 == lstat(src_path.c_str(), &src_stbuf) && 152 S_ISREG(src_stbuf.st_mode)) { 153 should_diff_from_source = !utils::SetContainsKey(visited_src_inodes, 154 src_stbuf.st_ino); 155 visited_src_inodes.insert(src_stbuf.st_ino); 156 } 157 158 off_t size = chunk_size == -1 ? dst_size : chunk_size; 159 off_t step = size; 160 for (off_t offset = 0; offset < dst_size; offset += step) { 161 if (offset + size >= dst_size) { 162 size = -1; // Read through the end of the file. 163 } 164 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::DeltaReadFile( 165 graph, 166 Vertex::kInvalidIndex, 167 blocks, 168 (should_diff_from_source ? old_root : kEmptyPath), 169 new_root, 170 fs_iter.GetPartialPath(), 171 offset, 172 size, 173 data_fd, 174 data_file_size)); 175 } 176 } 177 return true; 178} 179 180// Reads blocks from image_path that are not yet marked as being written in the 181// blocks array. These blocks that remain are either unchanged files or 182// non-file-data blocks. We compare each of them to the old image, and compress 183// the ones that changed into a single REPLACE_BZ operation. This updates a 184// newly created node in the graph to write these blocks and writes the 185// appropriate blob to blobs_fd. Reads and updates blobs_length. 186bool ReadUnwrittenBlocks(const vector<Block>& blocks, 187 int blobs_fd, 188 off_t* blobs_length, 189 const string& old_image_path, 190 const string& new_image_path, 191 Vertex* vertex) { 192 vertex->file_name = "<rootfs-non-file-data>"; 193 194 DeltaArchiveManifest_InstallOperation* out_op = &vertex->op; 195 int new_image_fd = open(new_image_path.c_str(), O_RDONLY, 000); 196 TEST_AND_RETURN_FALSE_ERRNO(new_image_fd >= 0); 197 ScopedFdCloser new_image_fd_closer(&new_image_fd); 198 int old_image_fd = open(old_image_path.c_str(), O_RDONLY, 000); 199 TEST_AND_RETURN_FALSE_ERRNO(old_image_fd >= 0); 200 ScopedFdCloser old_image_fd_closer(&old_image_fd); 201 202 string temp_file_path; 203 TEST_AND_RETURN_FALSE(utils::MakeTempFile("CrAU_temp_data.XXXXXX", 204 &temp_file_path, 205 nullptr)); 206 207 FILE* file = fopen(temp_file_path.c_str(), "w"); 208 TEST_AND_RETURN_FALSE(file); 209 int err = BZ_OK; 210 211 BZFILE* bz_file = BZ2_bzWriteOpen(&err, 212 file, 213 9, // max compression 214 0, // verbosity 215 0); // default work factor 216 TEST_AND_RETURN_FALSE(err == BZ_OK); 217 218 vector<Extent> extents; 219 vector<Block>::size_type block_count = 0; 220 221 LOG(INFO) << "Appending unwritten blocks to extents"; 222 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { 223 if (blocks[i].writer != Vertex::kInvalidIndex) 224 continue; 225 graph_utils::AppendBlockToExtents(&extents, i); 226 block_count++; 227 } 228 229 // Code will handle buffers of any size that's a multiple of kBlockSize, 230 // so we arbitrarily set it to 1024 * kBlockSize. 231 chromeos::Blob new_buf(1024 * kBlockSize); 232 chromeos::Blob old_buf(1024 * kBlockSize); 233 234 LOG(INFO) << "Scanning " << block_count << " unwritten blocks"; 235 vector<Extent> changed_extents; 236 vector<Block>::size_type changed_block_count = 0; 237 vector<Block>::size_type blocks_copied_count = 0; 238 239 // For each extent in extents, write the unchanged blocks into BZ2_bzWrite, 240 // which sends it to an output file. We use the temporary buffers to hold the 241 // old and new data, which may be smaller than the extent, so in that case we 242 // have to loop to get the extent's data (that's the inner while loop). 243 for (const Extent& extent : extents) { 244 vector<Block>::size_type blocks_read = 0; 245 float printed_progress = -1; 246 while (blocks_read < extent.num_blocks()) { 247 const uint64_t copy_first_block = extent.start_block() + blocks_read; 248 const int copy_block_cnt = 249 min(new_buf.size() / kBlockSize, 250 static_cast<chromeos::Blob::size_type>( 251 extent.num_blocks() - blocks_read)); 252 const size_t count = copy_block_cnt * kBlockSize; 253 const off_t offset = copy_first_block * kBlockSize; 254 ssize_t rc = pread(new_image_fd, new_buf.data(), count, offset); 255 TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); 256 TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == count); 257 258 rc = pread(old_image_fd, old_buf.data(), count, offset); 259 TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); 260 TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == count); 261 262 // Compare each block in the buffer to its counterpart in the old image 263 // and only compress it if its content has changed. 264 int buf_offset = 0; 265 for (int i = 0; i < copy_block_cnt; ++i) { 266 int buf_end_offset = buf_offset + kBlockSize; 267 if (!std::equal(new_buf.begin() + buf_offset, 268 new_buf.begin() + buf_end_offset, 269 old_buf.begin() + buf_offset)) { 270 BZ2_bzWrite(&err, bz_file, &new_buf[buf_offset], kBlockSize); 271 TEST_AND_RETURN_FALSE(err == BZ_OK); 272 const uint64_t block_idx = copy_first_block + i; 273 if (blocks[block_idx].reader != Vertex::kInvalidIndex) { 274 graph_utils::AddReadBeforeDep(vertex, blocks[block_idx].reader, 275 block_idx); 276 } 277 graph_utils::AppendBlockToExtents(&changed_extents, block_idx); 278 changed_block_count++; 279 } 280 buf_offset = buf_end_offset; 281 } 282 283 blocks_read += copy_block_cnt; 284 blocks_copied_count += copy_block_cnt; 285 float current_progress = 286 static_cast<float>(blocks_copied_count) / block_count; 287 if (printed_progress + 0.1 < current_progress || 288 blocks_copied_count == block_count) { 289 LOG(INFO) << "progress: " << current_progress; 290 printed_progress = current_progress; 291 } 292 } 293 } 294 BZ2_bzWriteClose(&err, bz_file, 0, nullptr, nullptr); 295 TEST_AND_RETURN_FALSE(err == BZ_OK); 296 bz_file = nullptr; 297 TEST_AND_RETURN_FALSE_ERRNO(0 == fclose(file)); 298 file = nullptr; 299 300 LOG(INFO) << "Compressed " << changed_block_count << " blocks (" 301 << block_count - changed_block_count << " blocks unchanged)"; 302 chromeos::Blob compressed_data; 303 if (changed_block_count > 0) { 304 LOG(INFO) << "Reading compressed data off disk"; 305 TEST_AND_RETURN_FALSE(utils::ReadFile(temp_file_path, &compressed_data)); 306 } 307 TEST_AND_RETURN_FALSE(unlink(temp_file_path.c_str()) == 0); 308 309 // Add node to graph to write these blocks 310 out_op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); 311 out_op->set_data_offset(*blobs_length); 312 out_op->set_data_length(compressed_data.size()); 313 LOG(INFO) << "Rootfs non-data blocks compressed take up " 314 << compressed_data.size(); 315 *blobs_length += compressed_data.size(); 316 out_op->set_dst_length(kBlockSize * changed_block_count); 317 DeltaDiffGenerator::StoreExtents(changed_extents, 318 out_op->mutable_dst_extents()); 319 320 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, 321 compressed_data.data(), 322 compressed_data.size())); 323 LOG(INFO) << "Done processing unwritten blocks"; 324 return true; 325} 326 327// Writes the uint64_t passed in in host-endian to the file as big-endian. 328// Returns true on success. 329bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) { 330 uint64_t value_be = htobe64(value); 331 TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be))); 332 return true; 333} 334 335// Adds each operation from |graph| to |out_manifest| in the order specified by 336// |order| while building |out_op_name_map| with operation to name 337// mappings. Adds all |kernel_ops| to |out_manifest|. Filters out no-op 338// operations. 339void InstallOperationsToManifest( 340 const Graph& graph, 341 const vector<Vertex::Index>& order, 342 const vector<DeltaArchiveManifest_InstallOperation>& kernel_ops, 343 DeltaArchiveManifest* out_manifest, 344 OperationNameMap* out_op_name_map) { 345 for (Vertex::Index vertex_index : order) { 346 const Vertex& vertex = graph[vertex_index]; 347 const DeltaArchiveManifest_InstallOperation& add_op = vertex.op; 348 if (DeltaDiffGenerator::IsNoopOperation(add_op)) { 349 continue; 350 } 351 DeltaArchiveManifest_InstallOperation* op = 352 out_manifest->add_install_operations(); 353 *op = add_op; 354 string name = vertex.file_name; 355 if (vertex.chunk_offset || vertex.chunk_size != -1) { 356 string offset = base::Int64ToString(vertex.chunk_offset); 357 if (vertex.chunk_size != -1) { 358 name += " [" + offset + ", " + 359 base::Int64ToString(vertex.chunk_offset + vertex.chunk_size - 1) + 360 "]"; 361 } else { 362 name += " [" + offset + ", end]"; 363 } 364 } 365 (*out_op_name_map)[op] = name; 366 } 367 for (vector<DeltaArchiveManifest_InstallOperation>::const_iterator it = 368 kernel_ops.begin(); it != kernel_ops.end(); ++it) { 369 const DeltaArchiveManifest_InstallOperation& add_op = *it; 370 if (DeltaDiffGenerator::IsNoopOperation(add_op)) { 371 continue; 372 } 373 DeltaArchiveManifest_InstallOperation* op = 374 out_manifest->add_kernel_install_operations(); 375 *op = add_op; 376 } 377} 378 379// Delta compresses a kernel partition |new_kernel_part| with knowledge of the 380// old kernel partition |old_kernel_part|. If |old_kernel_part| is an empty 381// string, generates a full update of the partition. 382bool DeltaCompressKernelPartition( 383 const string& old_kernel_part, 384 const string& new_kernel_part, 385 vector<DeltaArchiveManifest_InstallOperation>* ops, 386 int blobs_fd, 387 off_t* blobs_length) { 388 LOG(INFO) << "Delta compressing kernel partition..."; 389 LOG_IF(INFO, old_kernel_part.empty()) << "Generating full kernel update..."; 390 391 DeltaArchiveManifest_InstallOperation op; 392 chromeos::Blob data; 393 TEST_AND_RETURN_FALSE( 394 DeltaDiffGenerator::ReadFileToDiff(old_kernel_part, 395 new_kernel_part, 396 0, // chunk_offset 397 -1, // chunk_size 398 true, // bsdiff_allowed 399 &data, 400 &op, 401 false)); 402 403 // Check if the operation writes nothing. 404 if (op.dst_extents_size() == 0) { 405 if (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 406 LOG(INFO) << "Empty MOVE operation, nothing to do."; 407 return true; 408 } else { 409 LOG(ERROR) << "Empty non-MOVE operation"; 410 return false; 411 } 412 } 413 414 // Write the data. 415 if (op.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) { 416 op.set_data_offset(*blobs_length); 417 op.set_data_length(data.size()); 418 } 419 420 // Add the new install operation. 421 ops->clear(); 422 ops->push_back(op); 423 424 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, data.data(), data.size())); 425 *blobs_length += data.size(); 426 427 LOG(INFO) << "Done delta compressing kernel partition: " 428 << kInstallOperationTypes[op.type()]; 429 return true; 430} 431 432struct DeltaObject { 433 DeltaObject(const string& in_name, const int in_type, const off_t in_size) 434 : name(in_name), 435 type(in_type), 436 size(in_size) {} 437 bool operator <(const DeltaObject& object) const { 438 return (size != object.size) ? (size < object.size) : (name < object.name); 439 } 440 string name; 441 int type; 442 off_t size; 443}; 444 445void ReportPayloadUsage(const DeltaArchiveManifest& manifest, 446 const int64_t manifest_metadata_size, 447 const OperationNameMap& op_name_map) { 448 vector<DeltaObject> objects; 449 off_t total_size = 0; 450 451 // Rootfs install operations. 452 for (int i = 0; i < manifest.install_operations_size(); ++i) { 453 const DeltaArchiveManifest_InstallOperation& op = 454 manifest.install_operations(i); 455 objects.push_back(DeltaObject(op_name_map.find(&op)->second, 456 op.type(), 457 op.data_length())); 458 total_size += op.data_length(); 459 } 460 461 // Kernel install operations. 462 for (int i = 0; i < manifest.kernel_install_operations_size(); ++i) { 463 const DeltaArchiveManifest_InstallOperation& op = 464 manifest.kernel_install_operations(i); 465 objects.push_back(DeltaObject(base::StringPrintf("<kernel-operation-%d>", 466 i), 467 op.type(), 468 op.data_length())); 469 total_size += op.data_length(); 470 } 471 472 objects.push_back(DeltaObject("<manifest-metadata>", 473 -1, 474 manifest_metadata_size)); 475 total_size += manifest_metadata_size; 476 477 std::sort(objects.begin(), objects.end()); 478 479 static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n"; 480 for (const DeltaObject& object : objects) { 481 fprintf(stderr, kFormatString, 482 object.size * 100.0 / total_size, 483 static_cast<intmax_t>(object.size), 484 object.type >= 0 ? kInstallOperationTypes[object.type] : "-", 485 object.name.c_str()); 486 } 487 fprintf(stderr, kFormatString, 488 100.0, static_cast<intmax_t>(total_size), "", "<total>"); 489} 490 491// Process a range of blocks from |range_start| to |range_end| in the extent at 492// position |*idx_p| of |extents|. If |do_remove| is true, this range will be 493// removed, which may cause the extent to be trimmed, split or removed entirely. 494// The value of |*idx_p| is updated to point to the next extent to be processed. 495// Returns true iff the next extent to process is a new or updated one. 496bool ProcessExtentBlockRange(vector<Extent>* extents, size_t* idx_p, 497 const bool do_remove, uint64_t range_start, 498 uint64_t range_end) { 499 size_t idx = *idx_p; 500 uint64_t start_block = (*extents)[idx].start_block(); 501 uint64_t num_blocks = (*extents)[idx].num_blocks(); 502 uint64_t range_size = range_end - range_start; 503 504 if (do_remove) { 505 if (range_size == num_blocks) { 506 // Remove the entire extent. 507 extents->erase(extents->begin() + idx); 508 } else if (range_end == num_blocks) { 509 // Trim the end of the extent. 510 (*extents)[idx].set_num_blocks(num_blocks - range_size); 511 idx++; 512 } else if (range_start == 0) { 513 // Trim the head of the extent. 514 (*extents)[idx].set_start_block(start_block + range_size); 515 (*extents)[idx].set_num_blocks(num_blocks - range_size); 516 } else { 517 // Trim the middle, splitting the remainder into two parts. 518 (*extents)[idx].set_num_blocks(range_start); 519 Extent e; 520 e.set_start_block(start_block + range_end); 521 e.set_num_blocks(num_blocks - range_end); 522 idx++; 523 extents->insert(extents->begin() + idx, e); 524 } 525 } else if (range_end == num_blocks) { 526 // Done with this extent. 527 idx++; 528 } else { 529 return false; 530 } 531 532 *idx_p = idx; 533 return true; 534} 535 536// Remove identical corresponding block ranges in |src_extents| and 537// |dst_extents|. Used for preventing moving of blocks onto themselves during 538// MOVE operations. The value of |total_bytes| indicates the actual length of 539// content; this may be slightly less than the total size of blocks, in which 540// case the last block is only partly occupied with data. Returns the total 541// number of bytes removed. 542size_t RemoveIdenticalBlockRanges(vector<Extent>* src_extents, 543 vector<Extent>* dst_extents, 544 const size_t total_bytes) { 545 size_t src_idx = 0; 546 size_t dst_idx = 0; 547 uint64_t src_offset = 0, dst_offset = 0; 548 bool new_src = true, new_dst = true; 549 size_t removed_bytes = 0, nonfull_block_bytes; 550 bool do_remove = false; 551 while (src_idx < src_extents->size() && dst_idx < dst_extents->size()) { 552 if (new_src) { 553 src_offset = 0; 554 new_src = false; 555 } 556 if (new_dst) { 557 dst_offset = 0; 558 new_dst = false; 559 } 560 561 do_remove = ((*src_extents)[src_idx].start_block() + src_offset == 562 (*dst_extents)[dst_idx].start_block() + dst_offset); 563 564 uint64_t src_num_blocks = (*src_extents)[src_idx].num_blocks(); 565 uint64_t dst_num_blocks = (*dst_extents)[dst_idx].num_blocks(); 566 uint64_t min_num_blocks = min(src_num_blocks - src_offset, 567 dst_num_blocks - dst_offset); 568 uint64_t prev_src_offset = src_offset; 569 uint64_t prev_dst_offset = dst_offset; 570 src_offset += min_num_blocks; 571 dst_offset += min_num_blocks; 572 573 new_src = ProcessExtentBlockRange(src_extents, &src_idx, do_remove, 574 prev_src_offset, src_offset); 575 new_dst = ProcessExtentBlockRange(dst_extents, &dst_idx, do_remove, 576 prev_dst_offset, dst_offset); 577 if (do_remove) 578 removed_bytes += min_num_blocks * kBlockSize; 579 } 580 581 // If we removed the last block and this block is only partly used by file 582 // content, deduct the unused portion from the total removed byte count. 583 if (do_remove && (nonfull_block_bytes = total_bytes % kBlockSize)) 584 removed_bytes -= kBlockSize - nonfull_block_bytes; 585 586 return removed_bytes; 587} 588 589} // namespace 590 591void DeltaDiffGenerator::CheckGraph(const Graph& graph) { 592 for (const Vertex& v : graph) { 593 CHECK(v.op.has_type()); 594 } 595} 596 597bool DeltaDiffGenerator::DeltaReadFile(Graph* graph, 598 Vertex::Index existing_vertex, 599 vector<Block>* blocks, 600 const string& old_root, 601 const string& new_root, 602 const string& path, // within new_root 603 off_t chunk_offset, 604 off_t chunk_size, 605 int data_fd, 606 off_t* data_file_size) { 607 chromeos::Blob data; 608 DeltaArchiveManifest_InstallOperation operation; 609 610 string old_path = (old_root == kEmptyPath) ? kEmptyPath : 611 old_root + path; 612 613 // If bsdiff breaks again, blacklist the problem file by using: 614 // bsdiff_allowed = (path != "/foo/bar") 615 // 616 // TODO(dgarrett): chromium-os:15274 connect this test to the command line. 617 bool bsdiff_allowed = true; 618 619 if (utils::FileSize(new_root + path) > kMaxBsdiffDestinationSize) 620 bsdiff_allowed = false; 621 622 if (!bsdiff_allowed) 623 LOG(INFO) << "bsdiff blacklisting: " << path; 624 625 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_path, 626 new_root + path, 627 chunk_offset, 628 chunk_size, 629 bsdiff_allowed, 630 &data, 631 &operation, 632 true)); 633 634 // Check if the operation writes nothing. 635 if (operation.dst_extents_size() == 0) { 636 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 637 LOG(INFO) << "Empty MOVE operation (" << new_root + path << "), skipping"; 638 return true; 639 } else { 640 LOG(ERROR) << "Empty non-MOVE operation"; 641 return false; 642 } 643 } 644 645 // Write the data 646 if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE) { 647 operation.set_data_offset(*data_file_size); 648 operation.set_data_length(data.size()); 649 } 650 651 TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, data.data(), data.size())); 652 *data_file_size += data.size(); 653 654 // Now, insert into graph and blocks vector 655 Vertex::Index vertex = existing_vertex; 656 if (vertex == Vertex::kInvalidIndex) { 657 graph->resize(graph->size() + 1); 658 vertex = graph->size() - 1; 659 } 660 (*graph)[vertex].op = operation; 661 CHECK((*graph)[vertex].op.has_type()); 662 (*graph)[vertex].file_name = path; 663 (*graph)[vertex].chunk_offset = chunk_offset; 664 (*graph)[vertex].chunk_size = chunk_size; 665 666 if (blocks) 667 TEST_AND_RETURN_FALSE(InplaceGenerator::AddInstallOpToBlocksVector( 668 (*graph)[vertex].op, 669 *graph, 670 vertex, 671 blocks)); 672 return true; 673} 674 675 676bool DeltaDiffGenerator::ReadFileToDiff( 677 const string& old_filename, 678 const string& new_filename, 679 off_t chunk_offset, 680 off_t chunk_size, 681 bool bsdiff_allowed, 682 chromeos::Blob* out_data, 683 DeltaArchiveManifest_InstallOperation* out_op, 684 bool gather_extents) { 685 // Read new data in 686 chromeos::Blob new_data; 687 TEST_AND_RETURN_FALSE( 688 utils::ReadFileChunk(new_filename, chunk_offset, chunk_size, &new_data)); 689 690 TEST_AND_RETURN_FALSE(!new_data.empty()); 691 TEST_AND_RETURN_FALSE(chunk_size == -1 || 692 static_cast<off_t>(new_data.size()) <= chunk_size); 693 694 chromeos::Blob new_data_bz; 695 TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz)); 696 CHECK(!new_data_bz.empty()); 697 698 chromeos::Blob data; // Data blob that will be written to delta file. 699 700 DeltaArchiveManifest_InstallOperation operation; 701 size_t current_best_size = 0; 702 if (new_data.size() <= new_data_bz.size()) { 703 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE); 704 current_best_size = new_data.size(); 705 data = new_data; 706 } else { 707 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); 708 current_best_size = new_data_bz.size(); 709 data = new_data_bz; 710 } 711 712 // Do we have an original file to consider? 713 off_t old_size = 0; 714 bool original = !old_filename.empty(); 715 if (original && (old_size = utils::FileSize(old_filename)) < 0) { 716 // If stat-ing the old file fails, it should be because it doesn't exist. 717 TEST_AND_RETURN_FALSE(!utils::FileExists(old_filename.c_str())); 718 original = false; 719 } 720 721 chromeos::Blob old_data; 722 if (original) { 723 // Read old data 724 TEST_AND_RETURN_FALSE( 725 utils::ReadFileChunk( 726 old_filename, chunk_offset, chunk_size, &old_data)); 727 if (old_data == new_data) { 728 // No change in data. 729 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE); 730 current_best_size = 0; 731 data.clear(); 732 } else if (!old_data.empty() && bsdiff_allowed) { 733 // If the source file is considered bsdiff safe (no bsdiff bugs 734 // triggered), see if BSDIFF encoding is smaller. 735 base::FilePath old_chunk; 736 TEST_AND_RETURN_FALSE(base::CreateTemporaryFile(&old_chunk)); 737 ScopedPathUnlinker old_unlinker(old_chunk.value()); 738 TEST_AND_RETURN_FALSE( 739 utils::WriteFile(old_chunk.value().c_str(), 740 old_data.data(), old_data.size())); 741 base::FilePath new_chunk; 742 TEST_AND_RETURN_FALSE(base::CreateTemporaryFile(&new_chunk)); 743 ScopedPathUnlinker new_unlinker(new_chunk.value()); 744 TEST_AND_RETURN_FALSE( 745 utils::WriteFile(new_chunk.value().c_str(), 746 new_data.data(), new_data.size())); 747 748 chromeos::Blob bsdiff_delta; 749 TEST_AND_RETURN_FALSE( 750 BsdiffFiles(old_chunk.value(), new_chunk.value(), &bsdiff_delta)); 751 CHECK_GT(bsdiff_delta.size(), static_cast<chromeos::Blob::size_type>(0)); 752 if (bsdiff_delta.size() < current_best_size) { 753 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF); 754 current_best_size = bsdiff_delta.size(); 755 data = bsdiff_delta; 756 } 757 } 758 } 759 760 // Set parameters of the operations 761 CHECK_EQ(data.size(), current_best_size); 762 763 vector<Extent> src_extents, dst_extents; 764 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE || 765 operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF) { 766 if (gather_extents) { 767 TEST_AND_RETURN_FALSE( 768 GatherExtents(old_filename, 769 chunk_offset, 770 chunk_size, 771 &src_extents)); 772 } else { 773 Extent* src_extent = operation.add_src_extents(); 774 src_extent->set_start_block(0); 775 src_extent->set_num_blocks((old_size + kBlockSize - 1) / kBlockSize); 776 } 777 operation.set_src_length(old_data.size()); 778 } 779 780 if (gather_extents) { 781 TEST_AND_RETURN_FALSE( 782 GatherExtents(new_filename, 783 chunk_offset, 784 chunk_size, 785 &dst_extents)); 786 } else { 787 Extent* dst_extent = operation.add_dst_extents(); 788 dst_extent->set_start_block(0); 789 dst_extent->set_num_blocks((new_data.size() + kBlockSize - 1) / kBlockSize); 790 } 791 operation.set_dst_length(new_data.size()); 792 793 if (gather_extents) { 794 // Remove identical src/dst block ranges in MOVE operations. 795 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 796 size_t removed_bytes = RemoveIdenticalBlockRanges( 797 &src_extents, &dst_extents, new_data.size()); 798 799 // Adjust the file length field accordingly. 800 if (removed_bytes) { 801 operation.set_src_length(old_data.size() - removed_bytes); 802 operation.set_dst_length(new_data.size() - removed_bytes); 803 } 804 } 805 806 // Embed extents in the operation. 807 DeltaDiffGenerator::StoreExtents(src_extents, 808 operation.mutable_src_extents()); 809 DeltaDiffGenerator::StoreExtents(dst_extents, 810 operation.mutable_dst_extents()); 811 } 812 813 out_data->swap(data); 814 *out_op = operation; 815 816 return true; 817} 818 819bool DeltaDiffGenerator::InitializePartitionInfo(bool is_kernel, 820 const string& partition, 821 PartitionInfo* info) { 822 int64_t size = 0; 823 if (is_kernel) { 824 size = utils::FileSize(partition); 825 } else { 826 int block_count = 0, block_size = 0; 827 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(partition, 828 &block_count, 829 &block_size)); 830 size = static_cast<int64_t>(block_count) * block_size; 831 } 832 TEST_AND_RETURN_FALSE(size > 0); 833 info->set_size(size); 834 OmahaHashCalculator hasher; 835 TEST_AND_RETURN_FALSE(hasher.UpdateFile(partition, size) == size); 836 TEST_AND_RETURN_FALSE(hasher.Finalize()); 837 const chromeos::Blob& hash = hasher.raw_hash(); 838 info->set_hash(hash.data(), hash.size()); 839 LOG(INFO) << partition << ": size=" << size << " hash=" << hasher.hash(); 840 return true; 841} 842 843bool InitializePartitionInfos(const string& old_kernel, 844 const string& new_kernel, 845 const string& old_rootfs, 846 const string& new_rootfs, 847 DeltaArchiveManifest* manifest) { 848 if (!old_kernel.empty()) { 849 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 850 true, 851 old_kernel, 852 manifest->mutable_old_kernel_info())); 853 } 854 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 855 true, 856 new_kernel, 857 manifest->mutable_new_kernel_info())); 858 if (!old_rootfs.empty()) { 859 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 860 false, 861 old_rootfs, 862 manifest->mutable_old_rootfs_info())); 863 } 864 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 865 false, 866 new_rootfs, 867 manifest->mutable_new_rootfs_info())); 868 return true; 869} 870 871// Stores all Extents in 'extents' into 'out'. 872void DeltaDiffGenerator::StoreExtents( 873 const vector<Extent>& extents, 874 google::protobuf::RepeatedPtrField<Extent>* out) { 875 for (const Extent& extent : extents) { 876 Extent* new_extent = out->Add(); 877 *new_extent = extent; 878 } 879} 880 881// Returns true if |op| is a no-op operation that doesn't do any useful work 882// (e.g., a move operation that copies blocks onto themselves). 883bool DeltaDiffGenerator::IsNoopOperation( 884 const DeltaArchiveManifest_InstallOperation& op) { 885 return (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE && 886 ExpandExtents(op.src_extents()) == ExpandExtents(op.dst_extents())); 887} 888 889bool DeltaDiffGenerator::ReorderDataBlobs( 890 DeltaArchiveManifest* manifest, 891 const string& data_blobs_path, 892 const string& new_data_blobs_path) { 893 int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0); 894 TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0); 895 ScopedFdCloser in_fd_closer(&in_fd); 896 897 DirectFileWriter writer; 898 TEST_AND_RETURN_FALSE( 899 writer.Open(new_data_blobs_path.c_str(), 900 O_WRONLY | O_TRUNC | O_CREAT, 901 0644) == 0); 902 ScopedFileWriterCloser writer_closer(&writer); 903 uint64_t out_file_size = 0; 904 905 for (int i = 0; i < (manifest->install_operations_size() + 906 manifest->kernel_install_operations_size()); i++) { 907 DeltaArchiveManifest_InstallOperation* op = nullptr; 908 if (i < manifest->install_operations_size()) { 909 op = manifest->mutable_install_operations(i); 910 } else { 911 op = manifest->mutable_kernel_install_operations( 912 i - manifest->install_operations_size()); 913 } 914 if (!op->has_data_offset()) 915 continue; 916 CHECK(op->has_data_length()); 917 chromeos::Blob buf(op->data_length()); 918 ssize_t rc = pread(in_fd, buf.data(), buf.size(), op->data_offset()); 919 TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size())); 920 921 // Add the hash of the data blobs for this operation 922 TEST_AND_RETURN_FALSE(AddOperationHash(op, buf)); 923 924 op->set_data_offset(out_file_size); 925 TEST_AND_RETURN_FALSE(writer.Write(buf.data(), buf.size())); 926 out_file_size += buf.size(); 927 } 928 return true; 929} 930 931bool DeltaDiffGenerator::AddOperationHash( 932 DeltaArchiveManifest_InstallOperation* op, 933 const chromeos::Blob& buf) { 934 OmahaHashCalculator hasher; 935 936 TEST_AND_RETURN_FALSE(hasher.Update(buf.data(), buf.size())); 937 TEST_AND_RETURN_FALSE(hasher.Finalize()); 938 939 const chromeos::Blob& hash = hasher.raw_hash(); 940 op->set_data_sha256_hash(hash.data(), hash.size()); 941 return true; 942} 943 944bool DeltaDiffGenerator::GenerateDeltaUpdateFile( 945 const string& old_root, 946 const string& old_image, 947 const string& new_root, 948 const string& new_image, 949 const string& old_kernel_part, 950 const string& new_kernel_part, 951 const string& output_path, 952 const string& private_key_path, 953 off_t chunk_size, 954 size_t rootfs_partition_size, 955 uint32_t minor_version, 956 const ImageInfo* old_image_info, 957 const ImageInfo* new_image_info, 958 uint64_t* metadata_size) { 959 TEST_AND_RETURN_FALSE(chunk_size == -1 || chunk_size % kBlockSize == 0); 960 int old_image_block_count = 0, old_image_block_size = 0; 961 int new_image_block_count = 0, new_image_block_size = 0; 962 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(new_image, 963 &new_image_block_count, 964 &new_image_block_size)); 965 if (!old_image.empty()) { 966 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(old_image, 967 &old_image_block_count, 968 &old_image_block_size)); 969 TEST_AND_RETURN_FALSE(old_image_block_size == new_image_block_size); 970 LOG_IF(WARNING, old_image_block_count != new_image_block_count) 971 << "Old and new images have different block counts."; 972 973 // If new_image_info is present, old_image_info must be present. 974 TEST_AND_RETURN_FALSE(!old_image_info == !new_image_info); 975 } else { 976 // old_image_info must not be present for a full update. 977 TEST_AND_RETURN_FALSE(!old_image_info); 978 } 979 980 // Sanity checks for the partition size. 981 TEST_AND_RETURN_FALSE(rootfs_partition_size % kBlockSize == 0); 982 size_t fs_size = static_cast<size_t>(new_image_block_size) * 983 new_image_block_count; 984 LOG(INFO) << "Rootfs partition size: " << rootfs_partition_size; 985 LOG(INFO) << "Actual filesystem size: " << fs_size; 986 TEST_AND_RETURN_FALSE(rootfs_partition_size >= fs_size); 987 988 // Sanity check kernel partition arg 989 TEST_AND_RETURN_FALSE(utils::FileSize(new_kernel_part) >= 0); 990 991 vector<Block> blocks(max(old_image_block_count, new_image_block_count)); 992 LOG(INFO) << "Invalid block index: " << Vertex::kInvalidIndex; 993 LOG(INFO) << "Block count: " << blocks.size(); 994 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { 995 CHECK(blocks[i].reader == Vertex::kInvalidIndex); 996 CHECK(blocks[i].writer == Vertex::kInvalidIndex); 997 } 998 Graph graph; 999 CheckGraph(graph); 1000 1001 const string kTempFileTemplate("CrAU_temp_data.XXXXXX"); 1002 string temp_file_path; 1003 unique_ptr<ScopedPathUnlinker> temp_file_unlinker; 1004 off_t data_file_size = 0; 1005 1006 LOG(INFO) << "Reading files..."; 1007 1008 // Create empty protobuf Manifest object 1009 DeltaArchiveManifest manifest; 1010 1011 vector<DeltaArchiveManifest_InstallOperation> kernel_ops; 1012 1013 vector<Vertex::Index> final_order; 1014 Vertex::Index scratch_vertex = Vertex::kInvalidIndex; 1015 { 1016 int fd; 1017 TEST_AND_RETURN_FALSE( 1018 utils::MakeTempFile(kTempFileTemplate, &temp_file_path, &fd)); 1019 temp_file_unlinker.reset(new ScopedPathUnlinker(temp_file_path)); 1020 TEST_AND_RETURN_FALSE(fd >= 0); 1021 ScopedFdCloser fd_closer(&fd); 1022 if (!old_image.empty()) { 1023 // Delta update 1024 1025 // Set the minor version for this payload. 1026 LOG(INFO) << "Adding Delta Minor Version."; 1027 manifest.set_minor_version(minor_version); 1028 1029 TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph, 1030 &blocks, 1031 old_root, 1032 new_root, 1033 chunk_size, 1034 fd, 1035 &data_file_size)); 1036 LOG(INFO) << "done reading normal files"; 1037 CheckGraph(graph); 1038 1039 LOG(INFO) << "Starting metadata processing"; 1040 TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata(&graph, 1041 &blocks, 1042 old_image, 1043 new_image, 1044 fd, 1045 &data_file_size)); 1046 LOG(INFO) << "Done metadata processing"; 1047 CheckGraph(graph); 1048 1049 graph.resize(graph.size() + 1); 1050 TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks, 1051 fd, 1052 &data_file_size, 1053 old_image, 1054 new_image, 1055 &graph.back())); 1056 if (graph.back().op.data_length() == 0) { 1057 LOG(INFO) << "No unwritten blocks to write, omitting operation"; 1058 graph.pop_back(); 1059 } 1060 1061 // Final scratch block (if there's space) 1062 if (blocks.size() < (rootfs_partition_size / kBlockSize)) { 1063 scratch_vertex = graph.size(); 1064 graph.resize(graph.size() + 1); 1065 InplaceGenerator::CreateScratchNode( 1066 blocks.size(), 1067 (rootfs_partition_size / kBlockSize) - blocks.size(), 1068 &graph.back()); 1069 } 1070 1071 // Read kernel partition 1072 TEST_AND_RETURN_FALSE(DeltaCompressKernelPartition(old_kernel_part, 1073 new_kernel_part, 1074 &kernel_ops, 1075 fd, 1076 &data_file_size)); 1077 1078 LOG(INFO) << "done reading kernel"; 1079 CheckGraph(graph); 1080 1081 LOG(INFO) << "Creating edges..."; 1082 InplaceGenerator::CreateEdges(&graph, blocks); 1083 LOG(INFO) << "Done creating edges"; 1084 CheckGraph(graph); 1085 1086 TEST_AND_RETURN_FALSE(InplaceGenerator::ConvertGraphToDag( 1087 &graph, 1088 new_root, 1089 fd, 1090 &data_file_size, 1091 &final_order, 1092 scratch_vertex)); 1093 } else { 1094 // Full update 1095 off_t new_image_size = 1096 static_cast<off_t>(new_image_block_count) * new_image_block_size; 1097 TEST_AND_RETURN_FALSE(FullUpdateGenerator::Run(&graph, 1098 new_kernel_part, 1099 new_image, 1100 new_image_size, 1101 fd, 1102 &data_file_size, 1103 kFullUpdateChunkSize, 1104 kBlockSize, 1105 &kernel_ops, 1106 &final_order)); 1107 1108 // Set the minor version for this payload. 1109 LOG(INFO) << "Adding Full Minor Version."; 1110 manifest.set_minor_version(DeltaPerformer::kFullPayloadMinorVersion); 1111 } 1112 } 1113 1114 if (old_image_info) 1115 *(manifest.mutable_old_image_info()) = *old_image_info; 1116 1117 if (new_image_info) 1118 *(manifest.mutable_new_image_info()) = *new_image_info; 1119 1120 OperationNameMap op_name_map; 1121 CheckGraph(graph); 1122 InstallOperationsToManifest(graph, 1123 final_order, 1124 kernel_ops, 1125 &manifest, 1126 &op_name_map); 1127 CheckGraph(graph); 1128 manifest.set_block_size(kBlockSize); 1129 1130 // Reorder the data blobs with the newly ordered manifest 1131 string ordered_blobs_path; 1132 TEST_AND_RETURN_FALSE(utils::MakeTempFile( 1133 "CrAU_temp_data.ordered.XXXXXX", 1134 &ordered_blobs_path, 1135 nullptr)); 1136 ScopedPathUnlinker ordered_blobs_unlinker(ordered_blobs_path); 1137 TEST_AND_RETURN_FALSE(ReorderDataBlobs(&manifest, 1138 temp_file_path, 1139 ordered_blobs_path)); 1140 temp_file_unlinker.reset(); 1141 1142 // Check that install op blobs are in order. 1143 uint64_t next_blob_offset = 0; 1144 { 1145 for (int i = 0; i < (manifest.install_operations_size() + 1146 manifest.kernel_install_operations_size()); i++) { 1147 DeltaArchiveManifest_InstallOperation* op = 1148 i < manifest.install_operations_size() ? 1149 manifest.mutable_install_operations(i) : 1150 manifest.mutable_kernel_install_operations( 1151 i - manifest.install_operations_size()); 1152 if (op->has_data_offset()) { 1153 if (op->data_offset() != next_blob_offset) { 1154 LOG(FATAL) << "bad blob offset! " << op->data_offset() << " != " 1155 << next_blob_offset; 1156 } 1157 next_blob_offset += op->data_length(); 1158 } 1159 } 1160 } 1161 1162 // Signatures appear at the end of the blobs. Note the offset in the 1163 // manifest 1164 if (!private_key_path.empty()) { 1165 uint64_t signature_blob_length = 0; 1166 TEST_AND_RETURN_FALSE( 1167 PayloadSigner::SignatureBlobLength(vector<string>(1, private_key_path), 1168 &signature_blob_length)); 1169 AddSignatureOp(next_blob_offset, signature_blob_length, &manifest); 1170 } 1171 1172 TEST_AND_RETURN_FALSE(InitializePartitionInfos(old_kernel_part, 1173 new_kernel_part, 1174 old_image, 1175 new_image, 1176 &manifest)); 1177 1178 // Serialize protobuf 1179 string serialized_manifest; 1180 1181 CheckGraph(graph); 1182 TEST_AND_RETURN_FALSE(manifest.AppendToString(&serialized_manifest)); 1183 CheckGraph(graph); 1184 1185 LOG(INFO) << "Writing final delta file header..."; 1186 DirectFileWriter writer; 1187 TEST_AND_RETURN_FALSE_ERRNO(writer.Open(output_path.c_str(), 1188 O_WRONLY | O_CREAT | O_TRUNC, 1189 0644) == 0); 1190 ScopedFileWriterCloser writer_closer(&writer); 1191 1192 // Write header 1193 TEST_AND_RETURN_FALSE(writer.Write(kDeltaMagic, strlen(kDeltaMagic))); 1194 1195 // Write version number 1196 TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, kVersionNumber)); 1197 1198 // Write protobuf length 1199 TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, 1200 serialized_manifest.size())); 1201 1202 // Write protobuf 1203 LOG(INFO) << "Writing final delta file protobuf... " 1204 << serialized_manifest.size(); 1205 TEST_AND_RETURN_FALSE(writer.Write(serialized_manifest.data(), 1206 serialized_manifest.size())); 1207 1208 // Append the data blobs 1209 LOG(INFO) << "Writing final delta file data blobs..."; 1210 int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0); 1211 ScopedFdCloser blobs_fd_closer(&blobs_fd); 1212 TEST_AND_RETURN_FALSE(blobs_fd >= 0); 1213 for (;;) { 1214 char buf[kBlockSize]; 1215 ssize_t rc = read(blobs_fd, buf, sizeof(buf)); 1216 if (0 == rc) { 1217 // EOF 1218 break; 1219 } 1220 TEST_AND_RETURN_FALSE_ERRNO(rc > 0); 1221 TEST_AND_RETURN_FALSE(writer.Write(buf, rc)); 1222 } 1223 1224 // Write signature blob. 1225 if (!private_key_path.empty()) { 1226 LOG(INFO) << "Signing the update..."; 1227 chromeos::Blob signature_blob; 1228 TEST_AND_RETURN_FALSE(PayloadSigner::SignPayload( 1229 output_path, 1230 vector<string>(1, private_key_path), 1231 &signature_blob)); 1232 TEST_AND_RETURN_FALSE(writer.Write(signature_blob.data(), 1233 signature_blob.size())); 1234 } 1235 1236 *metadata_size = 1237 strlen(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size(); 1238 ReportPayloadUsage(manifest, *metadata_size, op_name_map); 1239 1240 LOG(INFO) << "All done. Successfully created delta file with " 1241 << "metadata size = " << *metadata_size; 1242 return true; 1243} 1244 1245// Runs the bsdiff tool on two files and returns the resulting delta in 1246// 'out'. Returns true on success. 1247bool DeltaDiffGenerator::BsdiffFiles(const string& old_file, 1248 const string& new_file, 1249 chromeos::Blob* out) { 1250 const string kPatchFile = "delta.patchXXXXXX"; 1251 string patch_file_path; 1252 1253 TEST_AND_RETURN_FALSE( 1254 utils::MakeTempFile(kPatchFile, &patch_file_path, nullptr)); 1255 1256 vector<string> cmd; 1257 cmd.push_back(kBsdiffPath); 1258 cmd.push_back(old_file); 1259 cmd.push_back(new_file); 1260 cmd.push_back(patch_file_path); 1261 1262 int rc = 1; 1263 chromeos::Blob patch_file; 1264 TEST_AND_RETURN_FALSE(Subprocess::SynchronousExec(cmd, &rc, nullptr)); 1265 TEST_AND_RETURN_FALSE(rc == 0); 1266 TEST_AND_RETURN_FALSE(utils::ReadFile(patch_file_path, out)); 1267 unlink(patch_file_path.c_str()); 1268 return true; 1269} 1270 1271void DeltaDiffGenerator::AddSignatureOp(uint64_t signature_blob_offset, 1272 uint64_t signature_blob_length, 1273 DeltaArchiveManifest* manifest) { 1274 LOG(INFO) << "Making room for signature in file"; 1275 manifest->set_signatures_offset(signature_blob_offset); 1276 LOG(INFO) << "set? " << manifest->has_signatures_offset(); 1277 // Add a dummy op at the end to appease older clients 1278 DeltaArchiveManifest_InstallOperation* dummy_op = 1279 manifest->add_kernel_install_operations(); 1280 dummy_op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE); 1281 dummy_op->set_data_offset(signature_blob_offset); 1282 manifest->set_signatures_offset(signature_blob_offset); 1283 dummy_op->set_data_length(signature_blob_length); 1284 manifest->set_signatures_size(signature_blob_length); 1285 Extent* dummy_extent = dummy_op->add_dst_extents(); 1286 // Tell the dummy op to write this data to a big sparse hole 1287 dummy_extent->set_start_block(kSparseHole); 1288 dummy_extent->set_num_blocks((signature_blob_length + kBlockSize - 1) / 1289 kBlockSize); 1290} 1291 1292}; // namespace chromeos_update_engine 1293