delta_diff_generator.cc revision fffd23e1ed1903beeb893640c611001175380186
13e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust// Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 23e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust// Use of this source code is governed by a BSD-style license that can be 33e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust// found in the LICENSE file. 43e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust 53e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust#include "update_engine/payload_generator/delta_diff_generator.h" 63e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust 73e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust#include <errno.h> 83e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust#include <fcntl.h> 93e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust#include <inttypes.h> 103e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust#include <sys/stat.h> 113e7cb52cbf4ca537cafb6617b20387225672ad1aCedric Beust#include <sys/types.h> 12 13#include <algorithm> 14#include <map> 15#include <memory> 16#include <string> 17#include <utility> 18#include <vector> 19 20#include <base/files/file_path.h> 21#include <base/files/file_util.h> 22#include <base/logging.h> 23#include <base/strings/stringprintf.h> 24#include <base/strings/string_number_conversions.h> 25#include <base/strings/string_util.h> 26#include <bzlib.h> 27 28#include "update_engine/bzip.h" 29#include "update_engine/delta_performer.h" 30#include "update_engine/file_writer.h" 31#include "update_engine/omaha_hash_calculator.h" 32#include "update_engine/payload_constants.h" 33#include "update_engine/payload_generator/extent_mapper.h" 34#include "update_engine/payload_generator/filesystem_iterator.h" 35#include "update_engine/payload_generator/full_update_generator.h" 36#include "update_engine/payload_generator/graph_types.h" 37#include "update_engine/payload_generator/graph_utils.h" 38#include "update_engine/payload_generator/inplace_generator.h" 39#include "update_engine/payload_generator/metadata.h" 40#include "update_engine/payload_generator/payload_signer.h" 41#include "update_engine/payload_verifier.h" 42#include "update_engine/subprocess.h" 43#include "update_engine/update_metadata.pb.h" 44#include "update_engine/utils.h" 45 46using std::map; 47using std::max; 48using std::min; 49using std::set; 50using std::string; 51using std::unique_ptr; 52using std::vector; 53 54namespace { 55 56const uint64_t kVersionNumber = 1; 57const uint64_t kFullUpdateChunkSize = 1024 * 1024; // bytes 58 59// The maximum destination size allowed for bsdiff. In general, bsdiff should 60// work for arbitrary big files, but the payload generation and payload 61// application requires a significant amount of RAM. We put a hard-limit of 62// 200 MiB that should not affect any released board, but will limit the 63// Chrome binary in ASan builders. 64const off_t kMaxBsdiffDestinationSize = 200 * 1024 * 1024; // bytes 65 66static const char* kInstallOperationTypes[] = { 67 "REPLACE", 68 "REPLACE_BZ", 69 "MOVE", 70 "BSDIFF", 71 "SOURCE_COPY", 72 "SOURCE_BSDIFF" 73}; 74 75} // namespace 76 77namespace chromeos_update_engine { 78 79typedef DeltaDiffGenerator::Block Block; 80typedef map<const DeltaArchiveManifest_InstallOperation*, 81 string> OperationNameMap; 82 83// bytes 84const size_t kRootFSPartitionSize = static_cast<size_t>(2) * 1024 * 1024 * 1024; 85const size_t kBlockSize = 4096; // bytes 86const char* const kEmptyPath = ""; 87const char* const kBsdiffPath = "bsdiff"; 88 89const uint32_t kInPlaceMinorPayloadVersion = 1; 90const uint32_t kSourceMinorPayloadVersion = 2; 91 92// Needed for testing purposes, in case we can't use actual filesystem objects. 93// TODO(garnold) (chromium:331965) Replace this hack with a properly injected 94// parameter in form of a mockable abstract class. 95bool (*get_extents_with_chunk_func)(const string&, off_t, off_t, 96 vector<Extent>*) = 97 extent_mapper::ExtentsForFileChunkFibmap; 98 99namespace { 100 101// Stores all the extents of |path| into |extents|. Returns true on success. 102bool GatherExtents(const string& path, 103 off_t chunk_offset, 104 off_t chunk_size, 105 vector<Extent>* extents) { 106 extents->clear(); 107 TEST_AND_RETURN_FALSE( 108 get_extents_with_chunk_func( 109 path, chunk_offset, chunk_size, extents)); 110 return true; 111} 112 113// For each regular file within new_root, creates a node in the graph, 114// determines the best way to compress it (REPLACE, REPLACE_BZ, COPY, BSDIFF), 115// and writes any necessary data to the end of data_fd. 116bool DeltaReadFiles(Graph* graph, 117 vector<Block>* blocks, 118 const string& old_root, 119 const string& new_root, 120 off_t chunk_size, 121 int data_fd, 122 off_t* data_file_size, 123 bool src_ops_allowed) { 124 set<ino_t> visited_inodes; 125 set<ino_t> visited_src_inodes; 126 for (FilesystemIterator fs_iter(new_root, 127 set<string>{"/lost+found"}); 128 !fs_iter.IsEnd(); fs_iter.Increment()) { 129 // We never diff symlinks (here, we check that dst file is not a symlink). 130 if (!S_ISREG(fs_iter.GetStat().st_mode)) 131 continue; 132 133 // Make sure we visit each inode only once. 134 if (utils::SetContainsKey(visited_inodes, fs_iter.GetStat().st_ino)) 135 continue; 136 visited_inodes.insert(fs_iter.GetStat().st_ino); 137 off_t dst_size = fs_iter.GetFileSize(); 138 if (dst_size == 0) 139 continue; 140 141 LOG(INFO) << "Encoding file " << fs_iter.GetPartialPath(); 142 143 // We can't visit each dst image inode more than once, as that would 144 // duplicate work. Here, we avoid visiting each source image inode 145 // more than once. Technically, we could have multiple operations 146 // that read the same blocks from the source image for diffing, but 147 // we choose not to avoid complexity. Eventually we will move away 148 // from using a graph/cycle detection/etc to generate diffs, and at that 149 // time, it will be easy (non-complex) to have many operations read 150 // from the same source blocks. At that time, this code can die. -adlr 151 bool should_diff_from_source = false; 152 string src_path = old_root + fs_iter.GetPartialPath(); 153 struct stat src_stbuf; 154 // We never diff symlinks (here, we check that src file is not a symlink). 155 if (0 == lstat(src_path.c_str(), &src_stbuf) && 156 S_ISREG(src_stbuf.st_mode)) { 157 should_diff_from_source = !utils::SetContainsKey(visited_src_inodes, 158 src_stbuf.st_ino); 159 visited_src_inodes.insert(src_stbuf.st_ino); 160 } 161 162 off_t size = chunk_size == -1 ? dst_size : chunk_size; 163 off_t step = size; 164 for (off_t offset = 0; offset < dst_size; offset += step) { 165 if (offset + size >= dst_size) { 166 size = -1; // Read through the end of the file. 167 } 168 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::DeltaReadFile( 169 graph, 170 Vertex::kInvalidIndex, 171 blocks, 172 (should_diff_from_source ? old_root : kEmptyPath), 173 new_root, 174 fs_iter.GetPartialPath(), 175 offset, 176 size, 177 data_fd, 178 data_file_size, 179 src_ops_allowed)); 180 } 181 } 182 return true; 183} 184 185// Reads blocks from image_path that are not yet marked as being written in the 186// blocks array. These blocks that remain are either unchanged files or 187// non-file-data blocks. We compare each of them to the old image, and compress 188// the ones that changed into a single REPLACE_BZ operation. This updates a 189// newly created node in the graph to write these blocks and writes the 190// appropriate blob to blobs_fd. Reads and updates blobs_length. 191bool ReadUnwrittenBlocks(const vector<Block>& blocks, 192 int blobs_fd, 193 off_t* blobs_length, 194 const string& old_image_path, 195 const string& new_image_path, 196 Vertex* vertex) { 197 vertex->file_name = "<rootfs-non-file-data>"; 198 199 DeltaArchiveManifest_InstallOperation* out_op = &vertex->op; 200 int new_image_fd = open(new_image_path.c_str(), O_RDONLY, 000); 201 TEST_AND_RETURN_FALSE_ERRNO(new_image_fd >= 0); 202 ScopedFdCloser new_image_fd_closer(&new_image_fd); 203 int old_image_fd = open(old_image_path.c_str(), O_RDONLY, 000); 204 TEST_AND_RETURN_FALSE_ERRNO(old_image_fd >= 0); 205 ScopedFdCloser old_image_fd_closer(&old_image_fd); 206 207 string temp_file_path; 208 TEST_AND_RETURN_FALSE(utils::MakeTempFile("CrAU_temp_data.XXXXXX", 209 &temp_file_path, 210 nullptr)); 211 212 FILE* file = fopen(temp_file_path.c_str(), "w"); 213 TEST_AND_RETURN_FALSE(file); 214 int err = BZ_OK; 215 216 BZFILE* bz_file = BZ2_bzWriteOpen(&err, 217 file, 218 9, // max compression 219 0, // verbosity 220 0); // default work factor 221 TEST_AND_RETURN_FALSE(err == BZ_OK); 222 223 vector<Extent> extents; 224 vector<Block>::size_type block_count = 0; 225 226 LOG(INFO) << "Appending unwritten blocks to extents"; 227 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { 228 if (blocks[i].writer != Vertex::kInvalidIndex) 229 continue; 230 graph_utils::AppendBlockToExtents(&extents, i); 231 block_count++; 232 } 233 234 // Code will handle buffers of any size that's a multiple of kBlockSize, 235 // so we arbitrarily set it to 1024 * kBlockSize. 236 chromeos::Blob new_buf(1024 * kBlockSize); 237 chromeos::Blob old_buf(1024 * kBlockSize); 238 239 LOG(INFO) << "Scanning " << block_count << " unwritten blocks"; 240 vector<Extent> changed_extents; 241 vector<Block>::size_type changed_block_count = 0; 242 vector<Block>::size_type blocks_copied_count = 0; 243 244 // For each extent in extents, write the unchanged blocks into BZ2_bzWrite, 245 // which sends it to an output file. We use the temporary buffers to hold the 246 // old and new data, which may be smaller than the extent, so in that case we 247 // have to loop to get the extent's data (that's the inner while loop). 248 for (const Extent& extent : extents) { 249 vector<Block>::size_type blocks_read = 0; 250 float printed_progress = -1; 251 while (blocks_read < extent.num_blocks()) { 252 const uint64_t copy_first_block = extent.start_block() + blocks_read; 253 const int copy_block_cnt = 254 min(new_buf.size() / kBlockSize, 255 static_cast<chromeos::Blob::size_type>( 256 extent.num_blocks() - blocks_read)); 257 const size_t count = copy_block_cnt * kBlockSize; 258 const off_t offset = copy_first_block * kBlockSize; 259 ssize_t rc = pread(new_image_fd, new_buf.data(), count, offset); 260 TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); 261 TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == count); 262 263 rc = pread(old_image_fd, old_buf.data(), count, offset); 264 TEST_AND_RETURN_FALSE_ERRNO(rc >= 0); 265 TEST_AND_RETURN_FALSE(static_cast<size_t>(rc) == count); 266 267 // Compare each block in the buffer to its counterpart in the old image 268 // and only compress it if its content has changed. 269 int buf_offset = 0; 270 for (int i = 0; i < copy_block_cnt; ++i) { 271 int buf_end_offset = buf_offset + kBlockSize; 272 if (!std::equal(new_buf.begin() + buf_offset, 273 new_buf.begin() + buf_end_offset, 274 old_buf.begin() + buf_offset)) { 275 BZ2_bzWrite(&err, bz_file, &new_buf[buf_offset], kBlockSize); 276 TEST_AND_RETURN_FALSE(err == BZ_OK); 277 const uint64_t block_idx = copy_first_block + i; 278 if (blocks[block_idx].reader != Vertex::kInvalidIndex) { 279 graph_utils::AddReadBeforeDep(vertex, blocks[block_idx].reader, 280 block_idx); 281 } 282 graph_utils::AppendBlockToExtents(&changed_extents, block_idx); 283 changed_block_count++; 284 } 285 buf_offset = buf_end_offset; 286 } 287 288 blocks_read += copy_block_cnt; 289 blocks_copied_count += copy_block_cnt; 290 float current_progress = 291 static_cast<float>(blocks_copied_count) / block_count; 292 if (printed_progress + 0.1 < current_progress || 293 blocks_copied_count == block_count) { 294 LOG(INFO) << "progress: " << current_progress; 295 printed_progress = current_progress; 296 } 297 } 298 } 299 BZ2_bzWriteClose(&err, bz_file, 0, nullptr, nullptr); 300 TEST_AND_RETURN_FALSE(err == BZ_OK); 301 bz_file = nullptr; 302 TEST_AND_RETURN_FALSE_ERRNO(0 == fclose(file)); 303 file = nullptr; 304 305 LOG(INFO) << "Compressed " << changed_block_count << " blocks (" 306 << block_count - changed_block_count << " blocks unchanged)"; 307 chromeos::Blob compressed_data; 308 if (changed_block_count > 0) { 309 LOG(INFO) << "Reading compressed data off disk"; 310 TEST_AND_RETURN_FALSE(utils::ReadFile(temp_file_path, &compressed_data)); 311 } 312 TEST_AND_RETURN_FALSE(unlink(temp_file_path.c_str()) == 0); 313 314 // Add node to graph to write these blocks 315 out_op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); 316 out_op->set_data_offset(*blobs_length); 317 out_op->set_data_length(compressed_data.size()); 318 LOG(INFO) << "Rootfs non-data blocks compressed take up " 319 << compressed_data.size(); 320 *blobs_length += compressed_data.size(); 321 out_op->set_dst_length(kBlockSize * changed_block_count); 322 DeltaDiffGenerator::StoreExtents(changed_extents, 323 out_op->mutable_dst_extents()); 324 325 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, 326 compressed_data.data(), 327 compressed_data.size())); 328 LOG(INFO) << "Done processing unwritten blocks"; 329 return true; 330} 331 332// Writes the uint64_t passed in in host-endian to the file as big-endian. 333// Returns true on success. 334bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) { 335 uint64_t value_be = htobe64(value); 336 TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be))); 337 return true; 338} 339 340// Adds each operation from |graph| to |out_manifest| in the order specified by 341// |order| while building |out_op_name_map| with operation to name 342// mappings. Adds all |kernel_ops| to |out_manifest|. Filters out no-op 343// operations. 344void InstallOperationsToManifest( 345 const Graph& graph, 346 const vector<Vertex::Index>& order, 347 const vector<DeltaArchiveManifest_InstallOperation>& kernel_ops, 348 DeltaArchiveManifest* out_manifest, 349 OperationNameMap* out_op_name_map) { 350 for (Vertex::Index vertex_index : order) { 351 const Vertex& vertex = graph[vertex_index]; 352 const DeltaArchiveManifest_InstallOperation& add_op = vertex.op; 353 if (DeltaDiffGenerator::IsNoopOperation(add_op)) { 354 continue; 355 } 356 DeltaArchiveManifest_InstallOperation* op = 357 out_manifest->add_install_operations(); 358 *op = add_op; 359 string name = vertex.file_name; 360 if (vertex.chunk_offset || vertex.chunk_size != -1) { 361 string offset = base::Int64ToString(vertex.chunk_offset); 362 if (vertex.chunk_size != -1) { 363 name += " [" + offset + ", " + 364 base::Int64ToString(vertex.chunk_offset + vertex.chunk_size - 1) + 365 "]"; 366 } else { 367 name += " [" + offset + ", end]"; 368 } 369 } 370 (*out_op_name_map)[op] = name; 371 } 372 for (vector<DeltaArchiveManifest_InstallOperation>::const_iterator it = 373 kernel_ops.begin(); it != kernel_ops.end(); ++it) { 374 const DeltaArchiveManifest_InstallOperation& add_op = *it; 375 if (DeltaDiffGenerator::IsNoopOperation(add_op)) { 376 continue; 377 } 378 DeltaArchiveManifest_InstallOperation* op = 379 out_manifest->add_kernel_install_operations(); 380 *op = add_op; 381 } 382} 383 384// Delta compresses a kernel partition |new_kernel_part| with knowledge of the 385// old kernel partition |old_kernel_part|. If |old_kernel_part| is an empty 386// string, generates a full update of the partition. 387bool DeltaCompressKernelPartition( 388 const string& old_kernel_part, 389 const string& new_kernel_part, 390 vector<DeltaArchiveManifest_InstallOperation>* ops, 391 int blobs_fd, 392 off_t* blobs_length, 393 bool src_ops_allowed) { 394 LOG(INFO) << "Delta compressing kernel partition..."; 395 LOG_IF(INFO, old_kernel_part.empty()) << "Generating full kernel update..."; 396 397 DeltaArchiveManifest_InstallOperation op; 398 chromeos::Blob data; 399 TEST_AND_RETURN_FALSE( 400 DeltaDiffGenerator::ReadFileToDiff(old_kernel_part, 401 new_kernel_part, 402 0, // chunk_offset 403 -1, // chunk_size 404 true, // bsdiff_allowed 405 &data, 406 &op, 407 false, 408 src_ops_allowed)); 409 410 // Check if the operation writes nothing. 411 if (op.dst_extents_size() == 0) { 412 if (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 413 LOG(INFO) << "Empty MOVE operation, nothing to do."; 414 return true; 415 } else { 416 LOG(ERROR) << "Empty non-MOVE operation"; 417 return false; 418 } 419 } 420 421 // Write the data. 422 if (op.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE && 423 op.type() != DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) { 424 op.set_data_offset(*blobs_length); 425 op.set_data_length(data.size()); 426 } 427 428 // Add the new install operation. 429 ops->clear(); 430 ops->push_back(op); 431 432 TEST_AND_RETURN_FALSE(utils::WriteAll(blobs_fd, data.data(), data.size())); 433 *blobs_length += data.size(); 434 435 LOG(INFO) << "Done delta compressing kernel partition: " 436 << kInstallOperationTypes[op.type()]; 437 return true; 438} 439 440struct DeltaObject { 441 DeltaObject(const string& in_name, const int in_type, const off_t in_size) 442 : name(in_name), 443 type(in_type), 444 size(in_size) {} 445 bool operator <(const DeltaObject& object) const { 446 return (size != object.size) ? (size < object.size) : (name < object.name); 447 } 448 string name; 449 int type; 450 off_t size; 451}; 452 453void ReportPayloadUsage(const DeltaArchiveManifest& manifest, 454 const int64_t manifest_metadata_size, 455 const OperationNameMap& op_name_map) { 456 vector<DeltaObject> objects; 457 off_t total_size = 0; 458 459 // Rootfs install operations. 460 for (int i = 0; i < manifest.install_operations_size(); ++i) { 461 const DeltaArchiveManifest_InstallOperation& op = 462 manifest.install_operations(i); 463 objects.push_back(DeltaObject(op_name_map.find(&op)->second, 464 op.type(), 465 op.data_length())); 466 total_size += op.data_length(); 467 } 468 469 // Kernel install operations. 470 for (int i = 0; i < manifest.kernel_install_operations_size(); ++i) { 471 const DeltaArchiveManifest_InstallOperation& op = 472 manifest.kernel_install_operations(i); 473 objects.push_back(DeltaObject(base::StringPrintf("<kernel-operation-%d>", 474 i), 475 op.type(), 476 op.data_length())); 477 total_size += op.data_length(); 478 } 479 480 objects.push_back(DeltaObject("<manifest-metadata>", 481 -1, 482 manifest_metadata_size)); 483 total_size += manifest_metadata_size; 484 485 std::sort(objects.begin(), objects.end()); 486 487 static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n"; 488 for (const DeltaObject& object : objects) { 489 fprintf(stderr, kFormatString, 490 object.size * 100.0 / total_size, 491 static_cast<intmax_t>(object.size), 492 object.type >= 0 ? kInstallOperationTypes[object.type] : "-", 493 object.name.c_str()); 494 } 495 fprintf(stderr, kFormatString, 496 100.0, static_cast<intmax_t>(total_size), "", "<total>"); 497} 498 499// Process a range of blocks from |range_start| to |range_end| in the extent at 500// position |*idx_p| of |extents|. If |do_remove| is true, this range will be 501// removed, which may cause the extent to be trimmed, split or removed entirely. 502// The value of |*idx_p| is updated to point to the next extent to be processed. 503// Returns true iff the next extent to process is a new or updated one. 504bool ProcessExtentBlockRange(vector<Extent>* extents, size_t* idx_p, 505 const bool do_remove, uint64_t range_start, 506 uint64_t range_end) { 507 size_t idx = *idx_p; 508 uint64_t start_block = (*extents)[idx].start_block(); 509 uint64_t num_blocks = (*extents)[idx].num_blocks(); 510 uint64_t range_size = range_end - range_start; 511 512 if (do_remove) { 513 if (range_size == num_blocks) { 514 // Remove the entire extent. 515 extents->erase(extents->begin() + idx); 516 } else if (range_end == num_blocks) { 517 // Trim the end of the extent. 518 (*extents)[idx].set_num_blocks(num_blocks - range_size); 519 idx++; 520 } else if (range_start == 0) { 521 // Trim the head of the extent. 522 (*extents)[idx].set_start_block(start_block + range_size); 523 (*extents)[idx].set_num_blocks(num_blocks - range_size); 524 } else { 525 // Trim the middle, splitting the remainder into two parts. 526 (*extents)[idx].set_num_blocks(range_start); 527 Extent e; 528 e.set_start_block(start_block + range_end); 529 e.set_num_blocks(num_blocks - range_end); 530 idx++; 531 extents->insert(extents->begin() + idx, e); 532 } 533 } else if (range_end == num_blocks) { 534 // Done with this extent. 535 idx++; 536 } else { 537 return false; 538 } 539 540 *idx_p = idx; 541 return true; 542} 543 544// Remove identical corresponding block ranges in |src_extents| and 545// |dst_extents|. Used for preventing moving of blocks onto themselves during 546// MOVE operations. The value of |total_bytes| indicates the actual length of 547// content; this may be slightly less than the total size of blocks, in which 548// case the last block is only partly occupied with data. Returns the total 549// number of bytes removed. 550size_t RemoveIdenticalBlockRanges(vector<Extent>* src_extents, 551 vector<Extent>* dst_extents, 552 const size_t total_bytes) { 553 size_t src_idx = 0; 554 size_t dst_idx = 0; 555 uint64_t src_offset = 0, dst_offset = 0; 556 bool new_src = true, new_dst = true; 557 size_t removed_bytes = 0, nonfull_block_bytes; 558 bool do_remove = false; 559 while (src_idx < src_extents->size() && dst_idx < dst_extents->size()) { 560 if (new_src) { 561 src_offset = 0; 562 new_src = false; 563 } 564 if (new_dst) { 565 dst_offset = 0; 566 new_dst = false; 567 } 568 569 do_remove = ((*src_extents)[src_idx].start_block() + src_offset == 570 (*dst_extents)[dst_idx].start_block() + dst_offset); 571 572 uint64_t src_num_blocks = (*src_extents)[src_idx].num_blocks(); 573 uint64_t dst_num_blocks = (*dst_extents)[dst_idx].num_blocks(); 574 uint64_t min_num_blocks = min(src_num_blocks - src_offset, 575 dst_num_blocks - dst_offset); 576 uint64_t prev_src_offset = src_offset; 577 uint64_t prev_dst_offset = dst_offset; 578 src_offset += min_num_blocks; 579 dst_offset += min_num_blocks; 580 581 new_src = ProcessExtentBlockRange(src_extents, &src_idx, do_remove, 582 prev_src_offset, src_offset); 583 new_dst = ProcessExtentBlockRange(dst_extents, &dst_idx, do_remove, 584 prev_dst_offset, dst_offset); 585 if (do_remove) 586 removed_bytes += min_num_blocks * kBlockSize; 587 } 588 589 // If we removed the last block and this block is only partly used by file 590 // content, deduct the unused portion from the total removed byte count. 591 if (do_remove && (nonfull_block_bytes = total_bytes % kBlockSize)) 592 removed_bytes -= kBlockSize - nonfull_block_bytes; 593 594 return removed_bytes; 595} 596 597} // namespace 598 599void DeltaDiffGenerator::CheckGraph(const Graph& graph) { 600 for (const Vertex& v : graph) { 601 CHECK(v.op.has_type()); 602 } 603} 604 605bool DeltaDiffGenerator::DeltaReadFile(Graph* graph, 606 Vertex::Index existing_vertex, 607 vector<Block>* blocks, 608 const string& old_root, 609 const string& new_root, 610 const string& path, // within new_root 611 off_t chunk_offset, 612 off_t chunk_size, 613 int data_fd, 614 off_t* data_file_size, 615 bool src_ops_allowed) { 616 chromeos::Blob data; 617 DeltaArchiveManifest_InstallOperation operation; 618 619 string old_path = (old_root == kEmptyPath) ? kEmptyPath : 620 old_root + path; 621 622 // If bsdiff breaks again, blacklist the problem file by using: 623 // bsdiff_allowed = (path != "/foo/bar") 624 // 625 // TODO(dgarrett): chromium-os:15274 connect this test to the command line. 626 bool bsdiff_allowed = true; 627 628 if (utils::FileSize(new_root + path) > kMaxBsdiffDestinationSize) 629 bsdiff_allowed = false; 630 631 if (!bsdiff_allowed) 632 LOG(INFO) << "bsdiff blacklisting: " << path; 633 634 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::ReadFileToDiff(old_path, 635 new_root + path, 636 chunk_offset, 637 chunk_size, 638 bsdiff_allowed, 639 &data, 640 &operation, 641 true, 642 src_ops_allowed)); 643 644 // Check if the operation writes nothing. 645 if (operation.dst_extents_size() == 0) { 646 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 647 LOG(INFO) << "Empty MOVE operation (" 648 << new_root + path << "), skipping"; 649 return true; 650 } else { 651 LOG(ERROR) << "Empty non-MOVE operation"; 652 return false; 653 } 654 } 655 656 // Write the data 657 if (operation.type() != DeltaArchiveManifest_InstallOperation_Type_MOVE && 658 operation.type() != 659 DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY) { 660 operation.set_data_offset(*data_file_size); 661 operation.set_data_length(data.size()); 662 } 663 664 TEST_AND_RETURN_FALSE(utils::WriteAll(data_fd, data.data(), data.size())); 665 *data_file_size += data.size(); 666 667 // Now, insert into graph and blocks vector 668 Vertex::Index vertex = existing_vertex; 669 if (vertex == Vertex::kInvalidIndex) { 670 graph->resize(graph->size() + 1); 671 vertex = graph->size() - 1; 672 } 673 (*graph)[vertex].op = operation; 674 CHECK((*graph)[vertex].op.has_type()); 675 (*graph)[vertex].file_name = path; 676 (*graph)[vertex].chunk_offset = chunk_offset; 677 (*graph)[vertex].chunk_size = chunk_size; 678 679 if (blocks) 680 TEST_AND_RETURN_FALSE(InplaceGenerator::AddInstallOpToBlocksVector( 681 (*graph)[vertex].op, 682 *graph, 683 vertex, 684 blocks)); 685 return true; 686} 687 688 689bool DeltaDiffGenerator::ReadFileToDiff( 690 const string& old_filename, 691 const string& new_filename, 692 off_t chunk_offset, 693 off_t chunk_size, 694 bool bsdiff_allowed, 695 chromeos::Blob* out_data, 696 DeltaArchiveManifest_InstallOperation* out_op, 697 bool gather_extents, 698 bool src_ops_allowed) { 699 // Read new data in 700 chromeos::Blob new_data; 701 TEST_AND_RETURN_FALSE( 702 utils::ReadFileChunk(new_filename, chunk_offset, chunk_size, &new_data)); 703 704 TEST_AND_RETURN_FALSE(!new_data.empty()); 705 TEST_AND_RETURN_FALSE(chunk_size == -1 || 706 static_cast<off_t>(new_data.size()) <= chunk_size); 707 708 chromeos::Blob new_data_bz; 709 TEST_AND_RETURN_FALSE(BzipCompress(new_data, &new_data_bz)); 710 CHECK(!new_data_bz.empty()); 711 712 chromeos::Blob data; // Data blob that will be written to delta file. 713 714 DeltaArchiveManifest_InstallOperation operation; 715 size_t current_best_size = 0; 716 if (new_data.size() <= new_data_bz.size()) { 717 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE); 718 current_best_size = new_data.size(); 719 data = new_data; 720 } else { 721 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ); 722 current_best_size = new_data_bz.size(); 723 data = new_data_bz; 724 } 725 726 // Do we have an original file to consider? 727 off_t old_size = 0; 728 bool original = !old_filename.empty(); 729 if (original && (old_size = utils::FileSize(old_filename)) < 0) { 730 // If stat-ing the old file fails, it should be because it doesn't exist. 731 TEST_AND_RETURN_FALSE(!utils::FileExists(old_filename.c_str())); 732 original = false; 733 } 734 735 chromeos::Blob old_data; 736 if (original) { 737 // Read old data 738 TEST_AND_RETURN_FALSE( 739 utils::ReadFileChunk( 740 old_filename, chunk_offset, chunk_size, &old_data)); 741 if (old_data == new_data) { 742 // No change in data. 743 if (src_ops_allowed) { 744 operation.set_type( 745 DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY); 746 } else { 747 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_MOVE); 748 } 749 current_best_size = 0; 750 data.clear(); 751 } else if (!old_data.empty() && bsdiff_allowed) { 752 // If the source file is considered bsdiff safe (no bsdiff bugs 753 // triggered), see if BSDIFF encoding is smaller. 754 base::FilePath old_chunk; 755 TEST_AND_RETURN_FALSE(base::CreateTemporaryFile(&old_chunk)); 756 ScopedPathUnlinker old_unlinker(old_chunk.value()); 757 TEST_AND_RETURN_FALSE( 758 utils::WriteFile(old_chunk.value().c_str(), 759 old_data.data(), old_data.size())); 760 base::FilePath new_chunk; 761 TEST_AND_RETURN_FALSE(base::CreateTemporaryFile(&new_chunk)); 762 ScopedPathUnlinker new_unlinker(new_chunk.value()); 763 TEST_AND_RETURN_FALSE( 764 utils::WriteFile(new_chunk.value().c_str(), 765 new_data.data(), new_data.size())); 766 767 chromeos::Blob bsdiff_delta; 768 TEST_AND_RETURN_FALSE( 769 BsdiffFiles(old_chunk.value(), new_chunk.value(), &bsdiff_delta)); 770 CHECK_GT(bsdiff_delta.size(), static_cast<chromeos::Blob::size_type>(0)); 771 if (bsdiff_delta.size() < current_best_size) { 772 if (src_ops_allowed) { 773 operation.set_type( 774 DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF); 775 } else { 776 operation.set_type(DeltaArchiveManifest_InstallOperation_Type_BSDIFF); 777 } 778 current_best_size = bsdiff_delta.size(); 779 data = bsdiff_delta; 780 } 781 } 782 } 783 784 // Set parameters of the operations 785 CHECK_EQ(data.size(), current_best_size); 786 787 vector<Extent> src_extents, dst_extents; 788 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE || 789 operation.type() == DeltaArchiveManifest_InstallOperation_Type_BSDIFF || 790 operation.type() == 791 DeltaArchiveManifest_InstallOperation_Type_SOURCE_COPY || 792 operation.type() == 793 DeltaArchiveManifest_InstallOperation_Type_SOURCE_BSDIFF) { 794 if (gather_extents) { 795 TEST_AND_RETURN_FALSE( 796 GatherExtents(old_filename, 797 chunk_offset, 798 chunk_size, 799 &src_extents)); 800 } else { 801 Extent* src_extent = operation.add_src_extents(); 802 src_extent->set_start_block(0); 803 src_extent->set_num_blocks((old_size + kBlockSize - 1) / kBlockSize); 804 } 805 operation.set_src_length(old_data.size()); 806 } 807 808 if (gather_extents) { 809 TEST_AND_RETURN_FALSE( 810 GatherExtents(new_filename, 811 chunk_offset, 812 chunk_size, 813 &dst_extents)); 814 } else { 815 Extent* dst_extent = operation.add_dst_extents(); 816 dst_extent->set_start_block(0); 817 dst_extent->set_num_blocks((new_data.size() + kBlockSize - 1) / kBlockSize); 818 } 819 operation.set_dst_length(new_data.size()); 820 821 if (gather_extents) { 822 // Remove identical src/dst block ranges in MOVE operations. 823 if (operation.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE) { 824 size_t removed_bytes = RemoveIdenticalBlockRanges( 825 &src_extents, &dst_extents, new_data.size()); 826 827 // Adjust the file length field accordingly. 828 if (removed_bytes) { 829 operation.set_src_length(old_data.size() - removed_bytes); 830 operation.set_dst_length(new_data.size() - removed_bytes); 831 } 832 } 833 834 // Embed extents in the operation. 835 DeltaDiffGenerator::StoreExtents(src_extents, 836 operation.mutable_src_extents()); 837 DeltaDiffGenerator::StoreExtents(dst_extents, 838 operation.mutable_dst_extents()); 839 } 840 841 out_data->swap(data); 842 *out_op = operation; 843 844 return true; 845} 846 847bool DeltaDiffGenerator::InitializePartitionInfo(bool is_kernel, 848 const string& partition, 849 PartitionInfo* info) { 850 int64_t size = 0; 851 if (is_kernel) { 852 size = utils::FileSize(partition); 853 } else { 854 int block_count = 0, block_size = 0; 855 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(partition, 856 &block_count, 857 &block_size)); 858 size = static_cast<int64_t>(block_count) * block_size; 859 } 860 TEST_AND_RETURN_FALSE(size > 0); 861 info->set_size(size); 862 OmahaHashCalculator hasher; 863 TEST_AND_RETURN_FALSE(hasher.UpdateFile(partition, size) == size); 864 TEST_AND_RETURN_FALSE(hasher.Finalize()); 865 const chromeos::Blob& hash = hasher.raw_hash(); 866 info->set_hash(hash.data(), hash.size()); 867 LOG(INFO) << partition << ": size=" << size << " hash=" << hasher.hash(); 868 return true; 869} 870 871bool InitializePartitionInfos(const string& old_kernel, 872 const string& new_kernel, 873 const string& old_rootfs, 874 const string& new_rootfs, 875 DeltaArchiveManifest* manifest) { 876 if (!old_kernel.empty()) { 877 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 878 true, 879 old_kernel, 880 manifest->mutable_old_kernel_info())); 881 } 882 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 883 true, 884 new_kernel, 885 manifest->mutable_new_kernel_info())); 886 if (!old_rootfs.empty()) { 887 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 888 false, 889 old_rootfs, 890 manifest->mutable_old_rootfs_info())); 891 } 892 TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo( 893 false, 894 new_rootfs, 895 manifest->mutable_new_rootfs_info())); 896 return true; 897} 898 899// Stores all Extents in 'extents' into 'out'. 900void DeltaDiffGenerator::StoreExtents( 901 const vector<Extent>& extents, 902 google::protobuf::RepeatedPtrField<Extent>* out) { 903 for (const Extent& extent : extents) { 904 Extent* new_extent = out->Add(); 905 *new_extent = extent; 906 } 907} 908 909// Returns true if |op| is a no-op operation that doesn't do any useful work 910// (e.g., a move operation that copies blocks onto themselves). 911bool DeltaDiffGenerator::IsNoopOperation( 912 const DeltaArchiveManifest_InstallOperation& op) { 913 return (op.type() == DeltaArchiveManifest_InstallOperation_Type_MOVE && 914 ExpandExtents(op.src_extents()) == ExpandExtents(op.dst_extents())); 915} 916 917bool DeltaDiffGenerator::ReorderDataBlobs( 918 DeltaArchiveManifest* manifest, 919 const string& data_blobs_path, 920 const string& new_data_blobs_path) { 921 int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0); 922 TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0); 923 ScopedFdCloser in_fd_closer(&in_fd); 924 925 DirectFileWriter writer; 926 TEST_AND_RETURN_FALSE( 927 writer.Open(new_data_blobs_path.c_str(), 928 O_WRONLY | O_TRUNC | O_CREAT, 929 0644) == 0); 930 ScopedFileWriterCloser writer_closer(&writer); 931 uint64_t out_file_size = 0; 932 933 for (int i = 0; i < (manifest->install_operations_size() + 934 manifest->kernel_install_operations_size()); i++) { 935 DeltaArchiveManifest_InstallOperation* op = nullptr; 936 if (i < manifest->install_operations_size()) { 937 op = manifest->mutable_install_operations(i); 938 } else { 939 op = manifest->mutable_kernel_install_operations( 940 i - manifest->install_operations_size()); 941 } 942 if (!op->has_data_offset()) 943 continue; 944 CHECK(op->has_data_length()); 945 chromeos::Blob buf(op->data_length()); 946 ssize_t rc = pread(in_fd, buf.data(), buf.size(), op->data_offset()); 947 TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size())); 948 949 // Add the hash of the data blobs for this operation 950 TEST_AND_RETURN_FALSE(AddOperationHash(op, buf)); 951 952 op->set_data_offset(out_file_size); 953 TEST_AND_RETURN_FALSE(writer.Write(buf.data(), buf.size())); 954 out_file_size += buf.size(); 955 } 956 return true; 957} 958 959bool DeltaDiffGenerator::AddOperationHash( 960 DeltaArchiveManifest_InstallOperation* op, 961 const chromeos::Blob& buf) { 962 OmahaHashCalculator hasher; 963 964 TEST_AND_RETURN_FALSE(hasher.Update(buf.data(), buf.size())); 965 TEST_AND_RETURN_FALSE(hasher.Finalize()); 966 967 const chromeos::Blob& hash = hasher.raw_hash(); 968 op->set_data_sha256_hash(hash.data(), hash.size()); 969 return true; 970} 971 972vector<Vertex::Index> DeltaDiffGenerator::OrderIndices(const Graph& graph) { 973 vector<Vertex::Index> order(graph.size()); 974 std::iota(order.begin(), order.end(), 0); 975 return order; 976} 977 978bool DeltaDiffGenerator::GenerateDeltaUpdateFile( 979 const string& old_root, 980 const string& old_image, 981 const string& new_root, 982 const string& new_image, 983 const string& old_kernel_part, 984 const string& new_kernel_part, 985 const string& output_path, 986 const string& private_key_path, 987 off_t chunk_size, 988 size_t rootfs_partition_size, 989 uint32_t minor_version, 990 const ImageInfo* old_image_info, 991 const ImageInfo* new_image_info, 992 uint64_t* metadata_size) { 993 TEST_AND_RETURN_FALSE(chunk_size == -1 || chunk_size % kBlockSize == 0); 994 int old_image_block_count = 0, old_image_block_size = 0; 995 int new_image_block_count = 0, new_image_block_size = 0; 996 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(new_image, 997 &new_image_block_count, 998 &new_image_block_size)); 999 if (!old_image.empty()) { 1000 TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(old_image, 1001 &old_image_block_count, 1002 &old_image_block_size)); 1003 TEST_AND_RETURN_FALSE(old_image_block_size == new_image_block_size); 1004 LOG_IF(WARNING, old_image_block_count != new_image_block_count) 1005 << "Old and new images have different block counts."; 1006 1007 // If new_image_info is present, old_image_info must be present. 1008 TEST_AND_RETURN_FALSE(!old_image_info == !new_image_info); 1009 } else { 1010 // old_image_info must not be present for a full update. 1011 TEST_AND_RETURN_FALSE(!old_image_info); 1012 } 1013 1014 // Sanity checks for the partition size. 1015 TEST_AND_RETURN_FALSE(rootfs_partition_size % kBlockSize == 0); 1016 size_t fs_size = static_cast<size_t>(new_image_block_size) * 1017 new_image_block_count; 1018 LOG(INFO) << "Rootfs partition size: " << rootfs_partition_size; 1019 LOG(INFO) << "Actual filesystem size: " << fs_size; 1020 TEST_AND_RETURN_FALSE(rootfs_partition_size >= fs_size); 1021 1022 // Sanity check kernel partition arg 1023 TEST_AND_RETURN_FALSE(utils::FileSize(new_kernel_part) >= 0); 1024 1025 vector<Block> blocks(max(old_image_block_count, new_image_block_count)); 1026 LOG(INFO) << "Invalid block index: " << Vertex::kInvalidIndex; 1027 LOG(INFO) << "Block count: " << blocks.size(); 1028 for (vector<Block>::size_type i = 0; i < blocks.size(); i++) { 1029 CHECK(blocks[i].reader == Vertex::kInvalidIndex); 1030 CHECK(blocks[i].writer == Vertex::kInvalidIndex); 1031 } 1032 Graph graph; 1033 CheckGraph(graph); 1034 1035 const string kTempFileTemplate("CrAU_temp_data.XXXXXX"); 1036 string temp_file_path; 1037 unique_ptr<ScopedPathUnlinker> temp_file_unlinker; 1038 off_t data_file_size = 0; 1039 1040 LOG(INFO) << "Reading files..."; 1041 1042 // Create empty protobuf Manifest object 1043 DeltaArchiveManifest manifest; 1044 1045 vector<DeltaArchiveManifest_InstallOperation> kernel_ops; 1046 1047 vector<Vertex::Index> final_order; 1048 Vertex::Index scratch_vertex = Vertex::kInvalidIndex; 1049 { 1050 int fd; 1051 TEST_AND_RETURN_FALSE( 1052 utils::MakeTempFile(kTempFileTemplate, &temp_file_path, &fd)); 1053 temp_file_unlinker.reset(new ScopedPathUnlinker(temp_file_path)); 1054 TEST_AND_RETURN_FALSE(fd >= 0); 1055 ScopedFdCloser fd_closer(&fd); 1056 if (!old_image.empty()) { 1057 // Delta update 1058 1059 // Set the minor version for this payload. 1060 LOG(INFO) << "Adding Delta Minor Version."; 1061 manifest.set_minor_version(minor_version); 1062 if (minor_version == kInPlaceMinorPayloadVersion) { 1063 TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph, 1064 &blocks, 1065 old_root, 1066 new_root, 1067 chunk_size, 1068 fd, 1069 &data_file_size, 1070 false)); // src_ops_allowed 1071 LOG(INFO) << "done reading normal files"; 1072 CheckGraph(graph); 1073 1074 LOG(INFO) << "Starting metadata processing"; 1075 TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata(&graph, 1076 &blocks, 1077 old_image, 1078 new_image, 1079 fd, 1080 &data_file_size)); 1081 LOG(INFO) << "Done metadata processing"; 1082 CheckGraph(graph); 1083 1084 graph.resize(graph.size() + 1); 1085 TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks, 1086 fd, 1087 &data_file_size, 1088 old_image, 1089 new_image, 1090 &graph.back())); 1091 if (graph.back().op.data_length() == 0) { 1092 LOG(INFO) << "No unwritten blocks to write, omitting operation"; 1093 graph.pop_back(); 1094 } 1095 1096 // Final scratch block (if there's space) 1097 if (blocks.size() < (rootfs_partition_size / kBlockSize)) { 1098 scratch_vertex = graph.size(); 1099 graph.resize(graph.size() + 1); 1100 InplaceGenerator::CreateScratchNode( 1101 blocks.size(), 1102 (rootfs_partition_size / kBlockSize) - blocks.size(), 1103 &graph.back()); 1104 } 1105 1106 // Read kernel partition 1107 TEST_AND_RETURN_FALSE( 1108 DeltaCompressKernelPartition(old_kernel_part, 1109 new_kernel_part, 1110 &kernel_ops, 1111 fd, 1112 &data_file_size, 1113 false)); // src_ops_allowed 1114 1115 LOG(INFO) << "done reading kernel"; 1116 CheckGraph(graph); 1117 1118 LOG(INFO) << "Creating edges..."; 1119 InplaceGenerator::CreateEdges(&graph, blocks); 1120 LOG(INFO) << "Done creating edges"; 1121 CheckGraph(graph); 1122 1123 TEST_AND_RETURN_FALSE(InplaceGenerator::ConvertGraphToDag( 1124 &graph, 1125 new_root, 1126 fd, 1127 &data_file_size, 1128 &final_order, 1129 scratch_vertex)); 1130 } else if (minor_version == kSourceMinorPayloadVersion) { 1131 TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph, 1132 &blocks, 1133 old_root, 1134 new_root, 1135 chunk_size, 1136 fd, 1137 &data_file_size, 1138 true)); // src_ops_allowed 1139 1140 LOG(INFO) << "done reading normal files"; 1141 CheckGraph(graph); 1142 1143 // Read kernel partition 1144 TEST_AND_RETURN_FALSE( 1145 DeltaCompressKernelPartition(old_kernel_part, 1146 new_kernel_part, 1147 &kernel_ops, 1148 fd, 1149 &data_file_size, 1150 true)); // src_ops_allowed 1151 LOG(INFO) << "done reading kernel"; 1152 CheckGraph(graph); 1153 1154 graph.resize(graph.size() + 1); 1155 TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks, 1156 fd, 1157 &data_file_size, 1158 old_image, 1159 new_image, 1160 &graph.back())); 1161 if (graph.back().op.data_length() == 0) { 1162 LOG(INFO) << "No unwritten blocks to write, omitting operation"; 1163 graph.pop_back(); 1164 } 1165 1166 final_order = OrderIndices(graph); 1167 } else { 1168 LOG(ERROR) << "Unsupported minor version given for delta payload: " 1169 << minor_version; 1170 return false; 1171 } 1172 } else { 1173 // Full update 1174 off_t new_image_size = 1175 static_cast<off_t>(new_image_block_count) * new_image_block_size; 1176 TEST_AND_RETURN_FALSE(FullUpdateGenerator::Run(&graph, 1177 new_kernel_part, 1178 new_image, 1179 new_image_size, 1180 fd, 1181 &data_file_size, 1182 kFullUpdateChunkSize, 1183 kBlockSize, 1184 &kernel_ops, 1185 &final_order)); 1186 1187 // Set the minor version for this payload. 1188 LOG(INFO) << "Adding Full Minor Version."; 1189 manifest.set_minor_version(DeltaPerformer::kFullPayloadMinorVersion); 1190 } 1191 } 1192 1193 if (old_image_info) 1194 *(manifest.mutable_old_image_info()) = *old_image_info; 1195 1196 if (new_image_info) 1197 *(manifest.mutable_new_image_info()) = *new_image_info; 1198 1199 OperationNameMap op_name_map; 1200 CheckGraph(graph); 1201 InstallOperationsToManifest(graph, 1202 final_order, 1203 kernel_ops, 1204 &manifest, 1205 &op_name_map); 1206 CheckGraph(graph); 1207 manifest.set_block_size(kBlockSize); 1208 1209 // Reorder the data blobs with the newly ordered manifest 1210 string ordered_blobs_path; 1211 TEST_AND_RETURN_FALSE(utils::MakeTempFile( 1212 "CrAU_temp_data.ordered.XXXXXX", 1213 &ordered_blobs_path, 1214 nullptr)); 1215 ScopedPathUnlinker ordered_blobs_unlinker(ordered_blobs_path); 1216 TEST_AND_RETURN_FALSE(ReorderDataBlobs(&manifest, 1217 temp_file_path, 1218 ordered_blobs_path)); 1219 temp_file_unlinker.reset(); 1220 1221 // Check that install op blobs are in order. 1222 uint64_t next_blob_offset = 0; 1223 { 1224 for (int i = 0; i < (manifest.install_operations_size() + 1225 manifest.kernel_install_operations_size()); i++) { 1226 DeltaArchiveManifest_InstallOperation* op = 1227 i < manifest.install_operations_size() ? 1228 manifest.mutable_install_operations(i) : 1229 manifest.mutable_kernel_install_operations( 1230 i - manifest.install_operations_size()); 1231 if (op->has_data_offset()) { 1232 if (op->data_offset() != next_blob_offset) { 1233 LOG(FATAL) << "bad blob offset! " << op->data_offset() << " != " 1234 << next_blob_offset; 1235 } 1236 next_blob_offset += op->data_length(); 1237 } 1238 } 1239 } 1240 1241 // Signatures appear at the end of the blobs. Note the offset in the 1242 // manifest 1243 if (!private_key_path.empty()) { 1244 uint64_t signature_blob_length = 0; 1245 TEST_AND_RETURN_FALSE( 1246 PayloadSigner::SignatureBlobLength(vector<string>(1, private_key_path), 1247 &signature_blob_length)); 1248 AddSignatureOp(next_blob_offset, signature_blob_length, &manifest); 1249 } 1250 1251 TEST_AND_RETURN_FALSE(InitializePartitionInfos(old_kernel_part, 1252 new_kernel_part, 1253 old_image, 1254 new_image, 1255 &manifest)); 1256 1257 // Serialize protobuf 1258 string serialized_manifest; 1259 1260 CheckGraph(graph); 1261 TEST_AND_RETURN_FALSE(manifest.AppendToString(&serialized_manifest)); 1262 CheckGraph(graph); 1263 1264 LOG(INFO) << "Writing final delta file header..."; 1265 DirectFileWriter writer; 1266 TEST_AND_RETURN_FALSE_ERRNO(writer.Open(output_path.c_str(), 1267 O_WRONLY | O_CREAT | O_TRUNC, 1268 0644) == 0); 1269 ScopedFileWriterCloser writer_closer(&writer); 1270 1271 // Write header 1272 TEST_AND_RETURN_FALSE(writer.Write(kDeltaMagic, strlen(kDeltaMagic))); 1273 1274 // Write version number 1275 TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, kVersionNumber)); 1276 1277 // Write protobuf length 1278 TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, 1279 serialized_manifest.size())); 1280 1281 // Write protobuf 1282 LOG(INFO) << "Writing final delta file protobuf... " 1283 << serialized_manifest.size(); 1284 TEST_AND_RETURN_FALSE(writer.Write(serialized_manifest.data(), 1285 serialized_manifest.size())); 1286 1287 // Append the data blobs 1288 LOG(INFO) << "Writing final delta file data blobs..."; 1289 int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0); 1290 ScopedFdCloser blobs_fd_closer(&blobs_fd); 1291 TEST_AND_RETURN_FALSE(blobs_fd >= 0); 1292 for (;;) { 1293 char buf[kBlockSize]; 1294 ssize_t rc = read(blobs_fd, buf, sizeof(buf)); 1295 if (0 == rc) { 1296 // EOF 1297 break; 1298 } 1299 TEST_AND_RETURN_FALSE_ERRNO(rc > 0); 1300 TEST_AND_RETURN_FALSE(writer.Write(buf, rc)); 1301 } 1302 1303 // Write signature blob. 1304 if (!private_key_path.empty()) { 1305 LOG(INFO) << "Signing the update..."; 1306 chromeos::Blob signature_blob; 1307 TEST_AND_RETURN_FALSE(PayloadSigner::SignPayload( 1308 output_path, 1309 vector<string>(1, private_key_path), 1310 &signature_blob)); 1311 TEST_AND_RETURN_FALSE(writer.Write(signature_blob.data(), 1312 signature_blob.size())); 1313 } 1314 1315 *metadata_size = 1316 strlen(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size(); 1317 ReportPayloadUsage(manifest, *metadata_size, op_name_map); 1318 1319 LOG(INFO) << "All done. Successfully created delta file with " 1320 << "metadata size = " << *metadata_size; 1321 return true; 1322} 1323 1324// Runs the bsdiff tool on two files and returns the resulting delta in 1325// 'out'. Returns true on success. 1326bool DeltaDiffGenerator::BsdiffFiles(const string& old_file, 1327 const string& new_file, 1328 chromeos::Blob* out) { 1329 const string kPatchFile = "delta.patchXXXXXX"; 1330 string patch_file_path; 1331 1332 TEST_AND_RETURN_FALSE( 1333 utils::MakeTempFile(kPatchFile, &patch_file_path, nullptr)); 1334 1335 vector<string> cmd; 1336 cmd.push_back(kBsdiffPath); 1337 cmd.push_back(old_file); 1338 cmd.push_back(new_file); 1339 cmd.push_back(patch_file_path); 1340 1341 int rc = 1; 1342 chromeos::Blob patch_file; 1343 TEST_AND_RETURN_FALSE(Subprocess::SynchronousExec(cmd, &rc, nullptr)); 1344 TEST_AND_RETURN_FALSE(rc == 0); 1345 TEST_AND_RETURN_FALSE(utils::ReadFile(patch_file_path, out)); 1346 unlink(patch_file_path.c_str()); 1347 return true; 1348} 1349 1350void DeltaDiffGenerator::AddSignatureOp(uint64_t signature_blob_offset, 1351 uint64_t signature_blob_length, 1352 DeltaArchiveManifest* manifest) { 1353 LOG(INFO) << "Making room for signature in file"; 1354 manifest->set_signatures_offset(signature_blob_offset); 1355 LOG(INFO) << "set? " << manifest->has_signatures_offset(); 1356 // Add a dummy op at the end to appease older clients 1357 DeltaArchiveManifest_InstallOperation* dummy_op = 1358 manifest->add_kernel_install_operations(); 1359 dummy_op->set_type(DeltaArchiveManifest_InstallOperation_Type_REPLACE); 1360 dummy_op->set_data_offset(signature_blob_offset); 1361 manifest->set_signatures_offset(signature_blob_offset); 1362 dummy_op->set_data_length(signature_blob_length); 1363 manifest->set_signatures_size(signature_blob_length); 1364 Extent* dummy_extent = dummy_op->add_dst_extents(); 1365 // Tell the dummy op to write this data to a big sparse hole 1366 dummy_extent->set_start_block(kSparseHole); 1367 dummy_extent->set_num_blocks((signature_blob_length + kBlockSize - 1) / 1368 kBlockSize); 1369} 1370 1371}; // namespace chromeos_update_engine 1372