full_update_generator.cc revision 8cc502dacbccdab96824d42287f230ce04004784 
1// Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "update_engine/payload_generator/full_update_generator.h"
6
7#include <fcntl.h>
8#include <inttypes.h>
9
10#include <algorithm>
11#include <deque>
12#include <memory>
13
14#include <base/format_macros.h>
15#include <base/strings/string_util.h>
16#include <base/strings/stringprintf.h>
17#include <base/synchronization/lock.h>
18#include <base/threading/simple_thread.h>
19#include <chromeos/secure_blob.h>
20
21#include "update_engine/bzip.h"
22#include "update_engine/utils.h"
23
24using std::string;
25using std::vector;
26
27namespace chromeos_update_engine {
28
29namespace {
30
31const size_t kDefaultFullChunkSize = 1024 * 1024;  // 1 MiB
32
33// This class encapsulates a full update chunk processing thread work. The
34// processor reads a chunk of data from the input file descriptor and compresses
35// it. The processor will destroy itself when the work is done.
36class ChunkProcessor : public base::DelegateSimpleThread::Delegate {
37 public:
38  // Read a chunk of |size| bytes from |fd| starting at offset |offset|.
39  ChunkProcessor(int fd, off_t offset, size_t size,
40                 BlobFileWriter* blob_file, AnnotatedOperation* aop)
41    : fd_(fd),
42      offset_(offset),
43      size_(size),
44      blob_file_(blob_file),
45      aop_(aop) {}
46  // We use a default move constructor since all the data members are POD types.
47  ChunkProcessor(ChunkProcessor&&) = default;
48  ~ChunkProcessor() override = default;
49
50  // Overrides DelegateSimpleThread::Delegate.
51  // Run() handles the read from |fd| in a thread-safe way, and stores the
52  // new operation to generate the region starting at |offset| of size |size|
53  // in the output operation |aop|. The associated blob data is stored in
54  // |blob_fd| and |blob_file_size| is updated.
55  void Run() override;
56
57 private:
58  bool ProcessChunk();
59
60  // Stores the operation blob in the |blob_fd_| and updates the
61  // |blob_file_size| accordingly.
62  // This method is thread-safe since it uses a mutex to access the file.
63  // Returns the data offset where the data was written to.
64  off_t StoreBlob(const chromeos::Blob& blob);
65
66  // Work parameters.
67  int fd_;
68  off_t offset_;
69  size_t size_;
70  BlobFileWriter* blob_file_;
71  AnnotatedOperation* aop_;
72
73  DISALLOW_COPY_AND_ASSIGN(ChunkProcessor);
74};
75
76void ChunkProcessor::Run() {
77  if (!ProcessChunk()) {
78    LOG(ERROR) << "Error processing region at " << offset_ << " of size "
79               << size_;
80  }
81}
82
83bool ChunkProcessor::ProcessChunk() {
84  chromeos::Blob buffer_in_(size_);
85  chromeos::Blob op_blob;
86  ssize_t bytes_read = -1;
87  TEST_AND_RETURN_FALSE(utils::PReadAll(fd_,
88                                        buffer_in_.data(),
89                                        buffer_in_.size(),
90                                        offset_,
91                                        &bytes_read));
92  TEST_AND_RETURN_FALSE(bytes_read == static_cast<ssize_t>(size_));
93  TEST_AND_RETURN_FALSE(BzipCompress(buffer_in_, &op_blob));
94
95  DeltaArchiveManifest_InstallOperation_Type op_type =
96      DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ;
97
98  if (op_blob.size() >= buffer_in_.size()) {
99    // A REPLACE is cheaper than a REPLACE_BZ in this case.
100    op_blob = std::move(buffer_in_);
101    op_type = DeltaArchiveManifest_InstallOperation_Type_REPLACE;
102  }
103
104  off_t op_offset = blob_file_->StoreBlob(op_blob);
105  TEST_AND_RETURN_FALSE(op_offset >= 0);
106  aop_->op.set_data_offset(op_offset);
107  aop_->op.set_data_length(op_blob.size());
108  aop_->op.set_type(op_type);
109  return true;
110}
111
112}  // namespace
113
114bool FullUpdateGenerator::GenerateOperations(
115    const PayloadGenerationConfig& config,
116    BlobFileWriter* blob_file,
117    vector<AnnotatedOperation>* rootfs_ops,
118    vector<AnnotatedOperation>* kernel_ops) {
119  TEST_AND_RETURN_FALSE(config.Validate());
120
121  // FullUpdateGenerator requires a positive chunk_size, otherwise there will
122  // be only one operation with the whole partition which should not be allowed.
123  // For performance reasons, we force a small default hard limit of 1 MiB. This
124  // limit can be changed in the config, and we will use the smaller of the two
125  // soft/hard limits.
126  size_t full_chunk_size;
127  if (config.hard_chunk_size >= 0) {
128    full_chunk_size = std::min(static_cast<size_t>(config.hard_chunk_size),
129                               config.soft_chunk_size);
130  } else {
131    full_chunk_size = std::min(kDefaultFullChunkSize, config.soft_chunk_size);
132    LOG(INFO) << "No chunk_size provided, using the default chunk_size for the "
133              << "full operations: " << full_chunk_size << " bytes.";
134  }
135  TEST_AND_RETURN_FALSE(full_chunk_size > 0);
136  TEST_AND_RETURN_FALSE(full_chunk_size % config.block_size == 0);
137
138  TEST_AND_RETURN_FALSE(GenerateOperationsForPartition(
139      config.target.rootfs,
140      config.block_size,
141      full_chunk_size / config.block_size,
142      blob_file,
143      rootfs_ops));
144  TEST_AND_RETURN_FALSE(GenerateOperationsForPartition(
145      config.target.kernel,
146      config.block_size,
147      full_chunk_size / config.block_size,
148      blob_file,
149      kernel_ops));
150  return true;
151}
152
153bool FullUpdateGenerator::GenerateOperationsForPartition(
154    const PartitionConfig& new_part,
155    size_t block_size,
156    size_t chunk_blocks,
157    BlobFileWriter* blob_file,
158    vector<AnnotatedOperation>* aops) {
159  size_t max_threads = std::max(sysconf(_SC_NPROCESSORS_ONLN), 4L);
160  LOG(INFO) << "Compressing partition " << PartitionNameString(new_part.name)
161            << " from " << new_part.path << " splitting in chunks of "
162            << chunk_blocks << " blocks (" << block_size
163            << " bytes each) using " << max_threads << " threads";
164
165  int in_fd = open(new_part.path.c_str(), O_RDONLY, 0);
166  TEST_AND_RETURN_FALSE(in_fd >= 0);
167  ScopedFdCloser in_fd_closer(&in_fd);
168
169  // We potentially have all the ChunkProcessors in memory but only
170  // |max_threads| will actually hold a block in memory while we process.
171  size_t partition_blocks = new_part.size / block_size;
172  size_t num_chunks = (partition_blocks + chunk_blocks - 1) / chunk_blocks;
173  aops->resize(num_chunks);
174  vector<ChunkProcessor> chunk_processors;
175  chunk_processors.reserve(num_chunks);
176  blob_file->SetTotalBlobs(num_chunks);
177
178  const string part_name_str = PartitionNameString(new_part.name);
179
180  for (size_t i = 0; i < num_chunks; ++i) {
181    size_t start_block = i * chunk_blocks;
182    // The last chunk could be smaller.
183    size_t num_blocks = std::min(chunk_blocks,
184                                 partition_blocks - i * chunk_blocks);
185
186    // Preset all the static information about the operations. The
187    // ChunkProcessor will set the rest.
188    AnnotatedOperation* aop = aops->data() + i;
189    aop->name = base::StringPrintf("<%s-operation-%" PRIuS ">",
190                                   part_name_str.c_str(), i);
191    Extent* dst_extent = aop->op.add_dst_extents();
192    dst_extent->set_start_block(start_block);
193    dst_extent->set_num_blocks(num_blocks);
194
195    chunk_processors.emplace_back(
196        in_fd,
197        static_cast<off_t>(start_block) * block_size,
198        num_blocks * block_size,
199        blob_file,
200        aop);
201  }
202
203  // Thread pool used for worker threads.
204  base::DelegateSimpleThreadPool thread_pool("full-update-generator",
205                                             max_threads);
206  thread_pool.Start();
207  for (ChunkProcessor& processor : chunk_processors)
208    thread_pool.AddWork(&processor);
209  thread_pool.JoinAll();
210  // All the operations must have a type set at this point. Otherwise, a
211  // ChunkProcessor failed to complete.
212  for (const AnnotatedOperation& aop : *aops) {
213    if (!aop.op.has_type())
214      return false;
215  }
216  return true;
217}
218
219}  // namespace chromeos_update_engine
220