full_update_generator.cc revision d41606fb3c25f6cb8191e154c87391ab285a8a2f 
1//
2// Copyright (C) 2012 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8//      http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include "update_engine/payload_generator/full_update_generator.h"
18
19#include <fcntl.h>
20#include <inttypes.h>
21
22#include <algorithm>
23#include <deque>
24#include <memory>
25
26#include <base/format_macros.h>
27#include <base/strings/string_util.h>
28#include <base/strings/stringprintf.h>
29#include <base/synchronization/lock.h>
30#include <base/threading/simple_thread.h>
31#include <brillo/secure_blob.h>
32
33#include "update_engine/common/utils.h"
34#include "update_engine/payload_generator/bzip.h"
35#include "update_engine/payload_generator/xz.h"
36
37using std::vector;
38
39namespace chromeos_update_engine {
40
41namespace {
42
43const size_t kDefaultFullChunkSize = 1024 * 1024;  // 1 MiB
44
45// This class encapsulates a full update chunk processing thread work. The
46// processor reads a chunk of data from the input file descriptor and compresses
47// it. The processor will destroy itself when the work is done.
48class ChunkProcessor : public base::DelegateSimpleThread::Delegate {
49 public:
50  // Read a chunk of |size| bytes from |fd| starting at offset |offset|.
51  ChunkProcessor(const PayloadVersion& version,
52                 int fd,
53                 off_t offset,
54                 size_t size,
55                 BlobFileWriter* blob_file,
56                 AnnotatedOperation* aop)
57      : version_(version),
58        fd_(fd),
59        offset_(offset),
60        size_(size),
61        blob_file_(blob_file),
62        aop_(aop) {}
63  // We use a default move constructor since all the data members are POD types.
64  ChunkProcessor(ChunkProcessor&&) = default;
65  ~ChunkProcessor() override = default;
66
67  // Overrides DelegateSimpleThread::Delegate.
68  // Run() handles the read from |fd| in a thread-safe way, and stores the
69  // new operation to generate the region starting at |offset| of size |size|
70  // in the output operation |aop|. The associated blob data is stored in
71  // |blob_fd| and |blob_file_size| is updated.
72  void Run() override;
73
74 private:
75  bool ProcessChunk();
76
77  // Work parameters.
78  const PayloadVersion& version_;
79  int fd_;
80  off_t offset_;
81  size_t size_;
82  BlobFileWriter* blob_file_;
83  AnnotatedOperation* aop_;
84
85  DISALLOW_COPY_AND_ASSIGN(ChunkProcessor);
86};
87
88void ChunkProcessor::Run() {
89  if (!ProcessChunk()) {
90    LOG(ERROR) << "Error processing region at " << offset_ << " of size "
91               << size_;
92  }
93}
94
95bool ChunkProcessor::ProcessChunk() {
96  brillo::Blob buffer_in_(size_);
97  brillo::Blob op_blob;
98  ssize_t bytes_read = -1;
99  TEST_AND_RETURN_FALSE(utils::PReadAll(fd_,
100                                        buffer_in_.data(),
101                                        buffer_in_.size(),
102                                        offset_,
103                                        &bytes_read));
104  TEST_AND_RETURN_FALSE(bytes_read == static_cast<ssize_t>(size_));
105
106  TEST_AND_RETURN_FALSE(BzipCompress(buffer_in_, &op_blob));
107  InstallOperation_Type op_type = InstallOperation::REPLACE_BZ;
108
109  if (version_.OperationAllowed(InstallOperation::REPLACE_XZ)) {
110    brillo::Blob xz_blob;
111    if (XzCompress(buffer_in_, &xz_blob) && xz_blob.size() < op_blob.size()) {
112      op_blob = std::move(xz_blob);
113      op_type = InstallOperation::REPLACE_XZ;
114    }
115  }
116
117  if (op_blob.size() >= buffer_in_.size()) {
118    // A REPLACE is cheaper than a REPLACE_BZ in this case.
119    op_blob = std::move(buffer_in_);
120    op_type = InstallOperation::REPLACE;
121  }
122
123  TEST_AND_RETURN_FALSE(aop_->SetOperationBlob(&op_blob, blob_file_));
124  aop_->op.set_type(op_type);
125  return true;
126}
127
128}  // namespace
129
130bool FullUpdateGenerator::GenerateOperations(
131    const PayloadGenerationConfig& config,
132    const PartitionConfig& old_part,
133    const PartitionConfig& new_part,
134    BlobFileWriter* blob_file,
135    vector<AnnotatedOperation>* aops) {
136  TEST_AND_RETURN_FALSE(new_part.ValidateExists());
137
138  // FullUpdateGenerator requires a positive chunk_size, otherwise there will
139  // be only one operation with the whole partition which should not be allowed.
140  // For performance reasons, we force a small default hard limit of 1 MiB. This
141  // limit can be changed in the config, and we will use the smaller of the two
142  // soft/hard limits.
143  size_t full_chunk_size;
144  if (config.hard_chunk_size >= 0) {
145    full_chunk_size = std::min(static_cast<size_t>(config.hard_chunk_size),
146                               config.soft_chunk_size);
147  } else {
148    full_chunk_size = std::min(kDefaultFullChunkSize, config.soft_chunk_size);
149    LOG(INFO) << "No chunk_size provided, using the default chunk_size for the "
150              << "full operations: " << full_chunk_size << " bytes.";
151  }
152  TEST_AND_RETURN_FALSE(full_chunk_size > 0);
153  TEST_AND_RETURN_FALSE(full_chunk_size % config.block_size == 0);
154
155  size_t chunk_blocks = full_chunk_size / config.block_size;
156  size_t max_threads = std::max(sysconf(_SC_NPROCESSORS_ONLN), 4L);
157  LOG(INFO) << "Compressing partition " << new_part.name
158            << " from " << new_part.path << " splitting in chunks of "
159            << chunk_blocks << " blocks (" << config.block_size
160            << " bytes each) using " << max_threads << " threads";
161
162  int in_fd = open(new_part.path.c_str(), O_RDONLY, 0);
163  TEST_AND_RETURN_FALSE(in_fd >= 0);
164  ScopedFdCloser in_fd_closer(&in_fd);
165
166  // We potentially have all the ChunkProcessors in memory but only
167  // |max_threads| will actually hold a block in memory while we process.
168  size_t partition_blocks = new_part.size / config.block_size;
169  size_t num_chunks = (partition_blocks + chunk_blocks - 1) / chunk_blocks;
170  aops->resize(num_chunks);
171  vector<ChunkProcessor> chunk_processors;
172  chunk_processors.reserve(num_chunks);
173  blob_file->SetTotalBlobs(num_chunks);
174
175  for (size_t i = 0; i < num_chunks; ++i) {
176    size_t start_block = i * chunk_blocks;
177    // The last chunk could be smaller.
178    size_t num_blocks = std::min(chunk_blocks,
179                                 partition_blocks - i * chunk_blocks);
180
181    // Preset all the static information about the operations. The
182    // ChunkProcessor will set the rest.
183    AnnotatedOperation* aop = aops->data() + i;
184    aop->name = base::StringPrintf("<%s-operation-%" PRIuS ">",
185                                   new_part.name.c_str(), i);
186    Extent* dst_extent = aop->op.add_dst_extents();
187    dst_extent->set_start_block(start_block);
188    dst_extent->set_num_blocks(num_blocks);
189
190    chunk_processors.emplace_back(
191        config.version,
192        in_fd,
193        static_cast<off_t>(start_block) * config.block_size,
194        num_blocks * config.block_size,
195        blob_file,
196        aop);
197  }
198
199  // Thread pool used for worker threads.
200  base::DelegateSimpleThreadPool thread_pool("full-update-generator",
201                                             max_threads);
202  thread_pool.Start();
203  for (ChunkProcessor& processor : chunk_processors)
204    thread_pool.AddWork(&processor);
205  thread_pool.JoinAll();
206
207  // All the work done, disable logging.
208  blob_file->SetTotalBlobs(0);
209
210  // All the operations must have a type set at this point. Otherwise, a
211  // ChunkProcessor failed to complete.
212  for (const AnnotatedOperation& aop : *aops) {
213    if (!aop.op.has_type())
214      return false;
215  }
216  return true;
217}
218
219}  // namespace chromeos_update_engine
220