payload_file.cc revision 6f20dd4fc8861d93d188cd27323d2f9746464aaf 
1//
2// Copyright (C) 2015 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8//      http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include "update_engine/payload_generator/payload_file.h"
18
19#include <endian.h>
20
21#include <algorithm>
22
23#include "update_engine/delta_performer.h"
24#include "update_engine/file_writer.h"
25#include "update_engine/omaha_hash_calculator.h"
26#include "update_engine/payload_constants.h"
27#include "update_engine/payload_generator/annotated_operation.h"
28#include "update_engine/payload_generator/delta_diff_generator.h"
29#include "update_engine/payload_generator/delta_diff_utils.h"
30#include "update_engine/payload_generator/payload_signer.h"
31
32using std::string;
33using std::vector;
34
35namespace chromeos_update_engine {
36
37namespace {
38
39static const char* kInstallOperationTypes[] = {
40  "REPLACE",
41  "REPLACE_BZ",
42  "MOVE",
43  "BSDIFF",
44  "SOURCE_COPY",
45  "SOURCE_BSDIFF"
46};
47
48struct DeltaObject {
49  DeltaObject(const string& in_name, const int in_type, const off_t in_size)
50      : name(in_name),
51        type(in_type),
52        size(in_size) {}
53  bool operator <(const DeltaObject& object) const {
54    return (size != object.size) ? (size < object.size) : (name < object.name);
55  }
56  string name;
57  int type;
58  off_t size;
59};
60
61// Writes the uint64_t passed in in host-endian to the file as big-endian.
62// Returns true on success.
63bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) {
64  uint64_t value_be = htobe64(value);
65  TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)));
66  return true;
67}
68
69}  // namespace
70
71const vector<PartitionName> PayloadFile::partition_disk_order_ = {
72  PartitionName::kRootfs,
73  PartitionName::kKernel,
74};
75
76bool PayloadFile::Init(const PayloadGenerationConfig& config) {
77  major_version_ = config.major_version;
78  TEST_AND_RETURN_FALSE(major_version_ == kChromeOSMajorPayloadVersion ||
79                        major_version_ == kBrilloMajorPayloadVersion);
80  manifest_.set_minor_version(config.minor_version);
81
82  if (!config.source.ImageInfoIsEmpty())
83    *(manifest_.mutable_old_image_info()) = config.source.image_info;
84
85  if (!config.target.ImageInfoIsEmpty())
86    *(manifest_.mutable_new_image_info()) = config.target.image_info;
87
88  manifest_.set_block_size(config.block_size);
89
90  // Initialize the PartitionInfo objects if present.
91  if (!config.source.kernel.path.empty()) {
92    TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(
93        config.source.kernel,
94        manifest_.mutable_old_kernel_info()));
95  }
96  TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(
97      config.target.kernel,
98      manifest_.mutable_new_kernel_info()));
99  if (!config.source.rootfs.path.empty()) {
100    TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(
101        config.source.rootfs,
102        manifest_.mutable_old_rootfs_info()));
103  }
104  TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(
105      config.target.rootfs,
106      manifest_.mutable_new_rootfs_info()));
107  return true;
108}
109
110void PayloadFile::AddPartitionOperations(
111    PartitionName name,
112    const vector<AnnotatedOperation>& aops) {
113  aops_map_[name].insert(aops_map_[name].end(), aops.begin(), aops.end());
114}
115
116bool PayloadFile::WritePayload(const string& payload_file,
117                               const string& data_blobs_path,
118                               const string& private_key_path,
119                               uint64_t* medatata_size_out) {
120  // Reorder the data blobs with the manifest_.
121  string ordered_blobs_path;
122  TEST_AND_RETURN_FALSE(utils::MakeTempFile(
123      "CrAU_temp_data.ordered.XXXXXX",
124      &ordered_blobs_path,
125      nullptr));
126  ScopedPathUnlinker ordered_blobs_unlinker(ordered_blobs_path);
127  TEST_AND_RETURN_FALSE(ReorderDataBlobs(data_blobs_path, ordered_blobs_path));
128
129  // Copy the operations from the aops_map_ to the manifest.
130  manifest_.clear_install_operations();
131  manifest_.clear_kernel_install_operations();
132  for (PartitionName name : partition_disk_order_) {
133    for (const AnnotatedOperation& aop : aops_map_[name]) {
134      if (name == PartitionName::kKernel) {
135        *manifest_.add_kernel_install_operations() = aop.op;
136      } else {
137        *manifest_.add_install_operations() = aop.op;
138      }
139    }
140  }
141
142  // Check that install op blobs are in order.
143  uint64_t next_blob_offset = 0;
144  {
145    for (int i = 0; i < (manifest_.install_operations_size() +
146                         manifest_.kernel_install_operations_size()); i++) {
147      InstallOperation* op = i < manifest_.install_operations_size()
148                                 ? manifest_.mutable_install_operations(i)
149                                 : manifest_.mutable_kernel_install_operations(
150                                       i - manifest_.install_operations_size());
151      if (op->has_data_offset()) {
152        if (op->data_offset() != next_blob_offset) {
153          LOG(FATAL) << "bad blob offset! " << op->data_offset() << " != "
154                     << next_blob_offset;
155        }
156        next_blob_offset += op->data_length();
157      }
158    }
159  }
160
161  // Signatures appear at the end of the blobs. Note the offset in the
162  // manifest_.
163  if (!private_key_path.empty()) {
164    uint64_t signature_blob_length = 0;
165    TEST_AND_RETURN_FALSE(
166        PayloadSigner::SignatureBlobLength(vector<string>(1, private_key_path),
167                                           &signature_blob_length));
168    AddSignatureOp(next_blob_offset, signature_blob_length, &manifest_);
169  }
170
171    // Serialize protobuf
172  string serialized_manifest;
173  TEST_AND_RETURN_FALSE(manifest_.AppendToString(&serialized_manifest));
174
175  LOG(INFO) << "Writing final delta file header...";
176  DirectFileWriter writer;
177  TEST_AND_RETURN_FALSE_ERRNO(writer.Open(payload_file.c_str(),
178                                          O_WRONLY | O_CREAT | O_TRUNC,
179                                          0644) == 0);
180  ScopedFileWriterCloser writer_closer(&writer);
181
182  // Write header
183  TEST_AND_RETURN_FALSE(writer.Write(kDeltaMagic, strlen(kDeltaMagic)));
184
185  // Write major version number
186  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, major_version_));
187
188  // Write protobuf length
189  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer,
190                                               serialized_manifest.size()));
191
192  // Write protobuf
193  LOG(INFO) << "Writing final delta file protobuf... "
194            << serialized_manifest.size();
195  TEST_AND_RETURN_FALSE(writer.Write(serialized_manifest.data(),
196                                     serialized_manifest.size()));
197
198  // Append the data blobs
199  LOG(INFO) << "Writing final delta file data blobs...";
200  int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0);
201  ScopedFdCloser blobs_fd_closer(&blobs_fd);
202  TEST_AND_RETURN_FALSE(blobs_fd >= 0);
203  for (;;) {
204    vector<char> buf(1024 * 1024);
205    ssize_t rc = read(blobs_fd, buf.data(), buf.size());
206    if (0 == rc) {
207      // EOF
208      break;
209    }
210    TEST_AND_RETURN_FALSE_ERRNO(rc > 0);
211    TEST_AND_RETURN_FALSE(writer.Write(buf.data(), rc));
212  }
213
214  // Write signature blob.
215  if (!private_key_path.empty()) {
216    LOG(INFO) << "Signing the update...";
217    chromeos::Blob signature_blob;
218    TEST_AND_RETURN_FALSE(PayloadSigner::SignPayload(
219        payload_file,
220        vector<string>(1, private_key_path),
221        &signature_blob));
222    TEST_AND_RETURN_FALSE(writer.Write(signature_blob.data(),
223                                       signature_blob.size()));
224  }
225
226  *medatata_size_out =
227      strlen(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size();
228  ReportPayloadUsage(*medatata_size_out);
229  return true;
230}
231
232bool PayloadFile::ReorderDataBlobs(
233    const string& data_blobs_path,
234    const string& new_data_blobs_path) {
235  int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0);
236  TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0);
237  ScopedFdCloser in_fd_closer(&in_fd);
238
239  DirectFileWriter writer;
240  TEST_AND_RETURN_FALSE(
241      writer.Open(new_data_blobs_path.c_str(),
242                  O_WRONLY | O_TRUNC | O_CREAT,
243                  0644) == 0);
244  ScopedFileWriterCloser writer_closer(&writer);
245  uint64_t out_file_size = 0;
246
247  for (PartitionName name : partition_disk_order_) {
248    for (AnnotatedOperation& aop : aops_map_[name]) {
249      if (!aop.op.has_data_offset())
250        continue;
251      CHECK(aop.op.has_data_length());
252      chromeos::Blob buf(aop.op.data_length());
253      ssize_t rc = pread(in_fd, buf.data(), buf.size(), aop.op.data_offset());
254      TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size()));
255
256      // Add the hash of the data blobs for this operation
257      TEST_AND_RETURN_FALSE(AddOperationHash(&aop.op, buf));
258
259      aop.op.set_data_offset(out_file_size);
260      TEST_AND_RETURN_FALSE(writer.Write(buf.data(), buf.size()));
261      out_file_size += buf.size();
262    }
263  }
264  return true;
265}
266
267bool PayloadFile::AddOperationHash(InstallOperation* op,
268                                   const chromeos::Blob& buf) {
269  OmahaHashCalculator hasher;
270  TEST_AND_RETURN_FALSE(hasher.Update(buf.data(), buf.size()));
271  TEST_AND_RETURN_FALSE(hasher.Finalize());
272  const chromeos::Blob& hash = hasher.raw_hash();
273  op->set_data_sha256_hash(hash.data(), hash.size());
274  return true;
275}
276
277void PayloadFile::ReportPayloadUsage(uint64_t metadata_size) const {
278  vector<DeltaObject> objects;
279  off_t total_size = 0;
280
281  for (PartitionName name : partition_disk_order_) {
282    const auto& partition_aops = aops_map_.find(name);
283    if (partition_aops == aops_map_.end())
284      continue;
285    for (const AnnotatedOperation& aop : partition_aops->second) {
286      objects.push_back(DeltaObject(aop.name,
287                                    aop.op.type(),
288                                    aop.op.data_length()));
289      total_size += aop.op.data_length();
290    }
291  }
292
293  objects.push_back(DeltaObject("<manifest-metadata>",
294                                -1,
295                                metadata_size));
296  total_size += metadata_size;
297
298  std::sort(objects.begin(), objects.end());
299
300  static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n";
301  for (const DeltaObject& object : objects) {
302    fprintf(stderr, kFormatString,
303            object.size * 100.0 / total_size,
304            static_cast<intmax_t>(object.size),
305            object.type >= 0 ? kInstallOperationTypes[object.type] : "-",
306            object.name.c_str());
307  }
308  fprintf(stderr, kFormatString,
309          100.0, static_cast<intmax_t>(total_size), "", "<total>");
310}
311
312void AddSignatureOp(uint64_t signature_blob_offset,
313                    uint64_t signature_blob_length,
314                    DeltaArchiveManifest* manifest) {
315  LOG(INFO) << "Making room for signature in file";
316  manifest->set_signatures_offset(signature_blob_offset);
317  LOG(INFO) << "set? " << manifest->has_signatures_offset();
318  // Add a dummy op at the end to appease older clients
319  InstallOperation* dummy_op = manifest->add_kernel_install_operations();
320  dummy_op->set_type(InstallOperation::REPLACE);
321  dummy_op->set_data_offset(signature_blob_offset);
322  manifest->set_signatures_offset(signature_blob_offset);
323  dummy_op->set_data_length(signature_blob_length);
324  manifest->set_signatures_size(signature_blob_length);
325  Extent* dummy_extent = dummy_op->add_dst_extents();
326  // Tell the dummy op to write this data to a big sparse hole
327  dummy_extent->set_start_block(kSparseHole);
328  dummy_extent->set_num_blocks((signature_blob_length + kBlockSize - 1) /
329                               kBlockSize);
330}
331
332}  // namespace chromeos_update_engine
333