payload_file.cc revision 2703ef4466066d64d8021904e233b120f38c0272 
1//
2// Copyright (C) 2015 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8//      http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include "update_engine/payload_generator/payload_file.h"
18
19#include <endian.h>
20
21#include <algorithm>
22
23#include "update_engine/common/hash_calculator.h"
24#include "update_engine/payload_consumer/delta_performer.h"
25#include "update_engine/payload_consumer/file_writer.h"
26#include "update_engine/payload_consumer/payload_constants.h"
27#include "update_engine/payload_generator/annotated_operation.h"
28#include "update_engine/payload_generator/delta_diff_utils.h"
29#include "update_engine/payload_generator/payload_signer.h"
30
31using std::string;
32using std::vector;
33
34namespace chromeos_update_engine {
35
36namespace {
37
38struct DeltaObject {
39  DeltaObject(const string& in_name, const int in_type, const off_t in_size)
40      : name(in_name),
41        type(in_type),
42        size(in_size) {}
43  bool operator <(const DeltaObject& object) const {
44    return (size != object.size) ? (size < object.size) : (name < object.name);
45  }
46  string name;
47  int type;
48  off_t size;
49};
50
51// Writes the uint64_t passed in in host-endian to the file as big-endian.
52// Returns true on success.
53bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) {
54  uint64_t value_be = htobe64(value);
55  TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)));
56  return true;
57}
58
59}  // namespace
60
61bool PayloadFile::Init(const PayloadGenerationConfig& config) {
62  TEST_AND_RETURN_FALSE(config.version.Validate());
63  major_version_ = config.version.major;
64  manifest_.set_minor_version(config.version.minor);
65
66  if (!config.source.ImageInfoIsEmpty())
67    *(manifest_.mutable_old_image_info()) = config.source.image_info;
68
69  if (!config.target.ImageInfoIsEmpty())
70    *(manifest_.mutable_new_image_info()) = config.target.image_info;
71
72  manifest_.set_block_size(config.block_size);
73  return true;
74}
75
76bool PayloadFile::AddPartition(const PartitionConfig& old_conf,
77                               const PartitionConfig& new_conf,
78                               const vector<AnnotatedOperation>& aops) {
79  // Check partitions order for Chrome OS
80  if (major_version_ == kChromeOSMajorPayloadVersion) {
81    const vector<const char*> part_order = { kLegacyPartitionNameRoot,
82                                             kLegacyPartitionNameKernel };
83    TEST_AND_RETURN_FALSE(part_vec_.size() < part_order.size());
84    TEST_AND_RETURN_FALSE(new_conf.name == part_order[part_vec_.size()]);
85  }
86  Partition part;
87  part.name = new_conf.name;
88  part.aops = aops;
89  part.postinstall = new_conf.postinstall;
90  // Initialize the PartitionInfo objects if present.
91  if (!old_conf.path.empty())
92    TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(old_conf,
93                                                              &part.old_info));
94  TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(new_conf,
95                                                            &part.new_info));
96  part_vec_.push_back(std::move(part));
97  return true;
98}
99
100bool PayloadFile::WritePayload(const string& payload_file,
101                               const string& data_blobs_path,
102                               const string& private_key_path,
103                               uint64_t* metadata_size_out) {
104  // Reorder the data blobs with the manifest_.
105  string ordered_blobs_path;
106  TEST_AND_RETURN_FALSE(utils::MakeTempFile(
107      "CrAU_temp_data.ordered.XXXXXX",
108      &ordered_blobs_path,
109      nullptr));
110  ScopedPathUnlinker ordered_blobs_unlinker(ordered_blobs_path);
111  TEST_AND_RETURN_FALSE(ReorderDataBlobs(data_blobs_path, ordered_blobs_path));
112
113  // Check that install op blobs are in order.
114  uint64_t next_blob_offset = 0;
115  for (const auto& part : part_vec_) {
116    for (const auto& aop : part.aops) {
117      if (!aop.op.has_data_offset())
118        continue;
119      if (aop.op.data_offset() != next_blob_offset) {
120        LOG(FATAL) << "bad blob offset! " << aop.op.data_offset() << " != "
121                   << next_blob_offset;
122      }
123      next_blob_offset += aop.op.data_length();
124    }
125  }
126
127  // Copy the operations and partition info from the part_vec_ to the manifest.
128  manifest_.clear_install_operations();
129  manifest_.clear_kernel_install_operations();
130  manifest_.clear_partitions();
131  for (const auto& part : part_vec_) {
132    if (major_version_ == kBrilloMajorPayloadVersion) {
133      PartitionUpdate* partition = manifest_.add_partitions();
134      partition->set_partition_name(part.name);
135      if (part.postinstall.run) {
136        partition->set_run_postinstall(true);
137        if (!part.postinstall.path.empty())
138          partition->set_postinstall_path(part.postinstall.path);
139        if (!part.postinstall.filesystem_type.empty())
140          partition->set_filesystem_type(part.postinstall.filesystem_type);
141        partition->set_postinstall_optional(part.postinstall.optional);
142      }
143      for (const AnnotatedOperation& aop : part.aops) {
144        *partition->add_operations() = aop.op;
145      }
146      if (part.old_info.has_size() || part.old_info.has_hash())
147        *(partition->mutable_old_partition_info()) = part.old_info;
148      if (part.new_info.has_size() || part.new_info.has_hash())
149        *(partition->mutable_new_partition_info()) = part.new_info;
150    } else {
151      // major_version_ == kChromeOSMajorPayloadVersion
152      if (part.name == kLegacyPartitionNameKernel) {
153        for (const AnnotatedOperation& aop : part.aops)
154          *manifest_.add_kernel_install_operations() = aop.op;
155        if (part.old_info.has_size() || part.old_info.has_hash())
156          *manifest_.mutable_old_kernel_info() = part.old_info;
157        if (part.new_info.has_size() || part.new_info.has_hash())
158          *manifest_.mutable_new_kernel_info() = part.new_info;
159      } else {
160        for (const AnnotatedOperation& aop : part.aops)
161          *manifest_.add_install_operations() = aop.op;
162        if (part.old_info.has_size() || part.old_info.has_hash())
163          *manifest_.mutable_old_rootfs_info() = part.old_info;
164        if (part.new_info.has_size() || part.new_info.has_hash())
165          *manifest_.mutable_new_rootfs_info() = part.new_info;
166      }
167    }
168  }
169
170  // Signatures appear at the end of the blobs. Note the offset in the
171  // manifest_.
172  uint64_t signature_blob_length = 0;
173  if (!private_key_path.empty()) {
174    TEST_AND_RETURN_FALSE(
175        PayloadSigner::SignatureBlobLength(vector<string>(1, private_key_path),
176                                           &signature_blob_length));
177    PayloadSigner::AddSignatureToManifest(
178        next_blob_offset, signature_blob_length,
179        major_version_ == kChromeOSMajorPayloadVersion, &manifest_);
180  }
181
182  // Serialize protobuf
183  string serialized_manifest;
184  TEST_AND_RETURN_FALSE(manifest_.AppendToString(&serialized_manifest));
185
186  uint64_t metadata_size =
187      sizeof(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size();
188
189  LOG(INFO) << "Writing final delta file header...";
190  DirectFileWriter writer;
191  TEST_AND_RETURN_FALSE_ERRNO(writer.Open(payload_file.c_str(),
192                                          O_WRONLY | O_CREAT | O_TRUNC,
193                                          0644) == 0);
194  ScopedFileWriterCloser writer_closer(&writer);
195
196  // Write header
197  TEST_AND_RETURN_FALSE_ERRNO(writer.Write(kDeltaMagic, sizeof(kDeltaMagic)));
198
199  // Write major version number
200  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, major_version_));
201
202  // Write protobuf length
203  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer,
204                                               serialized_manifest.size()));
205
206  // Write metadata signature size.
207  uint32_t metadata_signature_size = 0;
208  if (major_version_ == kBrilloMajorPayloadVersion) {
209    // Metadata signature has the same size as payload signature, because they
210    // are both the same kind of signature for the same kind of hash.
211    uint32_t metadata_signature_size = htobe32(signature_blob_length);
212    TEST_AND_RETURN_FALSE_ERRNO(writer.Write(&metadata_signature_size,
213                                             sizeof(metadata_signature_size)));
214    metadata_size += sizeof(metadata_signature_size);
215    // Set correct size instead of big endian size.
216    metadata_signature_size = signature_blob_length;
217  }
218
219  // Write protobuf
220  LOG(INFO) << "Writing final delta file protobuf... "
221            << serialized_manifest.size();
222  TEST_AND_RETURN_FALSE_ERRNO(
223      writer.Write(serialized_manifest.data(), serialized_manifest.size()));
224
225  // Write metadata signature blob.
226  if (major_version_ == kBrilloMajorPayloadVersion &&
227      !private_key_path.empty()) {
228    brillo::Blob metadata_hash, metadata_signature;
229    TEST_AND_RETURN_FALSE(HashCalculator::RawHashOfFile(payload_file,
230                                                             metadata_size,
231                                                             &metadata_hash));
232    TEST_AND_RETURN_FALSE(
233        PayloadSigner::SignHashWithKeys(metadata_hash,
234                                        vector<string>(1, private_key_path),
235                                        &metadata_signature));
236    TEST_AND_RETURN_FALSE_ERRNO(
237        writer.Write(metadata_signature.data(), metadata_signature.size()));
238  }
239
240  // Append the data blobs
241  LOG(INFO) << "Writing final delta file data blobs...";
242  int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0);
243  ScopedFdCloser blobs_fd_closer(&blobs_fd);
244  TEST_AND_RETURN_FALSE(blobs_fd >= 0);
245  for (;;) {
246    vector<char> buf(1024 * 1024);
247    ssize_t rc = read(blobs_fd, buf.data(), buf.size());
248    if (0 == rc) {
249      // EOF
250      break;
251    }
252    TEST_AND_RETURN_FALSE_ERRNO(rc > 0);
253    TEST_AND_RETURN_FALSE_ERRNO(writer.Write(buf.data(), rc));
254  }
255
256  // Write payload signature blob.
257  if (!private_key_path.empty()) {
258    LOG(INFO) << "Signing the update...";
259    brillo::Blob signature_blob;
260    TEST_AND_RETURN_FALSE(PayloadSigner::SignPayload(
261        payload_file,
262        vector<string>(1, private_key_path),
263        metadata_size,
264        metadata_signature_size,
265        metadata_size + metadata_signature_size + manifest_.signatures_offset(),
266        &signature_blob));
267    TEST_AND_RETURN_FALSE_ERRNO(
268        writer.Write(signature_blob.data(), signature_blob.size()));
269  }
270
271  ReportPayloadUsage(metadata_size);
272  *metadata_size_out = metadata_size;
273  return true;
274}
275
276bool PayloadFile::ReorderDataBlobs(
277    const string& data_blobs_path,
278    const string& new_data_blobs_path) {
279  int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0);
280  TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0);
281  ScopedFdCloser in_fd_closer(&in_fd);
282
283  DirectFileWriter writer;
284  int rc = writer.Open(
285      new_data_blobs_path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644);
286  if (rc != 0) {
287    PLOG(ERROR) << "Error creating " << new_data_blobs_path;
288    return false;
289  }
290  ScopedFileWriterCloser writer_closer(&writer);
291  uint64_t out_file_size = 0;
292
293  for (auto& part : part_vec_) {
294    for (AnnotatedOperation& aop : part.aops) {
295      if (!aop.op.has_data_offset())
296        continue;
297      CHECK(aop.op.has_data_length());
298      brillo::Blob buf(aop.op.data_length());
299      ssize_t rc = pread(in_fd, buf.data(), buf.size(), aop.op.data_offset());
300      TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size()));
301
302      // Add the hash of the data blobs for this operation
303      TEST_AND_RETURN_FALSE(AddOperationHash(&aop.op, buf));
304
305      aop.op.set_data_offset(out_file_size);
306      TEST_AND_RETURN_FALSE_ERRNO(writer.Write(buf.data(), buf.size()));
307      out_file_size += buf.size();
308    }
309  }
310  return true;
311}
312
313bool PayloadFile::AddOperationHash(InstallOperation* op,
314                                   const brillo::Blob& buf) {
315  brillo::Blob hash;
316  TEST_AND_RETURN_FALSE(HashCalculator::RawHashOfData(buf, &hash));
317  op->set_data_sha256_hash(hash.data(), hash.size());
318  return true;
319}
320
321void PayloadFile::ReportPayloadUsage(uint64_t metadata_size) const {
322  vector<DeltaObject> objects;
323  off_t total_size = 0;
324
325  for (const auto& part : part_vec_) {
326    for (const AnnotatedOperation& aop : part.aops) {
327      objects.push_back(DeltaObject(aop.name,
328                                    aop.op.type(),
329                                    aop.op.data_length()));
330      total_size += aop.op.data_length();
331    }
332  }
333
334  objects.push_back(DeltaObject("<manifest-metadata>",
335                                -1,
336                                metadata_size));
337  total_size += metadata_size;
338
339  std::sort(objects.begin(), objects.end());
340
341  static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n";
342  for (const DeltaObject& object : objects) {
343    fprintf(
344        stderr, kFormatString,
345        object.size * 100.0 / total_size,
346        static_cast<intmax_t>(object.size),
347        (object.type >= 0 ? InstallOperationTypeName(
348                                static_cast<InstallOperation_Type>(object.type))
349                          : "-"),
350        object.name.c_str());
351  }
352  fprintf(stderr, kFormatString,
353          100.0, static_cast<intmax_t>(total_size), "", "<total>");
354}
355
356}  // namespace chromeos_update_engine
357