payload_file.cc revision 8e768e9359d01008b286db1f6c54a6c3972a56b9 
1//
2// Copyright (C) 2015 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8//      http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include "update_engine/payload_generator/payload_file.h"
18
19#include <endian.h>
20
21#include <algorithm>
22
23#include "update_engine/common/hash_calculator.h"
24#include "update_engine/payload_consumer/delta_performer.h"
25#include "update_engine/payload_consumer/file_writer.h"
26#include "update_engine/payload_consumer/payload_constants.h"
27#include "update_engine/payload_generator/annotated_operation.h"
28#include "update_engine/payload_generator/delta_diff_utils.h"
29#include "update_engine/payload_generator/payload_signer.h"
30
31using std::string;
32using std::vector;
33
34namespace chromeos_update_engine {
35
36namespace {
37
38struct DeltaObject {
39  DeltaObject(const string& in_name, const int in_type, const off_t in_size)
40      : name(in_name),
41        type(in_type),
42        size(in_size) {}
43  bool operator <(const DeltaObject& object) const {
44    return (size != object.size) ? (size < object.size) : (name < object.name);
45  }
46  string name;
47  int type;
48  off_t size;
49};
50
51// Writes the uint64_t passed in in host-endian to the file as big-endian.
52// Returns true on success.
53bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) {
54  uint64_t value_be = htobe64(value);
55  TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)));
56  return true;
57}
58
59}  // namespace
60
61bool PayloadFile::Init(const PayloadGenerationConfig& config) {
62  TEST_AND_RETURN_FALSE(config.version.Validate());
63  major_version_ = config.version.major;
64  manifest_.set_minor_version(config.version.minor);
65
66  if (!config.source.ImageInfoIsEmpty())
67    *(manifest_.mutable_old_image_info()) = config.source.image_info;
68
69  if (!config.target.ImageInfoIsEmpty())
70    *(manifest_.mutable_new_image_info()) = config.target.image_info;
71
72  manifest_.set_block_size(config.block_size);
73  manifest_.set_max_timestamp(config.max_timestamp);
74  return true;
75}
76
77bool PayloadFile::AddPartition(const PartitionConfig& old_conf,
78                               const PartitionConfig& new_conf,
79                               const vector<AnnotatedOperation>& aops) {
80  // Check partitions order for Chrome OS
81  if (major_version_ == kChromeOSMajorPayloadVersion) {
82    const vector<const char*> part_order = { kLegacyPartitionNameRoot,
83                                             kLegacyPartitionNameKernel };
84    TEST_AND_RETURN_FALSE(part_vec_.size() < part_order.size());
85    TEST_AND_RETURN_FALSE(new_conf.name == part_order[part_vec_.size()]);
86  }
87  Partition part;
88  part.name = new_conf.name;
89  part.aops = aops;
90  part.postinstall = new_conf.postinstall;
91  // Initialize the PartitionInfo objects if present.
92  if (!old_conf.path.empty())
93    TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(old_conf,
94                                                              &part.old_info));
95  TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(new_conf,
96                                                            &part.new_info));
97  part_vec_.push_back(std::move(part));
98  return true;
99}
100
101bool PayloadFile::WritePayload(const string& payload_file,
102                               const string& data_blobs_path,
103                               const string& private_key_path,
104                               uint64_t* metadata_size_out) {
105  // Reorder the data blobs with the manifest_.
106  string ordered_blobs_path;
107  TEST_AND_RETURN_FALSE(utils::MakeTempFile(
108      "CrAU_temp_data.ordered.XXXXXX",
109      &ordered_blobs_path,
110      nullptr));
111  ScopedPathUnlinker ordered_blobs_unlinker(ordered_blobs_path);
112  TEST_AND_RETURN_FALSE(ReorderDataBlobs(data_blobs_path, ordered_blobs_path));
113
114  // Check that install op blobs are in order.
115  uint64_t next_blob_offset = 0;
116  for (const auto& part : part_vec_) {
117    for (const auto& aop : part.aops) {
118      if (!aop.op.has_data_offset())
119        continue;
120      if (aop.op.data_offset() != next_blob_offset) {
121        LOG(FATAL) << "bad blob offset! " << aop.op.data_offset() << " != "
122                   << next_blob_offset;
123      }
124      next_blob_offset += aop.op.data_length();
125    }
126  }
127
128  // Copy the operations and partition info from the part_vec_ to the manifest.
129  manifest_.clear_install_operations();
130  manifest_.clear_kernel_install_operations();
131  manifest_.clear_partitions();
132  for (const auto& part : part_vec_) {
133    if (major_version_ == kBrilloMajorPayloadVersion) {
134      PartitionUpdate* partition = manifest_.add_partitions();
135      partition->set_partition_name(part.name);
136      if (part.postinstall.run) {
137        partition->set_run_postinstall(true);
138        if (!part.postinstall.path.empty())
139          partition->set_postinstall_path(part.postinstall.path);
140        if (!part.postinstall.filesystem_type.empty())
141          partition->set_filesystem_type(part.postinstall.filesystem_type);
142        partition->set_postinstall_optional(part.postinstall.optional);
143      }
144      for (const AnnotatedOperation& aop : part.aops) {
145        *partition->add_operations() = aop.op;
146      }
147      if (part.old_info.has_size() || part.old_info.has_hash())
148        *(partition->mutable_old_partition_info()) = part.old_info;
149      if (part.new_info.has_size() || part.new_info.has_hash())
150        *(partition->mutable_new_partition_info()) = part.new_info;
151    } else {
152      // major_version_ == kChromeOSMajorPayloadVersion
153      if (part.name == kLegacyPartitionNameKernel) {
154        for (const AnnotatedOperation& aop : part.aops)
155          *manifest_.add_kernel_install_operations() = aop.op;
156        if (part.old_info.has_size() || part.old_info.has_hash())
157          *manifest_.mutable_old_kernel_info() = part.old_info;
158        if (part.new_info.has_size() || part.new_info.has_hash())
159          *manifest_.mutable_new_kernel_info() = part.new_info;
160      } else {
161        for (const AnnotatedOperation& aop : part.aops)
162          *manifest_.add_install_operations() = aop.op;
163        if (part.old_info.has_size() || part.old_info.has_hash())
164          *manifest_.mutable_old_rootfs_info() = part.old_info;
165        if (part.new_info.has_size() || part.new_info.has_hash())
166          *manifest_.mutable_new_rootfs_info() = part.new_info;
167      }
168    }
169  }
170
171  // Signatures appear at the end of the blobs. Note the offset in the
172  // manifest_.
173  uint64_t signature_blob_length = 0;
174  if (!private_key_path.empty()) {
175    TEST_AND_RETURN_FALSE(
176        PayloadSigner::SignatureBlobLength(vector<string>(1, private_key_path),
177                                           &signature_blob_length));
178    PayloadSigner::AddSignatureToManifest(
179        next_blob_offset, signature_blob_length,
180        major_version_ == kChromeOSMajorPayloadVersion, &manifest_);
181  }
182
183  // Serialize protobuf
184  string serialized_manifest;
185  TEST_AND_RETURN_FALSE(manifest_.AppendToString(&serialized_manifest));
186
187  uint64_t metadata_size =
188      sizeof(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size();
189
190  LOG(INFO) << "Writing final delta file header...";
191  DirectFileWriter writer;
192  TEST_AND_RETURN_FALSE_ERRNO(writer.Open(payload_file.c_str(),
193                                          O_WRONLY | O_CREAT | O_TRUNC,
194                                          0644) == 0);
195  ScopedFileWriterCloser writer_closer(&writer);
196
197  // Write header
198  TEST_AND_RETURN_FALSE_ERRNO(writer.Write(kDeltaMagic, sizeof(kDeltaMagic)));
199
200  // Write major version number
201  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, major_version_));
202
203  // Write protobuf length
204  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer,
205                                               serialized_manifest.size()));
206
207  // Write metadata signature size.
208  uint32_t metadata_signature_size = 0;
209  if (major_version_ == kBrilloMajorPayloadVersion) {
210    // Metadata signature has the same size as payload signature, because they
211    // are both the same kind of signature for the same kind of hash.
212    uint32_t metadata_signature_size = htobe32(signature_blob_length);
213    TEST_AND_RETURN_FALSE_ERRNO(writer.Write(&metadata_signature_size,
214                                             sizeof(metadata_signature_size)));
215    metadata_size += sizeof(metadata_signature_size);
216    // Set correct size instead of big endian size.
217    metadata_signature_size = signature_blob_length;
218  }
219
220  // Write protobuf
221  LOG(INFO) << "Writing final delta file protobuf... "
222            << serialized_manifest.size();
223  TEST_AND_RETURN_FALSE_ERRNO(
224      writer.Write(serialized_manifest.data(), serialized_manifest.size()));
225
226  // Write metadata signature blob.
227  if (major_version_ == kBrilloMajorPayloadVersion &&
228      !private_key_path.empty()) {
229    brillo::Blob metadata_hash, metadata_signature;
230    TEST_AND_RETURN_FALSE(HashCalculator::RawHashOfFile(payload_file,
231                                                             metadata_size,
232                                                             &metadata_hash));
233    TEST_AND_RETURN_FALSE(
234        PayloadSigner::SignHashWithKeys(metadata_hash,
235                                        vector<string>(1, private_key_path),
236                                        &metadata_signature));
237    TEST_AND_RETURN_FALSE_ERRNO(
238        writer.Write(metadata_signature.data(), metadata_signature.size()));
239  }
240
241  // Append the data blobs
242  LOG(INFO) << "Writing final delta file data blobs...";
243  int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0);
244  ScopedFdCloser blobs_fd_closer(&blobs_fd);
245  TEST_AND_RETURN_FALSE(blobs_fd >= 0);
246  for (;;) {
247    vector<char> buf(1024 * 1024);
248    ssize_t rc = read(blobs_fd, buf.data(), buf.size());
249    if (0 == rc) {
250      // EOF
251      break;
252    }
253    TEST_AND_RETURN_FALSE_ERRNO(rc > 0);
254    TEST_AND_RETURN_FALSE_ERRNO(writer.Write(buf.data(), rc));
255  }
256
257  // Write payload signature blob.
258  if (!private_key_path.empty()) {
259    LOG(INFO) << "Signing the update...";
260    brillo::Blob signature_blob;
261    TEST_AND_RETURN_FALSE(PayloadSigner::SignPayload(
262        payload_file,
263        vector<string>(1, private_key_path),
264        metadata_size,
265        metadata_signature_size,
266        metadata_size + metadata_signature_size + manifest_.signatures_offset(),
267        &signature_blob));
268    TEST_AND_RETURN_FALSE_ERRNO(
269        writer.Write(signature_blob.data(), signature_blob.size()));
270  }
271
272  ReportPayloadUsage(metadata_size);
273  *metadata_size_out = metadata_size;
274  return true;
275}
276
277bool PayloadFile::ReorderDataBlobs(
278    const string& data_blobs_path,
279    const string& new_data_blobs_path) {
280  int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0);
281  TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0);
282  ScopedFdCloser in_fd_closer(&in_fd);
283
284  DirectFileWriter writer;
285  int rc = writer.Open(
286      new_data_blobs_path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0644);
287  if (rc != 0) {
288    PLOG(ERROR) << "Error creating " << new_data_blobs_path;
289    return false;
290  }
291  ScopedFileWriterCloser writer_closer(&writer);
292  uint64_t out_file_size = 0;
293
294  for (auto& part : part_vec_) {
295    for (AnnotatedOperation& aop : part.aops) {
296      if (!aop.op.has_data_offset())
297        continue;
298      CHECK(aop.op.has_data_length());
299      brillo::Blob buf(aop.op.data_length());
300      ssize_t rc = pread(in_fd, buf.data(), buf.size(), aop.op.data_offset());
301      TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size()));
302
303      // Add the hash of the data blobs for this operation
304      TEST_AND_RETURN_FALSE(AddOperationHash(&aop.op, buf));
305
306      aop.op.set_data_offset(out_file_size);
307      TEST_AND_RETURN_FALSE_ERRNO(writer.Write(buf.data(), buf.size()));
308      out_file_size += buf.size();
309    }
310  }
311  return true;
312}
313
314bool PayloadFile::AddOperationHash(InstallOperation* op,
315                                   const brillo::Blob& buf) {
316  brillo::Blob hash;
317  TEST_AND_RETURN_FALSE(HashCalculator::RawHashOfData(buf, &hash));
318  op->set_data_sha256_hash(hash.data(), hash.size());
319  return true;
320}
321
322void PayloadFile::ReportPayloadUsage(uint64_t metadata_size) const {
323  vector<DeltaObject> objects;
324  off_t total_size = 0;
325
326  for (const auto& part : part_vec_) {
327    for (const AnnotatedOperation& aop : part.aops) {
328      objects.push_back(DeltaObject(aop.name,
329                                    aop.op.type(),
330                                    aop.op.data_length()));
331      total_size += aop.op.data_length();
332    }
333  }
334
335  objects.push_back(DeltaObject("<manifest-metadata>",
336                                -1,
337                                metadata_size));
338  total_size += metadata_size;
339
340  std::sort(objects.begin(), objects.end());
341
342  static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n";
343  for (const DeltaObject& object : objects) {
344    fprintf(
345        stderr, kFormatString,
346        object.size * 100.0 / total_size,
347        static_cast<intmax_t>(object.size),
348        (object.type >= 0 ? InstallOperationTypeName(
349                                static_cast<InstallOperation_Type>(object.type))
350                          : "-"),
351        object.name.c_str());
352  }
353  fprintf(stderr, kFormatString,
354          100.0, static_cast<intmax_t>(total_size), "", "<total>");
355}
356
357}  // namespace chromeos_update_engine
358