payload_file.cc revision b8060e435f0edb8efef4891a99fa18f642e01aa2 
1//
2// Copyright (C) 2015 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8//      http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include "update_engine/payload_generator/payload_file.h"
18
19#include <endian.h>
20
21#include <algorithm>
22
23#include "update_engine/delta_performer.h"
24#include "update_engine/file_writer.h"
25#include "update_engine/omaha_hash_calculator.h"
26#include "update_engine/payload_constants.h"
27#include "update_engine/payload_generator/annotated_operation.h"
28#include "update_engine/payload_generator/delta_diff_generator.h"
29#include "update_engine/payload_generator/delta_diff_utils.h"
30#include "update_engine/payload_generator/payload_signer.h"
31
32using std::string;
33using std::vector;
34
35namespace chromeos_update_engine {
36
37namespace {
38
39static const char* kInstallOperationTypes[] = {
40  "REPLACE",
41  "REPLACE_BZ",
42  "MOVE",
43  "BSDIFF",
44  "SOURCE_COPY",
45  "SOURCE_BSDIFF"
46};
47
48struct DeltaObject {
49  DeltaObject(const string& in_name, const int in_type, const off_t in_size)
50      : name(in_name),
51        type(in_type),
52        size(in_size) {}
53  bool operator <(const DeltaObject& object) const {
54    return (size != object.size) ? (size < object.size) : (name < object.name);
55  }
56  string name;
57  int type;
58  off_t size;
59};
60
61// Writes the uint64_t passed in in host-endian to the file as big-endian.
62// Returns true on success.
63bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) {
64  uint64_t value_be = htobe64(value);
65  TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)));
66  return true;
67}
68
69}  // namespace
70
71bool PayloadFile::Init(const PayloadGenerationConfig& config) {
72  major_version_ = config.major_version;
73  TEST_AND_RETURN_FALSE(major_version_ == kChromeOSMajorPayloadVersion ||
74                        major_version_ == kBrilloMajorPayloadVersion);
75  manifest_.set_minor_version(config.minor_version);
76
77  if (!config.source.ImageInfoIsEmpty())
78    *(manifest_.mutable_old_image_info()) = config.source.image_info;
79
80  if (!config.target.ImageInfoIsEmpty())
81    *(manifest_.mutable_new_image_info()) = config.target.image_info;
82
83  manifest_.set_block_size(config.block_size);
84  return true;
85}
86
87bool PayloadFile::AddPartition(const PartitionConfig& old_conf,
88                               const PartitionConfig& new_conf,
89                               const vector<AnnotatedOperation>& aops) {
90  // Check partitions order for Chrome OS
91  if (major_version_ == kChromeOSMajorPayloadVersion) {
92    const vector<const char*> part_order = { kLegacyPartitionNameRoot,
93                                             kLegacyPartitionNameKernel };
94    TEST_AND_RETURN_FALSE(part_vec_.size() < part_order.size());
95    TEST_AND_RETURN_FALSE(new_conf.name == part_order[part_vec_.size()]);
96  }
97  Partition part;
98  part.name = new_conf.name;
99  part.aops = aops;
100  // Initialize the PartitionInfo objects if present.
101  if (!old_conf.path.empty())
102    TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(old_conf,
103                                                              &part.old_info));
104  TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(new_conf,
105                                                            &part.new_info));
106  part_vec_.push_back(std::move(part));
107  return true;
108}
109
110bool PayloadFile::WritePayload(const string& payload_file,
111                               const string& data_blobs_path,
112                               const string& private_key_path,
113                               uint64_t* medatata_size_out) {
114  // Reorder the data blobs with the manifest_.
115  string ordered_blobs_path;
116  TEST_AND_RETURN_FALSE(utils::MakeTempFile(
117      "CrAU_temp_data.ordered.XXXXXX",
118      &ordered_blobs_path,
119      nullptr));
120  ScopedPathUnlinker ordered_blobs_unlinker(ordered_blobs_path);
121  TEST_AND_RETURN_FALSE(ReorderDataBlobs(data_blobs_path, ordered_blobs_path));
122
123  // Check that install op blobs are in order.
124  uint64_t next_blob_offset = 0;
125  for (const auto& part : part_vec_) {
126    for (const auto& aop : part.aops) {
127      if (!aop.op.has_data_offset())
128        continue;
129      if (aop.op.data_offset() != next_blob_offset) {
130        LOG(FATAL) << "bad blob offset! " << aop.op.data_offset() << " != "
131                   << next_blob_offset;
132      }
133      next_blob_offset += aop.op.data_length();
134    }
135  }
136
137  // Copy the operations and partition info from the part_vec_ to the manifest.
138  manifest_.clear_install_operations();
139  manifest_.clear_kernel_install_operations();
140  manifest_.clear_partitions();
141  for (const auto& part : part_vec_) {
142    if (major_version_ == kBrilloMajorPayloadVersion) {
143      PartitionUpdate* partition = manifest_.add_partitions();
144      partition->set_partition_name(part.name);
145      for (const AnnotatedOperation& aop : part.aops) {
146        *partition->add_operations() = aop.op;
147      }
148      if (part.old_info.has_size() || part.old_info.has_hash())
149        *(partition->mutable_old_partition_info()) = part.old_info;
150      if (part.new_info.has_size() || part.new_info.has_hash())
151        *(partition->mutable_new_partition_info()) = part.new_info;
152    } else {
153      // major_version_ == kChromeOSMajorPayloadVersion
154      if (part.name == kLegacyPartitionNameKernel) {
155        for (const AnnotatedOperation& aop : part.aops)
156          *manifest_.add_kernel_install_operations() = aop.op;
157        if (part.old_info.has_size() || part.old_info.has_hash())
158          *manifest_.mutable_old_kernel_info() = part.old_info;
159        if (part.new_info.has_size() || part.new_info.has_hash())
160          *manifest_.mutable_new_kernel_info() = part.new_info;
161      } else {
162        for (const AnnotatedOperation& aop : part.aops)
163          *manifest_.add_install_operations() = aop.op;
164        if (part.old_info.has_size() || part.old_info.has_hash())
165          *manifest_.mutable_old_rootfs_info() = part.old_info;
166        if (part.new_info.has_size() || part.new_info.has_hash())
167          *manifest_.mutable_new_rootfs_info() = part.new_info;
168      }
169    }
170  }
171
172  // Signatures appear at the end of the blobs. Note the offset in the
173  // manifest_.
174  if (!private_key_path.empty()) {
175    uint64_t signature_blob_length = 0;
176    TEST_AND_RETURN_FALSE(
177        PayloadSigner::SignatureBlobLength(vector<string>(1, private_key_path),
178                                           &signature_blob_length));
179    AddSignatureOp(next_blob_offset, signature_blob_length, &manifest_);
180  }
181
182  // Serialize protobuf
183  string serialized_manifest;
184  TEST_AND_RETURN_FALSE(manifest_.AppendToString(&serialized_manifest));
185
186  LOG(INFO) << "Writing final delta file header...";
187  DirectFileWriter writer;
188  TEST_AND_RETURN_FALSE_ERRNO(writer.Open(payload_file.c_str(),
189                                          O_WRONLY | O_CREAT | O_TRUNC,
190                                          0644) == 0);
191  ScopedFileWriterCloser writer_closer(&writer);
192
193  // Write header
194  TEST_AND_RETURN_FALSE(writer.Write(kDeltaMagic, sizeof(kDeltaMagic)));
195
196  // Write major version number
197  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, major_version_));
198
199  // Write protobuf length
200  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer,
201                                               serialized_manifest.size()));
202
203  if (major_version_ == kBrilloMajorPayloadVersion) {
204    // Write metadata signature size.
205    uint32_t zero = htobe32(0);
206    TEST_AND_RETURN_FALSE(writer.Write(&zero, sizeof(zero)));
207  }
208
209  // Write protobuf
210  LOG(INFO) << "Writing final delta file protobuf... "
211            << serialized_manifest.size();
212  TEST_AND_RETURN_FALSE(writer.Write(serialized_manifest.data(),
213                                     serialized_manifest.size()));
214
215  // Append the data blobs
216  LOG(INFO) << "Writing final delta file data blobs...";
217  int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0);
218  ScopedFdCloser blobs_fd_closer(&blobs_fd);
219  TEST_AND_RETURN_FALSE(blobs_fd >= 0);
220  for (;;) {
221    vector<char> buf(1024 * 1024);
222    ssize_t rc = read(blobs_fd, buf.data(), buf.size());
223    if (0 == rc) {
224      // EOF
225      break;
226    }
227    TEST_AND_RETURN_FALSE_ERRNO(rc > 0);
228    TEST_AND_RETURN_FALSE(writer.Write(buf.data(), rc));
229  }
230
231  // Write signature blob.
232  if (!private_key_path.empty()) {
233    LOG(INFO) << "Signing the update...";
234    chromeos::Blob signature_blob;
235    TEST_AND_RETURN_FALSE(PayloadSigner::SignPayload(
236        payload_file,
237        vector<string>(1, private_key_path),
238        &signature_blob));
239    TEST_AND_RETURN_FALSE(writer.Write(signature_blob.data(),
240                                       signature_blob.size()));
241  }
242
243  *medatata_size_out =
244      sizeof(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size();
245  ReportPayloadUsage(*medatata_size_out);
246  return true;
247}
248
249bool PayloadFile::ReorderDataBlobs(
250    const string& data_blobs_path,
251    const string& new_data_blobs_path) {
252  int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0);
253  TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0);
254  ScopedFdCloser in_fd_closer(&in_fd);
255
256  DirectFileWriter writer;
257  TEST_AND_RETURN_FALSE(
258      writer.Open(new_data_blobs_path.c_str(),
259                  O_WRONLY | O_TRUNC | O_CREAT,
260                  0644) == 0);
261  ScopedFileWriterCloser writer_closer(&writer);
262  uint64_t out_file_size = 0;
263
264  for (auto& part: part_vec_) {
265    for (AnnotatedOperation& aop : part.aops) {
266      if (!aop.op.has_data_offset())
267        continue;
268      CHECK(aop.op.has_data_length());
269      chromeos::Blob buf(aop.op.data_length());
270      ssize_t rc = pread(in_fd, buf.data(), buf.size(), aop.op.data_offset());
271      TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size()));
272
273      // Add the hash of the data blobs for this operation
274      TEST_AND_RETURN_FALSE(AddOperationHash(&aop.op, buf));
275
276      aop.op.set_data_offset(out_file_size);
277      TEST_AND_RETURN_FALSE(writer.Write(buf.data(), buf.size()));
278      out_file_size += buf.size();
279    }
280  }
281  return true;
282}
283
284bool PayloadFile::AddOperationHash(InstallOperation* op,
285                                   const chromeos::Blob& buf) {
286  OmahaHashCalculator hasher;
287  TEST_AND_RETURN_FALSE(hasher.Update(buf.data(), buf.size()));
288  TEST_AND_RETURN_FALSE(hasher.Finalize());
289  const chromeos::Blob& hash = hasher.raw_hash();
290  op->set_data_sha256_hash(hash.data(), hash.size());
291  return true;
292}
293
294void PayloadFile::ReportPayloadUsage(uint64_t metadata_size) const {
295  vector<DeltaObject> objects;
296  off_t total_size = 0;
297
298  for (const auto& part : part_vec_) {
299    for (const AnnotatedOperation& aop : part.aops) {
300      objects.push_back(DeltaObject(aop.name,
301                                    aop.op.type(),
302                                    aop.op.data_length()));
303      total_size += aop.op.data_length();
304    }
305  }
306
307  objects.push_back(DeltaObject("<manifest-metadata>",
308                                -1,
309                                metadata_size));
310  total_size += metadata_size;
311
312  std::sort(objects.begin(), objects.end());
313
314  static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n";
315  for (const DeltaObject& object : objects) {
316    fprintf(stderr, kFormatString,
317            object.size * 100.0 / total_size,
318            static_cast<intmax_t>(object.size),
319            object.type >= 0 ? kInstallOperationTypes[object.type] : "-",
320            object.name.c_str());
321  }
322  fprintf(stderr, kFormatString,
323          100.0, static_cast<intmax_t>(total_size), "", "<total>");
324}
325
326void AddSignatureOp(uint64_t signature_blob_offset,
327                    uint64_t signature_blob_length,
328                    DeltaArchiveManifest* manifest) {
329  LOG(INFO) << "Making room for signature in file";
330  manifest->set_signatures_offset(signature_blob_offset);
331  LOG(INFO) << "set? " << manifest->has_signatures_offset();
332  // Add a dummy op at the end to appease older clients
333  InstallOperation* dummy_op = manifest->add_kernel_install_operations();
334  dummy_op->set_type(InstallOperation::REPLACE);
335  dummy_op->set_data_offset(signature_blob_offset);
336  manifest->set_signatures_offset(signature_blob_offset);
337  dummy_op->set_data_length(signature_blob_length);
338  manifest->set_signatures_size(signature_blob_length);
339  Extent* dummy_extent = dummy_op->add_dst_extents();
340  // Tell the dummy op to write this data to a big sparse hole
341  dummy_extent->set_start_block(kSparseHole);
342  dummy_extent->set_num_blocks((signature_blob_length + kBlockSize - 1) /
343                               kBlockSize);
344}
345
346}  // namespace chromeos_update_engine
347