payload_file.cc revision 46e9b1735a16518d563b3c7e7227408f8fc3e891 
1//
2// Copyright (C) 2015 The Android Open Source Project
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8//      http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16
17#include "update_engine/payload_generator/payload_file.h"
18
19#include <algorithm>
20
21#include "update_engine/delta_performer.h"
22#include "update_engine/file_writer.h"
23#include "update_engine/omaha_hash_calculator.h"
24#include "update_engine/payload_constants.h"
25#include "update_engine/payload_generator/annotated_operation.h"
26#include "update_engine/payload_generator/delta_diff_generator.h"
27#include "update_engine/payload_generator/delta_diff_utils.h"
28#include "update_engine/payload_generator/payload_signer.h"
29
30using std::string;
31using std::vector;
32
33namespace chromeos_update_engine {
34
35namespace {
36
37static const char* kInstallOperationTypes[] = {
38  "REPLACE",
39  "REPLACE_BZ",
40  "MOVE",
41  "BSDIFF",
42  "SOURCE_COPY",
43  "SOURCE_BSDIFF"
44};
45
46struct DeltaObject {
47  DeltaObject(const string& in_name, const int in_type, const off_t in_size)
48      : name(in_name),
49        type(in_type),
50        size(in_size) {}
51  bool operator <(const DeltaObject& object) const {
52    return (size != object.size) ? (size < object.size) : (name < object.name);
53  }
54  string name;
55  int type;
56  off_t size;
57};
58
59// Writes the uint64_t passed in in host-endian to the file as big-endian.
60// Returns true on success.
61bool WriteUint64AsBigEndian(FileWriter* writer, const uint64_t value) {
62  uint64_t value_be = htobe64(value);
63  TEST_AND_RETURN_FALSE(writer->Write(&value_be, sizeof(value_be)));
64  return true;
65}
66
67}  // namespace
68
69const vector<PartitionName> PayloadFile::partition_disk_order_ = {
70  PartitionName::kRootfs,
71  PartitionName::kKernel,
72};
73
74bool PayloadFile::Init(const PayloadGenerationConfig& config) {
75  major_version_ = config.major_version;
76  TEST_AND_RETURN_FALSE(major_version_ == kChromeOSMajorPayloadVersion ||
77                        major_version_ == kBrilloMajorPayloadVersion);
78  manifest_.set_minor_version(config.minor_version);
79
80  if (!config.source.ImageInfoIsEmpty())
81    *(manifest_.mutable_old_image_info()) = config.source.image_info;
82
83  if (!config.target.ImageInfoIsEmpty())
84    *(manifest_.mutable_new_image_info()) = config.target.image_info;
85
86  manifest_.set_block_size(config.block_size);
87
88  // Initialize the PartitionInfo objects if present.
89  if (!config.source.kernel.path.empty()) {
90    TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(
91        config.source.kernel,
92        manifest_.mutable_old_kernel_info()));
93  }
94  TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(
95      config.target.kernel,
96      manifest_.mutable_new_kernel_info()));
97  if (!config.source.rootfs.path.empty()) {
98    TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(
99        config.source.rootfs,
100        manifest_.mutable_old_rootfs_info()));
101  }
102  TEST_AND_RETURN_FALSE(diff_utils::InitializePartitionInfo(
103      config.target.rootfs,
104      manifest_.mutable_new_rootfs_info()));
105  return true;
106}
107
108void PayloadFile::AddPartitionOperations(
109    PartitionName name,
110    const vector<AnnotatedOperation>& aops) {
111  aops_map_[name].insert(aops_map_[name].end(), aops.begin(), aops.end());
112}
113
114bool PayloadFile::WritePayload(const string& payload_file,
115                               const string& data_blobs_path,
116                               const string& private_key_path,
117                               uint64_t* medatata_size_out) {
118  // Reorder the data blobs with the manifest_.
119  string ordered_blobs_path;
120  TEST_AND_RETURN_FALSE(utils::MakeTempFile(
121      "CrAU_temp_data.ordered.XXXXXX",
122      &ordered_blobs_path,
123      nullptr));
124  ScopedPathUnlinker ordered_blobs_unlinker(ordered_blobs_path);
125  TEST_AND_RETURN_FALSE(ReorderDataBlobs(data_blobs_path, ordered_blobs_path));
126
127  // Copy the operations from the aops_map_ to the manifest.
128  manifest_.clear_install_operations();
129  manifest_.clear_kernel_install_operations();
130  for (PartitionName name : partition_disk_order_) {
131    for (const AnnotatedOperation& aop : aops_map_[name]) {
132      if (name == PartitionName::kKernel) {
133        *manifest_.add_kernel_install_operations() = aop.op;
134      } else {
135        *manifest_.add_install_operations() = aop.op;
136      }
137    }
138  }
139
140  // Check that install op blobs are in order.
141  uint64_t next_blob_offset = 0;
142  {
143    for (int i = 0; i < (manifest_.install_operations_size() +
144                         manifest_.kernel_install_operations_size()); i++) {
145      InstallOperation* op = i < manifest_.install_operations_size()
146                                 ? manifest_.mutable_install_operations(i)
147                                 : manifest_.mutable_kernel_install_operations(
148                                       i - manifest_.install_operations_size());
149      if (op->has_data_offset()) {
150        if (op->data_offset() != next_blob_offset) {
151          LOG(FATAL) << "bad blob offset! " << op->data_offset() << " != "
152                     << next_blob_offset;
153        }
154        next_blob_offset += op->data_length();
155      }
156    }
157  }
158
159  // Signatures appear at the end of the blobs. Note the offset in the
160  // manifest_.
161  if (!private_key_path.empty()) {
162    uint64_t signature_blob_length = 0;
163    TEST_AND_RETURN_FALSE(
164        PayloadSigner::SignatureBlobLength(vector<string>(1, private_key_path),
165                                           &signature_blob_length));
166    AddSignatureOp(next_blob_offset, signature_blob_length, &manifest_);
167  }
168
169    // Serialize protobuf
170  string serialized_manifest;
171  TEST_AND_RETURN_FALSE(manifest_.AppendToString(&serialized_manifest));
172
173  LOG(INFO) << "Writing final delta file header...";
174  DirectFileWriter writer;
175  TEST_AND_RETURN_FALSE_ERRNO(writer.Open(payload_file.c_str(),
176                                          O_WRONLY | O_CREAT | O_TRUNC,
177                                          0644) == 0);
178  ScopedFileWriterCloser writer_closer(&writer);
179
180  // Write header
181  TEST_AND_RETURN_FALSE(writer.Write(kDeltaMagic, strlen(kDeltaMagic)));
182
183  // Write major version number
184  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, major_version_));
185
186  // Write protobuf length
187  TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer,
188                                               serialized_manifest.size()));
189
190  // Write protobuf
191  LOG(INFO) << "Writing final delta file protobuf... "
192            << serialized_manifest.size();
193  TEST_AND_RETURN_FALSE(writer.Write(serialized_manifest.data(),
194                                     serialized_manifest.size()));
195
196  // Append the data blobs
197  LOG(INFO) << "Writing final delta file data blobs...";
198  int blobs_fd = open(ordered_blobs_path.c_str(), O_RDONLY, 0);
199  ScopedFdCloser blobs_fd_closer(&blobs_fd);
200  TEST_AND_RETURN_FALSE(blobs_fd >= 0);
201  for (;;) {
202    vector<char> buf(1024 * 1024);
203    ssize_t rc = read(blobs_fd, buf.data(), buf.size());
204    if (0 == rc) {
205      // EOF
206      break;
207    }
208    TEST_AND_RETURN_FALSE_ERRNO(rc > 0);
209    TEST_AND_RETURN_FALSE(writer.Write(buf.data(), rc));
210  }
211
212  // Write signature blob.
213  if (!private_key_path.empty()) {
214    LOG(INFO) << "Signing the update...";
215    chromeos::Blob signature_blob;
216    TEST_AND_RETURN_FALSE(PayloadSigner::SignPayload(
217        payload_file,
218        vector<string>(1, private_key_path),
219        &signature_blob));
220    TEST_AND_RETURN_FALSE(writer.Write(signature_blob.data(),
221                                       signature_blob.size()));
222  }
223
224  *medatata_size_out =
225      strlen(kDeltaMagic) + 2 * sizeof(uint64_t) + serialized_manifest.size();
226  ReportPayloadUsage(*medatata_size_out);
227  return true;
228}
229
230bool PayloadFile::ReorderDataBlobs(
231    const string& data_blobs_path,
232    const string& new_data_blobs_path) {
233  int in_fd = open(data_blobs_path.c_str(), O_RDONLY, 0);
234  TEST_AND_RETURN_FALSE_ERRNO(in_fd >= 0);
235  ScopedFdCloser in_fd_closer(&in_fd);
236
237  DirectFileWriter writer;
238  TEST_AND_RETURN_FALSE(
239      writer.Open(new_data_blobs_path.c_str(),
240                  O_WRONLY | O_TRUNC | O_CREAT,
241                  0644) == 0);
242  ScopedFileWriterCloser writer_closer(&writer);
243  uint64_t out_file_size = 0;
244
245  for (PartitionName name : partition_disk_order_) {
246    for (AnnotatedOperation& aop : aops_map_[name]) {
247      if (!aop.op.has_data_offset())
248        continue;
249      CHECK(aop.op.has_data_length());
250      chromeos::Blob buf(aop.op.data_length());
251      ssize_t rc = pread(in_fd, buf.data(), buf.size(), aop.op.data_offset());
252      TEST_AND_RETURN_FALSE(rc == static_cast<ssize_t>(buf.size()));
253
254      // Add the hash of the data blobs for this operation
255      TEST_AND_RETURN_FALSE(AddOperationHash(&aop.op, buf));
256
257      aop.op.set_data_offset(out_file_size);
258      TEST_AND_RETURN_FALSE(writer.Write(buf.data(), buf.size()));
259      out_file_size += buf.size();
260    }
261  }
262  return true;
263}
264
265bool PayloadFile::AddOperationHash(InstallOperation* op,
266                                   const chromeos::Blob& buf) {
267  OmahaHashCalculator hasher;
268  TEST_AND_RETURN_FALSE(hasher.Update(buf.data(), buf.size()));
269  TEST_AND_RETURN_FALSE(hasher.Finalize());
270  const chromeos::Blob& hash = hasher.raw_hash();
271  op->set_data_sha256_hash(hash.data(), hash.size());
272  return true;
273}
274
275void PayloadFile::ReportPayloadUsage(uint64_t metadata_size) const {
276  vector<DeltaObject> objects;
277  off_t total_size = 0;
278
279  for (PartitionName name : partition_disk_order_) {
280    const auto& partition_aops = aops_map_.find(name);
281    if (partition_aops == aops_map_.end())
282      continue;
283    for (const AnnotatedOperation& aop : partition_aops->second) {
284      objects.push_back(DeltaObject(aop.name,
285                                    aop.op.type(),
286                                    aop.op.data_length()));
287      total_size += aop.op.data_length();
288    }
289  }
290
291  objects.push_back(DeltaObject("<manifest-metadata>",
292                                -1,
293                                metadata_size));
294  total_size += metadata_size;
295
296  std::sort(objects.begin(), objects.end());
297
298  static const char kFormatString[] = "%6.2f%% %10jd %-10s %s\n";
299  for (const DeltaObject& object : objects) {
300    fprintf(stderr, kFormatString,
301            object.size * 100.0 / total_size,
302            static_cast<intmax_t>(object.size),
303            object.type >= 0 ? kInstallOperationTypes[object.type] : "-",
304            object.name.c_str());
305  }
306  fprintf(stderr, kFormatString,
307          100.0, static_cast<intmax_t>(total_size), "", "<total>");
308}
309
310void AddSignatureOp(uint64_t signature_blob_offset,
311                    uint64_t signature_blob_length,
312                    DeltaArchiveManifest* manifest) {
313  LOG(INFO) << "Making room for signature in file";
314  manifest->set_signatures_offset(signature_blob_offset);
315  LOG(INFO) << "set? " << manifest->has_signatures_offset();
316  // Add a dummy op at the end to appease older clients
317  InstallOperation* dummy_op = manifest->add_kernel_install_operations();
318  dummy_op->set_type(InstallOperation::REPLACE);
319  dummy_op->set_data_offset(signature_blob_offset);
320  manifest->set_signatures_offset(signature_blob_offset);
321  dummy_op->set_data_length(signature_blob_length);
322  manifest->set_signatures_size(signature_blob_length);
323  Extent* dummy_extent = dummy_op->add_dst_extents();
324  // Tell the dummy op to write this data to a big sparse hole
325  dummy_extent->set_start_block(kSparseHole);
326  dummy_extent->set_num_blocks((signature_blob_length + kBlockSize - 1) /
327                               kBlockSize);
328}
329
330}  // namespace chromeos_update_engine
331