ab_generator.cc revision ebdf17d4202c67933764135bfc1cece629829201
176d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman// 276d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman// Copyright (C) 2015 The Android Open Source Project 376d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman// 476d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman// Licensed under the Apache License, Version 2.0 (the "License"); 576d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman// you may not use this file except in compliance with the License. 676d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman// You may obtain a copy of the License at 776d05dc695b06c4e987bb8078f78032441e1430cGreg Hartman// 8// http://www.apache.org/licenses/LICENSE-2.0 9// 10// Unless required by applicable law or agreed to in writing, software 11// distributed under the License is distributed on an "AS IS" BASIS, 12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13// See the License for the specific language governing permissions and 14// limitations under the License. 15// 16 17#include "update_engine/payload_generator/ab_generator.h" 18 19#include <algorithm> 20 21#include <base/strings/stringprintf.h> 22 23#include "update_engine/bzip.h" 24#include "update_engine/delta_performer.h" 25#include "update_engine/payload_generator/annotated_operation.h" 26#include "update_engine/payload_generator/delta_diff_generator.h" 27#include "update_engine/payload_generator/delta_diff_utils.h" 28#include "update_engine/utils.h" 29 30using std::string; 31using std::vector; 32 33namespace chromeos_update_engine { 34 35bool ABGenerator::GenerateOperations( 36 const PayloadGenerationConfig& config, 37 const PartitionConfig& old_part, 38 const PartitionConfig& new_part, 39 BlobFileWriter* blob_file, 40 vector<AnnotatedOperation>* aops) { 41 42 ssize_t hard_chunk_blocks = (config.hard_chunk_size == -1 ? -1 : 43 config.hard_chunk_size / config.block_size); 44 size_t soft_chunk_blocks = config.soft_chunk_size / config.block_size; 45 46 aops->clear(); 47 TEST_AND_RETURN_FALSE(diff_utils::DeltaReadPartition( 48 aops, 49 old_part, 50 new_part, 51 hard_chunk_blocks, 52 soft_chunk_blocks, 53 blob_file, 54 true)); // src_ops_allowed 55 LOG(INFO) << "done reading " << PartitionNameString(new_part.name); 56 57 TEST_AND_RETURN_FALSE(FragmentOperations(aops, 58 new_part.path, 59 blob_file)); 60 SortOperationsByDestination(aops); 61 62 // Use the soft_chunk_size when merging operations to prevent merging all 63 // the operations into a huge one if there's no hard limit. 64 size_t merge_chunk_blocks = soft_chunk_blocks; 65 if (hard_chunk_blocks != -1 && 66 static_cast<size_t>(hard_chunk_blocks) < soft_chunk_blocks) { 67 merge_chunk_blocks = hard_chunk_blocks; 68 } 69 70 TEST_AND_RETURN_FALSE(MergeOperations(aops, 71 merge_chunk_blocks, 72 new_part.path, 73 blob_file)); 74 return true; 75} 76 77void ABGenerator::SortOperationsByDestination( 78 vector<AnnotatedOperation>* aops) { 79 sort(aops->begin(), aops->end(), diff_utils::CompareAopsByDestination); 80} 81 82bool ABGenerator::FragmentOperations( 83 vector<AnnotatedOperation>* aops, 84 const string& target_part_path, 85 BlobFileWriter* blob_file) { 86 vector<AnnotatedOperation> fragmented_aops; 87 for (const AnnotatedOperation& aop : *aops) { 88 if (aop.op.type() == InstallOperation::SOURCE_COPY) { 89 TEST_AND_RETURN_FALSE(SplitSourceCopy(aop, &fragmented_aops)); 90 } else if ((aop.op.type() == InstallOperation::REPLACE) || 91 (aop.op.type() == InstallOperation::REPLACE_BZ)) { 92 TEST_AND_RETURN_FALSE(SplitReplaceOrReplaceBz(aop, &fragmented_aops, 93 target_part_path, 94 blob_file)); 95 } else { 96 fragmented_aops.push_back(aop); 97 } 98 } 99 *aops = fragmented_aops; 100 return true; 101} 102 103bool ABGenerator::SplitSourceCopy( 104 const AnnotatedOperation& original_aop, 105 vector<AnnotatedOperation>* result_aops) { 106 InstallOperation original_op = original_aop.op; 107 TEST_AND_RETURN_FALSE(original_op.type() == InstallOperation::SOURCE_COPY); 108 // Keeps track of the index of curr_src_ext. 109 int curr_src_ext_index = 0; 110 Extent curr_src_ext = original_op.src_extents(curr_src_ext_index); 111 for (int i = 0; i < original_op.dst_extents_size(); i++) { 112 Extent dst_ext = original_op.dst_extents(i); 113 // The new operation which will have only one dst extent. 114 InstallOperation new_op; 115 uint64_t blocks_left = dst_ext.num_blocks(); 116 while (blocks_left > 0) { 117 if (curr_src_ext.num_blocks() <= blocks_left) { 118 // If the curr_src_ext is smaller than dst_ext, add it. 119 blocks_left -= curr_src_ext.num_blocks(); 120 *(new_op.add_src_extents()) = curr_src_ext; 121 if (curr_src_ext_index + 1 < original_op.src_extents().size()) { 122 curr_src_ext = original_op.src_extents(++curr_src_ext_index); 123 } else { 124 break; 125 } 126 } else { 127 // Split src_exts that are bigger than the dst_ext we're dealing with. 128 Extent first_ext; 129 first_ext.set_num_blocks(blocks_left); 130 first_ext.set_start_block(curr_src_ext.start_block()); 131 *(new_op.add_src_extents()) = first_ext; 132 // Keep the second half of the split op. 133 curr_src_ext.set_num_blocks(curr_src_ext.num_blocks() - blocks_left); 134 curr_src_ext.set_start_block(curr_src_ext.start_block() + blocks_left); 135 blocks_left -= first_ext.num_blocks(); 136 } 137 } 138 // Fix up our new operation and add it to the results. 139 new_op.set_type(InstallOperation::SOURCE_COPY); 140 *(new_op.add_dst_extents()) = dst_ext; 141 new_op.set_src_length(dst_ext.num_blocks() * kBlockSize); 142 new_op.set_dst_length(dst_ext.num_blocks() * kBlockSize); 143 144 AnnotatedOperation new_aop; 145 new_aop.op = new_op; 146 new_aop.name = base::StringPrintf("%s:%d", original_aop.name.c_str(), i); 147 result_aops->push_back(new_aop); 148 } 149 if (curr_src_ext_index != original_op.src_extents().size() - 1) { 150 LOG(FATAL) << "Incorrectly split SOURCE_COPY operation. Did not use all " 151 << "source extents."; 152 } 153 return true; 154} 155 156bool ABGenerator::SplitReplaceOrReplaceBz( 157 const AnnotatedOperation& original_aop, 158 vector<AnnotatedOperation>* result_aops, 159 const string& target_part_path, 160 BlobFileWriter* blob_file) { 161 InstallOperation original_op = original_aop.op; 162 const bool is_replace = original_op.type() == InstallOperation::REPLACE; 163 TEST_AND_RETURN_FALSE(is_replace || 164 original_op.type() == InstallOperation::REPLACE_BZ); 165 166 uint32_t data_offset = original_op.data_offset(); 167 for (int i = 0; i < original_op.dst_extents_size(); i++) { 168 Extent dst_ext = original_op.dst_extents(i); 169 // Make a new operation with only one dst extent. 170 InstallOperation new_op; 171 *(new_op.add_dst_extents()) = dst_ext; 172 uint32_t data_size = dst_ext.num_blocks() * kBlockSize; 173 new_op.set_dst_length(data_size); 174 // If this is a REPLACE, attempt to reuse portions of the existing blob. 175 if (is_replace) { 176 new_op.set_type(InstallOperation::REPLACE); 177 new_op.set_data_length(data_size); 178 new_op.set_data_offset(data_offset); 179 data_offset += data_size; 180 } 181 182 AnnotatedOperation new_aop; 183 new_aop.op = new_op; 184 new_aop.name = base::StringPrintf("%s:%d", original_aop.name.c_str(), i); 185 TEST_AND_RETURN_FALSE(AddDataAndSetType(&new_aop, target_part_path, 186 blob_file)); 187 188 result_aops->push_back(new_aop); 189 } 190 return true; 191} 192 193bool ABGenerator::MergeOperations(vector<AnnotatedOperation>* aops, 194 size_t chunk_blocks, 195 const string& target_part_path, 196 BlobFileWriter* blob_file) { 197 vector<AnnotatedOperation> new_aops; 198 for (const AnnotatedOperation& curr_aop : *aops) { 199 if (new_aops.empty()) { 200 new_aops.push_back(curr_aop); 201 continue; 202 } 203 AnnotatedOperation& last_aop = new_aops.back(); 204 205 if (last_aop.op.dst_extents_size() <= 0 || 206 curr_aop.op.dst_extents_size() <= 0) { 207 new_aops.push_back(curr_aop); 208 continue; 209 } 210 uint32_t last_dst_idx = last_aop.op.dst_extents_size() - 1; 211 uint32_t last_end_block = 212 last_aop.op.dst_extents(last_dst_idx).start_block() + 213 last_aop.op.dst_extents(last_dst_idx).num_blocks(); 214 uint32_t curr_start_block = curr_aop.op.dst_extents(0).start_block(); 215 uint32_t combined_block_count = 216 last_aop.op.dst_extents(last_dst_idx).num_blocks() + 217 curr_aop.op.dst_extents(0).num_blocks(); 218 bool good_op_type = curr_aop.op.type() == InstallOperation::SOURCE_COPY || 219 curr_aop.op.type() == InstallOperation::REPLACE || 220 curr_aop.op.type() == InstallOperation::REPLACE_BZ; 221 if (good_op_type && 222 last_aop.op.type() == curr_aop.op.type() && 223 last_end_block == curr_start_block && 224 combined_block_count <= chunk_blocks) { 225 // If the operations have the same type (which is a type that we can 226 // merge), are contiguous, are fragmented to have one destination extent, 227 // and their combined block count would be less than chunk size, merge 228 // them. 229 last_aop.name = base::StringPrintf("%s,%s", 230 last_aop.name.c_str(), 231 curr_aop.name.c_str()); 232 233 ExtendExtents(last_aop.op.mutable_src_extents(), 234 curr_aop.op.src_extents()); 235 if (curr_aop.op.src_length() > 0) 236 last_aop.op.set_src_length(last_aop.op.src_length() + 237 curr_aop.op.src_length()); 238 ExtendExtents(last_aop.op.mutable_dst_extents(), 239 curr_aop.op.dst_extents()); 240 if (curr_aop.op.dst_length() > 0) 241 last_aop.op.set_dst_length(last_aop.op.dst_length() + 242 curr_aop.op.dst_length()); 243 // Set the data length to zero so we know to add the blob later. 244 if (curr_aop.op.type() == InstallOperation::REPLACE || 245 curr_aop.op.type() == InstallOperation::REPLACE_BZ) { 246 last_aop.op.set_data_length(0); 247 } 248 } else { 249 // Otherwise just include the extent as is. 250 new_aops.push_back(curr_aop); 251 } 252 } 253 254 // Set the blobs for REPLACE/REPLACE_BZ operations that have been merged. 255 for (AnnotatedOperation& curr_aop : new_aops) { 256 if (curr_aop.op.data_length() == 0 && 257 (curr_aop.op.type() == InstallOperation::REPLACE || 258 curr_aop.op.type() == InstallOperation::REPLACE_BZ)) { 259 TEST_AND_RETURN_FALSE(AddDataAndSetType(&curr_aop, target_part_path, 260 blob_file)); 261 } 262 } 263 264 *aops = new_aops; 265 return true; 266} 267 268bool ABGenerator::AddDataAndSetType(AnnotatedOperation* aop, 269 const string& target_part_path, 270 BlobFileWriter* blob_file) { 271 TEST_AND_RETURN_FALSE(aop->op.type() == InstallOperation::REPLACE || 272 aop->op.type() == InstallOperation::REPLACE_BZ); 273 274 chromeos::Blob data(aop->op.dst_length()); 275 vector<Extent> dst_extents; 276 ExtentsToVector(aop->op.dst_extents(), &dst_extents); 277 TEST_AND_RETURN_FALSE(utils::ReadExtents(target_part_path, 278 dst_extents, 279 &data, 280 data.size(), 281 kBlockSize)); 282 283 chromeos::Blob data_bz; 284 TEST_AND_RETURN_FALSE(BzipCompress(data, &data_bz)); 285 CHECK(!data_bz.empty()); 286 287 chromeos::Blob* data_p = nullptr; 288 InstallOperation_Type new_op_type; 289 if (data_bz.size() < data.size()) { 290 new_op_type = InstallOperation::REPLACE_BZ; 291 data_p = &data_bz; 292 } else { 293 new_op_type = InstallOperation::REPLACE; 294 data_p = &data; 295 } 296 297 // If the operation doesn't point to a data blob, then we add it. 298 if (aop->op.type() != new_op_type || 299 aop->op.data_length() != data_p->size()) { 300 aop->op.set_type(new_op_type); 301 aop->SetOperationBlob(data_p, blob_file); 302 } 303 304 return true; 305} 306 307} // namespace chromeos_update_engine 308