1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "image.h" 18 19#include <memory> 20#include <string> 21#include <vector> 22 23#include "base/unix_file/fd_file.h" 24#include "class_linker-inl.h" 25#include "common_compiler_test.h" 26#include "debug/method_debug_info.h" 27#include "driver/compiler_options.h" 28#include "elf_writer.h" 29#include "elf_writer_quick.h" 30#include "gc/space/image_space.h" 31#include "image_writer.h" 32#include "linker/multi_oat_relative_patcher.h" 33#include "lock_word.h" 34#include "mirror/object-inl.h" 35#include "oat_writer.h" 36#include "scoped_thread_state_change.h" 37#include "signal_catcher.h" 38#include "utils.h" 39 40namespace art { 41 42static const uintptr_t kRequestedImageBase = ART_BASE_ADDRESS; 43 44struct CompilationHelper { 45 std::vector<std::string> dex_file_locations; 46 std::vector<ScratchFile> image_locations; 47 std::vector<std::unique_ptr<const DexFile>> extra_dex_files; 48 std::vector<ScratchFile> image_files; 49 std::vector<ScratchFile> oat_files; 50 std::string image_dir; 51 52 void Compile(CompilerDriver* driver, 53 ImageHeader::StorageMode storage_mode); 54 55 std::vector<size_t> GetImageObjectSectionSizes(); 56 57 ~CompilationHelper(); 58}; 59 60class ImageTest : public CommonCompilerTest { 61 protected: 62 virtual void SetUp() { 63 ReserveImageSpace(); 64 CommonCompilerTest::SetUp(); 65 } 66 67 void TestWriteRead(ImageHeader::StorageMode storage_mode); 68 69 void Compile(ImageHeader::StorageMode storage_mode, 70 CompilationHelper& out_helper, 71 const std::string& extra_dex = "", 72 const std::string& image_class = ""); 73 74 std::unordered_set<std::string>* GetImageClasses() OVERRIDE { 75 return new std::unordered_set<std::string>(image_classes_); 76 } 77 78 private: 79 std::unordered_set<std::string> image_classes_; 80}; 81 82CompilationHelper::~CompilationHelper() { 83 for (ScratchFile& image_file : image_files) { 84 image_file.Unlink(); 85 } 86 for (ScratchFile& oat_file : oat_files) { 87 oat_file.Unlink(); 88 } 89 const int rmdir_result = rmdir(image_dir.c_str()); 90 CHECK_EQ(0, rmdir_result); 91} 92 93std::vector<size_t> CompilationHelper::GetImageObjectSectionSizes() { 94 std::vector<size_t> ret; 95 for (ScratchFile& image_file : image_files) { 96 std::unique_ptr<File> file(OS::OpenFileForReading(image_file.GetFilename().c_str())); 97 CHECK(file.get() != nullptr); 98 ImageHeader image_header; 99 CHECK_EQ(file->ReadFully(&image_header, sizeof(image_header)), true); 100 CHECK(image_header.IsValid()); 101 ret.push_back(image_header.GetImageSize()); 102 } 103 return ret; 104} 105 106void CompilationHelper::Compile(CompilerDriver* driver, 107 ImageHeader::StorageMode storage_mode) { 108 ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); 109 std::vector<const DexFile*> class_path = class_linker->GetBootClassPath(); 110 111 for (const std::unique_ptr<const DexFile>& dex_file : extra_dex_files) { 112 { 113 ScopedObjectAccess soa(Thread::Current()); 114 // Inject in boot class path so that the compiler driver can see it. 115 class_linker->AppendToBootClassPath(soa.Self(), *dex_file.get()); 116 } 117 class_path.push_back(dex_file.get()); 118 } 119 120 // Enable write for dex2dex. 121 for (const DexFile* dex_file : class_path) { 122 dex_file_locations.push_back(dex_file->GetLocation()); 123 if (dex_file->IsReadOnly()) { 124 dex_file->EnableWrite(); 125 } 126 } 127 128 { 129 // Create a generic tmp file, to be the base of the .art and .oat temporary files. 130 ScratchFile location; 131 for (int i = 0; i < static_cast<int>(class_path.size()); ++i) { 132 std::string cur_location(StringPrintf("%s-%d.art", location.GetFilename().c_str(), i)); 133 image_locations.push_back(ScratchFile(cur_location)); 134 } 135 } 136 std::vector<std::string> image_filenames; 137 for (ScratchFile& file : image_locations) { 138 std::string image_filename(GetSystemImageFilename(file.GetFilename().c_str(), kRuntimeISA)); 139 image_filenames.push_back(image_filename); 140 size_t pos = image_filename.rfind('/'); 141 CHECK_NE(pos, std::string::npos) << image_filename; 142 if (image_dir.empty()) { 143 image_dir = image_filename.substr(0, pos); 144 int mkdir_result = mkdir(image_dir.c_str(), 0700); 145 CHECK_EQ(0, mkdir_result) << image_dir; 146 } 147 image_files.push_back(ScratchFile(OS::CreateEmptyFile(image_filename.c_str()))); 148 } 149 150 std::vector<std::string> oat_filenames; 151 for (const std::string& image_filename : image_filenames) { 152 std::string oat_filename(image_filename.substr(0, image_filename.size() - strlen("art")) + "oat"); 153 oat_files.push_back(ScratchFile(OS::CreateEmptyFile(oat_filename.c_str()))); 154 oat_filenames.push_back(oat_filename); 155 } 156 157 std::unordered_map<const DexFile*, size_t> dex_file_to_oat_index_map; 158 std::vector<const char*> oat_filename_vector; 159 for (const std::string& file : oat_filenames) { 160 oat_filename_vector.push_back(file.c_str()); 161 } 162 std::vector<const char*> image_filename_vector; 163 for (const std::string& file : image_filenames) { 164 image_filename_vector.push_back(file.c_str()); 165 } 166 size_t image_idx = 0; 167 for (const DexFile* dex_file : class_path) { 168 dex_file_to_oat_index_map.emplace(dex_file, image_idx); 169 ++image_idx; 170 } 171 // TODO: compile_pic should be a test argument. 172 std::unique_ptr<ImageWriter> writer(new ImageWriter(*driver, 173 kRequestedImageBase, 174 /*compile_pic*/false, 175 /*compile_app_image*/false, 176 storage_mode, 177 oat_filename_vector, 178 dex_file_to_oat_index_map)); 179 { 180 { 181 jobject class_loader = nullptr; 182 TimingLogger timings("ImageTest::WriteRead", false, false); 183 TimingLogger::ScopedTiming t("CompileAll", &timings); 184 driver->SetDexFilesForOatFile(class_path); 185 driver->CompileAll(class_loader, class_path, &timings); 186 187 t.NewTiming("WriteElf"); 188 SafeMap<std::string, std::string> key_value_store; 189 std::vector<const char*> dex_filename_vector; 190 for (size_t i = 0; i < class_path.size(); ++i) { 191 dex_filename_vector.push_back(""); 192 } 193 key_value_store.Put(OatHeader::kBootClassPathKey, 194 gc::space::ImageSpace::GetMultiImageBootClassPath( 195 dex_filename_vector, 196 oat_filename_vector, 197 image_filename_vector)); 198 199 std::vector<std::unique_ptr<ElfWriter>> elf_writers; 200 std::vector<std::unique_ptr<OatWriter>> oat_writers; 201 for (ScratchFile& oat_file : oat_files) { 202 elf_writers.emplace_back(CreateElfWriterQuick(driver->GetInstructionSet(), 203 driver->GetInstructionSetFeatures(), 204 &driver->GetCompilerOptions(), 205 oat_file.GetFile())); 206 elf_writers.back()->Start(); 207 oat_writers.emplace_back(new OatWriter(/*compiling_boot_image*/true, &timings)); 208 } 209 210 std::vector<OutputStream*> rodata; 211 std::vector<std::unique_ptr<MemMap>> opened_dex_files_map; 212 std::vector<std::unique_ptr<const DexFile>> opened_dex_files; 213 // Now that we have finalized key_value_store_, start writing the oat file. 214 for (size_t i = 0, size = oat_writers.size(); i != size; ++i) { 215 const DexFile* dex_file = class_path[i]; 216 rodata.push_back(elf_writers[i]->StartRoData()); 217 ArrayRef<const uint8_t> raw_dex_file( 218 reinterpret_cast<const uint8_t*>(&dex_file->GetHeader()), 219 dex_file->GetHeader().file_size_); 220 oat_writers[i]->AddRawDexFileSource(raw_dex_file, 221 dex_file->GetLocation().c_str(), 222 dex_file->GetLocationChecksum()); 223 224 std::unique_ptr<MemMap> cur_opened_dex_files_map; 225 std::vector<std::unique_ptr<const DexFile>> cur_opened_dex_files; 226 bool dex_files_ok = oat_writers[i]->WriteAndOpenDexFiles( 227 rodata.back(), 228 oat_files[i].GetFile(), 229 driver->GetInstructionSet(), 230 driver->GetInstructionSetFeatures(), 231 &key_value_store, 232 /* verify */ false, // Dex files may be dex-to-dex-ed, don't verify. 233 &cur_opened_dex_files_map, 234 &cur_opened_dex_files); 235 ASSERT_TRUE(dex_files_ok); 236 237 if (cur_opened_dex_files_map != nullptr) { 238 opened_dex_files_map.push_back(std::move(cur_opened_dex_files_map)); 239 for (std::unique_ptr<const DexFile>& cur_dex_file : cur_opened_dex_files) { 240 // dex_file_oat_index_map_.emplace(dex_file.get(), i); 241 opened_dex_files.push_back(std::move(cur_dex_file)); 242 } 243 } else { 244 ASSERT_TRUE(cur_opened_dex_files.empty()); 245 } 246 } 247 248 bool image_space_ok = writer->PrepareImageAddressSpace(); 249 ASSERT_TRUE(image_space_ok); 250 251 for (size_t i = 0, size = oat_files.size(); i != size; ++i) { 252 linker::MultiOatRelativePatcher patcher(driver->GetInstructionSet(), 253 driver->GetInstructionSetFeatures()); 254 OatWriter* const oat_writer = oat_writers[i].get(); 255 ElfWriter* const elf_writer = elf_writers[i].get(); 256 std::vector<const DexFile*> cur_dex_files(1u, class_path[i]); 257 oat_writer->PrepareLayout(driver, writer.get(), cur_dex_files, &patcher); 258 size_t rodata_size = oat_writer->GetOatHeader().GetExecutableOffset(); 259 size_t text_size = oat_writer->GetSize() - rodata_size; 260 elf_writer->SetLoadedSectionSizes(rodata_size, text_size, oat_writer->GetBssSize()); 261 262 writer->UpdateOatFileLayout(i, 263 elf_writer->GetLoadedSize(), 264 oat_writer->GetOatDataOffset(), 265 oat_writer->GetSize()); 266 267 bool rodata_ok = oat_writer->WriteRodata(rodata[i]); 268 ASSERT_TRUE(rodata_ok); 269 elf_writer->EndRoData(rodata[i]); 270 271 OutputStream* text = elf_writer->StartText(); 272 bool text_ok = oat_writer->WriteCode(text); 273 ASSERT_TRUE(text_ok); 274 elf_writer->EndText(text); 275 276 bool header_ok = oat_writer->WriteHeader(elf_writer->GetStream(), 0u, 0u, 0u); 277 ASSERT_TRUE(header_ok); 278 279 writer->UpdateOatFileHeader(i, oat_writer->GetOatHeader()); 280 281 elf_writer->WriteDynamicSection(); 282 elf_writer->WriteDebugInfo(oat_writer->GetMethodDebugInfo()); 283 elf_writer->WritePatchLocations(oat_writer->GetAbsolutePatchLocations()); 284 285 bool success = elf_writer->End(); 286 ASSERT_TRUE(success); 287 } 288 } 289 290 bool success_image = writer->Write(kInvalidFd, 291 image_filename_vector, 292 oat_filename_vector); 293 ASSERT_TRUE(success_image); 294 295 for (size_t i = 0, size = oat_filenames.size(); i != size; ++i) { 296 const char* oat_filename = oat_filenames[i].c_str(); 297 std::unique_ptr<File> oat_file(OS::OpenFileReadWrite(oat_filename)); 298 ASSERT_TRUE(oat_file != nullptr); 299 bool success_fixup = ElfWriter::Fixup(oat_file.get(), 300 writer->GetOatDataBegin(i)); 301 ASSERT_TRUE(success_fixup); 302 ASSERT_EQ(oat_file->FlushCloseOrErase(), 0) << "Could not flush and close oat file " 303 << oat_filename; 304 } 305 } 306} 307 308void ImageTest::Compile(ImageHeader::StorageMode storage_mode, 309 CompilationHelper& helper, 310 const std::string& extra_dex, 311 const std::string& image_class) { 312 if (!image_class.empty()) { 313 image_classes_.insert(image_class); 314 } 315 CreateCompilerDriver(Compiler::kOptimizing, kRuntimeISA, kIsTargetBuild ? 2U : 16U); 316 // Set inline filter values. 317 compiler_options_->SetInlineDepthLimit(CompilerOptions::kDefaultInlineDepthLimit); 318 compiler_options_->SetInlineMaxCodeUnits(CompilerOptions::kDefaultInlineMaxCodeUnits); 319 image_classes_.clear(); 320 if (!extra_dex.empty()) { 321 helper.extra_dex_files = OpenTestDexFiles(extra_dex.c_str()); 322 } 323 helper.Compile(compiler_driver_.get(), storage_mode); 324 if (!image_class.empty()) { 325 // Make sure the class got initialized. 326 ScopedObjectAccess soa(Thread::Current()); 327 ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); 328 mirror::Class* klass = class_linker->FindSystemClass(Thread::Current(), image_class.c_str()); 329 EXPECT_TRUE(klass != nullptr); 330 EXPECT_TRUE(klass->IsInitialized()); 331 } 332} 333 334void ImageTest::TestWriteRead(ImageHeader::StorageMode storage_mode) { 335 CompilationHelper helper; 336 Compile(storage_mode, /*out*/ helper); 337 std::vector<uint64_t> image_file_sizes; 338 for (ScratchFile& image_file : helper.image_files) { 339 std::unique_ptr<File> file(OS::OpenFileForReading(image_file.GetFilename().c_str())); 340 ASSERT_TRUE(file.get() != nullptr); 341 ImageHeader image_header; 342 ASSERT_EQ(file->ReadFully(&image_header, sizeof(image_header)), true); 343 ASSERT_TRUE(image_header.IsValid()); 344 const auto& bitmap_section = image_header.GetImageSection(ImageHeader::kSectionImageBitmap); 345 ASSERT_GE(bitmap_section.Offset(), sizeof(image_header)); 346 ASSERT_NE(0U, bitmap_section.Size()); 347 348 gc::Heap* heap = Runtime::Current()->GetHeap(); 349 ASSERT_TRUE(heap->HaveContinuousSpaces()); 350 gc::space::ContinuousSpace* space = heap->GetNonMovingSpace(); 351 ASSERT_FALSE(space->IsImageSpace()); 352 ASSERT_TRUE(space != nullptr); 353 ASSERT_TRUE(space->IsMallocSpace()); 354 355 image_file_sizes.push_back(file->GetLength()); 356 } 357 358 ASSERT_TRUE(compiler_driver_->GetImageClasses() != nullptr); 359 std::unordered_set<std::string> image_classes(*compiler_driver_->GetImageClasses()); 360 361 // Need to delete the compiler since it has worker threads which are attached to runtime. 362 compiler_driver_.reset(); 363 364 // Tear down old runtime before making a new one, clearing out misc state. 365 366 // Remove the reservation of the memory for use to load the image. 367 // Need to do this before we reset the runtime. 368 UnreserveImageSpace(); 369 370 helper.extra_dex_files.clear(); 371 runtime_.reset(); 372 java_lang_dex_file_ = nullptr; 373 374 MemMap::Init(); 375 376 RuntimeOptions options; 377 std::string image("-Ximage:"); 378 image.append(helper.image_locations[0].GetFilename()); 379 options.push_back(std::make_pair(image.c_str(), static_cast<void*>(nullptr))); 380 // By default the compiler this creates will not include patch information. 381 options.push_back(std::make_pair("-Xnorelocate", nullptr)); 382 383 if (!Runtime::Create(options, false)) { 384 LOG(FATAL) << "Failed to create runtime"; 385 return; 386 } 387 runtime_.reset(Runtime::Current()); 388 // Runtime::Create acquired the mutator_lock_ that is normally given away when we Runtime::Start, 389 // give it away now and then switch to a more managable ScopedObjectAccess. 390 Thread::Current()->TransitionFromRunnableToSuspended(kNative); 391 ScopedObjectAccess soa(Thread::Current()); 392 ASSERT_TRUE(runtime_.get() != nullptr); 393 class_linker_ = runtime_->GetClassLinker(); 394 395 gc::Heap* heap = Runtime::Current()->GetHeap(); 396 ASSERT_TRUE(heap->HasBootImageSpace()); 397 ASSERT_TRUE(heap->GetNonMovingSpace()->IsMallocSpace()); 398 399 // We loaded the runtime with an explicit image, so it must exist. 400 ASSERT_EQ(heap->GetBootImageSpaces().size(), image_file_sizes.size()); 401 for (size_t i = 0; i < helper.dex_file_locations.size(); ++i) { 402 std::unique_ptr<const DexFile> dex( 403 LoadExpectSingleDexFile(helper.dex_file_locations[i].c_str())); 404 ASSERT_TRUE(dex != nullptr); 405 uint64_t image_file_size = image_file_sizes[i]; 406 gc::space::ImageSpace* image_space = heap->GetBootImageSpaces()[i]; 407 ASSERT_TRUE(image_space != nullptr); 408 if (storage_mode == ImageHeader::kStorageModeUncompressed) { 409 // Uncompressed, image should be smaller than file. 410 ASSERT_LE(image_space->GetImageHeader().GetImageSize(), image_file_size); 411 } else if (image_file_size > 16 * KB) { 412 // Compressed, file should be smaller than image. Not really valid for small images. 413 ASSERT_LE(image_file_size, image_space->GetImageHeader().GetImageSize()); 414 } 415 416 image_space->VerifyImageAllocations(); 417 uint8_t* image_begin = image_space->Begin(); 418 uint8_t* image_end = image_space->End(); 419 if (i == 0) { 420 // This check is only valid for image 0. 421 CHECK_EQ(kRequestedImageBase, reinterpret_cast<uintptr_t>(image_begin)); 422 } 423 for (size_t j = 0; j < dex->NumClassDefs(); ++j) { 424 const DexFile::ClassDef& class_def = dex->GetClassDef(j); 425 const char* descriptor = dex->GetClassDescriptor(class_def); 426 mirror::Class* klass = class_linker_->FindSystemClass(soa.Self(), descriptor); 427 EXPECT_TRUE(klass != nullptr) << descriptor; 428 if (image_classes.find(descriptor) == image_classes.end()) { 429 EXPECT_TRUE(reinterpret_cast<uint8_t*>(klass) >= image_end || 430 reinterpret_cast<uint8_t*>(klass) < image_begin) << descriptor; 431 } else { 432 // Image classes should be located inside the image. 433 EXPECT_LT(image_begin, reinterpret_cast<uint8_t*>(klass)) << descriptor; 434 EXPECT_LT(reinterpret_cast<uint8_t*>(klass), image_end) << descriptor; 435 } 436 EXPECT_TRUE(Monitor::IsValidLockWord(klass->GetLockWord(false))); 437 } 438 } 439} 440 441TEST_F(ImageTest, WriteReadUncompressed) { 442 TestWriteRead(ImageHeader::kStorageModeUncompressed); 443} 444 445TEST_F(ImageTest, WriteReadLZ4) { 446 TestWriteRead(ImageHeader::kStorageModeLZ4); 447} 448 449TEST_F(ImageTest, WriteReadLZ4HC) { 450 TestWriteRead(ImageHeader::kStorageModeLZ4HC); 451} 452 453TEST_F(ImageTest, TestImageLayout) { 454 std::vector<size_t> image_sizes; 455 std::vector<size_t> image_sizes_extra; 456 // Compile multi-image with ImageLayoutA being the last image. 457 { 458 CompilationHelper helper; 459 Compile(ImageHeader::kStorageModeUncompressed, helper, "ImageLayoutA", "LMyClass;"); 460 image_sizes = helper.GetImageObjectSectionSizes(); 461 } 462 TearDown(); 463 runtime_.reset(); 464 SetUp(); 465 // Compile multi-image with ImageLayoutB being the last image. 466 { 467 CompilationHelper helper; 468 Compile(ImageHeader::kStorageModeUncompressed, helper, "ImageLayoutB", "LMyClass;"); 469 image_sizes_extra = helper.GetImageObjectSectionSizes(); 470 } 471 // Make sure that the new stuff in the clinit in ImageLayoutB is in the last image and not in the 472 // first two images. 473 ASSERT_EQ(image_sizes.size(), image_sizes.size()); 474 // Sizes of the images should be the same. These sizes are for the whole image unrounded. 475 for (size_t i = 0; i < image_sizes.size() - 1; ++i) { 476 EXPECT_EQ(image_sizes[i], image_sizes_extra[i]); 477 } 478 // Last image should be larger since it has a hash map and a string. 479 EXPECT_LT(image_sizes.back(), image_sizes_extra.back()); 480} 481 482TEST_F(ImageTest, ImageHeaderIsValid) { 483 uint32_t image_begin = ART_BASE_ADDRESS; 484 uint32_t image_size_ = 16 * KB; 485 uint32_t image_roots = ART_BASE_ADDRESS + (1 * KB); 486 uint32_t oat_checksum = 0; 487 uint32_t oat_file_begin = ART_BASE_ADDRESS + (4 * KB); // page aligned 488 uint32_t oat_data_begin = ART_BASE_ADDRESS + (8 * KB); // page aligned 489 uint32_t oat_data_end = ART_BASE_ADDRESS + (9 * KB); 490 uint32_t oat_file_end = ART_BASE_ADDRESS + (10 * KB); 491 ImageSection sections[ImageHeader::kSectionCount]; 492 ImageHeader image_header(image_begin, 493 image_size_, 494 sections, 495 image_roots, 496 oat_checksum, 497 oat_file_begin, 498 oat_data_begin, 499 oat_data_end, 500 oat_file_end, 501 /*boot_image_begin*/0U, 502 /*boot_image_size*/0U, 503 /*boot_oat_begin*/0U, 504 /*boot_oat_size_*/0U, 505 sizeof(void*), 506 /*compile_pic*/false, 507 /*is_pic*/false, 508 ImageHeader::kDefaultStorageMode, 509 /*data_size*/0u); 510 ASSERT_TRUE(image_header.IsValid()); 511 ASSERT_TRUE(!image_header.IsAppImage()); 512 513 char* magic = const_cast<char*>(image_header.GetMagic()); 514 strcpy(magic, ""); // bad magic 515 ASSERT_FALSE(image_header.IsValid()); 516 strcpy(magic, "art\n000"); // bad version 517 ASSERT_FALSE(image_header.IsValid()); 518} 519 520} // namespace art 521