1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "image.h"
18
19#include <memory>
20#include <string>
21#include <vector>
22
23#include "base/unix_file/fd_file.h"
24#include "class_linker-inl.h"
25#include "common_compiler_test.h"
26#include "debug/method_debug_info.h"
27#include "driver/compiler_options.h"
28#include "elf_writer.h"
29#include "elf_writer_quick.h"
30#include "gc/space/image_space.h"
31#include "image_writer.h"
32#include "linker/multi_oat_relative_patcher.h"
33#include "lock_word.h"
34#include "mirror/object-inl.h"
35#include "oat_writer.h"
36#include "scoped_thread_state_change.h"
37#include "signal_catcher.h"
38#include "utils.h"
39
40namespace art {
41
42static const uintptr_t kRequestedImageBase = ART_BASE_ADDRESS;
43
44struct CompilationHelper {
45  std::vector<std::string> dex_file_locations;
46  std::vector<ScratchFile> image_locations;
47  std::vector<std::unique_ptr<const DexFile>> extra_dex_files;
48  std::vector<ScratchFile> image_files;
49  std::vector<ScratchFile> oat_files;
50  std::string image_dir;
51
52  void Compile(CompilerDriver* driver,
53               ImageHeader::StorageMode storage_mode);
54
55  std::vector<size_t> GetImageObjectSectionSizes();
56
57  ~CompilationHelper();
58};
59
60class ImageTest : public CommonCompilerTest {
61 protected:
62  virtual void SetUp() {
63    ReserveImageSpace();
64    CommonCompilerTest::SetUp();
65  }
66
67  void TestWriteRead(ImageHeader::StorageMode storage_mode);
68
69  void Compile(ImageHeader::StorageMode storage_mode,
70               CompilationHelper& out_helper,
71               const std::string& extra_dex = "",
72               const std::string& image_class = "");
73
74  std::unordered_set<std::string>* GetImageClasses() OVERRIDE {
75    return new std::unordered_set<std::string>(image_classes_);
76  }
77
78 private:
79  std::unordered_set<std::string> image_classes_;
80};
81
82CompilationHelper::~CompilationHelper() {
83  for (ScratchFile& image_file : image_files) {
84    image_file.Unlink();
85  }
86  for (ScratchFile& oat_file : oat_files) {
87    oat_file.Unlink();
88  }
89  const int rmdir_result = rmdir(image_dir.c_str());
90  CHECK_EQ(0, rmdir_result);
91}
92
93std::vector<size_t> CompilationHelper::GetImageObjectSectionSizes() {
94  std::vector<size_t> ret;
95  for (ScratchFile& image_file : image_files) {
96    std::unique_ptr<File> file(OS::OpenFileForReading(image_file.GetFilename().c_str()));
97    CHECK(file.get() != nullptr);
98    ImageHeader image_header;
99    CHECK_EQ(file->ReadFully(&image_header, sizeof(image_header)), true);
100    CHECK(image_header.IsValid());
101    ret.push_back(image_header.GetImageSize());
102  }
103  return ret;
104}
105
106void CompilationHelper::Compile(CompilerDriver* driver,
107                                ImageHeader::StorageMode storage_mode) {
108  ClassLinker* class_linker = Runtime::Current()->GetClassLinker();
109  std::vector<const DexFile*> class_path = class_linker->GetBootClassPath();
110
111  for (const std::unique_ptr<const DexFile>& dex_file : extra_dex_files) {
112    {
113      ScopedObjectAccess soa(Thread::Current());
114      // Inject in boot class path so that the compiler driver can see it.
115      class_linker->AppendToBootClassPath(soa.Self(), *dex_file.get());
116    }
117    class_path.push_back(dex_file.get());
118  }
119
120  // Enable write for dex2dex.
121  for (const DexFile* dex_file : class_path) {
122    dex_file_locations.push_back(dex_file->GetLocation());
123    if (dex_file->IsReadOnly()) {
124      dex_file->EnableWrite();
125    }
126  }
127
128  {
129    // Create a generic tmp file, to be the base of the .art and .oat temporary files.
130    ScratchFile location;
131    for (int i = 0; i < static_cast<int>(class_path.size()); ++i) {
132      std::string cur_location(StringPrintf("%s-%d.art", location.GetFilename().c_str(), i));
133      image_locations.push_back(ScratchFile(cur_location));
134    }
135  }
136  std::vector<std::string> image_filenames;
137  for (ScratchFile& file : image_locations) {
138    std::string image_filename(GetSystemImageFilename(file.GetFilename().c_str(), kRuntimeISA));
139    image_filenames.push_back(image_filename);
140    size_t pos = image_filename.rfind('/');
141    CHECK_NE(pos, std::string::npos) << image_filename;
142    if (image_dir.empty()) {
143      image_dir = image_filename.substr(0, pos);
144      int mkdir_result = mkdir(image_dir.c_str(), 0700);
145      CHECK_EQ(0, mkdir_result) << image_dir;
146    }
147    image_files.push_back(ScratchFile(OS::CreateEmptyFile(image_filename.c_str())));
148  }
149
150  std::vector<std::string> oat_filenames;
151  for (const std::string& image_filename : image_filenames) {
152    std::string oat_filename(image_filename.substr(0, image_filename.size() - strlen("art")) + "oat");
153    oat_files.push_back(ScratchFile(OS::CreateEmptyFile(oat_filename.c_str())));
154    oat_filenames.push_back(oat_filename);
155  }
156
157  std::unordered_map<const DexFile*, size_t> dex_file_to_oat_index_map;
158  std::vector<const char*> oat_filename_vector;
159  for (const std::string& file : oat_filenames) {
160    oat_filename_vector.push_back(file.c_str());
161  }
162  std::vector<const char*> image_filename_vector;
163  for (const std::string& file : image_filenames) {
164    image_filename_vector.push_back(file.c_str());
165  }
166  size_t image_idx = 0;
167  for (const DexFile* dex_file : class_path) {
168    dex_file_to_oat_index_map.emplace(dex_file, image_idx);
169    ++image_idx;
170  }
171  // TODO: compile_pic should be a test argument.
172  std::unique_ptr<ImageWriter> writer(new ImageWriter(*driver,
173                                                      kRequestedImageBase,
174                                                      /*compile_pic*/false,
175                                                      /*compile_app_image*/false,
176                                                      storage_mode,
177                                                      oat_filename_vector,
178                                                      dex_file_to_oat_index_map));
179  {
180    {
181      jobject class_loader = nullptr;
182      TimingLogger timings("ImageTest::WriteRead", false, false);
183      TimingLogger::ScopedTiming t("CompileAll", &timings);
184      driver->SetDexFilesForOatFile(class_path);
185      driver->CompileAll(class_loader, class_path, &timings);
186
187      t.NewTiming("WriteElf");
188      SafeMap<std::string, std::string> key_value_store;
189      std::vector<const char*> dex_filename_vector;
190      for (size_t i = 0; i < class_path.size(); ++i) {
191        dex_filename_vector.push_back("");
192      }
193      key_value_store.Put(OatHeader::kBootClassPathKey,
194                          gc::space::ImageSpace::GetMultiImageBootClassPath(
195                              dex_filename_vector,
196                              oat_filename_vector,
197                              image_filename_vector));
198
199      std::vector<std::unique_ptr<ElfWriter>> elf_writers;
200      std::vector<std::unique_ptr<OatWriter>> oat_writers;
201      for (ScratchFile& oat_file : oat_files) {
202        elf_writers.emplace_back(CreateElfWriterQuick(driver->GetInstructionSet(),
203                                                      driver->GetInstructionSetFeatures(),
204                                                      &driver->GetCompilerOptions(),
205                                                      oat_file.GetFile()));
206        elf_writers.back()->Start();
207        oat_writers.emplace_back(new OatWriter(/*compiling_boot_image*/true, &timings));
208      }
209
210      std::vector<OutputStream*> rodata;
211      std::vector<std::unique_ptr<MemMap>> opened_dex_files_map;
212      std::vector<std::unique_ptr<const DexFile>> opened_dex_files;
213      // Now that we have finalized key_value_store_, start writing the oat file.
214      for (size_t i = 0, size = oat_writers.size(); i != size; ++i) {
215        const DexFile* dex_file = class_path[i];
216        rodata.push_back(elf_writers[i]->StartRoData());
217        ArrayRef<const uint8_t> raw_dex_file(
218            reinterpret_cast<const uint8_t*>(&dex_file->GetHeader()),
219            dex_file->GetHeader().file_size_);
220        oat_writers[i]->AddRawDexFileSource(raw_dex_file,
221                                            dex_file->GetLocation().c_str(),
222                                            dex_file->GetLocationChecksum());
223
224        std::unique_ptr<MemMap> cur_opened_dex_files_map;
225        std::vector<std::unique_ptr<const DexFile>> cur_opened_dex_files;
226        bool dex_files_ok = oat_writers[i]->WriteAndOpenDexFiles(
227            rodata.back(),
228            oat_files[i].GetFile(),
229            driver->GetInstructionSet(),
230            driver->GetInstructionSetFeatures(),
231            &key_value_store,
232            /* verify */ false,           // Dex files may be dex-to-dex-ed, don't verify.
233            &cur_opened_dex_files_map,
234            &cur_opened_dex_files);
235        ASSERT_TRUE(dex_files_ok);
236
237        if (cur_opened_dex_files_map != nullptr) {
238          opened_dex_files_map.push_back(std::move(cur_opened_dex_files_map));
239          for (std::unique_ptr<const DexFile>& cur_dex_file : cur_opened_dex_files) {
240            // dex_file_oat_index_map_.emplace(dex_file.get(), i);
241            opened_dex_files.push_back(std::move(cur_dex_file));
242          }
243        } else {
244          ASSERT_TRUE(cur_opened_dex_files.empty());
245        }
246      }
247
248      bool image_space_ok = writer->PrepareImageAddressSpace();
249      ASSERT_TRUE(image_space_ok);
250
251      for (size_t i = 0, size = oat_files.size(); i != size; ++i) {
252        linker::MultiOatRelativePatcher patcher(driver->GetInstructionSet(),
253                                                driver->GetInstructionSetFeatures());
254        OatWriter* const oat_writer = oat_writers[i].get();
255        ElfWriter* const elf_writer = elf_writers[i].get();
256        std::vector<const DexFile*> cur_dex_files(1u, class_path[i]);
257        oat_writer->PrepareLayout(driver, writer.get(), cur_dex_files, &patcher);
258        size_t rodata_size = oat_writer->GetOatHeader().GetExecutableOffset();
259        size_t text_size = oat_writer->GetSize() - rodata_size;
260        elf_writer->SetLoadedSectionSizes(rodata_size, text_size, oat_writer->GetBssSize());
261
262        writer->UpdateOatFileLayout(i,
263                                    elf_writer->GetLoadedSize(),
264                                    oat_writer->GetOatDataOffset(),
265                                    oat_writer->GetSize());
266
267        bool rodata_ok = oat_writer->WriteRodata(rodata[i]);
268        ASSERT_TRUE(rodata_ok);
269        elf_writer->EndRoData(rodata[i]);
270
271        OutputStream* text = elf_writer->StartText();
272        bool text_ok = oat_writer->WriteCode(text);
273        ASSERT_TRUE(text_ok);
274        elf_writer->EndText(text);
275
276        bool header_ok = oat_writer->WriteHeader(elf_writer->GetStream(), 0u, 0u, 0u);
277        ASSERT_TRUE(header_ok);
278
279        writer->UpdateOatFileHeader(i, oat_writer->GetOatHeader());
280
281        elf_writer->WriteDynamicSection();
282        elf_writer->WriteDebugInfo(oat_writer->GetMethodDebugInfo());
283        elf_writer->WritePatchLocations(oat_writer->GetAbsolutePatchLocations());
284
285        bool success = elf_writer->End();
286        ASSERT_TRUE(success);
287      }
288    }
289
290    bool success_image = writer->Write(kInvalidFd,
291                                       image_filename_vector,
292                                       oat_filename_vector);
293    ASSERT_TRUE(success_image);
294
295    for (size_t i = 0, size = oat_filenames.size(); i != size; ++i) {
296      const char* oat_filename = oat_filenames[i].c_str();
297      std::unique_ptr<File> oat_file(OS::OpenFileReadWrite(oat_filename));
298      ASSERT_TRUE(oat_file != nullptr);
299      bool success_fixup = ElfWriter::Fixup(oat_file.get(),
300                                            writer->GetOatDataBegin(i));
301      ASSERT_TRUE(success_fixup);
302      ASSERT_EQ(oat_file->FlushCloseOrErase(), 0) << "Could not flush and close oat file "
303                                                  << oat_filename;
304    }
305  }
306}
307
308void ImageTest::Compile(ImageHeader::StorageMode storage_mode,
309                        CompilationHelper& helper,
310                        const std::string& extra_dex,
311                        const std::string& image_class) {
312  if (!image_class.empty()) {
313    image_classes_.insert(image_class);
314  }
315  CreateCompilerDriver(Compiler::kOptimizing, kRuntimeISA, kIsTargetBuild ? 2U : 16U);
316  // Set inline filter values.
317  compiler_options_->SetInlineDepthLimit(CompilerOptions::kDefaultInlineDepthLimit);
318  compiler_options_->SetInlineMaxCodeUnits(CompilerOptions::kDefaultInlineMaxCodeUnits);
319  image_classes_.clear();
320  if (!extra_dex.empty()) {
321    helper.extra_dex_files = OpenTestDexFiles(extra_dex.c_str());
322  }
323  helper.Compile(compiler_driver_.get(), storage_mode);
324  if (!image_class.empty()) {
325    // Make sure the class got initialized.
326    ScopedObjectAccess soa(Thread::Current());
327    ClassLinker* const class_linker = Runtime::Current()->GetClassLinker();
328    mirror::Class* klass = class_linker->FindSystemClass(Thread::Current(), image_class.c_str());
329    EXPECT_TRUE(klass != nullptr);
330    EXPECT_TRUE(klass->IsInitialized());
331  }
332}
333
334void ImageTest::TestWriteRead(ImageHeader::StorageMode storage_mode) {
335  CompilationHelper helper;
336  Compile(storage_mode, /*out*/ helper);
337  std::vector<uint64_t> image_file_sizes;
338  for (ScratchFile& image_file : helper.image_files) {
339    std::unique_ptr<File> file(OS::OpenFileForReading(image_file.GetFilename().c_str()));
340    ASSERT_TRUE(file.get() != nullptr);
341    ImageHeader image_header;
342    ASSERT_EQ(file->ReadFully(&image_header, sizeof(image_header)), true);
343    ASSERT_TRUE(image_header.IsValid());
344    const auto& bitmap_section = image_header.GetImageSection(ImageHeader::kSectionImageBitmap);
345    ASSERT_GE(bitmap_section.Offset(), sizeof(image_header));
346    ASSERT_NE(0U, bitmap_section.Size());
347
348    gc::Heap* heap = Runtime::Current()->GetHeap();
349    ASSERT_TRUE(heap->HaveContinuousSpaces());
350    gc::space::ContinuousSpace* space = heap->GetNonMovingSpace();
351    ASSERT_FALSE(space->IsImageSpace());
352    ASSERT_TRUE(space != nullptr);
353    ASSERT_TRUE(space->IsMallocSpace());
354
355    image_file_sizes.push_back(file->GetLength());
356  }
357
358  ASSERT_TRUE(compiler_driver_->GetImageClasses() != nullptr);
359  std::unordered_set<std::string> image_classes(*compiler_driver_->GetImageClasses());
360
361  // Need to delete the compiler since it has worker threads which are attached to runtime.
362  compiler_driver_.reset();
363
364  // Tear down old runtime before making a new one, clearing out misc state.
365
366  // Remove the reservation of the memory for use to load the image.
367  // Need to do this before we reset the runtime.
368  UnreserveImageSpace();
369
370  helper.extra_dex_files.clear();
371  runtime_.reset();
372  java_lang_dex_file_ = nullptr;
373
374  MemMap::Init();
375
376  RuntimeOptions options;
377  std::string image("-Ximage:");
378  image.append(helper.image_locations[0].GetFilename());
379  options.push_back(std::make_pair(image.c_str(), static_cast<void*>(nullptr)));
380  // By default the compiler this creates will not include patch information.
381  options.push_back(std::make_pair("-Xnorelocate", nullptr));
382
383  if (!Runtime::Create(options, false)) {
384    LOG(FATAL) << "Failed to create runtime";
385    return;
386  }
387  runtime_.reset(Runtime::Current());
388  // Runtime::Create acquired the mutator_lock_ that is normally given away when we Runtime::Start,
389  // give it away now and then switch to a more managable ScopedObjectAccess.
390  Thread::Current()->TransitionFromRunnableToSuspended(kNative);
391  ScopedObjectAccess soa(Thread::Current());
392  ASSERT_TRUE(runtime_.get() != nullptr);
393  class_linker_ = runtime_->GetClassLinker();
394
395  gc::Heap* heap = Runtime::Current()->GetHeap();
396  ASSERT_TRUE(heap->HasBootImageSpace());
397  ASSERT_TRUE(heap->GetNonMovingSpace()->IsMallocSpace());
398
399  // We loaded the runtime with an explicit image, so it must exist.
400  ASSERT_EQ(heap->GetBootImageSpaces().size(), image_file_sizes.size());
401  for (size_t i = 0; i < helper.dex_file_locations.size(); ++i) {
402    std::unique_ptr<const DexFile> dex(
403        LoadExpectSingleDexFile(helper.dex_file_locations[i].c_str()));
404    ASSERT_TRUE(dex != nullptr);
405    uint64_t image_file_size = image_file_sizes[i];
406    gc::space::ImageSpace* image_space = heap->GetBootImageSpaces()[i];
407    ASSERT_TRUE(image_space != nullptr);
408    if (storage_mode == ImageHeader::kStorageModeUncompressed) {
409      // Uncompressed, image should be smaller than file.
410      ASSERT_LE(image_space->GetImageHeader().GetImageSize(), image_file_size);
411    } else if (image_file_size > 16 * KB) {
412      // Compressed, file should be smaller than image. Not really valid for small images.
413      ASSERT_LE(image_file_size, image_space->GetImageHeader().GetImageSize());
414    }
415
416    image_space->VerifyImageAllocations();
417    uint8_t* image_begin = image_space->Begin();
418    uint8_t* image_end = image_space->End();
419    if (i == 0) {
420      // This check is only valid for image 0.
421      CHECK_EQ(kRequestedImageBase, reinterpret_cast<uintptr_t>(image_begin));
422    }
423    for (size_t j = 0; j < dex->NumClassDefs(); ++j) {
424      const DexFile::ClassDef& class_def = dex->GetClassDef(j);
425      const char* descriptor = dex->GetClassDescriptor(class_def);
426      mirror::Class* klass = class_linker_->FindSystemClass(soa.Self(), descriptor);
427      EXPECT_TRUE(klass != nullptr) << descriptor;
428      if (image_classes.find(descriptor) == image_classes.end()) {
429        EXPECT_TRUE(reinterpret_cast<uint8_t*>(klass) >= image_end ||
430                    reinterpret_cast<uint8_t*>(klass) < image_begin) << descriptor;
431      } else {
432        // Image classes should be located inside the image.
433        EXPECT_LT(image_begin, reinterpret_cast<uint8_t*>(klass)) << descriptor;
434        EXPECT_LT(reinterpret_cast<uint8_t*>(klass), image_end) << descriptor;
435      }
436      EXPECT_TRUE(Monitor::IsValidLockWord(klass->GetLockWord(false)));
437    }
438  }
439}
440
441TEST_F(ImageTest, WriteReadUncompressed) {
442  TestWriteRead(ImageHeader::kStorageModeUncompressed);
443}
444
445TEST_F(ImageTest, WriteReadLZ4) {
446  TestWriteRead(ImageHeader::kStorageModeLZ4);
447}
448
449TEST_F(ImageTest, WriteReadLZ4HC) {
450  TestWriteRead(ImageHeader::kStorageModeLZ4HC);
451}
452
453TEST_F(ImageTest, TestImageLayout) {
454  std::vector<size_t> image_sizes;
455  std::vector<size_t> image_sizes_extra;
456  // Compile multi-image with ImageLayoutA being the last image.
457  {
458    CompilationHelper helper;
459    Compile(ImageHeader::kStorageModeUncompressed, helper, "ImageLayoutA", "LMyClass;");
460    image_sizes = helper.GetImageObjectSectionSizes();
461  }
462  TearDown();
463  runtime_.reset();
464  SetUp();
465  // Compile multi-image with ImageLayoutB being the last image.
466  {
467    CompilationHelper helper;
468    Compile(ImageHeader::kStorageModeUncompressed, helper, "ImageLayoutB", "LMyClass;");
469    image_sizes_extra = helper.GetImageObjectSectionSizes();
470  }
471  // Make sure that the new stuff in the clinit in ImageLayoutB is in the last image and not in the
472  // first two images.
473  ASSERT_EQ(image_sizes.size(), image_sizes.size());
474  // Sizes of the images should be the same. These sizes are for the whole image unrounded.
475  for (size_t i = 0; i < image_sizes.size() - 1; ++i) {
476    EXPECT_EQ(image_sizes[i], image_sizes_extra[i]);
477  }
478  // Last image should be larger since it has a hash map and a string.
479  EXPECT_LT(image_sizes.back(), image_sizes_extra.back());
480}
481
482TEST_F(ImageTest, ImageHeaderIsValid) {
483    uint32_t image_begin = ART_BASE_ADDRESS;
484    uint32_t image_size_ = 16 * KB;
485    uint32_t image_roots = ART_BASE_ADDRESS + (1 * KB);
486    uint32_t oat_checksum = 0;
487    uint32_t oat_file_begin = ART_BASE_ADDRESS + (4 * KB);  // page aligned
488    uint32_t oat_data_begin = ART_BASE_ADDRESS + (8 * KB);  // page aligned
489    uint32_t oat_data_end = ART_BASE_ADDRESS + (9 * KB);
490    uint32_t oat_file_end = ART_BASE_ADDRESS + (10 * KB);
491    ImageSection sections[ImageHeader::kSectionCount];
492    ImageHeader image_header(image_begin,
493                             image_size_,
494                             sections,
495                             image_roots,
496                             oat_checksum,
497                             oat_file_begin,
498                             oat_data_begin,
499                             oat_data_end,
500                             oat_file_end,
501                             /*boot_image_begin*/0U,
502                             /*boot_image_size*/0U,
503                             /*boot_oat_begin*/0U,
504                             /*boot_oat_size_*/0U,
505                             sizeof(void*),
506                             /*compile_pic*/false,
507                             /*is_pic*/false,
508                             ImageHeader::kDefaultStorageMode,
509                             /*data_size*/0u);
510    ASSERT_TRUE(image_header.IsValid());
511    ASSERT_TRUE(!image_header.IsAppImage());
512
513    char* magic = const_cast<char*>(image_header.GetMagic());
514    strcpy(magic, "");  // bad magic
515    ASSERT_FALSE(image_header.IsValid());
516    strcpy(magic, "art\n000");  // bad version
517    ASSERT_FALSE(image_header.IsValid());
518}
519
520}  // namespace art
521