1// Copyright 2017 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/wasm/wasm-code-specialization.h"
6
7#include "src/assembler-inl.h"
8#include "src/objects-inl.h"
9#include "src/source-position-table.h"
10#include "src/wasm/decoder.h"
11#include "src/wasm/wasm-module.h"
12#include "src/wasm/wasm-opcodes.h"
13
14using namespace v8::internal;
15using namespace v8::internal::wasm;
16
17namespace {
18
19int ExtractDirectCallIndex(wasm::Decoder& decoder, const byte* pc) {
20  DCHECK_EQ(static_cast<int>(kExprCallFunction), static_cast<int>(*pc));
21  decoder.Reset(pc + 1, pc + 6);
22  uint32_t call_idx = decoder.consume_u32v("call index");
23  DCHECK(decoder.ok());
24  DCHECK_GE(kMaxInt, call_idx);
25  return static_cast<int>(call_idx);
26}
27
28int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator,
29                                       size_t offset_l) {
30  DCHECK_GE(kMaxInt, offset_l);
31  int offset = static_cast<int>(offset_l);
32  DCHECK(!iterator.done());
33  int byte_pos;
34  do {
35    byte_pos = iterator.source_position().ScriptOffset();
36    iterator.Advance();
37  } while (!iterator.done() && iterator.code_offset() <= offset);
38  return byte_pos;
39}
40
41class PatchDirectCallsHelper {
42 public:
43  PatchDirectCallsHelper(WasmInstanceObject* instance, Code* code)
44      : source_pos_it(code->source_position_table()),
45        decoder(nullptr, nullptr) {
46    FixedArray* deopt_data = code->deoptimization_data();
47    DCHECK_EQ(2, deopt_data->length());
48    WasmCompiledModule* comp_mod = instance->compiled_module();
49    int func_index = Smi::cast(deopt_data->get(1))->value();
50    func_bytes = comp_mod->module_bytes()->GetChars() +
51                 comp_mod->module()->functions[func_index].code_start_offset;
52  }
53
54  SourcePositionTableIterator source_pos_it;
55  Decoder decoder;
56  const byte* func_bytes;
57};
58
59}  // namespace
60
61CodeSpecialization::CodeSpecialization(Isolate* isolate, Zone* zone)
62    : objects_to_relocate(isolate->heap(), ZoneAllocationPolicy(zone)) {}
63
64CodeSpecialization::~CodeSpecialization() {}
65
66void CodeSpecialization::RelocateMemoryReferences(Address old_start,
67                                                  uint32_t old_size,
68                                                  Address new_start,
69                                                  uint32_t new_size) {
70  DCHECK(old_mem_start == nullptr && old_mem_size == 0 &&
71         new_mem_start == nullptr && new_mem_size == 0);
72  DCHECK(old_start != new_start || old_size != new_size);
73  old_mem_start = old_start;
74  old_mem_size = old_size;
75  new_mem_start = new_start;
76  new_mem_size = new_size;
77}
78
79void CodeSpecialization::RelocateGlobals(Address old_start, Address new_start) {
80  DCHECK(old_globals_start == 0 && new_globals_start == 0);
81  DCHECK(old_start != 0 || new_start != 0);
82  old_globals_start = old_start;
83  new_globals_start = new_start;
84}
85
86void CodeSpecialization::PatchTableSize(uint32_t old_size, uint32_t new_size) {
87  DCHECK(old_function_table_size == 0 && new_function_table_size == 0);
88  DCHECK(old_size != 0 || new_size != 0);
89  old_function_table_size = old_size;
90  new_function_table_size = new_size;
91}
92
93void CodeSpecialization::RelocateDirectCalls(
94    Handle<WasmInstanceObject> instance) {
95  DCHECK(relocate_direct_calls_instance.is_null());
96  DCHECK(!instance.is_null());
97  relocate_direct_calls_instance = instance;
98}
99
100void CodeSpecialization::RelocateObject(Handle<Object> old_obj,
101                                        Handle<Object> new_obj) {
102  DCHECK(!old_obj.is_null() && !new_obj.is_null());
103  has_objects_to_relocate = true;
104  objects_to_relocate.Set(*old_obj, new_obj);
105}
106
107bool CodeSpecialization::ApplyToWholeInstance(
108    WasmInstanceObject* instance, ICacheFlushMode icache_flush_mode) {
109  DisallowHeapAllocation no_gc;
110  WasmCompiledModule* compiled_module = instance->compiled_module();
111  FixedArray* code_table = compiled_module->ptr_to_code_table();
112  WasmModule* module = compiled_module->module();
113  std::vector<WasmFunction>* wasm_functions =
114      &compiled_module->module()->functions;
115  DCHECK_EQ(wasm_functions->size() +
116                compiled_module->module()->num_exported_functions,
117            code_table->length());
118
119  bool changed = false;
120  int func_index = module->num_imported_functions;
121
122  // Patch all wasm functions.
123  for (int num_wasm_functions = static_cast<int>(wasm_functions->size());
124       func_index < num_wasm_functions; ++func_index) {
125    Code* wasm_function = Code::cast(code_table->get(func_index));
126    changed |= ApplyToWasmCode(wasm_function, icache_flush_mode);
127  }
128
129  // Patch all exported functions.
130  for (auto exp : module->export_table) {
131    if (exp.kind != kExternalFunction) continue;
132    Code* export_wrapper = Code::cast(code_table->get(func_index));
133    DCHECK_EQ(Code::JS_TO_WASM_FUNCTION, export_wrapper->kind());
134    // There must be exactly one call to WASM_FUNCTION or WASM_TO_JS_FUNCTION.
135    int num_wasm_calls = 0;
136    for (RelocIterator it(export_wrapper,
137                          RelocInfo::ModeMask(RelocInfo::CODE_TARGET));
138         !it.done(); it.next()) {
139      DCHECK(RelocInfo::IsCodeTarget(it.rinfo()->rmode()));
140      Code* code = Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
141      // Ignore calls to other builtins like ToNumber.
142      if (code->kind() != Code::WASM_FUNCTION &&
143          code->kind() != Code::WASM_TO_JS_FUNCTION &&
144          code->builtin_index() != Builtins::kIllegal)
145        continue;
146      ++num_wasm_calls;
147      Code* new_code = Code::cast(code_table->get(exp.index));
148      DCHECK(new_code->kind() == Code::WASM_FUNCTION ||
149             new_code->kind() == Code::WASM_TO_JS_FUNCTION);
150      it.rinfo()->set_target_address(new_code->instruction_start(),
151                                     UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH);
152      changed = true;
153    }
154    DCHECK_EQ(1, num_wasm_calls);
155    func_index++;
156  }
157  DCHECK_EQ(code_table->length(), func_index);
158  return changed;
159}
160
161bool CodeSpecialization::ApplyToWasmCode(Code* code,
162                                         ICacheFlushMode icache_flush_mode) {
163  DisallowHeapAllocation no_gc;
164  DCHECK_EQ(Code::WASM_FUNCTION, code->kind());
165
166  bool reloc_mem_addr = old_mem_start != new_mem_start;
167  bool reloc_mem_size = old_mem_size != new_mem_size;
168  bool reloc_globals = old_globals_start || new_globals_start;
169  bool patch_table_size = old_function_table_size || new_function_table_size;
170  bool reloc_direct_calls = !relocate_direct_calls_instance.is_null();
171  bool reloc_objects = has_objects_to_relocate;
172
173  int reloc_mode = 0;
174  auto add_mode = [&reloc_mode](bool cond, RelocInfo::Mode mode) {
175    if (cond) reloc_mode |= RelocInfo::ModeMask(mode);
176  };
177  add_mode(reloc_mem_addr, RelocInfo::WASM_MEMORY_REFERENCE);
178  add_mode(reloc_mem_size, RelocInfo::WASM_MEMORY_SIZE_REFERENCE);
179  add_mode(reloc_globals, RelocInfo::WASM_GLOBAL_REFERENCE);
180  add_mode(patch_table_size, RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE);
181  add_mode(reloc_direct_calls, RelocInfo::CODE_TARGET);
182  add_mode(reloc_objects, RelocInfo::EMBEDDED_OBJECT);
183
184  std::unique_ptr<PatchDirectCallsHelper> patch_direct_calls_helper;
185  bool changed = false;
186
187  for (RelocIterator it(code, reloc_mode); !it.done(); it.next()) {
188    RelocInfo::Mode mode = it.rinfo()->rmode();
189    switch (mode) {
190      case RelocInfo::WASM_MEMORY_REFERENCE:
191        DCHECK(reloc_mem_addr);
192        it.rinfo()->update_wasm_memory_reference(old_mem_start, new_mem_start,
193                                                 icache_flush_mode);
194        changed = true;
195        break;
196      case RelocInfo::WASM_MEMORY_SIZE_REFERENCE:
197        DCHECK(reloc_mem_size);
198        it.rinfo()->update_wasm_memory_size(old_mem_size, new_mem_size,
199                                            icache_flush_mode);
200        changed = true;
201        break;
202      case RelocInfo::WASM_GLOBAL_REFERENCE:
203        DCHECK(reloc_globals);
204        it.rinfo()->update_wasm_global_reference(
205            old_globals_start, new_globals_start, icache_flush_mode);
206        changed = true;
207        break;
208      case RelocInfo::CODE_TARGET: {
209        DCHECK(reloc_direct_calls);
210        Code* old_code =
211            Code::GetCodeFromTargetAddress(it.rinfo()->target_address());
212        // Skip everything which is not a wasm call (stack checks, traps, ...).
213        if (old_code->kind() != Code::WASM_FUNCTION &&
214            old_code->kind() != Code::WASM_TO_JS_FUNCTION &&
215            old_code->builtin_index() != Builtins::kIllegal)
216          continue;
217        // Iterate simultaneously over the relocation information and the source
218        // position table. For each call in the reloc info, move the source
219        // position iterator forward to that position to find the byte offset of
220        // the respective call. Then extract the call index from the module wire
221        // bytes to find the new compiled function.
222        size_t offset = it.rinfo()->pc() - code->instruction_start();
223        if (!patch_direct_calls_helper) {
224          patch_direct_calls_helper.reset(new PatchDirectCallsHelper(
225              *relocate_direct_calls_instance, code));
226        }
227        int byte_pos = AdvanceSourcePositionTableIterator(
228            patch_direct_calls_helper->source_pos_it, offset);
229        int called_func_index = ExtractDirectCallIndex(
230            patch_direct_calls_helper->decoder,
231            patch_direct_calls_helper->func_bytes + byte_pos);
232        FixedArray* code_table =
233            relocate_direct_calls_instance->compiled_module()
234                ->ptr_to_code_table();
235        Code* new_code = Code::cast(code_table->get(called_func_index));
236        it.rinfo()->set_target_address(new_code->instruction_start(),
237                                       UPDATE_WRITE_BARRIER, icache_flush_mode);
238        changed = true;
239      } break;
240      case RelocInfo::EMBEDDED_OBJECT: {
241        DCHECK(reloc_objects);
242        Object* old = it.rinfo()->target_object();
243        Handle<Object>* new_obj = objects_to_relocate.Find(old);
244        if (new_obj) {
245          it.rinfo()->set_target_object(**new_obj, UPDATE_WRITE_BARRIER,
246                                        icache_flush_mode);
247          changed = true;
248        }
249      } break;
250      case RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE:
251        DCHECK(patch_table_size);
252        it.rinfo()->update_wasm_function_table_size_reference(
253            old_function_table_size, new_function_table_size,
254            icache_flush_mode);
255        changed = true;
256        break;
257      default:
258        UNREACHABLE();
259    }
260  }
261
262  return changed;
263}
264