1// Copyright 2016 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "src/compiler/simd-scalar-lowering.h"
6#include "src/compiler/diamond.h"
7#include "src/compiler/linkage.h"
8#include "src/compiler/node-matchers.h"
9#include "src/compiler/node-properties.h"
10
11#include "src/compiler/node.h"
12#include "src/objects-inl.h"
13#include "src/wasm/wasm-module.h"
14
15namespace v8 {
16namespace internal {
17namespace compiler {
18
19SimdScalarLowering::SimdScalarLowering(
20    Graph* graph, MachineOperatorBuilder* machine,
21    CommonOperatorBuilder* common, Zone* zone,
22    Signature<MachineRepresentation>* signature)
23    : zone_(zone),
24      graph_(graph),
25      machine_(machine),
26      common_(common),
27      state_(graph, 3),
28      stack_(zone),
29      replacements_(nullptr),
30      signature_(signature),
31      placeholder_(
32          graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())),
33      parameter_count_after_lowering_(-1) {
34  DCHECK_NOT_NULL(graph);
35  DCHECK_NOT_NULL(graph->end());
36  replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
37  memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
38}
39
40void SimdScalarLowering::LowerGraph() {
41  stack_.push_back({graph()->end(), 0});
42  state_.Set(graph()->end(), State::kOnStack);
43  replacements_[graph()->end()->id()].type = SimdType::kInt32;
44
45  while (!stack_.empty()) {
46    NodeState& top = stack_.back();
47    if (top.input_index == top.node->InputCount()) {
48      // All inputs of top have already been lowered, now lower top.
49      stack_.pop_back();
50      state_.Set(top.node, State::kVisited);
51      LowerNode(top.node);
52    } else {
53      // Push the next input onto the stack.
54      Node* input = top.node->InputAt(top.input_index++);
55      if (state_.Get(input) == State::kUnvisited) {
56        SetLoweredType(input, top.node);
57        if (input->opcode() == IrOpcode::kPhi) {
58          // To break cycles with phi nodes we push phis on a separate stack so
59          // that they are processed after all other nodes.
60          PreparePhiReplacement(input);
61          stack_.push_front({input, 0});
62        } else if (input->opcode() == IrOpcode::kEffectPhi ||
63                   input->opcode() == IrOpcode::kLoop) {
64          stack_.push_front({input, 0});
65        } else {
66          stack_.push_back({input, 0});
67        }
68        state_.Set(input, State::kOnStack);
69      }
70    }
71  }
72}
73
74#define FOREACH_INT32X4_OPCODE(V) \
75  V(Int32x4Add)                   \
76  V(Int32x4ExtractLane)           \
77  V(CreateInt32x4)                \
78  V(Int32x4ReplaceLane)
79
80#define FOREACH_FLOAT32X4_OPCODE(V) \
81  V(Float32x4Add)                   \
82  V(Float32x4ExtractLane)           \
83  V(CreateFloat32x4)                \
84  V(Float32x4ReplaceLane)
85
86void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
87  switch (node->opcode()) {
88#define CASE_STMT(name) case IrOpcode::k##name:
89    FOREACH_INT32X4_OPCODE(CASE_STMT)
90    case IrOpcode::kReturn:
91    case IrOpcode::kParameter:
92    case IrOpcode::kCall: {
93      replacements_[node->id()].type = SimdType::kInt32;
94      break;
95    }
96      FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
97        replacements_[node->id()].type = SimdType::kFloat32;
98        break;
99      }
100#undef CASE_STMT
101    default:
102      replacements_[node->id()].type = replacements_[output->id()].type;
103  }
104}
105
106static int GetParameterIndexAfterLowering(
107    Signature<MachineRepresentation>* signature, int old_index) {
108  // In function calls, the simd128 types are passed as 4 Int32 types. The
109  // parameters are typecast to the types as needed for various operations.
110  int result = old_index;
111  for (int i = 0; i < old_index; ++i) {
112    if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
113      result += 3;
114    }
115  }
116  return result;
117}
118
119int SimdScalarLowering::GetParameterCountAfterLowering() {
120  if (parameter_count_after_lowering_ == -1) {
121    // GetParameterIndexAfterLowering(parameter_count) returns the parameter
122    // count after lowering.
123    parameter_count_after_lowering_ = GetParameterIndexAfterLowering(
124        signature(), static_cast<int>(signature()->parameter_count()));
125  }
126  return parameter_count_after_lowering_;
127}
128
129static int GetReturnCountAfterLowering(
130    Signature<MachineRepresentation>* signature) {
131  int result = static_cast<int>(signature->return_count());
132  for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
133    if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
134      result += 3;
135    }
136  }
137  return result;
138}
139
140void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices) {
141  new_indices[0] = index;
142  for (size_t i = 1; i < kMaxLanes; ++i) {
143    new_indices[i] = graph()->NewNode(machine()->Int32Add(), index,
144                                      graph()->NewNode(common()->Int32Constant(
145                                          static_cast<int>(i) * kLaneWidth)));
146  }
147}
148
149void SimdScalarLowering::LowerLoadOp(MachineRepresentation rep, Node* node,
150                                     const Operator* load_op) {
151  if (rep == MachineRepresentation::kSimd128) {
152    Node* base = node->InputAt(0);
153    Node* index = node->InputAt(1);
154    Node* indices[kMaxLanes];
155    GetIndexNodes(index, indices);
156    Node* rep_nodes[kMaxLanes];
157    rep_nodes[0] = node;
158    NodeProperties::ChangeOp(rep_nodes[0], load_op);
159    if (node->InputCount() > 2) {
160      DCHECK(node->InputCount() > 3);
161      Node* effect_input = node->InputAt(2);
162      Node* control_input = node->InputAt(3);
163      rep_nodes[3] = graph()->NewNode(load_op, base, indices[3], effect_input,
164                                      control_input);
165      rep_nodes[2] = graph()->NewNode(load_op, base, indices[2], rep_nodes[3],
166                                      control_input);
167      rep_nodes[1] = graph()->NewNode(load_op, base, indices[1], rep_nodes[2],
168                                      control_input);
169      rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
170    } else {
171      for (size_t i = 1; i < kMaxLanes; ++i) {
172        rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
173      }
174    }
175    ReplaceNode(node, rep_nodes);
176  } else {
177    DefaultLowering(node);
178  }
179}
180
181void SimdScalarLowering::LowerStoreOp(MachineRepresentation rep, Node* node,
182                                      const Operator* store_op,
183                                      SimdType rep_type) {
184  if (rep == MachineRepresentation::kSimd128) {
185    Node* base = node->InputAt(0);
186    Node* index = node->InputAt(1);
187    Node* indices[kMaxLanes];
188    GetIndexNodes(index, indices);
189    DCHECK(node->InputCount() > 2);
190    Node* value = node->InputAt(2);
191    DCHECK(HasReplacement(1, value));
192    Node* rep_nodes[kMaxLanes];
193    rep_nodes[0] = node;
194    Node** rep_inputs = GetReplacementsWithType(value, rep_type);
195    rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
196    NodeProperties::ChangeOp(node, store_op);
197    if (node->InputCount() > 3) {
198      DCHECK(node->InputCount() > 4);
199      Node* effect_input = node->InputAt(3);
200      Node* control_input = node->InputAt(4);
201      rep_nodes[3] = graph()->NewNode(store_op, base, indices[3], rep_inputs[3],
202                                      effect_input, control_input);
203      rep_nodes[2] = graph()->NewNode(store_op, base, indices[2], rep_inputs[2],
204                                      rep_nodes[3], control_input);
205      rep_nodes[1] = graph()->NewNode(store_op, base, indices[1], rep_inputs[1],
206                                      rep_nodes[2], control_input);
207      rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
208
209    } else {
210      for (size_t i = 1; i < kMaxLanes; ++i) {
211        rep_nodes[i] =
212            graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
213      }
214    }
215
216    ReplaceNode(node, rep_nodes);
217  } else {
218    DefaultLowering(node);
219  }
220}
221
222void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType rep_type,
223                                       const Operator* op) {
224  DCHECK(node->InputCount() == 2);
225  Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
226  Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
227  Node* rep_node[kMaxLanes];
228  for (int i = 0; i < kMaxLanes; ++i) {
229    rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
230  }
231  ReplaceNode(node, rep_node);
232}
233
234void SimdScalarLowering::LowerNode(Node* node) {
235  SimdType rep_type = ReplacementType(node);
236  switch (node->opcode()) {
237    case IrOpcode::kStart: {
238      int parameter_count = GetParameterCountAfterLowering();
239      // Only exchange the node if the parameter count actually changed.
240      if (parameter_count != static_cast<int>(signature()->parameter_count())) {
241        int delta =
242            parameter_count - static_cast<int>(signature()->parameter_count());
243        int new_output_count = node->op()->ValueOutputCount() + delta;
244        NodeProperties::ChangeOp(node, common()->Start(new_output_count));
245      }
246      break;
247    }
248    case IrOpcode::kParameter: {
249      DCHECK(node->InputCount() == 1);
250      // Only exchange the node if the parameter count actually changed. We do
251      // not even have to do the default lowering because the the start node,
252      // the only input of a parameter node, only changes if the parameter count
253      // changes.
254      if (GetParameterCountAfterLowering() !=
255          static_cast<int>(signature()->parameter_count())) {
256        int old_index = ParameterIndexOf(node->op());
257        int new_index = GetParameterIndexAfterLowering(signature(), old_index);
258        if (old_index == new_index) {
259          NodeProperties::ChangeOp(node, common()->Parameter(new_index));
260
261          Node* new_node[kMaxLanes];
262          for (int i = 0; i < kMaxLanes; ++i) {
263            new_node[i] = nullptr;
264          }
265          new_node[0] = node;
266          if (signature()->GetParam(old_index) ==
267              MachineRepresentation::kSimd128) {
268            for (int i = 1; i < kMaxLanes; ++i) {
269              new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
270                                             graph()->start());
271            }
272          }
273          ReplaceNode(node, new_node);
274        }
275      }
276      break;
277    }
278    case IrOpcode::kLoad: {
279      MachineRepresentation rep =
280          LoadRepresentationOf(node->op()).representation();
281      const Operator* load_op;
282      if (rep_type == SimdType::kInt32) {
283        load_op = machine()->Load(MachineType::Int32());
284      } else if (rep_type == SimdType::kFloat32) {
285        load_op = machine()->Load(MachineType::Float32());
286      }
287      LowerLoadOp(rep, node, load_op);
288      break;
289    }
290    case IrOpcode::kUnalignedLoad: {
291      MachineRepresentation rep =
292          UnalignedLoadRepresentationOf(node->op()).representation();
293      const Operator* load_op;
294      if (rep_type == SimdType::kInt32) {
295        load_op = machine()->UnalignedLoad(MachineType::Int32());
296      } else if (rep_type == SimdType::kFloat32) {
297        load_op = machine()->UnalignedLoad(MachineType::Float32());
298      }
299      LowerLoadOp(rep, node, load_op);
300      break;
301    }
302    case IrOpcode::kStore: {
303      MachineRepresentation rep =
304          StoreRepresentationOf(node->op()).representation();
305      WriteBarrierKind write_barrier_kind =
306          StoreRepresentationOf(node->op()).write_barrier_kind();
307      const Operator* store_op;
308      if (rep_type == SimdType::kInt32) {
309        store_op = machine()->Store(StoreRepresentation(
310            MachineRepresentation::kWord32, write_barrier_kind));
311      } else {
312        store_op = machine()->Store(StoreRepresentation(
313            MachineRepresentation::kFloat32, write_barrier_kind));
314      }
315      LowerStoreOp(rep, node, store_op, rep_type);
316      break;
317    }
318    case IrOpcode::kUnalignedStore: {
319      MachineRepresentation rep = UnalignedStoreRepresentationOf(node->op());
320      const Operator* store_op;
321      if (rep_type == SimdType::kInt32) {
322        store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
323      } else {
324        store_op = machine()->UnalignedStore(MachineRepresentation::kFloat32);
325      }
326      LowerStoreOp(rep, node, store_op, rep_type);
327      break;
328    }
329    case IrOpcode::kReturn: {
330      DefaultLowering(node);
331      int new_return_count = GetReturnCountAfterLowering(signature());
332      if (static_cast<int>(signature()->return_count()) != new_return_count) {
333        NodeProperties::ChangeOp(node, common()->Return(new_return_count));
334      }
335      break;
336    }
337    case IrOpcode::kCall: {
338      // TODO(turbofan): Make WASM code const-correct wrt. CallDescriptor.
339      CallDescriptor* descriptor =
340          const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
341      if (DefaultLowering(node) ||
342          (descriptor->ReturnCount() == 1 &&
343           descriptor->GetReturnType(0) == MachineType::Simd128())) {
344        // We have to adjust the call descriptor.
345        const Operator* op =
346            common()->Call(wasm::ModuleEnv::GetI32WasmCallDescriptorForSimd(
347                zone(), descriptor));
348        NodeProperties::ChangeOp(node, op);
349      }
350      if (descriptor->ReturnCount() == 1 &&
351          descriptor->GetReturnType(0) == MachineType::Simd128()) {
352        // We access the additional return values through projections.
353        Node* rep_node[kMaxLanes];
354        for (int i = 0; i < kMaxLanes; ++i) {
355          rep_node[i] =
356              graph()->NewNode(common()->Projection(i), node, graph()->start());
357        }
358        ReplaceNode(node, rep_node);
359      }
360      break;
361    }
362    case IrOpcode::kPhi: {
363      MachineRepresentation rep = PhiRepresentationOf(node->op());
364      if (rep == MachineRepresentation::kSimd128) {
365        // The replacement nodes have already been created, we only have to
366        // replace placeholder nodes.
367        Node** rep_node = GetReplacements(node);
368        for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
369          Node** rep_input =
370              GetReplacementsWithType(node->InputAt(i), rep_type);
371          for (int j = 0; j < kMaxLanes; j++) {
372            rep_node[j]->ReplaceInput(i, rep_input[j]);
373          }
374        }
375      } else {
376        DefaultLowering(node);
377      }
378      break;
379    }
380    case IrOpcode::kInt32x4Add: {
381      LowerBinaryOp(node, rep_type, machine()->Int32Add());
382      break;
383    }
384    case IrOpcode::kFloat32x4Add: {
385      LowerBinaryOp(node, rep_type, machine()->Float32Add());
386      break;
387    }
388    case IrOpcode::kCreateInt32x4:
389    case IrOpcode::kCreateFloat32x4: {
390      Node* rep_node[kMaxLanes];
391      for (int i = 0; i < kMaxLanes; ++i) {
392        if (HasReplacement(0, node->InputAt(i))) {
393          rep_node[i] = GetReplacements(node->InputAt(i))[0];
394        } else {
395          rep_node[i] = node->InputAt(i);
396        }
397      }
398      ReplaceNode(node, rep_node);
399      break;
400    }
401    case IrOpcode::kInt32x4ExtractLane:
402    case IrOpcode::kFloat32x4ExtractLane: {
403      int32_t lane = OpParameter<int32_t>(node);
404      Node* rep_node[kMaxLanes] = {
405          GetReplacementsWithType(node->InputAt(0), rep_type)[lane], nullptr,
406          nullptr, nullptr};
407      ReplaceNode(node, rep_node);
408      break;
409    }
410    case IrOpcode::kInt32x4ReplaceLane:
411    case IrOpcode::kFloat32x4ReplaceLane: {
412      DCHECK_EQ(2, node->InputCount());
413      Node* repNode = node->InputAt(1);
414      int32_t lane = OpParameter<int32_t>(node);
415      DCHECK(lane >= 0 && lane <= 3);
416      Node** rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
417      if (HasReplacement(0, repNode)) {
418        rep_node[lane] = GetReplacements(repNode)[0];
419      } else {
420        rep_node[lane] = repNode;
421      }
422      ReplaceNode(node, rep_node);
423      break;
424    }
425    default: { DefaultLowering(node); }
426  }
427}
428
429bool SimdScalarLowering::DefaultLowering(Node* node) {
430  bool something_changed = false;
431  for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
432    Node* input = node->InputAt(i);
433    if (HasReplacement(0, input)) {
434      something_changed = true;
435      node->ReplaceInput(i, GetReplacements(input)[0]);
436    }
437    if (HasReplacement(1, input)) {
438      something_changed = true;
439      for (int j = 1; j < kMaxLanes; j++) {
440        node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
441      }
442    }
443  }
444  return something_changed;
445}
446
447void SimdScalarLowering::ReplaceNode(Node* old, Node** new_node) {
448  // if new_low == nullptr, then also new_high == nullptr.
449  DCHECK(new_node[0] != nullptr ||
450         (new_node[1] == nullptr && new_node[2] == nullptr &&
451          new_node[3] == nullptr));
452  for (int i = 0; i < kMaxLanes; ++i) {
453    replacements_[old->id()].node[i] = new_node[i];
454  }
455}
456
457bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
458  return replacements_[node->id()].node[index] != nullptr;
459}
460
461SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
462  return replacements_[node->id()].type;
463}
464
465Node** SimdScalarLowering::GetReplacements(Node* node) {
466  Node** result = replacements_[node->id()].node;
467  DCHECK(result);
468  return result;
469}
470
471Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
472  Node** replacements = GetReplacements(node);
473  if (ReplacementType(node) == type) {
474    return GetReplacements(node);
475  }
476  Node** result = zone()->NewArray<Node*>(kMaxLanes);
477  if (ReplacementType(node) == SimdType::kInt32 && type == SimdType::kFloat32) {
478    for (int i = 0; i < kMaxLanes; ++i) {
479      if (replacements[i] != nullptr) {
480        result[i] = graph()->NewNode(machine()->BitcastInt32ToFloat32(),
481                                     replacements[i]);
482      } else {
483        result[i] = nullptr;
484      }
485    }
486  } else {
487    for (int i = 0; i < kMaxLanes; ++i) {
488      if (replacements[i] != nullptr) {
489        result[i] = graph()->NewNode(machine()->BitcastFloat32ToInt32(),
490                                     replacements[i]);
491      } else {
492        result[i] = nullptr;
493      }
494    }
495  }
496  return result;
497}
498
499void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
500  MachineRepresentation rep = PhiRepresentationOf(phi->op());
501  if (rep == MachineRepresentation::kSimd128) {
502    // We have to create the replacements for a phi node before we actually
503    // lower the phi to break potential cycles in the graph. The replacements of
504    // input nodes do not exist yet, so we use a placeholder node to pass the
505    // graph verifier.
506    int value_count = phi->op()->ValueInputCount();
507    SimdType type = ReplacementType(phi);
508    Node** inputs_rep[kMaxLanes];
509    for (int i = 0; i < kMaxLanes; ++i) {
510      inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
511      inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
512    }
513    for (int i = 0; i < value_count; ++i) {
514      for (int j = 0; j < kMaxLanes; j++) {
515        inputs_rep[j][i] = placeholder_;
516      }
517    }
518    Node* rep_nodes[kMaxLanes];
519    for (int i = 0; i < kMaxLanes; ++i) {
520      if (type == SimdType::kInt32) {
521        rep_nodes[i] = graph()->NewNode(
522            common()->Phi(MachineRepresentation::kWord32, value_count),
523            value_count + 1, inputs_rep[i], false);
524      } else if (type == SimdType::kFloat32) {
525        rep_nodes[i] = graph()->NewNode(
526            common()->Phi(MachineRepresentation::kFloat32, value_count),
527            value_count + 1, inputs_rep[i], false);
528      } else {
529        UNREACHABLE();
530      }
531    }
532    ReplaceNode(phi, rep_nodes);
533  }
534}
535}  // namespace compiler
536}  // namespace internal
537}  // namespace v8
538