1c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch// Copyright 2016 the V8 project authors. All rights reserved. 2c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch// Use of this source code is governed by a BSD-style license that can be 3c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch// found in the LICENSE file. 4c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 5c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#include "src/compiler/simd-scalar-lowering.h" 6c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#include "src/compiler/diamond.h" 7c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#include "src/compiler/linkage.h" 8c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#include "src/compiler/node-matchers.h" 9c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#include "src/compiler/node-properties.h" 10c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 11c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#include "src/compiler/node.h" 1262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch#include "src/objects-inl.h" 13c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#include "src/wasm/wasm-module.h" 14c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 15c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochnamespace v8 { 16c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochnamespace internal { 17c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochnamespace compiler { 18c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 19c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen MurdochSimdScalarLowering::SimdScalarLowering( 20c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Graph* graph, MachineOperatorBuilder* machine, 21c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch CommonOperatorBuilder* common, Zone* zone, 22c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Signature<MachineRepresentation>* signature) 23c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch : zone_(zone), 24c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch graph_(graph), 25c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch machine_(machine), 26c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch common_(common), 27c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch state_(graph, 3), 28c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch stack_(zone), 29c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements_(nullptr), 30c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch signature_(signature), 31c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch placeholder_( 32c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch graph->NewNode(common->Parameter(-2, "placeholder"), graph->start())), 33c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch parameter_count_after_lowering_(-1) { 34c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch DCHECK_NOT_NULL(graph); 35c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch DCHECK_NOT_NULL(graph->end()); 36c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements_ = zone->NewArray<Replacement>(graph->NodeCount()); 37c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount()); 38c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 39c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 40c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochvoid SimdScalarLowering::LowerGraph() { 41c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch stack_.push_back({graph()->end(), 0}); 42c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch state_.Set(graph()->end(), State::kOnStack); 43c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements_[graph()->end()->id()].type = SimdType::kInt32; 44c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 45c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch while (!stack_.empty()) { 46c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch NodeState& top = stack_.back(); 47c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (top.input_index == top.node->InputCount()) { 48c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // All inputs of top have already been lowered, now lower top. 49c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch stack_.pop_back(); 50c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch state_.Set(top.node, State::kVisited); 51c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch LowerNode(top.node); 52c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } else { 53c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // Push the next input onto the stack. 54c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node* input = top.node->InputAt(top.input_index++); 55c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (state_.Get(input) == State::kUnvisited) { 56c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch SetLoweredType(input, top.node); 57c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (input->opcode() == IrOpcode::kPhi) { 58c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // To break cycles with phi nodes we push phis on a separate stack so 59c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // that they are processed after all other nodes. 60c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch PreparePhiReplacement(input); 61c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch stack_.push_front({input, 0}); 6262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else if (input->opcode() == IrOpcode::kEffectPhi || 6362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch input->opcode() == IrOpcode::kLoop) { 6462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch stack_.push_front({input, 0}); 65c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } else { 66c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch stack_.push_back({input, 0}); 67c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 68c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch state_.Set(input, State::kOnStack); 69c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 70c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 71c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 72c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 73c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 74c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#define FOREACH_INT32X4_OPCODE(V) \ 75c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch V(Int32x4Add) \ 76c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch V(Int32x4ExtractLane) \ 7762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch V(CreateInt32x4) \ 7862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch V(Int32x4ReplaceLane) 79c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 80c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#define FOREACH_FLOAT32X4_OPCODE(V) \ 81c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch V(Float32x4Add) \ 82c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch V(Float32x4ExtractLane) \ 8362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch V(CreateFloat32x4) \ 8462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch V(Float32x4ReplaceLane) 85c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 86c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochvoid SimdScalarLowering::SetLoweredType(Node* node, Node* output) { 87c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch switch (node->opcode()) { 88c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#define CASE_STMT(name) case IrOpcode::k##name: 89c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch FOREACH_INT32X4_OPCODE(CASE_STMT) 90c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kReturn: 91c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kParameter: 92c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kCall: { 93c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements_[node->id()].type = SimdType::kInt32; 94c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 95c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 96c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch FOREACH_FLOAT32X4_OPCODE(CASE_STMT) { 97c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements_[node->id()].type = SimdType::kFloat32; 98c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 99c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 100c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch#undef CASE_STMT 101c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch default: 102c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements_[node->id()].type = replacements_[output->id()].type; 103c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 104c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 105c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 106c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochstatic int GetParameterIndexAfterLowering( 107c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Signature<MachineRepresentation>* signature, int old_index) { 108c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // In function calls, the simd128 types are passed as 4 Int32 types. The 109c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // parameters are typecast to the types as needed for various operations. 110c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int result = old_index; 11162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < old_index; ++i) { 112c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (signature->GetParam(i) == MachineRepresentation::kSimd128) { 113c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch result += 3; 114c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 115c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 116c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return result; 117c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 118c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 119c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochint SimdScalarLowering::GetParameterCountAfterLowering() { 120c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (parameter_count_after_lowering_ == -1) { 121c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // GetParameterIndexAfterLowering(parameter_count) returns the parameter 122c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // count after lowering. 123c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch parameter_count_after_lowering_ = GetParameterIndexAfterLowering( 124c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch signature(), static_cast<int>(signature()->parameter_count())); 125c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 126c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return parameter_count_after_lowering_; 127c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 128c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 129c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochstatic int GetReturnCountAfterLowering( 130c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Signature<MachineRepresentation>* signature) { 131c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int result = static_cast<int>(signature->return_count()); 13262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) { 133c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (signature->GetReturn(i) == MachineRepresentation::kSimd128) { 134c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch result += 3; 135c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 136c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 137c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return result; 138c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 139c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 14062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdochvoid SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices) { 14162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch new_indices[0] = index; 14262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (size_t i = 1; i < kMaxLanes; ++i) { 14362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch new_indices[i] = graph()->NewNode(machine()->Int32Add(), index, 14462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch graph()->NewNode(common()->Int32Constant( 14562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch static_cast<int>(i) * kLaneWidth))); 14662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 14762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch} 14862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch 14962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdochvoid SimdScalarLowering::LowerLoadOp(MachineRepresentation rep, Node* node, 15062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch const Operator* load_op) { 15162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (rep == MachineRepresentation::kSimd128) { 15262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* base = node->InputAt(0); 15362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* index = node->InputAt(1); 15462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* indices[kMaxLanes]; 15562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch GetIndexNodes(index, indices); 15662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* rep_nodes[kMaxLanes]; 15762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[0] = node; 15862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch NodeProperties::ChangeOp(rep_nodes[0], load_op); 15962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (node->InputCount() > 2) { 16062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DCHECK(node->InputCount() > 3); 16162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* effect_input = node->InputAt(2); 16262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* control_input = node->InputAt(3); 16362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[3] = graph()->NewNode(load_op, base, indices[3], effect_input, 16462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch control_input); 16562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[2] = graph()->NewNode(load_op, base, indices[2], rep_nodes[3], 16662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch control_input); 16762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[1] = graph()->NewNode(load_op, base, indices[1], rep_nodes[2], 16862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch control_input); 16962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[0]->ReplaceInput(2, rep_nodes[1]); 17062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else { 17162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (size_t i = 1; i < kMaxLanes; ++i) { 17262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]); 17362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 17462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 17562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch ReplaceNode(node, rep_nodes); 17662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else { 17762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DefaultLowering(node); 17862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 17962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch} 18062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch 18162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdochvoid SimdScalarLowering::LowerStoreOp(MachineRepresentation rep, Node* node, 18262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch const Operator* store_op, 18362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch SimdType rep_type) { 18462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (rep == MachineRepresentation::kSimd128) { 18562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* base = node->InputAt(0); 18662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* index = node->InputAt(1); 18762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* indices[kMaxLanes]; 18862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch GetIndexNodes(index, indices); 18962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DCHECK(node->InputCount() > 2); 19062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* value = node->InputAt(2); 19162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DCHECK(HasReplacement(1, value)); 19262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* rep_nodes[kMaxLanes]; 19362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[0] = node; 19462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node** rep_inputs = GetReplacementsWithType(value, rep_type); 19562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[0]->ReplaceInput(2, rep_inputs[0]); 19662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch NodeProperties::ChangeOp(node, store_op); 19762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (node->InputCount() > 3) { 19862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DCHECK(node->InputCount() > 4); 19962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* effect_input = node->InputAt(3); 20062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* control_input = node->InputAt(4); 20162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[3] = graph()->NewNode(store_op, base, indices[3], rep_inputs[3], 20262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch effect_input, control_input); 20362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[2] = graph()->NewNode(store_op, base, indices[2], rep_inputs[2], 20462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[3], control_input); 20562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[1] = graph()->NewNode(store_op, base, indices[1], rep_inputs[1], 20662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[2], control_input); 20762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[0]->ReplaceInput(3, rep_nodes[1]); 20862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch 20962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else { 21062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (size_t i = 1; i < kMaxLanes; ++i) { 21162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_nodes[i] = 21262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch graph()->NewNode(store_op, base, indices[i], rep_inputs[i]); 21362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 21462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 21562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch 21662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch ReplaceNode(node, rep_nodes); 21762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else { 21862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DefaultLowering(node); 21962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 22062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch} 22162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch 22262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdochvoid SimdScalarLowering::LowerBinaryOp(Node* node, SimdType rep_type, 22362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch const Operator* op) { 22462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DCHECK(node->InputCount() == 2); 22562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type); 22662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type); 22762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* rep_node[kMaxLanes]; 22862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 22962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]); 23062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 23162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch ReplaceNode(node, rep_node); 23262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch} 23362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch 234c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochvoid SimdScalarLowering::LowerNode(Node* node) { 235c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch SimdType rep_type = ReplacementType(node); 236c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch switch (node->opcode()) { 237c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kStart: { 238c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int parameter_count = GetParameterCountAfterLowering(); 239c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // Only exchange the node if the parameter count actually changed. 240c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (parameter_count != static_cast<int>(signature()->parameter_count())) { 241c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int delta = 242c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch parameter_count - static_cast<int>(signature()->parameter_count()); 243c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int new_output_count = node->op()->ValueOutputCount() + delta; 244c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch NodeProperties::ChangeOp(node, common()->Start(new_output_count)); 245c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 246c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 247c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 248c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kParameter: { 249c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch DCHECK(node->InputCount() == 1); 250c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // Only exchange the node if the parameter count actually changed. We do 251c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // not even have to do the default lowering because the the start node, 252c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // the only input of a parameter node, only changes if the parameter count 253c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // changes. 254c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (GetParameterCountAfterLowering() != 255c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch static_cast<int>(signature()->parameter_count())) { 256c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int old_index = ParameterIndexOf(node->op()); 257c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int new_index = GetParameterIndexAfterLowering(signature(), old_index); 258c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (old_index == new_index) { 259c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch NodeProperties::ChangeOp(node, common()->Parameter(new_index)); 260c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 261c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node* new_node[kMaxLanes]; 26262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 263c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch new_node[i] = nullptr; 264c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 265c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch new_node[0] = node; 266c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (signature()->GetParam(old_index) == 267c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch MachineRepresentation::kSimd128) { 26862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 1; i < kMaxLanes; ++i) { 269c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch new_node[i] = graph()->NewNode(common()->Parameter(new_index + i), 270c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch graph()->start()); 271c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 272c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 273c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch ReplaceNode(node, new_node); 274c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 275c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 276c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 277c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 27862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch case IrOpcode::kLoad: { 27962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch MachineRepresentation rep = 28062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch LoadRepresentationOf(node->op()).representation(); 28162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch const Operator* load_op; 28262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (rep_type == SimdType::kInt32) { 28362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch load_op = machine()->Load(MachineType::Int32()); 28462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else if (rep_type == SimdType::kFloat32) { 28562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch load_op = machine()->Load(MachineType::Float32()); 28662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 28762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch LowerLoadOp(rep, node, load_op); 28862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch break; 28962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 29062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch case IrOpcode::kUnalignedLoad: { 29162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch MachineRepresentation rep = 29262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch UnalignedLoadRepresentationOf(node->op()).representation(); 29362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch const Operator* load_op; 29462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (rep_type == SimdType::kInt32) { 29562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch load_op = machine()->UnalignedLoad(MachineType::Int32()); 29662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else if (rep_type == SimdType::kFloat32) { 29762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch load_op = machine()->UnalignedLoad(MachineType::Float32()); 29862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 29962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch LowerLoadOp(rep, node, load_op); 30062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch break; 30162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 30262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch case IrOpcode::kStore: { 30362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch MachineRepresentation rep = 30462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch StoreRepresentationOf(node->op()).representation(); 30562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch WriteBarrierKind write_barrier_kind = 30662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch StoreRepresentationOf(node->op()).write_barrier_kind(); 30762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch const Operator* store_op; 30862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (rep_type == SimdType::kInt32) { 30962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch store_op = machine()->Store(StoreRepresentation( 31062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch MachineRepresentation::kWord32, write_barrier_kind)); 31162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else { 31262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch store_op = machine()->Store(StoreRepresentation( 31362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch MachineRepresentation::kFloat32, write_barrier_kind)); 31462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 31562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch LowerStoreOp(rep, node, store_op, rep_type); 31662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch break; 31762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 31862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch case IrOpcode::kUnalignedStore: { 31962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch MachineRepresentation rep = UnalignedStoreRepresentationOf(node->op()); 32062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch const Operator* store_op; 32162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (rep_type == SimdType::kInt32) { 32262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch store_op = machine()->UnalignedStore(MachineRepresentation::kWord32); 32362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else { 32462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch store_op = machine()->UnalignedStore(MachineRepresentation::kFloat32); 32562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 32662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch LowerStoreOp(rep, node, store_op, rep_type); 32762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch break; 32862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 329c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kReturn: { 330c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch DefaultLowering(node); 331c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int new_return_count = GetReturnCountAfterLowering(signature()); 332c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (static_cast<int>(signature()->return_count()) != new_return_count) { 333c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch NodeProperties::ChangeOp(node, common()->Return(new_return_count)); 334c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 335c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 336c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 337c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kCall: { 338c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // TODO(turbofan): Make WASM code const-correct wrt. CallDescriptor. 339c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch CallDescriptor* descriptor = 340c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch const_cast<CallDescriptor*>(CallDescriptorOf(node->op())); 341c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (DefaultLowering(node) || 342c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch (descriptor->ReturnCount() == 1 && 343c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch descriptor->GetReturnType(0) == MachineType::Simd128())) { 344c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // We have to adjust the call descriptor. 345c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch const Operator* op = 346c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch common()->Call(wasm::ModuleEnv::GetI32WasmCallDescriptorForSimd( 347c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch zone(), descriptor)); 348c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch NodeProperties::ChangeOp(node, op); 349c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 350c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (descriptor->ReturnCount() == 1 && 351c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch descriptor->GetReturnType(0) == MachineType::Simd128()) { 352c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // We access the additional return values through projections. 353c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node* rep_node[kMaxLanes]; 35462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 355c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch rep_node[i] = 356c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch graph()->NewNode(common()->Projection(i), node, graph()->start()); 357c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 358c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch ReplaceNode(node, rep_node); 359c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 360c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 361c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 362c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kPhi: { 363c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch MachineRepresentation rep = PhiRepresentationOf(node->op()); 364c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (rep == MachineRepresentation::kSimd128) { 365c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // The replacement nodes have already been created, we only have to 366c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // replace placeholder nodes. 367c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node** rep_node = GetReplacements(node); 36862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < node->op()->ValueInputCount(); ++i) { 369c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node** rep_input = 370c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch GetReplacementsWithType(node->InputAt(i), rep_type); 371c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch for (int j = 0; j < kMaxLanes; j++) { 372c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch rep_node[j]->ReplaceInput(i, rep_input[j]); 373c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 374c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 375c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } else { 376c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch DefaultLowering(node); 377c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 378c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 379c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 380c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kInt32x4Add: { 38162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch LowerBinaryOp(node, rep_type, machine()->Int32Add()); 382c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 383c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 384c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kFloat32x4Add: { 38562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch LowerBinaryOp(node, rep_type, machine()->Float32Add()); 386c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 387c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 38862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch case IrOpcode::kCreateInt32x4: 389c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kCreateFloat32x4: { 390c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node* rep_node[kMaxLanes]; 39162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 39262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (HasReplacement(0, node->InputAt(i))) { 39362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_node[i] = GetReplacements(node->InputAt(i))[0]; 39462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else { 39562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_node[i] = node->InputAt(i); 39662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 397c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 398c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch ReplaceNode(node, rep_node); 399c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 400c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 40162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch case IrOpcode::kInt32x4ExtractLane: 402c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch case IrOpcode::kFloat32x4ExtractLane: { 40362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch int32_t lane = OpParameter<int32_t>(node); 404c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node* rep_node[kMaxLanes] = { 405c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch GetReplacementsWithType(node->InputAt(0), rep_type)[lane], nullptr, 406c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch nullptr, nullptr}; 407c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch ReplaceNode(node, rep_node); 408c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch break; 409c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 41062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch case IrOpcode::kInt32x4ReplaceLane: 41162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch case IrOpcode::kFloat32x4ReplaceLane: { 41262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DCHECK_EQ(2, node->InputCount()); 41362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node* repNode = node->InputAt(1); 41462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch int32_t lane = OpParameter<int32_t>(node); 41562ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch DCHECK(lane >= 0 && lane <= 3); 41662ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch Node** rep_node = GetReplacementsWithType(node->InputAt(0), rep_type); 41762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch if (HasReplacement(0, repNode)) { 41862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_node[lane] = GetReplacements(repNode)[0]; 41962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } else { 42062ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch rep_node[lane] = repNode; 42162ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 42262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch ReplaceNode(node, rep_node); 42362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch break; 42462ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch } 425c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch default: { DefaultLowering(node); } 426c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 427c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 428c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 429c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochbool SimdScalarLowering::DefaultLowering(Node* node) { 430c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch bool something_changed = false; 431c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) { 432c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node* input = node->InputAt(i); 433c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (HasReplacement(0, input)) { 434c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch something_changed = true; 435c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch node->ReplaceInput(i, GetReplacements(input)[0]); 436c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 437c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (HasReplacement(1, input)) { 438c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch something_changed = true; 439c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch for (int j = 1; j < kMaxLanes; j++) { 440c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch node->InsertInput(zone(), i + j, GetReplacements(input)[j]); 441c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 442c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 443c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 444c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return something_changed; 445c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 446c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 447c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochvoid SimdScalarLowering::ReplaceNode(Node* old, Node** new_node) { 448c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // if new_low == nullptr, then also new_high == nullptr. 449c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch DCHECK(new_node[0] != nullptr || 450c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch (new_node[1] == nullptr && new_node[2] == nullptr && 451c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch new_node[3] == nullptr)); 45262ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 453c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements_[old->id()].node[i] = new_node[i]; 454c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 455c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 456c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 457c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochbool SimdScalarLowering::HasReplacement(size_t index, Node* node) { 458c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return replacements_[node->id()].node[index] != nullptr; 459c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 460c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 461c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen MurdochSimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) { 462c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return replacements_[node->id()].type; 463c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 464c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 465c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen MurdochNode** SimdScalarLowering::GetReplacements(Node* node) { 466c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node** result = replacements_[node->id()].node; 467c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch DCHECK(result); 468c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return result; 469c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 470c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 471c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen MurdochNode** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) { 472c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node** replacements = GetReplacements(node); 473c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (ReplacementType(node) == type) { 474c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return GetReplacements(node); 475c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 476c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node** result = zone()->NewArray<Node*>(kMaxLanes); 477c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (ReplacementType(node) == SimdType::kInt32 && type == SimdType::kFloat32) { 47862ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 479c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (replacements[i] != nullptr) { 480c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch result[i] = graph()->NewNode(machine()->BitcastInt32ToFloat32(), 481c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements[i]); 482c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } else { 483c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch result[i] = nullptr; 484c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 485c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 486c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } else { 48762ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 488c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (replacements[i] != nullptr) { 489c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch result[i] = graph()->NewNode(machine()->BitcastFloat32ToInt32(), 490c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch replacements[i]); 491c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } else { 492c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch result[i] = nullptr; 493c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 494c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 495c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 496c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch return result; 497c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 498c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch 499c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdochvoid SimdScalarLowering::PreparePhiReplacement(Node* phi) { 500c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch MachineRepresentation rep = PhiRepresentationOf(phi->op()); 501c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (rep == MachineRepresentation::kSimd128) { 502c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // We have to create the replacements for a phi node before we actually 503c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // lower the phi to break potential cycles in the graph. The replacements of 504c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // input nodes do not exist yet, so we use a placeholder node to pass the 505c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch // graph verifier. 506c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch int value_count = phi->op()->ValueInputCount(); 507c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch SimdType type = ReplacementType(phi); 508c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node** inputs_rep[kMaxLanes]; 50962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 510c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1); 511c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0); 512c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 51362ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < value_count; ++i) { 514c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch for (int j = 0; j < kMaxLanes; j++) { 515c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch inputs_rep[j][i] = placeholder_; 516c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 517c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 518c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch Node* rep_nodes[kMaxLanes]; 51962ed631aa0ff23db68a47fd423efa9c019ff2c9eBen Murdoch for (int i = 0; i < kMaxLanes; ++i) { 520c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch if (type == SimdType::kInt32) { 521c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch rep_nodes[i] = graph()->NewNode( 522c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch common()->Phi(MachineRepresentation::kWord32, value_count), 523c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch value_count + 1, inputs_rep[i], false); 524c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } else if (type == SimdType::kFloat32) { 525c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch rep_nodes[i] = graph()->NewNode( 526c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch common()->Phi(MachineRepresentation::kFloat32, value_count), 527c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch value_count + 1, inputs_rep[i], false); 528c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } else { 529c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch UNREACHABLE(); 530c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 531c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 532c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch ReplaceNode(phi, rep_nodes); 533c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch } 534c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} 535c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} // namespace compiler 536c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} // namespace internal 537c8c1d9e03f4babd16833b0f8ccf6aab5fa6e8c7aBen Murdoch} // namespace v8 538