1// Copyright 2014 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <algorithm>
6
7#include "src/base/adapters.h"
8#include "src/compiler/instruction-selector-impl.h"
9#include "src/compiler/node-matchers.h"
10#include "src/compiler/node-properties.h"
11
12namespace v8 {
13namespace internal {
14namespace compiler {
15
16// Adds X64-specific methods for generating operands.
17class X64OperandGenerator final : public OperandGenerator {
18 public:
19  explicit X64OperandGenerator(InstructionSelector* selector)
20      : OperandGenerator(selector) {}
21
22  bool CanBeImmediate(Node* node) {
23    switch (node->opcode()) {
24      case IrOpcode::kInt32Constant:
25      case IrOpcode::kRelocatableInt32Constant:
26        return true;
27      case IrOpcode::kInt64Constant: {
28        const int64_t value = OpParameter<int64_t>(node);
29        return value == static_cast<int64_t>(static_cast<int32_t>(value));
30      }
31      case IrOpcode::kNumberConstant: {
32        const double value = OpParameter<double>(node);
33        return bit_cast<int64_t>(value) == 0;
34      }
35      default:
36        return false;
37    }
38  }
39
40  int32_t GetImmediateIntegerValue(Node* node) {
41    DCHECK(CanBeImmediate(node));
42    if (node->opcode() == IrOpcode::kInt32Constant) {
43      return OpParameter<int32_t>(node);
44    }
45    DCHECK_EQ(IrOpcode::kInt64Constant, node->opcode());
46    return static_cast<int32_t>(OpParameter<int64_t>(node));
47  }
48
49  bool CanBeMemoryOperand(InstructionCode opcode, Node* node, Node* input,
50                          int effect_level) {
51    if (input->opcode() != IrOpcode::kLoad ||
52        !selector()->CanCover(node, input)) {
53      return false;
54    }
55    if (effect_level != selector()->GetEffectLevel(input)) {
56      return false;
57    }
58    MachineRepresentation rep =
59        LoadRepresentationOf(input->op()).representation();
60    switch (opcode) {
61      case kX64Cmp:
62      case kX64Test:
63        return rep == MachineRepresentation::kWord64 || IsAnyTagged(rep);
64      case kX64Cmp32:
65      case kX64Test32:
66        return rep == MachineRepresentation::kWord32;
67      case kX64Cmp16:
68      case kX64Test16:
69        return rep == MachineRepresentation::kWord16;
70      case kX64Cmp8:
71      case kX64Test8:
72        return rep == MachineRepresentation::kWord8;
73      default:
74        break;
75    }
76    return false;
77  }
78
79  AddressingMode GenerateMemoryOperandInputs(Node* index, int scale_exponent,
80                                             Node* base, Node* displacement,
81                                             DisplacementMode displacement_mode,
82                                             InstructionOperand inputs[],
83                                             size_t* input_count) {
84    AddressingMode mode = kMode_MRI;
85    if (base != nullptr) {
86      inputs[(*input_count)++] = UseRegister(base);
87      if (index != nullptr) {
88        DCHECK(scale_exponent >= 0 && scale_exponent <= 3);
89        inputs[(*input_count)++] = UseRegister(index);
90        if (displacement != nullptr) {
91          inputs[(*input_count)++] = displacement_mode
92                                         ? UseNegatedImmediate(displacement)
93                                         : UseImmediate(displacement);
94          static const AddressingMode kMRnI_modes[] = {kMode_MR1I, kMode_MR2I,
95                                                       kMode_MR4I, kMode_MR8I};
96          mode = kMRnI_modes[scale_exponent];
97        } else {
98          static const AddressingMode kMRn_modes[] = {kMode_MR1, kMode_MR2,
99                                                      kMode_MR4, kMode_MR8};
100          mode = kMRn_modes[scale_exponent];
101        }
102      } else {
103        if (displacement == nullptr) {
104          mode = kMode_MR;
105        } else {
106          inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
107                                         ? UseNegatedImmediate(displacement)
108                                         : UseImmediate(displacement);
109          mode = kMode_MRI;
110        }
111      }
112    } else {
113      DCHECK_NOT_NULL(index);
114      DCHECK(scale_exponent >= 0 && scale_exponent <= 3);
115      inputs[(*input_count)++] = UseRegister(index);
116      if (displacement != nullptr) {
117        inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
118                                       ? UseNegatedImmediate(displacement)
119                                       : UseImmediate(displacement);
120        static const AddressingMode kMnI_modes[] = {kMode_MRI, kMode_M2I,
121                                                    kMode_M4I, kMode_M8I};
122        mode = kMnI_modes[scale_exponent];
123      } else {
124        static const AddressingMode kMn_modes[] = {kMode_MR, kMode_MR1,
125                                                   kMode_M4, kMode_M8};
126        mode = kMn_modes[scale_exponent];
127        if (mode == kMode_MR1) {
128          // [%r1 + %r1*1] has a smaller encoding than [%r1*2+0]
129          inputs[(*input_count)++] = UseRegister(index);
130        }
131      }
132    }
133    return mode;
134  }
135
136  AddressingMode GetEffectiveAddressMemoryOperand(Node* operand,
137                                                  InstructionOperand inputs[],
138                                                  size_t* input_count) {
139    if (selector()->CanAddressRelativeToRootsRegister()) {
140      LoadMatcher<ExternalReferenceMatcher> m(operand);
141      if (m.index().HasValue() && m.object().HasValue()) {
142        Address const kRootsRegisterValue =
143            kRootRegisterBias +
144            reinterpret_cast<Address>(
145                selector()->isolate()->heap()->roots_array_start());
146        ptrdiff_t const delta =
147            m.index().Value() +
148            (m.object().Value().address() - kRootsRegisterValue);
149        if (is_int32(delta)) {
150          inputs[(*input_count)++] = TempImmediate(static_cast<int32_t>(delta));
151          return kMode_Root;
152        }
153      }
154    }
155    BaseWithIndexAndDisplacement64Matcher m(operand, AddressOption::kAllowAll);
156    DCHECK(m.matches());
157    if ((m.displacement() == nullptr || CanBeImmediate(m.displacement()))) {
158      return GenerateMemoryOperandInputs(
159          m.index(), m.scale(), m.base(), m.displacement(),
160          m.displacement_mode(), inputs, input_count);
161    } else {
162      inputs[(*input_count)++] = UseRegister(operand->InputAt(0));
163      inputs[(*input_count)++] = UseRegister(operand->InputAt(1));
164      return kMode_MR1;
165    }
166  }
167
168  bool CanBeBetterLeftOperand(Node* node) const {
169    return !selector()->IsLive(node);
170  }
171};
172
173namespace {
174
175ArchOpcode GetLoadOpcode(LoadRepresentation load_rep) {
176  ArchOpcode opcode = kArchNop;
177  switch (load_rep.representation()) {
178    case MachineRepresentation::kFloat32:
179      opcode = kX64Movss;
180      break;
181    case MachineRepresentation::kFloat64:
182      opcode = kX64Movsd;
183      break;
184    case MachineRepresentation::kBit:  // Fall through.
185    case MachineRepresentation::kWord8:
186      opcode = load_rep.IsSigned() ? kX64Movsxbl : kX64Movzxbl;
187      break;
188    case MachineRepresentation::kWord16:
189      opcode = load_rep.IsSigned() ? kX64Movsxwl : kX64Movzxwl;
190      break;
191    case MachineRepresentation::kWord32:
192      opcode = kX64Movl;
193      break;
194    case MachineRepresentation::kTaggedSigned:   // Fall through.
195    case MachineRepresentation::kTaggedPointer:  // Fall through.
196    case MachineRepresentation::kTagged:  // Fall through.
197    case MachineRepresentation::kWord64:
198      opcode = kX64Movq;
199      break;
200    case MachineRepresentation::kSimd128:  // Fall through.
201    case MachineRepresentation::kNone:
202      UNREACHABLE();
203      break;
204  }
205  return opcode;
206}
207
208}  // namespace
209
210void InstructionSelector::VisitLoad(Node* node) {
211  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
212  X64OperandGenerator g(this);
213
214  ArchOpcode opcode = GetLoadOpcode(load_rep);
215  InstructionOperand outputs[1];
216  outputs[0] = g.DefineAsRegister(node);
217  InstructionOperand inputs[3];
218  size_t input_count = 0;
219  AddressingMode mode =
220      g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
221  InstructionCode code = opcode | AddressingModeField::encode(mode);
222  Emit(code, 1, outputs, input_count, inputs);
223}
224
225void InstructionSelector::VisitProtectedLoad(Node* node) {
226  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
227  X64OperandGenerator g(this);
228
229  ArchOpcode opcode = GetLoadOpcode(load_rep);
230  InstructionOperand outputs[1];
231  outputs[0] = g.DefineAsRegister(node);
232  InstructionOperand inputs[4];
233  size_t input_count = 0;
234  AddressingMode mode =
235      g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
236  // Add the context parameter as an input.
237  inputs[input_count++] = g.UseUniqueRegister(node->InputAt(2));
238  // Add the source position as an input
239  inputs[input_count++] = g.UseImmediate(node->InputAt(3));
240  InstructionCode code = opcode | AddressingModeField::encode(mode);
241  Emit(code, 1, outputs, input_count, inputs);
242}
243
244void InstructionSelector::VisitStore(Node* node) {
245  X64OperandGenerator g(this);
246  Node* base = node->InputAt(0);
247  Node* index = node->InputAt(1);
248  Node* value = node->InputAt(2);
249
250  StoreRepresentation store_rep = StoreRepresentationOf(node->op());
251  WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
252  MachineRepresentation rep = store_rep.representation();
253
254  if (write_barrier_kind != kNoWriteBarrier) {
255    DCHECK(CanBeTaggedPointer(rep));
256    AddressingMode addressing_mode;
257    InstructionOperand inputs[3];
258    size_t input_count = 0;
259    inputs[input_count++] = g.UseUniqueRegister(base);
260    if (g.CanBeImmediate(index)) {
261      inputs[input_count++] = g.UseImmediate(index);
262      addressing_mode = kMode_MRI;
263    } else {
264      inputs[input_count++] = g.UseUniqueRegister(index);
265      addressing_mode = kMode_MR1;
266    }
267    inputs[input_count++] = g.UseUniqueRegister(value);
268    RecordWriteMode record_write_mode = RecordWriteMode::kValueIsAny;
269    switch (write_barrier_kind) {
270      case kNoWriteBarrier:
271        UNREACHABLE();
272        break;
273      case kMapWriteBarrier:
274        record_write_mode = RecordWriteMode::kValueIsMap;
275        break;
276      case kPointerWriteBarrier:
277        record_write_mode = RecordWriteMode::kValueIsPointer;
278        break;
279      case kFullWriteBarrier:
280        record_write_mode = RecordWriteMode::kValueIsAny;
281        break;
282    }
283    InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
284    size_t const temp_count = arraysize(temps);
285    InstructionCode code = kArchStoreWithWriteBarrier;
286    code |= AddressingModeField::encode(addressing_mode);
287    code |= MiscField::encode(static_cast<int>(record_write_mode));
288    Emit(code, 0, nullptr, input_count, inputs, temp_count, temps);
289  } else {
290    ArchOpcode opcode = kArchNop;
291    switch (rep) {
292      case MachineRepresentation::kFloat32:
293        opcode = kX64Movss;
294        break;
295      case MachineRepresentation::kFloat64:
296        opcode = kX64Movsd;
297        break;
298      case MachineRepresentation::kBit:  // Fall through.
299      case MachineRepresentation::kWord8:
300        opcode = kX64Movb;
301        break;
302      case MachineRepresentation::kWord16:
303        opcode = kX64Movw;
304        break;
305      case MachineRepresentation::kWord32:
306        opcode = kX64Movl;
307        break;
308      case MachineRepresentation::kTaggedSigned:   // Fall through.
309      case MachineRepresentation::kTaggedPointer:  // Fall through.
310      case MachineRepresentation::kTagged:  // Fall through.
311      case MachineRepresentation::kWord64:
312        opcode = kX64Movq;
313        break;
314      case MachineRepresentation::kSimd128:  // Fall through.
315      case MachineRepresentation::kNone:
316        UNREACHABLE();
317        return;
318    }
319    InstructionOperand inputs[4];
320    size_t input_count = 0;
321    AddressingMode addressing_mode =
322        g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
323    InstructionCode code =
324        opcode | AddressingModeField::encode(addressing_mode);
325    InstructionOperand value_operand =
326        g.CanBeImmediate(value) ? g.UseImmediate(value) : g.UseRegister(value);
327    inputs[input_count++] = value_operand;
328    Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count,
329         inputs);
330  }
331}
332
333// Architecture supports unaligned access, therefore VisitLoad is used instead
334void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); }
335
336// Architecture supports unaligned access, therefore VisitStore is used instead
337void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); }
338
339void InstructionSelector::VisitCheckedLoad(Node* node) {
340  CheckedLoadRepresentation load_rep = CheckedLoadRepresentationOf(node->op());
341  X64OperandGenerator g(this);
342  Node* const buffer = node->InputAt(0);
343  Node* const offset = node->InputAt(1);
344  Node* const length = node->InputAt(2);
345  ArchOpcode opcode = kArchNop;
346  switch (load_rep.representation()) {
347    case MachineRepresentation::kWord8:
348      opcode = load_rep.IsSigned() ? kCheckedLoadInt8 : kCheckedLoadUint8;
349      break;
350    case MachineRepresentation::kWord16:
351      opcode = load_rep.IsSigned() ? kCheckedLoadInt16 : kCheckedLoadUint16;
352      break;
353    case MachineRepresentation::kWord32:
354      opcode = kCheckedLoadWord32;
355      break;
356    case MachineRepresentation::kWord64:
357      opcode = kCheckedLoadWord64;
358      break;
359    case MachineRepresentation::kFloat32:
360      opcode = kCheckedLoadFloat32;
361      break;
362    case MachineRepresentation::kFloat64:
363      opcode = kCheckedLoadFloat64;
364      break;
365    case MachineRepresentation::kBit:      // Fall through.
366    case MachineRepresentation::kSimd128:  // Fall through.
367    case MachineRepresentation::kTaggedSigned:   // Fall through.
368    case MachineRepresentation::kTaggedPointer:  // Fall through.
369    case MachineRepresentation::kTagged:   // Fall through.
370    case MachineRepresentation::kNone:
371      UNREACHABLE();
372      return;
373  }
374  if (offset->opcode() == IrOpcode::kInt32Add && CanCover(node, offset)) {
375    Int32Matcher mlength(length);
376    Int32BinopMatcher moffset(offset);
377    if (mlength.HasValue() && moffset.right().HasValue() &&
378        moffset.right().Value() >= 0 &&
379        mlength.Value() >= moffset.right().Value()) {
380      Emit(opcode, g.DefineAsRegister(node), g.UseRegister(buffer),
381           g.UseRegister(moffset.left().node()),
382           g.UseImmediate(moffset.right().node()), g.UseImmediate(length));
383      return;
384    }
385  }
386  InstructionOperand length_operand =
387      g.CanBeImmediate(length) ? g.UseImmediate(length) : g.UseRegister(length);
388  Emit(opcode, g.DefineAsRegister(node), g.UseRegister(buffer),
389       g.UseRegister(offset), g.TempImmediate(0), length_operand);
390}
391
392
393void InstructionSelector::VisitCheckedStore(Node* node) {
394  MachineRepresentation rep = CheckedStoreRepresentationOf(node->op());
395  X64OperandGenerator g(this);
396  Node* const buffer = node->InputAt(0);
397  Node* const offset = node->InputAt(1);
398  Node* const length = node->InputAt(2);
399  Node* const value = node->InputAt(3);
400  ArchOpcode opcode = kArchNop;
401  switch (rep) {
402    case MachineRepresentation::kWord8:
403      opcode = kCheckedStoreWord8;
404      break;
405    case MachineRepresentation::kWord16:
406      opcode = kCheckedStoreWord16;
407      break;
408    case MachineRepresentation::kWord32:
409      opcode = kCheckedStoreWord32;
410      break;
411    case MachineRepresentation::kWord64:
412      opcode = kCheckedStoreWord64;
413      break;
414    case MachineRepresentation::kFloat32:
415      opcode = kCheckedStoreFloat32;
416      break;
417    case MachineRepresentation::kFloat64:
418      opcode = kCheckedStoreFloat64;
419      break;
420    case MachineRepresentation::kBit:      // Fall through.
421    case MachineRepresentation::kSimd128:  // Fall through.
422    case MachineRepresentation::kTaggedSigned:   // Fall through.
423    case MachineRepresentation::kTaggedPointer:  // Fall through.
424    case MachineRepresentation::kTagged:   // Fall through.
425    case MachineRepresentation::kNone:
426      UNREACHABLE();
427      return;
428  }
429  InstructionOperand value_operand =
430      g.CanBeImmediate(value) ? g.UseImmediate(value) : g.UseRegister(value);
431  if (offset->opcode() == IrOpcode::kInt32Add && CanCover(node, offset)) {
432    Int32Matcher mlength(length);
433    Int32BinopMatcher moffset(offset);
434    if (mlength.HasValue() && moffset.right().HasValue() &&
435        moffset.right().Value() >= 0 &&
436        mlength.Value() >= moffset.right().Value()) {
437      Emit(opcode, g.NoOutput(), g.UseRegister(buffer),
438           g.UseRegister(moffset.left().node()),
439           g.UseImmediate(moffset.right().node()), g.UseImmediate(length),
440           value_operand);
441      return;
442    }
443  }
444  InstructionOperand length_operand =
445      g.CanBeImmediate(length) ? g.UseImmediate(length) : g.UseRegister(length);
446  Emit(opcode, g.NoOutput(), g.UseRegister(buffer), g.UseRegister(offset),
447       g.TempImmediate(0), length_operand, value_operand);
448}
449
450
451// Shared routine for multiple binary operations.
452static void VisitBinop(InstructionSelector* selector, Node* node,
453                       InstructionCode opcode, FlagsContinuation* cont) {
454  X64OperandGenerator g(selector);
455  Int32BinopMatcher m(node);
456  Node* left = m.left().node();
457  Node* right = m.right().node();
458  InstructionOperand inputs[4];
459  size_t input_count = 0;
460  InstructionOperand outputs[2];
461  size_t output_count = 0;
462
463  // TODO(turbofan): match complex addressing modes.
464  if (left == right) {
465    // If both inputs refer to the same operand, enforce allocating a register
466    // for both of them to ensure that we don't end up generating code like
467    // this:
468    //
469    //   mov rax, [rbp-0x10]
470    //   add rax, [rbp-0x10]
471    //   jo label
472    InstructionOperand const input = g.UseRegister(left);
473    inputs[input_count++] = input;
474    inputs[input_count++] = input;
475  } else if (g.CanBeImmediate(right)) {
476    inputs[input_count++] = g.UseRegister(left);
477    inputs[input_count++] = g.UseImmediate(right);
478  } else {
479    if (node->op()->HasProperty(Operator::kCommutative) &&
480        g.CanBeBetterLeftOperand(right)) {
481      std::swap(left, right);
482    }
483    inputs[input_count++] = g.UseRegister(left);
484    inputs[input_count++] = g.Use(right);
485  }
486
487  if (cont->IsBranch()) {
488    inputs[input_count++] = g.Label(cont->true_block());
489    inputs[input_count++] = g.Label(cont->false_block());
490  }
491
492  outputs[output_count++] = g.DefineSameAsFirst(node);
493  if (cont->IsSet()) {
494    outputs[output_count++] = g.DefineAsRegister(cont->result());
495  }
496
497  DCHECK_NE(0u, input_count);
498  DCHECK_NE(0u, output_count);
499  DCHECK_GE(arraysize(inputs), input_count);
500  DCHECK_GE(arraysize(outputs), output_count);
501
502  opcode = cont->Encode(opcode);
503  if (cont->IsDeoptimize()) {
504    selector->EmitDeoptimize(opcode, output_count, outputs, input_count, inputs,
505                             cont->reason(), cont->frame_state());
506  } else {
507    selector->Emit(opcode, output_count, outputs, input_count, inputs);
508  }
509}
510
511
512// Shared routine for multiple binary operations.
513static void VisitBinop(InstructionSelector* selector, Node* node,
514                       InstructionCode opcode) {
515  FlagsContinuation cont;
516  VisitBinop(selector, node, opcode, &cont);
517}
518
519
520void InstructionSelector::VisitWord32And(Node* node) {
521  X64OperandGenerator g(this);
522  Uint32BinopMatcher m(node);
523  if (m.right().Is(0xff)) {
524    Emit(kX64Movzxbl, g.DefineAsRegister(node), g.Use(m.left().node()));
525  } else if (m.right().Is(0xffff)) {
526    Emit(kX64Movzxwl, g.DefineAsRegister(node), g.Use(m.left().node()));
527  } else {
528    VisitBinop(this, node, kX64And32);
529  }
530}
531
532
533void InstructionSelector::VisitWord64And(Node* node) {
534  VisitBinop(this, node, kX64And);
535}
536
537
538void InstructionSelector::VisitWord32Or(Node* node) {
539  VisitBinop(this, node, kX64Or32);
540}
541
542
543void InstructionSelector::VisitWord64Or(Node* node) {
544  VisitBinop(this, node, kX64Or);
545}
546
547
548void InstructionSelector::VisitWord32Xor(Node* node) {
549  X64OperandGenerator g(this);
550  Uint32BinopMatcher m(node);
551  if (m.right().Is(-1)) {
552    Emit(kX64Not32, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
553  } else {
554    VisitBinop(this, node, kX64Xor32);
555  }
556}
557
558
559void InstructionSelector::VisitWord64Xor(Node* node) {
560  X64OperandGenerator g(this);
561  Uint64BinopMatcher m(node);
562  if (m.right().Is(-1)) {
563    Emit(kX64Not, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
564  } else {
565    VisitBinop(this, node, kX64Xor);
566  }
567}
568
569
570namespace {
571
572// Shared routine for multiple 32-bit shift operations.
573// TODO(bmeurer): Merge this with VisitWord64Shift using template magic?
574void VisitWord32Shift(InstructionSelector* selector, Node* node,
575                      ArchOpcode opcode) {
576  X64OperandGenerator g(selector);
577  Int32BinopMatcher m(node);
578  Node* left = m.left().node();
579  Node* right = m.right().node();
580
581  if (g.CanBeImmediate(right)) {
582    selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
583                   g.UseImmediate(right));
584  } else {
585    selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
586                   g.UseFixed(right, rcx));
587  }
588}
589
590
591// Shared routine for multiple 64-bit shift operations.
592// TODO(bmeurer): Merge this with VisitWord32Shift using template magic?
593void VisitWord64Shift(InstructionSelector* selector, Node* node,
594                      ArchOpcode opcode) {
595  X64OperandGenerator g(selector);
596  Int64BinopMatcher m(node);
597  Node* left = m.left().node();
598  Node* right = m.right().node();
599
600  if (g.CanBeImmediate(right)) {
601    selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
602                   g.UseImmediate(right));
603  } else {
604    if (m.right().IsWord64And()) {
605      Int64BinopMatcher mright(right);
606      if (mright.right().Is(0x3F)) {
607        right = mright.left().node();
608      }
609    }
610    selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
611                   g.UseFixed(right, rcx));
612  }
613}
614
615void EmitLea(InstructionSelector* selector, InstructionCode opcode,
616             Node* result, Node* index, int scale, Node* base,
617             Node* displacement, DisplacementMode displacement_mode) {
618  X64OperandGenerator g(selector);
619
620  InstructionOperand inputs[4];
621  size_t input_count = 0;
622  AddressingMode mode =
623      g.GenerateMemoryOperandInputs(index, scale, base, displacement,
624                                    displacement_mode, inputs, &input_count);
625
626  DCHECK_NE(0u, input_count);
627  DCHECK_GE(arraysize(inputs), input_count);
628
629  InstructionOperand outputs[1];
630  outputs[0] = g.DefineAsRegister(result);
631
632  opcode = AddressingModeField::encode(mode) | opcode;
633
634  selector->Emit(opcode, 1, outputs, input_count, inputs);
635}
636
637}  // namespace
638
639
640void InstructionSelector::VisitWord32Shl(Node* node) {
641  Int32ScaleMatcher m(node, true);
642  if (m.matches()) {
643    Node* index = node->InputAt(0);
644    Node* base = m.power_of_two_plus_one() ? index : nullptr;
645    EmitLea(this, kX64Lea32, node, index, m.scale(), base, nullptr,
646            kPositiveDisplacement);
647    return;
648  }
649  VisitWord32Shift(this, node, kX64Shl32);
650}
651
652
653void InstructionSelector::VisitWord64Shl(Node* node) {
654  X64OperandGenerator g(this);
655  Int64ScaleMatcher m(node, true);
656  if (m.matches()) {
657    Node* index = node->InputAt(0);
658    Node* base = m.power_of_two_plus_one() ? index : nullptr;
659    EmitLea(this, kX64Lea, node, index, m.scale(), base, nullptr,
660            kPositiveDisplacement);
661    return;
662  } else {
663    Int64BinopMatcher m(node);
664    if ((m.left().IsChangeInt32ToInt64() ||
665         m.left().IsChangeUint32ToUint64()) &&
666        m.right().IsInRange(32, 63)) {
667      // There's no need to sign/zero-extend to 64-bit if we shift out the upper
668      // 32 bits anyway.
669      Emit(kX64Shl, g.DefineSameAsFirst(node),
670           g.UseRegister(m.left().node()->InputAt(0)),
671           g.UseImmediate(m.right().node()));
672      return;
673    }
674  }
675  VisitWord64Shift(this, node, kX64Shl);
676}
677
678
679void InstructionSelector::VisitWord32Shr(Node* node) {
680  VisitWord32Shift(this, node, kX64Shr32);
681}
682
683namespace {
684bool TryMatchLoadWord64AndShiftRight(InstructionSelector* selector, Node* node,
685                                     InstructionCode opcode) {
686  DCHECK(IrOpcode::kWord64Sar == node->opcode() ||
687         IrOpcode::kWord64Shr == node->opcode());
688  X64OperandGenerator g(selector);
689  Int64BinopMatcher m(node);
690  if (selector->CanCover(m.node(), m.left().node()) && m.left().IsLoad() &&
691      m.right().Is(32)) {
692    // Just load and sign-extend the interesting 4 bytes instead. This happens,
693    // for example, when we're loading and untagging SMIs.
694    BaseWithIndexAndDisplacement64Matcher mleft(m.left().node(),
695                                                AddressOption::kAllowAll);
696    if (mleft.matches() && (mleft.displacement() == nullptr ||
697                            g.CanBeImmediate(mleft.displacement()))) {
698      size_t input_count = 0;
699      InstructionOperand inputs[3];
700      AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
701          m.left().node(), inputs, &input_count);
702      if (mleft.displacement() == nullptr) {
703        // Make sure that the addressing mode indicates the presence of an
704        // immediate displacement. It seems that we never use M1 and M2, but we
705        // handle them here anyways.
706        switch (mode) {
707          case kMode_MR:
708            mode = kMode_MRI;
709            break;
710          case kMode_MR1:
711            mode = kMode_MR1I;
712            break;
713          case kMode_MR2:
714            mode = kMode_MR2I;
715            break;
716          case kMode_MR4:
717            mode = kMode_MR4I;
718            break;
719          case kMode_MR8:
720            mode = kMode_MR8I;
721            break;
722          case kMode_M1:
723            mode = kMode_M1I;
724            break;
725          case kMode_M2:
726            mode = kMode_M2I;
727            break;
728          case kMode_M4:
729            mode = kMode_M4I;
730            break;
731          case kMode_M8:
732            mode = kMode_M8I;
733            break;
734          case kMode_None:
735          case kMode_MRI:
736          case kMode_MR1I:
737          case kMode_MR2I:
738          case kMode_MR4I:
739          case kMode_MR8I:
740          case kMode_M1I:
741          case kMode_M2I:
742          case kMode_M4I:
743          case kMode_M8I:
744          case kMode_Root:
745            UNREACHABLE();
746        }
747        inputs[input_count++] = ImmediateOperand(ImmediateOperand::INLINE, 4);
748      } else {
749        int32_t displacement = g.GetImmediateIntegerValue(mleft.displacement());
750        inputs[input_count - 1] =
751            ImmediateOperand(ImmediateOperand::INLINE, displacement + 4);
752      }
753      InstructionOperand outputs[] = {g.DefineAsRegister(node)};
754      InstructionCode code = opcode | AddressingModeField::encode(mode);
755      selector->Emit(code, 1, outputs, input_count, inputs);
756      return true;
757    }
758  }
759  return false;
760}
761}  // namespace
762
763void InstructionSelector::VisitWord64Shr(Node* node) {
764  if (TryMatchLoadWord64AndShiftRight(this, node, kX64Movl)) return;
765  VisitWord64Shift(this, node, kX64Shr);
766}
767
768void InstructionSelector::VisitWord32Sar(Node* node) {
769  X64OperandGenerator g(this);
770  Int32BinopMatcher m(node);
771  if (CanCover(m.node(), m.left().node()) && m.left().IsWord32Shl()) {
772    Int32BinopMatcher mleft(m.left().node());
773    if (mleft.right().Is(16) && m.right().Is(16)) {
774      Emit(kX64Movsxwl, g.DefineAsRegister(node), g.Use(mleft.left().node()));
775      return;
776    } else if (mleft.right().Is(24) && m.right().Is(24)) {
777      Emit(kX64Movsxbl, g.DefineAsRegister(node), g.Use(mleft.left().node()));
778      return;
779    }
780  }
781  VisitWord32Shift(this, node, kX64Sar32);
782}
783
784void InstructionSelector::VisitWord64Sar(Node* node) {
785  if (TryMatchLoadWord64AndShiftRight(this, node, kX64Movsxlq)) return;
786  VisitWord64Shift(this, node, kX64Sar);
787}
788
789
790void InstructionSelector::VisitWord32Ror(Node* node) {
791  VisitWord32Shift(this, node, kX64Ror32);
792}
793
794
795void InstructionSelector::VisitWord64Ror(Node* node) {
796  VisitWord64Shift(this, node, kX64Ror);
797}
798
799
800void InstructionSelector::VisitWord64Clz(Node* node) {
801  X64OperandGenerator g(this);
802  Emit(kX64Lzcnt, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
803}
804
805
806void InstructionSelector::VisitWord32Clz(Node* node) {
807  X64OperandGenerator g(this);
808  Emit(kX64Lzcnt32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
809}
810
811
812void InstructionSelector::VisitWord64Ctz(Node* node) {
813  X64OperandGenerator g(this);
814  Emit(kX64Tzcnt, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
815}
816
817
818void InstructionSelector::VisitWord32Ctz(Node* node) {
819  X64OperandGenerator g(this);
820  Emit(kX64Tzcnt32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
821}
822
823
824void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); }
825
826
827void InstructionSelector::VisitWord64ReverseBits(Node* node) { UNREACHABLE(); }
828
829void InstructionSelector::VisitWord64ReverseBytes(Node* node) { UNREACHABLE(); }
830
831void InstructionSelector::VisitWord32ReverseBytes(Node* node) { UNREACHABLE(); }
832
833void InstructionSelector::VisitWord32Popcnt(Node* node) {
834  X64OperandGenerator g(this);
835  Emit(kX64Popcnt32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
836}
837
838
839void InstructionSelector::VisitWord64Popcnt(Node* node) {
840  X64OperandGenerator g(this);
841  Emit(kX64Popcnt, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
842}
843
844
845void InstructionSelector::VisitInt32Add(Node* node) {
846  X64OperandGenerator g(this);
847
848  // Try to match the Add to a leal pattern
849  BaseWithIndexAndDisplacement32Matcher m(node);
850  if (m.matches() &&
851      (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
852    EmitLea(this, kX64Lea32, node, m.index(), m.scale(), m.base(),
853            m.displacement(), m.displacement_mode());
854    return;
855  }
856
857  // No leal pattern match, use addl
858  VisitBinop(this, node, kX64Add32);
859}
860
861
862void InstructionSelector::VisitInt64Add(Node* node) {
863  X64OperandGenerator g(this);
864
865  // Try to match the Add to a leaq pattern
866  BaseWithIndexAndDisplacement64Matcher m(node);
867  if (m.matches() &&
868      (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
869    EmitLea(this, kX64Lea, node, m.index(), m.scale(), m.base(),
870            m.displacement(), m.displacement_mode());
871    return;
872  }
873
874  // No leal pattern match, use addq
875  VisitBinop(this, node, kX64Add);
876}
877
878
879void InstructionSelector::VisitInt64AddWithOverflow(Node* node) {
880  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
881    FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
882    return VisitBinop(this, node, kX64Add, &cont);
883  }
884  FlagsContinuation cont;
885  VisitBinop(this, node, kX64Add, &cont);
886}
887
888
889void InstructionSelector::VisitInt32Sub(Node* node) {
890  X64OperandGenerator g(this);
891  Int32BinopMatcher m(node);
892  if (m.left().Is(0)) {
893    Emit(kX64Neg32, g.DefineSameAsFirst(node), g.UseRegister(m.right().node()));
894  } else {
895    if (m.right().HasValue() && g.CanBeImmediate(m.right().node())) {
896      // Turn subtractions of constant values into immediate "leal" instructions
897      // by negating the value.
898      Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI),
899           g.DefineAsRegister(node), g.UseRegister(m.left().node()),
900           g.TempImmediate(-m.right().Value()));
901      return;
902    }
903    VisitBinop(this, node, kX64Sub32);
904  }
905}
906
907
908void InstructionSelector::VisitInt64Sub(Node* node) {
909  X64OperandGenerator g(this);
910  Int64BinopMatcher m(node);
911  if (m.left().Is(0)) {
912    Emit(kX64Neg, g.DefineSameAsFirst(node), g.UseRegister(m.right().node()));
913  } else {
914    if (m.right().HasValue() && g.CanBeImmediate(m.right().node())) {
915      // Turn subtractions of constant values into immediate "leaq" instructions
916      // by negating the value.
917      Emit(kX64Lea | AddressingModeField::encode(kMode_MRI),
918           g.DefineAsRegister(node), g.UseRegister(m.left().node()),
919           g.TempImmediate(-static_cast<int32_t>(m.right().Value())));
920      return;
921    }
922    VisitBinop(this, node, kX64Sub);
923  }
924}
925
926
927void InstructionSelector::VisitInt64SubWithOverflow(Node* node) {
928  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
929    FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
930    return VisitBinop(this, node, kX64Sub, &cont);
931  }
932  FlagsContinuation cont;
933  VisitBinop(this, node, kX64Sub, &cont);
934}
935
936
937namespace {
938
939void VisitMul(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
940  X64OperandGenerator g(selector);
941  Int32BinopMatcher m(node);
942  Node* left = m.left().node();
943  Node* right = m.right().node();
944  if (g.CanBeImmediate(right)) {
945    selector->Emit(opcode, g.DefineAsRegister(node), g.Use(left),
946                   g.UseImmediate(right));
947  } else {
948    if (g.CanBeBetterLeftOperand(right)) {
949      std::swap(left, right);
950    }
951    selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
952                   g.Use(right));
953  }
954}
955
956void VisitMulHigh(InstructionSelector* selector, Node* node,
957                  ArchOpcode opcode) {
958  X64OperandGenerator g(selector);
959  Node* left = node->InputAt(0);
960  Node* right = node->InputAt(1);
961  if (selector->IsLive(left) && !selector->IsLive(right)) {
962    std::swap(left, right);
963  }
964  InstructionOperand temps[] = {g.TempRegister(rax)};
965  // TODO(turbofan): We use UseUniqueRegister here to improve register
966  // allocation.
967  selector->Emit(opcode, g.DefineAsFixed(node, rdx), g.UseFixed(left, rax),
968                 g.UseUniqueRegister(right), arraysize(temps), temps);
969}
970
971
972void VisitDiv(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
973  X64OperandGenerator g(selector);
974  InstructionOperand temps[] = {g.TempRegister(rdx)};
975  selector->Emit(
976      opcode, g.DefineAsFixed(node, rax), g.UseFixed(node->InputAt(0), rax),
977      g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
978}
979
980
981void VisitMod(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
982  X64OperandGenerator g(selector);
983  InstructionOperand temps[] = {g.TempRegister(rax)};
984  selector->Emit(
985      opcode, g.DefineAsFixed(node, rdx), g.UseFixed(node->InputAt(0), rax),
986      g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
987}
988
989}  // namespace
990
991
992void InstructionSelector::VisitInt32Mul(Node* node) {
993  Int32ScaleMatcher m(node, true);
994  if (m.matches()) {
995    Node* index = node->InputAt(0);
996    Node* base = m.power_of_two_plus_one() ? index : nullptr;
997    EmitLea(this, kX64Lea32, node, index, m.scale(), base, nullptr,
998            kPositiveDisplacement);
999    return;
1000  }
1001  VisitMul(this, node, kX64Imul32);
1002}
1003
1004void InstructionSelector::VisitInt32MulWithOverflow(Node* node) {
1005  // TODO(mvstanton): Use Int32ScaleMatcher somehow.
1006  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1007    FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1008    return VisitBinop(this, node, kX64Imul32, &cont);
1009  }
1010  FlagsContinuation cont;
1011  VisitBinop(this, node, kX64Imul32, &cont);
1012}
1013
1014void InstructionSelector::VisitInt64Mul(Node* node) {
1015  VisitMul(this, node, kX64Imul);
1016}
1017
1018void InstructionSelector::VisitInt32MulHigh(Node* node) {
1019  VisitMulHigh(this, node, kX64ImulHigh32);
1020}
1021
1022
1023void InstructionSelector::VisitInt32Div(Node* node) {
1024  VisitDiv(this, node, kX64Idiv32);
1025}
1026
1027
1028void InstructionSelector::VisitInt64Div(Node* node) {
1029  VisitDiv(this, node, kX64Idiv);
1030}
1031
1032
1033void InstructionSelector::VisitUint32Div(Node* node) {
1034  VisitDiv(this, node, kX64Udiv32);
1035}
1036
1037
1038void InstructionSelector::VisitUint64Div(Node* node) {
1039  VisitDiv(this, node, kX64Udiv);
1040}
1041
1042
1043void InstructionSelector::VisitInt32Mod(Node* node) {
1044  VisitMod(this, node, kX64Idiv32);
1045}
1046
1047
1048void InstructionSelector::VisitInt64Mod(Node* node) {
1049  VisitMod(this, node, kX64Idiv);
1050}
1051
1052
1053void InstructionSelector::VisitUint32Mod(Node* node) {
1054  VisitMod(this, node, kX64Udiv32);
1055}
1056
1057
1058void InstructionSelector::VisitUint64Mod(Node* node) {
1059  VisitMod(this, node, kX64Udiv);
1060}
1061
1062
1063void InstructionSelector::VisitUint32MulHigh(Node* node) {
1064  VisitMulHigh(this, node, kX64UmulHigh32);
1065}
1066
1067
1068void InstructionSelector::VisitChangeFloat32ToFloat64(Node* node) {
1069  X64OperandGenerator g(this);
1070  Emit(kSSEFloat32ToFloat64, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1071}
1072
1073
1074void InstructionSelector::VisitChangeInt32ToFloat64(Node* node) {
1075  X64OperandGenerator g(this);
1076  Emit(kSSEInt32ToFloat64, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1077}
1078
1079
1080void InstructionSelector::VisitChangeUint32ToFloat64(Node* node) {
1081  X64OperandGenerator g(this);
1082  Emit(kSSEUint32ToFloat64, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1083}
1084
1085
1086void InstructionSelector::VisitChangeFloat64ToInt32(Node* node) {
1087  X64OperandGenerator g(this);
1088  Emit(kSSEFloat64ToInt32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1089}
1090
1091
1092void InstructionSelector::VisitChangeFloat64ToUint32(Node* node) {
1093  X64OperandGenerator g(this);
1094  Emit(kSSEFloat64ToUint32 | MiscField::encode(1), g.DefineAsRegister(node),
1095       g.Use(node->InputAt(0)));
1096}
1097
1098void InstructionSelector::VisitTruncateFloat64ToUint32(Node* node) {
1099  X64OperandGenerator g(this);
1100  Emit(kSSEFloat64ToUint32 | MiscField::encode(0), g.DefineAsRegister(node),
1101       g.Use(node->InputAt(0)));
1102}
1103
1104void InstructionSelector::VisitTruncateFloat32ToInt32(Node* node) {
1105  X64OperandGenerator g(this);
1106  Emit(kSSEFloat32ToInt32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1107}
1108
1109
1110void InstructionSelector::VisitTruncateFloat32ToUint32(Node* node) {
1111  X64OperandGenerator g(this);
1112  Emit(kSSEFloat32ToUint32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1113}
1114
1115
1116void InstructionSelector::VisitTryTruncateFloat32ToInt64(Node* node) {
1117  X64OperandGenerator g(this);
1118  InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1119  InstructionOperand outputs[2];
1120  size_t output_count = 0;
1121  outputs[output_count++] = g.DefineAsRegister(node);
1122
1123  Node* success_output = NodeProperties::FindProjection(node, 1);
1124  if (success_output) {
1125    outputs[output_count++] = g.DefineAsRegister(success_output);
1126  }
1127
1128  Emit(kSSEFloat32ToInt64, output_count, outputs, 1, inputs);
1129}
1130
1131
1132void InstructionSelector::VisitTryTruncateFloat64ToInt64(Node* node) {
1133  X64OperandGenerator g(this);
1134  InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1135  InstructionOperand outputs[2];
1136  size_t output_count = 0;
1137  outputs[output_count++] = g.DefineAsRegister(node);
1138
1139  Node* success_output = NodeProperties::FindProjection(node, 1);
1140  if (success_output) {
1141    outputs[output_count++] = g.DefineAsRegister(success_output);
1142  }
1143
1144  Emit(kSSEFloat64ToInt64, output_count, outputs, 1, inputs);
1145}
1146
1147
1148void InstructionSelector::VisitTryTruncateFloat32ToUint64(Node* node) {
1149  X64OperandGenerator g(this);
1150  InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1151  InstructionOperand outputs[2];
1152  size_t output_count = 0;
1153  outputs[output_count++] = g.DefineAsRegister(node);
1154
1155  Node* success_output = NodeProperties::FindProjection(node, 1);
1156  if (success_output) {
1157    outputs[output_count++] = g.DefineAsRegister(success_output);
1158  }
1159
1160  Emit(kSSEFloat32ToUint64, output_count, outputs, 1, inputs);
1161}
1162
1163
1164void InstructionSelector::VisitTryTruncateFloat64ToUint64(Node* node) {
1165  X64OperandGenerator g(this);
1166  InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1167  InstructionOperand outputs[2];
1168  size_t output_count = 0;
1169  outputs[output_count++] = g.DefineAsRegister(node);
1170
1171  Node* success_output = NodeProperties::FindProjection(node, 1);
1172  if (success_output) {
1173    outputs[output_count++] = g.DefineAsRegister(success_output);
1174  }
1175
1176  Emit(kSSEFloat64ToUint64, output_count, outputs, 1, inputs);
1177}
1178
1179
1180void InstructionSelector::VisitChangeInt32ToInt64(Node* node) {
1181  X64OperandGenerator g(this);
1182  Node* const value = node->InputAt(0);
1183  if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
1184    LoadRepresentation load_rep = LoadRepresentationOf(value->op());
1185    MachineRepresentation rep = load_rep.representation();
1186    InstructionCode opcode = kArchNop;
1187    switch (rep) {
1188      case MachineRepresentation::kBit:  // Fall through.
1189      case MachineRepresentation::kWord8:
1190        opcode = load_rep.IsSigned() ? kX64Movsxbq : kX64Movzxbq;
1191        break;
1192      case MachineRepresentation::kWord16:
1193        opcode = load_rep.IsSigned() ? kX64Movsxwq : kX64Movzxwq;
1194        break;
1195      case MachineRepresentation::kWord32:
1196        opcode = load_rep.IsSigned() ? kX64Movsxlq : kX64Movl;
1197        break;
1198      default:
1199        UNREACHABLE();
1200        return;
1201    }
1202    InstructionOperand outputs[] = {g.DefineAsRegister(node)};
1203    size_t input_count = 0;
1204    InstructionOperand inputs[3];
1205    AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1206        node->InputAt(0), inputs, &input_count);
1207    opcode |= AddressingModeField::encode(mode);
1208    Emit(opcode, 1, outputs, input_count, inputs);
1209  } else {
1210    Emit(kX64Movsxlq, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1211  }
1212}
1213
1214namespace {
1215
1216bool ZeroExtendsWord32ToWord64(Node* node) {
1217  switch (node->opcode()) {
1218    case IrOpcode::kWord32And:
1219    case IrOpcode::kWord32Or:
1220    case IrOpcode::kWord32Xor:
1221    case IrOpcode::kWord32Shl:
1222    case IrOpcode::kWord32Shr:
1223    case IrOpcode::kWord32Sar:
1224    case IrOpcode::kWord32Ror:
1225    case IrOpcode::kWord32Equal:
1226    case IrOpcode::kInt32Add:
1227    case IrOpcode::kInt32Sub:
1228    case IrOpcode::kInt32Mul:
1229    case IrOpcode::kInt32MulHigh:
1230    case IrOpcode::kInt32Div:
1231    case IrOpcode::kInt32LessThan:
1232    case IrOpcode::kInt32LessThanOrEqual:
1233    case IrOpcode::kInt32Mod:
1234    case IrOpcode::kUint32Div:
1235    case IrOpcode::kUint32LessThan:
1236    case IrOpcode::kUint32LessThanOrEqual:
1237    case IrOpcode::kUint32Mod:
1238    case IrOpcode::kUint32MulHigh:
1239      // These 32-bit operations implicitly zero-extend to 64-bit on x64, so the
1240      // zero-extension is a no-op.
1241      return true;
1242    case IrOpcode::kProjection: {
1243      Node* const value = node->InputAt(0);
1244      switch (value->opcode()) {
1245        case IrOpcode::kInt32AddWithOverflow:
1246        case IrOpcode::kInt32SubWithOverflow:
1247        case IrOpcode::kInt32MulWithOverflow:
1248          return true;
1249        default:
1250          return false;
1251      }
1252    }
1253    case IrOpcode::kLoad: {
1254      // The movzxbl/movsxbl/movzxwl/movsxwl operations implicitly zero-extend
1255      // to 64-bit on x64,
1256      // so the zero-extension is a no-op.
1257      LoadRepresentation load_rep = LoadRepresentationOf(node->op());
1258      switch (load_rep.representation()) {
1259        case MachineRepresentation::kWord8:
1260        case MachineRepresentation::kWord16:
1261          return true;
1262        default:
1263          return false;
1264      }
1265    }
1266    default:
1267      return false;
1268  }
1269}
1270
1271}  // namespace
1272
1273void InstructionSelector::VisitChangeUint32ToUint64(Node* node) {
1274  X64OperandGenerator g(this);
1275  Node* value = node->InputAt(0);
1276  if (ZeroExtendsWord32ToWord64(value)) {
1277    // These 32-bit operations implicitly zero-extend to 64-bit on x64, so the
1278    // zero-extension is a no-op.
1279    return EmitIdentity(node);
1280  }
1281  Emit(kX64Movl, g.DefineAsRegister(node), g.Use(value));
1282}
1283
1284
1285namespace {
1286
1287void VisitRO(InstructionSelector* selector, Node* node,
1288             InstructionCode opcode) {
1289  X64OperandGenerator g(selector);
1290  selector->Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1291}
1292
1293
1294void VisitRR(InstructionSelector* selector, Node* node,
1295             InstructionCode opcode) {
1296  X64OperandGenerator g(selector);
1297  selector->Emit(opcode, g.DefineAsRegister(node),
1298                 g.UseRegister(node->InputAt(0)));
1299}
1300
1301void VisitRRO(InstructionSelector* selector, Node* node,
1302              InstructionCode opcode) {
1303  X64OperandGenerator g(selector);
1304  selector->Emit(opcode, g.DefineSameAsFirst(node),
1305                 g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)));
1306}
1307
1308void VisitFloatBinop(InstructionSelector* selector, Node* node,
1309                     ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
1310  X64OperandGenerator g(selector);
1311  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
1312  InstructionOperand operand1 = g.Use(node->InputAt(1));
1313  if (selector->IsSupported(AVX)) {
1314    selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1);
1315  } else {
1316    selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
1317  }
1318}
1319
1320
1321void VisitFloatUnop(InstructionSelector* selector, Node* node, Node* input,
1322                    ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
1323  X64OperandGenerator g(selector);
1324  if (selector->IsSupported(AVX)) {
1325    selector->Emit(avx_opcode, g.DefineAsRegister(node), g.Use(input));
1326  } else {
1327    selector->Emit(sse_opcode, g.DefineSameAsFirst(node), g.UseRegister(input));
1328  }
1329}
1330
1331}  // namespace
1332
1333
1334void InstructionSelector::VisitTruncateFloat64ToFloat32(Node* node) {
1335  VisitRO(this, node, kSSEFloat64ToFloat32);
1336}
1337
1338void InstructionSelector::VisitTruncateFloat64ToWord32(Node* node) {
1339  VisitRR(this, node, kArchTruncateDoubleToI);
1340}
1341
1342
1343void InstructionSelector::VisitTruncateInt64ToInt32(Node* node) {
1344  X64OperandGenerator g(this);
1345  Node* value = node->InputAt(0);
1346  if (CanCover(node, value)) {
1347    switch (value->opcode()) {
1348      case IrOpcode::kWord64Sar:
1349      case IrOpcode::kWord64Shr: {
1350        Int64BinopMatcher m(value);
1351        if (m.right().Is(32)) {
1352          if (TryMatchLoadWord64AndShiftRight(this, value, kX64Movl)) {
1353            return EmitIdentity(node);
1354          }
1355          Emit(kX64Shr, g.DefineSameAsFirst(node),
1356               g.UseRegister(m.left().node()), g.TempImmediate(32));
1357          return;
1358        }
1359        break;
1360      }
1361      default:
1362        break;
1363    }
1364  }
1365  Emit(kX64Movl, g.DefineAsRegister(node), g.Use(value));
1366}
1367
1368void InstructionSelector::VisitRoundFloat64ToInt32(Node* node) {
1369  VisitRO(this, node, kSSEFloat64ToInt32);
1370}
1371
1372void InstructionSelector::VisitRoundInt32ToFloat32(Node* node) {
1373  X64OperandGenerator g(this);
1374  Emit(kSSEInt32ToFloat32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1375}
1376
1377
1378void InstructionSelector::VisitRoundInt64ToFloat32(Node* node) {
1379  X64OperandGenerator g(this);
1380  Emit(kSSEInt64ToFloat32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1381}
1382
1383
1384void InstructionSelector::VisitRoundInt64ToFloat64(Node* node) {
1385  X64OperandGenerator g(this);
1386  Emit(kSSEInt64ToFloat64, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1387}
1388
1389
1390void InstructionSelector::VisitRoundUint32ToFloat32(Node* node) {
1391  X64OperandGenerator g(this);
1392  Emit(kSSEUint32ToFloat32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1393}
1394
1395
1396void InstructionSelector::VisitRoundUint64ToFloat32(Node* node) {
1397  X64OperandGenerator g(this);
1398  InstructionOperand temps[] = {g.TempRegister()};
1399  Emit(kSSEUint64ToFloat32, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
1400       arraysize(temps), temps);
1401}
1402
1403
1404void InstructionSelector::VisitRoundUint64ToFloat64(Node* node) {
1405  X64OperandGenerator g(this);
1406  InstructionOperand temps[] = {g.TempRegister()};
1407  Emit(kSSEUint64ToFloat64, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
1408       arraysize(temps), temps);
1409}
1410
1411
1412void InstructionSelector::VisitBitcastFloat32ToInt32(Node* node) {
1413  X64OperandGenerator g(this);
1414  Emit(kX64BitcastFI, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1415}
1416
1417
1418void InstructionSelector::VisitBitcastFloat64ToInt64(Node* node) {
1419  X64OperandGenerator g(this);
1420  Emit(kX64BitcastDL, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1421}
1422
1423
1424void InstructionSelector::VisitBitcastInt32ToFloat32(Node* node) {
1425  X64OperandGenerator g(this);
1426  Emit(kX64BitcastIF, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1427}
1428
1429
1430void InstructionSelector::VisitBitcastInt64ToFloat64(Node* node) {
1431  X64OperandGenerator g(this);
1432  Emit(kX64BitcastLD, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1433}
1434
1435
1436void InstructionSelector::VisitFloat32Add(Node* node) {
1437  VisitFloatBinop(this, node, kAVXFloat32Add, kSSEFloat32Add);
1438}
1439
1440
1441void InstructionSelector::VisitFloat32Sub(Node* node) {
1442  VisitFloatBinop(this, node, kAVXFloat32Sub, kSSEFloat32Sub);
1443}
1444
1445void InstructionSelector::VisitFloat32Mul(Node* node) {
1446  VisitFloatBinop(this, node, kAVXFloat32Mul, kSSEFloat32Mul);
1447}
1448
1449
1450void InstructionSelector::VisitFloat32Div(Node* node) {
1451  VisitFloatBinop(this, node, kAVXFloat32Div, kSSEFloat32Div);
1452}
1453
1454
1455void InstructionSelector::VisitFloat32Abs(Node* node) {
1456  VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat32Abs, kSSEFloat32Abs);
1457}
1458
1459
1460void InstructionSelector::VisitFloat32Sqrt(Node* node) {
1461  VisitRO(this, node, kSSEFloat32Sqrt);
1462}
1463
1464void InstructionSelector::VisitFloat32Max(Node* node) {
1465  VisitRRO(this, node, kSSEFloat32Max);
1466}
1467
1468void InstructionSelector::VisitFloat32Min(Node* node) {
1469  VisitRRO(this, node, kSSEFloat32Min);
1470}
1471
1472void InstructionSelector::VisitFloat64Add(Node* node) {
1473  VisitFloatBinop(this, node, kAVXFloat64Add, kSSEFloat64Add);
1474}
1475
1476
1477void InstructionSelector::VisitFloat64Sub(Node* node) {
1478  VisitFloatBinop(this, node, kAVXFloat64Sub, kSSEFloat64Sub);
1479}
1480
1481void InstructionSelector::VisitFloat64Mul(Node* node) {
1482  VisitFloatBinop(this, node, kAVXFloat64Mul, kSSEFloat64Mul);
1483}
1484
1485
1486void InstructionSelector::VisitFloat64Div(Node* node) {
1487  VisitFloatBinop(this, node, kAVXFloat64Div, kSSEFloat64Div);
1488}
1489
1490
1491void InstructionSelector::VisitFloat64Mod(Node* node) {
1492  X64OperandGenerator g(this);
1493  InstructionOperand temps[] = {g.TempRegister(rax)};
1494  Emit(kSSEFloat64Mod, g.DefineSameAsFirst(node),
1495       g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)), 1,
1496       temps);
1497}
1498
1499
1500void InstructionSelector::VisitFloat64Max(Node* node) {
1501  VisitRRO(this, node, kSSEFloat64Max);
1502}
1503
1504
1505void InstructionSelector::VisitFloat64Min(Node* node) {
1506  VisitRRO(this, node, kSSEFloat64Min);
1507}
1508
1509
1510void InstructionSelector::VisitFloat64Abs(Node* node) {
1511  VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat64Abs, kSSEFloat64Abs);
1512}
1513
1514void InstructionSelector::VisitFloat64Sqrt(Node* node) {
1515  VisitRO(this, node, kSSEFloat64Sqrt);
1516}
1517
1518
1519void InstructionSelector::VisitFloat32RoundDown(Node* node) {
1520  VisitRR(this, node, kSSEFloat32Round | MiscField::encode(kRoundDown));
1521}
1522
1523
1524void InstructionSelector::VisitFloat64RoundDown(Node* node) {
1525  VisitRR(this, node, kSSEFloat64Round | MiscField::encode(kRoundDown));
1526}
1527
1528
1529void InstructionSelector::VisitFloat32RoundUp(Node* node) {
1530  VisitRR(this, node, kSSEFloat32Round | MiscField::encode(kRoundUp));
1531}
1532
1533
1534void InstructionSelector::VisitFloat64RoundUp(Node* node) {
1535  VisitRR(this, node, kSSEFloat64Round | MiscField::encode(kRoundUp));
1536}
1537
1538
1539void InstructionSelector::VisitFloat32RoundTruncate(Node* node) {
1540  VisitRR(this, node, kSSEFloat32Round | MiscField::encode(kRoundToZero));
1541}
1542
1543
1544void InstructionSelector::VisitFloat64RoundTruncate(Node* node) {
1545  VisitRR(this, node, kSSEFloat64Round | MiscField::encode(kRoundToZero));
1546}
1547
1548
1549void InstructionSelector::VisitFloat64RoundTiesAway(Node* node) {
1550  UNREACHABLE();
1551}
1552
1553
1554void InstructionSelector::VisitFloat32RoundTiesEven(Node* node) {
1555  VisitRR(this, node, kSSEFloat32Round | MiscField::encode(kRoundToNearest));
1556}
1557
1558
1559void InstructionSelector::VisitFloat64RoundTiesEven(Node* node) {
1560  VisitRR(this, node, kSSEFloat64Round | MiscField::encode(kRoundToNearest));
1561}
1562
1563void InstructionSelector::VisitFloat32Neg(Node* node) {
1564  VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat32Neg, kSSEFloat32Neg);
1565}
1566
1567void InstructionSelector::VisitFloat64Neg(Node* node) {
1568  VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat64Neg, kSSEFloat64Neg);
1569}
1570
1571void InstructionSelector::VisitFloat64Ieee754Binop(Node* node,
1572                                                   InstructionCode opcode) {
1573  X64OperandGenerator g(this);
1574  Emit(opcode, g.DefineAsFixed(node, xmm0), g.UseFixed(node->InputAt(0), xmm0),
1575       g.UseFixed(node->InputAt(1), xmm1))
1576      ->MarkAsCall();
1577}
1578
1579void InstructionSelector::VisitFloat64Ieee754Unop(Node* node,
1580                                                  InstructionCode opcode) {
1581  X64OperandGenerator g(this);
1582  Emit(opcode, g.DefineAsFixed(node, xmm0), g.UseFixed(node->InputAt(0), xmm0))
1583      ->MarkAsCall();
1584}
1585
1586void InstructionSelector::EmitPrepareArguments(
1587    ZoneVector<PushParameter>* arguments, const CallDescriptor* descriptor,
1588    Node* node) {
1589  X64OperandGenerator g(this);
1590
1591  // Prepare for C function call.
1592  if (descriptor->IsCFunctionCall()) {
1593    Emit(kArchPrepareCallCFunction |
1594             MiscField::encode(static_cast<int>(descriptor->ParameterCount())),
1595         0, nullptr, 0, nullptr);
1596
1597    // Poke any stack arguments.
1598    for (size_t n = 0; n < arguments->size(); ++n) {
1599      PushParameter input = (*arguments)[n];
1600      if (input.node()) {
1601        int slot = static_cast<int>(n);
1602        InstructionOperand value = g.CanBeImmediate(input.node())
1603                                       ? g.UseImmediate(input.node())
1604                                       : g.UseRegister(input.node());
1605        Emit(kX64Poke | MiscField::encode(slot), g.NoOutput(), value);
1606      }
1607    }
1608  } else {
1609    // Push any stack arguments.
1610    for (PushParameter input : base::Reversed(*arguments)) {
1611      // TODO(titzer): X64Push cannot handle stack->stack double moves
1612      // because there is no way to encode fixed double slots.
1613      InstructionOperand value =
1614          g.CanBeImmediate(input.node())
1615              ? g.UseImmediate(input.node())
1616              : IsSupported(ATOM) ||
1617                        sequence()->IsFP(GetVirtualRegister(input.node()))
1618                    ? g.UseRegister(input.node())
1619                    : g.Use(input.node());
1620      Emit(kX64Push, g.NoOutput(), value);
1621    }
1622  }
1623}
1624
1625
1626bool InstructionSelector::IsTailCallAddressImmediate() { return true; }
1627
1628int InstructionSelector::GetTempsCountForTailCallFromJSFunction() { return 3; }
1629
1630namespace {
1631
1632void VisitCompareWithMemoryOperand(InstructionSelector* selector,
1633                                   InstructionCode opcode, Node* left,
1634                                   InstructionOperand right,
1635                                   FlagsContinuation* cont) {
1636  DCHECK(left->opcode() == IrOpcode::kLoad);
1637  X64OperandGenerator g(selector);
1638  size_t input_count = 0;
1639  InstructionOperand inputs[6];
1640  AddressingMode addressing_mode =
1641      g.GetEffectiveAddressMemoryOperand(left, inputs, &input_count);
1642  opcode |= AddressingModeField::encode(addressing_mode);
1643  opcode = cont->Encode(opcode);
1644  inputs[input_count++] = right;
1645
1646  if (cont->IsBranch()) {
1647    inputs[input_count++] = g.Label(cont->true_block());
1648    inputs[input_count++] = g.Label(cont->false_block());
1649    selector->Emit(opcode, 0, nullptr, input_count, inputs);
1650  } else if (cont->IsDeoptimize()) {
1651    selector->EmitDeoptimize(opcode, 0, nullptr, input_count, inputs,
1652                             cont->reason(), cont->frame_state());
1653  } else {
1654    DCHECK(cont->IsSet());
1655    InstructionOperand output = g.DefineAsRegister(cont->result());
1656    selector->Emit(opcode, 1, &output, input_count, inputs);
1657  }
1658}
1659
1660// Shared routine for multiple compare operations.
1661void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1662                  InstructionOperand left, InstructionOperand right,
1663                  FlagsContinuation* cont) {
1664  X64OperandGenerator g(selector);
1665  opcode = cont->Encode(opcode);
1666  if (cont->IsBranch()) {
1667    selector->Emit(opcode, g.NoOutput(), left, right,
1668                   g.Label(cont->true_block()), g.Label(cont->false_block()));
1669  } else if (cont->IsDeoptimize()) {
1670    selector->EmitDeoptimize(opcode, g.NoOutput(), left, right, cont->reason(),
1671                             cont->frame_state());
1672  } else {
1673    DCHECK(cont->IsSet());
1674    selector->Emit(opcode, g.DefineAsRegister(cont->result()), left, right);
1675  }
1676}
1677
1678
1679// Shared routine for multiple compare operations.
1680void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1681                  Node* left, Node* right, FlagsContinuation* cont,
1682                  bool commutative) {
1683  X64OperandGenerator g(selector);
1684  if (commutative && g.CanBeBetterLeftOperand(right)) {
1685    std::swap(left, right);
1686  }
1687  VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
1688}
1689
1690// Tries to match the size of the given opcode to that of the operands, if
1691// possible.
1692InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
1693                                    Node* right, FlagsContinuation* cont) {
1694  // Currently, if one of the two operands is not a Load, we don't know what its
1695  // machine representation is, so we bail out.
1696  // TODO(epertoso): we can probably get some size information out of immediates
1697  // and phi nodes.
1698  if (left->opcode() != IrOpcode::kLoad || right->opcode() != IrOpcode::kLoad) {
1699    return opcode;
1700  }
1701  // If the load representations don't match, both operands will be
1702  // zero/sign-extended to 32bit.
1703  MachineType left_type = LoadRepresentationOf(left->op());
1704  MachineType right_type = LoadRepresentationOf(right->op());
1705  if (left_type == right_type) {
1706    switch (left_type.representation()) {
1707      case MachineRepresentation::kBit:
1708      case MachineRepresentation::kWord8: {
1709        if (opcode == kX64Test32) return kX64Test8;
1710        if (opcode == kX64Cmp32) {
1711          if (left_type.semantic() == MachineSemantic::kUint32) {
1712            cont->OverwriteUnsignedIfSigned();
1713          } else {
1714            CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1715          }
1716          return kX64Cmp8;
1717        }
1718        break;
1719      }
1720      case MachineRepresentation::kWord16:
1721        if (opcode == kX64Test32) return kX64Test16;
1722        if (opcode == kX64Cmp32) {
1723          if (left_type.semantic() == MachineSemantic::kUint32) {
1724            cont->OverwriteUnsignedIfSigned();
1725          } else {
1726            CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1727          }
1728          return kX64Cmp16;
1729        }
1730        break;
1731      default:
1732        break;
1733    }
1734  }
1735  return opcode;
1736}
1737
1738// Shared routine for multiple word compare operations.
1739void VisitWordCompare(InstructionSelector* selector, Node* node,
1740                      InstructionCode opcode, FlagsContinuation* cont) {
1741  X64OperandGenerator g(selector);
1742  Node* left = node->InputAt(0);
1743  Node* right = node->InputAt(1);
1744
1745  opcode = TryNarrowOpcodeSize(opcode, left, right, cont);
1746
1747  // If one of the two inputs is an immediate, make sure it's on the right, or
1748  // if one of the two inputs is a memory operand, make sure it's on the left.
1749  int effect_level = selector->GetEffectLevel(node);
1750  if (cont->IsBranch()) {
1751    effect_level = selector->GetEffectLevel(
1752        cont->true_block()->PredecessorAt(0)->control_input());
1753  }
1754
1755  if ((!g.CanBeImmediate(right) && g.CanBeImmediate(left)) ||
1756      (g.CanBeMemoryOperand(opcode, node, right, effect_level) &&
1757       !g.CanBeMemoryOperand(opcode, node, left, effect_level))) {
1758    if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1759    std::swap(left, right);
1760  }
1761
1762  // Match immediates on right side of comparison.
1763  if (g.CanBeImmediate(right)) {
1764    if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
1765      return VisitCompareWithMemoryOperand(selector, opcode, left,
1766                                           g.UseImmediate(right), cont);
1767    }
1768    return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right),
1769                        cont);
1770  }
1771
1772  // Match memory operands on left side of comparison.
1773  if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
1774    return VisitCompareWithMemoryOperand(selector, opcode, left,
1775                                         g.UseRegister(right), cont);
1776  }
1777
1778  if (g.CanBeBetterLeftOperand(right)) {
1779    if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1780    std::swap(left, right);
1781  }
1782
1783  return VisitCompare(selector, opcode, left, right, cont,
1784                      node->op()->HasProperty(Operator::kCommutative));
1785}
1786
1787// Shared routine for 64-bit word comparison operations.
1788void VisitWord64Compare(InstructionSelector* selector, Node* node,
1789                        FlagsContinuation* cont) {
1790  X64OperandGenerator g(selector);
1791  if (selector->CanUseRootsRegister()) {
1792    Heap* const heap = selector->isolate()->heap();
1793    Heap::RootListIndex root_index;
1794    HeapObjectBinopMatcher m(node);
1795    if (m.right().HasValue() &&
1796        heap->IsRootHandle(m.right().Value(), &root_index)) {
1797      if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1798      InstructionCode opcode =
1799          kX64Cmp | AddressingModeField::encode(kMode_Root);
1800      return VisitCompare(
1801          selector, opcode,
1802          g.TempImmediate((root_index * kPointerSize) - kRootRegisterBias),
1803          g.UseRegister(m.left().node()), cont);
1804    } else if (m.left().HasValue() &&
1805               heap->IsRootHandle(m.left().Value(), &root_index)) {
1806      InstructionCode opcode =
1807          kX64Cmp | AddressingModeField::encode(kMode_Root);
1808      return VisitCompare(
1809          selector, opcode,
1810          g.TempImmediate((root_index * kPointerSize) - kRootRegisterBias),
1811          g.UseRegister(m.right().node()), cont);
1812    }
1813  }
1814  Int64BinopMatcher m(node);
1815  if (m.left().IsLoad() && m.right().IsLoadStackPointer()) {
1816    LoadMatcher<ExternalReferenceMatcher> mleft(m.left().node());
1817    ExternalReference js_stack_limit =
1818        ExternalReference::address_of_stack_limit(selector->isolate());
1819    if (mleft.object().Is(js_stack_limit) && mleft.index().Is(0)) {
1820      // Compare(Load(js_stack_limit), LoadStackPointer)
1821      if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1822      InstructionCode opcode = cont->Encode(kX64StackCheck);
1823      if (cont->IsBranch()) {
1824        selector->Emit(opcode, g.NoOutput(), g.Label(cont->true_block()),
1825                       g.Label(cont->false_block()));
1826      } else if (cont->IsDeoptimize()) {
1827        selector->EmitDeoptimize(opcode, 0, nullptr, 0, nullptr, cont->reason(),
1828                                 cont->frame_state());
1829      } else {
1830        DCHECK(cont->IsSet());
1831        selector->Emit(opcode, g.DefineAsRegister(cont->result()));
1832      }
1833      return;
1834    }
1835  }
1836  VisitWordCompare(selector, node, kX64Cmp, cont);
1837}
1838
1839
1840// Shared routine for comparison with zero.
1841void VisitCompareZero(InstructionSelector* selector, Node* node,
1842                      InstructionCode opcode, FlagsContinuation* cont) {
1843  X64OperandGenerator g(selector);
1844  VisitCompare(selector, opcode, g.Use(node), g.TempImmediate(0), cont);
1845}
1846
1847
1848// Shared routine for multiple float32 compare operations (inputs commuted).
1849void VisitFloat32Compare(InstructionSelector* selector, Node* node,
1850                         FlagsContinuation* cont) {
1851  Node* const left = node->InputAt(0);
1852  Node* const right = node->InputAt(1);
1853  InstructionCode const opcode =
1854      selector->IsSupported(AVX) ? kAVXFloat32Cmp : kSSEFloat32Cmp;
1855  VisitCompare(selector, opcode, right, left, cont, false);
1856}
1857
1858
1859// Shared routine for multiple float64 compare operations (inputs commuted).
1860void VisitFloat64Compare(InstructionSelector* selector, Node* node,
1861                         FlagsContinuation* cont) {
1862  Node* const left = node->InputAt(0);
1863  Node* const right = node->InputAt(1);
1864  InstructionCode const opcode =
1865      selector->IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
1866  VisitCompare(selector, opcode, right, left, cont, false);
1867}
1868
1869// Shared routine for word comparison against zero.
1870void VisitWordCompareZero(InstructionSelector* selector, Node* user,
1871                          Node* value, FlagsContinuation* cont) {
1872  // Try to combine with comparisons against 0 by simply inverting the branch.
1873  while (value->opcode() == IrOpcode::kWord32Equal &&
1874         selector->CanCover(user, value)) {
1875    Int32BinopMatcher m(value);
1876    if (!m.right().Is(0)) break;
1877
1878    user = value;
1879    value = m.left().node();
1880    cont->Negate();
1881  }
1882
1883  if (selector->CanCover(user, value)) {
1884    switch (value->opcode()) {
1885      case IrOpcode::kWord32Equal:
1886        cont->OverwriteAndNegateIfEqual(kEqual);
1887        return VisitWordCompare(selector, value, kX64Cmp32, cont);
1888      case IrOpcode::kInt32LessThan:
1889        cont->OverwriteAndNegateIfEqual(kSignedLessThan);
1890        return VisitWordCompare(selector, value, kX64Cmp32, cont);
1891      case IrOpcode::kInt32LessThanOrEqual:
1892        cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
1893        return VisitWordCompare(selector, value, kX64Cmp32, cont);
1894      case IrOpcode::kUint32LessThan:
1895        cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
1896        return VisitWordCompare(selector, value, kX64Cmp32, cont);
1897      case IrOpcode::kUint32LessThanOrEqual:
1898        cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
1899        return VisitWordCompare(selector, value, kX64Cmp32, cont);
1900      case IrOpcode::kWord64Equal: {
1901        cont->OverwriteAndNegateIfEqual(kEqual);
1902        Int64BinopMatcher m(value);
1903        if (m.right().Is(0)) {
1904          // Try to combine the branch with a comparison.
1905          Node* const user = m.node();
1906          Node* const value = m.left().node();
1907          if (selector->CanCover(user, value)) {
1908            switch (value->opcode()) {
1909              case IrOpcode::kInt64Sub:
1910                return VisitWord64Compare(selector, value, cont);
1911              case IrOpcode::kWord64And:
1912                return VisitWordCompare(selector, value, kX64Test, cont);
1913              default:
1914                break;
1915            }
1916          }
1917          return VisitCompareZero(selector, value, kX64Cmp, cont);
1918        }
1919        return VisitWord64Compare(selector, value, cont);
1920      }
1921      case IrOpcode::kInt64LessThan:
1922        cont->OverwriteAndNegateIfEqual(kSignedLessThan);
1923        return VisitWord64Compare(selector, value, cont);
1924      case IrOpcode::kInt64LessThanOrEqual:
1925        cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
1926        return VisitWord64Compare(selector, value, cont);
1927      case IrOpcode::kUint64LessThan:
1928        cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
1929        return VisitWord64Compare(selector, value, cont);
1930      case IrOpcode::kUint64LessThanOrEqual:
1931        cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
1932        return VisitWord64Compare(selector, value, cont);
1933      case IrOpcode::kFloat32Equal:
1934        cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
1935        return VisitFloat32Compare(selector, value, cont);
1936      case IrOpcode::kFloat32LessThan:
1937        cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
1938        return VisitFloat32Compare(selector, value, cont);
1939      case IrOpcode::kFloat32LessThanOrEqual:
1940        cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
1941        return VisitFloat32Compare(selector, value, cont);
1942      case IrOpcode::kFloat64Equal:
1943        cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
1944        return VisitFloat64Compare(selector, value, cont);
1945      case IrOpcode::kFloat64LessThan: {
1946        Float64BinopMatcher m(value);
1947        if (m.left().Is(0.0) && m.right().IsFloat64Abs()) {
1948          // This matches the pattern
1949          //
1950          //   Float64LessThan(#0.0, Float64Abs(x))
1951          //
1952          // which TurboFan generates for NumberToBoolean in the general case,
1953          // and which evaluates to false if x is 0, -0 or NaN. We can compile
1954          // this to a simple (v)ucomisd using not_equal flags condition, which
1955          // avoids the costly Float64Abs.
1956          cont->OverwriteAndNegateIfEqual(kNotEqual);
1957          InstructionCode const opcode =
1958              selector->IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
1959          return VisitCompare(selector, opcode, m.left().node(),
1960                              m.right().InputAt(0), cont, false);
1961        }
1962        cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
1963        return VisitFloat64Compare(selector, value, cont);
1964      }
1965      case IrOpcode::kFloat64LessThanOrEqual:
1966        cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
1967        return VisitFloat64Compare(selector, value, cont);
1968      case IrOpcode::kProjection:
1969        // Check if this is the overflow output projection of an
1970        // <Operation>WithOverflow node.
1971        if (ProjectionIndexOf(value->op()) == 1u) {
1972          // We cannot combine the <Operation>WithOverflow with this branch
1973          // unless the 0th projection (the use of the actual value of the
1974          // <Operation> is either nullptr, which means there's no use of the
1975          // actual value, or was already defined, which means it is scheduled
1976          // *AFTER* this branch).
1977          Node* const node = value->InputAt(0);
1978          Node* const result = NodeProperties::FindProjection(node, 0);
1979          if (result == nullptr || selector->IsDefined(result)) {
1980            switch (node->opcode()) {
1981              case IrOpcode::kInt32AddWithOverflow:
1982                cont->OverwriteAndNegateIfEqual(kOverflow);
1983                return VisitBinop(selector, node, kX64Add32, cont);
1984              case IrOpcode::kInt32SubWithOverflow:
1985                cont->OverwriteAndNegateIfEqual(kOverflow);
1986                return VisitBinop(selector, node, kX64Sub32, cont);
1987              case IrOpcode::kInt32MulWithOverflow:
1988                cont->OverwriteAndNegateIfEqual(kOverflow);
1989                return VisitBinop(selector, node, kX64Imul32, cont);
1990              case IrOpcode::kInt64AddWithOverflow:
1991                cont->OverwriteAndNegateIfEqual(kOverflow);
1992                return VisitBinop(selector, node, kX64Add, cont);
1993              case IrOpcode::kInt64SubWithOverflow:
1994                cont->OverwriteAndNegateIfEqual(kOverflow);
1995                return VisitBinop(selector, node, kX64Sub, cont);
1996              default:
1997                break;
1998            }
1999          }
2000        }
2001        break;
2002      case IrOpcode::kInt32Sub:
2003        return VisitWordCompare(selector, value, kX64Cmp32, cont);
2004      case IrOpcode::kInt64Sub:
2005        return VisitWord64Compare(selector, value, cont);
2006      case IrOpcode::kWord32And:
2007        return VisitWordCompare(selector, value, kX64Test32, cont);
2008      case IrOpcode::kWord64And:
2009        return VisitWordCompare(selector, value, kX64Test, cont);
2010      default:
2011        break;
2012    }
2013  }
2014
2015  // Branch could not be combined with a compare, emit compare against 0.
2016  VisitCompareZero(selector, value, kX64Cmp32, cont);
2017}
2018
2019}  // namespace
2020
2021void InstructionSelector::VisitBranch(Node* branch, BasicBlock* tbranch,
2022                                      BasicBlock* fbranch) {
2023  FlagsContinuation cont(kNotEqual, tbranch, fbranch);
2024  VisitWordCompareZero(this, branch, branch->InputAt(0), &cont);
2025}
2026
2027void InstructionSelector::VisitDeoptimizeIf(Node* node) {
2028  FlagsContinuation cont = FlagsContinuation::ForDeoptimize(
2029      kNotEqual, DeoptimizeReasonOf(node->op()), node->InputAt(1));
2030  VisitWordCompareZero(this, node, node->InputAt(0), &cont);
2031}
2032
2033void InstructionSelector::VisitDeoptimizeUnless(Node* node) {
2034  FlagsContinuation cont = FlagsContinuation::ForDeoptimize(
2035      kEqual, DeoptimizeReasonOf(node->op()), node->InputAt(1));
2036  VisitWordCompareZero(this, node, node->InputAt(0), &cont);
2037}
2038
2039void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) {
2040  X64OperandGenerator g(this);
2041  InstructionOperand value_operand = g.UseRegister(node->InputAt(0));
2042
2043  // Emit either ArchTableSwitch or ArchLookupSwitch.
2044  size_t table_space_cost = 4 + sw.value_range;
2045  size_t table_time_cost = 3;
2046  size_t lookup_space_cost = 3 + 2 * sw.case_count;
2047  size_t lookup_time_cost = sw.case_count;
2048  if (sw.case_count > 4 &&
2049      table_space_cost + 3 * table_time_cost <=
2050          lookup_space_cost + 3 * lookup_time_cost &&
2051      sw.min_value > std::numeric_limits<int32_t>::min()) {
2052    InstructionOperand index_operand = g.TempRegister();
2053    if (sw.min_value) {
2054      // The leal automatically zero extends, so result is a valid 64-bit index.
2055      Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI), index_operand,
2056           value_operand, g.TempImmediate(-sw.min_value));
2057    } else {
2058      // Zero extend, because we use it as 64-bit index into the jump table.
2059      Emit(kX64Movl, index_operand, value_operand);
2060    }
2061    // Generate a table lookup.
2062    return EmitTableSwitch(sw, index_operand);
2063  }
2064
2065  // Generate a sequence of conditional jumps.
2066  return EmitLookupSwitch(sw, value_operand);
2067}
2068
2069
2070void InstructionSelector::VisitWord32Equal(Node* const node) {
2071  Node* user = node;
2072  FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2073  Int32BinopMatcher m(user);
2074  if (m.right().Is(0)) {
2075    Node* value = m.left().node();
2076
2077    // Try to combine with comparisons against 0 by simply inverting the branch.
2078    while (CanCover(user, value) && value->opcode() == IrOpcode::kWord32Equal) {
2079      Int32BinopMatcher m(value);
2080      if (m.right().Is(0)) {
2081        user = value;
2082        value = m.left().node();
2083        cont.Negate();
2084      } else {
2085        break;
2086      }
2087    }
2088
2089    // Try to combine the branch with a comparison.
2090    if (CanCover(user, value)) {
2091      switch (value->opcode()) {
2092        case IrOpcode::kInt32Sub:
2093          return VisitWordCompare(this, value, kX64Cmp32, &cont);
2094        case IrOpcode::kWord32And:
2095          return VisitWordCompare(this, value, kX64Test32, &cont);
2096        default:
2097          break;
2098      }
2099    }
2100    return VisitCompareZero(this, value, kX64Cmp32, &cont);
2101  }
2102  VisitWordCompare(this, node, kX64Cmp32, &cont);
2103}
2104
2105
2106void InstructionSelector::VisitInt32LessThan(Node* node) {
2107  FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2108  VisitWordCompare(this, node, kX64Cmp32, &cont);
2109}
2110
2111
2112void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) {
2113  FlagsContinuation cont =
2114      FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2115  VisitWordCompare(this, node, kX64Cmp32, &cont);
2116}
2117
2118
2119void InstructionSelector::VisitUint32LessThan(Node* node) {
2120  FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2121  VisitWordCompare(this, node, kX64Cmp32, &cont);
2122}
2123
2124
2125void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) {
2126  FlagsContinuation cont =
2127      FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2128  VisitWordCompare(this, node, kX64Cmp32, &cont);
2129}
2130
2131
2132void InstructionSelector::VisitWord64Equal(Node* const node) {
2133  FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2134  Int64BinopMatcher m(node);
2135  if (m.right().Is(0)) {
2136    // Try to combine the equality check with a comparison.
2137    Node* const user = m.node();
2138    Node* const value = m.left().node();
2139    if (CanCover(user, value)) {
2140      switch (value->opcode()) {
2141        case IrOpcode::kInt64Sub:
2142          return VisitWord64Compare(this, value, &cont);
2143        case IrOpcode::kWord64And:
2144          return VisitWordCompare(this, value, kX64Test, &cont);
2145        default:
2146          break;
2147      }
2148    }
2149  }
2150  VisitWord64Compare(this, node, &cont);
2151}
2152
2153
2154void InstructionSelector::VisitInt32AddWithOverflow(Node* node) {
2155  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2156    FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2157    return VisitBinop(this, node, kX64Add32, &cont);
2158  }
2159  FlagsContinuation cont;
2160  VisitBinop(this, node, kX64Add32, &cont);
2161}
2162
2163
2164void InstructionSelector::VisitInt32SubWithOverflow(Node* node) {
2165  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2166    FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2167    return VisitBinop(this, node, kX64Sub32, &cont);
2168  }
2169  FlagsContinuation cont;
2170  VisitBinop(this, node, kX64Sub32, &cont);
2171}
2172
2173
2174void InstructionSelector::VisitInt64LessThan(Node* node) {
2175  FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2176  VisitWord64Compare(this, node, &cont);
2177}
2178
2179
2180void InstructionSelector::VisitInt64LessThanOrEqual(Node* node) {
2181  FlagsContinuation cont =
2182      FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2183  VisitWord64Compare(this, node, &cont);
2184}
2185
2186
2187void InstructionSelector::VisitUint64LessThan(Node* node) {
2188  FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2189  VisitWord64Compare(this, node, &cont);
2190}
2191
2192
2193void InstructionSelector::VisitUint64LessThanOrEqual(Node* node) {
2194  FlagsContinuation cont =
2195      FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2196  VisitWord64Compare(this, node, &cont);
2197}
2198
2199
2200void InstructionSelector::VisitFloat32Equal(Node* node) {
2201  FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
2202  VisitFloat32Compare(this, node, &cont);
2203}
2204
2205
2206void InstructionSelector::VisitFloat32LessThan(Node* node) {
2207  FlagsContinuation cont =
2208      FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
2209  VisitFloat32Compare(this, node, &cont);
2210}
2211
2212
2213void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) {
2214  FlagsContinuation cont =
2215      FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
2216  VisitFloat32Compare(this, node, &cont);
2217}
2218
2219
2220void InstructionSelector::VisitFloat64Equal(Node* node) {
2221  FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
2222  VisitFloat64Compare(this, node, &cont);
2223}
2224
2225void InstructionSelector::VisitFloat64LessThan(Node* node) {
2226  Float64BinopMatcher m(node);
2227  if (m.left().Is(0.0) && m.right().IsFloat64Abs()) {
2228    // This matches the pattern
2229    //
2230    //   Float64LessThan(#0.0, Float64Abs(x))
2231    //
2232    // which TurboFan generates for NumberToBoolean in the general case,
2233    // and which evaluates to false if x is 0, -0 or NaN. We can compile
2234    // this to a simple (v)ucomisd using not_equal flags condition, which
2235    // avoids the costly Float64Abs.
2236    FlagsContinuation cont = FlagsContinuation::ForSet(kNotEqual, node);
2237    InstructionCode const opcode =
2238        IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2239    return VisitCompare(this, opcode, m.left().node(), m.right().InputAt(0),
2240                        &cont, false);
2241  }
2242  FlagsContinuation cont =
2243      FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
2244  VisitFloat64Compare(this, node, &cont);
2245}
2246
2247void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) {
2248  FlagsContinuation cont =
2249      FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
2250  VisitFloat64Compare(this, node, &cont);
2251}
2252
2253
2254void InstructionSelector::VisitFloat64ExtractLowWord32(Node* node) {
2255  X64OperandGenerator g(this);
2256  Emit(kSSEFloat64ExtractLowWord32, g.DefineAsRegister(node),
2257       g.Use(node->InputAt(0)));
2258}
2259
2260
2261void InstructionSelector::VisitFloat64ExtractHighWord32(Node* node) {
2262  X64OperandGenerator g(this);
2263  Emit(kSSEFloat64ExtractHighWord32, g.DefineAsRegister(node),
2264       g.Use(node->InputAt(0)));
2265}
2266
2267
2268void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) {
2269  X64OperandGenerator g(this);
2270  Node* left = node->InputAt(0);
2271  Node* right = node->InputAt(1);
2272  Float64Matcher mleft(left);
2273  if (mleft.HasValue() && (bit_cast<uint64_t>(mleft.Value()) >> 32) == 0u) {
2274    Emit(kSSEFloat64LoadLowWord32, g.DefineAsRegister(node), g.Use(right));
2275    return;
2276  }
2277  Emit(kSSEFloat64InsertLowWord32, g.DefineSameAsFirst(node),
2278       g.UseRegister(left), g.Use(right));
2279}
2280
2281
2282void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
2283  X64OperandGenerator g(this);
2284  Node* left = node->InputAt(0);
2285  Node* right = node->InputAt(1);
2286  Emit(kSSEFloat64InsertHighWord32, g.DefineSameAsFirst(node),
2287       g.UseRegister(left), g.Use(right));
2288}
2289
2290void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
2291  X64OperandGenerator g(this);
2292  Emit(kSSEFloat64SilenceNaN, g.DefineSameAsFirst(node),
2293       g.UseRegister(node->InputAt(0)));
2294}
2295
2296void InstructionSelector::VisitAtomicLoad(Node* node) {
2297  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
2298  DCHECK(load_rep.representation() == MachineRepresentation::kWord8 ||
2299         load_rep.representation() == MachineRepresentation::kWord16 ||
2300         load_rep.representation() == MachineRepresentation::kWord32);
2301  USE(load_rep);
2302  VisitLoad(node);
2303}
2304
2305void InstructionSelector::VisitAtomicStore(Node* node) {
2306  X64OperandGenerator g(this);
2307  Node* base = node->InputAt(0);
2308  Node* index = node->InputAt(1);
2309  Node* value = node->InputAt(2);
2310
2311  MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
2312  ArchOpcode opcode = kArchNop;
2313  switch (rep) {
2314    case MachineRepresentation::kWord8:
2315      opcode = kX64Xchgb;
2316      break;
2317    case MachineRepresentation::kWord16:
2318      opcode = kX64Xchgw;
2319      break;
2320    case MachineRepresentation::kWord32:
2321      opcode = kX64Xchgl;
2322      break;
2323    default:
2324      UNREACHABLE();
2325      return;
2326  }
2327  AddressingMode addressing_mode;
2328  InstructionOperand inputs[4];
2329  size_t input_count = 0;
2330  inputs[input_count++] = g.UseUniqueRegister(base);
2331  if (g.CanBeImmediate(index)) {
2332    inputs[input_count++] = g.UseImmediate(index);
2333    addressing_mode = kMode_MRI;
2334  } else {
2335    inputs[input_count++] = g.UseUniqueRegister(index);
2336    addressing_mode = kMode_MR1;
2337  }
2338  inputs[input_count++] = g.UseUniqueRegister(value);
2339  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
2340  Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count, inputs);
2341}
2342
2343void InstructionSelector::VisitCreateInt32x4(Node* node) {
2344  X64OperandGenerator g(this);
2345  Emit(kX64Int32x4Create, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
2346}
2347
2348void InstructionSelector::VisitInt32x4ExtractLane(Node* node) {
2349  X64OperandGenerator g(this);
2350  Emit(kX64Int32x4ExtractLane, g.DefineAsRegister(node),
2351       g.UseRegister(node->InputAt(0)), g.UseImmediate(node->InputAt(1)));
2352}
2353
2354// static
2355MachineOperatorBuilder::Flags
2356InstructionSelector::SupportedMachineOperatorFlags() {
2357  MachineOperatorBuilder::Flags flags =
2358      MachineOperatorBuilder::kWord32ShiftIsSafe |
2359      MachineOperatorBuilder::kWord32Ctz | MachineOperatorBuilder::kWord64Ctz;
2360  if (CpuFeatures::IsSupported(POPCNT)) {
2361    flags |= MachineOperatorBuilder::kWord32Popcnt |
2362             MachineOperatorBuilder::kWord64Popcnt;
2363  }
2364  if (CpuFeatures::IsSupported(SSE4_1)) {
2365    flags |= MachineOperatorBuilder::kFloat32RoundDown |
2366             MachineOperatorBuilder::kFloat64RoundDown |
2367             MachineOperatorBuilder::kFloat32RoundUp |
2368             MachineOperatorBuilder::kFloat64RoundUp |
2369             MachineOperatorBuilder::kFloat32RoundTruncate |
2370             MachineOperatorBuilder::kFloat64RoundTruncate |
2371             MachineOperatorBuilder::kFloat32RoundTiesEven |
2372             MachineOperatorBuilder::kFloat64RoundTiesEven;
2373  }
2374  return flags;
2375}
2376
2377// static
2378MachineOperatorBuilder::AlignmentRequirements
2379InstructionSelector::AlignmentRequirements() {
2380  return MachineOperatorBuilder::AlignmentRequirements::
2381      FullUnalignedAccessSupport();
2382}
2383
2384}  // namespace compiler
2385}  // namespace internal
2386}  // namespace v8
2387