1//===- subzero/src/IceInstX86Base.h - Generic x86 instructions -*- C++ -*--===//
2//
3//                        The Subzero Code Generator
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// \file
11/// \brief This file defines the InstX86Base template class, as well as the
12/// generic X86 Instruction class hierarchy.
13///
14/// Only X86 instructions common across all/most X86 targets should be defined
15/// here, with target-specific instructions declared in the target's traits.
16///
17//===----------------------------------------------------------------------===//
18
19#ifndef SUBZERO_SRC_ICEINSTX86BASE_H
20#define SUBZERO_SRC_ICEINSTX86BASE_H
21
22#include "IceDefs.h"
23#include "IceInst.h"
24#include "IceOperand.h"
25
26namespace Ice {
27
28#ifndef X86NAMESPACE
29#error "You must define the X86 Target namespace."
30#endif
31
32namespace X86NAMESPACE {
33
34template <typename TraitsType> struct InstImpl {
35  using Traits = TraitsType;
36  using Assembler = typename Traits::Assembler;
37  using AssemblerLabel = typename Assembler::Label;
38  using AssemblerImmediate = typename Assembler::Immediate;
39  using TargetLowering = typename Traits::TargetLowering;
40  using Address = typename Traits::Address;
41  using X86Operand = typename Traits::X86Operand;
42  using X86OperandMem = typename Traits::X86OperandMem;
43  using VariableSplit = typename Traits::VariableSplit;
44
45  using GPRRegister = typename Traits::RegisterSet::GPRRegister;
46  using RegisterSet = typename Traits::RegisterSet;
47  using XmmRegister = typename Traits::RegisterSet::XmmRegister;
48
49  using Cond = typename Traits::Cond;
50  using BrCond = typename Traits::Cond::BrCond;
51  using CmppsCond = typename Traits::Cond::CmppsCond;
52
53  template <typename SReg_t, typename DReg_t>
54  using CastEmitterRegOp =
55      typename Traits::Assembler::template CastEmitterRegOp<SReg_t, DReg_t>;
56  template <typename SReg_t, typename DReg_t>
57  using ThreeOpImmEmitter =
58      typename Traits::Assembler::template ThreeOpImmEmitter<SReg_t, DReg_t>;
59  using GPREmitterAddrOp = typename Traits::Assembler::GPREmitterAddrOp;
60  using GPREmitterRegOp = typename Traits::Assembler::GPREmitterRegOp;
61  using GPREmitterShiftD = typename Traits::Assembler::GPREmitterShiftD;
62  using GPREmitterShiftOp = typename Traits::Assembler::GPREmitterShiftOp;
63  using GPREmitterOneOp = typename Traits::Assembler::GPREmitterOneOp;
64  using XmmEmitterRegOp = typename Traits::Assembler::XmmEmitterRegOp;
65  using XmmEmitterShiftOp = typename Traits::Assembler::XmmEmitterShiftOp;
66  using XmmEmitterMovOps = typename Traits::Assembler::XmmEmitterMovOps;
67
68  class InstX86Base : public InstTarget {
69    InstX86Base() = delete;
70    InstX86Base(const InstX86Base &) = delete;
71    InstX86Base &operator=(const InstX86Base &) = delete;
72
73  public:
74    enum InstKindX86 {
75      k__Start = Inst::Target,
76      Adc,
77      AdcRMW,
78      Add,
79      AddRMW,
80      Addps,
81      Addss,
82      And,
83      Andnps,
84      Andps,
85      AndRMW,
86      Blendvps,
87      Br,
88      Bsf,
89      Bsr,
90      Bswap,
91      Call,
92      Cbwdq,
93      Cmov,
94      Cmpps,
95      Cmpxchg,
96      Cmpxchg8b,
97      Cvt,
98      Div,
99      Divps,
100      Divss,
101      FakeRMW,
102      Fld,
103      Fstp,
104      GetIP,
105      Icmp,
106      Idiv,
107      Imul,
108      ImulImm,
109      Insertps,
110      Int3,
111      Jmp,
112      Label,
113      Lea,
114      Load,
115      Mfence,
116      Minps,
117      Maxps,
118      Minss,
119      Maxss,
120      Mov,
121      Movd,
122      Movmsk,
123      Movp,
124      Movq,
125      MovssRegs,
126      Movsx,
127      Movzx,
128      Mul,
129      Mulps,
130      Mulss,
131      Neg,
132      Nop,
133      Or,
134      Orps,
135      OrRMW,
136      Padd,
137      Padds,
138      Paddus,
139      Pand,
140      Pandn,
141      Pblendvb,
142      Pcmpeq,
143      Pcmpgt,
144      Pextr,
145      Pinsr,
146      Pmull,
147      Pmulhw,
148      Pmulhuw,
149      Pmaddwd,
150      Pmuludq,
151      Pop,
152      Por,
153      Pshufb,
154      Pshufd,
155      Punpckl,
156      Punpckh,
157      Packss,
158      Packus,
159      Psll,
160      Psra,
161      Psrl,
162      Psub,
163      Psubs,
164      Psubus,
165      Push,
166      Pxor,
167      Ret,
168      Rol,
169      Round,
170      Sar,
171      Sbb,
172      SbbRMW,
173      Setcc,
174      Shl,
175      Shld,
176      Shr,
177      Shrd,
178      Shufps,
179      Sqrt,
180      Store,
181      StoreP,
182      StoreQ,
183      StoreD,
184      Sub,
185      SubRMW,
186      Subps,
187      Subss,
188      Test,
189      Ucomiss,
190      UD2,
191      Xadd,
192      Xchg,
193      Xor,
194      Xorps,
195      XorRMW,
196
197      /// Intel Architecture Code Analyzer markers. These are not executable so
198      /// must only be used for analysis.
199      IacaStart,
200      IacaEnd
201    };
202
203    enum SseSuffix { None, Packed, Unpack, Scalar, Integral, Pack };
204
205    static const char *getWidthString(Type Ty);
206    static const char *getFldString(Type Ty);
207    static BrCond getOppositeCondition(BrCond Cond);
208    void dump(const Cfg *Func) const override;
209
210    // Shared emit routines for common forms of instructions.
211    void emitTwoAddress(const Cfg *Func, const char *Opcode,
212                        const char *Suffix = "") const;
213
214    static TargetLowering *getTarget(const Cfg *Func) {
215      return static_cast<TargetLowering *>(Func->getTarget());
216    }
217
218  protected:
219    InstX86Base(Cfg *Func, InstKindX86 Kind, SizeT Maxsrcs, Variable *Dest)
220        : InstTarget(Func, static_cast<InstKind>(Kind), Maxsrcs, Dest) {}
221
222    static bool isClassof(const Inst *Instr, InstKindX86 MyKind) {
223      return Instr->getKind() == static_cast<InstKind>(MyKind);
224    }
225    // Most instructions that operate on vector arguments require vector memory
226    // operands to be fully aligned (16-byte alignment for PNaCl vector types).
227    // The stack frame layout and call ABI ensure proper alignment for stack
228    // operands, but memory operands (originating from load/store bitcode
229    // instructions) only have element-size alignment guarantees. This function
230    // validates that none of the operands is a memory operand of vector type,
231    // calling report_fatal_error() if one is found. This function should be
232    // called during emission, and maybe also in the ctor (as long as that fits
233    // the lowering style).
234    void validateVectorAddrMode() const {
235      if (this->getDest())
236        this->validateVectorAddrModeOpnd(this->getDest());
237      for (SizeT i = 0; i < this->getSrcSize(); ++i) {
238        this->validateVectorAddrModeOpnd(this->getSrc(i));
239      }
240    }
241
242  private:
243    static void validateVectorAddrModeOpnd(const Operand *Opnd) {
244      if (llvm::isa<X86OperandMem>(Opnd) && isVectorType(Opnd->getType())) {
245        llvm::report_fatal_error("Possible misaligned vector memory operation");
246      }
247    }
248  };
249
250  /// InstX86FakeRMW represents a non-atomic read-modify-write operation on a
251  /// memory location. An InstX86FakeRMW is a "fake" instruction in that it
252  /// still needs to be lowered to some actual RMW instruction.
253  ///
254  /// If A is some memory address, D is some data value to apply, and OP is an
255  /// arithmetic operator, the instruction operates as: (*A) = (*A) OP D
256  class InstX86FakeRMW final : public InstX86Base {
257    InstX86FakeRMW() = delete;
258    InstX86FakeRMW(const InstX86FakeRMW &) = delete;
259    InstX86FakeRMW &operator=(const InstX86FakeRMW &) = delete;
260
261  public:
262    static InstX86FakeRMW *create(Cfg *Func, Operand *Data, Operand *Addr,
263                                  Variable *Beacon, InstArithmetic::OpKind Op,
264                                  uint32_t Align = 1) {
265      // TODO(stichnot): Stop ignoring alignment specification.
266      (void)Align;
267      return new (Func->allocate<InstX86FakeRMW>())
268          InstX86FakeRMW(Func, Data, Addr, Op, Beacon);
269    }
270    Operand *getAddr() const { return this->getSrc(1); }
271    Operand *getData() const { return this->getSrc(0); }
272    InstArithmetic::OpKind getOp() const { return Op; }
273    Variable *getBeacon() const {
274      return llvm::cast<Variable>(this->getSrc(2));
275    }
276    void dump(const Cfg *Func) const override;
277    static bool classof(const Inst *Instr) {
278      return InstX86Base::isClassof(Instr, InstX86Base::FakeRMW);
279    }
280
281  private:
282    InstArithmetic::OpKind Op;
283    InstX86FakeRMW(Cfg *Func, Operand *Data, Operand *Addr,
284                   InstArithmetic::OpKind Op, Variable *Beacon);
285  };
286
287  class InstX86GetIP final : public InstX86Base {
288    InstX86GetIP() = delete;
289    InstX86GetIP(const InstX86GetIP &) = delete;
290    InstX86GetIP &operator=(const InstX86GetIP &) = delete;
291
292  public:
293    static InstX86GetIP *create(Cfg *Func, Variable *Dest) {
294      return new (Func->allocate<InstX86GetIP>()) InstX86GetIP(Func, Dest);
295    }
296    void emit(const Cfg *Func) const override;
297    void emitIAS(const Cfg *Func) const override;
298    void dump(const Cfg *Func) const override;
299    static bool classof(const Inst *Instr) {
300      return InstX86Base::isClassof(Instr, InstX86Base::GetIP);
301    }
302
303  private:
304    InstX86GetIP(Cfg *Func, Variable *Dest);
305  };
306
307  /// InstX86Label represents an intra-block label that is the target of an
308  /// intra-block branch. The offset between the label and the branch must be
309  /// fit into one byte (considered "near"). These are used for lowering i1
310  /// calculations, Select instructions, and 64-bit compares on a 32-bit
311  /// architecture, without basic block splitting. Basic block splitting is not
312  /// so desirable for several reasons, one of which is the impact on decisions
313  /// based on whether a variable's live range spans multiple basic blocks.
314  ///
315  /// Intra-block control flow must be used with caution. Consider the sequence
316  /// for "c = (a >= b ? x : y)".
317  ///     cmp a, b
318  ///     br lt, L1
319  ///     mov c, x
320  ///     jmp L2
321  ///   L1:
322  ///     mov c, y
323  ///   L2:
324  ///
325  /// Labels L1 and L2 are intra-block labels. Without knowledge of the
326  /// intra-block control flow, liveness analysis will determine the "mov c, x"
327  /// instruction to be dead. One way to prevent this is to insert a
328  /// "FakeUse(c)" instruction anywhere between the two "mov c, ..."
329  /// instructions, e.g.:
330  ///
331  ///     cmp a, b
332  ///     br lt, L1
333  ///     mov c, x
334  ///     jmp L2
335  ///     FakeUse(c)
336  ///   L1:
337  ///     mov c, y
338  ///   L2:
339  ///
340  /// The down-side is that "mov c, x" can never be dead-code eliminated even if
341  /// there are no uses of c. As unlikely as this situation is, it may be
342  /// prevented by running dead code elimination before lowering.
343  class InstX86Label final : public InstX86Base {
344    InstX86Label() = delete;
345    InstX86Label(const InstX86Label &) = delete;
346    InstX86Label &operator=(const InstX86Label &) = delete;
347
348  public:
349    static InstX86Label *create(Cfg *Func, TargetLowering *Target) {
350      return new (Func->allocate<InstX86Label>()) InstX86Label(Func, Target);
351    }
352    uint32_t getEmitInstCount() const override { return 0; }
353    GlobalString getLabelName() const { return Name; }
354    SizeT getLabelNumber() const { return LabelNumber; }
355    bool isLabel() const override { return true; }
356    void emit(const Cfg *Func) const override;
357    void emitIAS(const Cfg *Func) const override;
358    void dump(const Cfg *Func) const override;
359    void setRelocOffset(RelocOffset *Value) { OffsetReloc = Value; }
360
361  private:
362    InstX86Label(Cfg *Func, TargetLowering *Target);
363
364    SizeT LabelNumber; // used for unique label generation.
365    RelocOffset *OffsetReloc = nullptr;
366    GlobalString Name;
367  };
368
369  /// Conditional and unconditional branch instruction.
370  class InstX86Br final : public InstX86Base {
371    InstX86Br() = delete;
372    InstX86Br(const InstX86Br &) = delete;
373    InstX86Br &operator=(const InstX86Br &) = delete;
374
375  public:
376    enum Mode { Near, Far };
377
378    /// Create a conditional branch to a node.
379    static InstX86Br *create(Cfg *Func, CfgNode *TargetTrue,
380                             CfgNode *TargetFalse, BrCond Condition,
381                             Mode Kind) {
382      assert(Condition != Cond::Br_None);
383      constexpr InstX86Label *NoLabel = nullptr;
384      return new (Func->allocate<InstX86Br>())
385          InstX86Br(Func, TargetTrue, TargetFalse, NoLabel, Condition, Kind);
386    }
387    /// Create an unconditional branch to a node.
388    static InstX86Br *create(Cfg *Func, CfgNode *Target, Mode Kind) {
389      constexpr CfgNode *NoCondTarget = nullptr;
390      constexpr InstX86Label *NoLabel = nullptr;
391      return new (Func->allocate<InstX86Br>())
392          InstX86Br(Func, NoCondTarget, Target, NoLabel, Cond::Br_None, Kind);
393    }
394    /// Create a non-terminator conditional branch to a node, with a fallthrough
395    /// to the next instruction in the current node. This is used for switch
396    /// lowering.
397    static InstX86Br *create(Cfg *Func, CfgNode *Target, BrCond Condition,
398                             Mode Kind) {
399      assert(Condition != Cond::Br_None);
400      constexpr CfgNode *NoUncondTarget = nullptr;
401      constexpr InstX86Label *NoLabel = nullptr;
402      return new (Func->allocate<InstX86Br>())
403          InstX86Br(Func, Target, NoUncondTarget, NoLabel, Condition, Kind);
404    }
405    /// Create a conditional intra-block branch (or unconditional, if
406    /// Condition==Br_None) to a label in the current block.
407    static InstX86Br *create(Cfg *Func, InstX86Label *Label, BrCond Condition,
408                             Mode Kind) {
409      constexpr CfgNode *NoCondTarget = nullptr;
410      constexpr CfgNode *NoUncondTarget = nullptr;
411      return new (Func->allocate<InstX86Br>())
412          InstX86Br(Func, NoCondTarget, NoUncondTarget, Label, Condition, Kind);
413    }
414    const CfgNode *getTargetTrue() const { return TargetTrue; }
415    const CfgNode *getTargetFalse() const { return TargetFalse; }
416    bool isNear() const { return Kind == Near; }
417    bool optimizeBranch(const CfgNode *NextNode);
418    uint32_t getEmitInstCount() const override {
419      uint32_t Sum = 0;
420      if (Label)
421        ++Sum;
422      if (getTargetTrue())
423        ++Sum;
424      if (getTargetFalse())
425        ++Sum;
426      return Sum;
427    }
428    bool isUnconditionalBranch() const override {
429      return !Label && Condition == Cond::Br_None;
430    }
431    const Inst *getIntraBlockBranchTarget() const override { return Label; }
432    bool repointEdges(CfgNode *OldNode, CfgNode *NewNode) override;
433    void emit(const Cfg *Func) const override;
434    void emitIAS(const Cfg *Func) const override;
435    void dump(const Cfg *Func) const override;
436    static bool classof(const Inst *Instr) {
437      return InstX86Base::isClassof(Instr, InstX86Base::Br);
438    }
439
440  private:
441    InstX86Br(Cfg *Func, const CfgNode *TargetTrue, const CfgNode *TargetFalse,
442              const InstX86Label *Label, BrCond Condition, Mode Kind);
443
444    BrCond Condition;
445    const CfgNode *TargetTrue;
446    const CfgNode *TargetFalse;
447    const InstX86Label *Label; // Intra-block branch target
448    const Mode Kind;
449  };
450
451  /// Jump to a target outside this function, such as tailcall, nacljump,
452  /// naclret, unreachable. This is different from a Branch instruction in that
453  /// there is no intra-function control flow to represent.
454  class InstX86Jmp final : public InstX86Base {
455    InstX86Jmp() = delete;
456    InstX86Jmp(const InstX86Jmp &) = delete;
457    InstX86Jmp &operator=(const InstX86Jmp &) = delete;
458
459  public:
460    static InstX86Jmp *create(Cfg *Func, Operand *Target) {
461      return new (Func->allocate<InstX86Jmp>()) InstX86Jmp(Func, Target);
462    }
463    Operand *getJmpTarget() const { return this->getSrc(0); }
464    void emit(const Cfg *Func) const override;
465    void emitIAS(const Cfg *Func) const override;
466    void dump(const Cfg *Func) const override;
467    static bool classof(const Inst *Instr) {
468      return InstX86Base::isClassof(Instr, InstX86Base::Jmp);
469    }
470
471  private:
472    InstX86Jmp(Cfg *Func, Operand *Target);
473  };
474
475  /// Call instruction. Arguments should have already been pushed.
476  class InstX86Call final : public InstX86Base {
477    InstX86Call() = delete;
478    InstX86Call(const InstX86Call &) = delete;
479    InstX86Call &operator=(const InstX86Call &) = delete;
480
481  public:
482    static InstX86Call *create(Cfg *Func, Variable *Dest, Operand *CallTarget) {
483      return new (Func->allocate<InstX86Call>())
484          InstX86Call(Func, Dest, CallTarget);
485    }
486    Operand *getCallTarget() const { return this->getSrc(0); }
487    void emit(const Cfg *Func) const override;
488    void emitIAS(const Cfg *Func) const override;
489    void dump(const Cfg *Func) const override;
490    static bool classof(const Inst *Instr) {
491      return InstX86Base::isClassof(Instr, InstX86Base::Call);
492    }
493
494  private:
495    InstX86Call(Cfg *Func, Variable *Dest, Operand *CallTarget);
496  };
497
498  /// Emit a one-operand (GPR) instruction.
499  static void emitIASOpTyGPR(const Cfg *Func, Type Ty, const Operand *Var,
500                             const GPREmitterOneOp &Emitter);
501
502  static void emitIASAsAddrOpTyGPR(const Cfg *Func, Type Ty, const Operand *Op0,
503                                   const Operand *Op1,
504                                   const GPREmitterAddrOp &Emitter);
505
506  static void emitIASGPRShift(const Cfg *Func, Type Ty, const Variable *Var,
507                              const Operand *Src,
508                              const GPREmitterShiftOp &Emitter);
509
510  static void emitIASAddrOpTyGPR(const Cfg *Func, Type Ty, const Address &Addr,
511                                 const Operand *Src,
512                                 const GPREmitterAddrOp &Emitter);
513
514  static void emitIASRegOpTyXMM(const Cfg *Func, Type Ty, const Variable *Var,
515                                const Operand *Src,
516                                const XmmEmitterRegOp &Emitter);
517
518  static void emitIASGPRShiftDouble(const Cfg *Func, const Variable *Dest,
519                                    const Operand *Src1Op,
520                                    const Operand *Src2Op,
521                                    const GPREmitterShiftD &Emitter);
522
523  template <typename DReg_t, typename SReg_t, DReg_t (*destEnc)(RegNumT),
524            SReg_t (*srcEnc)(RegNumT)>
525  static void emitIASCastRegOp(const Cfg *Func, Type DestTy,
526                               const Variable *Dest, Type SrcTy,
527                               const Operand *Src,
528                               const CastEmitterRegOp<DReg_t, SReg_t> &Emitter);
529
530  template <typename DReg_t, typename SReg_t, DReg_t (*destEnc)(RegNumT),
531            SReg_t (*srcEnc)(RegNumT)>
532  static void
533  emitIASThreeOpImmOps(const Cfg *Func, Type DispatchTy, const Variable *Dest,
534                       const Operand *Src0, const Operand *Src1,
535                       const ThreeOpImmEmitter<DReg_t, SReg_t> Emitter);
536
537  static void emitIASMovlikeXMM(const Cfg *Func, const Variable *Dest,
538                                const Operand *Src,
539                                const XmmEmitterMovOps Emitter);
540
541  static void emitVariableBlendInst(const char *Opcode, const Inst *Instr,
542                                    const Cfg *Func);
543
544  static void emitIASVariableBlendInst(const Inst *Instr, const Cfg *Func,
545                                       const XmmEmitterRegOp &Emitter);
546
547  static void emitIASXmmShift(const Cfg *Func, Type Ty, const Variable *Var,
548                              const Operand *Src,
549                              const XmmEmitterShiftOp &Emitter);
550
551  /// Emit a two-operand (GPR) instruction, where the dest operand is a Variable
552  /// that's guaranteed to be a register.
553  template <bool VarCanBeByte = true, bool SrcCanBeByte = true>
554  static void emitIASRegOpTyGPR(const Cfg *Func, bool IsLea, Type Ty,
555                                const Variable *Dst, const Operand *Src,
556                                const GPREmitterRegOp &Emitter);
557
558  /// Instructions of the form x := op(x).
559  template <typename InstX86Base::InstKindX86 K>
560  class InstX86BaseInplaceopGPR : public InstX86Base {
561    InstX86BaseInplaceopGPR() = delete;
562    InstX86BaseInplaceopGPR(const InstX86BaseInplaceopGPR &) = delete;
563    InstX86BaseInplaceopGPR &
564    operator=(const InstX86BaseInplaceopGPR &) = delete;
565
566  public:
567    using Base = InstX86BaseInplaceopGPR<K>;
568
569    void emit(const Cfg *Func) const override {
570      if (!BuildDefs::dump())
571        return;
572      Ostream &Str = Func->getContext()->getStrEmit();
573      assert(this->getSrcSize() == 1);
574      Str << "\t" << Opcode << "\t";
575      this->getSrc(0)->emit(Func);
576    }
577    void emitIAS(const Cfg *Func) const override {
578      assert(this->getSrcSize() == 1);
579      const Variable *Var = this->getDest();
580      Type Ty = Var->getType();
581      emitIASOpTyGPR(Func, Ty, Var, Emitter);
582    }
583    void dump(const Cfg *Func) const override {
584      if (!BuildDefs::dump())
585        return;
586      Ostream &Str = Func->getContext()->getStrDump();
587      this->dumpDest(Func);
588      Str << " = " << Opcode << "." << this->getDest()->getType() << " ";
589      this->dumpSources(Func);
590    }
591    static bool classof(const Inst *Instr) {
592      return InstX86Base::isClassof(Instr, InstX86Base::K);
593    }
594
595  protected:
596    InstX86BaseInplaceopGPR(Cfg *Func, Operand *SrcDest)
597        : InstX86Base(Func, K, 1, llvm::dyn_cast<Variable>(SrcDest)) {
598      this->addSource(SrcDest);
599    }
600
601  private:
602    static const char *Opcode;
603    static const GPREmitterOneOp Emitter;
604  };
605
606  /// Instructions of the form x := op(y).
607  template <typename InstX86Base::InstKindX86 K>
608  class InstX86BaseUnaryopGPR : public InstX86Base {
609    InstX86BaseUnaryopGPR() = delete;
610    InstX86BaseUnaryopGPR(const InstX86BaseUnaryopGPR &) = delete;
611    InstX86BaseUnaryopGPR &operator=(const InstX86BaseUnaryopGPR &) = delete;
612
613  public:
614    using Base = InstX86BaseUnaryopGPR<K>;
615
616    void emit(const Cfg *Func) const override {
617      if (!BuildDefs::dump())
618        return;
619      Ostream &Str = Func->getContext()->getStrEmit();
620      assert(this->getSrcSize() == 1);
621      Type SrcTy = this->getSrc(0)->getType();
622      Type DestTy = this->getDest()->getType();
623      Str << "\t" << Opcode << this->getWidthString(SrcTy);
624      // Movsx and movzx need both the source and dest type width letter to
625      // define the operation. The other unary operations have the same source
626      // and dest type and as a result need only one letter.
627      if (SrcTy != DestTy)
628        Str << this->getWidthString(DestTy);
629      Str << "\t";
630      this->getSrc(0)->emit(Func);
631      Str << ", ";
632      this->getDest()->emit(Func);
633    }
634    void emitIAS(const Cfg *Func) const override {
635      assert(this->getSrcSize() == 1);
636      const Variable *Var = this->getDest();
637      Type Ty = Var->getType();
638      const Operand *Src = this->getSrc(0);
639      constexpr bool IsLea = K == InstX86Base::Lea;
640
641      if (IsLea) {
642        if (auto *Add = deoptLeaToAddOrNull(Func)) {
643          Add->emitIAS(Func);
644          return;
645        }
646      }
647      emitIASRegOpTyGPR(Func, IsLea, Ty, Var, Src, Emitter);
648    }
649    void dump(const Cfg *Func) const override {
650      if (!BuildDefs::dump())
651        return;
652      Ostream &Str = Func->getContext()->getStrDump();
653      this->dumpDest(Func);
654      Str << " = " << Opcode << "." << this->getSrc(0)->getType() << " ";
655      this->dumpSources(Func);
656    }
657
658    static bool classof(const Inst *Instr) {
659      return InstX86Base::isClassof(Instr, InstX86Base::K);
660    }
661
662  protected:
663    InstX86BaseUnaryopGPR(Cfg *Func, Variable *Dest, Operand *Src)
664        : InstX86Base(Func, K, 1, Dest) {
665      this->addSource(Src);
666    }
667
668    Inst *deoptLeaToAddOrNull(const Cfg *Func) const {
669      // Revert back to Add when the Lea is a 2-address instruction.
670      // Caller has to emit, this just produces the add instruction.
671      if (auto *MemOp = llvm::dyn_cast<X86OperandMem>(this->getSrc(0))) {
672        if (getFlags().getAggressiveLea() &&
673            MemOp->getBase()->getRegNum() == this->getDest()->getRegNum() &&
674            MemOp->getIndex() == nullptr && MemOp->getShift() == 0) {
675          auto *Add = InstImpl<TraitsType>::InstX86Add::create(
676              const_cast<Cfg *>(Func), this->getDest(), MemOp->getOffset());
677          // TODO(manasijm): Remove const_cast by emitting code for add
678          // directly.
679          return Add;
680        }
681      }
682      return nullptr;
683    }
684
685    static const char *Opcode;
686    static const GPREmitterRegOp Emitter;
687  };
688
689  template <typename InstX86Base::InstKindX86 K>
690  class InstX86BaseUnaryopXmm : public InstX86Base {
691    InstX86BaseUnaryopXmm() = delete;
692    InstX86BaseUnaryopXmm(const InstX86BaseUnaryopXmm &) = delete;
693    InstX86BaseUnaryopXmm &operator=(const InstX86BaseUnaryopXmm &) = delete;
694
695  public:
696    using Base = InstX86BaseUnaryopXmm<K>;
697
698    void emit(const Cfg *Func) const override {
699      if (!BuildDefs::dump())
700        return;
701      Ostream &Str = Func->getContext()->getStrEmit();
702      assert(this->getSrcSize() == 1);
703      Str << "\t" << Opcode << "\t";
704      this->getSrc(0)->emit(Func);
705      Str << ", ";
706      this->getDest()->emit(Func);
707    }
708    void emitIAS(const Cfg *Func) const override {
709      Type Ty = this->getDest()->getType();
710      assert(this->getSrcSize() == 1);
711      emitIASRegOpTyXMM(Func, Ty, this->getDest(), this->getSrc(0), Emitter);
712    }
713    void dump(const Cfg *Func) const override {
714      if (!BuildDefs::dump())
715        return;
716      Ostream &Str = Func->getContext()->getStrDump();
717      this->dumpDest(Func);
718      Str << " = " << Opcode << "." << this->getDest()->getType() << " ";
719      this->dumpSources(Func);
720    }
721    static bool classof(const Inst *Instr) {
722      return InstX86Base::isClassof(Instr, InstX86Base::K);
723    }
724
725  protected:
726    InstX86BaseUnaryopXmm(Cfg *Func, Variable *Dest, Operand *Src)
727        : InstX86Base(Func, K, 1, Dest) {
728      this->addSource(Src);
729    }
730
731    static const char *Opcode;
732    static const XmmEmitterRegOp Emitter;
733  };
734
735  template <typename InstX86Base::InstKindX86 K>
736  class InstX86BaseBinopGPRShift : public InstX86Base {
737    InstX86BaseBinopGPRShift() = delete;
738    InstX86BaseBinopGPRShift(const InstX86BaseBinopGPRShift &) = delete;
739    InstX86BaseBinopGPRShift &
740    operator=(const InstX86BaseBinopGPRShift &) = delete;
741
742  public:
743    using Base = InstX86BaseBinopGPRShift<K>;
744
745    void emit(const Cfg *Func) const override {
746      if (!BuildDefs::dump())
747        return;
748      this->emitTwoAddress(Func, Opcode);
749    }
750    void emitIAS(const Cfg *Func) const override {
751      Type Ty = this->getDest()->getType();
752      assert(this->getSrcSize() == 2);
753      emitIASGPRShift(Func, Ty, this->getDest(), this->getSrc(1), Emitter);
754    }
755    void dump(const Cfg *Func) const override {
756      if (!BuildDefs::dump())
757        return;
758      Ostream &Str = Func->getContext()->getStrDump();
759      this->dumpDest(Func);
760      Str << " = " << Opcode << "." << this->getDest()->getType() << " ";
761      this->dumpSources(Func);
762    }
763    static bool classof(const Inst *Instr) {
764      return InstX86Base::isClassof(Instr, InstX86Base::K);
765    }
766
767  protected:
768    InstX86BaseBinopGPRShift(Cfg *Func, Variable *Dest, Operand *Source)
769        : InstX86Base(Func, K, 2, Dest) {
770      this->addSource(Dest);
771      this->addSource(Source);
772    }
773
774    static const char *Opcode;
775    static const GPREmitterShiftOp Emitter;
776  };
777
778  template <typename InstX86Base::InstKindX86 K>
779  class InstX86BaseBinopGPR : public InstX86Base {
780    InstX86BaseBinopGPR() = delete;
781    InstX86BaseBinopGPR(const InstX86BaseBinopGPR &) = delete;
782    InstX86BaseBinopGPR &operator=(const InstX86BaseBinopGPR &) = delete;
783
784  public:
785    using Base = InstX86BaseBinopGPR<K>;
786
787    void emit(const Cfg *Func) const override {
788      if (!BuildDefs::dump())
789        return;
790      this->emitTwoAddress(Func, Opcode);
791    }
792    void emitIAS(const Cfg *Func) const override {
793      Type Ty = this->getDest()->getType();
794      assert(this->getSrcSize() == 2);
795      constexpr bool ThisIsLEA = K == InstX86Base::Lea;
796      static_assert(!ThisIsLEA, "Lea should be a unaryop.");
797      emitIASRegOpTyGPR(Func, !ThisIsLEA, Ty, this->getDest(), this->getSrc(1),
798                        Emitter);
799    }
800    void dump(const Cfg *Func) const override {
801      if (!BuildDefs::dump())
802        return;
803      Ostream &Str = Func->getContext()->getStrDump();
804      this->dumpDest(Func);
805      Str << " = " << Opcode << "." << this->getDest()->getType() << " ";
806      this->dumpSources(Func);
807    }
808    static bool classof(const Inst *Instr) {
809      return InstX86Base::isClassof(Instr, InstX86Base::K);
810    }
811
812  protected:
813    InstX86BaseBinopGPR(Cfg *Func, Variable *Dest, Operand *Source)
814        : InstX86Base(Func, K, 2, Dest) {
815      this->addSource(Dest);
816      this->addSource(Source);
817    }
818
819    static const char *Opcode;
820    static const GPREmitterRegOp Emitter;
821  };
822
823  template <typename InstX86Base::InstKindX86 K>
824  class InstX86BaseBinopRMW : public InstX86Base {
825    InstX86BaseBinopRMW() = delete;
826    InstX86BaseBinopRMW(const InstX86BaseBinopRMW &) = delete;
827    InstX86BaseBinopRMW &operator=(const InstX86BaseBinopRMW &) = delete;
828
829  public:
830    using Base = InstX86BaseBinopRMW<K>;
831
832    void emit(const Cfg *Func) const override {
833      if (!BuildDefs::dump())
834        return;
835      this->emitTwoAddress(Func, Opcode);
836    }
837    void emitIAS(const Cfg *Func) const override {
838      Type Ty = this->getSrc(0)->getType();
839      assert(this->getSrcSize() == 2);
840      emitIASAsAddrOpTyGPR(Func, Ty, this->getSrc(0), this->getSrc(1), Emitter);
841    }
842    void dump(const Cfg *Func) const override {
843      if (!BuildDefs::dump())
844        return;
845      Ostream &Str = Func->getContext()->getStrDump();
846      Str << Opcode << "." << this->getSrc(0)->getType() << " ";
847      this->dumpSources(Func);
848    }
849    static bool classof(const Inst *Instr) {
850      return InstX86Base::isClassof(Instr, InstX86Base::K);
851    }
852
853  protected:
854    InstX86BaseBinopRMW(Cfg *Func, X86OperandMem *DestSrc0, Operand *Src1)
855        : InstX86Base(Func, K, 2, nullptr) {
856      this->addSource(DestSrc0);
857      this->addSource(Src1);
858    }
859
860    static const char *Opcode;
861    static const GPREmitterAddrOp Emitter;
862  };
863
864  template <typename InstX86Base::InstKindX86 K, bool NeedsElementType,
865            typename InstX86Base::SseSuffix Suffix>
866  class InstX86BaseBinopXmm : public InstX86Base {
867    InstX86BaseBinopXmm() = delete;
868    InstX86BaseBinopXmm(const InstX86BaseBinopXmm &) = delete;
869    InstX86BaseBinopXmm &operator=(const InstX86BaseBinopXmm &) = delete;
870
871  public:
872    using Base = InstX86BaseBinopXmm<K, NeedsElementType, Suffix>;
873
874    void emit(const Cfg *Func) const override {
875      if (!BuildDefs::dump())
876        return;
877      this->validateVectorAddrMode();
878      const Type DestTy = ArithmeticTypeOverride == IceType_void
879                              ? this->getDest()->getType()
880                              : ArithmeticTypeOverride;
881      const char *SuffixString = "";
882      switch (Suffix) {
883      case InstX86Base::SseSuffix::None:
884        break;
885      case InstX86Base::SseSuffix::Packed:
886        SuffixString = Traits::TypeAttributes[DestTy].PdPsString;
887        break;
888      case InstX86Base::SseSuffix::Unpack:
889        SuffixString = Traits::TypeAttributes[DestTy].UnpackString;
890        break;
891      case InstX86Base::SseSuffix::Scalar:
892        SuffixString = Traits::TypeAttributes[DestTy].SdSsString;
893        break;
894      case InstX86Base::SseSuffix::Integral:
895        SuffixString = Traits::TypeAttributes[DestTy].IntegralString;
896        break;
897      case InstX86Base::SseSuffix::Pack:
898        SuffixString = Traits::TypeAttributes[DestTy].PackString;
899        break;
900      }
901      this->emitTwoAddress(Func, Opcode, SuffixString);
902    }
903    void emitIAS(const Cfg *Func) const override {
904      this->validateVectorAddrMode();
905      Type Ty = this->getDest()->getType();
906      if (NeedsElementType)
907        Ty = typeElementType(Ty);
908      assert(this->getSrcSize() == 2);
909      emitIASRegOpTyXMM(Func, Ty, this->getDest(), this->getSrc(1), Emitter);
910    }
911    void dump(const Cfg *Func) const override {
912      if (!BuildDefs::dump())
913        return;
914      Ostream &Str = Func->getContext()->getStrDump();
915      this->dumpDest(Func);
916      Str << " = " << Opcode << "." << this->getDest()->getType() << " ";
917      this->dumpSources(Func);
918    }
919    static bool classof(const Inst *Instr) {
920      return InstX86Base::isClassof(Instr, InstX86Base::K);
921    }
922
923  protected:
924    InstX86BaseBinopXmm(Cfg *Func, Variable *Dest, Operand *Source,
925                        Type ArithmeticTypeOverride = IceType_void)
926        : InstX86Base(Func, K, 2, Dest),
927          ArithmeticTypeOverride(ArithmeticTypeOverride) {
928      this->addSource(Dest);
929      this->addSource(Source);
930    }
931
932    const Type ArithmeticTypeOverride;
933    static const char *Opcode;
934    static const XmmEmitterRegOp Emitter;
935  };
936
937  template <typename InstX86Base::InstKindX86 K, bool AllowAllTypes = false>
938  class InstX86BaseBinopXmmShift : public InstX86Base {
939    InstX86BaseBinopXmmShift() = delete;
940    InstX86BaseBinopXmmShift(const InstX86BaseBinopXmmShift &) = delete;
941    InstX86BaseBinopXmmShift &
942    operator=(const InstX86BaseBinopXmmShift &) = delete;
943
944  public:
945    using Base = InstX86BaseBinopXmmShift<K, AllowAllTypes>;
946
947    void emit(const Cfg *Func) const override {
948      if (!BuildDefs::dump())
949        return;
950      this->validateVectorAddrMode();
951      // Shift operations are always integral, and hence always need a suffix.
952      const Type DestTy = this->getDest()->getType();
953      this->emitTwoAddress(Func, this->Opcode,
954                           Traits::TypeAttributes[DestTy].IntegralString);
955    }
956    void emitIAS(const Cfg *Func) const override {
957      this->validateVectorAddrMode();
958      Type Ty = this->getDest()->getType();
959      assert(AllowAllTypes || isVectorType(Ty));
960      Type ElementTy = typeElementType(Ty);
961      assert(this->getSrcSize() == 2);
962      emitIASXmmShift(Func, ElementTy, this->getDest(), this->getSrc(1),
963                      Emitter);
964    }
965    void dump(const Cfg *Func) const override {
966      if (!BuildDefs::dump())
967        return;
968      Ostream &Str = Func->getContext()->getStrDump();
969      this->dumpDest(Func);
970      Str << " = " << Opcode << "." << this->getDest()->getType() << " ";
971      this->dumpSources(Func);
972    }
973    static bool classof(const Inst *Instr) {
974      return InstX86Base::isClassof(Instr, InstX86Base::K);
975    }
976
977  protected:
978    InstX86BaseBinopXmmShift(Cfg *Func, Variable *Dest, Operand *Source)
979        : InstX86Base(Func, K, 2, Dest) {
980      this->addSource(Dest);
981      this->addSource(Source);
982    }
983
984    static const char *Opcode;
985    static const XmmEmitterShiftOp Emitter;
986  };
987
988  template <typename InstX86Base::InstKindX86 K>
989  class InstX86BaseTernop : public InstX86Base {
990    InstX86BaseTernop() = delete;
991    InstX86BaseTernop(const InstX86BaseTernop &) = delete;
992    InstX86BaseTernop &operator=(const InstX86BaseTernop &) = delete;
993
994  public:
995    using Base = InstX86BaseTernop<K>;
996
997    void emit(const Cfg *Func) const override {
998      if (!BuildDefs::dump())
999        return;
1000      Ostream &Str = Func->getContext()->getStrEmit();
1001      assert(this->getSrcSize() == 3);
1002      Str << "\t" << Opcode << "\t";
1003      this->getSrc(2)->emit(Func);
1004      Str << ", ";
1005      this->getSrc(1)->emit(Func);
1006      Str << ", ";
1007      this->getDest()->emit(Func);
1008    }
1009    void dump(const Cfg *Func) const override {
1010      if (!BuildDefs::dump())
1011        return;
1012      Ostream &Str = Func->getContext()->getStrDump();
1013      this->dumpDest(Func);
1014      Str << " = " << Opcode << "." << this->getDest()->getType() << " ";
1015      this->dumpSources(Func);
1016    }
1017    static bool classof(const Inst *Instr) {
1018      return InstX86Base::isClassof(Instr, InstX86Base::K);
1019    }
1020
1021  protected:
1022    InstX86BaseTernop(Cfg *Func, Variable *Dest, Operand *Source1,
1023                      Operand *Source2)
1024        : InstX86Base(Func, K, 3, Dest) {
1025      this->addSource(Dest);
1026      this->addSource(Source1);
1027      this->addSource(Source2);
1028    }
1029
1030    static const char *Opcode;
1031  };
1032
1033  // Instructions of the form x := y op z
1034  template <typename InstX86Base::InstKindX86 K>
1035  class InstX86BaseThreeAddressop : public InstX86Base {
1036    InstX86BaseThreeAddressop() = delete;
1037    InstX86BaseThreeAddressop(const InstX86BaseThreeAddressop &) = delete;
1038    InstX86BaseThreeAddressop &
1039    operator=(const InstX86BaseThreeAddressop &) = delete;
1040
1041  public:
1042    using Base = InstX86BaseThreeAddressop<K>;
1043
1044    void emit(const Cfg *Func) const override {
1045      if (!BuildDefs::dump())
1046        return;
1047      Ostream &Str = Func->getContext()->getStrEmit();
1048      assert(this->getSrcSize() == 2);
1049      Str << "\t" << Opcode << "\t";
1050      this->getSrc(1)->emit(Func);
1051      Str << ", ";
1052      this->getSrc(0)->emit(Func);
1053      Str << ", ";
1054      this->getDest()->emit(Func);
1055    }
1056    void dump(const Cfg *Func) const override {
1057      if (!BuildDefs::dump())
1058        return;
1059      Ostream &Str = Func->getContext()->getStrDump();
1060      this->dumpDest(Func);
1061      Str << " = " << Opcode << "." << this->getDest()->getType() << " ";
1062      this->dumpSources(Func);
1063    }
1064    static bool classof(const Inst *Instr) {
1065      return InstX86Base::isClassof(Instr, InstX86Base::K);
1066    }
1067
1068  protected:
1069    InstX86BaseThreeAddressop(Cfg *Func, Variable *Dest, Operand *Source0,
1070                              Operand *Source1)
1071        : InstX86Base(Func, K, 2, Dest) {
1072      this->addSource(Source0);
1073      this->addSource(Source1);
1074    }
1075
1076    static const char *Opcode;
1077  };
1078
1079  /// Base class for assignment instructions
1080  template <typename InstX86Base::InstKindX86 K>
1081  class InstX86BaseMovlike : public InstX86Base {
1082    InstX86BaseMovlike() = delete;
1083    InstX86BaseMovlike(const InstX86BaseMovlike &) = delete;
1084    InstX86BaseMovlike &operator=(const InstX86BaseMovlike &) = delete;
1085
1086  public:
1087    using Base = InstX86BaseMovlike<K>;
1088
1089    bool isRedundantAssign() const override {
1090      if (const auto *SrcVar =
1091              llvm::dyn_cast<const Variable>(this->getSrc(0))) {
1092        if (SrcVar->hasReg() && this->Dest->hasReg()) {
1093          // An assignment between physical registers is considered redundant if
1094          // they have the same base register and the same encoding. E.g.:
1095          //   mov cl, ecx ==> redundant
1096          //   mov ch, ecx ==> not redundant due to different encodings
1097          //   mov ch, ebp ==> not redundant due to different base registers
1098          //   mov ecx, ecx ==> redundant, and dangerous in x86-64. i64 zexting
1099          //                    is handled by Inst86Zext.
1100          const auto SrcReg = SrcVar->getRegNum();
1101          const auto DestReg = this->Dest->getRegNum();
1102          return (Traits::getEncoding(SrcReg) ==
1103                  Traits::getEncoding(DestReg)) &&
1104                 (Traits::getBaseReg(SrcReg) == Traits::getBaseReg(DestReg));
1105        }
1106      }
1107      return checkForRedundantAssign(this->getDest(), this->getSrc(0));
1108    }
1109    bool isVarAssign() const override {
1110      return llvm::isa<Variable>(this->getSrc(0));
1111    }
1112    void dump(const Cfg *Func) const override {
1113      if (!BuildDefs::dump())
1114        return;
1115      Ostream &Str = Func->getContext()->getStrDump();
1116      Str << Opcode << "." << this->getDest()->getType() << " ";
1117      this->dumpDest(Func);
1118      Str << ", ";
1119      this->dumpSources(Func);
1120    }
1121    static bool classof(const Inst *Instr) {
1122      return InstX86Base::isClassof(Instr, InstX86Base::K);
1123    }
1124
1125  protected:
1126    InstX86BaseMovlike(Cfg *Func, Variable *Dest, Operand *Source)
1127        : InstX86Base(Func, K, 1, Dest) {
1128      this->addSource(Source);
1129      // For an integer assignment, make sure it's either a same-type assignment
1130      // or a truncation.
1131      assert(!isScalarIntegerType(Dest->getType()) ||
1132             (typeWidthInBytes(Dest->getType()) <=
1133              typeWidthInBytes(Source->getType())));
1134    }
1135
1136    static const char *Opcode;
1137  };
1138
1139  class InstX86Bswap : public InstX86BaseInplaceopGPR<InstX86Base::Bswap> {
1140  public:
1141    static InstX86Bswap *create(Cfg *Func, Operand *SrcDest) {
1142      return new (Func->allocate<InstX86Bswap>()) InstX86Bswap(Func, SrcDest);
1143    }
1144
1145  private:
1146    InstX86Bswap(Cfg *Func, Operand *SrcDest)
1147        : InstX86BaseInplaceopGPR<InstX86Base::Bswap>(Func, SrcDest) {}
1148  };
1149
1150  class InstX86Neg : public InstX86BaseInplaceopGPR<InstX86Base::Neg> {
1151  public:
1152    static InstX86Neg *create(Cfg *Func, Operand *SrcDest) {
1153      return new (Func->allocate<InstX86Neg>()) InstX86Neg(Func, SrcDest);
1154    }
1155
1156  private:
1157    InstX86Neg(Cfg *Func, Operand *SrcDest)
1158        : InstX86BaseInplaceopGPR<InstX86Base::Neg>(Func, SrcDest) {}
1159  };
1160
1161  class InstX86Bsf : public InstX86BaseUnaryopGPR<InstX86Base::Bsf> {
1162  public:
1163    static InstX86Bsf *create(Cfg *Func, Variable *Dest, Operand *Src) {
1164      return new (Func->allocate<InstX86Bsf>()) InstX86Bsf(Func, Dest, Src);
1165    }
1166
1167  private:
1168    InstX86Bsf(Cfg *Func, Variable *Dest, Operand *Src)
1169        : InstX86BaseUnaryopGPR<InstX86Base::Bsf>(Func, Dest, Src) {}
1170  };
1171
1172  class InstX86Bsr : public InstX86BaseUnaryopGPR<InstX86Base::Bsr> {
1173  public:
1174    static InstX86Bsr *create(Cfg *Func, Variable *Dest, Operand *Src) {
1175      return new (Func->allocate<InstX86Bsr>()) InstX86Bsr(Func, Dest, Src);
1176    }
1177
1178  private:
1179    InstX86Bsr(Cfg *Func, Variable *Dest, Operand *Src)
1180        : InstX86BaseUnaryopGPR<InstX86Base::Bsr>(Func, Dest, Src) {}
1181  };
1182
1183  class InstX86Lea : public InstX86BaseUnaryopGPR<InstX86Base::Lea> {
1184  public:
1185    static InstX86Lea *create(Cfg *Func, Variable *Dest, Operand *Src) {
1186      return new (Func->allocate<InstX86Lea>()) InstX86Lea(Func, Dest, Src);
1187    }
1188
1189    void emit(const Cfg *Func) const override;
1190
1191  private:
1192    InstX86Lea(Cfg *Func, Variable *Dest, Operand *Src)
1193        : InstX86BaseUnaryopGPR<InstX86Base::Lea>(Func, Dest, Src) {}
1194  };
1195
1196  // Cbwdq instruction - wrapper for cbw, cwd, and cdq
1197  class InstX86Cbwdq : public InstX86BaseUnaryopGPR<InstX86Base::Cbwdq> {
1198  public:
1199    static InstX86Cbwdq *create(Cfg *Func, Variable *Dest, Operand *Src) {
1200      return new (Func->allocate<InstX86Cbwdq>()) InstX86Cbwdq(Func, Dest, Src);
1201    }
1202
1203    void emit(const Cfg *Func) const override;
1204    void emitIAS(const Cfg *Func) const override;
1205
1206  private:
1207    InstX86Cbwdq(Cfg *Func, Variable *Dest, Operand *Src)
1208        : InstX86BaseUnaryopGPR<InstX86Base::Cbwdq>(Func, Dest, Src) {}
1209  };
1210
1211  class InstX86Movsx : public InstX86BaseUnaryopGPR<InstX86Base::Movsx> {
1212  public:
1213    static InstX86Movsx *create(Cfg *Func, Variable *Dest, Operand *Src) {
1214      assert(typeWidthInBytes(Dest->getType()) >
1215             typeWidthInBytes(Src->getType()));
1216      return new (Func->allocate<InstX86Movsx>()) InstX86Movsx(Func, Dest, Src);
1217    }
1218
1219    void emitIAS(const Cfg *Func) const override;
1220
1221  private:
1222    InstX86Movsx(Cfg *Func, Variable *Dest, Operand *Src)
1223        : InstX86BaseUnaryopGPR<InstX86Base::Movsx>(Func, Dest, Src) {}
1224  };
1225
1226  class InstX86Movzx : public InstX86BaseUnaryopGPR<InstX86Base::Movzx> {
1227  public:
1228    static InstX86Movzx *create(Cfg *Func, Variable *Dest, Operand *Src) {
1229      assert(typeWidthInBytes(Dest->getType()) >
1230             typeWidthInBytes(Src->getType()));
1231      return new (Func->allocate<InstX86Movzx>()) InstX86Movzx(Func, Dest, Src);
1232    }
1233
1234    void emit(const Cfg *Func) const override;
1235
1236    void emitIAS(const Cfg *Func) const override;
1237
1238    void setMustKeep() { MustKeep = true; }
1239
1240  private:
1241    bool MustKeep = false;
1242
1243    InstX86Movzx(Cfg *Func, Variable *Dest, Operand *Src)
1244        : InstX86BaseUnaryopGPR<InstX86Base::Movzx>(Func, Dest, Src) {}
1245
1246    bool mayBeElided(const Variable *Dest, const Operand *Src) const;
1247  };
1248
1249  class InstX86Movd : public InstX86BaseUnaryopXmm<InstX86Base::Movd> {
1250  public:
1251    static InstX86Movd *create(Cfg *Func, Variable *Dest, Operand *Src) {
1252      return new (Func->allocate<InstX86Movd>()) InstX86Movd(Func, Dest, Src);
1253    }
1254
1255    void emit(const Cfg *Func) const override;
1256
1257    void emitIAS(const Cfg *Func) const override;
1258
1259  private:
1260    InstX86Movd(Cfg *Func, Variable *Dest, Operand *Src)
1261        : InstX86BaseUnaryopXmm<InstX86Base::Movd>(Func, Dest, Src) {}
1262  };
1263
1264  class InstX86Movmsk final : public InstX86Base {
1265    InstX86Movmsk() = delete;
1266    InstX86Movmsk(const InstX86Movmsk &) = delete;
1267    InstX86Movmsk &operator=(const InstX86Movmsk &) = delete;
1268
1269  public:
1270    static InstX86Movmsk *create(Cfg *Func, Variable *Dest, Operand *Source) {
1271      return new (Func->allocate<InstX86Movmsk>())
1272          InstX86Movmsk(Func, Dest, Source);
1273    }
1274    void emit(const Cfg *Func) const override;
1275    void emitIAS(const Cfg *Func) const override;
1276    void dump(const Cfg *Func) const override;
1277    static bool classof(const Inst *Instr) {
1278      return InstX86Base::isClassof(Instr, InstX86Base::InstX86Movmsk);
1279    }
1280
1281  private:
1282    InstX86Movmsk(Cfg *Func, Variable *Dest, Operand *Source);
1283  };
1284
1285  class InstX86Sqrt : public InstX86BaseUnaryopXmm<InstX86Base::Sqrt> {
1286  public:
1287    static InstX86Sqrt *create(Cfg *Func, Variable *Dest, Operand *Src) {
1288      return new (Func->allocate<InstX86Sqrt>()) InstX86Sqrt(Func, Dest, Src);
1289    }
1290
1291    virtual void emit(const Cfg *Func) const override;
1292
1293  private:
1294    InstX86Sqrt(Cfg *Func, Variable *Dest, Operand *Src)
1295        : InstX86BaseUnaryopXmm<InstX86Base::Sqrt>(Func, Dest, Src) {}
1296  };
1297
1298  /// Move/assignment instruction - wrapper for mov/movss/movsd.
1299  class InstX86Mov : public InstX86BaseMovlike<InstX86Base::Mov> {
1300  public:
1301    static InstX86Mov *create(Cfg *Func, Variable *Dest, Operand *Source) {
1302      assert(!isScalarIntegerType(Dest->getType()) ||
1303             (typeWidthInBytes(Dest->getType()) <=
1304              typeWidthInBytes(Source->getType())));
1305      return new (Func->allocate<InstX86Mov>()) InstX86Mov(Func, Dest, Source);
1306    }
1307
1308    void emit(const Cfg *Func) const override;
1309    void emitIAS(const Cfg *Func) const override;
1310
1311  private:
1312    InstX86Mov(Cfg *Func, Variable *Dest, Operand *Source)
1313        : InstX86BaseMovlike<InstX86Base::Mov>(Func, Dest, Source) {}
1314  };
1315
1316  /// Move packed - copy 128 bit values between XMM registers, or mem128 and XMM
1317  /// registers.
1318  class InstX86Movp : public InstX86BaseMovlike<InstX86Base::Movp> {
1319  public:
1320    static InstX86Movp *create(Cfg *Func, Variable *Dest, Operand *Source) {
1321      return new (Func->allocate<InstX86Movp>())
1322          InstX86Movp(Func, Dest, Source);
1323    }
1324
1325    void emit(const Cfg *Func) const override;
1326    void emitIAS(const Cfg *Func) const override;
1327
1328  private:
1329    InstX86Movp(Cfg *Func, Variable *Dest, Operand *Source)
1330        : InstX86BaseMovlike<InstX86Base::Movp>(Func, Dest, Source) {}
1331  };
1332
1333  /// Movq - copy between XMM registers, or mem64 and XMM registers.
1334  class InstX86Movq : public InstX86BaseMovlike<InstX86Base::Movq> {
1335  public:
1336    static InstX86Movq *create(Cfg *Func, Variable *Dest, Operand *Source) {
1337      return new (Func->allocate<InstX86Movq>())
1338          InstX86Movq(Func, Dest, Source);
1339    }
1340
1341    void emit(const Cfg *Func) const override;
1342    void emitIAS(const Cfg *Func) const override;
1343
1344  private:
1345    InstX86Movq(Cfg *Func, Variable *Dest, Operand *Source)
1346        : InstX86BaseMovlike<InstX86Base::Movq>(Func, Dest, Source) {}
1347  };
1348
1349  class InstX86Add : public InstX86BaseBinopGPR<InstX86Base::Add> {
1350  public:
1351    static InstX86Add *create(Cfg *Func, Variable *Dest, Operand *Source) {
1352      return new (Func->allocate<InstX86Add>()) InstX86Add(Func, Dest, Source);
1353    }
1354
1355  private:
1356    InstX86Add(Cfg *Func, Variable *Dest, Operand *Source)
1357        : InstX86BaseBinopGPR<InstX86Base::Add>(Func, Dest, Source) {}
1358  };
1359
1360  class InstX86AddRMW : public InstX86BaseBinopRMW<InstX86Base::AddRMW> {
1361  public:
1362    static InstX86AddRMW *create(Cfg *Func, X86OperandMem *DestSrc0,
1363                                 Operand *Src1) {
1364      return new (Func->allocate<InstX86AddRMW>())
1365          InstX86AddRMW(Func, DestSrc0, Src1);
1366    }
1367
1368  private:
1369    InstX86AddRMW(Cfg *Func, X86OperandMem *DestSrc0, Operand *Src1)
1370        : InstX86BaseBinopRMW<InstX86Base::AddRMW>(Func, DestSrc0, Src1) {}
1371  };
1372
1373  class InstX86Addps
1374      : public InstX86BaseBinopXmm<InstX86Base::Addps, true,
1375                                   InstX86Base::SseSuffix::Packed> {
1376  public:
1377    static InstX86Addps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1378      return new (Func->allocate<InstX86Addps>())
1379          InstX86Addps(Func, Dest, Source);
1380    }
1381
1382  private:
1383    InstX86Addps(Cfg *Func, Variable *Dest, Operand *Source)
1384        : InstX86BaseBinopXmm<InstX86Base::Addps, true,
1385                              InstX86Base::SseSuffix::Packed>(Func, Dest,
1386                                                              Source) {}
1387  };
1388
1389  class InstX86Adc : public InstX86BaseBinopGPR<InstX86Base::Adc> {
1390  public:
1391    static InstX86Adc *create(Cfg *Func, Variable *Dest, Operand *Source) {
1392      return new (Func->allocate<InstX86Adc>()) InstX86Adc(Func, Dest, Source);
1393    }
1394
1395  private:
1396    InstX86Adc(Cfg *Func, Variable *Dest, Operand *Source)
1397        : InstX86BaseBinopGPR<InstX86Base::Adc>(Func, Dest, Source) {}
1398  };
1399
1400  class InstX86AdcRMW : public InstX86BaseBinopRMW<InstX86Base::AdcRMW> {
1401  public:
1402    static InstX86AdcRMW *create(Cfg *Func, X86OperandMem *DestSrc0,
1403                                 Operand *Src1) {
1404      return new (Func->allocate<InstX86AdcRMW>())
1405          InstX86AdcRMW(Func, DestSrc0, Src1);
1406    }
1407
1408  private:
1409    InstX86AdcRMW(Cfg *Func, X86OperandMem *DestSrc0, Operand *Src1)
1410        : InstX86BaseBinopRMW<InstX86Base::AdcRMW>(Func, DestSrc0, Src1) {}
1411  };
1412
1413  class InstX86Addss
1414      : public InstX86BaseBinopXmm<InstX86Base::Addss, false,
1415                                   InstX86Base::SseSuffix::Scalar> {
1416  public:
1417    static InstX86Addss *create(Cfg *Func, Variable *Dest, Operand *Source) {
1418      return new (Func->allocate<InstX86Addss>())
1419          InstX86Addss(Func, Dest, Source);
1420    }
1421
1422  private:
1423    InstX86Addss(Cfg *Func, Variable *Dest, Operand *Source)
1424        : InstX86BaseBinopXmm<InstX86Base::Addss, false,
1425                              InstX86Base::SseSuffix::Scalar>(Func, Dest,
1426                                                              Source) {}
1427  };
1428
1429  class InstX86Padd
1430      : public InstX86BaseBinopXmm<InstX86Base::Padd, true,
1431                                   InstX86Base::SseSuffix::Integral> {
1432  public:
1433    static InstX86Padd *create(Cfg *Func, Variable *Dest, Operand *Source) {
1434      return new (Func->allocate<InstX86Padd>())
1435          InstX86Padd(Func, Dest, Source);
1436    }
1437
1438  private:
1439    InstX86Padd(Cfg *Func, Variable *Dest, Operand *Source)
1440        : InstX86BaseBinopXmm<InstX86Base::Padd, true,
1441                              InstX86Base::SseSuffix::Integral>(Func, Dest,
1442                                                                Source) {}
1443  };
1444
1445  class InstX86Padds
1446      : public InstX86BaseBinopXmm<InstX86Base::Padds, true,
1447                                   InstX86Base::SseSuffix::Integral> {
1448  public:
1449    static InstX86Padds *create(Cfg *Func, Variable *Dest, Operand *Source) {
1450      return new (Func->allocate<InstX86Padds>())
1451          InstX86Padds(Func, Dest, Source);
1452    }
1453
1454  private:
1455    InstX86Padds(Cfg *Func, Variable *Dest, Operand *Source)
1456        : InstX86BaseBinopXmm<InstX86Base::Padds, true,
1457                              InstX86Base::SseSuffix::Integral>(Func, Dest,
1458                                                                Source) {}
1459  };
1460
1461  class InstX86Paddus
1462      : public InstX86BaseBinopXmm<InstX86Base::Paddus, true,
1463                                   InstX86Base::SseSuffix::Integral> {
1464  public:
1465    static InstX86Paddus *create(Cfg *Func, Variable *Dest, Operand *Source) {
1466      return new (Func->allocate<InstX86Paddus>())
1467          InstX86Paddus(Func, Dest, Source);
1468    }
1469
1470  private:
1471    InstX86Paddus(Cfg *Func, Variable *Dest, Operand *Source)
1472        : InstX86BaseBinopXmm<InstX86Base::Paddus, true,
1473                              InstX86Base::SseSuffix::Integral>(Func, Dest,
1474                                                                Source) {}
1475  };
1476
1477  class InstX86Sub : public InstX86BaseBinopGPR<InstX86Base::Sub> {
1478  public:
1479    static InstX86Sub *create(Cfg *Func, Variable *Dest, Operand *Source) {
1480      return new (Func->allocate<InstX86Sub>()) InstX86Sub(Func, Dest, Source);
1481    }
1482
1483  private:
1484    InstX86Sub(Cfg *Func, Variable *Dest, Operand *Source)
1485        : InstX86BaseBinopGPR<InstX86Base::Sub>(Func, Dest, Source) {}
1486  };
1487
1488  class InstX86SubRMW : public InstX86BaseBinopRMW<InstX86Base::SubRMW> {
1489  public:
1490    static InstX86SubRMW *create(Cfg *Func, X86OperandMem *DestSrc0,
1491                                 Operand *Src1) {
1492      return new (Func->allocate<InstX86SubRMW>())
1493          InstX86SubRMW(Func, DestSrc0, Src1);
1494    }
1495
1496  private:
1497    InstX86SubRMW(Cfg *Func, X86OperandMem *DestSrc0, Operand *Src1)
1498        : InstX86BaseBinopRMW<InstX86Base::SubRMW>(Func, DestSrc0, Src1) {}
1499  };
1500
1501  class InstX86Subps
1502      : public InstX86BaseBinopXmm<InstX86Base::Subps, true,
1503                                   InstX86Base::SseSuffix::Packed> {
1504  public:
1505    static InstX86Subps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1506      return new (Func->allocate<InstX86Subps>())
1507          InstX86Subps(Func, Dest, Source);
1508    }
1509
1510  private:
1511    InstX86Subps(Cfg *Func, Variable *Dest, Operand *Source)
1512        : InstX86BaseBinopXmm<InstX86Base::Subps, true,
1513                              InstX86Base::SseSuffix::Packed>(Func, Dest,
1514                                                              Source) {}
1515  };
1516
1517  class InstX86Subss
1518      : public InstX86BaseBinopXmm<InstX86Base::Subss, false,
1519                                   InstX86Base::SseSuffix::Scalar> {
1520  public:
1521    static InstX86Subss *create(Cfg *Func, Variable *Dest, Operand *Source) {
1522      return new (Func->allocate<InstX86Subss>())
1523          InstX86Subss(Func, Dest, Source);
1524    }
1525
1526  private:
1527    InstX86Subss(Cfg *Func, Variable *Dest, Operand *Source)
1528        : InstX86BaseBinopXmm<InstX86Base::Subss, false,
1529                              InstX86Base::SseSuffix::Scalar>(Func, Dest,
1530                                                              Source) {}
1531  };
1532
1533  class InstX86Sbb : public InstX86BaseBinopGPR<InstX86Base::Sbb> {
1534  public:
1535    static InstX86Sbb *create(Cfg *Func, Variable *Dest, Operand *Source) {
1536      return new (Func->allocate<InstX86Sbb>()) InstX86Sbb(Func, Dest, Source);
1537    }
1538
1539  private:
1540    InstX86Sbb(Cfg *Func, Variable *Dest, Operand *Source)
1541        : InstX86BaseBinopGPR<InstX86Base::Sbb>(Func, Dest, Source) {}
1542  };
1543
1544  class InstX86SbbRMW : public InstX86BaseBinopRMW<InstX86Base::SbbRMW> {
1545  public:
1546    static InstX86SbbRMW *create(Cfg *Func, X86OperandMem *DestSrc0,
1547                                 Operand *Src1) {
1548      return new (Func->allocate<InstX86SbbRMW>())
1549          InstX86SbbRMW(Func, DestSrc0, Src1);
1550    }
1551
1552  private:
1553    InstX86SbbRMW(Cfg *Func, X86OperandMem *DestSrc0, Operand *Src1)
1554        : InstX86BaseBinopRMW<InstX86Base::SbbRMW>(Func, DestSrc0, Src1) {}
1555  };
1556
1557  class InstX86Psub
1558      : public InstX86BaseBinopXmm<InstX86Base::Psub, true,
1559                                   InstX86Base::SseSuffix::Integral> {
1560  public:
1561    static InstX86Psub *create(Cfg *Func, Variable *Dest, Operand *Source) {
1562      return new (Func->allocate<InstX86Psub>())
1563          InstX86Psub(Func, Dest, Source);
1564    }
1565
1566  private:
1567    InstX86Psub(Cfg *Func, Variable *Dest, Operand *Source)
1568        : InstX86BaseBinopXmm<InstX86Base::Psub, true,
1569                              InstX86Base::SseSuffix::Integral>(Func, Dest,
1570                                                                Source) {}
1571  };
1572
1573  class InstX86Psubs
1574      : public InstX86BaseBinopXmm<InstX86Base::Psubs, true,
1575                                   InstX86Base::SseSuffix::Integral> {
1576  public:
1577    static InstX86Psubs *create(Cfg *Func, Variable *Dest, Operand *Source) {
1578      return new (Func->allocate<InstX86Psubs>())
1579          InstX86Psubs(Func, Dest, Source);
1580    }
1581
1582  private:
1583    InstX86Psubs(Cfg *Func, Variable *Dest, Operand *Source)
1584        : InstX86BaseBinopXmm<InstX86Base::Psubs, true,
1585                              InstX86Base::SseSuffix::Integral>(Func, Dest,
1586                                                                Source) {}
1587  };
1588
1589  class InstX86Psubus
1590      : public InstX86BaseBinopXmm<InstX86Base::Psubus, true,
1591                                   InstX86Base::SseSuffix::Integral> {
1592  public:
1593    static InstX86Psubus *create(Cfg *Func, Variable *Dest, Operand *Source) {
1594      return new (Func->allocate<InstX86Psubus>())
1595          InstX86Psubus(Func, Dest, Source);
1596    }
1597
1598  private:
1599    InstX86Psubus(Cfg *Func, Variable *Dest, Operand *Source)
1600        : InstX86BaseBinopXmm<InstX86Base::Psubus, true,
1601                              InstX86Base::SseSuffix::Integral>(Func, Dest,
1602                                                                Source) {}
1603  };
1604
1605  class InstX86And : public InstX86BaseBinopGPR<InstX86Base::And> {
1606  public:
1607    static InstX86And *create(Cfg *Func, Variable *Dest, Operand *Source) {
1608      return new (Func->allocate<InstX86And>()) InstX86And(Func, Dest, Source);
1609    }
1610
1611  private:
1612    InstX86And(Cfg *Func, Variable *Dest, Operand *Source)
1613        : InstX86BaseBinopGPR<InstX86Base::And>(Func, Dest, Source) {}
1614  };
1615
1616  class InstX86Andnps
1617      : public InstX86BaseBinopXmm<InstX86Base::Andnps, true,
1618                                   InstX86Base::SseSuffix::Packed> {
1619  public:
1620    static InstX86Andnps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1621      return new (Func->allocate<InstX86Andnps>())
1622          InstX86Andnps(Func, Dest, Source);
1623    }
1624
1625  private:
1626    InstX86Andnps(Cfg *Func, Variable *Dest, Operand *Source)
1627        : InstX86BaseBinopXmm<InstX86Base::Andnps, true,
1628                              InstX86Base::SseSuffix::Packed>(Func, Dest,
1629                                                              Source) {}
1630  };
1631
1632  class InstX86Andps
1633      : public InstX86BaseBinopXmm<InstX86Base::Andps, true,
1634                                   InstX86Base::SseSuffix::Packed> {
1635  public:
1636    static InstX86Andps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1637      return new (Func->allocate<InstX86Andps>())
1638          InstX86Andps(Func, Dest, Source);
1639    }
1640
1641  private:
1642    InstX86Andps(Cfg *Func, Variable *Dest, Operand *Source)
1643        : InstX86BaseBinopXmm<InstX86Base::Andps, true,
1644                              InstX86Base::SseSuffix::Packed>(Func, Dest,
1645                                                              Source) {}
1646  };
1647
1648  class InstX86AndRMW : public InstX86BaseBinopRMW<InstX86Base::AndRMW> {
1649  public:
1650    static InstX86AndRMW *create(Cfg *Func, X86OperandMem *DestSrc0,
1651                                 Operand *Src1) {
1652      return new (Func->allocate<InstX86AndRMW>())
1653          InstX86AndRMW(Func, DestSrc0, Src1);
1654    }
1655
1656  private:
1657    InstX86AndRMW(Cfg *Func, X86OperandMem *DestSrc0, Operand *Src1)
1658        : InstX86BaseBinopRMW<InstX86Base::AndRMW>(Func, DestSrc0, Src1) {}
1659  };
1660
1661  class InstX86Pand : public InstX86BaseBinopXmm<InstX86Base::Pand, false,
1662                                                 InstX86Base::SseSuffix::None> {
1663  public:
1664    static InstX86Pand *create(Cfg *Func, Variable *Dest, Operand *Source) {
1665      return new (Func->allocate<InstX86Pand>())
1666          InstX86Pand(Func, Dest, Source);
1667    }
1668
1669  private:
1670    InstX86Pand(Cfg *Func, Variable *Dest, Operand *Source)
1671        : InstX86BaseBinopXmm<InstX86Base::Pand, false,
1672                              InstX86Base::SseSuffix::None>(Func, Dest,
1673                                                            Source) {}
1674  };
1675
1676  class InstX86Pandn
1677      : public InstX86BaseBinopXmm<InstX86Base::Pandn, false,
1678                                   InstX86Base::SseSuffix::None> {
1679  public:
1680    static InstX86Pandn *create(Cfg *Func, Variable *Dest, Operand *Source) {
1681      return new (Func->allocate<InstX86Pandn>())
1682          InstX86Pandn(Func, Dest, Source);
1683    }
1684
1685  private:
1686    InstX86Pandn(Cfg *Func, Variable *Dest, Operand *Source)
1687        : InstX86BaseBinopXmm<InstX86Base::Pandn, false,
1688                              InstX86Base::SseSuffix::None>(Func, Dest,
1689                                                            Source) {}
1690  };
1691
1692  class InstX86Maxss
1693      : public InstX86BaseBinopXmm<InstX86Base::Maxss, true,
1694                                   InstX86Base::SseSuffix::Scalar> {
1695  public:
1696    static InstX86Maxss *create(Cfg *Func, Variable *Dest, Operand *Source) {
1697      return new (Func->allocate<InstX86Maxss>())
1698          InstX86Maxss(Func, Dest, Source);
1699    }
1700
1701  private:
1702    InstX86Maxss(Cfg *Func, Variable *Dest, Operand *Source)
1703        : InstX86BaseBinopXmm<InstX86Base::Maxss, true,
1704                              InstX86Base::SseSuffix::Scalar>(Func, Dest,
1705                                                              Source) {}
1706  };
1707
1708  class InstX86Minss
1709      : public InstX86BaseBinopXmm<InstX86Base::Minss, true,
1710                                   InstX86Base::SseSuffix::Scalar> {
1711  public:
1712    static InstX86Minss *create(Cfg *Func, Variable *Dest, Operand *Source) {
1713      return new (Func->allocate<InstX86Minss>())
1714          InstX86Minss(Func, Dest, Source);
1715    }
1716
1717  private:
1718    InstX86Minss(Cfg *Func, Variable *Dest, Operand *Source)
1719        : InstX86BaseBinopXmm<InstX86Base::Minss, true,
1720                              InstX86Base::SseSuffix::Scalar>(Func, Dest,
1721                                                              Source) {}
1722  };
1723
1724  class InstX86Maxps
1725      : public InstX86BaseBinopXmm<InstX86Base::Maxps, true,
1726                                   InstX86Base::SseSuffix::None> {
1727  public:
1728    static InstX86Maxps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1729      return new (Func->allocate<InstX86Maxps>())
1730          InstX86Maxps(Func, Dest, Source);
1731    }
1732
1733  private:
1734    InstX86Maxps(Cfg *Func, Variable *Dest, Operand *Source)
1735        : InstX86BaseBinopXmm<InstX86Base::Maxps, true,
1736                              InstX86Base::SseSuffix::None>(Func, Dest,
1737                                                            Source) {}
1738  };
1739
1740  class InstX86Minps
1741      : public InstX86BaseBinopXmm<InstX86Base::Minps, true,
1742                                   InstX86Base::SseSuffix::None> {
1743  public:
1744    static InstX86Minps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1745      return new (Func->allocate<InstX86Minps>())
1746          InstX86Minps(Func, Dest, Source);
1747    }
1748
1749  private:
1750    InstX86Minps(Cfg *Func, Variable *Dest, Operand *Source)
1751        : InstX86BaseBinopXmm<InstX86Base::Minps, true,
1752                              InstX86Base::SseSuffix::None>(Func, Dest,
1753                                                            Source) {}
1754  };
1755
1756  class InstX86Or : public InstX86BaseBinopGPR<InstX86Base::Or> {
1757  public:
1758    static InstX86Or *create(Cfg *Func, Variable *Dest, Operand *Source) {
1759      return new (Func->allocate<InstX86Or>()) InstX86Or(Func, Dest, Source);
1760    }
1761
1762  private:
1763    InstX86Or(Cfg *Func, Variable *Dest, Operand *Source)
1764        : InstX86BaseBinopGPR<InstX86Base::Or>(Func, Dest, Source) {}
1765  };
1766
1767  class InstX86Orps
1768      : public InstX86BaseBinopXmm<InstX86Base::Orps, true,
1769                                   InstX86Base::SseSuffix::Packed> {
1770  public:
1771    static InstX86Orps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1772      return new (Func->allocate<InstX86Orps>())
1773          InstX86Orps(Func, Dest, Source);
1774    }
1775
1776  private:
1777    InstX86Orps(Cfg *Func, Variable *Dest, Operand *Source)
1778        : InstX86BaseBinopXmm<InstX86Base::Orps, true,
1779                              InstX86Base::SseSuffix::Packed>(Func, Dest,
1780                                                              Source) {}
1781  };
1782
1783  class InstX86OrRMW : public InstX86BaseBinopRMW<InstX86Base::OrRMW> {
1784  public:
1785    static InstX86OrRMW *create(Cfg *Func, X86OperandMem *DestSrc0,
1786                                Operand *Src1) {
1787      return new (Func->allocate<InstX86OrRMW>())
1788          InstX86OrRMW(Func, DestSrc0, Src1);
1789    }
1790
1791  private:
1792    InstX86OrRMW(Cfg *Func, X86OperandMem *DestSrc0, Operand *Src1)
1793        : InstX86BaseBinopRMW<InstX86Base::OrRMW>(Func, DestSrc0, Src1) {}
1794  };
1795
1796  class InstX86Por : public InstX86BaseBinopXmm<InstX86Base::Por, false,
1797                                                InstX86Base::SseSuffix::None> {
1798  public:
1799    static InstX86Por *create(Cfg *Func, Variable *Dest, Operand *Source) {
1800      return new (Func->allocate<InstX86Por>()) InstX86Por(Func, Dest, Source);
1801    }
1802
1803  private:
1804    InstX86Por(Cfg *Func, Variable *Dest, Operand *Source)
1805        : InstX86BaseBinopXmm<InstX86Base::Por, false,
1806                              InstX86Base::SseSuffix::None>(Func, Dest,
1807                                                            Source) {}
1808  };
1809
1810  class InstX86Xor : public InstX86BaseBinopGPR<InstX86Base::Xor> {
1811  public:
1812    static InstX86Xor *create(Cfg *Func, Variable *Dest, Operand *Source) {
1813      return new (Func->allocate<InstX86Xor>()) InstX86Xor(Func, Dest, Source);
1814    }
1815
1816  private:
1817    InstX86Xor(Cfg *Func, Variable *Dest, Operand *Source)
1818        : InstX86BaseBinopGPR<InstX86Base::Xor>(Func, Dest, Source) {}
1819  };
1820
1821  class InstX86Xorps
1822      : public InstX86BaseBinopXmm<InstX86Base::Xorps, true,
1823                                   InstX86Base::SseSuffix::Packed> {
1824  public:
1825    static InstX86Xorps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1826      return new (Func->allocate<InstX86Xorps>())
1827          InstX86Xorps(Func, Dest, Source);
1828    }
1829
1830  private:
1831    InstX86Xorps(Cfg *Func, Variable *Dest, Operand *Source)
1832        : InstX86BaseBinopXmm<InstX86Base::Xorps, true,
1833                              InstX86Base::SseSuffix::Packed>(Func, Dest,
1834                                                              Source) {}
1835  };
1836
1837  class InstX86XorRMW : public InstX86BaseBinopRMW<InstX86Base::XorRMW> {
1838  public:
1839    static InstX86XorRMW *create(Cfg *Func, X86OperandMem *DestSrc0,
1840                                 Operand *Src1) {
1841      return new (Func->allocate<InstX86XorRMW>())
1842          InstX86XorRMW(Func, DestSrc0, Src1);
1843    }
1844
1845  private:
1846    InstX86XorRMW(Cfg *Func, X86OperandMem *DestSrc0, Operand *Src1)
1847        : InstX86BaseBinopRMW<InstX86Base::XorRMW>(Func, DestSrc0, Src1) {}
1848  };
1849
1850  class InstX86Pxor : public InstX86BaseBinopXmm<InstX86Base::Pxor, false,
1851                                                 InstX86Base::SseSuffix::None> {
1852  public:
1853    static InstX86Pxor *create(Cfg *Func, Variable *Dest, Operand *Source) {
1854      return new (Func->allocate<InstX86Pxor>())
1855          InstX86Pxor(Func, Dest, Source);
1856    }
1857
1858  private:
1859    InstX86Pxor(Cfg *Func, Variable *Dest, Operand *Source)
1860        : InstX86BaseBinopXmm<InstX86Base::Pxor, false,
1861                              InstX86Base::SseSuffix::None>(Func, Dest,
1862                                                            Source) {}
1863  };
1864
1865  class InstX86Imul : public InstX86BaseBinopGPR<InstX86Base::Imul> {
1866  public:
1867    static InstX86Imul *create(Cfg *Func, Variable *Dest, Operand *Source) {
1868      return new (Func->allocate<InstX86Imul>())
1869          InstX86Imul(Func, Dest, Source);
1870    }
1871
1872    void emit(const Cfg *Func) const override;
1873    void emitIAS(const Cfg *Func) const override;
1874
1875  private:
1876    InstX86Imul(Cfg *Func, Variable *Dest, Operand *Source)
1877        : InstX86BaseBinopGPR<InstX86Base::Imul>(Func, Dest, Source) {}
1878  };
1879
1880  class InstX86ImulImm
1881      : public InstX86BaseThreeAddressop<InstX86Base::ImulImm> {
1882  public:
1883    static InstX86ImulImm *create(Cfg *Func, Variable *Dest, Operand *Source0,
1884                                  Operand *Source1) {
1885      return new (Func->allocate<InstX86ImulImm>())
1886          InstX86ImulImm(Func, Dest, Source0, Source1);
1887    }
1888
1889    void emit(const Cfg *Func) const override;
1890    void emitIAS(const Cfg *Func) const override;
1891
1892  private:
1893    InstX86ImulImm(Cfg *Func, Variable *Dest, Operand *Source0,
1894                   Operand *Source1)
1895        : InstX86BaseThreeAddressop<InstX86Base::ImulImm>(Func, Dest, Source0,
1896                                                          Source1) {}
1897  };
1898
1899  class InstX86Mulps
1900      : public InstX86BaseBinopXmm<InstX86Base::Mulps, true,
1901                                   InstX86Base::SseSuffix::Packed> {
1902  public:
1903    static InstX86Mulps *create(Cfg *Func, Variable *Dest, Operand *Source) {
1904      return new (Func->allocate<InstX86Mulps>())
1905          InstX86Mulps(Func, Dest, Source);
1906    }
1907
1908  private:
1909    InstX86Mulps(Cfg *Func, Variable *Dest, Operand *Source)
1910        : InstX86BaseBinopXmm<InstX86Base::Mulps, true,
1911                              InstX86Base::SseSuffix::Packed>(Func, Dest,
1912                                                              Source) {}
1913  };
1914
1915  class InstX86Mulss
1916      : public InstX86BaseBinopXmm<InstX86Base::Mulss, false,
1917                                   InstX86Base::SseSuffix::Scalar> {
1918  public:
1919    static InstX86Mulss *create(Cfg *Func, Variable *Dest, Operand *Source) {
1920      return new (Func->allocate<InstX86Mulss>())
1921          InstX86Mulss(Func, Dest, Source);
1922    }
1923
1924  private:
1925    InstX86Mulss(Cfg *Func, Variable *Dest, Operand *Source)
1926        : InstX86BaseBinopXmm<InstX86Base::Mulss, false,
1927                              InstX86Base::SseSuffix::Scalar>(Func, Dest,
1928                                                              Source) {}
1929  };
1930
1931  class InstX86Pmull
1932      : public InstX86BaseBinopXmm<InstX86Base::Pmull, true,
1933                                   InstX86Base::SseSuffix::Integral> {
1934  public:
1935    static InstX86Pmull *create(Cfg *Func, Variable *Dest, Operand *Source) {
1936      bool TypesAreValid =
1937          Dest->getType() == IceType_v4i32 || Dest->getType() == IceType_v8i16;
1938      auto *Target = InstX86Base::getTarget(Func);
1939      bool InstructionSetIsValid =
1940          Dest->getType() == IceType_v8i16 ||
1941          Target->getInstructionSet() >= Traits::SSE4_1;
1942      (void)TypesAreValid;
1943      (void)InstructionSetIsValid;
1944      assert(TypesAreValid);
1945      assert(InstructionSetIsValid);
1946      return new (Func->allocate<InstX86Pmull>())
1947          InstX86Pmull(Func, Dest, Source);
1948    }
1949
1950  private:
1951    InstX86Pmull(Cfg *Func, Variable *Dest, Operand *Source)
1952        : InstX86BaseBinopXmm<InstX86Base::Pmull, true,
1953                              InstX86Base::SseSuffix::Integral>(Func, Dest,
1954                                                                Source) {}
1955  };
1956
1957  class InstX86Pmulhw
1958      : public InstX86BaseBinopXmm<InstX86Base::Pmulhw, false,
1959                                   InstX86Base::SseSuffix::None> {
1960  public:
1961    static InstX86Pmulhw *create(Cfg *Func, Variable *Dest, Operand *Source) {
1962      assert(Dest->getType() == IceType_v8i16 &&
1963             Source->getType() == IceType_v8i16);
1964      return new (Func->allocate<InstX86Pmulhw>())
1965          InstX86Pmulhw(Func, Dest, Source);
1966    }
1967
1968  private:
1969    InstX86Pmulhw(Cfg *Func, Variable *Dest, Operand *Source)
1970        : InstX86BaseBinopXmm<InstX86Base::Pmulhw, false,
1971                              InstX86Base::SseSuffix::None>(Func, Dest,
1972                                                            Source) {}
1973  };
1974
1975  class InstX86Pmulhuw
1976      : public InstX86BaseBinopXmm<InstX86Base::Pmulhuw, false,
1977                                   InstX86Base::SseSuffix::None> {
1978  public:
1979    static InstX86Pmulhuw *create(Cfg *Func, Variable *Dest, Operand *Source) {
1980      assert(Dest->getType() == IceType_v8i16 &&
1981             Source->getType() == IceType_v8i16);
1982      return new (Func->allocate<InstX86Pmulhuw>())
1983          InstX86Pmulhuw(Func, Dest, Source);
1984    }
1985
1986  private:
1987    InstX86Pmulhuw(Cfg *Func, Variable *Dest, Operand *Source)
1988        : InstX86BaseBinopXmm<InstX86Base::Pmulhuw, false,
1989                              InstX86Base::SseSuffix::None>(Func, Dest,
1990                                                            Source) {}
1991  };
1992
1993  class InstX86Pmaddwd
1994      : public InstX86BaseBinopXmm<InstX86Base::Pmaddwd, false,
1995                                   InstX86Base::SseSuffix::None> {
1996  public:
1997    static InstX86Pmaddwd *create(Cfg *Func, Variable *Dest, Operand *Source) {
1998      assert(Dest->getType() == IceType_v8i16 &&
1999             Source->getType() == IceType_v8i16);
2000      return new (Func->allocate<InstX86Pmaddwd>())
2001          InstX86Pmaddwd(Func, Dest, Source);
2002    }
2003
2004  private:
2005    InstX86Pmaddwd(Cfg *Func, Variable *Dest, Operand *Source)
2006        : InstX86BaseBinopXmm<InstX86Base::Pmaddwd, false,
2007                              InstX86Base::SseSuffix::None>(Func, Dest,
2008                                                            Source) {}
2009  };
2010
2011  class InstX86Pmuludq
2012      : public InstX86BaseBinopXmm<InstX86Base::Pmuludq, false,
2013                                   InstX86Base::SseSuffix::None> {
2014  public:
2015    static InstX86Pmuludq *create(Cfg *Func, Variable *Dest, Operand *Source) {
2016      assert(Dest->getType() == IceType_v4i32 &&
2017             Source->getType() == IceType_v4i32);
2018      return new (Func->allocate<InstX86Pmuludq>())
2019          InstX86Pmuludq(Func, Dest, Source);
2020    }
2021
2022  private:
2023    InstX86Pmuludq(Cfg *Func, Variable *Dest, Operand *Source)
2024        : InstX86BaseBinopXmm<InstX86Base::Pmuludq, false,
2025                              InstX86Base::SseSuffix::None>(Func, Dest,
2026                                                            Source) {}
2027  };
2028
2029  class InstX86Divps
2030      : public InstX86BaseBinopXmm<InstX86Base::Divps, true,
2031                                   InstX86Base::SseSuffix::Packed> {
2032  public:
2033    static InstX86Divps *create(Cfg *Func, Variable *Dest, Operand *Source) {
2034      return new (Func->allocate<InstX86Divps>())
2035          InstX86Divps(Func, Dest, Source);
2036    }
2037
2038  private:
2039    InstX86Divps(Cfg *Func, Variable *Dest, Operand *Source)
2040        : InstX86BaseBinopXmm<InstX86Base::Divps, true,
2041                              InstX86Base::SseSuffix::Packed>(Func, Dest,
2042                                                              Source) {}
2043  };
2044
2045  class InstX86Divss
2046      : public InstX86BaseBinopXmm<InstX86Base::Divss, false,
2047                                   InstX86Base::SseSuffix::Scalar> {
2048  public:
2049    static InstX86Divss *create(Cfg *Func, Variable *Dest, Operand *Source) {
2050      return new (Func->allocate<InstX86Divss>())
2051          InstX86Divss(Func, Dest, Source);
2052    }
2053
2054  private:
2055    InstX86Divss(Cfg *Func, Variable *Dest, Operand *Source)
2056        : InstX86BaseBinopXmm<InstX86Base::Divss, false,
2057                              InstX86Base::SseSuffix::Scalar>(Func, Dest,
2058                                                              Source) {}
2059  };
2060
2061  class InstX86Rol : public InstX86BaseBinopGPRShift<InstX86Base::Rol> {
2062  public:
2063    static InstX86Rol *create(Cfg *Func, Variable *Dest, Operand *Source) {
2064      return new (Func->allocate<InstX86Rol>()) InstX86Rol(Func, Dest, Source);
2065    }
2066
2067  private:
2068    InstX86Rol(Cfg *Func, Variable *Dest, Operand *Source)
2069        : InstX86BaseBinopGPRShift<InstX86Base::Rol>(Func, Dest, Source) {}
2070  };
2071
2072  class InstX86Shl : public InstX86BaseBinopGPRShift<InstX86Base::Shl> {
2073  public:
2074    static InstX86Shl *create(Cfg *Func, Variable *Dest, Operand *Source) {
2075      return new (Func->allocate<InstX86Shl>()) InstX86Shl(Func, Dest, Source);
2076    }
2077
2078  private:
2079    InstX86Shl(Cfg *Func, Variable *Dest, Operand *Source)
2080        : InstX86BaseBinopGPRShift<InstX86Base::Shl>(Func, Dest, Source) {}
2081  };
2082
2083  class InstX86Psll : public InstX86BaseBinopXmmShift<InstX86Base::Psll> {
2084  public:
2085    static InstX86Psll *create(Cfg *Func, Variable *Dest, Operand *Source) {
2086      assert(
2087          Dest->getType() == IceType_v8i16 || Dest->getType() == IceType_v8i1 ||
2088          Dest->getType() == IceType_v4i32 || Dest->getType() == IceType_v4i1);
2089      return new (Func->allocate<InstX86Psll>())
2090          InstX86Psll(Func, Dest, Source);
2091    }
2092
2093  private:
2094    InstX86Psll(Cfg *Func, Variable *Dest, Operand *Source)
2095        : InstX86BaseBinopXmmShift<InstX86Base::Psll>(Func, Dest, Source) {}
2096  };
2097
2098  class InstX86Psrl : public InstX86BaseBinopXmmShift<InstX86Base::Psrl, true> {
2099  public:
2100    static InstX86Psrl *create(Cfg *Func, Variable *Dest, Operand *Source) {
2101      return new (Func->allocate<InstX86Psrl>())
2102          InstX86Psrl(Func, Dest, Source);
2103    }
2104
2105  private:
2106    InstX86Psrl(Cfg *Func, Variable *Dest, Operand *Source)
2107        : InstX86BaseBinopXmmShift<InstX86Base::Psrl, true>(Func, Dest,
2108                                                            Source) {}
2109  };
2110
2111  class InstX86Shr : public InstX86BaseBinopGPRShift<InstX86Base::Shr> {
2112  public:
2113    static InstX86Shr *create(Cfg *Func, Variable *Dest, Operand *Source) {
2114      return new (Func->allocate<InstX86Shr>()) InstX86Shr(Func, Dest, Source);
2115    }
2116
2117  private:
2118    InstX86Shr(Cfg *Func, Variable *Dest, Operand *Source)
2119        : InstX86BaseBinopGPRShift<InstX86Base::Shr>(Func, Dest, Source) {}
2120  };
2121
2122  class InstX86Sar : public InstX86BaseBinopGPRShift<InstX86Base::Sar> {
2123  public:
2124    static InstX86Sar *create(Cfg *Func, Variable *Dest, Operand *Source) {
2125      return new (Func->allocate<InstX86Sar>()) InstX86Sar(Func, Dest, Source);
2126    }
2127
2128  private:
2129    InstX86Sar(Cfg *Func, Variable *Dest, Operand *Source)
2130        : InstX86BaseBinopGPRShift<InstX86Base::Sar>(Func, Dest, Source) {}
2131  };
2132
2133  class InstX86Psra : public InstX86BaseBinopXmmShift<InstX86Base::Psra> {
2134  public:
2135    static InstX86Psra *create(Cfg *Func, Variable *Dest, Operand *Source) {
2136      assert(
2137          Dest->getType() == IceType_v8i16 || Dest->getType() == IceType_v8i1 ||
2138          Dest->getType() == IceType_v4i32 || Dest->getType() == IceType_v4i1);
2139      return new (Func->allocate<InstX86Psra>())
2140          InstX86Psra(Func, Dest, Source);
2141    }
2142
2143  private:
2144    InstX86Psra(Cfg *Func, Variable *Dest, Operand *Source)
2145        : InstX86BaseBinopXmmShift<InstX86Base::Psra>(Func, Dest, Source) {}
2146  };
2147
2148  class InstX86Pcmpeq
2149      : public InstX86BaseBinopXmm<InstX86Base::Pcmpeq, true,
2150                                   InstX86Base::SseSuffix::Integral> {
2151  public:
2152    static InstX86Pcmpeq *create(Cfg *Func, Variable *Dest, Operand *Source,
2153                                 Type ArithmeticTypeOverride = IceType_void) {
2154      const Type Ty = ArithmeticTypeOverride == IceType_void
2155                          ? Dest->getType()
2156                          : ArithmeticTypeOverride;
2157      (void)Ty;
2158      assert((Ty != IceType_f64 && Ty != IceType_i64) ||
2159             InstX86Base::getTarget(Func)->getInstructionSet() >=
2160                 Traits::SSE4_1);
2161      return new (Func->allocate<InstX86Pcmpeq>())
2162          InstX86Pcmpeq(Func, Dest, Source, ArithmeticTypeOverride);
2163    }
2164
2165  private:
2166    InstX86Pcmpeq(Cfg *Func, Variable *Dest, Operand *Source,
2167                  Type ArithmeticTypeOverride)
2168        : InstX86BaseBinopXmm<InstX86Base::Pcmpeq, true,
2169                              InstX86Base::SseSuffix::Integral>(
2170              Func, Dest, Source, ArithmeticTypeOverride) {}
2171  };
2172
2173  class InstX86Pcmpgt
2174      : public InstX86BaseBinopXmm<InstX86Base::Pcmpgt, true,
2175                                   InstX86Base::SseSuffix::Integral> {
2176  public:
2177    static InstX86Pcmpgt *create(Cfg *Func, Variable *Dest, Operand *Source) {
2178      assert(Dest->getType() != IceType_f64 ||
2179             InstX86Base::getTarget(Func)->getInstructionSet() >=
2180                 Traits::SSE4_1);
2181      return new (Func->allocate<InstX86Pcmpgt>())
2182          InstX86Pcmpgt(Func, Dest, Source);
2183    }
2184
2185  private:
2186    InstX86Pcmpgt(Cfg *Func, Variable *Dest, Operand *Source)
2187        : InstX86BaseBinopXmm<InstX86Base::Pcmpgt, true,
2188                              InstX86Base::SseSuffix::Integral>(Func, Dest,
2189                                                                Source) {}
2190  };
2191
2192  /// movss is only a binary operation when the source and dest operands are
2193  /// both registers (the high bits of dest are left untouched). In other cases,
2194  /// it behaves like a copy (mov-like) operation (and the high bits of dest are
2195  /// cleared). InstX86Movss will assert that both its source and dest operands
2196  /// are registers, so the lowering code should use _mov instead of _movss in
2197  /// cases where a copy operation is intended.
2198  class InstX86MovssRegs
2199      : public InstX86BaseBinopXmm<InstX86Base::MovssRegs, false,
2200                                   InstX86Base::SseSuffix::None> {
2201  public:
2202    static InstX86MovssRegs *create(Cfg *Func, Variable *Dest,
2203                                    Operand *Source) {
2204      return new (Func->allocate<InstX86MovssRegs>())
2205          InstX86MovssRegs(Func, Dest, Source);
2206    }
2207
2208    void emitIAS(const Cfg *Func) const override;
2209
2210  private:
2211    InstX86MovssRegs(Cfg *Func, Variable *Dest, Operand *Source)
2212        : InstX86BaseBinopXmm<InstX86Base::MovssRegs, false,
2213                              InstX86Base::SseSuffix::None>(Func, Dest,
2214                                                            Source) {}
2215  };
2216
2217  class InstX86Idiv : public InstX86BaseTernop<InstX86Base::Idiv> {
2218  public:
2219    static InstX86Idiv *create(Cfg *Func, Variable *Dest, Operand *Source1,
2220                               Operand *Source2) {
2221      return new (Func->allocate<InstX86Idiv>())
2222          InstX86Idiv(Func, Dest, Source1, Source2);
2223    }
2224
2225    void emit(const Cfg *Func) const override;
2226    void emitIAS(const Cfg *Func) const override;
2227
2228  private:
2229    InstX86Idiv(Cfg *Func, Variable *Dest, Operand *Source1, Operand *Source2)
2230        : InstX86BaseTernop<InstX86Base::Idiv>(Func, Dest, Source1, Source2) {}
2231  };
2232
2233  class InstX86Div : public InstX86BaseTernop<InstX86Base::Div> {
2234  public:
2235    static InstX86Div *create(Cfg *Func, Variable *Dest, Operand *Source1,
2236                              Operand *Source2) {
2237      return new (Func->allocate<InstX86Div>())
2238          InstX86Div(Func, Dest, Source1, Source2);
2239    }
2240
2241    void emit(const Cfg *Func) const override;
2242    void emitIAS(const Cfg *Func) const override;
2243
2244  private:
2245    InstX86Div(Cfg *Func, Variable *Dest, Operand *Source1, Operand *Source2)
2246        : InstX86BaseTernop<InstX86Base::Div>(Func, Dest, Source1, Source2) {}
2247  };
2248
2249  class InstX86Insertps : public InstX86BaseTernop<InstX86Base::Insertps> {
2250  public:
2251    static InstX86Insertps *create(Cfg *Func, Variable *Dest, Operand *Source1,
2252                                   Operand *Source2) {
2253      return new (Func->allocate<InstX86Insertps>())
2254          InstX86Insertps(Func, Dest, Source1, Source2);
2255    }
2256
2257    void emitIAS(const Cfg *Func) const override;
2258
2259  private:
2260    InstX86Insertps(Cfg *Func, Variable *Dest, Operand *Source1,
2261                    Operand *Source2)
2262        : InstX86BaseTernop<InstX86Base::Insertps>(Func, Dest, Source1,
2263                                                   Source2) {}
2264  };
2265
2266  class InstX86Pinsr : public InstX86BaseTernop<InstX86Base::Pinsr> {
2267  public:
2268    static InstX86Pinsr *create(Cfg *Func, Variable *Dest, Operand *Source1,
2269                                Operand *Source2) {
2270      // pinsrb and pinsrd are SSE4.1 instructions.
2271      assert(
2272          Dest->getType() == IceType_v8i16 || Dest->getType() == IceType_v8i1 ||
2273          InstX86Base::getTarget(Func)->getInstructionSet() >= Traits::SSE4_1);
2274      return new (Func->allocate<InstX86Pinsr>())
2275          InstX86Pinsr(Func, Dest, Source1, Source2);
2276    }
2277
2278    void emit(const Cfg *Func) const override;
2279    void emitIAS(const Cfg *Func) const override;
2280
2281  private:
2282    InstX86Pinsr(Cfg *Func, Variable *Dest, Operand *Source1, Operand *Source2)
2283        : InstX86BaseTernop<InstX86Base::Pinsr>(Func, Dest, Source1, Source2) {}
2284  };
2285
2286  class InstX86Shufps : public InstX86BaseTernop<InstX86Base::Shufps> {
2287  public:
2288    static InstX86Shufps *create(Cfg *Func, Variable *Dest, Operand *Source1,
2289                                 Operand *Source2) {
2290      return new (Func->allocate<InstX86Shufps>())
2291          InstX86Shufps(Func, Dest, Source1, Source2);
2292    }
2293
2294    void emitIAS(const Cfg *Func) const override;
2295
2296  private:
2297    InstX86Shufps(Cfg *Func, Variable *Dest, Operand *Source1, Operand *Source2)
2298        : InstX86BaseTernop<InstX86Base::Shufps>(Func, Dest, Source1, Source2) {
2299    }
2300  };
2301
2302  class InstX86Blendvps : public InstX86BaseTernop<InstX86Base::Blendvps> {
2303  public:
2304    static InstX86Blendvps *create(Cfg *Func, Variable *Dest, Operand *Source1,
2305                                   Operand *Source2) {
2306      assert(InstX86Base::getTarget(Func)->getInstructionSet() >=
2307             Traits::SSE4_1);
2308      return new (Func->allocate<InstX86Blendvps>())
2309          InstX86Blendvps(Func, Dest, Source1, Source2);
2310    }
2311
2312    void emit(const Cfg *Func) const override;
2313    void emitIAS(const Cfg *Fund) const override;
2314
2315  private:
2316    InstX86Blendvps(Cfg *Func, Variable *Dest, Operand *Source1,
2317                    Operand *Source2)
2318        : InstX86BaseTernop<InstX86Base::Blendvps>(Func, Dest, Source1,
2319                                                   Source2) {}
2320  };
2321
2322  class InstX86Pblendvb : public InstX86BaseTernop<InstX86Base::Pblendvb> {
2323  public:
2324    static InstX86Pblendvb *create(Cfg *Func, Variable *Dest, Operand *Source1,
2325                                   Operand *Source2) {
2326      assert(InstX86Base::getTarget(Func)->getInstructionSet() >=
2327             Traits::SSE4_1);
2328      return new (Func->allocate<InstX86Pblendvb>())
2329          InstX86Pblendvb(Func, Dest, Source1, Source2);
2330    }
2331
2332    void emit(const Cfg *Func) const override;
2333    void emitIAS(const Cfg *Func) const override;
2334
2335  private:
2336    InstX86Pblendvb(Cfg *Func, Variable *Dest, Operand *Source1,
2337                    Operand *Source2)
2338        : InstX86BaseTernop<InstX86Base::Pblendvb>(Func, Dest, Source1,
2339                                                   Source2) {}
2340  };
2341
2342  class InstX86Pextr : public InstX86BaseThreeAddressop<InstX86Base::Pextr> {
2343  public:
2344    static InstX86Pextr *create(Cfg *Func, Variable *Dest, Operand *Source0,
2345                                Operand *Source1) {
2346      assert(Source0->getType() == IceType_v8i16 ||
2347             Source0->getType() == IceType_v8i1 ||
2348             InstX86Base::getTarget(Func)->getInstructionSet() >=
2349                 Traits::SSE4_1);
2350      return new (Func->allocate<InstX86Pextr>())
2351          InstX86Pextr(Func, Dest, Source0, Source1);
2352    }
2353
2354    void emit(const Cfg *Func) const override;
2355    void emitIAS(const Cfg *Func) const override;
2356
2357  private:
2358    InstX86Pextr(Cfg *Func, Variable *Dest, Operand *Source0, Operand *Source1)
2359        : InstX86BaseThreeAddressop<InstX86Base::Pextr>(Func, Dest, Source0,
2360                                                        Source1) {}
2361  };
2362
2363  class InstX86Pshufd : public InstX86BaseThreeAddressop<InstX86Base::Pshufd> {
2364  public:
2365    static InstX86Pshufd *create(Cfg *Func, Variable *Dest, Operand *Source0,
2366                                 Operand *Source1) {
2367      return new (Func->allocate<InstX86Pshufd>())
2368          InstX86Pshufd(Func, Dest, Source0, Source1);
2369    }
2370
2371    void emitIAS(const Cfg *Func) const override;
2372
2373  private:
2374    InstX86Pshufd(Cfg *Func, Variable *Dest, Operand *Source0, Operand *Source1)
2375        : InstX86BaseThreeAddressop<InstX86Base::Pshufd>(Func, Dest, Source0,
2376                                                         Source1) {}
2377  };
2378
2379  /// Base class for a lockable x86-32 instruction (emits a locked prefix).
2380  class InstX86BaseLockable : public InstX86Base {
2381    InstX86BaseLockable() = delete;
2382    InstX86BaseLockable(const InstX86BaseLockable &) = delete;
2383    InstX86BaseLockable &operator=(const InstX86BaseLockable &) = delete;
2384
2385  protected:
2386    bool Locked;
2387
2388    InstX86BaseLockable(Cfg *Func, typename InstX86Base::InstKindX86 Kind,
2389                        SizeT Maxsrcs, Variable *Dest, bool Locked)
2390        : InstX86Base(Func, Kind, Maxsrcs, Dest), Locked(Locked) {
2391      // Assume that such instructions are used for Atomics and be careful with
2392      // optimizations.
2393      this->HasSideEffects = Locked;
2394    }
2395  };
2396
2397  /// Mul instruction - unsigned multiply.
2398  class InstX86Mul final : public InstX86Base {
2399    InstX86Mul() = delete;
2400    InstX86Mul(const InstX86Mul &) = delete;
2401    InstX86Mul &operator=(const InstX86Mul &) = delete;
2402
2403  public:
2404    static InstX86Mul *create(Cfg *Func, Variable *Dest, Variable *Source1,
2405                              Operand *Source2) {
2406      return new (Func->allocate<InstX86Mul>())
2407          InstX86Mul(Func, Dest, Source1, Source2);
2408    }
2409    void emit(const Cfg *Func) const override;
2410    void emitIAS(const Cfg *Func) const override;
2411    void dump(const Cfg *Func) const override;
2412    static bool classof(const Inst *Instr) {
2413      return InstX86Base::isClassof(Instr, InstX86Base::Mul);
2414    }
2415
2416  private:
2417    InstX86Mul(Cfg *Func, Variable *Dest, Variable *Source1, Operand *Source2);
2418  };
2419
2420  /// Shld instruction - shift across a pair of operands.
2421  class InstX86Shld final : public InstX86Base {
2422    InstX86Shld() = delete;
2423    InstX86Shld(const InstX86Shld &) = delete;
2424    InstX86Shld &operator=(const InstX86Shld &) = delete;
2425
2426  public:
2427    static InstX86Shld *create(Cfg *Func, Variable *Dest, Variable *Source1,
2428                               Operand *Source2) {
2429      return new (Func->allocate<InstX86Shld>())
2430          InstX86Shld(Func, Dest, Source1, Source2);
2431    }
2432    void emit(const Cfg *Func) const override;
2433    void emitIAS(const Cfg *Func) const override;
2434    void dump(const Cfg *Func) const override;
2435    static bool classof(const Inst *Instr) {
2436      return InstX86Base::isClassof(Instr, InstX86Base::Shld);
2437    }
2438
2439  private:
2440    InstX86Shld(Cfg *Func, Variable *Dest, Variable *Source1, Operand *Source2);
2441  };
2442
2443  /// Shrd instruction - shift across a pair of operands.
2444  class InstX86Shrd final : public InstX86Base {
2445    InstX86Shrd() = delete;
2446    InstX86Shrd(const InstX86Shrd &) = delete;
2447    InstX86Shrd &operator=(const InstX86Shrd &) = delete;
2448
2449  public:
2450    static InstX86Shrd *create(Cfg *Func, Variable *Dest, Variable *Source1,
2451                               Operand *Source2) {
2452      return new (Func->allocate<InstX86Shrd>())
2453          InstX86Shrd(Func, Dest, Source1, Source2);
2454    }
2455    void emit(const Cfg *Func) const override;
2456    void emitIAS(const Cfg *Func) const override;
2457    void dump(const Cfg *Func) const override;
2458    static bool classof(const Inst *Instr) {
2459      return InstX86Base::isClassof(Instr, InstX86Base::Shrd);
2460    }
2461
2462  private:
2463    InstX86Shrd(Cfg *Func, Variable *Dest, Variable *Source1, Operand *Source2);
2464  };
2465
2466  /// Conditional move instruction.
2467  class InstX86Cmov final : public InstX86Base {
2468    InstX86Cmov() = delete;
2469    InstX86Cmov(const InstX86Cmov &) = delete;
2470    InstX86Cmov &operator=(const InstX86Cmov &) = delete;
2471
2472  public:
2473    static InstX86Cmov *create(Cfg *Func, Variable *Dest, Operand *Source,
2474                               BrCond Cond) {
2475      return new (Func->allocate<InstX86Cmov>())
2476          InstX86Cmov(Func, Dest, Source, Cond);
2477    }
2478    void emit(const Cfg *Func) const override;
2479    void emitIAS(const Cfg *Func) const override;
2480    void dump(const Cfg *Func) const override;
2481    static bool classof(const Inst *Instr) {
2482      return InstX86Base::isClassof(Instr, InstX86Base::Cmov);
2483    }
2484
2485  private:
2486    InstX86Cmov(Cfg *Func, Variable *Dest, Operand *Source, BrCond Cond);
2487
2488    BrCond Condition;
2489  };
2490
2491  /// Cmpps instruction - compare packed singled-precision floating point values
2492  class InstX86Cmpps final : public InstX86Base {
2493    InstX86Cmpps() = delete;
2494    InstX86Cmpps(const InstX86Cmpps &) = delete;
2495    InstX86Cmpps &operator=(const InstX86Cmpps &) = delete;
2496
2497  public:
2498    static InstX86Cmpps *create(Cfg *Func, Variable *Dest, Operand *Source,
2499                                CmppsCond Condition) {
2500      return new (Func->allocate<InstX86Cmpps>())
2501          InstX86Cmpps(Func, Dest, Source, Condition);
2502    }
2503    void emit(const Cfg *Func) const override;
2504    void emitIAS(const Cfg *Func) const override;
2505    void dump(const Cfg *Func) const override;
2506    static bool classof(const Inst *Instr) {
2507      return InstX86Base::isClassof(Instr, InstX86Base::Cmpps);
2508    }
2509
2510  private:
2511    InstX86Cmpps(Cfg *Func, Variable *Dest, Operand *Source, CmppsCond Cond);
2512
2513    CmppsCond Condition;
2514  };
2515
2516  /// Cmpxchg instruction - cmpxchg <dest>, <desired> will compare if <dest>
2517  /// equals eax. If so, the ZF is set and <desired> is stored in <dest>. If
2518  /// not, ZF is cleared and <dest> is copied to eax (or subregister). <dest>
2519  /// can be a register or memory, while <desired> must be a register. It is
2520  /// the user's responsibility to mark eax with a FakeDef.
2521  class InstX86Cmpxchg final : public InstX86BaseLockable {
2522    InstX86Cmpxchg() = delete;
2523    InstX86Cmpxchg(const InstX86Cmpxchg &) = delete;
2524    InstX86Cmpxchg &operator=(const InstX86Cmpxchg &) = delete;
2525
2526  public:
2527    static InstX86Cmpxchg *create(Cfg *Func, Operand *DestOrAddr, Variable *Eax,
2528                                  Variable *Desired, bool Locked) {
2529      return new (Func->allocate<InstX86Cmpxchg>())
2530          InstX86Cmpxchg(Func, DestOrAddr, Eax, Desired, Locked);
2531    }
2532    void emit(const Cfg *Func) const override;
2533    void emitIAS(const Cfg *Func) const override;
2534    void dump(const Cfg *Func) const override;
2535    static bool classof(const Inst *Instr) {
2536      return InstX86Base::isClassof(Instr, InstX86Base::Cmpxchg);
2537    }
2538
2539  private:
2540    InstX86Cmpxchg(Cfg *Func, Operand *DestOrAddr, Variable *Eax,
2541                   Variable *Desired, bool Locked);
2542  };
2543
2544  /// Cmpxchg8b instruction - cmpxchg8b <m64> will compare if <m64> equals
2545  /// edx:eax. If so, the ZF is set and ecx:ebx is stored in <m64>. If not, ZF
2546  /// is cleared and <m64> is copied to edx:eax. The caller is responsible for
2547  /// inserting FakeDefs to mark edx and eax as modified. <m64> must be a memory
2548  /// operand.
2549  class InstX86Cmpxchg8b final : public InstX86BaseLockable {
2550    InstX86Cmpxchg8b() = delete;
2551    InstX86Cmpxchg8b(const InstX86Cmpxchg8b &) = delete;
2552    InstX86Cmpxchg8b &operator=(const InstX86Cmpxchg8b &) = delete;
2553
2554  public:
2555    static InstX86Cmpxchg8b *create(Cfg *Func, X86OperandMem *Dest,
2556                                    Variable *Edx, Variable *Eax, Variable *Ecx,
2557                                    Variable *Ebx, bool Locked) {
2558      return new (Func->allocate<InstX86Cmpxchg8b>())
2559          InstX86Cmpxchg8b(Func, Dest, Edx, Eax, Ecx, Ebx, Locked);
2560    }
2561    void emit(const Cfg *Func) const override;
2562    void emitIAS(const Cfg *Func) const override;
2563    void dump(const Cfg *Func) const override;
2564    static bool classof(const Inst *Instr) {
2565      return InstX86Base::isClassof(Instr, InstX86Base::Cmpxchg8b);
2566    }
2567
2568  private:
2569    InstX86Cmpxchg8b(Cfg *Func, X86OperandMem *Dest, Variable *Edx,
2570                     Variable *Eax, Variable *Ecx, Variable *Ebx, bool Locked);
2571  };
2572
2573  /// Cvt instruction - wrapper for cvtsX2sY where X and Y are in {s,d,i} as
2574  /// appropriate.  s=float, d=double, i=int. X and Y are determined from
2575  /// dest/src types. Sign and zero extension on the integer operand needs to be
2576  /// done separately.
2577  class InstX86Cvt final : public InstX86Base {
2578    InstX86Cvt() = delete;
2579    InstX86Cvt(const InstX86Cvt &) = delete;
2580    InstX86Cvt &operator=(const InstX86Cvt &) = delete;
2581
2582  public:
2583    enum CvtVariant { Si2ss, Tss2si, Ss2si, Float2float, Dq2ps, Tps2dq, Ps2dq };
2584    static InstX86Cvt *create(Cfg *Func, Variable *Dest, Operand *Source,
2585                              CvtVariant Variant) {
2586      return new (Func->allocate<InstX86Cvt>())
2587          InstX86Cvt(Func, Dest, Source, Variant);
2588    }
2589    void emit(const Cfg *Func) const override;
2590    void emitIAS(const Cfg *Func) const override;
2591    void dump(const Cfg *Func) const override;
2592    static bool classof(const Inst *Instr) {
2593      return InstX86Base::isClassof(Instr, InstX86Base::Cvt);
2594    }
2595    bool isTruncating() const { return Variant == Tss2si || Variant == Tps2dq; }
2596
2597  private:
2598    CvtVariant Variant;
2599    InstX86Cvt(Cfg *Func, Variable *Dest, Operand *Source, CvtVariant Variant);
2600  };
2601
2602  /// Round instruction
2603  class InstX86Round final
2604      : public InstX86BaseThreeAddressop<InstX86Base::Round> {
2605  public:
2606    static InstX86Round *create(Cfg *Func, Variable *Dest, Operand *Source,
2607                                Operand *Imm) {
2608      return new (Func->allocate<InstX86Round>())
2609          InstX86Round(Func, Dest, Source, Imm);
2610    }
2611
2612    void emit(const Cfg *Func) const override;
2613    void emitIAS(const Cfg *Func) const override;
2614
2615  private:
2616    InstX86Round(Cfg *Func, Variable *Dest, Operand *Source, Operand *Imm)
2617        : InstX86BaseThreeAddressop<InstX86Base::Round>(Func, Dest, Source,
2618                                                        Imm) {}
2619  };
2620
2621  /// cmp - Integer compare instruction.
2622  class InstX86Icmp final : public InstX86Base {
2623    InstX86Icmp() = delete;
2624    InstX86Icmp(const InstX86Icmp &) = delete;
2625    InstX86Icmp &operator=(const InstX86Icmp &) = delete;
2626
2627  public:
2628    static InstX86Icmp *create(Cfg *Func, Operand *Src1, Operand *Src2) {
2629      return new (Func->allocate<InstX86Icmp>()) InstX86Icmp(Func, Src1, Src2);
2630    }
2631    void emit(const Cfg *Func) const override;
2632    void emitIAS(const Cfg *Func) const override;
2633    void dump(const Cfg *Func) const override;
2634    static bool classof(const Inst *Instr) {
2635      return InstX86Base::isClassof(Instr, InstX86Base::Icmp);
2636    }
2637
2638  private:
2639    InstX86Icmp(Cfg *Func, Operand *Src1, Operand *Src2);
2640  };
2641
2642  /// ucomiss/ucomisd - floating-point compare instruction.
2643  class InstX86Ucomiss final : public InstX86Base {
2644    InstX86Ucomiss() = delete;
2645    InstX86Ucomiss(const InstX86Ucomiss &) = delete;
2646    InstX86Ucomiss &operator=(const InstX86Ucomiss &) = delete;
2647
2648  public:
2649    static InstX86Ucomiss *create(Cfg *Func, Operand *Src1, Operand *Src2) {
2650      return new (Func->allocate<InstX86Ucomiss>())
2651          InstX86Ucomiss(Func, Src1, Src2);
2652    }
2653    void emit(const Cfg *Func) const override;
2654    void emitIAS(const Cfg *Func) const override;
2655    void dump(const Cfg *Func) const override;
2656    static bool classof(const Inst *Instr) {
2657      return InstX86Base::isClassof(Instr, InstX86Base::Ucomiss);
2658    }
2659
2660  private:
2661    InstX86Ucomiss(Cfg *Func, Operand *Src1, Operand *Src2);
2662  };
2663
2664  /// UD2 instruction.
2665  class InstX86UD2 final : public InstX86Base {
2666    InstX86UD2() = delete;
2667    InstX86UD2(const InstX86UD2 &) = delete;
2668    InstX86UD2 &operator=(const InstX86UD2 &) = delete;
2669
2670  public:
2671    static InstX86UD2 *create(Cfg *Func) {
2672      return new (Func->allocate<InstX86UD2>()) InstX86UD2(Func);
2673    }
2674    void emit(const Cfg *Func) const override;
2675    void emitIAS(const Cfg *Func) const override;
2676    void dump(const Cfg *Func) const override;
2677    static bool classof(const Inst *Instr) {
2678      return InstX86Base::isClassof(Instr, InstX86Base::UD2);
2679    }
2680
2681  private:
2682    explicit InstX86UD2(Cfg *Func);
2683  };
2684
2685  /// Int3 instruction.
2686  class InstX86Int3 final : public InstX86Base {
2687    InstX86Int3() = delete;
2688    InstX86Int3(const InstX86Int3 &) = delete;
2689    InstX86Int3 &operator=(const InstX86Int3 &) = delete;
2690
2691  public:
2692    static InstX86Int3 *create(Cfg *Func) {
2693      return new (Func->allocate<InstX86Int3>()) InstX86Int3(Func);
2694    }
2695    void emit(const Cfg *Func) const override;
2696    void emitIAS(const Cfg *Func) const override;
2697    void dump(const Cfg *Func) const override;
2698    static bool classof(const Inst *Instr) {
2699      return InstX86Base::isClassof(Instr, InstX86Base::Int3);
2700    }
2701
2702  private:
2703    explicit InstX86Int3(Cfg *Func);
2704  };
2705
2706  /// Test instruction.
2707  class InstX86Test final : public InstX86Base {
2708    InstX86Test() = delete;
2709    InstX86Test(const InstX86Test &) = delete;
2710    InstX86Test &operator=(const InstX86Test &) = delete;
2711
2712  public:
2713    static InstX86Test *create(Cfg *Func, Operand *Source1, Operand *Source2) {
2714      return new (Func->allocate<InstX86Test>())
2715          InstX86Test(Func, Source1, Source2);
2716    }
2717    void emit(const Cfg *Func) const override;
2718    void emitIAS(const Cfg *Func) const override;
2719    void dump(const Cfg *Func) const override;
2720    static bool classof(const Inst *Instr) {
2721      return InstX86Base::isClassof(Instr, InstX86Base::Test);
2722    }
2723
2724  private:
2725    InstX86Test(Cfg *Func, Operand *Source1, Operand *Source2);
2726  };
2727
2728  /// Mfence instruction.
2729  class InstX86Mfence final : public InstX86Base {
2730    InstX86Mfence() = delete;
2731    InstX86Mfence(const InstX86Mfence &) = delete;
2732    InstX86Mfence &operator=(const InstX86Mfence &) = delete;
2733
2734  public:
2735    static InstX86Mfence *create(Cfg *Func) {
2736      return new (Func->allocate<InstX86Mfence>()) InstX86Mfence(Func);
2737    }
2738    void emit(const Cfg *Func) const override;
2739    void emitIAS(const Cfg *Func) const override;
2740    void dump(const Cfg *Func) const override;
2741    static bool classof(const Inst *Instr) {
2742      return InstX86Base::isClassof(Instr, InstX86Base::Mfence);
2743    }
2744
2745  private:
2746    explicit InstX86Mfence(Cfg *Func);
2747  };
2748
2749  /// This is essentially a "mov" instruction with anX86OperandMem operand
2750  /// instead of Variable as the destination. It's important for liveness that
2751  /// there is no Dest operand.
2752  class InstX86Store final : public InstX86Base {
2753    InstX86Store() = delete;
2754    InstX86Store(const InstX86Store &) = delete;
2755    InstX86Store &operator=(const InstX86Store &) = delete;
2756
2757  public:
2758    static InstX86Store *create(Cfg *Func, Operand *Value, X86Operand *Mem) {
2759      return new (Func->allocate<InstX86Store>())
2760          InstX86Store(Func, Value, Mem);
2761    }
2762    void emit(const Cfg *Func) const override;
2763    void emitIAS(const Cfg *Func) const override;
2764    void dump(const Cfg *Func) const override;
2765    static bool classof(const Inst *Instr) {
2766      return InstX86Base::isClassof(Instr, InstX86Base::Store);
2767    }
2768
2769  private:
2770    InstX86Store(Cfg *Func, Operand *Value, X86Operand *Mem);
2771  };
2772
2773  /// This is essentially a vector "mov" instruction with an typename
2774  /// X86OperandMem operand instead of Variable as the destination. It's
2775  /// important for liveness that there is no Dest operand. The source must be
2776  /// an Xmm register, since Dest is mem.
2777  class InstX86StoreP final : public InstX86Base {
2778    InstX86StoreP() = delete;
2779    InstX86StoreP(const InstX86StoreP &) = delete;
2780    InstX86StoreP &operator=(const InstX86StoreP &) = delete;
2781
2782  public:
2783    static InstX86StoreP *create(Cfg *Func, Variable *Value,
2784                                 X86OperandMem *Mem) {
2785      return new (Func->allocate<InstX86StoreP>())
2786          InstX86StoreP(Func, Value, Mem);
2787    }
2788    void emit(const Cfg *Func) const override;
2789    void emitIAS(const Cfg *Func) const override;
2790    void dump(const Cfg *Func) const override;
2791    static bool classof(const Inst *Instr) {
2792      return InstX86Base::isClassof(Instr, InstX86Base::StoreP);
2793    }
2794
2795  private:
2796    InstX86StoreP(Cfg *Func, Variable *Value, X86OperandMem *Mem);
2797  };
2798
2799  class InstX86StoreQ final : public InstX86Base {
2800    InstX86StoreQ() = delete;
2801    InstX86StoreQ(const InstX86StoreQ &) = delete;
2802    InstX86StoreQ &operator=(const InstX86StoreQ &) = delete;
2803
2804  public:
2805    static InstX86StoreQ *create(Cfg *Func, Operand *Value,
2806                                 X86OperandMem *Mem) {
2807      return new (Func->allocate<InstX86StoreQ>())
2808          InstX86StoreQ(Func, Value, Mem);
2809    }
2810    void emit(const Cfg *Func) const override;
2811    void emitIAS(const Cfg *Func) const override;
2812    void dump(const Cfg *Func) const override;
2813    static bool classof(const Inst *Instr) {
2814      return InstX86Base::isClassof(Instr, InstX86Base::StoreQ);
2815    }
2816
2817  private:
2818    InstX86StoreQ(Cfg *Func, Operand *Value, X86OperandMem *Mem);
2819  };
2820
2821  class InstX86StoreD final : public InstX86Base {
2822    InstX86StoreD() = delete;
2823    InstX86StoreD(const InstX86StoreD &) = delete;
2824    InstX86StoreD &operator=(const InstX86StoreD &) = delete;
2825
2826  public:
2827    static InstX86StoreD *create(Cfg *Func, Operand *Value,
2828                                 X86OperandMem *Mem) {
2829      return new (Func->allocate<InstX86StoreD>())
2830          InstX86StoreD(Func, Value, Mem);
2831    }
2832    void emit(const Cfg *Func) const override;
2833    void emitIAS(const Cfg *Func) const override;
2834    void dump(const Cfg *Func) const override;
2835    static bool classof(const Inst *Instr) {
2836      return InstX86Base::isClassof(Instr, InstX86Base::StoreQ);
2837    }
2838
2839  private:
2840    InstX86StoreD(Cfg *Func, Operand *Value, X86OperandMem *Mem);
2841  };
2842
2843  /// Nop instructions of varying length
2844  class InstX86Nop final : public InstX86Base {
2845    InstX86Nop() = delete;
2846    InstX86Nop(const InstX86Nop &) = delete;
2847    InstX86Nop &operator=(const InstX86Nop &) = delete;
2848
2849  public:
2850    // TODO: Replace with enum.
2851    using NopVariant = unsigned;
2852
2853    static InstX86Nop *create(Cfg *Func, NopVariant Variant) {
2854      return new (Func->allocate<InstX86Nop>()) InstX86Nop(Func, Variant);
2855    }
2856    void emit(const Cfg *Func) const override;
2857    void emitIAS(const Cfg *Func) const override;
2858    void dump(const Cfg *Func) const override;
2859    static bool classof(const Inst *Instr) {
2860      return InstX86Base::isClassof(Instr, InstX86Base::Nop);
2861    }
2862
2863  private:
2864    InstX86Nop(Cfg *Func, NopVariant Length);
2865
2866    NopVariant Variant;
2867  };
2868
2869  /// Fld - load a value onto the x87 FP stack.
2870  class InstX86Fld final : public InstX86Base {
2871    InstX86Fld() = delete;
2872    InstX86Fld(const InstX86Fld &) = delete;
2873    InstX86Fld &operator=(const InstX86Fld &) = delete;
2874
2875  public:
2876    static InstX86Fld *create(Cfg *Func, Operand *Src) {
2877      return new (Func->allocate<InstX86Fld>()) InstX86Fld(Func, Src);
2878    }
2879    void emit(const Cfg *Func) const override;
2880    void emitIAS(const Cfg *Func) const override;
2881    void dump(const Cfg *Func) const override;
2882    static bool classof(const Inst *Instr) {
2883      return InstX86Base::isClassof(Instr, InstX86Base::Fld);
2884    }
2885
2886  private:
2887    InstX86Fld(Cfg *Func, Operand *Src);
2888  };
2889
2890  /// Fstp - store x87 st(0) into memory and pop st(0).
2891  class InstX86Fstp final : public InstX86Base {
2892    InstX86Fstp() = delete;
2893    InstX86Fstp(const InstX86Fstp &) = delete;
2894    InstX86Fstp &operator=(const InstX86Fstp &) = delete;
2895
2896  public:
2897    static InstX86Fstp *create(Cfg *Func, Variable *Dest) {
2898      return new (Func->allocate<InstX86Fstp>()) InstX86Fstp(Func, Dest);
2899    }
2900    void emit(const Cfg *Func) const override;
2901    void emitIAS(const Cfg *Func) const override;
2902    void dump(const Cfg *Func) const override;
2903    static bool classof(const Inst *Instr) {
2904      return InstX86Base::isClassof(Instr, InstX86Base::Fstp);
2905    }
2906
2907  private:
2908    InstX86Fstp(Cfg *Func, Variable *Dest);
2909  };
2910
2911  class InstX86Pop final : public InstX86Base {
2912    InstX86Pop() = delete;
2913    InstX86Pop(const InstX86Pop &) = delete;
2914    InstX86Pop &operator=(const InstX86Pop &) = delete;
2915
2916  public:
2917    static InstX86Pop *create(Cfg *Func, Variable *Dest) {
2918      return new (Func->allocate<InstX86Pop>()) InstX86Pop(Func, Dest);
2919    }
2920    void emit(const Cfg *Func) const override;
2921    void emitIAS(const Cfg *Func) const override;
2922    void dump(const Cfg *Func) const override;
2923    static bool classof(const Inst *Instr) {
2924      return InstX86Base::isClassof(Instr, InstX86Base::Pop);
2925    }
2926
2927  private:
2928    InstX86Pop(Cfg *Func, Variable *Dest);
2929  };
2930
2931  class InstX86Push final : public InstX86Base {
2932    InstX86Push() = delete;
2933    InstX86Push(const InstX86Push &) = delete;
2934    InstX86Push &operator=(const InstX86Push &) = delete;
2935
2936  public:
2937    static InstX86Push *create(Cfg *Func, InstX86Label *Label) {
2938      return new (Func->allocate<InstX86Push>()) InstX86Push(Func, Label);
2939    }
2940    static InstX86Push *create(Cfg *Func, Operand *Source) {
2941      return new (Func->allocate<InstX86Push>()) InstX86Push(Func, Source);
2942    }
2943    void emit(const Cfg *Func) const override;
2944    void emitIAS(const Cfg *Func) const override;
2945    void dump(const Cfg *Func) const override;
2946    static bool classof(const Inst *Instr) {
2947      return InstX86Base::isClassof(Instr, InstX86Base::Push);
2948    }
2949
2950  private:
2951    InstX86Label *Label = nullptr;
2952
2953    InstX86Push(Cfg *Func, Operand *Source);
2954    InstX86Push(Cfg *Func, InstX86Label *Label);
2955  };
2956
2957  /// Ret instruction. Currently only supports the "ret" version that does not
2958  /// pop arguments. This instruction takes a Source operand (for non-void
2959  /// returning functions) for liveness analysis, though a FakeUse before the
2960  /// ret would do just as well.
2961  class InstX86Ret final : public InstX86Base {
2962    InstX86Ret() = delete;
2963    InstX86Ret(const InstX86Ret &) = delete;
2964    InstX86Ret &operator=(const InstX86Ret &) = delete;
2965
2966  public:
2967    static InstX86Ret *create(Cfg *Func, Variable *Source = nullptr) {
2968      return new (Func->allocate<InstX86Ret>()) InstX86Ret(Func, Source);
2969    }
2970    void emit(const Cfg *Func) const override;
2971    void emitIAS(const Cfg *Func) const override;
2972    void dump(const Cfg *Func) const override;
2973    static bool classof(const Inst *Instr) {
2974      return InstX86Base::isClassof(Instr, InstX86Base::Ret);
2975    }
2976
2977  private:
2978    InstX86Ret(Cfg *Func, Variable *Source);
2979  };
2980
2981  /// Conditional set-byte instruction.
2982  class InstX86Setcc final : public InstX86Base {
2983    InstX86Setcc() = delete;
2984    InstX86Setcc(const InstX86Cmov &) = delete;
2985    InstX86Setcc &operator=(const InstX86Setcc &) = delete;
2986
2987  public:
2988    static InstX86Setcc *create(Cfg *Func, Variable *Dest, BrCond Cond) {
2989      return new (Func->allocate<InstX86Setcc>())
2990          InstX86Setcc(Func, Dest, Cond);
2991    }
2992    void emit(const Cfg *Func) const override;
2993    void emitIAS(const Cfg *Func) const override;
2994    void dump(const Cfg *Func) const override;
2995    static bool classof(const Inst *Instr) {
2996      return InstX86Base::isClassof(Instr, InstX86Base::Setcc);
2997    }
2998
2999  private:
3000    InstX86Setcc(Cfg *Func, Variable *Dest, BrCond Cond);
3001
3002    const BrCond Condition;
3003  };
3004
3005  /// Exchanging Add instruction. Exchanges the first operand (destination
3006  /// operand) with the second operand (source operand), then loads the sum of
3007  /// the two values into the destination operand. The destination may be a
3008  /// register or memory, while the source must be a register.
3009  ///
3010  /// Both the dest and source are updated. The caller should then insert a
3011  /// FakeDef to reflect the second udpate.
3012  class InstX86Xadd final : public InstX86BaseLockable {
3013    InstX86Xadd() = delete;
3014    InstX86Xadd(const InstX86Xadd &) = delete;
3015    InstX86Xadd &operator=(const InstX86Xadd &) = delete;
3016
3017  public:
3018    static InstX86Xadd *create(Cfg *Func, Operand *Dest, Variable *Source,
3019                               bool Locked) {
3020      return new (Func->allocate<InstX86Xadd>())
3021          InstX86Xadd(Func, Dest, Source, Locked);
3022    }
3023    void emit(const Cfg *Func) const override;
3024    void emitIAS(const Cfg *Func) const override;
3025    void dump(const Cfg *Func) const override;
3026    static bool classof(const Inst *Instr) {
3027      return InstX86Base::isClassof(Instr, InstX86Base::Xadd);
3028    }
3029
3030  private:
3031    InstX86Xadd(Cfg *Func, Operand *Dest, Variable *Source, bool Locked);
3032  };
3033
3034  /// Exchange instruction. Exchanges the first operand (destination operand)
3035  /// with the second operand (source operand). At least one of the operands
3036  /// must be a register (and the other can be reg or mem). Both the Dest and
3037  /// Source are updated. If there is a memory operand, then the instruction is
3038  /// automatically "locked" without the need for a lock prefix.
3039  class InstX86Xchg final : public InstX86Base {
3040    InstX86Xchg() = delete;
3041    InstX86Xchg(const InstX86Xchg &) = delete;
3042    InstX86Xchg &operator=(const InstX86Xchg &) = delete;
3043
3044  public:
3045    static InstX86Xchg *create(Cfg *Func, Operand *Dest, Variable *Source) {
3046      return new (Func->allocate<InstX86Xchg>())
3047          InstX86Xchg(Func, Dest, Source);
3048    }
3049    void emit(const Cfg *Func) const override;
3050    void emitIAS(const Cfg *Func) const override;
3051    void dump(const Cfg *Func) const override;
3052    static bool classof(const Inst *Instr) {
3053      return InstX86Base::isClassof(Instr, InstX86Base::Xchg);
3054    }
3055
3056  private:
3057    InstX86Xchg(Cfg *Func, Operand *Dest, Variable *Source);
3058  };
3059
3060  /// Start marker for the Intel Architecture Code Analyzer. This is not an
3061  /// executable instruction and must only be used for analysis.
3062  class InstX86IacaStart final : public InstX86Base {
3063    InstX86IacaStart() = delete;
3064    InstX86IacaStart(const InstX86IacaStart &) = delete;
3065    InstX86IacaStart &operator=(const InstX86IacaStart &) = delete;
3066
3067  public:
3068    static InstX86IacaStart *create(Cfg *Func) {
3069      return new (Func->allocate<InstX86IacaStart>()) InstX86IacaStart(Func);
3070    }
3071    void emit(const Cfg *Func) const override;
3072    void emitIAS(const Cfg *Func) const override;
3073    void dump(const Cfg *Func) const override;
3074    static bool classof(const Inst *Instr) {
3075      return InstX86Base::isClassof(Instr, InstX86Base::IacaStart);
3076    }
3077
3078  private:
3079    InstX86IacaStart(Cfg *Func);
3080  };
3081
3082  /// End marker for the Intel Architecture Code Analyzer. This is not an
3083  /// executable instruction and must only be used for analysis.
3084  class InstX86IacaEnd final : public InstX86Base {
3085    InstX86IacaEnd() = delete;
3086    InstX86IacaEnd(const InstX86IacaEnd &) = delete;
3087    InstX86IacaEnd &operator=(const InstX86IacaEnd &) = delete;
3088
3089  public:
3090    static InstX86IacaEnd *create(Cfg *Func) {
3091      return new (Func->allocate<InstX86IacaEnd>()) InstX86IacaEnd(Func);
3092    }
3093    void emit(const Cfg *Func) const override;
3094    void emitIAS(const Cfg *Func) const override;
3095    void dump(const Cfg *Func) const override;
3096    static bool classof(const Inst *Instr) {
3097      return InstX86Base::isClassof(Instr, InstX86Base::IacaEnd);
3098    }
3099
3100  private:
3101    InstX86IacaEnd(Cfg *Func);
3102  };
3103
3104  class InstX86Pshufb
3105      : public InstX86BaseBinopXmm<InstX86Base::Pshufb, false,
3106                                   InstX86Base::SseSuffix::None> {
3107  public:
3108    static InstX86Pshufb *create(Cfg *Func, Variable *Dest, Operand *Source) {
3109      return new (Func->allocate<InstX86Pshufb>())
3110          InstX86Pshufb(Func, Dest, Source);
3111    }
3112
3113  private:
3114    InstX86Pshufb(Cfg *Func, Variable *Dest, Operand *Source)
3115        : InstX86BaseBinopXmm<InstX86Base::Pshufb, false,
3116                              InstX86Base::SseSuffix::None>(Func, Dest,
3117                                                            Source) {}
3118  };
3119
3120  class InstX86Punpckl
3121      : public InstX86BaseBinopXmm<InstX86Base::Punpckl, false,
3122                                   InstX86Base::SseSuffix::Unpack> {
3123  public:
3124    static InstX86Punpckl *create(Cfg *Func, Variable *Dest, Operand *Source) {
3125      return new (Func->allocate<InstX86Punpckl>())
3126          InstX86Punpckl(Func, Dest, Source);
3127    }
3128
3129  private:
3130    InstX86Punpckl(Cfg *Func, Variable *Dest, Operand *Source)
3131        : InstX86BaseBinopXmm<InstX86Base::Punpckl, false,
3132                              InstX86Base::SseSuffix::Unpack>(Func, Dest,
3133                                                              Source) {}
3134  };
3135
3136  class InstX86Punpckh
3137      : public InstX86BaseBinopXmm<InstX86Base::Punpckh, false,
3138                                   InstX86Base::SseSuffix::Unpack> {
3139  public:
3140    static InstX86Punpckh *create(Cfg *Func, Variable *Dest, Operand *Source) {
3141      return new (Func->allocate<InstX86Punpckh>())
3142          InstX86Punpckh(Func, Dest, Source);
3143    }
3144
3145  private:
3146    InstX86Punpckh(Cfg *Func, Variable *Dest, Operand *Source)
3147        : InstX86BaseBinopXmm<InstX86Base::Punpckh, false,
3148                              InstX86Base::SseSuffix::Unpack>(Func, Dest,
3149                                                              Source) {}
3150  };
3151
3152  class InstX86Packss
3153      : public InstX86BaseBinopXmm<InstX86Base::Packss, false,
3154                                   InstX86Base::SseSuffix::Pack> {
3155  public:
3156    static InstX86Packss *create(Cfg *Func, Variable *Dest, Operand *Source) {
3157      return new (Func->allocate<InstX86Packss>())
3158          InstX86Packss(Func, Dest, Source);
3159    }
3160
3161  private:
3162    InstX86Packss(Cfg *Func, Variable *Dest, Operand *Source)
3163        : InstX86BaseBinopXmm<InstX86Base::Packss, false,
3164                              InstX86Base::SseSuffix::Pack>(Func, Dest,
3165                                                            Source) {}
3166  };
3167
3168  class InstX86Packus
3169      : public InstX86BaseBinopXmm<InstX86Base::Packus, false,
3170                                   InstX86Base::SseSuffix::Pack> {
3171  public:
3172    static InstX86Packus *create(Cfg *Func, Variable *Dest, Operand *Source) {
3173      return new (Func->allocate<InstX86Packus>())
3174          InstX86Packus(Func, Dest, Source);
3175    }
3176
3177  private:
3178    InstX86Packus(Cfg *Func, Variable *Dest, Operand *Source)
3179        : InstX86BaseBinopXmm<InstX86Base::Packus, false,
3180                              InstX86Base::SseSuffix::Pack>(Func, Dest,
3181                                                            Source) {}
3182  };
3183
3184}; // struct InstImpl
3185
3186/// struct Insts is a template that can be used to instantiate all the X86
3187/// instructions for a target with a simple
3188///
3189/// using Insts = ::Ice::X86NAMESPACE::Insts<TraitsType>;
3190template <typename TraitsType> struct Insts {
3191  using GetIP = typename InstImpl<TraitsType>::InstX86GetIP;
3192  using FakeRMW = typename InstImpl<TraitsType>::InstX86FakeRMW;
3193  using Label = typename InstImpl<TraitsType>::InstX86Label;
3194
3195  using Call = typename InstImpl<TraitsType>::InstX86Call;
3196
3197  using Br = typename InstImpl<TraitsType>::InstX86Br;
3198  using Jmp = typename InstImpl<TraitsType>::InstX86Jmp;
3199  using Bswap = typename InstImpl<TraitsType>::InstX86Bswap;
3200  using Neg = typename InstImpl<TraitsType>::InstX86Neg;
3201  using Bsf = typename InstImpl<TraitsType>::InstX86Bsf;
3202  using Bsr = typename InstImpl<TraitsType>::InstX86Bsr;
3203  using Lea = typename InstImpl<TraitsType>::InstX86Lea;
3204  using Cbwdq = typename InstImpl<TraitsType>::InstX86Cbwdq;
3205  using Movsx = typename InstImpl<TraitsType>::InstX86Movsx;
3206  using Movzx = typename InstImpl<TraitsType>::InstX86Movzx;
3207  using Movd = typename InstImpl<TraitsType>::InstX86Movd;
3208  using Movmsk = typename InstImpl<TraitsType>::InstX86Movmsk;
3209  using Sqrt = typename InstImpl<TraitsType>::InstX86Sqrt;
3210  using Mov = typename InstImpl<TraitsType>::InstX86Mov;
3211  using Movp = typename InstImpl<TraitsType>::InstX86Movp;
3212  using Movq = typename InstImpl<TraitsType>::InstX86Movq;
3213  using Add = typename InstImpl<TraitsType>::InstX86Add;
3214  using AddRMW = typename InstImpl<TraitsType>::InstX86AddRMW;
3215  using Addps = typename InstImpl<TraitsType>::InstX86Addps;
3216  using Adc = typename InstImpl<TraitsType>::InstX86Adc;
3217  using AdcRMW = typename InstImpl<TraitsType>::InstX86AdcRMW;
3218  using Addss = typename InstImpl<TraitsType>::InstX86Addss;
3219  using Andnps = typename InstImpl<TraitsType>::InstX86Andnps;
3220  using Andps = typename InstImpl<TraitsType>::InstX86Andps;
3221  using Padd = typename InstImpl<TraitsType>::InstX86Padd;
3222  using Padds = typename InstImpl<TraitsType>::InstX86Padds;
3223  using Paddus = typename InstImpl<TraitsType>::InstX86Paddus;
3224  using Sub = typename InstImpl<TraitsType>::InstX86Sub;
3225  using SubRMW = typename InstImpl<TraitsType>::InstX86SubRMW;
3226  using Subps = typename InstImpl<TraitsType>::InstX86Subps;
3227  using Subss = typename InstImpl<TraitsType>::InstX86Subss;
3228  using Sbb = typename InstImpl<TraitsType>::InstX86Sbb;
3229  using SbbRMW = typename InstImpl<TraitsType>::InstX86SbbRMW;
3230  using Psub = typename InstImpl<TraitsType>::InstX86Psub;
3231  using Psubs = typename InstImpl<TraitsType>::InstX86Psubs;
3232  using Psubus = typename InstImpl<TraitsType>::InstX86Psubus;
3233  using And = typename InstImpl<TraitsType>::InstX86And;
3234  using AndRMW = typename InstImpl<TraitsType>::InstX86AndRMW;
3235  using Pand = typename InstImpl<TraitsType>::InstX86Pand;
3236  using Pandn = typename InstImpl<TraitsType>::InstX86Pandn;
3237  using Or = typename InstImpl<TraitsType>::InstX86Or;
3238  using Orps = typename InstImpl<TraitsType>::InstX86Orps;
3239  using OrRMW = typename InstImpl<TraitsType>::InstX86OrRMW;
3240  using Por = typename InstImpl<TraitsType>::InstX86Por;
3241  using Xor = typename InstImpl<TraitsType>::InstX86Xor;
3242  using Xorps = typename InstImpl<TraitsType>::InstX86Xorps;
3243  using XorRMW = typename InstImpl<TraitsType>::InstX86XorRMW;
3244  using Pxor = typename InstImpl<TraitsType>::InstX86Pxor;
3245  using Maxss = typename InstImpl<TraitsType>::InstX86Maxss;
3246  using Minss = typename InstImpl<TraitsType>::InstX86Minss;
3247  using Maxps = typename InstImpl<TraitsType>::InstX86Maxps;
3248  using Minps = typename InstImpl<TraitsType>::InstX86Minps;
3249  using Imul = typename InstImpl<TraitsType>::InstX86Imul;
3250  using ImulImm = typename InstImpl<TraitsType>::InstX86ImulImm;
3251  using Mulps = typename InstImpl<TraitsType>::InstX86Mulps;
3252  using Mulss = typename InstImpl<TraitsType>::InstX86Mulss;
3253  using Pmull = typename InstImpl<TraitsType>::InstX86Pmull;
3254  using Pmulhw = typename InstImpl<TraitsType>::InstX86Pmulhw;
3255  using Pmulhuw = typename InstImpl<TraitsType>::InstX86Pmulhuw;
3256  using Pmaddwd = typename InstImpl<TraitsType>::InstX86Pmaddwd;
3257  using Pmuludq = typename InstImpl<TraitsType>::InstX86Pmuludq;
3258  using Divps = typename InstImpl<TraitsType>::InstX86Divps;
3259  using Divss = typename InstImpl<TraitsType>::InstX86Divss;
3260  using Rol = typename InstImpl<TraitsType>::InstX86Rol;
3261  using Shl = typename InstImpl<TraitsType>::InstX86Shl;
3262  using Psll = typename InstImpl<TraitsType>::InstX86Psll;
3263  using Psrl = typename InstImpl<TraitsType>::InstX86Psrl;
3264  using Shr = typename InstImpl<TraitsType>::InstX86Shr;
3265  using Sar = typename InstImpl<TraitsType>::InstX86Sar;
3266  using Psra = typename InstImpl<TraitsType>::InstX86Psra;
3267  using Pcmpeq = typename InstImpl<TraitsType>::InstX86Pcmpeq;
3268  using Pcmpgt = typename InstImpl<TraitsType>::InstX86Pcmpgt;
3269  using MovssRegs = typename InstImpl<TraitsType>::InstX86MovssRegs;
3270  using Idiv = typename InstImpl<TraitsType>::InstX86Idiv;
3271  using Div = typename InstImpl<TraitsType>::InstX86Div;
3272  using Insertps = typename InstImpl<TraitsType>::InstX86Insertps;
3273  using Pinsr = typename InstImpl<TraitsType>::InstX86Pinsr;
3274  using Shufps = typename InstImpl<TraitsType>::InstX86Shufps;
3275  using Blendvps = typename InstImpl<TraitsType>::InstX86Blendvps;
3276  using Pblendvb = typename InstImpl<TraitsType>::InstX86Pblendvb;
3277  using Pextr = typename InstImpl<TraitsType>::InstX86Pextr;
3278  using Pshufd = typename InstImpl<TraitsType>::InstX86Pshufd;
3279  using Lockable = typename InstImpl<TraitsType>::InstX86BaseLockable;
3280  using Mul = typename InstImpl<TraitsType>::InstX86Mul;
3281  using Shld = typename InstImpl<TraitsType>::InstX86Shld;
3282  using Shrd = typename InstImpl<TraitsType>::InstX86Shrd;
3283  using Cmov = typename InstImpl<TraitsType>::InstX86Cmov;
3284  using Cmpps = typename InstImpl<TraitsType>::InstX86Cmpps;
3285  using Cmpxchg = typename InstImpl<TraitsType>::InstX86Cmpxchg;
3286  using Cmpxchg8b = typename InstImpl<TraitsType>::InstX86Cmpxchg8b;
3287  using Cvt = typename InstImpl<TraitsType>::InstX86Cvt;
3288  using Round = typename InstImpl<TraitsType>::InstX86Round;
3289  using Icmp = typename InstImpl<TraitsType>::InstX86Icmp;
3290  using Ucomiss = typename InstImpl<TraitsType>::InstX86Ucomiss;
3291  using UD2 = typename InstImpl<TraitsType>::InstX86UD2;
3292  using Int3 = typename InstImpl<TraitsType>::InstX86Int3;
3293  using Test = typename InstImpl<TraitsType>::InstX86Test;
3294  using Mfence = typename InstImpl<TraitsType>::InstX86Mfence;
3295  using Store = typename InstImpl<TraitsType>::InstX86Store;
3296  using StoreP = typename InstImpl<TraitsType>::InstX86StoreP;
3297  using StoreQ = typename InstImpl<TraitsType>::InstX86StoreQ;
3298  using StoreD = typename InstImpl<TraitsType>::InstX86StoreD;
3299  using Nop = typename InstImpl<TraitsType>::InstX86Nop;
3300  template <typename T = typename InstImpl<TraitsType>::Traits>
3301  using Fld =
3302      typename std::enable_if<T::UsesX87,
3303                              typename InstImpl<TraitsType>::InstX86Fld>::type;
3304  template <typename T = typename InstImpl<TraitsType>::Traits>
3305  using Fstp =
3306      typename std::enable_if<T::UsesX87,
3307                              typename InstImpl<TraitsType>::InstX86Fstp>::type;
3308  using Pop = typename InstImpl<TraitsType>::InstX86Pop;
3309  using Push = typename InstImpl<TraitsType>::InstX86Push;
3310  using Ret = typename InstImpl<TraitsType>::InstX86Ret;
3311  using Setcc = typename InstImpl<TraitsType>::InstX86Setcc;
3312  using Xadd = typename InstImpl<TraitsType>::InstX86Xadd;
3313  using Xchg = typename InstImpl<TraitsType>::InstX86Xchg;
3314
3315  using IacaStart = typename InstImpl<TraitsType>::InstX86IacaStart;
3316  using IacaEnd = typename InstImpl<TraitsType>::InstX86IacaEnd;
3317
3318  using Pshufb = typename InstImpl<TraitsType>::InstX86Pshufb;
3319  using Punpckl = typename InstImpl<TraitsType>::InstX86Punpckl;
3320  using Punpckh = typename InstImpl<TraitsType>::InstX86Punpckh;
3321  using Packss = typename InstImpl<TraitsType>::InstX86Packss;
3322  using Packus = typename InstImpl<TraitsType>::InstX86Packus;
3323};
3324
3325/// X86 Instructions have static data (particularly, opcodes and instruction
3326/// emitters). Each X86 target needs to define all of these, so this macro is
3327/// provided so that, if something changes, then all X86 targets will be updated
3328/// automatically.
3329#define X86INSTS_DEFINE_STATIC_DATA(X86NAMESPACE, TraitsType)                  \
3330  namespace Ice {                                                              \
3331  namespace X86NAMESPACE {                                                     \
3332  /* In-place ops */                                                           \
3333  template <>                                                                  \
3334  template <>                                                                  \
3335  const char *InstImpl<TraitsType>::InstX86Bswap::Base::Opcode = "bswap";      \
3336  template <>                                                                  \
3337  template <>                                                                  \
3338  const char *InstImpl<TraitsType>::InstX86Neg::Base::Opcode = "neg";          \
3339  /* Unary ops */                                                              \
3340  template <>                                                                  \
3341  template <>                                                                  \
3342  const char *InstImpl<TraitsType>::InstX86Bsf::Base::Opcode = "bsf";          \
3343  template <>                                                                  \
3344  template <>                                                                  \
3345  const char *InstImpl<TraitsType>::InstX86Bsr::Base::Opcode = "bsr";          \
3346  template <>                                                                  \
3347  template <>                                                                  \
3348  const char *InstImpl<TraitsType>::InstX86Lea::Base::Opcode = "lea";          \
3349  template <>                                                                  \
3350  template <>                                                                  \
3351  const char *InstImpl<TraitsType>::InstX86Movd::Base::Opcode = "movd";        \
3352  template <>                                                                  \
3353  template <>                                                                  \
3354  const char *InstImpl<TraitsType>::InstX86Movsx::Base::Opcode = "movs";       \
3355  template <>                                                                  \
3356  template <>                                                                  \
3357  const char *InstImpl<TraitsType>::InstX86Movzx::Base::Opcode = "movz";       \
3358  template <>                                                                  \
3359  template <>                                                                  \
3360  const char *InstImpl<TraitsType>::InstX86Sqrt::Base::Opcode = "sqrt";        \
3361  template <>                                                                  \
3362  template <>                                                                  \
3363  const char *InstImpl<TraitsType>::InstX86Cbwdq::Base::Opcode =               \
3364      "cbw/cwd/cdq";                                                           \
3365  /* Mov-like ops */                                                           \
3366  template <>                                                                  \
3367  template <>                                                                  \
3368  const char *InstImpl<TraitsType>::InstX86Mov::Base::Opcode = "mov";          \
3369  template <>                                                                  \
3370  template <>                                                                  \
3371  const char *InstImpl<TraitsType>::InstX86Movp::Base::Opcode = "movups";      \
3372  template <>                                                                  \
3373  template <>                                                                  \
3374  const char *InstImpl<TraitsType>::InstX86Movq::Base::Opcode = "movq";        \
3375  /* Binary ops */                                                             \
3376  template <>                                                                  \
3377  template <>                                                                  \
3378  const char *InstImpl<TraitsType>::InstX86Add::Base::Opcode = "add";          \
3379  template <>                                                                  \
3380  template <>                                                                  \
3381  const char *InstImpl<TraitsType>::InstX86AddRMW::Base::Opcode = "add";       \
3382  template <>                                                                  \
3383  template <>                                                                  \
3384  const char *InstImpl<TraitsType>::InstX86Addps::Base::Opcode = "add";        \
3385  template <>                                                                  \
3386  template <>                                                                  \
3387  const char *InstImpl<TraitsType>::InstX86Adc::Base::Opcode = "adc";          \
3388  template <>                                                                  \
3389  template <>                                                                  \
3390  const char *InstImpl<TraitsType>::InstX86AdcRMW::Base::Opcode = "adc";       \
3391  template <>                                                                  \
3392  template <>                                                                  \
3393  const char *InstImpl<TraitsType>::InstX86Addss::Base::Opcode = "add";        \
3394  template <>                                                                  \
3395  template <>                                                                  \
3396  const char *InstImpl<TraitsType>::InstX86Andnps::Base::Opcode = "andn";      \
3397  template <>                                                                  \
3398  template <>                                                                  \
3399  const char *InstImpl<TraitsType>::InstX86Andps::Base::Opcode = "and";        \
3400  template <>                                                                  \
3401  template <>                                                                  \
3402  const char *InstImpl<TraitsType>::InstX86Maxss::Base::Opcode = "max";        \
3403  template <>                                                                  \
3404  template <>                                                                  \
3405  const char *InstImpl<TraitsType>::InstX86Minss::Base::Opcode = "min";        \
3406  template <>                                                                  \
3407  template <>                                                                  \
3408  const char *InstImpl<TraitsType>::InstX86Maxps::Base::Opcode = "max";        \
3409  template <>                                                                  \
3410  template <>                                                                  \
3411  const char *InstImpl<TraitsType>::InstX86Minps::Base::Opcode = "min";        \
3412  template <>                                                                  \
3413  template <>                                                                  \
3414  const char *InstImpl<TraitsType>::InstX86Padd::Base::Opcode = "padd";        \
3415  template <>                                                                  \
3416  template <>                                                                  \
3417  const char *InstImpl<TraitsType>::InstX86Padds::Base::Opcode = "padds";      \
3418  template <>                                                                  \
3419  template <>                                                                  \
3420  const char *InstImpl<TraitsType>::InstX86Paddus::Base::Opcode = "paddus";    \
3421  template <>                                                                  \
3422  template <>                                                                  \
3423  const char *InstImpl<TraitsType>::InstX86Sub::Base::Opcode = "sub";          \
3424  template <>                                                                  \
3425  template <>                                                                  \
3426  const char *InstImpl<TraitsType>::InstX86SubRMW::Base::Opcode = "sub";       \
3427  template <>                                                                  \
3428  template <>                                                                  \
3429  const char *InstImpl<TraitsType>::InstX86Subps::Base::Opcode = "sub";        \
3430  template <>                                                                  \
3431  template <>                                                                  \
3432  const char *InstImpl<TraitsType>::InstX86Subss::Base::Opcode = "sub";        \
3433  template <>                                                                  \
3434  template <>                                                                  \
3435  const char *InstImpl<TraitsType>::InstX86Sbb::Base::Opcode = "sbb";          \
3436  template <>                                                                  \
3437  template <>                                                                  \
3438  const char *InstImpl<TraitsType>::InstX86SbbRMW::Base::Opcode = "sbb";       \
3439  template <>                                                                  \
3440  template <>                                                                  \
3441  const char *InstImpl<TraitsType>::InstX86Psub::Base::Opcode = "psub";        \
3442  template <>                                                                  \
3443  template <>                                                                  \
3444  const char *InstImpl<TraitsType>::InstX86Psubs::Base::Opcode = "psubs";      \
3445  template <>                                                                  \
3446  template <>                                                                  \
3447  const char *InstImpl<TraitsType>::InstX86Psubus::Base::Opcode = "psubus";    \
3448  template <>                                                                  \
3449  template <>                                                                  \
3450  const char *InstImpl<TraitsType>::InstX86And::Base::Opcode = "and";          \
3451  template <>                                                                  \
3452  template <>                                                                  \
3453  const char *InstImpl<TraitsType>::InstX86AndRMW::Base::Opcode = "and";       \
3454  template <>                                                                  \
3455  template <>                                                                  \
3456  const char *InstImpl<TraitsType>::InstX86Pand::Base::Opcode = "pand";        \
3457  template <>                                                                  \
3458  template <>                                                                  \
3459  const char *InstImpl<TraitsType>::InstX86Pandn::Base::Opcode = "pandn";      \
3460  template <>                                                                  \
3461  template <>                                                                  \
3462  const char *InstImpl<TraitsType>::InstX86Or::Base::Opcode = "or";            \
3463  template <>                                                                  \
3464  template <>                                                                  \
3465  const char *InstImpl<TraitsType>::InstX86Orps::Base::Opcode = "or";          \
3466  template <>                                                                  \
3467  template <>                                                                  \
3468  const char *InstImpl<TraitsType>::InstX86OrRMW::Base::Opcode = "or";         \
3469  template <>                                                                  \
3470  template <>                                                                  \
3471  const char *InstImpl<TraitsType>::InstX86Por::Base::Opcode = "por";          \
3472  template <>                                                                  \
3473  template <>                                                                  \
3474  const char *InstImpl<TraitsType>::InstX86Xor::Base::Opcode = "xor";          \
3475  template <>                                                                  \
3476  template <>                                                                  \
3477  const char *InstImpl<TraitsType>::InstX86Xorps::Base::Opcode = "xor";        \
3478  template <>                                                                  \
3479  template <>                                                                  \
3480  const char *InstImpl<TraitsType>::InstX86XorRMW::Base::Opcode = "xor";       \
3481  template <>                                                                  \
3482  template <>                                                                  \
3483  const char *InstImpl<TraitsType>::InstX86Pxor::Base::Opcode = "pxor";        \
3484  template <>                                                                  \
3485  template <>                                                                  \
3486  const char *InstImpl<TraitsType>::InstX86Imul::Base::Opcode = "imul";        \
3487  template <>                                                                  \
3488  template <>                                                                  \
3489  const char *InstImpl<TraitsType>::InstX86ImulImm::Base::Opcode = "imul";     \
3490  template <>                                                                  \
3491  template <>                                                                  \
3492  const char *InstImpl<TraitsType>::InstX86Mulps::Base::Opcode = "mul";        \
3493  template <>                                                                  \
3494  template <>                                                                  \
3495  const char *InstImpl<TraitsType>::InstX86Mulss::Base::Opcode = "mul";        \
3496  template <>                                                                  \
3497  template <>                                                                  \
3498  const char *InstImpl<TraitsType>::InstX86Pmull::Base::Opcode = "pmull";      \
3499  template <>                                                                  \
3500  template <>                                                                  \
3501  const char *InstImpl<TraitsType>::InstX86Pmulhw::Base::Opcode = "pmulhw";    \
3502  template <>                                                                  \
3503  template <>                                                                  \
3504  const char *InstImpl<TraitsType>::InstX86Pmulhuw::Base::Opcode = "pmulhuw";  \
3505  template <>                                                                  \
3506  template <>                                                                  \
3507  const char *InstImpl<TraitsType>::InstX86Pmaddwd::Base::Opcode = "pmaddwd";  \
3508  template <>                                                                  \
3509  template <>                                                                  \
3510  const char *InstImpl<TraitsType>::InstX86Pmuludq::Base::Opcode = "pmuludq";  \
3511  template <>                                                                  \
3512  template <>                                                                  \
3513  const char *InstImpl<TraitsType>::InstX86Div::Base::Opcode = "div";          \
3514  template <>                                                                  \
3515  template <>                                                                  \
3516  const char *InstImpl<TraitsType>::InstX86Divps::Base::Opcode = "div";        \
3517  template <>                                                                  \
3518  template <>                                                                  \
3519  const char *InstImpl<TraitsType>::InstX86Divss::Base::Opcode = "div";        \
3520  template <>                                                                  \
3521  template <>                                                                  \
3522  const char *InstImpl<TraitsType>::InstX86Idiv::Base::Opcode = "idiv";        \
3523  template <>                                                                  \
3524  template <>                                                                  \
3525  const char *InstImpl<TraitsType>::InstX86Rol::Base::Opcode = "rol";          \
3526  template <>                                                                  \
3527  template <>                                                                  \
3528  const char *InstImpl<TraitsType>::InstX86Shl::Base::Opcode = "shl";          \
3529  template <>                                                                  \
3530  template <>                                                                  \
3531  const char *InstImpl<TraitsType>::InstX86Psll::Base::Opcode = "psll";        \
3532  template <>                                                                  \
3533  template <>                                                                  \
3534  const char *InstImpl<TraitsType>::InstX86Shr::Base::Opcode = "shr";          \
3535  template <>                                                                  \
3536  template <>                                                                  \
3537  const char *InstImpl<TraitsType>::InstX86Sar::Base::Opcode = "sar";          \
3538  template <>                                                                  \
3539  template <>                                                                  \
3540  const char *InstImpl<TraitsType>::InstX86Psra::Base::Opcode = "psra";        \
3541  template <>                                                                  \
3542  template <>                                                                  \
3543  const char *InstImpl<TraitsType>::InstX86Psrl::Base::Opcode = "psrl";        \
3544  template <>                                                                  \
3545  template <>                                                                  \
3546  const char *InstImpl<TraitsType>::InstX86Pcmpeq::Base::Opcode = "pcmpeq";    \
3547  template <>                                                                  \
3548  template <>                                                                  \
3549  const char *InstImpl<TraitsType>::InstX86Pcmpgt::Base::Opcode = "pcmpgt";    \
3550  template <>                                                                  \
3551  template <>                                                                  \
3552  const char *InstImpl<TraitsType>::InstX86MovssRegs::Base::Opcode = "movss";  \
3553  /* Ternary ops */                                                            \
3554  template <>                                                                  \
3555  template <>                                                                  \
3556  const char *InstImpl<TraitsType>::InstX86Insertps::Base::Opcode =            \
3557      "insertps";                                                              \
3558  template <>                                                                  \
3559  template <>                                                                  \
3560  const char *InstImpl<TraitsType>::InstX86Round::Base::Opcode = "round";      \
3561  template <>                                                                  \
3562  template <>                                                                  \
3563  const char *InstImpl<TraitsType>::InstX86Shufps::Base::Opcode = "shufps";    \
3564  template <>                                                                  \
3565  template <>                                                                  \
3566  const char *InstImpl<TraitsType>::InstX86Pinsr::Base::Opcode = "pinsr";      \
3567  template <>                                                                  \
3568  template <>                                                                  \
3569  const char *InstImpl<TraitsType>::InstX86Blendvps::Base::Opcode =            \
3570      "blendvps";                                                              \
3571  template <>                                                                  \
3572  template <>                                                                  \
3573  const char *InstImpl<TraitsType>::InstX86Pblendvb::Base::Opcode =            \
3574      "pblendvb";                                                              \
3575  /* Three address ops */                                                      \
3576  template <>                                                                  \
3577  template <>                                                                  \
3578  const char *InstImpl<TraitsType>::InstX86Pextr::Base::Opcode = "pextr";      \
3579  template <>                                                                  \
3580  template <>                                                                  \
3581  const char *InstImpl<TraitsType>::InstX86Pshufd::Base::Opcode = "pshufd";    \
3582  template <>                                                                  \
3583  template <>                                                                  \
3584  const char *InstImpl<TraitsType>::InstX86Pshufb::Base::Opcode = "pshufb";    \
3585  template <>                                                                  \
3586  template <>                                                                  \
3587  const char *InstImpl<TraitsType>::InstX86Punpckl::Base::Opcode = "punpckl";  \
3588  template <>                                                                  \
3589  template <>                                                                  \
3590  const char *InstImpl<TraitsType>::InstX86Punpckh::Base::Opcode = "punpckh";  \
3591  template <>                                                                  \
3592  template <>                                                                  \
3593  const char *InstImpl<TraitsType>::InstX86Packss::Base::Opcode = "packss";    \
3594  template <>                                                                  \
3595  template <>                                                                  \
3596  const char *InstImpl<TraitsType>::InstX86Packus::Base::Opcode = "packus";    \
3597  /* Inplace GPR ops */                                                        \
3598  template <>                                                                  \
3599  template <>                                                                  \
3600  const InstImpl<TraitsType>::Assembler::GPREmitterOneOp                       \
3601      InstImpl<TraitsType>::InstX86Bswap::Base::Emitter = {                    \
3602          &InstImpl<TraitsType>::Assembler::bswap,                             \
3603          nullptr /* only a reg form exists */                                 \
3604  };                                                                           \
3605  template <>                                                                  \
3606  template <>                                                                  \
3607  const InstImpl<TraitsType>::Assembler::GPREmitterOneOp                       \
3608      InstImpl<TraitsType>::InstX86Neg::Base::Emitter = {                      \
3609          &InstImpl<TraitsType>::Assembler::neg,                               \
3610          &InstImpl<TraitsType>::Assembler::neg};                              \
3611                                                                               \
3612  /* Unary GPR ops */                                                          \
3613  template <>                                                                  \
3614  template <> /* uses specialized emitter. */                                  \
3615  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3616      InstImpl<TraitsType>::InstX86Cbwdq::Base::Emitter = {nullptr, nullptr,   \
3617                                                           nullptr};           \
3618  template <>                                                                  \
3619  template <>                                                                  \
3620  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3621      InstImpl<TraitsType>::InstX86Bsf::Base::Emitter = {                      \
3622          &InstImpl<TraitsType>::Assembler::bsf,                               \
3623          &InstImpl<TraitsType>::Assembler::bsf, nullptr};                     \
3624  template <>                                                                  \
3625  template <>                                                                  \
3626  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3627      InstImpl<TraitsType>::InstX86Bsr::Base::Emitter = {                      \
3628          &InstImpl<TraitsType>::Assembler::bsr,                               \
3629          &InstImpl<TraitsType>::Assembler::bsr, nullptr};                     \
3630  template <>                                                                  \
3631  template <>                                                                  \
3632  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3633      InstImpl<TraitsType>::InstX86Lea::Base::Emitter = {                      \
3634          /* reg/reg and reg/imm are illegal */ nullptr,                       \
3635          &InstImpl<TraitsType>::Assembler::lea, nullptr};                     \
3636  template <>                                                                  \
3637  template <>                                                                  \
3638  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3639      InstImpl<TraitsType>::InstX86Movsx::Base::Emitter = {                    \
3640          &InstImpl<TraitsType>::Assembler::movsx,                             \
3641          &InstImpl<TraitsType>::Assembler::movsx, nullptr};                   \
3642  template <>                                                                  \
3643  template <>                                                                  \
3644  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3645      InstImpl<TraitsType>::InstX86Movzx::Base::Emitter = {                    \
3646          &InstImpl<TraitsType>::Assembler::movzx,                             \
3647          &InstImpl<TraitsType>::Assembler::movzx, nullptr};                   \
3648                                                                               \
3649  /* Unary XMM ops */                                                          \
3650  template <>                                                                  \
3651  template <> /* uses specialized emitter. */                                  \
3652  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3653      InstImpl<TraitsType>::InstX86Movd::Base::Emitter = {nullptr, nullptr};   \
3654  template <>                                                                  \
3655  template <>                                                                  \
3656  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3657      InstImpl<TraitsType>::InstX86Sqrt::Base::Emitter = {                     \
3658          &InstImpl<TraitsType>::Assembler::sqrt,                              \
3659          &InstImpl<TraitsType>::Assembler::sqrt};                             \
3660                                                                               \
3661  /* Binary GPR ops */                                                         \
3662  template <>                                                                  \
3663  template <> /* uses specialized emitter. */                                  \
3664  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3665      InstImpl<TraitsType>::InstX86Imul::Base::Emitter = {nullptr, nullptr,    \
3666                                                          nullptr};            \
3667  template <>                                                                  \
3668  template <>                                                                  \
3669  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3670      InstImpl<TraitsType>::InstX86Add::Base::Emitter = {                      \
3671          &InstImpl<TraitsType>::Assembler::add,                               \
3672          &InstImpl<TraitsType>::Assembler::add,                               \
3673          &InstImpl<TraitsType>::Assembler::add};                              \
3674  template <>                                                                  \
3675  template <>                                                                  \
3676  const InstImpl<TraitsType>::Assembler::GPREmitterAddrOp                      \
3677      InstImpl<TraitsType>::InstX86AddRMW::Base::Emitter = {                   \
3678          &InstImpl<TraitsType>::Assembler::add,                               \
3679          &InstImpl<TraitsType>::Assembler::add};                              \
3680  template <>                                                                  \
3681  template <>                                                                  \
3682  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3683      InstImpl<TraitsType>::InstX86Adc::Base::Emitter = {                      \
3684          &InstImpl<TraitsType>::Assembler::adc,                               \
3685          &InstImpl<TraitsType>::Assembler::adc,                               \
3686          &InstImpl<TraitsType>::Assembler::adc};                              \
3687  template <>                                                                  \
3688  template <>                                                                  \
3689  const InstImpl<TraitsType>::Assembler::GPREmitterAddrOp                      \
3690      InstImpl<TraitsType>::InstX86AdcRMW::Base::Emitter = {                   \
3691          &InstImpl<TraitsType>::Assembler::adc,                               \
3692          &InstImpl<TraitsType>::Assembler::adc};                              \
3693  template <>                                                                  \
3694  template <>                                                                  \
3695  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3696      InstImpl<TraitsType>::InstX86And::Base::Emitter = {                      \
3697          &InstImpl<TraitsType>::Assembler::And,                               \
3698          &InstImpl<TraitsType>::Assembler::And,                               \
3699          &InstImpl<TraitsType>::Assembler::And};                              \
3700  template <>                                                                  \
3701  template <>                                                                  \
3702  const InstImpl<TraitsType>::Assembler::GPREmitterAddrOp                      \
3703      InstImpl<TraitsType>::InstX86AndRMW::Base::Emitter = {                   \
3704          &InstImpl<TraitsType>::Assembler::And,                               \
3705          &InstImpl<TraitsType>::Assembler::And};                              \
3706  template <>                                                                  \
3707  template <>                                                                  \
3708  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3709      InstImpl<TraitsType>::InstX86Or::Base::Emitter = {                       \
3710          &InstImpl<TraitsType>::Assembler::Or,                                \
3711          &InstImpl<TraitsType>::Assembler::Or,                                \
3712          &InstImpl<TraitsType>::Assembler::Or};                               \
3713  template <>                                                                  \
3714  template <>                                                                  \
3715  const InstImpl<TraitsType>::Assembler::GPREmitterAddrOp                      \
3716      InstImpl<TraitsType>::InstX86OrRMW::Base::Emitter = {                    \
3717          &InstImpl<TraitsType>::Assembler::Or,                                \
3718          &InstImpl<TraitsType>::Assembler::Or};                               \
3719  template <>                                                                  \
3720  template <>                                                                  \
3721  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3722      InstImpl<TraitsType>::InstX86Sbb::Base::Emitter = {                      \
3723          &InstImpl<TraitsType>::Assembler::sbb,                               \
3724          &InstImpl<TraitsType>::Assembler::sbb,                               \
3725          &InstImpl<TraitsType>::Assembler::sbb};                              \
3726  template <>                                                                  \
3727  template <>                                                                  \
3728  const InstImpl<TraitsType>::Assembler::GPREmitterAddrOp                      \
3729      InstImpl<TraitsType>::InstX86SbbRMW::Base::Emitter = {                   \
3730          &InstImpl<TraitsType>::Assembler::sbb,                               \
3731          &InstImpl<TraitsType>::Assembler::sbb};                              \
3732  template <>                                                                  \
3733  template <>                                                                  \
3734  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3735      InstImpl<TraitsType>::InstX86Sub::Base::Emitter = {                      \
3736          &InstImpl<TraitsType>::Assembler::sub,                               \
3737          &InstImpl<TraitsType>::Assembler::sub,                               \
3738          &InstImpl<TraitsType>::Assembler::sub};                              \
3739  template <>                                                                  \
3740  template <>                                                                  \
3741  const InstImpl<TraitsType>::Assembler::GPREmitterAddrOp                      \
3742      InstImpl<TraitsType>::InstX86SubRMW::Base::Emitter = {                   \
3743          &InstImpl<TraitsType>::Assembler::sub,                               \
3744          &InstImpl<TraitsType>::Assembler::sub};                              \
3745  template <>                                                                  \
3746  template <>                                                                  \
3747  const InstImpl<TraitsType>::Assembler::GPREmitterRegOp                       \
3748      InstImpl<TraitsType>::InstX86Xor::Base::Emitter = {                      \
3749          &InstImpl<TraitsType>::Assembler::Xor,                               \
3750          &InstImpl<TraitsType>::Assembler::Xor,                               \
3751          &InstImpl<TraitsType>::Assembler::Xor};                              \
3752  template <>                                                                  \
3753  template <>                                                                  \
3754  const InstImpl<TraitsType>::Assembler::GPREmitterAddrOp                      \
3755      InstImpl<TraitsType>::InstX86XorRMW::Base::Emitter = {                   \
3756          &InstImpl<TraitsType>::Assembler::Xor,                               \
3757          &InstImpl<TraitsType>::Assembler::Xor};                              \
3758                                                                               \
3759  /* Binary Shift GPR ops */                                                   \
3760  template <>                                                                  \
3761  template <>                                                                  \
3762  const InstImpl<TraitsType>::Assembler::GPREmitterShiftOp                     \
3763      InstImpl<TraitsType>::InstX86Rol::Base::Emitter = {                      \
3764          &InstImpl<TraitsType>::Assembler::rol,                               \
3765          &InstImpl<TraitsType>::Assembler::rol};                              \
3766  template <>                                                                  \
3767  template <>                                                                  \
3768  const InstImpl<TraitsType>::Assembler::GPREmitterShiftOp                     \
3769      InstImpl<TraitsType>::InstX86Sar::Base::Emitter = {                      \
3770          &InstImpl<TraitsType>::Assembler::sar,                               \
3771          &InstImpl<TraitsType>::Assembler::sar};                              \
3772  template <>                                                                  \
3773  template <>                                                                  \
3774  const InstImpl<TraitsType>::Assembler::GPREmitterShiftOp                     \
3775      InstImpl<TraitsType>::InstX86Shl::Base::Emitter = {                      \
3776          &InstImpl<TraitsType>::Assembler::shl,                               \
3777          &InstImpl<TraitsType>::Assembler::shl};                              \
3778  template <>                                                                  \
3779  template <>                                                                  \
3780  const InstImpl<TraitsType>::Assembler::GPREmitterShiftOp                     \
3781      InstImpl<TraitsType>::InstX86Shr::Base::Emitter = {                      \
3782          &InstImpl<TraitsType>::Assembler::shr,                               \
3783          &InstImpl<TraitsType>::Assembler::shr};                              \
3784                                                                               \
3785  /* Binary XMM ops */                                                         \
3786  template <>                                                                  \
3787  template <> /* uses specialized emitter. */                                  \
3788  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3789      InstImpl<TraitsType>::InstX86MovssRegs::Base::Emitter = {nullptr,        \
3790                                                               nullptr};       \
3791  template <>                                                                  \
3792  template <>                                                                  \
3793  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3794      InstImpl<TraitsType>::InstX86Addss::Base::Emitter = {                    \
3795          &InstImpl<TraitsType>::Assembler::addss,                             \
3796          &InstImpl<TraitsType>::Assembler::addss};                            \
3797  template <>                                                                  \
3798  template <>                                                                  \
3799  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3800      InstImpl<TraitsType>::InstX86Addps::Base::Emitter = {                    \
3801          &InstImpl<TraitsType>::Assembler::addps,                             \
3802          &InstImpl<TraitsType>::Assembler::addps};                            \
3803  template <>                                                                  \
3804  template <>                                                                  \
3805  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3806      InstImpl<TraitsType>::InstX86Divss::Base::Emitter = {                    \
3807          &InstImpl<TraitsType>::Assembler::divss,                             \
3808          &InstImpl<TraitsType>::Assembler::divss};                            \
3809  template <>                                                                  \
3810  template <>                                                                  \
3811  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3812      InstImpl<TraitsType>::InstX86Divps::Base::Emitter = {                    \
3813          &InstImpl<TraitsType>::Assembler::divps,                             \
3814          &InstImpl<TraitsType>::Assembler::divps};                            \
3815  template <>                                                                  \
3816  template <>                                                                  \
3817  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3818      InstImpl<TraitsType>::InstX86Mulss::Base::Emitter = {                    \
3819          &InstImpl<TraitsType>::Assembler::mulss,                             \
3820          &InstImpl<TraitsType>::Assembler::mulss};                            \
3821  template <>                                                                  \
3822  template <>                                                                  \
3823  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3824      InstImpl<TraitsType>::InstX86Mulps::Base::Emitter = {                    \
3825          &InstImpl<TraitsType>::Assembler::mulps,                             \
3826          &InstImpl<TraitsType>::Assembler::mulps};                            \
3827  template <>                                                                  \
3828  template <>                                                                  \
3829  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3830      InstImpl<TraitsType>::InstX86Padd::Base::Emitter = {                     \
3831          &InstImpl<TraitsType>::Assembler::padd,                              \
3832          &InstImpl<TraitsType>::Assembler::padd};                             \
3833  template <>                                                                  \
3834  template <>                                                                  \
3835  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3836      InstImpl<TraitsType>::InstX86Padds::Base::Emitter = {                    \
3837          &InstImpl<TraitsType>::Assembler::padds,                             \
3838          &InstImpl<TraitsType>::Assembler::padds};                            \
3839  template <>                                                                  \
3840  template <>                                                                  \
3841  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3842      InstImpl<TraitsType>::InstX86Paddus::Base::Emitter = {                   \
3843          &InstImpl<TraitsType>::Assembler::paddus,                            \
3844          &InstImpl<TraitsType>::Assembler::paddus};                           \
3845  template <>                                                                  \
3846  template <>                                                                  \
3847  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3848      InstImpl<TraitsType>::InstX86Pand::Base::Emitter = {                     \
3849          &InstImpl<TraitsType>::Assembler::pand,                              \
3850          &InstImpl<TraitsType>::Assembler::pand};                             \
3851  template <>                                                                  \
3852  template <>                                                                  \
3853  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3854      InstImpl<TraitsType>::InstX86Pandn::Base::Emitter = {                    \
3855          &InstImpl<TraitsType>::Assembler::pandn,                             \
3856          &InstImpl<TraitsType>::Assembler::pandn};                            \
3857  template <>                                                                  \
3858  template <>                                                                  \
3859  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3860      InstImpl<TraitsType>::InstX86Pcmpeq::Base::Emitter = {                   \
3861          &InstImpl<TraitsType>::Assembler::pcmpeq,                            \
3862          &InstImpl<TraitsType>::Assembler::pcmpeq};                           \
3863  template <>                                                                  \
3864  template <>                                                                  \
3865  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3866      InstImpl<TraitsType>::InstX86Pcmpgt::Base::Emitter = {                   \
3867          &InstImpl<TraitsType>::Assembler::pcmpgt,                            \
3868          &InstImpl<TraitsType>::Assembler::pcmpgt};                           \
3869  template <>                                                                  \
3870  template <>                                                                  \
3871  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3872      InstImpl<TraitsType>::InstX86Pmull::Base::Emitter = {                    \
3873          &InstImpl<TraitsType>::Assembler::pmull,                             \
3874          &InstImpl<TraitsType>::Assembler::pmull};                            \
3875  template <>                                                                  \
3876  template <>                                                                  \
3877  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3878      InstImpl<TraitsType>::InstX86Pmulhw::Base::Emitter = {                   \
3879          &InstImpl<TraitsType>::Assembler::pmulhw,                            \
3880          &InstImpl<TraitsType>::Assembler::pmulhw};                           \
3881  template <>                                                                  \
3882  template <>                                                                  \
3883  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3884      InstImpl<TraitsType>::InstX86Pmulhuw::Base::Emitter = {                  \
3885          &InstImpl<TraitsType>::Assembler::pmulhuw,                           \
3886          &InstImpl<TraitsType>::Assembler::pmulhuw};                          \
3887  template <>                                                                  \
3888  template <>                                                                  \
3889  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3890      InstImpl<TraitsType>::InstX86Pmaddwd::Base::Emitter = {                  \
3891          &InstImpl<TraitsType>::Assembler::pmaddwd,                           \
3892          &InstImpl<TraitsType>::Assembler::pmaddwd};                          \
3893  template <>                                                                  \
3894  template <>                                                                  \
3895  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3896      InstImpl<TraitsType>::InstX86Pmuludq::Base::Emitter = {                  \
3897          &InstImpl<TraitsType>::Assembler::pmuludq,                           \
3898          &InstImpl<TraitsType>::Assembler::pmuludq};                          \
3899  template <>                                                                  \
3900  template <>                                                                  \
3901  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3902      InstImpl<TraitsType>::InstX86Por::Base::Emitter = {                      \
3903          &InstImpl<TraitsType>::Assembler::por,                               \
3904          &InstImpl<TraitsType>::Assembler::por};                              \
3905  template <>                                                                  \
3906  template <>                                                                  \
3907  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3908      InstImpl<TraitsType>::InstX86Psub::Base::Emitter = {                     \
3909          &InstImpl<TraitsType>::Assembler::psub,                              \
3910          &InstImpl<TraitsType>::Assembler::psub};                             \
3911  template <>                                                                  \
3912  template <>                                                                  \
3913  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3914      InstImpl<TraitsType>::InstX86Psubs::Base::Emitter = {                    \
3915          &InstImpl<TraitsType>::Assembler::psubs,                             \
3916          &InstImpl<TraitsType>::Assembler::psubs};                            \
3917  template <>                                                                  \
3918  template <>                                                                  \
3919  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3920      InstImpl<TraitsType>::InstX86Psubus::Base::Emitter = {                   \
3921          &InstImpl<TraitsType>::Assembler::psubus,                            \
3922          &InstImpl<TraitsType>::Assembler::psubus};                           \
3923  template <>                                                                  \
3924  template <>                                                                  \
3925  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3926      InstImpl<TraitsType>::InstX86Pxor::Base::Emitter = {                     \
3927          &InstImpl<TraitsType>::Assembler::pxor,                              \
3928          &InstImpl<TraitsType>::Assembler::pxor};                             \
3929  template <>                                                                  \
3930  template <>                                                                  \
3931  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3932      InstImpl<TraitsType>::InstX86Subss::Base::Emitter = {                    \
3933          &InstImpl<TraitsType>::Assembler::subss,                             \
3934          &InstImpl<TraitsType>::Assembler::subss};                            \
3935  template <>                                                                  \
3936  template <>                                                                  \
3937  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3938      InstImpl<TraitsType>::InstX86Subps::Base::Emitter = {                    \
3939          &InstImpl<TraitsType>::Assembler::subps,                             \
3940          &InstImpl<TraitsType>::Assembler::subps};                            \
3941  template <>                                                                  \
3942  template <>                                                                  \
3943  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3944      InstImpl<TraitsType>::InstX86Andnps::Base::Emitter = {                   \
3945          &InstImpl<TraitsType>::Assembler::andnps,                            \
3946          &InstImpl<TraitsType>::Assembler::andnps};                           \
3947  template <>                                                                  \
3948  template <>                                                                  \
3949  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3950      InstImpl<TraitsType>::InstX86Andps::Base::Emitter = {                    \
3951          &InstImpl<TraitsType>::Assembler::andps,                             \
3952          &InstImpl<TraitsType>::Assembler::andps};                            \
3953  template <>                                                                  \
3954  template <>                                                                  \
3955  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3956      InstImpl<TraitsType>::InstX86Maxss::Base::Emitter = {                    \
3957          &InstImpl<TraitsType>::Assembler::maxss,                             \
3958          &InstImpl<TraitsType>::Assembler::maxss};                            \
3959  template <>                                                                  \
3960  template <>                                                                  \
3961  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3962      InstImpl<TraitsType>::InstX86Minss::Base::Emitter = {                    \
3963          &InstImpl<TraitsType>::Assembler::minss,                             \
3964          &InstImpl<TraitsType>::Assembler::minss};                            \
3965  template <>                                                                  \
3966  template <>                                                                  \
3967  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3968      InstImpl<TraitsType>::InstX86Maxps::Base::Emitter = {                    \
3969          &InstImpl<TraitsType>::Assembler::maxps,                             \
3970          &InstImpl<TraitsType>::Assembler::maxps};                            \
3971  template <>                                                                  \
3972  template <>                                                                  \
3973  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3974      InstImpl<TraitsType>::InstX86Minps::Base::Emitter = {                    \
3975          &InstImpl<TraitsType>::Assembler::minps,                             \
3976          &InstImpl<TraitsType>::Assembler::minps};                            \
3977  template <>                                                                  \
3978  template <>                                                                  \
3979  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3980      InstImpl<TraitsType>::InstX86Orps::Base::Emitter = {                     \
3981          &InstImpl<TraitsType>::Assembler::orps,                              \
3982          &InstImpl<TraitsType>::Assembler::orps};                             \
3983  template <>                                                                  \
3984  template <>                                                                  \
3985  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
3986      InstImpl<TraitsType>::InstX86Xorps::Base::Emitter = {                    \
3987          &InstImpl<TraitsType>::Assembler::xorps,                             \
3988          &InstImpl<TraitsType>::Assembler::xorps};                            \
3989                                                                               \
3990  /* Binary XMM Shift ops */                                                   \
3991  template <>                                                                  \
3992  template <>                                                                  \
3993  const InstImpl<TraitsType>::Assembler::XmmEmitterShiftOp                     \
3994      InstImpl<TraitsType>::InstX86Psll::Base::Emitter = {                     \
3995          &InstImpl<TraitsType>::Assembler::psll,                              \
3996          &InstImpl<TraitsType>::Assembler::psll,                              \
3997          &InstImpl<TraitsType>::Assembler::psll};                             \
3998  template <>                                                                  \
3999  template <>                                                                  \
4000  const InstImpl<TraitsType>::Assembler::XmmEmitterShiftOp                     \
4001      InstImpl<TraitsType>::InstX86Psra::Base::Emitter = {                     \
4002          &InstImpl<TraitsType>::Assembler::psra,                              \
4003          &InstImpl<TraitsType>::Assembler::psra,                              \
4004          &InstImpl<TraitsType>::Assembler::psra};                             \
4005  template <>                                                                  \
4006  template <>                                                                  \
4007  const InstImpl<TraitsType>::Assembler::XmmEmitterShiftOp                     \
4008      InstImpl<TraitsType>::InstX86Psrl::Base::Emitter = {                     \
4009          &InstImpl<TraitsType>::Assembler::psrl,                              \
4010          &InstImpl<TraitsType>::Assembler::psrl,                              \
4011          &InstImpl<TraitsType>::Assembler::psrl};                             \
4012  template <>                                                                  \
4013  template <>                                                                  \
4014  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
4015      InstImpl<TraitsType>::InstX86Pshufb::Base::Emitter = {                   \
4016          &InstImpl<TraitsType>::Assembler::pshufb,                            \
4017          &InstImpl<TraitsType>::Assembler::pshufb};                           \
4018  template <>                                                                  \
4019  template <>                                                                  \
4020  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
4021      InstImpl<TraitsType>::InstX86Punpckl::Base::Emitter = {                  \
4022          &InstImpl<TraitsType>::Assembler::punpckl,                           \
4023          &InstImpl<TraitsType>::Assembler::punpckl};                          \
4024  template <>                                                                  \
4025  template <>                                                                  \
4026  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
4027      InstImpl<TraitsType>::InstX86Punpckh::Base::Emitter = {                  \
4028          &InstImpl<TraitsType>::Assembler::punpckh,                           \
4029          &InstImpl<TraitsType>::Assembler::punpckh};                          \
4030  template <>                                                                  \
4031  template <>                                                                  \
4032  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
4033      InstImpl<TraitsType>::InstX86Packss::Base::Emitter = {                   \
4034          &InstImpl<TraitsType>::Assembler::packss,                            \
4035          &InstImpl<TraitsType>::Assembler::packss};                           \
4036  template <>                                                                  \
4037  template <>                                                                  \
4038  const InstImpl<TraitsType>::Assembler::XmmEmitterRegOp                       \
4039      InstImpl<TraitsType>::InstX86Packus::Base::Emitter = {                   \
4040          &InstImpl<TraitsType>::Assembler::packus,                            \
4041          &InstImpl<TraitsType>::Assembler::packus};                           \
4042  }                                                                            \
4043  }
4044
4045} // end of namespace X86NAMESPACE
4046} // end of namespace Ice
4047
4048#include "IceInstX86BaseImpl.h"
4049
4050#endif // SUBZERO_SRC_ICEINSTX86BASE_H
4051