1//===- subzero/src/IceTargetLoweringX8664.cpp - x86-64 lowering -----------===//
2//
3//                        The Subzero Code Generator
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// \file
11/// \brief Implements the TargetLoweringX8664 class, which consists almost
12/// entirely of the lowering sequence for each high-level instruction.
13///
14//===----------------------------------------------------------------------===//
15#include "IceTargetLoweringX8664.h"
16
17#include "IceDefs.h"
18#include "IceTargetLoweringX8664Traits.h"
19
20namespace X8664 {
21std::unique_ptr<::Ice::TargetLowering> createTargetLowering(::Ice::Cfg *Func) {
22  return ::Ice::X8664::TargetX8664::create(Func);
23}
24
25std::unique_ptr<::Ice::TargetDataLowering>
26createTargetDataLowering(::Ice::GlobalContext *Ctx) {
27  return ::Ice::X8664::TargetDataX86<::Ice::X8664::TargetX8664Traits>::create(
28      Ctx);
29}
30
31std::unique_ptr<::Ice::TargetHeaderLowering>
32createTargetHeaderLowering(::Ice::GlobalContext *Ctx) {
33  return ::Ice::X8664::TargetHeaderX86::create(Ctx);
34}
35
36void staticInit(::Ice::GlobalContext *Ctx) {
37  ::Ice::X8664::TargetX8664::staticInit(Ctx);
38}
39
40bool shouldBePooled(const class ::Ice::Constant *C) {
41  return ::Ice::X8664::TargetX8664::shouldBePooled(C);
42}
43
44::Ice::Type getPointerType() {
45  return ::Ice::X8664::TargetX8664::getPointerType();
46}
47
48} // end of namespace X8664
49
50namespace Ice {
51namespace X8664 {
52
53//------------------------------------------------------------------------------
54//      ______   ______     ______     __     ______   ______
55//     /\__  _\ /\  == \   /\  __ \   /\ \   /\__  _\ /\  ___\
56//     \/_/\ \/ \ \  __<   \ \  __ \  \ \ \  \/_/\ \/ \ \___  \
57//        \ \_\  \ \_\ \_\  \ \_\ \_\  \ \_\    \ \_\  \/\_____\
58//         \/_/   \/_/ /_/   \/_/\/_/   \/_/     \/_/   \/_____/
59//
60//------------------------------------------------------------------------------
61const TargetX8664Traits::TableFcmpType TargetX8664Traits::TableFcmp[] = {
62#define X(val, dflt, swapS, C1, C2, swapV, pred)                               \
63  {                                                                            \
64    dflt, swapS, X8664::Traits::Cond::C1, X8664::Traits::Cond::C2, swapV,      \
65        X8664::Traits::Cond::pred                                              \
66  }                                                                            \
67  ,
68    FCMPX8664_TABLE
69#undef X
70};
71
72const size_t TargetX8664Traits::TableFcmpSize = llvm::array_lengthof(TableFcmp);
73
74const TargetX8664Traits::TableIcmp32Type TargetX8664Traits::TableIcmp32[] = {
75#define X(val, C_32, C1_64, C2_64, C3_64)                                      \
76  { X8664::Traits::Cond::C_32 }                                                \
77  ,
78    ICMPX8664_TABLE
79#undef X
80};
81
82const size_t TargetX8664Traits::TableIcmp32Size =
83    llvm::array_lengthof(TableIcmp32);
84
85const TargetX8664Traits::TableIcmp64Type TargetX8664Traits::TableIcmp64[] = {
86#define X(val, C_32, C1_64, C2_64, C3_64)                                      \
87  {                                                                            \
88    X8664::Traits::Cond::C1_64, X8664::Traits::Cond::C2_64,                    \
89        X8664::Traits::Cond::C3_64                                             \
90  }                                                                            \
91  ,
92    ICMPX8664_TABLE
93#undef X
94};
95
96const size_t TargetX8664Traits::TableIcmp64Size =
97    llvm::array_lengthof(TableIcmp64);
98
99const TargetX8664Traits::TableTypeX8664AttributesType
100    TargetX8664Traits::TableTypeX8664Attributes[] = {
101#define X(tag, elty, cvt, sdss, pdps, spsd, int_, unpack, pack, width, fld)    \
102  { IceType_##elty }                                                           \
103  ,
104        ICETYPEX8664_TABLE
105#undef X
106};
107
108const size_t TargetX8664Traits::TableTypeX8664AttributesSize =
109    llvm::array_lengthof(TableTypeX8664Attributes);
110
111const uint32_t TargetX8664Traits::X86_STACK_ALIGNMENT_BYTES = 16;
112const char *TargetX8664Traits::TargetName = "X8664";
113
114template <>
115std::array<SmallBitVector, RCX86_NUM>
116    TargetX86Base<X8664::Traits>::TypeToRegisterSet = {{}};
117
118template <>
119std::array<SmallBitVector, RCX86_NUM>
120    TargetX86Base<X8664::Traits>::TypeToRegisterSetUnfiltered = {{}};
121
122template <>
123std::array<SmallBitVector,
124           TargetX86Base<X8664::Traits>::Traits::RegisterSet::Reg_NUM>
125    TargetX86Base<X8664::Traits>::RegisterAliases = {{}};
126
127template <>
128FixupKind TargetX86Base<X8664::Traits>::PcRelFixup =
129    TargetX86Base<X8664::Traits>::Traits::FK_PcRel;
130
131template <>
132FixupKind TargetX86Base<X8664::Traits>::AbsFixup =
133    TargetX86Base<X8664::Traits>::Traits::FK_Abs;
134
135//------------------------------------------------------------------------------
136//     __      ______  __     __  ______  ______  __  __   __  ______
137//    /\ \    /\  __ \/\ \  _ \ \/\  ___\/\  == \/\ \/\ "-.\ \/\  ___\
138//    \ \ \___\ \ \/\ \ \ \/ ".\ \ \  __\\ \  __<\ \ \ \ \-.  \ \ \__ \
139//     \ \_____\ \_____\ \__/".~\_\ \_____\ \_\ \_\ \_\ \_\\"\_\ \_____\
140//      \/_____/\/_____/\/_/   \/_/\/_____/\/_/ /_/\/_/\/_/ \/_/\/_____/
141//
142//------------------------------------------------------------------------------
143void TargetX8664::_add_sp(Operand *Adjustment) {
144  Variable *rsp =
145      getPhysicalRegister(Traits::RegisterSet::Reg_rsp, IceType_i64);
146  if (!NeedSandboxing) {
147    _add(rsp, Adjustment);
148    return;
149  }
150
151  Variable *esp =
152      getPhysicalRegister(Traits::RegisterSet::Reg_esp, IceType_i32);
153  Variable *r15 =
154      getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
155
156  // When incrementing rsp, NaCl sandboxing requires the following sequence
157  //
158  // .bundle_start
159  // add Adjustment, %esp
160  // add %r15, %rsp
161  // .bundle_end
162  //
163  // In Subzero, even though rsp and esp alias each other, defining one does not
164  // define the other. Therefore, we must emit
165  //
166  // .bundle_start
167  // %esp = fake-def %rsp
168  // add Adjustment, %esp
169  // %rsp = fake-def %esp
170  // add %r15, %rsp
171  // .bundle_end
172  //
173  // The fake-defs ensure that the
174  //
175  // add Adjustment, %esp
176  //
177  // instruction is not DCE'd.
178  AutoBundle _(this);
179  _redefined(Context.insert<InstFakeDef>(esp, rsp));
180  _add(esp, Adjustment);
181  _redefined(Context.insert<InstFakeDef>(rsp, esp));
182  _add(rsp, r15);
183}
184
185void TargetX8664::_mov_sp(Operand *NewValue) {
186  assert(NewValue->getType() == IceType_i32);
187
188  Variable *esp = getPhysicalRegister(Traits::RegisterSet::Reg_esp);
189  Variable *rsp =
190      getPhysicalRegister(Traits::RegisterSet::Reg_rsp, IceType_i64);
191
192  AutoBundle _(this);
193
194  _redefined(Context.insert<InstFakeDef>(esp, rsp));
195  _redefined(_mov(esp, NewValue));
196  _redefined(Context.insert<InstFakeDef>(rsp, esp));
197
198  if (!NeedSandboxing) {
199    return;
200  }
201
202  Variable *r15 =
203      getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
204  _add(rsp, r15);
205}
206
207void TargetX8664::_push_rbp() {
208  assert(NeedSandboxing);
209
210  Constant *_0 = Ctx->getConstantZero(IceType_i32);
211  Variable *ebp =
212      getPhysicalRegister(Traits::RegisterSet::Reg_ebp, IceType_i32);
213  Variable *rsp =
214      getPhysicalRegister(Traits::RegisterSet::Reg_rsp, IceType_i64);
215  auto *TopOfStack = llvm::cast<X86OperandMem>(
216      legalize(X86OperandMem::create(Func, IceType_i32, rsp, _0),
217               Legal_Reg | Legal_Mem));
218
219  // Emits a sequence:
220  //
221  //   .bundle_start
222  //   push 0
223  //   mov %ebp, %(rsp)
224  //   .bundle_end
225  //
226  // to avoid leaking the upper 32-bits (i.e., the sandbox address.)
227  AutoBundle _(this);
228  _push(_0);
229  Context.insert<typename Traits::Insts::Store>(ebp, TopOfStack);
230}
231
232void TargetX8664::_link_bp() {
233  Variable *esp =
234      getPhysicalRegister(Traits::RegisterSet::Reg_esp, IceType_i32);
235  Variable *rsp =
236      getPhysicalRegister(Traits::RegisterSet::Reg_rsp, Traits::WordType);
237  Variable *ebp =
238      getPhysicalRegister(Traits::RegisterSet::Reg_ebp, IceType_i32);
239  Variable *rbp =
240      getPhysicalRegister(Traits::RegisterSet::Reg_rbp, Traits::WordType);
241  Variable *r15 =
242      getPhysicalRegister(Traits::RegisterSet::Reg_r15, Traits::WordType);
243
244  if (!NeedSandboxing) {
245    _push(rbp);
246    _mov(rbp, rsp);
247  } else {
248    _push_rbp();
249
250    AutoBundle _(this);
251    _redefined(Context.insert<InstFakeDef>(ebp, rbp));
252    _redefined(Context.insert<InstFakeDef>(esp, rsp));
253    _mov(ebp, esp);
254    _redefined(Context.insert<InstFakeDef>(rsp, esp));
255    _add(rbp, r15);
256  }
257  // Keep ebp live for late-stage liveness analysis (e.g. asm-verbose mode).
258  Context.insert<InstFakeUse>(rbp);
259}
260
261void TargetX8664::_unlink_bp() {
262  Variable *rsp =
263      getPhysicalRegister(Traits::RegisterSet::Reg_rsp, IceType_i64);
264  Variable *rbp =
265      getPhysicalRegister(Traits::RegisterSet::Reg_rbp, IceType_i64);
266  Variable *ebp =
267      getPhysicalRegister(Traits::RegisterSet::Reg_ebp, IceType_i32);
268  // For late-stage liveness analysis (e.g. asm-verbose mode), adding a fake
269  // use of rsp before the assignment of rsp=rbp keeps previous rsp
270  // adjustments from being dead-code eliminated.
271  Context.insert<InstFakeUse>(rsp);
272  if (!NeedSandboxing) {
273    _mov(rsp, rbp);
274    _pop(rbp);
275  } else {
276    _mov_sp(ebp);
277
278    Variable *r15 =
279        getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
280    Variable *rcx =
281        getPhysicalRegister(Traits::RegisterSet::Reg_rcx, IceType_i64);
282    Variable *ecx =
283        getPhysicalRegister(Traits::RegisterSet::Reg_ecx, IceType_i32);
284
285    _pop(rcx);
286    Context.insert<InstFakeDef>(ecx, rcx);
287    AutoBundle _(this);
288    _mov(ebp, ecx);
289
290    _redefined(Context.insert<InstFakeDef>(rbp, ebp));
291    _add(rbp, r15);
292  }
293}
294
295void TargetX8664::_push_reg(Variable *Reg) {
296  Variable *rbp =
297      getPhysicalRegister(Traits::RegisterSet::Reg_rbp, Traits::WordType);
298  if (Reg != rbp || !NeedSandboxing) {
299    _push(Reg);
300  } else {
301    _push_rbp();
302  }
303}
304
305void TargetX8664::emitGetIP(CfgNode *Node) {
306  // No IP base register is needed on X86-64.
307  (void)Node;
308}
309
310namespace {
311bool isAssignedToRspOrRbp(const Variable *Var) {
312  if (Var == nullptr) {
313    return false;
314  }
315
316  if (Var->isRematerializable()) {
317    return true;
318  }
319
320  if (!Var->hasReg()) {
321    return false;
322  }
323
324  const auto RegNum = Var->getRegNum();
325  if ((RegNum == Traits::RegisterSet::Reg_rsp) ||
326      (RegNum == Traits::RegisterSet::Reg_rbp)) {
327    return true;
328  }
329
330  return false;
331}
332} // end of anonymous namespace
333
334Traits::X86OperandMem *TargetX8664::_sandbox_mem_reference(X86OperandMem *Mem) {
335  if (SandboxingType == ST_None) {
336    return Mem;
337  }
338
339  if (SandboxingType == ST_Nonsfi) {
340    llvm::report_fatal_error(
341        "_sandbox_mem_reference not implemented for nonsfi");
342  }
343
344  // In x86_64-nacl, all memory references are relative to a base register
345  // (%r15, %rsp, %rbp, or %rip).
346
347  Variable *Base = Mem->getBase();
348  Variable *Index = Mem->getIndex();
349  uint16_t Shift = 0;
350  Variable *ZeroReg = RebasePtr;
351  Constant *Offset = Mem->getOffset();
352  Variable *T = nullptr;
353
354  bool AbsoluteAddress = false;
355  if (Base == nullptr && Index == nullptr) {
356    if (llvm::isa<ConstantRelocatable>(Offset)) {
357      // Mem is RIP-relative. There's no need to rebase it.
358      return Mem;
359    }
360    // Offset is an absolute address, so we need to emit
361    //   Offset(%r15)
362    AbsoluteAddress = true;
363  }
364
365  if (Mem->getIsRebased()) {
366    // If Mem.IsRebased, then we don't need to update Mem, as it's already been
367    // updated to contain a reference to one of %rsp, %rbp, or %r15.
368    // We don't return early because we still need to zero extend Index.
369    assert(ZeroReg == Base || AbsoluteAddress || isAssignedToRspOrRbp(Base));
370    if (!AbsoluteAddress) {
371      // If Mem is an absolute address, no need to update ZeroReg (which is
372      // already set to %r15.)
373      ZeroReg = Base;
374    }
375    if (Index != nullptr) {
376      T = makeReg(IceType_i32);
377      _mov(T, Index);
378      Shift = Mem->getShift();
379    }
380  } else {
381    if (Base != nullptr) {
382      // If Base is a valid base pointer we don't need to use the RebasePtr. By
383      // doing this we might save us the need to zero extend the memory operand.
384      if (isAssignedToRspOrRbp(Base)) {
385        ZeroReg = Base;
386      } else {
387        T = Base;
388      }
389    }
390
391    if (Index != nullptr) {
392      assert(!Index->isRematerializable());
393      // If Index is not nullptr, it is mandatory that T is a nullptr.
394      // Otherwise, the lowering generated a memory operand with two registers.
395      // Note that Base might still be non-nullptr, but it must be a valid
396      // base register.
397      if (T != nullptr) {
398        llvm::report_fatal_error("memory reference contains base and index.");
399      }
400      // If the Index is not shifted, and it is a Valid Base, and the ZeroReg is
401      // still RebasePtr, then we do ZeroReg = Index, and hopefully prevent the
402      // need to zero-extend the memory operand (which may still happen -- see
403      // NeedLea below.)
404      if (Shift == 0 && isAssignedToRspOrRbp(Index) && ZeroReg == RebasePtr) {
405        ZeroReg = Index;
406      } else {
407        T = Index;
408        Shift = Mem->getShift();
409      }
410    }
411  }
412
413  // NeedsLea is a flag indicating whether Mem needs to be materialized to a GPR
414  // prior to being used. A LEA is needed if Mem.Offset is a constant
415  // relocatable with a nonzero offset, or if Mem.Offset is a nonzero immediate;
416  // but only when the address mode contains a "user" register other than the
417  // rsp/rbp/r15 base. In both these cases, the LEA is needed to ensure the
418  // sandboxed memory operand will only use the lower 32-bits of T+Offset.
419  bool NeedsLea = false;
420  if (!Mem->getIsRebased()) {
421    bool IsOffsetZero = false;
422    if (Offset == nullptr) {
423      IsOffsetZero = true;
424    } else if (const auto *CR = llvm::dyn_cast<ConstantRelocatable>(Offset)) {
425      IsOffsetZero = (CR->getOffset() == 0);
426    } else if (const auto *Imm = llvm::dyn_cast<ConstantInteger32>(Offset)) {
427      IsOffsetZero = (Imm->getValue() == 0);
428    } else {
429      llvm::report_fatal_error("Unexpected Offset type.");
430    }
431    if (!IsOffsetZero) {
432      if (Base != nullptr && Base != ZeroReg)
433        NeedsLea = true;
434      if (Index != nullptr && Index != ZeroReg)
435        NeedsLea = true;
436    }
437  }
438
439  RegNumT RegNum, RegNum32;
440  if (T != nullptr) {
441    if (T->hasReg()) {
442      RegNum = Traits::getGprForType(IceType_i64, T->getRegNum());
443      RegNum32 = Traits::getGprForType(IceType_i32, RegNum);
444      // At this point, if T was assigned to rsp/rbp, then we would have already
445      // made this the ZeroReg.
446      assert(RegNum != Traits::RegisterSet::Reg_rsp);
447      assert(RegNum != Traits::RegisterSet::Reg_rbp);
448    }
449
450    switch (T->getType()) {
451    default:
452      llvm::report_fatal_error("Mem pointer should be a 32-bit GPR.");
453    case IceType_i64:
454      // Even though "default:" would also catch T.Type == IceType_i64, an
455      // explicit 'case IceType_i64' shows that memory operands are always
456      // supposed to be 32-bits.
457      llvm::report_fatal_error("Mem pointer should not be a 64-bit GPR.");
458    case IceType_i32: {
459      Variable *T64 = makeReg(IceType_i64, RegNum);
460      auto *Movzx = _movzx(T64, T);
461      if (!NeedsLea) {
462        // This movzx is only needed when Mem does not need to be lea'd into a
463        // temporary. If an lea is going to be emitted, then eliding this movzx
464        // is safe because the emitted lea will write a 32-bit result --
465        // implicitly zero-extended to 64-bit.
466        Movzx->setMustKeep();
467      }
468      T = T64;
469    } break;
470    }
471  }
472
473  if (NeedsLea) {
474    Variable *NewT = makeReg(IceType_i32, RegNum32);
475    Variable *Base = T;
476    Variable *Index = T;
477    static constexpr bool NotRebased = false;
478    if (Shift == 0) {
479      Index = nullptr;
480    } else {
481      Base = nullptr;
482    }
483    _lea(NewT, Traits::X86OperandMem::create(
484                   Func, Mem->getType(), Base, Offset, Index, Shift,
485                   Traits::X86OperandMem::DefaultSegment, NotRebased));
486
487    T = makeReg(IceType_i64, RegNum);
488    _movzx(T, NewT);
489    Shift = 0;
490    Offset = nullptr;
491  }
492
493  static constexpr bool IsRebased = true;
494  return Traits::X86OperandMem::create(
495      Func, Mem->getType(), ZeroReg, Offset, T, Shift,
496      Traits::X86OperandMem::DefaultSegment, IsRebased);
497}
498
499void TargetX8664::_sub_sp(Operand *Adjustment) {
500  Variable *rsp =
501      getPhysicalRegister(Traits::RegisterSet::Reg_rsp, Traits::WordType);
502
503  if (NeedSandboxing) {
504    Variable *esp =
505        getPhysicalRegister(Traits::RegisterSet::Reg_esp, IceType_i32);
506    Variable *r15 =
507        getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
508
509    // .bundle_start
510    // sub Adjustment, %esp
511    // add %r15, %rsp
512    // .bundle_end
513    AutoBundle _(this);
514    _redefined(Context.insert<InstFakeDef>(esp, rsp));
515    _sub(esp, Adjustment);
516    _redefined(Context.insert<InstFakeDef>(rsp, esp));
517    _add(rsp, r15);
518  } else {
519    _sub(rsp, Adjustment);
520  }
521
522  // Add a fake use of the stack pointer, to prevent the stack pointer adustment
523  // from being dead-code eliminated in a function that doesn't return.
524  Context.insert<InstFakeUse>(rsp);
525}
526
527void TargetX8664::initRebasePtr() {
528  switch (SandboxingType) {
529  case ST_Nonsfi:
530    // Probably no implementation is needed, but error to be safe for now.
531    llvm::report_fatal_error(
532        "initRebasePtr() is not yet implemented on x32-nonsfi.");
533  case ST_NaCl:
534    RebasePtr = getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
535    break;
536  case ST_None:
537    // nothing.
538    break;
539  }
540}
541
542void TargetX8664::initSandbox() {
543  assert(SandboxingType == ST_NaCl);
544  Context.init(Func->getEntryNode());
545  Context.setInsertPoint(Context.getCur());
546  Variable *r15 =
547      getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
548  Context.insert<InstFakeDef>(r15);
549  Context.insert<InstFakeUse>(r15);
550}
551
552namespace {
553bool isRematerializable(const Variable *Var) {
554  return Var != nullptr && Var->isRematerializable();
555}
556} // end of anonymous namespace
557
558bool TargetX8664::legalizeOptAddrForSandbox(OptAddr *Addr) {
559  if (SandboxingType == ST_Nonsfi) {
560    llvm::report_fatal_error("Nonsfi not yet implemented for x8664.");
561  }
562
563  if (isRematerializable(Addr->Base)) {
564    if (Addr->Index == RebasePtr) {
565      Addr->Index = nullptr;
566      Addr->Shift = 0;
567    }
568    return true;
569  }
570
571  if (isRematerializable(Addr->Index)) {
572    if (Addr->Base == RebasePtr) {
573      Addr->Base = nullptr;
574    }
575    return true;
576  }
577
578  assert(Addr->Base != RebasePtr && Addr->Index != RebasePtr);
579
580  if (Addr->Base == nullptr) {
581    return true;
582  }
583
584  if (Addr->Index == nullptr) {
585    return true;
586  }
587
588  return false;
589}
590
591void TargetX8664::lowerIndirectJump(Variable *JumpTarget) {
592  std::unique_ptr<AutoBundle> Bundler;
593
594  if (!NeedSandboxing) {
595    if (JumpTarget->getType() != IceType_i64) {
596      Variable *T = makeReg(IceType_i64);
597      _movzx(T, JumpTarget);
598      JumpTarget = T;
599    }
600  } else {
601    Variable *T = makeReg(IceType_i32);
602    Variable *T64 = makeReg(IceType_i64);
603    Variable *r15 =
604        getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
605
606    _mov(T, JumpTarget);
607    Bundler = makeUnique<AutoBundle>(this);
608    const SizeT BundleSize =
609        1 << Func->getAssembler<>()->getBundleAlignLog2Bytes();
610    _and(T, Ctx->getConstantInt32(~(BundleSize - 1)));
611    _movzx(T64, T);
612    _add(T64, r15);
613    JumpTarget = T64;
614  }
615
616  _jmp(JumpTarget);
617}
618
619Inst *TargetX8664::emitCallToTarget(Operand *CallTarget, Variable *ReturnReg) {
620  Inst *NewCall = nullptr;
621  auto *CallTargetR = llvm::dyn_cast<Variable>(CallTarget);
622  if (NeedSandboxing) {
623    // In NaCl sandbox, calls are replaced by a push/jmp pair:
624    //
625    //     push .after_call
626    //     jmp CallTarget
627    //     .align bundle_size
628    // after_call:
629    //
630    // In order to emit this sequence, we need a temporary label ("after_call",
631    // in this example.)
632    //
633    // The operand to push is a ConstantRelocatable. The easy way to implement
634    // this sequence is to create a ConstantRelocatable(0, "after_call"), but
635    // this ends up creating more relocations for the linker to resolve.
636    // Therefore, we create a ConstantRelocatable from the name of the function
637    // being compiled (i.e., ConstantRelocatable(after_call - Func, Func).
638    //
639    // By default, ConstantRelocatables are emitted (in textual output) as
640    //
641    //  ConstantName + Offset
642    //
643    // ReturnReloc has an offset that is only known during binary emission.
644    // Therefore, we set a custom emit string for ReturnReloc that will be
645    // used instead. In this particular case, the code will be emitted as
646    //
647    //  push .after_call
648    InstX86Label *ReturnAddress = InstX86Label::create(Func, this);
649    auto *ReturnRelocOffset = RelocOffset::create(Func->getAssembler());
650    ReturnAddress->setRelocOffset(ReturnRelocOffset);
651    constexpr RelocOffsetT NoFixedOffset = 0;
652    const std::string EmitString =
653        BuildDefs::dump() ? ReturnAddress->getLabelName().toString() : "";
654    auto *ReturnReloc = ConstantRelocatable::create(
655        Func->getAssembler(), IceType_i32,
656        RelocatableTuple(NoFixedOffset, {ReturnRelocOffset},
657                         Func->getFunctionName(), EmitString));
658    /* AutoBundle scoping */ {
659      std::unique_ptr<AutoBundle> Bundler;
660      if (CallTargetR == nullptr) {
661        Bundler = makeUnique<AutoBundle>(this, InstBundleLock::Opt_PadToEnd);
662        _push(ReturnReloc);
663      } else {
664        Variable *T = makeReg(IceType_i32);
665        Variable *T64 = makeReg(IceType_i64);
666        Variable *r15 =
667            getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
668
669        _mov(T, CallTargetR);
670        Bundler = makeUnique<AutoBundle>(this, InstBundleLock::Opt_PadToEnd);
671        _push(ReturnReloc);
672        const SizeT BundleSize =
673            1 << Func->getAssembler<>()->getBundleAlignLog2Bytes();
674        _and(T, Ctx->getConstantInt32(~(BundleSize - 1)));
675        _movzx(T64, T);
676        _add(T64, r15);
677        CallTarget = T64;
678      }
679
680      NewCall = Context.insert<Traits::Insts::Jmp>(CallTarget);
681    }
682    if (ReturnReg != nullptr) {
683      Context.insert<InstFakeDef>(ReturnReg);
684    }
685
686    Context.insert(ReturnAddress);
687  } else {
688    if (CallTargetR != nullptr) {
689      // x86-64 in Subzero is ILP32. Therefore, CallTarget is i32, but the
690      // emitted call needs a i64 register (for textual asm.)
691      Variable *T = makeReg(IceType_i64);
692      _movzx(T, CallTargetR);
693      CallTarget = T;
694    }
695    NewCall = Context.insert<Traits::Insts::Call>(ReturnReg, CallTarget);
696  }
697  return NewCall;
698}
699
700Variable *TargetX8664::moveReturnValueToRegister(Operand *Value,
701                                                 Type ReturnType) {
702  if (isVectorType(ReturnType) || isScalarFloatingType(ReturnType)) {
703    return legalizeToReg(Value, Traits::RegisterSet::Reg_xmm0);
704  } else {
705    assert(ReturnType == IceType_i32 || ReturnType == IceType_i64);
706    Variable *Reg = nullptr;
707    _mov(Reg, Value,
708         Traits::getGprForType(ReturnType, Traits::RegisterSet::Reg_rax));
709    return Reg;
710  }
711}
712
713void TargetX8664::emitSandboxedReturn() {
714  Variable *T_rcx = makeReg(IceType_i64, Traits::RegisterSet::Reg_rcx);
715  Variable *T_ecx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ecx);
716  _pop(T_rcx);
717  _mov(T_ecx, T_rcx);
718  // lowerIndirectJump(T_ecx);
719  Variable *r15 =
720      getPhysicalRegister(Traits::RegisterSet::Reg_r15, IceType_i64);
721
722  /* AutoBundle scoping */ {
723    AutoBundle _(this);
724    const SizeT BundleSize =
725        1 << Func->getAssembler<>()->getBundleAlignLog2Bytes();
726    _and(T_ecx, Ctx->getConstantInt32(~(BundleSize - 1)));
727    Context.insert<InstFakeDef>(T_rcx, T_ecx);
728    _add(T_rcx, r15);
729
730    _jmp(T_rcx);
731  }
732}
733
734// In some cases, there are x-macros tables for both high-level and low-level
735// instructions/operands that use the same enum key value. The tables are kept
736// separate to maintain a proper separation between abstraction layers. There
737// is a risk that the tables could get out of sync if enum values are reordered
738// or if entries are added or deleted. The following dummy namespaces use
739// static_asserts to ensure everything is kept in sync.
740
741namespace {
742// Validate the enum values in FCMPX8664_TABLE.
743namespace dummy1 {
744// Define a temporary set of enum values based on low-level table entries.
745enum _tmp_enum {
746#define X(val, dflt, swapS, C1, C2, swapV, pred) _tmp_##val,
747  FCMPX8664_TABLE
748#undef X
749      _num
750};
751// Define a set of constants based on high-level table entries.
752#define X(tag, str) static const int _table1_##tag = InstFcmp::tag;
753ICEINSTFCMP_TABLE
754#undef X
755// Define a set of constants based on low-level table entries, and ensure the
756// table entry keys are consistent.
757#define X(val, dflt, swapS, C1, C2, swapV, pred)                               \
758  static const int _table2_##val = _tmp_##val;                                 \
759  static_assert(                                                               \
760      _table1_##val == _table2_##val,                                          \
761      "Inconsistency between FCMPX8664_TABLE and ICEINSTFCMP_TABLE");
762FCMPX8664_TABLE
763#undef X
764// Repeat the static asserts with respect to the high-level table entries in
765// case the high-level table has extra entries.
766#define X(tag, str)                                                            \
767  static_assert(                                                               \
768      _table1_##tag == _table2_##tag,                                          \
769      "Inconsistency between FCMPX8664_TABLE and ICEINSTFCMP_TABLE");
770ICEINSTFCMP_TABLE
771#undef X
772} // end of namespace dummy1
773
774// Validate the enum values in ICMPX8664_TABLE.
775namespace dummy2 {
776// Define a temporary set of enum values based on low-level table entries.
777enum _tmp_enum {
778#define X(val, C_32, C1_64, C2_64, C3_64) _tmp_##val,
779  ICMPX8664_TABLE
780#undef X
781      _num
782};
783// Define a set of constants based on high-level table entries.
784#define X(tag, reverse, str) static const int _table1_##tag = InstIcmp::tag;
785ICEINSTICMP_TABLE
786#undef X
787// Define a set of constants based on low-level table entries, and ensure the
788// table entry keys are consistent.
789#define X(val, C_32, C1_64, C2_64, C3_64)                                      \
790  static const int _table2_##val = _tmp_##val;                                 \
791  static_assert(                                                               \
792      _table1_##val == _table2_##val,                                          \
793      "Inconsistency between ICMPX8664_TABLE and ICEINSTICMP_TABLE");
794ICMPX8664_TABLE
795#undef X
796// Repeat the static asserts with respect to the high-level table entries in
797// case the high-level table has extra entries.
798#define X(tag, reverse, str)                                                   \
799  static_assert(                                                               \
800      _table1_##tag == _table2_##tag,                                          \
801      "Inconsistency between ICMPX8664_TABLE and ICEINSTICMP_TABLE");
802ICEINSTICMP_TABLE
803#undef X
804} // end of namespace dummy2
805
806// Validate the enum values in ICETYPEX8664_TABLE.
807namespace dummy3 {
808// Define a temporary set of enum values based on low-level table entries.
809enum _tmp_enum {
810#define X(tag, elty, cvt, sdss, pdps, spsd, int_, unpack, pack, width, fld)    \
811  _tmp_##tag,
812  ICETYPEX8664_TABLE
813#undef X
814      _num
815};
816// Define a set of constants based on high-level table entries.
817#define X(tag, sizeLog2, align, elts, elty, str, rcstr)                        \
818  static const int _table1_##tag = IceType_##tag;
819ICETYPE_TABLE
820#undef X
821// Define a set of constants based on low-level table entries, and ensure the
822// table entry keys are consistent.
823#define X(tag, elty, cvt, sdss, pdps, spsd, int_, unpack, pack, width, fld)    \
824  static const int _table2_##tag = _tmp_##tag;                                 \
825  static_assert(_table1_##tag == _table2_##tag,                                \
826                "Inconsistency between ICETYPEX8664_TABLE and ICETYPE_TABLE");
827ICETYPEX8664_TABLE
828#undef X
829// Repeat the static asserts with respect to the high-level table entries in
830// case the high-level table has extra entries.
831#define X(tag, sizeLog2, align, elts, elty, str, rcstr)                        \
832  static_assert(_table1_##tag == _table2_##tag,                                \
833                "Inconsistency between ICETYPEX8664_TABLE and ICETYPE_TABLE");
834ICETYPE_TABLE
835#undef X
836} // end of namespace dummy3
837} // end of anonymous namespace
838
839} // end of namespace X8664
840} // end of namespace Ice
841