1//===-- PPCJITInfo.cpp - Implement the JIT interfaces for the PowerPC -----===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the JIT interfaces for the 32-bit PowerPC target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "PPCJITInfo.h"
15#include "PPCRelocations.h"
16#include "PPCSubtarget.h"
17#include "llvm/IR/Function.h"
18#include "llvm/Support/Debug.h"
19#include "llvm/Support/ErrorHandling.h"
20#include "llvm/Support/Memory.h"
21#include "llvm/Support/raw_ostream.h"
22using namespace llvm;
23
24#define DEBUG_TYPE "jit"
25
26static TargetJITInfo::JITCompilerFn JITCompilerFunction;
27
28PPCJITInfo::PPCJITInfo(PPCSubtarget &STI)
29    : Subtarget(STI), is64Bit(STI.isPPC64()) {
30  useGOT = 0;
31}
32
33#define BUILD_ADDIS(RD,RS,IMM16) \
34  ((15 << 26) | ((RD) << 21) | ((RS) << 16) | ((IMM16) & 65535))
35#define BUILD_ORI(RD,RS,UIMM16) \
36  ((24 << 26) | ((RS) << 21) | ((RD) << 16) | ((UIMM16) & 65535))
37#define BUILD_ORIS(RD,RS,UIMM16) \
38  ((25 << 26) | ((RS) << 21) | ((RD) << 16) | ((UIMM16) & 65535))
39#define BUILD_RLDICR(RD,RS,SH,ME) \
40  ((30 << 26) | ((RS) << 21) | ((RD) << 16) | (((SH) & 31) << 11) | \
41   (((ME) & 63) << 6) | (1 << 2) | ((((SH) >> 5) & 1) << 1))
42#define BUILD_MTSPR(RS,SPR)      \
43  ((31 << 26) | ((RS) << 21) | ((SPR) << 16) | (467 << 1))
44#define BUILD_BCCTRx(BO,BI,LINK) \
45  ((19 << 26) | ((BO) << 21) | ((BI) << 16) | (528 << 1) | ((LINK) & 1))
46#define BUILD_B(TARGET, LINK) \
47  ((18 << 26) | (((TARGET) & 0x00FFFFFF) << 2) | ((LINK) & 1))
48
49// Pseudo-ops
50#define BUILD_LIS(RD,IMM16)    BUILD_ADDIS(RD,0,IMM16)
51#define BUILD_SLDI(RD,RS,IMM6) BUILD_RLDICR(RD,RS,IMM6,63-IMM6)
52#define BUILD_MTCTR(RS)        BUILD_MTSPR(RS,9)
53#define BUILD_BCTR(LINK)       BUILD_BCCTRx(20,0,LINK)
54
55static void EmitBranchToAt(uint64_t At, uint64_t To, bool isCall, bool is64Bit){
56  intptr_t Offset = ((intptr_t)To - (intptr_t)At) >> 2;
57  unsigned *AtI = (unsigned*)(intptr_t)At;
58
59  if (Offset >= -(1 << 23) && Offset < (1 << 23)) {   // In range?
60    AtI[0] = BUILD_B(Offset, isCall);     // b/bl target
61  } else if (!is64Bit) {
62    AtI[0] = BUILD_LIS(12, To >> 16);     // lis r12, hi16(address)
63    AtI[1] = BUILD_ORI(12, 12, To);       // ori r12, r12, lo16(address)
64    AtI[2] = BUILD_MTCTR(12);             // mtctr r12
65    AtI[3] = BUILD_BCTR(isCall);          // bctr/bctrl
66  } else {
67    AtI[0] = BUILD_LIS(12, To >> 48);      // lis r12, hi16(address)
68    AtI[1] = BUILD_ORI(12, 12, To >> 32);  // ori r12, r12, lo16(address)
69    AtI[2] = BUILD_SLDI(12, 12, 32);       // sldi r12, r12, 32
70    AtI[3] = BUILD_ORIS(12, 12, To >> 16); // oris r12, r12, hi16(address)
71    AtI[4] = BUILD_ORI(12, 12, To);        // ori r12, r12, lo16(address)
72    AtI[5] = BUILD_MTCTR(12);              // mtctr r12
73    AtI[6] = BUILD_BCTR(isCall);           // bctr/bctrl
74  }
75}
76
77extern "C" void PPC32CompilationCallback();
78extern "C" void PPC64CompilationCallback();
79
80// The first clause of the preprocessor directive looks wrong, but it is
81// necessary when compiling this code on non-PowerPC hosts.
82#if (!defined(__ppc__) && !defined(__powerpc__)) || defined(__powerpc64__) || defined(__ppc64__)
83void PPC32CompilationCallback() {
84  llvm_unreachable("This is not a 32bit PowerPC, you can't execute this!");
85}
86#elif !defined(__ELF__)
87// CompilationCallback stub - We can't use a C function with inline assembly in
88// it, because we the prolog/epilog inserted by GCC won't work for us.  Instead,
89// write our own wrapper, which does things our way, so we have complete control
90// over register saving and restoring.
91asm(
92    ".text\n"
93    ".align 2\n"
94    ".globl _PPC32CompilationCallback\n"
95"_PPC32CompilationCallback:\n"
96    // Make space for 8 ints r[3-10] and 13 doubles f[1-13] and the
97    // FIXME: need to save v[0-19] for altivec?
98    // FIXME: could shrink frame
99    // Set up a proper stack frame
100    // FIXME Layout
101    //   PowerPC32 ABI linkage    -  24 bytes
102    //                 parameters -  32 bytes
103    //   13 double registers      - 104 bytes
104    //   8 int registers          -  32 bytes
105    "mflr r0\n"
106    "stw r0,  8(r1)\n"
107    "stwu r1, -208(r1)\n"
108    // Save all int arg registers
109    "stw r10, 204(r1)\n"    "stw r9,  200(r1)\n"
110    "stw r8,  196(r1)\n"    "stw r7,  192(r1)\n"
111    "stw r6,  188(r1)\n"    "stw r5,  184(r1)\n"
112    "stw r4,  180(r1)\n"    "stw r3,  176(r1)\n"
113    // Save all call-clobbered FP regs.
114    "stfd f13, 168(r1)\n"   "stfd f12, 160(r1)\n"
115    "stfd f11, 152(r1)\n"   "stfd f10, 144(r1)\n"
116    "stfd f9,  136(r1)\n"   "stfd f8,  128(r1)\n"
117    "stfd f7,  120(r1)\n"   "stfd f6,  112(r1)\n"
118    "stfd f5,  104(r1)\n"   "stfd f4,   96(r1)\n"
119    "stfd f3,   88(r1)\n"   "stfd f2,   80(r1)\n"
120    "stfd f1,   72(r1)\n"
121    // Arguments to Compilation Callback:
122    // r3 - our lr (address of the call instruction in stub plus 4)
123    // r4 - stub's lr (address of instruction that called the stub plus 4)
124    // r5 - is64Bit - always 0.
125    "mr   r3, r0\n"
126    "lwz  r2, 208(r1)\n" // stub's frame
127    "lwz  r4, 8(r2)\n" // stub's lr
128    "li   r5, 0\n"       // 0 == 32 bit
129    "bl _LLVMPPCCompilationCallback\n"
130    "mtctr r3\n"
131    // Restore all int arg registers
132    "lwz r10, 204(r1)\n"    "lwz r9,  200(r1)\n"
133    "lwz r8,  196(r1)\n"    "lwz r7,  192(r1)\n"
134    "lwz r6,  188(r1)\n"    "lwz r5,  184(r1)\n"
135    "lwz r4,  180(r1)\n"    "lwz r3,  176(r1)\n"
136    // Restore all FP arg registers
137    "lfd f13, 168(r1)\n"    "lfd f12, 160(r1)\n"
138    "lfd f11, 152(r1)\n"    "lfd f10, 144(r1)\n"
139    "lfd f9,  136(r1)\n"    "lfd f8,  128(r1)\n"
140    "lfd f7,  120(r1)\n"    "lfd f6,  112(r1)\n"
141    "lfd f5,  104(r1)\n"    "lfd f4,   96(r1)\n"
142    "lfd f3,   88(r1)\n"    "lfd f2,   80(r1)\n"
143    "lfd f1,   72(r1)\n"
144    // Pop 3 frames off the stack and branch to target
145    "lwz  r1, 208(r1)\n"
146    "lwz  r2, 8(r1)\n"
147    "mtlr r2\n"
148    "bctr\n"
149    );
150
151#else
152// ELF PPC 32 support
153
154// CompilationCallback stub - We can't use a C function with inline assembly in
155// it, because we the prolog/epilog inserted by GCC won't work for us.  Instead,
156// write our own wrapper, which does things our way, so we have complete control
157// over register saving and restoring.
158asm(
159    ".text\n"
160    ".align 2\n"
161    ".globl PPC32CompilationCallback\n"
162"PPC32CompilationCallback:\n"
163    // Make space for 8 ints r[3-10] and 8 doubles f[1-8] and the
164    // FIXME: need to save v[0-19] for altivec?
165    // FIXME: could shrink frame
166    // Set up a proper stack frame
167    // FIXME Layout
168    //   8 double registers       -  64 bytes
169    //   8 int registers          -  32 bytes
170    "mflr 0\n"
171    "stw 0,  4(1)\n"
172    "stwu 1, -104(1)\n"
173    // Save all int arg registers
174    "stw 10, 100(1)\n"   "stw 9,  96(1)\n"
175    "stw 8,  92(1)\n"    "stw 7,  88(1)\n"
176    "stw 6,  84(1)\n"    "stw 5,  80(1)\n"
177    "stw 4,  76(1)\n"    "stw 3,  72(1)\n"
178    // Save all call-clobbered FP regs.
179    "stfd 8,  64(1)\n"
180    "stfd 7,  56(1)\n"   "stfd 6,  48(1)\n"
181    "stfd 5,  40(1)\n"   "stfd 4,  32(1)\n"
182    "stfd 3,  24(1)\n"   "stfd 2,  16(1)\n"
183    "stfd 1,  8(1)\n"
184    // Arguments to Compilation Callback:
185    // r3 - our lr (address of the call instruction in stub plus 4)
186    // r4 - stub's lr (address of instruction that called the stub plus 4)
187    // r5 - is64Bit - always 0.
188    "mr   3, 0\n"
189    "lwz  5, 104(1)\n" // stub's frame
190    "lwz  4, 4(5)\n" // stub's lr
191    "li   5, 0\n"       // 0 == 32 bit
192    "bl LLVMPPCCompilationCallback\n"
193    "mtctr 3\n"
194    // Restore all int arg registers
195    "lwz 10, 100(1)\n"   "lwz 9,  96(1)\n"
196    "lwz 8,  92(1)\n"    "lwz 7,  88(1)\n"
197    "lwz 6,  84(1)\n"    "lwz 5,  80(1)\n"
198    "lwz 4,  76(1)\n"    "lwz 3,  72(1)\n"
199    // Restore all FP arg registers
200    "lfd 8,  64(1)\n"
201    "lfd 7,  56(1)\n"    "lfd 6,  48(1)\n"
202    "lfd 5,  40(1)\n"    "lfd 4,  32(1)\n"
203    "lfd 3,  24(1)\n"    "lfd 2,  16(1)\n"
204    "lfd 1,  8(1)\n"
205    // Pop 3 frames off the stack and branch to target
206    "lwz  1, 104(1)\n"
207    "lwz  0, 4(1)\n"
208    "mtlr 0\n"
209    "bctr\n"
210    );
211#endif
212
213#if !defined(__powerpc64__) && !defined(__ppc64__)
214void PPC64CompilationCallback() {
215  llvm_unreachable("This is not a 64bit PowerPC, you can't execute this!");
216}
217#else
218#  ifdef __ELF__
219asm(
220    ".text\n"
221    ".align 2\n"
222    ".globl PPC64CompilationCallback\n"
223#if _CALL_ELF == 2
224    ".type PPC64CompilationCallback,@function\n"
225"PPC64CompilationCallback:\n"
226#else
227    ".section \".opd\",\"aw\",@progbits\n"
228    ".align 3\n"
229"PPC64CompilationCallback:\n"
230    ".quad .L.PPC64CompilationCallback,.TOC.@tocbase,0\n"
231    ".size PPC64CompilationCallback,24\n"
232    ".previous\n"
233    ".align 4\n"
234    ".type PPC64CompilationCallback,@function\n"
235".L.PPC64CompilationCallback:\n"
236#endif
237#  else
238asm(
239    ".text\n"
240    ".align 2\n"
241    ".globl _PPC64CompilationCallback\n"
242"_PPC64CompilationCallback:\n"
243#  endif
244    // Make space for 8 ints r[3-10] and 13 doubles f[1-13] and the
245    // FIXME: need to save v[0-19] for altivec?
246    // Set up a proper stack frame
247    // Layout
248    //   PowerPC64 ABI linkage    -  48 bytes
249    //                 parameters -  64 bytes
250    //   13 double registers      - 104 bytes
251    //   8 int registers          -  64 bytes
252    "mflr 0\n"
253    "std  0,  16(1)\n"
254    "stdu 1, -280(1)\n"
255    // Save all int arg registers
256    "std 10, 272(1)\n"    "std 9,  264(1)\n"
257    "std 8,  256(1)\n"    "std 7,  248(1)\n"
258    "std 6,  240(1)\n"    "std 5,  232(1)\n"
259    "std 4,  224(1)\n"    "std 3,  216(1)\n"
260    // Save all call-clobbered FP regs.
261    "stfd 13, 208(1)\n"    "stfd 12, 200(1)\n"
262    "stfd 11, 192(1)\n"    "stfd 10, 184(1)\n"
263    "stfd 9,  176(1)\n"    "stfd 8,  168(1)\n"
264    "stfd 7,  160(1)\n"    "stfd 6,  152(1)\n"
265    "stfd 5,  144(1)\n"    "stfd 4,  136(1)\n"
266    "stfd 3,  128(1)\n"    "stfd 2,  120(1)\n"
267    "stfd 1,  112(1)\n"
268    // Arguments to Compilation Callback:
269    // r3 - our lr (address of the call instruction in stub plus 4)
270    // r4 - stub's lr (address of instruction that called the stub plus 4)
271    // r5 - is64Bit - always 1.
272    "mr   3, 0\n"      // return address (still in r0)
273    "ld   5, 280(1)\n" // stub's frame
274    "ld   4, 16(5)\n"  // stub's lr
275    "li   5, 1\n"      // 1 == 64 bit
276#  ifdef __ELF__
277    "bl LLVMPPCCompilationCallback\n"
278    "nop\n"
279#  else
280    "bl _LLVMPPCCompilationCallback\n"
281#  endif
282    "mtctr 3\n"
283    // Restore all int arg registers
284    "ld 10, 272(1)\n"    "ld 9,  264(1)\n"
285    "ld 8,  256(1)\n"    "ld 7,  248(1)\n"
286    "ld 6,  240(1)\n"    "ld 5,  232(1)\n"
287    "ld 4,  224(1)\n"    "ld 3,  216(1)\n"
288    // Restore all FP arg registers
289    "lfd 13, 208(1)\n"    "lfd 12, 200(1)\n"
290    "lfd 11, 192(1)\n"    "lfd 10, 184(1)\n"
291    "lfd 9,  176(1)\n"    "lfd 8,  168(1)\n"
292    "lfd 7,  160(1)\n"    "lfd 6,  152(1)\n"
293    "lfd 5,  144(1)\n"    "lfd 4,  136(1)\n"
294    "lfd 3,  128(1)\n"    "lfd 2,  120(1)\n"
295    "lfd 1,  112(1)\n"
296    // Pop 3 frames off the stack and branch to target
297    "ld  1, 280(1)\n"
298    "ld  0, 16(1)\n"
299    "mtlr 0\n"
300    // XXX: any special TOC handling in the ELF case for JIT?
301    "bctr\n"
302    );
303#endif
304
305extern "C" {
306LLVM_LIBRARY_VISIBILITY void *
307LLVMPPCCompilationCallback(unsigned *StubCallAddrPlus4,
308                           unsigned *OrigCallAddrPlus4,
309                           bool is64Bit) {
310  // Adjust the pointer to the address of the call instruction in the stub
311  // emitted by emitFunctionStub, rather than the instruction after it.
312  unsigned *StubCallAddr = StubCallAddrPlus4 - 1;
313  unsigned *OrigCallAddr = OrigCallAddrPlus4 - 1;
314
315  void *Target = JITCompilerFunction(StubCallAddr);
316
317  // Check to see if *OrigCallAddr is a 'bl' instruction, and if we can rewrite
318  // it to branch directly to the destination.  If so, rewrite it so it does not
319  // need to go through the stub anymore.
320  unsigned OrigCallInst = *OrigCallAddr;
321  if ((OrigCallInst >> 26) == 18) {     // Direct call.
322    intptr_t Offset = ((intptr_t)Target - (intptr_t)OrigCallAddr) >> 2;
323
324    if (Offset >= -(1 << 23) && Offset < (1 << 23)) {   // In range?
325      // Clear the original target out.
326      OrigCallInst &= (63 << 26) | 3;
327      // Fill in the new target.
328      OrigCallInst |= (Offset & ((1 << 24)-1)) << 2;
329      // Replace the call.
330      *OrigCallAddr = OrigCallInst;
331    }
332  }
333
334  // Assert that we are coming from a stub that was created with our
335  // emitFunctionStub.
336  if ((*StubCallAddr >> 26) == 18)
337    StubCallAddr -= 3;
338  else {
339  assert((*StubCallAddr >> 26) == 19 && "Call in stub is not indirect!");
340    StubCallAddr -= is64Bit ? 9 : 6;
341  }
342
343  // Rewrite the stub with an unconditional branch to the target, for any users
344  // who took the address of the stub.
345  EmitBranchToAt((intptr_t)StubCallAddr, (intptr_t)Target, false, is64Bit);
346  sys::Memory::InvalidateInstructionCache(StubCallAddr, 7*4);
347
348  // Put the address of the target function to call and the address to return to
349  // after calling the target function in a place that is easy to get on the
350  // stack after we restore all regs.
351  return Target;
352}
353}
354
355
356
357TargetJITInfo::LazyResolverFn
358PPCJITInfo::getLazyResolverFunction(JITCompilerFn Fn) {
359  JITCompilerFunction = Fn;
360  return is64Bit ? PPC64CompilationCallback : PPC32CompilationCallback;
361}
362
363TargetJITInfo::StubLayout PPCJITInfo::getStubLayout() {
364  // The stub contains up to 10 4-byte instructions, aligned at 4 bytes: 3
365  // instructions to save the caller's address if this is a lazy-compilation
366  // stub, plus a 1-, 4-, or 7-instruction sequence to load an arbitrary address
367  // into a register and jump through it.
368  StubLayout Result = {10*4, 4};
369  return Result;
370}
371
372#if (defined(__POWERPC__) || defined (__ppc__) || defined(_POWER)) && \
373defined(__APPLE__)
374extern "C" void sys_icache_invalidate(const void *Addr, size_t len);
375#endif
376
377void *PPCJITInfo::emitFunctionStub(const Function* F, void *Fn,
378                                   JITCodeEmitter &JCE) {
379  // If this is just a call to an external function, emit a branch instead of a
380  // call.  The code is the same except for one bit of the last instruction.
381  if (Fn != (void*)(intptr_t)PPC32CompilationCallback &&
382      Fn != (void*)(intptr_t)PPC64CompilationCallback) {
383    void *Addr = (void*)JCE.getCurrentPCValue();
384    JCE.emitWordBE(0);
385    JCE.emitWordBE(0);
386    JCE.emitWordBE(0);
387    JCE.emitWordBE(0);
388    JCE.emitWordBE(0);
389    JCE.emitWordBE(0);
390    JCE.emitWordBE(0);
391    EmitBranchToAt((intptr_t)Addr, (intptr_t)Fn, false, is64Bit);
392    sys::Memory::InvalidateInstructionCache(Addr, 7*4);
393    return Addr;
394  }
395
396  void *Addr = (void*)JCE.getCurrentPCValue();
397  if (is64Bit) {
398    JCE.emitWordBE(0xf821ffb1);     // stdu r1,-80(r1)
399    JCE.emitWordBE(0x7d6802a6);     // mflr r11
400    JCE.emitWordBE(0xf9610060);     // std r11, 96(r1)
401  } else if (Subtarget.isDarwinABI()){
402    JCE.emitWordBE(0x9421ffe0);     // stwu r1,-32(r1)
403    JCE.emitWordBE(0x7d6802a6);     // mflr r11
404    JCE.emitWordBE(0x91610028);     // stw r11, 40(r1)
405  } else {
406    JCE.emitWordBE(0x9421ffe0);     // stwu r1,-32(r1)
407    JCE.emitWordBE(0x7d6802a6);     // mflr r11
408    JCE.emitWordBE(0x91610024);     // stw r11, 36(r1)
409  }
410  intptr_t BranchAddr = (intptr_t)JCE.getCurrentPCValue();
411  JCE.emitWordBE(0);
412  JCE.emitWordBE(0);
413  JCE.emitWordBE(0);
414  JCE.emitWordBE(0);
415  JCE.emitWordBE(0);
416  JCE.emitWordBE(0);
417  JCE.emitWordBE(0);
418  EmitBranchToAt(BranchAddr, (intptr_t)Fn, true, is64Bit);
419  sys::Memory::InvalidateInstructionCache(Addr, 10*4);
420  return Addr;
421}
422
423
424void PPCJITInfo::relocate(void *Function, MachineRelocation *MR,
425                          unsigned NumRelocs, unsigned char* GOTBase) {
426  for (unsigned i = 0; i != NumRelocs; ++i, ++MR) {
427    unsigned *RelocPos = (unsigned*)Function + MR->getMachineCodeOffset()/4;
428    intptr_t ResultPtr = (intptr_t)MR->getResultPointer();
429    switch ((PPC::RelocationType)MR->getRelocationType()) {
430    default: llvm_unreachable("Unknown relocation type!");
431    case PPC::reloc_pcrel_bx:
432      // PC-relative relocation for b and bl instructions.
433      ResultPtr = (ResultPtr-(intptr_t)RelocPos) >> 2;
434      assert(ResultPtr >= -(1 << 23) && ResultPtr < (1 << 23) &&
435             "Relocation out of range!");
436      *RelocPos |= (ResultPtr & ((1 << 24)-1))  << 2;
437      break;
438    case PPC::reloc_pcrel_bcx:
439      // PC-relative relocation for BLT,BLE,BEQ,BGE,BGT,BNE, or other
440      // bcx instructions.
441      ResultPtr = (ResultPtr-(intptr_t)RelocPos) >> 2;
442      assert(ResultPtr >= -(1 << 13) && ResultPtr < (1 << 13) &&
443             "Relocation out of range!");
444      *RelocPos |= (ResultPtr & ((1 << 14)-1))  << 2;
445      break;
446    case PPC::reloc_absolute_high:     // high bits of ref -> low 16 of instr
447    case PPC::reloc_absolute_low: {    // low bits of ref  -> low 16 of instr
448      ResultPtr += MR->getConstantVal();
449
450      // If this is a high-part access, get the high-part.
451      if (MR->getRelocationType() == PPC::reloc_absolute_high) {
452        // If the low part will have a carry (really a borrow) from the low
453        // 16-bits into the high 16, add a bit to borrow from.
454        if (((int)ResultPtr << 16) < 0)
455          ResultPtr += 1 << 16;
456        ResultPtr >>= 16;
457      }
458
459      // Do the addition then mask, so the addition does not overflow the 16-bit
460      // immediate section of the instruction.
461      unsigned LowBits  = (*RelocPos + ResultPtr) & 65535;
462      unsigned HighBits = *RelocPos & ~65535;
463      *RelocPos = LowBits | HighBits;  // Slam into low 16-bits
464      break;
465    }
466    case PPC::reloc_absolute_low_ix: {  // low bits of ref  -> low 14 of instr
467      ResultPtr += MR->getConstantVal();
468      // Do the addition then mask, so the addition does not overflow the 16-bit
469      // immediate section of the instruction.
470      unsigned LowBits  = (*RelocPos + ResultPtr) & 0xFFFC;
471      unsigned HighBits = *RelocPos & 0xFFFF0003;
472      *RelocPos = LowBits | HighBits;  // Slam into low 14-bits.
473      break;
474    }
475    }
476  }
477}
478
479void PPCJITInfo::replaceMachineCodeForFunction(void *Old, void *New) {
480  EmitBranchToAt((intptr_t)Old, (intptr_t)New, false, is64Bit);
481  sys::Memory::InvalidateInstructionCache(Old, 7*4);
482}
483