1/* 2 * Code for replacing ftrace calls with jumps. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * 6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 7 * 8 * Added function graph tracer code, taken from x86 that was written 9 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 10 * 11 */ 12 13#include <linux/spinlock.h> 14#include <linux/hardirq.h> 15#include <linux/uaccess.h> 16#include <linux/module.h> 17#include <linux/ftrace.h> 18#include <linux/percpu.h> 19#include <linux/init.h> 20#include <linux/list.h> 21 22#include <asm/cacheflush.h> 23#include <asm/code-patching.h> 24#include <asm/ftrace.h> 25#include <asm/syscall.h> 26 27 28#ifdef CONFIG_DYNAMIC_FTRACE 29static unsigned int 30ftrace_call_replace(unsigned long ip, unsigned long addr, int link) 31{ 32 unsigned int op; 33 34 addr = ppc_function_entry((void *)addr); 35 36 /* if (link) set op to 'bl' else 'b' */ 37 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0); 38 39 return op; 40} 41 42static int 43ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new) 44{ 45 unsigned int replaced; 46 47 /* 48 * Note: Due to modules and __init, code can 49 * disappear and change, we need to protect against faulting 50 * as well as code changing. We do this by using the 51 * probe_kernel_* functions. 52 * 53 * No real locking needed, this code is run through 54 * kstop_machine, or before SMP starts. 55 */ 56 57 /* read the text we want to modify */ 58 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE)) 59 return -EFAULT; 60 61 /* Make sure it is what we expect it to be */ 62 if (replaced != old) 63 return -EINVAL; 64 65 /* replace the text with the new text */ 66 if (probe_kernel_write((void *)ip, &new, MCOUNT_INSN_SIZE)) 67 return -EPERM; 68 69 flush_icache_range(ip, ip + 8); 70 71 return 0; 72} 73 74/* 75 * Helper functions that are the same for both PPC64 and PPC32. 76 */ 77static int test_24bit_addr(unsigned long ip, unsigned long addr) 78{ 79 80 /* use the create_branch to verify that this offset can be branched */ 81 return create_branch((unsigned int *)ip, addr, 0); 82} 83 84#ifdef CONFIG_MODULES 85 86static int is_bl_op(unsigned int op) 87{ 88 return (op & 0xfc000003) == 0x48000001; 89} 90 91static unsigned long find_bl_target(unsigned long ip, unsigned int op) 92{ 93 static int offset; 94 95 offset = (op & 0x03fffffc); 96 /* make it signed */ 97 if (offset & 0x02000000) 98 offset |= 0xfe000000; 99 100 return ip + (long)offset; 101} 102 103#ifdef CONFIG_PPC64 104static int 105__ftrace_make_nop(struct module *mod, 106 struct dyn_ftrace *rec, unsigned long addr) 107{ 108 unsigned int op; 109 unsigned int jmp[5]; 110 unsigned long ptr; 111 unsigned long ip = rec->ip; 112 unsigned long tramp; 113 int offset; 114 115 /* read where this goes */ 116 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) 117 return -EFAULT; 118 119 /* Make sure that that this is still a 24bit jump */ 120 if (!is_bl_op(op)) { 121 printk(KERN_ERR "Not expected bl: opcode is %x\n", op); 122 return -EINVAL; 123 } 124 125 /* lets find where the pointer goes */ 126 tramp = find_bl_target(ip, op); 127 128 /* 129 * On PPC64 the trampoline looks like: 130 * 0x3d, 0x82, 0x00, 0x00, addis r12,r2, <high> 131 * 0x39, 0x8c, 0x00, 0x00, addi r12,r12, <low> 132 * Where the bytes 2,3,6 and 7 make up the 32bit offset 133 * to the TOC that holds the pointer. 134 * to jump to. 135 * 0xf8, 0x41, 0x00, 0x28, std r2,40(r1) 136 * 0xe9, 0x6c, 0x00, 0x20, ld r11,32(r12) 137 * The actually address is 32 bytes from the offset 138 * into the TOC. 139 * 0xe8, 0x4c, 0x00, 0x28, ld r2,40(r12) 140 */ 141 142 pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc); 143 144 /* Find where the trampoline jumps to */ 145 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) { 146 printk(KERN_ERR "Failed to read %lx\n", tramp); 147 return -EFAULT; 148 } 149 150 pr_devel(" %08x %08x", jmp[0], jmp[1]); 151 152 /* verify that this is what we expect it to be */ 153 if (((jmp[0] & 0xffff0000) != 0x3d820000) || 154 ((jmp[1] & 0xffff0000) != 0x398c0000) || 155 (jmp[2] != 0xf8410028) || 156 (jmp[3] != 0xe96c0020) || 157 (jmp[4] != 0xe84c0028)) { 158 printk(KERN_ERR "Not a trampoline\n"); 159 return -EINVAL; 160 } 161 162 /* The bottom half is signed extended */ 163 offset = ((unsigned)((unsigned short)jmp[0]) << 16) + 164 (int)((short)jmp[1]); 165 166 pr_devel(" %x ", offset); 167 168 /* get the address this jumps too */ 169 tramp = mod->arch.toc + offset + 32; 170 pr_devel("toc: %lx", tramp); 171 172 if (probe_kernel_read(jmp, (void *)tramp, 8)) { 173 printk(KERN_ERR "Failed to read %lx\n", tramp); 174 return -EFAULT; 175 } 176 177 pr_devel(" %08x %08x\n", jmp[0], jmp[1]); 178 179 ptr = ((unsigned long)jmp[0] << 32) + jmp[1]; 180 181 /* This should match what was called */ 182 if (ptr != ppc_function_entry((void *)addr)) { 183 printk(KERN_ERR "addr does not match %lx\n", ptr); 184 return -EINVAL; 185 } 186 187 /* 188 * We want to nop the line, but the next line is 189 * 0xe8, 0x41, 0x00, 0x28 ld r2,40(r1) 190 * This needs to be turned to a nop too. 191 */ 192 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) 193 return -EFAULT; 194 195 if (op != 0xe8410028) { 196 printk(KERN_ERR "Next line is not ld! (%08x)\n", op); 197 return -EINVAL; 198 } 199 200 /* 201 * Milton Miller pointed out that we can not blindly do nops. 202 * If a task was preempted when calling a trace function, 203 * the nops will remove the way to restore the TOC in r2 204 * and the r2 TOC will get corrupted. 205 */ 206 207 /* 208 * Replace: 209 * bl <tramp> <==== will be replaced with "b 1f" 210 * ld r2,40(r1) 211 * 1: 212 */ 213 op = 0x48000008; /* b +8 */ 214 215 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE)) 216 return -EPERM; 217 218 219 flush_icache_range(ip, ip + 8); 220 221 return 0; 222} 223 224#else /* !PPC64 */ 225static int 226__ftrace_make_nop(struct module *mod, 227 struct dyn_ftrace *rec, unsigned long addr) 228{ 229 unsigned int op; 230 unsigned int jmp[4]; 231 unsigned long ip = rec->ip; 232 unsigned long tramp; 233 234 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 235 return -EFAULT; 236 237 /* Make sure that that this is still a 24bit jump */ 238 if (!is_bl_op(op)) { 239 printk(KERN_ERR "Not expected bl: opcode is %x\n", op); 240 return -EINVAL; 241 } 242 243 /* lets find where the pointer goes */ 244 tramp = find_bl_target(ip, op); 245 246 /* 247 * On PPC32 the trampoline looks like: 248 * 0x3d, 0x60, 0x00, 0x00 lis r11,sym@ha 249 * 0x39, 0x6b, 0x00, 0x00 addi r11,r11,sym@l 250 * 0x7d, 0x69, 0x03, 0xa6 mtctr r11 251 * 0x4e, 0x80, 0x04, 0x20 bctr 252 */ 253 254 pr_devel("ip:%lx jumps to %lx", ip, tramp); 255 256 /* Find where the trampoline jumps to */ 257 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) { 258 printk(KERN_ERR "Failed to read %lx\n", tramp); 259 return -EFAULT; 260 } 261 262 pr_devel(" %08x %08x ", jmp[0], jmp[1]); 263 264 /* verify that this is what we expect it to be */ 265 if (((jmp[0] & 0xffff0000) != 0x3d600000) || 266 ((jmp[1] & 0xffff0000) != 0x396b0000) || 267 (jmp[2] != 0x7d6903a6) || 268 (jmp[3] != 0x4e800420)) { 269 printk(KERN_ERR "Not a trampoline\n"); 270 return -EINVAL; 271 } 272 273 tramp = (jmp[1] & 0xffff) | 274 ((jmp[0] & 0xffff) << 16); 275 if (tramp & 0x8000) 276 tramp -= 0x10000; 277 278 pr_devel(" %lx ", tramp); 279 280 if (tramp != addr) { 281 printk(KERN_ERR 282 "Trampoline location %08lx does not match addr\n", 283 tramp); 284 return -EINVAL; 285 } 286 287 op = PPC_INST_NOP; 288 289 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE)) 290 return -EPERM; 291 292 flush_icache_range(ip, ip + 8); 293 294 return 0; 295} 296#endif /* PPC64 */ 297#endif /* CONFIG_MODULES */ 298 299int ftrace_make_nop(struct module *mod, 300 struct dyn_ftrace *rec, unsigned long addr) 301{ 302 unsigned long ip = rec->ip; 303 unsigned int old, new; 304 305 /* 306 * If the calling address is more that 24 bits away, 307 * then we had to use a trampoline to make the call. 308 * Otherwise just update the call site. 309 */ 310 if (test_24bit_addr(ip, addr)) { 311 /* within range */ 312 old = ftrace_call_replace(ip, addr, 1); 313 new = PPC_INST_NOP; 314 return ftrace_modify_code(ip, old, new); 315 } 316 317#ifdef CONFIG_MODULES 318 /* 319 * Out of range jumps are called from modules. 320 * We should either already have a pointer to the module 321 * or it has been passed in. 322 */ 323 if (!rec->arch.mod) { 324 if (!mod) { 325 printk(KERN_ERR "No module loaded addr=%lx\n", 326 addr); 327 return -EFAULT; 328 } 329 rec->arch.mod = mod; 330 } else if (mod) { 331 if (mod != rec->arch.mod) { 332 printk(KERN_ERR 333 "Record mod %p not equal to passed in mod %p\n", 334 rec->arch.mod, mod); 335 return -EINVAL; 336 } 337 /* nothing to do if mod == rec->arch.mod */ 338 } else 339 mod = rec->arch.mod; 340 341 return __ftrace_make_nop(mod, rec, addr); 342#else 343 /* We should not get here without modules */ 344 return -EINVAL; 345#endif /* CONFIG_MODULES */ 346} 347 348#ifdef CONFIG_MODULES 349#ifdef CONFIG_PPC64 350static int 351__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 352{ 353 unsigned int op[2]; 354 unsigned long ip = rec->ip; 355 356 /* read where this goes */ 357 if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2)) 358 return -EFAULT; 359 360 /* 361 * It should be pointing to two nops or 362 * b +8; ld r2,40(r1) 363 */ 364 if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) && 365 ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) { 366 printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]); 367 return -EINVAL; 368 } 369 370 /* If we never set up a trampoline to ftrace_caller, then bail */ 371 if (!rec->arch.mod->arch.tramp) { 372 printk(KERN_ERR "No ftrace trampoline\n"); 373 return -EINVAL; 374 } 375 376 /* create the branch to the trampoline */ 377 op[0] = create_branch((unsigned int *)ip, 378 rec->arch.mod->arch.tramp, BRANCH_SET_LINK); 379 if (!op[0]) { 380 printk(KERN_ERR "REL24 out of range!\n"); 381 return -EINVAL; 382 } 383 384 /* ld r2,40(r1) */ 385 op[1] = 0xe8410028; 386 387 pr_devel("write to %lx\n", rec->ip); 388 389 if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2)) 390 return -EPERM; 391 392 flush_icache_range(ip, ip + 8); 393 394 return 0; 395} 396#else 397static int 398__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 399{ 400 unsigned int op; 401 unsigned long ip = rec->ip; 402 403 /* read where this goes */ 404 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 405 return -EFAULT; 406 407 /* It should be pointing to a nop */ 408 if (op != PPC_INST_NOP) { 409 printk(KERN_ERR "Expected NOP but have %x\n", op); 410 return -EINVAL; 411 } 412 413 /* If we never set up a trampoline to ftrace_caller, then bail */ 414 if (!rec->arch.mod->arch.tramp) { 415 printk(KERN_ERR "No ftrace trampoline\n"); 416 return -EINVAL; 417 } 418 419 /* create the branch to the trampoline */ 420 op = create_branch((unsigned int *)ip, 421 rec->arch.mod->arch.tramp, BRANCH_SET_LINK); 422 if (!op) { 423 printk(KERN_ERR "REL24 out of range!\n"); 424 return -EINVAL; 425 } 426 427 pr_devel("write to %lx\n", rec->ip); 428 429 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE)) 430 return -EPERM; 431 432 flush_icache_range(ip, ip + 8); 433 434 return 0; 435} 436#endif /* CONFIG_PPC64 */ 437#endif /* CONFIG_MODULES */ 438 439int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 440{ 441 unsigned long ip = rec->ip; 442 unsigned int old, new; 443 444 /* 445 * If the calling address is more that 24 bits away, 446 * then we had to use a trampoline to make the call. 447 * Otherwise just update the call site. 448 */ 449 if (test_24bit_addr(ip, addr)) { 450 /* within range */ 451 old = PPC_INST_NOP; 452 new = ftrace_call_replace(ip, addr, 1); 453 return ftrace_modify_code(ip, old, new); 454 } 455 456#ifdef CONFIG_MODULES 457 /* 458 * Out of range jumps are called from modules. 459 * Being that we are converting from nop, it had better 460 * already have a module defined. 461 */ 462 if (!rec->arch.mod) { 463 printk(KERN_ERR "No module loaded\n"); 464 return -EINVAL; 465 } 466 467 return __ftrace_make_call(rec, addr); 468#else 469 /* We should not get here without modules */ 470 return -EINVAL; 471#endif /* CONFIG_MODULES */ 472} 473 474int ftrace_update_ftrace_func(ftrace_func_t func) 475{ 476 unsigned long ip = (unsigned long)(&ftrace_call); 477 unsigned int old, new; 478 int ret; 479 480 old = *(unsigned int *)&ftrace_call; 481 new = ftrace_call_replace(ip, (unsigned long)func, 1); 482 ret = ftrace_modify_code(ip, old, new); 483 484 return ret; 485} 486 487int __init ftrace_dyn_arch_init(void *data) 488{ 489 /* caller expects data to be zero */ 490 unsigned long *p = data; 491 492 *p = 0; 493 494 return 0; 495} 496#endif /* CONFIG_DYNAMIC_FTRACE */ 497 498#ifdef CONFIG_FUNCTION_GRAPH_TRACER 499 500#ifdef CONFIG_DYNAMIC_FTRACE 501extern void ftrace_graph_call(void); 502extern void ftrace_graph_stub(void); 503 504int ftrace_enable_ftrace_graph_caller(void) 505{ 506 unsigned long ip = (unsigned long)(&ftrace_graph_call); 507 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 508 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 509 unsigned int old, new; 510 511 old = ftrace_call_replace(ip, stub, 0); 512 new = ftrace_call_replace(ip, addr, 0); 513 514 return ftrace_modify_code(ip, old, new); 515} 516 517int ftrace_disable_ftrace_graph_caller(void) 518{ 519 unsigned long ip = (unsigned long)(&ftrace_graph_call); 520 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 521 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 522 unsigned int old, new; 523 524 old = ftrace_call_replace(ip, addr, 0); 525 new = ftrace_call_replace(ip, stub, 0); 526 527 return ftrace_modify_code(ip, old, new); 528} 529#endif /* CONFIG_DYNAMIC_FTRACE */ 530 531#ifdef CONFIG_PPC64 532extern void mod_return_to_handler(void); 533#endif 534 535/* 536 * Hook the return address and push it in the stack of return addrs 537 * in current thread info. 538 */ 539void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) 540{ 541 unsigned long old; 542 int faulted; 543 struct ftrace_graph_ent trace; 544 unsigned long return_hooker = (unsigned long)&return_to_handler; 545 546 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 547 return; 548 549#ifdef CONFIG_PPC64 550 /* non core kernel code needs to save and restore the TOC */ 551 if (REGION_ID(self_addr) != KERNEL_REGION_ID) 552 return_hooker = (unsigned long)&mod_return_to_handler; 553#endif 554 555 return_hooker = ppc_function_entry((void *)return_hooker); 556 557 /* 558 * Protect against fault, even if it shouldn't 559 * happen. This tool is too much intrusive to 560 * ignore such a protection. 561 */ 562 asm volatile( 563 "1: " PPC_LL "%[old], 0(%[parent])\n" 564 "2: " PPC_STL "%[return_hooker], 0(%[parent])\n" 565 " li %[faulted], 0\n" 566 "3:\n" 567 568 ".section .fixup, \"ax\"\n" 569 "4: li %[faulted], 1\n" 570 " b 3b\n" 571 ".previous\n" 572 573 ".section __ex_table,\"a\"\n" 574 PPC_LONG_ALIGN "\n" 575 PPC_LONG "1b,4b\n" 576 PPC_LONG "2b,4b\n" 577 ".previous" 578 579 : [old] "=&r" (old), [faulted] "=r" (faulted) 580 : [parent] "r" (parent), [return_hooker] "r" (return_hooker) 581 : "memory" 582 ); 583 584 if (unlikely(faulted)) { 585 ftrace_graph_stop(); 586 WARN_ON(1); 587 return; 588 } 589 590 if (ftrace_push_return_trace(old, self_addr, &trace.depth, 0) == -EBUSY) { 591 *parent = old; 592 return; 593 } 594 595 trace.func = self_addr; 596 597 /* Only trace if the calling function expects to */ 598 if (!ftrace_graph_entry(&trace)) { 599 current->curr_ret_stack--; 600 *parent = old; 601 } 602} 603#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 604 605#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64) 606unsigned long __init arch_syscall_addr(int nr) 607{ 608 return sys_call_table[nr*2]; 609} 610#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */ 611