1a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li/* 2a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * Dynamic function tracing support. 3a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * 4a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * Copyright (C) 2008 Shaohua Li <shaohua.li@intel.com> 5a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * 6a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * For licencing details, see COPYING. 7a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * 8a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * Defines low-level handling of mcount calls when the kernel 9a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * is compiled with the -pg flag. When using dynamic ftrace, the 10a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * mcount call-sites get patched lazily with NOP till they are 11a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * enabled. All code mutation routines here take effect atomically. 12a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li */ 13a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 14a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li#include <linux/uaccess.h> 15a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li#include <linux/ftrace.h> 16a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 17a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li#include <asm/cacheflush.h> 18a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li#include <asm/patch.h> 19a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 20a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li/* In IA64, each function will be added below two bundles with -pg option */ 21a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listatic unsigned char __attribute__((aligned(8))) 22a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Liftrace_orig_code[MCOUNT_INSN_SIZE] = { 23a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x02, 0x40, 0x31, 0x10, 0x80, 0x05, /* alloc r40=ar.pfs,12,8,0 */ 24a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0xb0, 0x02, 0x00, 0x00, 0x42, 0x40, /* mov r43=r0;; */ 25a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x05, 0x00, 0xc4, 0x00, /* mov r42=b0 */ 26a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x11, 0x48, 0x01, 0x02, 0x00, 0x21, /* mov r41=r1 */ 27a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /* nop.i 0x0 */ 28a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x08, 0x00, 0x00, 0x50 /* br.call.sptk.many b0 = _mcount;; */ 29a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li}; 30a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 31a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listruct ftrace_orig_insn { 32a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy1, dummy2, dummy3; 33a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy4:64-41+13; 34a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 imm20:20; 35a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy5:3; 36a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 sign:1; 37a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy6:4; 38a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li}; 39a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 40a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li/* mcount stub will be converted below for nop */ 41a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listatic unsigned char ftrace_nop_code[MCOUNT_INSN_SIZE] = { 42a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MII] nop.m 0x0 */ 43a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x30, 0x00, 0x00, 0x60, 0x00, 0x00, /* mov r3=ip */ 44a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x00, 0x00, 0x04, 0x00, /* nop.i 0x0 */ 45a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0x0 */ 46a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* nop.x 0x0;; */ 47a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x00, 0x00, 0x04, 0x00 48a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li}; 49a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 50a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listatic unsigned char *ftrace_nop_replace(void) 51a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li{ 52a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return ftrace_nop_code; 53a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li} 54a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 55a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li/* 56a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * mcount stub will be converted below for call 57a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * Note: Just the last instruction is changed against nop 58a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * */ 59a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listatic unsigned char __attribute__((aligned(8))) 60a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Liftrace_call_code[MCOUNT_INSN_SIZE] = { 61a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MII] nop.m 0x0 */ 62a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x30, 0x00, 0x00, 0x60, 0x00, 0x00, /* mov r3=ip */ 63a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x00, 0x00, 0x04, 0x00, /* nop.i 0x0 */ 64a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0x0 */ 65a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, /* brl.many .;;*/ 66a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 0xf8, 0xff, 0xff, 0xc8 67a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li}; 68a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 69a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listruct ftrace_call_insn { 70a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy1, dummy2; 71a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy3:48; 72a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 imm39_l:16; 73a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 imm39_h:23; 74a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy4:13; 75a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 imm20:20; 76a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy5:3; 77a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 i:1; 78a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li u64 dummy6:4; 79a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li}; 80a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 81a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listatic unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) 82a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li{ 83a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li struct ftrace_call_insn *code = (void *)ftrace_call_code; 84a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned long offset = addr - (ip + 0x10); 85a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 86a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li code->imm39_l = offset >> 24; 87a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li code->imm39_h = offset >> 40; 88a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li code->imm20 = offset >> 4; 89a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li code->i = offset >> 63; 90a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return ftrace_call_code; 91a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li} 92a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 93a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listatic int 94a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Liftrace_modify_code(unsigned long ip, unsigned char *old_code, 95a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned char *new_code, int do_check) 96a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li{ 97a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned char replaced[MCOUNT_INSN_SIZE]; 98a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 99a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li /* 100a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * Note: Due to modules and __init, code can 101a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * disappear and change, we need to protect against faulting 102a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * as well as code changing. We do this by using the 103a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * probe_kernel_* functions. 104a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * 105a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * No real locking needed, this code is run through 106a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li * kstop_machine, or before SMP starts. 107a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li */ 108a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 109a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (!do_check) 110a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li goto skip_check; 111a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 112a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li /* read the text we want to modify */ 113a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE)) 114a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return -EFAULT; 115a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 116a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li /* Make sure it is what we expect it to be */ 117a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0) 118a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return -EINVAL; 119a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 120a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Liskip_check: 121a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li /* replace the text with the new text */ 122a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (probe_kernel_write(((void *)ip), new_code, MCOUNT_INSN_SIZE)) 123a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return -EPERM; 124a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li flush_icache_range(ip, ip + MCOUNT_INSN_SIZE); 125a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 126a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return 0; 127a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li} 128a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 129a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Listatic int ftrace_make_nop_check(struct dyn_ftrace *rec, unsigned long addr) 130a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li{ 131a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned char __attribute__((aligned(8))) replaced[MCOUNT_INSN_SIZE]; 132a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned long ip = rec->ip; 133a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 134a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE)) 135a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return -EFAULT; 136a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (rec->flags & FTRACE_FL_CONVERTED) { 137a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li struct ftrace_call_insn *call_insn, *tmp_call; 138a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 139a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li call_insn = (void *)ftrace_call_code; 140a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li tmp_call = (void *)replaced; 141a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li call_insn->imm39_l = tmp_call->imm39_l; 142a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li call_insn->imm39_h = tmp_call->imm39_h; 143a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li call_insn->imm20 = tmp_call->imm20; 144a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li call_insn->i = tmp_call->i; 145a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (memcmp(replaced, ftrace_call_code, MCOUNT_INSN_SIZE) != 0) 146a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return -EINVAL; 147a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return 0; 148a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li } else { 149a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li struct ftrace_orig_insn *call_insn, *tmp_call; 150a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 151a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li call_insn = (void *)ftrace_orig_code; 152a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li tmp_call = (void *)replaced; 153a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li call_insn->sign = tmp_call->sign; 154a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li call_insn->imm20 = tmp_call->imm20; 155a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (memcmp(replaced, ftrace_orig_code, MCOUNT_INSN_SIZE) != 0) 156a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return -EINVAL; 157a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return 0; 158a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li } 159a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li} 160a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 161a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Liint ftrace_make_nop(struct module *mod, 162a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li struct dyn_ftrace *rec, unsigned long addr) 163a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li{ 164a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li int ret; 165a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li char *new; 166a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 167a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li ret = ftrace_make_nop_check(rec, addr); 168a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (ret) 169a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return ret; 170a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li new = ftrace_nop_replace(); 171a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return ftrace_modify_code(rec->ip, NULL, new, 0); 172a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li} 173a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 174a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Liint ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 175a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li{ 176a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned long ip = rec->ip; 177a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned char *old, *new; 178a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 179a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li old= ftrace_nop_replace(); 180a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li new = ftrace_call_replace(ip, addr); 181a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return ftrace_modify_code(ip, old, new, 1); 182a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li} 183a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 184a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li/* in IA64, _mcount can't directly call ftrace_stub. Only jump is ok */ 185a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Liint ftrace_update_ftrace_func(ftrace_func_t func) 186a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li{ 187a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned long ip; 188a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li unsigned long addr = ((struct fnptr *)ftrace_call)->ip; 189a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 190a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li if (func == ftrace_stub) 191a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return 0; 192a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li ip = ((struct fnptr *)func)->ip; 193a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 194a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li ia64_patch_imm64(addr + 2, ip); 195a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 196a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li flush_icache_range(addr, addr + 16); 197a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return 0; 198a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li} 199a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li 200a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li/* run from kstop_machine */ 2013a36cb11ca65cd6804972eaf1000378ba4384ea7Jiri Slabyint __init ftrace_dyn_arch_init(void) 202a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li{ 203a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li return 0; 204a14a07b8018b714e03a39ff2180c66e307ef4238Shaohua Li} 205