1/*
2 * Code for replacing ftrace calls with jumps.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 *
6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
7 *
8 * Added function graph tracer code, taken from x86 that was written
9 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
10 *
11 */
12
13#include <linux/spinlock.h>
14#include <linux/hardirq.h>
15#include <linux/uaccess.h>
16#include <linux/module.h>
17#include <linux/ftrace.h>
18#include <linux/percpu.h>
19#include <linux/init.h>
20#include <linux/list.h>
21
22#include <asm/cacheflush.h>
23#include <asm/code-patching.h>
24#include <asm/ftrace.h>
25#include <asm/syscall.h>
26
27
28#ifdef CONFIG_DYNAMIC_FTRACE
29static unsigned int
30ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
31{
32	unsigned int op;
33
34	addr = ppc_function_entry((void *)addr);
35
36	/* if (link) set op to 'bl' else 'b' */
37	op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
38
39	return op;
40}
41
42static int
43ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
44{
45	unsigned int replaced;
46
47	/*
48	 * Note: Due to modules and __init, code can
49	 *  disappear and change, we need to protect against faulting
50	 *  as well as code changing. We do this by using the
51	 *  probe_kernel_* functions.
52	 *
53	 * No real locking needed, this code is run through
54	 * kstop_machine, or before SMP starts.
55	 */
56
57	/* read the text we want to modify */
58	if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
59		return -EFAULT;
60
61	/* Make sure it is what we expect it to be */
62	if (replaced != old)
63		return -EINVAL;
64
65	/* replace the text with the new text */
66	if (probe_kernel_write((void *)ip, &new, MCOUNT_INSN_SIZE))
67		return -EPERM;
68
69	flush_icache_range(ip, ip + 8);
70
71	return 0;
72}
73
74/*
75 * Helper functions that are the same for both PPC64 and PPC32.
76 */
77static int test_24bit_addr(unsigned long ip, unsigned long addr)
78{
79
80	/* use the create_branch to verify that this offset can be branched */
81	return create_branch((unsigned int *)ip, addr, 0);
82}
83
84#ifdef CONFIG_MODULES
85
86static int is_bl_op(unsigned int op)
87{
88	return (op & 0xfc000003) == 0x48000001;
89}
90
91static unsigned long find_bl_target(unsigned long ip, unsigned int op)
92{
93	static int offset;
94
95	offset = (op & 0x03fffffc);
96	/* make it signed */
97	if (offset & 0x02000000)
98		offset |= 0xfe000000;
99
100	return ip + (long)offset;
101}
102
103#ifdef CONFIG_PPC64
104static int
105__ftrace_make_nop(struct module *mod,
106		  struct dyn_ftrace *rec, unsigned long addr)
107{
108	unsigned int op;
109	unsigned int jmp[5];
110	unsigned long ptr;
111	unsigned long ip = rec->ip;
112	unsigned long tramp;
113	int offset;
114
115	/* read where this goes */
116	if (probe_kernel_read(&op, (void *)ip, sizeof(int)))
117		return -EFAULT;
118
119	/* Make sure that that this is still a 24bit jump */
120	if (!is_bl_op(op)) {
121		printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
122		return -EINVAL;
123	}
124
125	/* lets find where the pointer goes */
126	tramp = find_bl_target(ip, op);
127
128	/*
129	 * On PPC64 the trampoline looks like:
130	 * 0x3d, 0x82, 0x00, 0x00,    addis   r12,r2, <high>
131	 * 0x39, 0x8c, 0x00, 0x00,    addi    r12,r12, <low>
132	 *   Where the bytes 2,3,6 and 7 make up the 32bit offset
133	 *   to the TOC that holds the pointer.
134	 *   to jump to.
135	 * 0xf8, 0x41, 0x00, 0x28,    std     r2,40(r1)
136	 * 0xe9, 0x6c, 0x00, 0x20,    ld      r11,32(r12)
137	 *   The actually address is 32 bytes from the offset
138	 *   into the TOC.
139	 * 0xe8, 0x4c, 0x00, 0x28,    ld      r2,40(r12)
140	 */
141
142	pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
143
144	/* Find where the trampoline jumps to */
145	if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
146		printk(KERN_ERR "Failed to read %lx\n", tramp);
147		return -EFAULT;
148	}
149
150	pr_devel(" %08x %08x", jmp[0], jmp[1]);
151
152	/* verify that this is what we expect it to be */
153	if (((jmp[0] & 0xffff0000) != 0x3d820000) ||
154	    ((jmp[1] & 0xffff0000) != 0x398c0000) ||
155	    (jmp[2] != 0xf8410028) ||
156	    (jmp[3] != 0xe96c0020) ||
157	    (jmp[4] != 0xe84c0028)) {
158		printk(KERN_ERR "Not a trampoline\n");
159		return -EINVAL;
160	}
161
162	/* The bottom half is signed extended */
163	offset = ((unsigned)((unsigned short)jmp[0]) << 16) +
164		(int)((short)jmp[1]);
165
166	pr_devel(" %x ", offset);
167
168	/* get the address this jumps too */
169	tramp = mod->arch.toc + offset + 32;
170	pr_devel("toc: %lx", tramp);
171
172	if (probe_kernel_read(jmp, (void *)tramp, 8)) {
173		printk(KERN_ERR "Failed to read %lx\n", tramp);
174		return -EFAULT;
175	}
176
177	pr_devel(" %08x %08x\n", jmp[0], jmp[1]);
178
179	ptr = ((unsigned long)jmp[0] << 32) + jmp[1];
180
181	/* This should match what was called */
182	if (ptr != ppc_function_entry((void *)addr)) {
183		printk(KERN_ERR "addr does not match %lx\n", ptr);
184		return -EINVAL;
185	}
186
187	/*
188	 * We want to nop the line, but the next line is
189	 *  0xe8, 0x41, 0x00, 0x28   ld r2,40(r1)
190	 * This needs to be turned to a nop too.
191	 */
192	if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE))
193		return -EFAULT;
194
195	if (op != 0xe8410028) {
196		printk(KERN_ERR "Next line is not ld! (%08x)\n", op);
197		return -EINVAL;
198	}
199
200	/*
201	 * Milton Miller pointed out that we can not blindly do nops.
202	 * If a task was preempted when calling a trace function,
203	 * the nops will remove the way to restore the TOC in r2
204	 * and the r2 TOC will get corrupted.
205	 */
206
207	/*
208	 * Replace:
209	 *   bl <tramp>  <==== will be replaced with "b 1f"
210	 *   ld r2,40(r1)
211	 *  1:
212	 */
213	op = 0x48000008;	/* b +8 */
214
215	if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
216		return -EPERM;
217
218
219	flush_icache_range(ip, ip + 8);
220
221	return 0;
222}
223
224#else /* !PPC64 */
225static int
226__ftrace_make_nop(struct module *mod,
227		  struct dyn_ftrace *rec, unsigned long addr)
228{
229	unsigned int op;
230	unsigned int jmp[4];
231	unsigned long ip = rec->ip;
232	unsigned long tramp;
233
234	if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
235		return -EFAULT;
236
237	/* Make sure that that this is still a 24bit jump */
238	if (!is_bl_op(op)) {
239		printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
240		return -EINVAL;
241	}
242
243	/* lets find where the pointer goes */
244	tramp = find_bl_target(ip, op);
245
246	/*
247	 * On PPC32 the trampoline looks like:
248	 *  0x3d, 0x60, 0x00, 0x00  lis r11,sym@ha
249	 *  0x39, 0x6b, 0x00, 0x00  addi r11,r11,sym@l
250	 *  0x7d, 0x69, 0x03, 0xa6  mtctr r11
251	 *  0x4e, 0x80, 0x04, 0x20  bctr
252	 */
253
254	pr_devel("ip:%lx jumps to %lx", ip, tramp);
255
256	/* Find where the trampoline jumps to */
257	if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
258		printk(KERN_ERR "Failed to read %lx\n", tramp);
259		return -EFAULT;
260	}
261
262	pr_devel(" %08x %08x ", jmp[0], jmp[1]);
263
264	/* verify that this is what we expect it to be */
265	if (((jmp[0] & 0xffff0000) != 0x3d600000) ||
266	    ((jmp[1] & 0xffff0000) != 0x396b0000) ||
267	    (jmp[2] != 0x7d6903a6) ||
268	    (jmp[3] != 0x4e800420)) {
269		printk(KERN_ERR "Not a trampoline\n");
270		return -EINVAL;
271	}
272
273	tramp = (jmp[1] & 0xffff) |
274		((jmp[0] & 0xffff) << 16);
275	if (tramp & 0x8000)
276		tramp -= 0x10000;
277
278	pr_devel(" %lx ", tramp);
279
280	if (tramp != addr) {
281		printk(KERN_ERR
282		       "Trampoline location %08lx does not match addr\n",
283		       tramp);
284		return -EINVAL;
285	}
286
287	op = PPC_INST_NOP;
288
289	if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
290		return -EPERM;
291
292	flush_icache_range(ip, ip + 8);
293
294	return 0;
295}
296#endif /* PPC64 */
297#endif /* CONFIG_MODULES */
298
299int ftrace_make_nop(struct module *mod,
300		    struct dyn_ftrace *rec, unsigned long addr)
301{
302	unsigned long ip = rec->ip;
303	unsigned int old, new;
304
305	/*
306	 * If the calling address is more that 24 bits away,
307	 * then we had to use a trampoline to make the call.
308	 * Otherwise just update the call site.
309	 */
310	if (test_24bit_addr(ip, addr)) {
311		/* within range */
312		old = ftrace_call_replace(ip, addr, 1);
313		new = PPC_INST_NOP;
314		return ftrace_modify_code(ip, old, new);
315	}
316
317#ifdef CONFIG_MODULES
318	/*
319	 * Out of range jumps are called from modules.
320	 * We should either already have a pointer to the module
321	 * or it has been passed in.
322	 */
323	if (!rec->arch.mod) {
324		if (!mod) {
325			printk(KERN_ERR "No module loaded addr=%lx\n",
326			       addr);
327			return -EFAULT;
328		}
329		rec->arch.mod = mod;
330	} else if (mod) {
331		if (mod != rec->arch.mod) {
332			printk(KERN_ERR
333			       "Record mod %p not equal to passed in mod %p\n",
334			       rec->arch.mod, mod);
335			return -EINVAL;
336		}
337		/* nothing to do if mod == rec->arch.mod */
338	} else
339		mod = rec->arch.mod;
340
341	return __ftrace_make_nop(mod, rec, addr);
342#else
343	/* We should not get here without modules */
344	return -EINVAL;
345#endif /* CONFIG_MODULES */
346}
347
348#ifdef CONFIG_MODULES
349#ifdef CONFIG_PPC64
350static int
351__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
352{
353	unsigned int op[2];
354	unsigned long ip = rec->ip;
355
356	/* read where this goes */
357	if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2))
358		return -EFAULT;
359
360	/*
361	 * It should be pointing to two nops or
362	 *  b +8; ld r2,40(r1)
363	 */
364	if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
365	    ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) {
366		printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
367		return -EINVAL;
368	}
369
370	/* If we never set up a trampoline to ftrace_caller, then bail */
371	if (!rec->arch.mod->arch.tramp) {
372		printk(KERN_ERR "No ftrace trampoline\n");
373		return -EINVAL;
374	}
375
376	/* create the branch to the trampoline */
377	op[0] = create_branch((unsigned int *)ip,
378			      rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
379	if (!op[0]) {
380		printk(KERN_ERR "REL24 out of range!\n");
381		return -EINVAL;
382	}
383
384	/* ld r2,40(r1) */
385	op[1] = 0xe8410028;
386
387	pr_devel("write to %lx\n", rec->ip);
388
389	if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2))
390		return -EPERM;
391
392	flush_icache_range(ip, ip + 8);
393
394	return 0;
395}
396#else
397static int
398__ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
399{
400	unsigned int op;
401	unsigned long ip = rec->ip;
402
403	/* read where this goes */
404	if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
405		return -EFAULT;
406
407	/* It should be pointing to a nop */
408	if (op != PPC_INST_NOP) {
409		printk(KERN_ERR "Expected NOP but have %x\n", op);
410		return -EINVAL;
411	}
412
413	/* If we never set up a trampoline to ftrace_caller, then bail */
414	if (!rec->arch.mod->arch.tramp) {
415		printk(KERN_ERR "No ftrace trampoline\n");
416		return -EINVAL;
417	}
418
419	/* create the branch to the trampoline */
420	op = create_branch((unsigned int *)ip,
421			   rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
422	if (!op) {
423		printk(KERN_ERR "REL24 out of range!\n");
424		return -EINVAL;
425	}
426
427	pr_devel("write to %lx\n", rec->ip);
428
429	if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
430		return -EPERM;
431
432	flush_icache_range(ip, ip + 8);
433
434	return 0;
435}
436#endif /* CONFIG_PPC64 */
437#endif /* CONFIG_MODULES */
438
439int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
440{
441	unsigned long ip = rec->ip;
442	unsigned int old, new;
443
444	/*
445	 * If the calling address is more that 24 bits away,
446	 * then we had to use a trampoline to make the call.
447	 * Otherwise just update the call site.
448	 */
449	if (test_24bit_addr(ip, addr)) {
450		/* within range */
451		old = PPC_INST_NOP;
452		new = ftrace_call_replace(ip, addr, 1);
453		return ftrace_modify_code(ip, old, new);
454	}
455
456#ifdef CONFIG_MODULES
457	/*
458	 * Out of range jumps are called from modules.
459	 * Being that we are converting from nop, it had better
460	 * already have a module defined.
461	 */
462	if (!rec->arch.mod) {
463		printk(KERN_ERR "No module loaded\n");
464		return -EINVAL;
465	}
466
467	return __ftrace_make_call(rec, addr);
468#else
469	/* We should not get here without modules */
470	return -EINVAL;
471#endif /* CONFIG_MODULES */
472}
473
474int ftrace_update_ftrace_func(ftrace_func_t func)
475{
476	unsigned long ip = (unsigned long)(&ftrace_call);
477	unsigned int old, new;
478	int ret;
479
480	old = *(unsigned int *)&ftrace_call;
481	new = ftrace_call_replace(ip, (unsigned long)func, 1);
482	ret = ftrace_modify_code(ip, old, new);
483
484	return ret;
485}
486
487int __init ftrace_dyn_arch_init(void *data)
488{
489	/* caller expects data to be zero */
490	unsigned long *p = data;
491
492	*p = 0;
493
494	return 0;
495}
496#endif /* CONFIG_DYNAMIC_FTRACE */
497
498#ifdef CONFIG_FUNCTION_GRAPH_TRACER
499
500#ifdef CONFIG_DYNAMIC_FTRACE
501extern void ftrace_graph_call(void);
502extern void ftrace_graph_stub(void);
503
504int ftrace_enable_ftrace_graph_caller(void)
505{
506	unsigned long ip = (unsigned long)(&ftrace_graph_call);
507	unsigned long addr = (unsigned long)(&ftrace_graph_caller);
508	unsigned long stub = (unsigned long)(&ftrace_graph_stub);
509	unsigned int old, new;
510
511	old = ftrace_call_replace(ip, stub, 0);
512	new = ftrace_call_replace(ip, addr, 0);
513
514	return ftrace_modify_code(ip, old, new);
515}
516
517int ftrace_disable_ftrace_graph_caller(void)
518{
519	unsigned long ip = (unsigned long)(&ftrace_graph_call);
520	unsigned long addr = (unsigned long)(&ftrace_graph_caller);
521	unsigned long stub = (unsigned long)(&ftrace_graph_stub);
522	unsigned int old, new;
523
524	old = ftrace_call_replace(ip, addr, 0);
525	new = ftrace_call_replace(ip, stub, 0);
526
527	return ftrace_modify_code(ip, old, new);
528}
529#endif /* CONFIG_DYNAMIC_FTRACE */
530
531#ifdef CONFIG_PPC64
532extern void mod_return_to_handler(void);
533#endif
534
535/*
536 * Hook the return address and push it in the stack of return addrs
537 * in current thread info.
538 */
539void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
540{
541	unsigned long old;
542	int faulted;
543	struct ftrace_graph_ent trace;
544	unsigned long return_hooker = (unsigned long)&return_to_handler;
545
546	if (unlikely(atomic_read(&current->tracing_graph_pause)))
547		return;
548
549#ifdef CONFIG_PPC64
550	/* non core kernel code needs to save and restore the TOC */
551	if (REGION_ID(self_addr) != KERNEL_REGION_ID)
552		return_hooker = (unsigned long)&mod_return_to_handler;
553#endif
554
555	return_hooker = ppc_function_entry((void *)return_hooker);
556
557	/*
558	 * Protect against fault, even if it shouldn't
559	 * happen. This tool is too much intrusive to
560	 * ignore such a protection.
561	 */
562	asm volatile(
563		"1: " PPC_LL "%[old], 0(%[parent])\n"
564		"2: " PPC_STL "%[return_hooker], 0(%[parent])\n"
565		"   li %[faulted], 0\n"
566		"3:\n"
567
568		".section .fixup, \"ax\"\n"
569		"4: li %[faulted], 1\n"
570		"   b 3b\n"
571		".previous\n"
572
573		".section __ex_table,\"a\"\n"
574			PPC_LONG_ALIGN "\n"
575			PPC_LONG "1b,4b\n"
576			PPC_LONG "2b,4b\n"
577		".previous"
578
579		: [old] "=&r" (old), [faulted] "=r" (faulted)
580		: [parent] "r" (parent), [return_hooker] "r" (return_hooker)
581		: "memory"
582	);
583
584	if (unlikely(faulted)) {
585		ftrace_graph_stop();
586		WARN_ON(1);
587		return;
588	}
589
590	if (ftrace_push_return_trace(old, self_addr, &trace.depth, 0) == -EBUSY) {
591		*parent = old;
592		return;
593	}
594
595	trace.func = self_addr;
596
597	/* Only trace if the calling function expects to */
598	if (!ftrace_graph_entry(&trace)) {
599		current->curr_ret_stack--;
600		*parent = old;
601	}
602}
603#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
604
605#if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64)
606unsigned long __init arch_syscall_addr(int nr)
607{
608	return sys_call_table[nr*2];
609}
610#endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */
611