2 * Code for replacing ftrace calls with jumps.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box.
8 * Added function graph tracer code, taken from x86 that was written
9 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt.
13 #include <linux/spinlock.h>
14 #include <linux/hardirq.h>
15 #include <linux/uaccess.h>
16 #include <linux/module.h>
17 #include <linux/ftrace.h>
18 #include <linux/percpu.h>
19 #include <linux/init.h>
20 #include <linux/list.h>
22 #include <asm/cacheflush.h>
23 #include <asm/code-patching.h>
24 #include <asm/ftrace.h>
27 #ifdef CONFIG_DYNAMIC_FTRACE
28 static unsigned int ftrace_nop_replace(void)
34 ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
38 addr = ppc_function_entry((void *)addr);
40 /* if (link) set op to 'bl' else 'b' */
41 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
47 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
49 unsigned int replaced;
52 * Note: Due to modules and __init, code can
53 * disappear and change, we need to protect against faulting
54 * as well as code changing. We do this by using the
55 * probe_kernel_* functions.
57 * No real locking needed, this code is run through
58 * kstop_machine, or before SMP starts.
61 /* read the text we want to modify */
62 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE))
65 /* Make sure it is what we expect it to be */
69 /* replace the text with the new text */
70 if (probe_kernel_write((void *)ip, &new, MCOUNT_INSN_SIZE))
73 flush_icache_range(ip, ip + 8);
79 * Helper functions that are the same for both PPC64 and PPC32.
81 static int test_24bit_addr(unsigned long ip, unsigned long addr)
84 /* use the create_branch to verify that this offset can be branched */
85 return create_branch((unsigned int *)ip, addr, 0);
90 static int is_bl_op(unsigned int op)
92 return (op & 0xfc000003) == 0x48000001;
95 static unsigned long find_bl_target(unsigned long ip, unsigned int op)
99 offset = (op & 0x03fffffc);
101 if (offset & 0x02000000)
102 offset |= 0xfe000000;
104 return ip + (long)offset;
109 __ftrace_make_nop(struct module *mod,
110 struct dyn_ftrace *rec, unsigned long addr)
115 unsigned long ip = rec->ip;
119 /* read where this goes */
120 if (probe_kernel_read(&op, (void *)ip, sizeof(int)))
123 /* Make sure that that this is still a 24bit jump */
125 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
129 /* lets find where the pointer goes */
130 tramp = find_bl_target(ip, op);
133 * On PPC64 the trampoline looks like:
134 * 0x3d, 0x82, 0x00, 0x00, addis r12,r2, <high>
135 * 0x39, 0x8c, 0x00, 0x00, addi r12,r12, <low>
136 * Where the bytes 2,3,6 and 7 make up the 32bit offset
137 * to the TOC that holds the pointer.
139 * 0xf8, 0x41, 0x00, 0x28, std r2,40(r1)
140 * 0xe9, 0x6c, 0x00, 0x20, ld r11,32(r12)
141 * The actually address is 32 bytes from the offset
143 * 0xe8, 0x4c, 0x00, 0x28, ld r2,40(r12)
146 pr_devel("ip:%lx jumps to %lx r2: %lx", ip, tramp, mod->arch.toc);
148 /* Find where the trampoline jumps to */
149 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
150 printk(KERN_ERR "Failed to read %lx\n", tramp);
154 pr_devel(" %08x %08x", jmp[0], jmp[1]);
156 /* verify that this is what we expect it to be */
157 if (((jmp[0] & 0xffff0000) != 0x3d820000) ||
158 ((jmp[1] & 0xffff0000) != 0x398c0000) ||
159 (jmp[2] != 0xf8410028) ||
160 (jmp[3] != 0xe96c0020) ||
161 (jmp[4] != 0xe84c0028)) {
162 printk(KERN_ERR "Not a trampoline\n");
166 /* The bottom half is signed extended */
167 offset = ((unsigned)((unsigned short)jmp[0]) << 16) +
168 (int)((short)jmp[1]);
170 pr_devel(" %x ", offset);
172 /* get the address this jumps too */
173 tramp = mod->arch.toc + offset + 32;
174 pr_devel("toc: %lx", tramp);
176 if (probe_kernel_read(jmp, (void *)tramp, 8)) {
177 printk(KERN_ERR "Failed to read %lx\n", tramp);
181 pr_devel(" %08x %08x\n", jmp[0], jmp[1]);
183 ptr = ((unsigned long)jmp[0] << 32) + jmp[1];
185 /* This should match what was called */
186 if (ptr != ppc_function_entry((void *)addr)) {
187 printk(KERN_ERR "addr does not match %lx\n", ptr);
192 * We want to nop the line, but the next line is
193 * 0xe8, 0x41, 0x00, 0x28 ld r2,40(r1)
194 * This needs to be turned to a nop too.
196 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE))
199 if (op != 0xe8410028) {
200 printk(KERN_ERR "Next line is not ld! (%08x)\n", op);
205 * Milton Miller pointed out that we can not blindly do nops.
206 * If a task was preempted when calling a trace function,
207 * the nops will remove the way to restore the TOC in r2
208 * and the r2 TOC will get corrupted.
213 * bl <tramp> <==== will be replaced with "b 1f"
217 op = 0x48000008; /* b +8 */
219 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
223 flush_icache_range(ip, ip + 8);
230 __ftrace_make_nop(struct module *mod,
231 struct dyn_ftrace *rec, unsigned long addr)
235 unsigned long ip = rec->ip;
238 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
241 /* Make sure that that this is still a 24bit jump */
243 printk(KERN_ERR "Not expected bl: opcode is %x\n", op);
247 /* lets find where the pointer goes */
248 tramp = find_bl_target(ip, op);
251 * On PPC32 the trampoline looks like:
252 * 0x3d, 0x60, 0x00, 0x00 lis r11,sym@ha
253 * 0x39, 0x6b, 0x00, 0x00 addi r11,r11,sym@l
254 * 0x7d, 0x69, 0x03, 0xa6 mtctr r11
255 * 0x4e, 0x80, 0x04, 0x20 bctr
258 pr_devel("ip:%lx jumps to %lx", ip, tramp);
260 /* Find where the trampoline jumps to */
261 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) {
262 printk(KERN_ERR "Failed to read %lx\n", tramp);
266 pr_devel(" %08x %08x ", jmp[0], jmp[1]);
268 /* verify that this is what we expect it to be */
269 if (((jmp[0] & 0xffff0000) != 0x3d600000) ||
270 ((jmp[1] & 0xffff0000) != 0x396b0000) ||
271 (jmp[2] != 0x7d6903a6) ||
272 (jmp[3] != 0x4e800420)) {
273 printk(KERN_ERR "Not a trampoline\n");
277 tramp = (jmp[1] & 0xffff) |
278 ((jmp[0] & 0xffff) << 16);
282 pr_devel(" %lx ", tramp);
286 "Trampoline location %08lx does not match addr\n",
293 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
296 flush_icache_range(ip, ip + 8);
301 #endif /* CONFIG_MODULES */
303 int ftrace_make_nop(struct module *mod,
304 struct dyn_ftrace *rec, unsigned long addr)
306 unsigned long ip = rec->ip;
307 unsigned int old, new;
310 * If the calling address is more that 24 bits away,
311 * then we had to use a trampoline to make the call.
312 * Otherwise just update the call site.
314 if (test_24bit_addr(ip, addr)) {
316 old = ftrace_call_replace(ip, addr, 1);
317 new = ftrace_nop_replace();
318 return ftrace_modify_code(ip, old, new);
321 #ifdef CONFIG_MODULES
323 * Out of range jumps are called from modules.
324 * We should either already have a pointer to the module
325 * or it has been passed in.
327 if (!rec->arch.mod) {
329 printk(KERN_ERR "No module loaded addr=%lx\n",
335 if (mod != rec->arch.mod) {
337 "Record mod %p not equal to passed in mod %p\n",
341 /* nothing to do if mod == rec->arch.mod */
345 return __ftrace_make_nop(mod, rec, addr);
347 /* We should not get here without modules */
349 #endif /* CONFIG_MODULES */
352 #ifdef CONFIG_MODULES
355 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
358 unsigned long ip = rec->ip;
360 /* read where this goes */
361 if (probe_kernel_read(op, (void *)ip, MCOUNT_INSN_SIZE * 2))
365 * It should be pointing to two nops or
368 if (((op[0] != 0x48000008) || (op[1] != 0xe8410028)) &&
369 ((op[0] != PPC_INST_NOP) || (op[1] != PPC_INST_NOP))) {
370 printk(KERN_ERR "Expected NOPs but have %x %x\n", op[0], op[1]);
374 /* If we never set up a trampoline to ftrace_caller, then bail */
375 if (!rec->arch.mod->arch.tramp) {
376 printk(KERN_ERR "No ftrace trampoline\n");
380 /* create the branch to the trampoline */
381 op[0] = create_branch((unsigned int *)ip,
382 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
384 printk(KERN_ERR "REL24 out of range!\n");
391 pr_devel("write to %lx\n", rec->ip);
393 if (probe_kernel_write((void *)ip, op, MCOUNT_INSN_SIZE * 2))
396 flush_icache_range(ip, ip + 8);
402 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
405 unsigned long ip = rec->ip;
407 /* read where this goes */
408 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE))
411 /* It should be pointing to a nop */
412 if (op != PPC_INST_NOP) {
413 printk(KERN_ERR "Expected NOP but have %x\n", op);
417 /* If we never set up a trampoline to ftrace_caller, then bail */
418 if (!rec->arch.mod->arch.tramp) {
419 printk(KERN_ERR "No ftrace trampoline\n");
423 /* create the branch to the trampoline */
424 op = create_branch((unsigned int *)ip,
425 rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
427 printk(KERN_ERR "REL24 out of range!\n");
431 pr_devel("write to %lx\n", rec->ip);
433 if (probe_kernel_write((void *)ip, &op, MCOUNT_INSN_SIZE))
436 flush_icache_range(ip, ip + 8);
440 #endif /* CONFIG_PPC64 */
441 #endif /* CONFIG_MODULES */
443 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
445 unsigned long ip = rec->ip;
446 unsigned int old, new;
449 * If the calling address is more that 24 bits away,
450 * then we had to use a trampoline to make the call.
451 * Otherwise just update the call site.
453 if (test_24bit_addr(ip, addr)) {
455 old = ftrace_nop_replace();
456 new = ftrace_call_replace(ip, addr, 1);
457 return ftrace_modify_code(ip, old, new);
460 #ifdef CONFIG_MODULES
462 * Out of range jumps are called from modules.
463 * Being that we are converting from nop, it had better
464 * already have a module defined.
466 if (!rec->arch.mod) {
467 printk(KERN_ERR "No module loaded\n");
471 return __ftrace_make_call(rec, addr);
473 /* We should not get here without modules */
475 #endif /* CONFIG_MODULES */
478 int ftrace_update_ftrace_func(ftrace_func_t func)
480 unsigned long ip = (unsigned long)(&ftrace_call);
481 unsigned int old, new;
484 old = *(unsigned int *)&ftrace_call;
485 new = ftrace_call_replace(ip, (unsigned long)func, 1);
486 ret = ftrace_modify_code(ip, old, new);
491 int __init ftrace_dyn_arch_init(void *data)
493 /* caller expects data to be zero */
494 unsigned long *p = data;
500 #endif /* CONFIG_DYNAMIC_FTRACE */
502 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
504 #ifdef CONFIG_DYNAMIC_FTRACE
505 extern void ftrace_graph_call(void);
506 extern void ftrace_graph_stub(void);
508 int ftrace_enable_ftrace_graph_caller(void)
510 unsigned long ip = (unsigned long)(&ftrace_graph_call);
511 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
512 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
513 unsigned int old, new;
515 old = ftrace_call_replace(ip, stub, 0);
516 new = ftrace_call_replace(ip, addr, 0);
518 return ftrace_modify_code(ip, old, new);
521 int ftrace_disable_ftrace_graph_caller(void)
523 unsigned long ip = (unsigned long)(&ftrace_graph_call);
524 unsigned long addr = (unsigned long)(&ftrace_graph_caller);
525 unsigned long stub = (unsigned long)(&ftrace_graph_stub);
526 unsigned int old, new;
528 old = ftrace_call_replace(ip, addr, 0);
529 new = ftrace_call_replace(ip, stub, 0);
531 return ftrace_modify_code(ip, old, new);
533 #endif /* CONFIG_DYNAMIC_FTRACE */
536 extern void mod_return_to_handler(void);
540 * Hook the return address and push it in the stack of return addrs
541 * in current thread info.
543 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr)
547 struct ftrace_graph_ent trace;
548 unsigned long return_hooker = (unsigned long)&return_to_handler;
550 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
554 /* non core kernel code needs to save and restore the TOC */
555 if (REGION_ID(self_addr) != KERNEL_REGION_ID)
556 return_hooker = (unsigned long)&mod_return_to_handler;
559 return_hooker = ppc_function_entry((void *)return_hooker);
562 * Protect against fault, even if it shouldn't
563 * happen. This tool is too much intrusive to
564 * ignore such a protection.
567 "1: " PPC_LL "%[old], 0(%[parent])\n"
568 "2: " PPC_STL "%[return_hooker], 0(%[parent])\n"
569 " li %[faulted], 0\n"
572 ".section .fixup, \"ax\"\n"
573 "4: li %[faulted], 1\n"
577 ".section __ex_table,\"a\"\n"
583 : [old] "=&r" (old), [faulted] "=r" (faulted)
584 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
588 if (unlikely(faulted)) {
594 if (ftrace_push_return_trace(old, self_addr, &trace.depth) == -EBUSY) {
599 trace.func = self_addr;
601 /* Only trace if the calling function expects to */
602 if (!ftrace_graph_entry(&trace)) {
603 current->curr_ret_stack--;
607 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */