Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/paulus/powerpc
[linux-2.6] / arch / powerpc / kernel / kprobes.c
1 /*
2  *  Kernel Probes (KProbes)
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17  *
18  * Copyright (C) IBM Corporation, 2002, 2004
19  *
20  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
21  *              Probes initial implementation ( includes contributions from
22  *              Rusty Russell).
23  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
24  *              interface to access function arguments.
25  * 2004-Nov     Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port
26  *              for PPC64
27  */
28
29 #include <linux/kprobes.h>
30 #include <linux/ptrace.h>
31 #include <linux/preempt.h>
32 #include <linux/module.h>
33 #include <linux/kdebug.h>
34 #include <asm/cacheflush.h>
35 #include <asm/sstep.h>
36 #include <asm/uaccess.h>
37
38 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
39 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
40
41 int __kprobes arch_prepare_kprobe(struct kprobe *p)
42 {
43         int ret = 0;
44         kprobe_opcode_t insn = *p->addr;
45
46         if ((unsigned long)p->addr & 0x03) {
47                 printk("Attempt to register kprobe at an unaligned address\n");
48                 ret = -EINVAL;
49         } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
50                 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
51                 ret = -EINVAL;
52         }
53
54         /* insn must be on a special executable page on ppc64 */
55         if (!ret) {
56                 p->ainsn.insn = get_insn_slot();
57                 if (!p->ainsn.insn)
58                         ret = -ENOMEM;
59         }
60
61         if (!ret) {
62                 memcpy(p->ainsn.insn, p->addr,
63                                 MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
64                 p->opcode = *p->addr;
65                 flush_icache_range((unsigned long)p->ainsn.insn,
66                         (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t));
67         }
68
69         p->ainsn.boostable = 0;
70         return ret;
71 }
72
73 void __kprobes arch_arm_kprobe(struct kprobe *p)
74 {
75         *p->addr = BREAKPOINT_INSTRUCTION;
76         flush_icache_range((unsigned long) p->addr,
77                            (unsigned long) p->addr + sizeof(kprobe_opcode_t));
78 }
79
80 void __kprobes arch_disarm_kprobe(struct kprobe *p)
81 {
82         *p->addr = p->opcode;
83         flush_icache_range((unsigned long) p->addr,
84                            (unsigned long) p->addr + sizeof(kprobe_opcode_t));
85 }
86
87 void __kprobes arch_remove_kprobe(struct kprobe *p)
88 {
89         mutex_lock(&kprobe_mutex);
90         free_insn_slot(p->ainsn.insn, 0);
91         mutex_unlock(&kprobe_mutex);
92 }
93
94 static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
95 {
96         regs->msr |= MSR_SE;
97
98         /*
99          * On powerpc we should single step on the original
100          * instruction even if the probed insn is a trap
101          * variant as values in regs could play a part in
102          * if the trap is taken or not
103          */
104         regs->nip = (unsigned long)p->ainsn.insn;
105 }
106
107 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
108 {
109         kcb->prev_kprobe.kp = kprobe_running();
110         kcb->prev_kprobe.status = kcb->kprobe_status;
111         kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
112 }
113
114 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
115 {
116         __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
117         kcb->kprobe_status = kcb->prev_kprobe.status;
118         kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
119 }
120
121 static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
122                                 struct kprobe_ctlblk *kcb)
123 {
124         __get_cpu_var(current_kprobe) = p;
125         kcb->kprobe_saved_msr = regs->msr;
126 }
127
128 /* Called with kretprobe_lock held */
129 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
130                                       struct pt_regs *regs)
131 {
132         ri->ret_addr = (kprobe_opcode_t *)regs->link;
133
134         /* Replace the return addr with trampoline addr */
135         regs->link = (unsigned long)kretprobe_trampoline;
136 }
137
138 static int __kprobes kprobe_handler(struct pt_regs *regs)
139 {
140         struct kprobe *p;
141         int ret = 0;
142         unsigned int *addr = (unsigned int *)regs->nip;
143         struct kprobe_ctlblk *kcb;
144
145         /*
146          * We don't want to be preempted for the entire
147          * duration of kprobe processing
148          */
149         preempt_disable();
150         kcb = get_kprobe_ctlblk();
151
152         /* Check we're not actually recursing */
153         if (kprobe_running()) {
154                 p = get_kprobe(addr);
155                 if (p) {
156                         kprobe_opcode_t insn = *p->ainsn.insn;
157                         if (kcb->kprobe_status == KPROBE_HIT_SS &&
158                                         is_trap(insn)) {
159                                 regs->msr &= ~MSR_SE;
160                                 regs->msr |= kcb->kprobe_saved_msr;
161                                 goto no_kprobe;
162                         }
163                         /* We have reentered the kprobe_handler(), since
164                          * another probe was hit while within the handler.
165                          * We here save the original kprobes variables and
166                          * just single step on the instruction of the new probe
167                          * without calling any user handlers.
168                          */
169                         save_previous_kprobe(kcb);
170                         set_current_kprobe(p, regs, kcb);
171                         kcb->kprobe_saved_msr = regs->msr;
172                         kprobes_inc_nmissed_count(p);
173                         prepare_singlestep(p, regs);
174                         kcb->kprobe_status = KPROBE_REENTER;
175                         return 1;
176                 } else {
177                         if (*addr != BREAKPOINT_INSTRUCTION) {
178                                 /* If trap variant, then it belongs not to us */
179                                 kprobe_opcode_t cur_insn = *addr;
180                                 if (is_trap(cur_insn))
181                                         goto no_kprobe;
182                                 /* The breakpoint instruction was removed by
183                                  * another cpu right after we hit, no further
184                                  * handling of this interrupt is appropriate
185                                  */
186                                 ret = 1;
187                                 goto no_kprobe;
188                         }
189                         p = __get_cpu_var(current_kprobe);
190                         if (p->break_handler && p->break_handler(p, regs)) {
191                                 goto ss_probe;
192                         }
193                 }
194                 goto no_kprobe;
195         }
196
197         p = get_kprobe(addr);
198         if (!p) {
199                 if (*addr != BREAKPOINT_INSTRUCTION) {
200                         /*
201                          * PowerPC has multiple variants of the "trap"
202                          * instruction. If the current instruction is a
203                          * trap variant, it could belong to someone else
204                          */
205                         kprobe_opcode_t cur_insn = *addr;
206                         if (is_trap(cur_insn))
207                                 goto no_kprobe;
208                         /*
209                          * The breakpoint instruction was removed right
210                          * after we hit it.  Another cpu has removed
211                          * either a probepoint or a debugger breakpoint
212                          * at this address.  In either case, no further
213                          * handling of this interrupt is appropriate.
214                          */
215                         ret = 1;
216                 }
217                 /* Not one of ours: let kernel handle it */
218                 goto no_kprobe;
219         }
220
221         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
222         set_current_kprobe(p, regs, kcb);
223         if (p->pre_handler && p->pre_handler(p, regs))
224                 /* handler has already set things up, so skip ss setup */
225                 return 1;
226
227 ss_probe:
228         if (p->ainsn.boostable >= 0) {
229                 unsigned int insn = *p->ainsn.insn;
230
231                 /* regs->nip is also adjusted if emulate_step returns 1 */
232                 ret = emulate_step(regs, insn);
233                 if (ret > 0) {
234                         /*
235                          * Once this instruction has been boosted
236                          * successfully, set the boostable flag
237                          */
238                         if (unlikely(p->ainsn.boostable == 0))
239                                 p->ainsn.boostable = 1;
240
241                         if (p->post_handler)
242                                 p->post_handler(p, regs, 0);
243
244                         kcb->kprobe_status = KPROBE_HIT_SSDONE;
245                         reset_current_kprobe();
246                         preempt_enable_no_resched();
247                         return 1;
248                 } else if (ret < 0) {
249                         /*
250                          * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
251                          * So, we should never get here... but, its still
252                          * good to catch them, just in case...
253                          */
254                         printk("Can't step on instruction %x\n", insn);
255                         BUG();
256                 } else if (ret == 0)
257                         /* This instruction can't be boosted */
258                         p->ainsn.boostable = -1;
259         }
260         prepare_singlestep(p, regs);
261         kcb->kprobe_status = KPROBE_HIT_SS;
262         return 1;
263
264 no_kprobe:
265         preempt_enable_no_resched();
266         return ret;
267 }
268
269 /*
270  * Function return probe trampoline:
271  *      - init_kprobes() establishes a probepoint here
272  *      - When the probed function returns, this probe
273  *              causes the handlers to fire
274  */
275 void kretprobe_trampoline_holder(void)
276 {
277         asm volatile(".global kretprobe_trampoline\n"
278                         "kretprobe_trampoline:\n"
279                         "nop\n");
280 }
281
282 /*
283  * Called when the probe at kretprobe trampoline is hit
284  */
285 int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
286 {
287         struct kretprobe_instance *ri = NULL;
288         struct hlist_head *head, empty_rp;
289         struct hlist_node *node, *tmp;
290         unsigned long flags, orig_ret_address = 0;
291         unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
292
293         INIT_HLIST_HEAD(&empty_rp);
294         spin_lock_irqsave(&kretprobe_lock, flags);
295         head = kretprobe_inst_table_head(current);
296
297         /*
298          * It is possible to have multiple instances associated with a given
299          * task either because an multiple functions in the call path
300          * have a return probe installed on them, and/or more then one return
301          * return probe was registered for a target function.
302          *
303          * We can handle this because:
304          *     - instances are always inserted at the head of the list
305          *     - when multiple return probes are registered for the same
306          *       function, the first instance's ret_addr will point to the
307          *       real return address, and all the rest will point to
308          *       kretprobe_trampoline
309          */
310         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
311                 if (ri->task != current)
312                         /* another task is sharing our hash bucket */
313                         continue;
314
315                 if (ri->rp && ri->rp->handler)
316                         ri->rp->handler(ri, regs);
317
318                 orig_ret_address = (unsigned long)ri->ret_addr;
319                 recycle_rp_inst(ri, &empty_rp);
320
321                 if (orig_ret_address != trampoline_address)
322                         /*
323                          * This is the real return address. Any other
324                          * instances associated with this task are for
325                          * other calls deeper on the call stack
326                          */
327                         break;
328         }
329
330         kretprobe_assert(ri, orig_ret_address, trampoline_address);
331         regs->nip = orig_ret_address;
332
333         reset_current_kprobe();
334         spin_unlock_irqrestore(&kretprobe_lock, flags);
335         preempt_enable_no_resched();
336
337         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
338                 hlist_del(&ri->hlist);
339                 kfree(ri);
340         }
341         /*
342          * By returning a non-zero value, we are telling
343          * kprobe_handler() that we don't want the post_handler
344          * to run (and have re-enabled preemption)
345          */
346         return 1;
347 }
348
349 /*
350  * Called after single-stepping.  p->addr is the address of the
351  * instruction whose first byte has been replaced by the "breakpoint"
352  * instruction.  To avoid the SMP problems that can occur when we
353  * temporarily put back the original opcode to single-step, we
354  * single-stepped a copy of the instruction.  The address of this
355  * copy is p->ainsn.insn.
356  */
357 static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs)
358 {
359         int ret;
360         unsigned int insn = *p->ainsn.insn;
361
362         regs->nip = (unsigned long)p->addr;
363         ret = emulate_step(regs, insn);
364         if (ret == 0)
365                 regs->nip = (unsigned long)p->addr + 4;
366 }
367
368 static int __kprobes post_kprobe_handler(struct pt_regs *regs)
369 {
370         struct kprobe *cur = kprobe_running();
371         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
372
373         if (!cur)
374                 return 0;
375
376         if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
377                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
378                 cur->post_handler(cur, regs, 0);
379         }
380
381         resume_execution(cur, regs);
382         regs->msr |= kcb->kprobe_saved_msr;
383
384         /*Restore back the original saved kprobes variables and continue. */
385         if (kcb->kprobe_status == KPROBE_REENTER) {
386                 restore_previous_kprobe(kcb);
387                 goto out;
388         }
389         reset_current_kprobe();
390 out:
391         preempt_enable_no_resched();
392
393         /*
394          * if somebody else is singlestepping across a probe point, msr
395          * will have SE set, in which case, continue the remaining processing
396          * of do_debug, as if this is not a probe hit.
397          */
398         if (regs->msr & MSR_SE)
399                 return 0;
400
401         return 1;
402 }
403
404 int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
405 {
406         struct kprobe *cur = kprobe_running();
407         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
408         const struct exception_table_entry *entry;
409
410         switch(kcb->kprobe_status) {
411         case KPROBE_HIT_SS:
412         case KPROBE_REENTER:
413                 /*
414                  * We are here because the instruction being single
415                  * stepped caused a page fault. We reset the current
416                  * kprobe and the nip points back to the probe address
417                  * and allow the page fault handler to continue as a
418                  * normal page fault.
419                  */
420                 regs->nip = (unsigned long)cur->addr;
421                 regs->msr &= ~MSR_SE;
422                 regs->msr |= kcb->kprobe_saved_msr;
423                 if (kcb->kprobe_status == KPROBE_REENTER)
424                         restore_previous_kprobe(kcb);
425                 else
426                         reset_current_kprobe();
427                 preempt_enable_no_resched();
428                 break;
429         case KPROBE_HIT_ACTIVE:
430         case KPROBE_HIT_SSDONE:
431                 /*
432                  * We increment the nmissed count for accounting,
433                  * we can also use npre/npostfault count for accouting
434                  * these specific fault cases.
435                  */
436                 kprobes_inc_nmissed_count(cur);
437
438                 /*
439                  * We come here because instructions in the pre/post
440                  * handler caused the page_fault, this could happen
441                  * if handler tries to access user space by
442                  * copy_from_user(), get_user() etc. Let the
443                  * user-specified handler try to fix it first.
444                  */
445                 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
446                         return 1;
447
448                 /*
449                  * In case the user-specified fault handler returned
450                  * zero, try to fix up.
451                  */
452                 if ((entry = search_exception_tables(regs->nip)) != NULL) {
453                         regs->nip = entry->fixup;
454                         return 1;
455                 }
456
457                 /*
458                  * fixup_exception() could not handle it,
459                  * Let do_page_fault() fix it.
460                  */
461                 break;
462         default:
463                 break;
464         }
465         return 0;
466 }
467
468 /*
469  * Wrapper routine to for handling exceptions.
470  */
471 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
472                                        unsigned long val, void *data)
473 {
474         struct die_args *args = (struct die_args *)data;
475         int ret = NOTIFY_DONE;
476
477         if (args->regs && user_mode(args->regs))
478                 return ret;
479
480         switch (val) {
481         case DIE_BPT:
482                 if (kprobe_handler(args->regs))
483                         ret = NOTIFY_STOP;
484                 break;
485         case DIE_SSTEP:
486                 if (post_kprobe_handler(args->regs))
487                         ret = NOTIFY_STOP;
488                 break;
489         default:
490                 break;
491         }
492         return ret;
493 }
494
495 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
496 {
497         struct jprobe *jp = container_of(p, struct jprobe, kp);
498         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
499
500         memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
501
502         /* setup return addr to the jprobe handler routine */
503 #ifdef CONFIG_PPC64
504         regs->nip = (unsigned long)(((func_descr_t *)jp->entry)->entry);
505         regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc);
506 #else
507         regs->nip = (unsigned long)jp->entry;
508 #endif
509
510         return 1;
511 }
512
513 void __kprobes jprobe_return(void)
514 {
515         asm volatile("trap" ::: "memory");
516 }
517
518 void __kprobes jprobe_return_end(void)
519 {
520 };
521
522 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
523 {
524         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
525
526         /*
527          * FIXME - we should ideally be validating that we got here 'cos
528          * of the "trap" in jprobe_return() above, before restoring the
529          * saved regs...
530          */
531         memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
532         preempt_enable_no_resched();
533         return 1;
534 }
535
536 static struct kprobe trampoline_p = {
537         .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
538         .pre_handler = trampoline_probe_handler
539 };
540
541 int __init arch_init_kprobes(void)
542 {
543         return register_kprobe(&trampoline_p);
544 }
545
546 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
547 {
548         if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
549                 return 1;
550
551         return 0;
552 }