[IA64] Avoid "u64 foo : 32;" for gcc3 vs. gcc4 compatibility
[linux-2.6] / arch / ia64 / kernel / kprobes.c
1 /*
2  *  Kernel Probes (KProbes)
3  *  arch/ia64/kernel/kprobes.c
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18  *
19  * Copyright (C) IBM Corporation, 2002, 2004
20  * Copyright (C) Intel Corporation, 2005
21  *
22  * 2005-Apr     Rusty Lynch <rusty.lynch@intel.com> and Anil S Keshavamurthy
23  *              <anil.s.keshavamurthy@intel.com> adapted from i386
24  */
25
26 #include <linux/config.h>
27 #include <linux/kprobes.h>
28 #include <linux/ptrace.h>
29 #include <linux/string.h>
30 #include <linux/slab.h>
31 #include <linux/preempt.h>
32 #include <linux/moduleloader.h>
33
34 #include <asm/pgtable.h>
35 #include <asm/kdebug.h>
36 #include <asm/sections.h>
37 #include <asm/uaccess.h>
38
39 extern void jprobe_inst_return(void);
40
41 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
42 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
43
44 enum instruction_type {A, I, M, F, B, L, X, u};
45 static enum instruction_type bundle_encoding[32][3] = {
46   { M, I, I },                          /* 00 */
47   { M, I, I },                          /* 01 */
48   { M, I, I },                          /* 02 */
49   { M, I, I },                          /* 03 */
50   { M, L, X },                          /* 04 */
51   { M, L, X },                          /* 05 */
52   { u, u, u },                          /* 06 */
53   { u, u, u },                          /* 07 */
54   { M, M, I },                          /* 08 */
55   { M, M, I },                          /* 09 */
56   { M, M, I },                          /* 0A */
57   { M, M, I },                          /* 0B */
58   { M, F, I },                          /* 0C */
59   { M, F, I },                          /* 0D */
60   { M, M, F },                          /* 0E */
61   { M, M, F },                          /* 0F */
62   { M, I, B },                          /* 10 */
63   { M, I, B },                          /* 11 */
64   { M, B, B },                          /* 12 */
65   { M, B, B },                          /* 13 */
66   { u, u, u },                          /* 14 */
67   { u, u, u },                          /* 15 */
68   { B, B, B },                          /* 16 */
69   { B, B, B },                          /* 17 */
70   { M, M, B },                          /* 18 */
71   { M, M, B },                          /* 19 */
72   { u, u, u },                          /* 1A */
73   { u, u, u },                          /* 1B */
74   { M, F, B },                          /* 1C */
75   { M, F, B },                          /* 1D */
76   { u, u, u },                          /* 1E */
77   { u, u, u },                          /* 1F */
78 };
79
80 /*
81  * In this function we check to see if the instruction
82  * is IP relative instruction and update the kprobe
83  * inst flag accordingly
84  */
85 static void __kprobes update_kprobe_inst_flag(uint template, uint  slot,
86                                               uint major_opcode,
87                                               unsigned long kprobe_inst,
88                                               struct kprobe *p)
89 {
90         p->ainsn.inst_flag = 0;
91         p->ainsn.target_br_reg = 0;
92
93         /* Check for Break instruction
94          * Bits 37:40 Major opcode to be zero
95          * Bits 27:32 X6 to be zero
96          * Bits 32:35 X3 to be zero
97          */
98         if ((!major_opcode) && (!((kprobe_inst >> 27) & 0x1FF)) ) {
99                 /* is a break instruction */
100                 p->ainsn.inst_flag |= INST_FLAG_BREAK_INST;
101                 return;
102         }
103
104         if (bundle_encoding[template][slot] == B) {
105                 switch (major_opcode) {
106                   case INDIRECT_CALL_OPCODE:
107                         p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG;
108                         p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7);
109                         break;
110                   case IP_RELATIVE_PREDICT_OPCODE:
111                   case IP_RELATIVE_BRANCH_OPCODE:
112                         p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR;
113                         break;
114                   case IP_RELATIVE_CALL_OPCODE:
115                         p->ainsn.inst_flag |= INST_FLAG_FIX_RELATIVE_IP_ADDR;
116                         p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG;
117                         p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7);
118                         break;
119                 }
120         } else if (bundle_encoding[template][slot] == X) {
121                 switch (major_opcode) {
122                   case LONG_CALL_OPCODE:
123                         p->ainsn.inst_flag |= INST_FLAG_FIX_BRANCH_REG;
124                         p->ainsn.target_br_reg = ((kprobe_inst >> 6) & 0x7);
125                   break;
126                 }
127         }
128         return;
129 }
130
131 /*
132  * In this function we check to see if the instruction
133  * on which we are inserting kprobe is supported.
134  * Returns 0 if supported
135  * Returns -EINVAL if unsupported
136  */
137 static int __kprobes unsupported_inst(uint template, uint  slot,
138                                       uint major_opcode,
139                                       unsigned long kprobe_inst,
140                                       struct kprobe *p)
141 {
142         unsigned long addr = (unsigned long)p->addr;
143
144         if (bundle_encoding[template][slot] == I) {
145                 switch (major_opcode) {
146                         case 0x0: //I_UNIT_MISC_OPCODE:
147                         /*
148                          * Check for Integer speculation instruction
149                          * - Bit 33-35 to be equal to 0x1
150                          */
151                         if (((kprobe_inst >> 33) & 0x7) == 1) {
152                                 printk(KERN_WARNING
153                                         "Kprobes on speculation inst at <0x%lx> not supported\n",
154                                         addr);
155                                 return -EINVAL;
156                         }
157
158                         /*
159                          * IP relative mov instruction
160                          *  - Bit 27-35 to be equal to 0x30
161                          */
162                         if (((kprobe_inst >> 27) & 0x1FF) == 0x30) {
163                                 printk(KERN_WARNING
164                                         "Kprobes on \"mov r1=ip\" at <0x%lx> not supported\n",
165                                         addr);
166                                 return -EINVAL;
167
168                         }
169                 }
170         }
171         return 0;
172 }
173
174
175 /*
176  * In this function we check to see if the instruction
177  * (qp) cmpx.crel.ctype p1,p2=r2,r3
178  * on which we are inserting kprobe is cmp instruction
179  * with ctype as unc.
180  */
181 static uint __kprobes is_cmp_ctype_unc_inst(uint template, uint slot,
182                                             uint major_opcode,
183                                             unsigned long kprobe_inst)
184 {
185         cmp_inst_t cmp_inst;
186         uint ctype_unc = 0;
187
188         if (!((bundle_encoding[template][slot] == I) ||
189                 (bundle_encoding[template][slot] == M)))
190                 goto out;
191
192         if (!((major_opcode == 0xC) || (major_opcode == 0xD) ||
193                 (major_opcode == 0xE)))
194                 goto out;
195
196         cmp_inst.l = kprobe_inst;
197         if ((cmp_inst.f.x2 == 0) || (cmp_inst.f.x2 == 1)) {
198                 /* Integere compare - Register Register (A6 type)*/
199                 if ((cmp_inst.f.tb == 0) && (cmp_inst.f.ta == 0)
200                                 &&(cmp_inst.f.c == 1))
201                         ctype_unc = 1;
202         } else if ((cmp_inst.f.x2 == 2)||(cmp_inst.f.x2 == 3)) {
203                 /* Integere compare - Immediate Register (A8 type)*/
204                 if ((cmp_inst.f.ta == 0) &&(cmp_inst.f.c == 1))
205                         ctype_unc = 1;
206         }
207 out:
208         return ctype_unc;
209 }
210
211 /*
212  * In this function we override the bundle with
213  * the break instruction at the given slot.
214  */
215 static void __kprobes prepare_break_inst(uint template, uint  slot,
216                                          uint major_opcode,
217                                          unsigned long kprobe_inst,
218                                          struct kprobe *p)
219 {
220         unsigned long break_inst = BREAK_INST;
221         bundle_t *bundle = &p->ainsn.insn.bundle;
222
223         /*
224          * Copy the original kprobe_inst qualifying predicate(qp)
225          * to the break instruction iff !is_cmp_ctype_unc_inst
226          * because for cmp instruction with ctype equal to unc,
227          * which is a special instruction always needs to be
228          * executed regradless of qp
229          */
230         if (!is_cmp_ctype_unc_inst(template, slot, major_opcode, kprobe_inst))
231                 break_inst |= (0x3f & kprobe_inst);
232
233         switch (slot) {
234           case 0:
235                 bundle->quad0.slot0 = break_inst;
236                 break;
237           case 1:
238                 bundle->quad0.slot1_p0 = break_inst;
239                 bundle->quad1.slot1_p1 = break_inst >> (64-46);
240                 break;
241           case 2:
242                 bundle->quad1.slot2 = break_inst;
243                 break;
244         }
245
246         /*
247          * Update the instruction flag, so that we can
248          * emulate the instruction properly after we
249          * single step on original instruction
250          */
251         update_kprobe_inst_flag(template, slot, major_opcode, kprobe_inst, p);
252 }
253
254 static inline void get_kprobe_inst(bundle_t *bundle, uint slot,
255                 unsigned long *kprobe_inst, uint *major_opcode)
256 {
257         unsigned long kprobe_inst_p0, kprobe_inst_p1;
258         unsigned int template;
259
260         template = bundle->quad0.template;
261
262         switch (slot) {
263           case 0:
264                 *major_opcode = (bundle->quad0.slot0 >> SLOT0_OPCODE_SHIFT);
265                 *kprobe_inst = bundle->quad0.slot0;
266                 break;
267           case 1:
268                 *major_opcode = (bundle->quad1.slot1_p1 >> SLOT1_p1_OPCODE_SHIFT);
269                 kprobe_inst_p0 = bundle->quad0.slot1_p0;
270                 kprobe_inst_p1 = bundle->quad1.slot1_p1;
271                 *kprobe_inst = kprobe_inst_p0 | (kprobe_inst_p1 << (64-46));
272                 break;
273           case 2:
274                 *major_opcode = (bundle->quad1.slot2 >> SLOT2_OPCODE_SHIFT);
275                 *kprobe_inst = bundle->quad1.slot2;
276                 break;
277         }
278 }
279
280 /* Returns non-zero if the addr is in the Interrupt Vector Table */
281 static inline int in_ivt_functions(unsigned long addr)
282 {
283         return (addr >= (unsigned long)__start_ivt_text
284                 && addr < (unsigned long)__end_ivt_text);
285 }
286
287 static int __kprobes valid_kprobe_addr(int template, int slot,
288                                        unsigned long addr)
289 {
290         if ((slot > 2) || ((bundle_encoding[template][1] == L) && slot > 1)) {
291                 printk(KERN_WARNING "Attempting to insert unaligned kprobe "
292                                 "at 0x%lx\n", addr);
293                 return -EINVAL;
294         }
295
296         if (in_ivt_functions(addr)) {
297                 printk(KERN_WARNING "Kprobes can't be inserted inside "
298                                 "IVT functions at 0x%lx\n", addr);
299                 return -EINVAL;
300         }
301
302         if (slot == 1 && bundle_encoding[template][1] != L) {
303                 printk(KERN_WARNING "Inserting kprobes on slot #1 "
304                        "is not supported\n");
305                 return -EINVAL;
306         }
307
308         return 0;
309 }
310
311 static inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
312 {
313         kcb->prev_kprobe.kp = kprobe_running();
314         kcb->prev_kprobe.status = kcb->kprobe_status;
315 }
316
317 static inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
318 {
319         __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
320         kcb->kprobe_status = kcb->prev_kprobe.status;
321 }
322
323 static inline void set_current_kprobe(struct kprobe *p,
324                         struct kprobe_ctlblk *kcb)
325 {
326         __get_cpu_var(current_kprobe) = p;
327 }
328
329 static void kretprobe_trampoline(void)
330 {
331 }
332
333 /*
334  * At this point the target function has been tricked into
335  * returning into our trampoline.  Lookup the associated instance
336  * and then:
337  *    - call the handler function
338  *    - cleanup by marking the instance as unused
339  *    - long jump back to the original return address
340  */
341 int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
342 {
343         struct kretprobe_instance *ri = NULL;
344         struct hlist_head *head;
345         struct hlist_node *node, *tmp;
346         unsigned long flags, orig_ret_address = 0;
347         unsigned long trampoline_address =
348                 ((struct fnptr *)kretprobe_trampoline)->ip;
349
350         spin_lock_irqsave(&kretprobe_lock, flags);
351         head = kretprobe_inst_table_head(current);
352
353         /*
354          * It is possible to have multiple instances associated with a given
355          * task either because an multiple functions in the call path
356          * have a return probe installed on them, and/or more then one return
357          * return probe was registered for a target function.
358          *
359          * We can handle this because:
360          *     - instances are always inserted at the head of the list
361          *     - when multiple return probes are registered for the same
362          *       function, the first instance's ret_addr will point to the
363          *       real return address, and all the rest will point to
364          *       kretprobe_trampoline
365          */
366         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
367                 if (ri->task != current)
368                         /* another task is sharing our hash bucket */
369                         continue;
370
371                 if (ri->rp && ri->rp->handler)
372                         ri->rp->handler(ri, regs);
373
374                 orig_ret_address = (unsigned long)ri->ret_addr;
375                 recycle_rp_inst(ri);
376
377                 if (orig_ret_address != trampoline_address)
378                         /*
379                          * This is the real return address. Any other
380                          * instances associated with this task are for
381                          * other calls deeper on the call stack
382                          */
383                         break;
384         }
385
386         BUG_ON(!orig_ret_address || (orig_ret_address == trampoline_address));
387         regs->cr_iip = orig_ret_address;
388
389         reset_current_kprobe();
390         spin_unlock_irqrestore(&kretprobe_lock, flags);
391         preempt_enable_no_resched();
392
393         /*
394          * By returning a non-zero value, we are telling
395          * kprobe_handler() that we don't want the post_handler
396          * to run (and have re-enabled preemption)
397          */
398         return 1;
399 }
400
401 /* Called with kretprobe_lock held */
402 void __kprobes arch_prepare_kretprobe(struct kretprobe *rp,
403                                       struct pt_regs *regs)
404 {
405         struct kretprobe_instance *ri;
406
407         if ((ri = get_free_rp_inst(rp)) != NULL) {
408                 ri->rp = rp;
409                 ri->task = current;
410                 ri->ret_addr = (kprobe_opcode_t *)regs->b0;
411
412                 /* Replace the return addr with trampoline addr */
413                 regs->b0 = ((struct fnptr *)kretprobe_trampoline)->ip;
414
415                 add_rp_inst(ri);
416         } else {
417                 rp->nmissed++;
418         }
419 }
420
421 int __kprobes arch_prepare_kprobe(struct kprobe *p)
422 {
423         unsigned long addr = (unsigned long) p->addr;
424         unsigned long *kprobe_addr = (unsigned long *)(addr & ~0xFULL);
425         unsigned long kprobe_inst=0;
426         unsigned int slot = addr & 0xf, template, major_opcode = 0;
427         bundle_t *bundle = &p->ainsn.insn.bundle;
428
429         memcpy(&p->opcode.bundle, kprobe_addr, sizeof(bundle_t));
430         memcpy(&p->ainsn.insn.bundle, kprobe_addr, sizeof(bundle_t));
431
432         template = bundle->quad0.template;
433
434         if(valid_kprobe_addr(template, slot, addr))
435                 return -EINVAL;
436
437         /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */
438         if (slot == 1 && bundle_encoding[template][1] == L)
439                 slot++;
440
441         /* Get kprobe_inst and major_opcode from the bundle */
442         get_kprobe_inst(bundle, slot, &kprobe_inst, &major_opcode);
443
444         if (unsupported_inst(template, slot, major_opcode, kprobe_inst, p))
445                         return -EINVAL;
446
447         prepare_break_inst(template, slot, major_opcode, kprobe_inst, p);
448
449         return 0;
450 }
451
452 void __kprobes arch_arm_kprobe(struct kprobe *p)
453 {
454         unsigned long addr = (unsigned long)p->addr;
455         unsigned long arm_addr = addr & ~0xFULL;
456
457         memcpy((char *)arm_addr, &p->ainsn.insn.bundle, sizeof(bundle_t));
458         flush_icache_range(arm_addr, arm_addr + sizeof(bundle_t));
459 }
460
461 void __kprobes arch_disarm_kprobe(struct kprobe *p)
462 {
463         unsigned long addr = (unsigned long)p->addr;
464         unsigned long arm_addr = addr & ~0xFULL;
465
466         /* p->opcode contains the original unaltered bundle */
467         memcpy((char *) arm_addr, (char *) &p->opcode.bundle, sizeof(bundle_t));
468         flush_icache_range(arm_addr, arm_addr + sizeof(bundle_t));
469 }
470
471 /*
472  * We are resuming execution after a single step fault, so the pt_regs
473  * structure reflects the register state after we executed the instruction
474  * located in the kprobe (p->ainsn.insn.bundle).  We still need to adjust
475  * the ip to point back to the original stack address. To set the IP address
476  * to original stack address, handle the case where we need to fixup the
477  * relative IP address and/or fixup branch register.
478  */
479 static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs)
480 {
481         unsigned long bundle_addr = ((unsigned long) (&p->opcode.bundle)) & ~0xFULL;
482         unsigned long resume_addr = (unsigned long)p->addr & ~0xFULL;
483         unsigned long template;
484         int slot = ((unsigned long)p->addr & 0xf);
485
486         template = p->opcode.bundle.quad0.template;
487
488         if (slot == 1 && bundle_encoding[template][1] == L)
489                 slot = 2;
490
491         if (p->ainsn.inst_flag) {
492
493                 if (p->ainsn.inst_flag & INST_FLAG_FIX_RELATIVE_IP_ADDR) {
494                         /* Fix relative IP address */
495                         regs->cr_iip = (regs->cr_iip - bundle_addr) + resume_addr;
496                 }
497
498                 if (p->ainsn.inst_flag & INST_FLAG_FIX_BRANCH_REG) {
499                 /*
500                  * Fix target branch register, software convention is
501                  * to use either b0 or b6 or b7, so just checking
502                  * only those registers
503                  */
504                         switch (p->ainsn.target_br_reg) {
505                         case 0:
506                                 if ((regs->b0 == bundle_addr) ||
507                                         (regs->b0 == bundle_addr + 0x10)) {
508                                         regs->b0 = (regs->b0 - bundle_addr) +
509                                                 resume_addr;
510                                 }
511                                 break;
512                         case 6:
513                                 if ((regs->b6 == bundle_addr) ||
514                                         (regs->b6 == bundle_addr + 0x10)) {
515                                         regs->b6 = (regs->b6 - bundle_addr) +
516                                                 resume_addr;
517                                 }
518                                 break;
519                         case 7:
520                                 if ((regs->b7 == bundle_addr) ||
521                                         (regs->b7 == bundle_addr + 0x10)) {
522                                         regs->b7 = (regs->b7 - bundle_addr) +
523                                                 resume_addr;
524                                 }
525                                 break;
526                         } /* end switch */
527                 }
528                 goto turn_ss_off;
529         }
530
531         if (slot == 2) {
532                 if (regs->cr_iip == bundle_addr + 0x10) {
533                         regs->cr_iip = resume_addr + 0x10;
534                 }
535         } else {
536                 if (regs->cr_iip == bundle_addr) {
537                         regs->cr_iip = resume_addr;
538                 }
539         }
540
541 turn_ss_off:
542         /* Turn off Single Step bit */
543         ia64_psr(regs)->ss = 0;
544 }
545
546 static void __kprobes prepare_ss(struct kprobe *p, struct pt_regs *regs)
547 {
548         unsigned long bundle_addr = (unsigned long) &p->opcode.bundle;
549         unsigned long slot = (unsigned long)p->addr & 0xf;
550
551         /* single step inline if break instruction */
552         if (p->ainsn.inst_flag == INST_FLAG_BREAK_INST)
553                 regs->cr_iip = (unsigned long)p->addr & ~0xFULL;
554         else
555                 regs->cr_iip = bundle_addr & ~0xFULL;
556
557         if (slot > 2)
558                 slot = 0;
559
560         ia64_psr(regs)->ri = slot;
561
562         /* turn on single stepping */
563         ia64_psr(regs)->ss = 1;
564 }
565
566 static int __kprobes is_ia64_break_inst(struct pt_regs *regs)
567 {
568         unsigned int slot = ia64_psr(regs)->ri;
569         unsigned int template, major_opcode;
570         unsigned long kprobe_inst;
571         unsigned long *kprobe_addr = (unsigned long *)regs->cr_iip;
572         bundle_t bundle;
573
574         memcpy(&bundle, kprobe_addr, sizeof(bundle_t));
575         template = bundle.quad0.template;
576
577         /* Move to slot 2, if bundle is MLX type and kprobe slot is 1 */
578         if (slot == 1 && bundle_encoding[template][1] == L)
579                 slot++;
580
581         /* Get Kprobe probe instruction at given slot*/
582         get_kprobe_inst(&bundle, slot, &kprobe_inst, &major_opcode);
583
584         /* For break instruction,
585          * Bits 37:40 Major opcode to be zero
586          * Bits 27:32 X6 to be zero
587          * Bits 32:35 X3 to be zero
588          */
589         if (major_opcode || ((kprobe_inst >> 27) & 0x1FF) ) {
590                 /* Not a break instruction */
591                 return 0;
592         }
593
594         /* Is a break instruction */
595         return 1;
596 }
597
598 static int __kprobes pre_kprobes_handler(struct die_args *args)
599 {
600         struct kprobe *p;
601         int ret = 0;
602         struct pt_regs *regs = args->regs;
603         kprobe_opcode_t *addr = (kprobe_opcode_t *)instruction_pointer(regs);
604         struct kprobe_ctlblk *kcb;
605
606         /*
607          * We don't want to be preempted for the entire
608          * duration of kprobe processing
609          */
610         preempt_disable();
611         kcb = get_kprobe_ctlblk();
612
613         /* Handle recursion cases */
614         if (kprobe_running()) {
615                 p = get_kprobe(addr);
616                 if (p) {
617                         if ((kcb->kprobe_status == KPROBE_HIT_SS) &&
618                              (p->ainsn.inst_flag == INST_FLAG_BREAK_INST)) {
619                                 ia64_psr(regs)->ss = 0;
620                                 goto no_kprobe;
621                         }
622                         /* We have reentered the pre_kprobe_handler(), since
623                          * another probe was hit while within the handler.
624                          * We here save the original kprobes variables and
625                          * just single step on the instruction of the new probe
626                          * without calling any user handlers.
627                          */
628                         save_previous_kprobe(kcb);
629                         set_current_kprobe(p, kcb);
630                         kprobes_inc_nmissed_count(p);
631                         prepare_ss(p, regs);
632                         kcb->kprobe_status = KPROBE_REENTER;
633                         return 1;
634                 } else if (args->err == __IA64_BREAK_JPROBE) {
635                         /*
636                          * jprobe instrumented function just completed
637                          */
638                         p = __get_cpu_var(current_kprobe);
639                         if (p->break_handler && p->break_handler(p, regs)) {
640                                 goto ss_probe;
641                         }
642                 } else if (!is_ia64_break_inst(regs)) {
643                         /* The breakpoint instruction was removed by
644                          * another cpu right after we hit, no further
645                          * handling of this interrupt is appropriate
646                          */
647                         ret = 1;
648                         goto no_kprobe;
649                 } else {
650                         /* Not our break */
651                         goto no_kprobe;
652                 }
653         }
654
655         p = get_kprobe(addr);
656         if (!p) {
657                 if (!is_ia64_break_inst(regs)) {
658                         /*
659                          * The breakpoint instruction was removed right
660                          * after we hit it.  Another cpu has removed
661                          * either a probepoint or a debugger breakpoint
662                          * at this address.  In either case, no further
663                          * handling of this interrupt is appropriate.
664                          */
665                         ret = 1;
666
667                 }
668
669                 /* Not one of our break, let kernel handle it */
670                 goto no_kprobe;
671         }
672
673         set_current_kprobe(p, kcb);
674         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
675
676         if (p->pre_handler && p->pre_handler(p, regs))
677                 /*
678                  * Our pre-handler is specifically requesting that we just
679                  * do a return.  This is used for both the jprobe pre-handler
680                  * and the kretprobe trampoline
681                  */
682                 return 1;
683
684 ss_probe:
685         prepare_ss(p, regs);
686         kcb->kprobe_status = KPROBE_HIT_SS;
687         return 1;
688
689 no_kprobe:
690         preempt_enable_no_resched();
691         return ret;
692 }
693
694 static int __kprobes post_kprobes_handler(struct pt_regs *regs)
695 {
696         struct kprobe *cur = kprobe_running();
697         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
698
699         if (!cur)
700                 return 0;
701
702         if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
703                 kcb->kprobe_status = KPROBE_HIT_SSDONE;
704                 cur->post_handler(cur, regs, 0);
705         }
706
707         resume_execution(cur, regs);
708
709         /*Restore back the original saved kprobes variables and continue. */
710         if (kcb->kprobe_status == KPROBE_REENTER) {
711                 restore_previous_kprobe(kcb);
712                 goto out;
713         }
714         reset_current_kprobe();
715
716 out:
717         preempt_enable_no_resched();
718         return 1;
719 }
720
721 static int __kprobes kprobes_fault_handler(struct pt_regs *regs, int trapnr)
722 {
723         struct kprobe *cur = kprobe_running();
724         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
725
726
727         switch(kcb->kprobe_status) {
728         case KPROBE_HIT_SS:
729         case KPROBE_REENTER:
730                 /*
731                  * We are here because the instruction being single
732                  * stepped caused a page fault. We reset the current
733                  * kprobe and the instruction pointer points back to
734                  * the probe address and allow the page fault handler
735                  * to continue as a normal page fault.
736                  */
737                 regs->cr_iip = ((unsigned long)cur->addr) & ~0xFULL;
738                 ia64_psr(regs)->ri = ((unsigned long)cur->addr) & 0xf;
739                 if (kcb->kprobe_status == KPROBE_REENTER)
740                         restore_previous_kprobe(kcb);
741                 else
742                         reset_current_kprobe();
743                 preempt_enable_no_resched();
744                 break;
745         case KPROBE_HIT_ACTIVE:
746         case KPROBE_HIT_SSDONE:
747                 /*
748                  * We increment the nmissed count for accounting,
749                  * we can also use npre/npostfault count for accouting
750                  * these specific fault cases.
751                  */
752                 kprobes_inc_nmissed_count(cur);
753
754                 /*
755                  * We come here because instructions in the pre/post
756                  * handler caused the page_fault, this could happen
757                  * if handler tries to access user space by
758                  * copy_from_user(), get_user() etc. Let the
759                  * user-specified handler try to fix it first.
760                  */
761                 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
762                         return 1;
763
764                 /*
765                  * Let ia64_do_page_fault() fix it.
766                  */
767                 break;
768         default:
769                 break;
770         }
771
772         return 0;
773 }
774
775 int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
776                                        unsigned long val, void *data)
777 {
778         struct die_args *args = (struct die_args *)data;
779         int ret = NOTIFY_DONE;
780
781         if (args->regs && user_mode(args->regs))
782                 return ret;
783
784         switch(val) {
785         case DIE_BREAK:
786                 /* err is break number from ia64_bad_break() */
787                 if (args->err == 0x80200 || args->err == 0x80300 || args->err == 0)
788                         if (pre_kprobes_handler(args))
789                                 ret = NOTIFY_STOP;
790                 break;
791         case DIE_FAULT:
792                 /* err is vector number from ia64_fault() */
793                 if (args->err == 36)
794                         if (post_kprobes_handler(args->regs))
795                                 ret = NOTIFY_STOP;
796                 break;
797         case DIE_PAGE_FAULT:
798                 /* kprobe_running() needs smp_processor_id() */
799                 preempt_disable();
800                 if (kprobe_running() &&
801                         kprobes_fault_handler(args->regs, args->trapnr))
802                         ret = NOTIFY_STOP;
803                 preempt_enable();
804         default:
805                 break;
806         }
807         return ret;
808 }
809
810 struct param_bsp_cfm {
811         unsigned long ip;
812         unsigned long *bsp;
813         unsigned long cfm;
814 };
815
816 static void ia64_get_bsp_cfm(struct unw_frame_info *info, void *arg)
817 {
818         unsigned long ip;
819         struct param_bsp_cfm *lp = arg;
820
821         do {
822                 unw_get_ip(info, &ip);
823                 if (ip == 0)
824                         break;
825                 if (ip == lp->ip) {
826                         unw_get_bsp(info, (unsigned long*)&lp->bsp);
827                         unw_get_cfm(info, (unsigned long*)&lp->cfm);
828                         return;
829                 }
830         } while (unw_unwind(info) >= 0);
831         lp->bsp = 0;
832         lp->cfm = 0;
833         return;
834 }
835
836 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
837 {
838         struct jprobe *jp = container_of(p, struct jprobe, kp);
839         unsigned long addr = ((struct fnptr *)(jp->entry))->ip;
840         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
841         struct param_bsp_cfm pa;
842         int bytes;
843
844         /*
845          * Callee owns the argument space and could overwrite it, eg
846          * tail call optimization. So to be absolutely safe
847          * we save the argument space before transfering the control
848          * to instrumented jprobe function which runs in
849          * the process context
850          */
851         pa.ip = regs->cr_iip;
852         unw_init_running(ia64_get_bsp_cfm, &pa);
853         bytes = (char *)ia64_rse_skip_regs(pa.bsp, pa.cfm & 0x3f)
854                                 - (char *)pa.bsp;
855         memcpy( kcb->jprobes_saved_stacked_regs,
856                 pa.bsp,
857                 bytes );
858         kcb->bsp = pa.bsp;
859         kcb->cfm = pa.cfm;
860
861         /* save architectural state */
862         kcb->jprobe_saved_regs = *regs;
863
864         /* after rfi, execute the jprobe instrumented function */
865         regs->cr_iip = addr & ~0xFULL;
866         ia64_psr(regs)->ri = addr & 0xf;
867         regs->r1 = ((struct fnptr *)(jp->entry))->gp;
868
869         /*
870          * fix the return address to our jprobe_inst_return() function
871          * in the jprobes.S file
872          */
873         regs->b0 = ((struct fnptr *)(jprobe_inst_return))->ip;
874
875         return 1;
876 }
877
878 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
879 {
880         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
881         int bytes;
882
883         /* restoring architectural state */
884         *regs = kcb->jprobe_saved_regs;
885
886         /* restoring the original argument space */
887         flush_register_stack();
888         bytes = (char *)ia64_rse_skip_regs(kcb->bsp, kcb->cfm & 0x3f)
889                                 - (char *)kcb->bsp;
890         memcpy( kcb->bsp,
891                 kcb->jprobes_saved_stacked_regs,
892                 bytes );
893         invalidate_stacked_regs();
894
895         preempt_enable_no_resched();
896         return 1;
897 }
898
899 static struct kprobe trampoline_p = {
900         .pre_handler = trampoline_probe_handler
901 };
902
903 int __init arch_init_kprobes(void)
904 {
905         trampoline_p.addr =
906                 (kprobe_opcode_t *)((struct fnptr *)kretprobe_trampoline)->ip;
907         return register_kprobe(&trampoline_p);
908 }