Pull linus into release branch
[linux-2.6] / arch / sparc / kernel / unaligned.c
1 /* $Id: unaligned.c,v 1.23 2001/12/21 00:54:31 davem Exp $
2  * unaligned.c: Unaligned load/store trap handling with special
3  *              cases for the kernel to do them more quickly.
4  *
5  * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
6  * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
7  */
8
9
10 #include <linux/kernel.h>
11 #include <linux/sched.h>
12 #include <linux/mm.h>
13 #include <linux/module.h>
14 #include <asm/ptrace.h>
15 #include <asm/processor.h>
16 #include <asm/system.h>
17 #include <asm/uaccess.h>
18 #include <linux/smp.h>
19 #include <linux/smp_lock.h>
20
21 /* #define DEBUG_MNA */
22
23 enum direction {
24         load,    /* ld, ldd, ldh, ldsh */
25         store,   /* st, std, sth, stsh */
26         both,    /* Swap, ldstub, etc. */
27         fpload,
28         fpstore,
29         invalid,
30 };
31
32 #ifdef DEBUG_MNA
33 static char *dirstrings[] = {
34   "load", "store", "both", "fpload", "fpstore", "invalid"
35 };
36 #endif
37
38 static inline enum direction decode_direction(unsigned int insn)
39 {
40         unsigned long tmp = (insn >> 21) & 1;
41
42         if(!tmp)
43                 return load;
44         else {
45                 if(((insn>>19)&0x3f) == 15)
46                         return both;
47                 else
48                         return store;
49         }
50 }
51
52 /* 8 = double-word, 4 = word, 2 = half-word */
53 static inline int decode_access_size(unsigned int insn)
54 {
55         insn = (insn >> 19) & 3;
56
57         if(!insn)
58                 return 4;
59         else if(insn == 3)
60                 return 8;
61         else if(insn == 2)
62                 return 2;
63         else {
64                 printk("Impossible unaligned trap. insn=%08x\n", insn);
65                 die_if_kernel("Byte sized unaligned access?!?!", current->thread.kregs);
66                 return 4; /* just to keep gcc happy. */
67         }
68 }
69
70 /* 0x400000 = signed, 0 = unsigned */
71 static inline int decode_signedness(unsigned int insn)
72 {
73         return (insn & 0x400000);
74 }
75
76 static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2,
77                                        unsigned int rd)
78 {
79         if(rs2 >= 16 || rs1 >= 16 || rd >= 16) {
80                 /* Wheee... */
81                 __asm__ __volatile__("save %sp, -0x40, %sp\n\t"
82                                      "save %sp, -0x40, %sp\n\t"
83                                      "save %sp, -0x40, %sp\n\t"
84                                      "save %sp, -0x40, %sp\n\t"
85                                      "save %sp, -0x40, %sp\n\t"
86                                      "save %sp, -0x40, %sp\n\t"
87                                      "save %sp, -0x40, %sp\n\t"
88                                      "restore; restore; restore; restore;\n\t"
89                                      "restore; restore; restore;\n\t");
90         }
91 }
92
93 static inline int sign_extend_imm13(int imm)
94 {
95         return imm << 19 >> 19;
96 }
97
98 static inline unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs)
99 {
100         struct reg_window *win;
101
102         if(reg < 16)
103                 return (!reg ? 0 : regs->u_regs[reg]);
104
105         /* Ho hum, the slightly complicated case. */
106         win = (struct reg_window *) regs->u_regs[UREG_FP];
107         return win->locals[reg - 16]; /* yes, I know what this does... */
108 }
109
110 static inline unsigned long safe_fetch_reg(unsigned int reg, struct pt_regs *regs)
111 {
112         struct reg_window __user *win;
113         unsigned long ret;
114
115         if (reg < 16)
116                 return (!reg ? 0 : regs->u_regs[reg]);
117
118         /* Ho hum, the slightly complicated case. */
119         win = (struct reg_window __user *) regs->u_regs[UREG_FP];
120
121         if ((unsigned long)win & 3)
122                 return -1;
123
124         if (get_user(ret, &win->locals[reg - 16]))
125                 return -1;
126
127         return ret;
128 }
129
130 static inline unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs)
131 {
132         struct reg_window *win;
133
134         if(reg < 16)
135                 return &regs->u_regs[reg];
136         win = (struct reg_window *) regs->u_regs[UREG_FP];
137         return &win->locals[reg - 16];
138 }
139
140 static unsigned long compute_effective_address(struct pt_regs *regs,
141                                                unsigned int insn)
142 {
143         unsigned int rs1 = (insn >> 14) & 0x1f;
144         unsigned int rs2 = insn & 0x1f;
145         unsigned int rd = (insn >> 25) & 0x1f;
146
147         if(insn & 0x2000) {
148                 maybe_flush_windows(rs1, 0, rd);
149                 return (fetch_reg(rs1, regs) + sign_extend_imm13(insn));
150         } else {
151                 maybe_flush_windows(rs1, rs2, rd);
152                 return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs));
153         }
154 }
155
156 unsigned long safe_compute_effective_address(struct pt_regs *regs,
157                                              unsigned int insn)
158 {
159         unsigned int rs1 = (insn >> 14) & 0x1f;
160         unsigned int rs2 = insn & 0x1f;
161         unsigned int rd = (insn >> 25) & 0x1f;
162
163         if(insn & 0x2000) {
164                 maybe_flush_windows(rs1, 0, rd);
165                 return (safe_fetch_reg(rs1, regs) + sign_extend_imm13(insn));
166         } else {
167                 maybe_flush_windows(rs1, rs2, rd);
168                 return (safe_fetch_reg(rs1, regs) + safe_fetch_reg(rs2, regs));
169         }
170 }
171
172 /* This is just to make gcc think panic does return... */
173 static void unaligned_panic(char *str)
174 {
175         panic(str);
176 }
177
178 #define do_integer_load(dest_reg, size, saddr, is_signed, errh) ({              \
179 __asm__ __volatile__ (                                                          \
180         "cmp    %1, 8\n\t"                                                      \
181         "be     9f\n\t"                                                         \
182         " cmp   %1, 4\n\t"                                                      \
183         "be     6f\n"                                                           \
184 "4:\t"  " ldub  [%2], %%l1\n"                                                   \
185 "5:\t"  "ldub   [%2 + 1], %%l2\n\t"                                             \
186         "sll    %%l1, 8, %%l1\n\t"                                              \
187         "tst    %3\n\t"                                                         \
188         "be     3f\n\t"                                                         \
189         " add   %%l1, %%l2, %%l1\n\t"                                           \
190         "sll    %%l1, 16, %%l1\n\t"                                             \
191         "sra    %%l1, 16, %%l1\n"                                               \
192 "3:\t"  "b      0f\n\t"                                                         \
193         " st    %%l1, [%0]\n"                                                   \
194 "6:\t"  "ldub   [%2 + 1], %%l2\n\t"                                             \
195         "sll    %%l1, 24, %%l1\n"                                               \
196 "7:\t"  "ldub   [%2 + 2], %%g7\n\t"                                             \
197         "sll    %%l2, 16, %%l2\n"                                               \
198 "8:\t"  "ldub   [%2 + 3], %%g1\n\t"                                             \
199         "sll    %%g7, 8, %%g7\n\t"                                              \
200         "or     %%l1, %%l2, %%l1\n\t"                                           \
201         "or     %%g7, %%g1, %%g7\n\t"                                           \
202         "or     %%l1, %%g7, %%l1\n\t"                                           \
203         "b      0f\n\t"                                                         \
204         " st    %%l1, [%0]\n"                                                   \
205 "9:\t"  "ldub   [%2], %%l1\n"                                                   \
206 "10:\t" "ldub   [%2 + 1], %%l2\n\t"                                             \
207         "sll    %%l1, 24, %%l1\n"                                               \
208 "11:\t" "ldub   [%2 + 2], %%g7\n\t"                                             \
209         "sll    %%l2, 16, %%l2\n"                                               \
210 "12:\t" "ldub   [%2 + 3], %%g1\n\t"                                             \
211         "sll    %%g7, 8, %%g7\n\t"                                              \
212         "or     %%l1, %%l2, %%l1\n\t"                                           \
213         "or     %%g7, %%g1, %%g7\n\t"                                           \
214         "or     %%l1, %%g7, %%g7\n"                                             \
215 "13:\t" "ldub   [%2 + 4], %%l1\n\t"                                             \
216         "st     %%g7, [%0]\n"                                                   \
217 "14:\t" "ldub   [%2 + 5], %%l2\n\t"                                             \
218         "sll    %%l1, 24, %%l1\n"                                               \
219 "15:\t" "ldub   [%2 + 6], %%g7\n\t"                                             \
220         "sll    %%l2, 16, %%l2\n"                                               \
221 "16:\t" "ldub   [%2 + 7], %%g1\n\t"                                             \
222         "sll    %%g7, 8, %%g7\n\t"                                              \
223         "or     %%l1, %%l2, %%l1\n\t"                                           \
224         "or     %%g7, %%g1, %%g7\n\t"                                           \
225         "or     %%l1, %%g7, %%g7\n\t"                                           \
226         "st     %%g7, [%0 + 4]\n"                                               \
227 "0:\n\n\t"                                                                      \
228         ".section __ex_table,#alloc\n\t"                                        \
229         ".word  4b, " #errh "\n\t"                                              \
230         ".word  5b, " #errh "\n\t"                                              \
231         ".word  6b, " #errh "\n\t"                                              \
232         ".word  7b, " #errh "\n\t"                                              \
233         ".word  8b, " #errh "\n\t"                                              \
234         ".word  9b, " #errh "\n\t"                                              \
235         ".word  10b, " #errh "\n\t"                                             \
236         ".word  11b, " #errh "\n\t"                                             \
237         ".word  12b, " #errh "\n\t"                                             \
238         ".word  13b, " #errh "\n\t"                                             \
239         ".word  14b, " #errh "\n\t"                                             \
240         ".word  15b, " #errh "\n\t"                                             \
241         ".word  16b, " #errh "\n\n\t"                                           \
242         ".previous\n\t"                                                         \
243         : : "r" (dest_reg), "r" (size), "r" (saddr), "r" (is_signed)            \
244         : "l1", "l2", "g7", "g1", "cc");                                        \
245 })
246         
247 #define store_common(dst_addr, size, src_val, errh) ({                          \
248 __asm__ __volatile__ (                                                          \
249         "ld     [%2], %%l1\n"                                                   \
250         "cmp    %1, 2\n\t"                                                      \
251         "be     2f\n\t"                                                         \
252         " cmp   %1, 4\n\t"                                                      \
253         "be     1f\n\t"                                                         \
254         " srl   %%l1, 24, %%l2\n\t"                                             \
255         "srl    %%l1, 16, %%g7\n"                                               \
256 "4:\t"  "stb    %%l2, [%0]\n\t"                                                 \
257         "srl    %%l1, 8, %%l2\n"                                                \
258 "5:\t"  "stb    %%g7, [%0 + 1]\n\t"                                             \
259         "ld     [%2 + 4], %%g7\n"                                               \
260 "6:\t"  "stb    %%l2, [%0 + 2]\n\t"                                             \
261         "srl    %%g7, 24, %%l2\n"                                               \
262 "7:\t"  "stb    %%l1, [%0 + 3]\n\t"                                             \
263         "srl    %%g7, 16, %%l1\n"                                               \
264 "8:\t"  "stb    %%l2, [%0 + 4]\n\t"                                             \
265         "srl    %%g7, 8, %%l2\n"                                                \
266 "9:\t"  "stb    %%l1, [%0 + 5]\n"                                               \
267 "10:\t" "stb    %%l2, [%0 + 6]\n\t"                                             \
268         "b      0f\n"                                                           \
269 "11:\t" " stb   %%g7, [%0 + 7]\n"                                               \
270 "1:\t"  "srl    %%l1, 16, %%g7\n"                                               \
271 "12:\t" "stb    %%l2, [%0]\n\t"                                                 \
272         "srl    %%l1, 8, %%l2\n"                                                \
273 "13:\t" "stb    %%g7, [%0 + 1]\n"                                               \
274 "14:\t" "stb    %%l2, [%0 + 2]\n\t"                                             \
275         "b      0f\n"                                                           \
276 "15:\t" " stb   %%l1, [%0 + 3]\n"                                               \
277 "2:\t"  "srl    %%l1, 8, %%l2\n"                                                \
278 "16:\t" "stb    %%l2, [%0]\n"                                                   \
279 "17:\t" "stb    %%l1, [%0 + 1]\n"                                               \
280 "0:\n\n\t"                                                                      \
281         ".section __ex_table,#alloc\n\t"                                        \
282         ".word  4b, " #errh "\n\t"                                              \
283         ".word  5b, " #errh "\n\t"                                              \
284         ".word  6b, " #errh "\n\t"                                              \
285         ".word  7b, " #errh "\n\t"                                              \
286         ".word  8b, " #errh "\n\t"                                              \
287         ".word  9b, " #errh "\n\t"                                              \
288         ".word  10b, " #errh "\n\t"                                             \
289         ".word  11b, " #errh "\n\t"                                             \
290         ".word  12b, " #errh "\n\t"                                             \
291         ".word  13b, " #errh "\n\t"                                             \
292         ".word  14b, " #errh "\n\t"                                             \
293         ".word  15b, " #errh "\n\t"                                             \
294         ".word  16b, " #errh "\n\t"                                             \
295         ".word  17b, " #errh "\n\n\t"                                           \
296         ".previous\n\t"                                                         \
297         : : "r" (dst_addr), "r" (size), "r" (src_val)                           \
298         : "l1", "l2", "g7", "g1", "cc");                                        \
299 })
300
301 #define do_integer_store(reg_num, size, dst_addr, regs, errh) ({                \
302         unsigned long *src_val;                                                 \
303         static unsigned long zero[2] = { 0, };                                  \
304                                                                                 \
305         if (reg_num) src_val = fetch_reg_addr(reg_num, regs);                   \
306         else {                                                                  \
307                 src_val = &zero[0];                                             \
308                 if (size == 8)                                                  \
309                         zero[1] = fetch_reg(1, regs);                           \
310         }                                                                       \
311         store_common(dst_addr, size, src_val, errh);                            \
312 })
313
314 extern void smp_capture(void);
315 extern void smp_release(void);
316
317 #define do_atomic(srcdest_reg, mem, errh) ({                                    \
318         unsigned long flags, tmp;                                               \
319                                                                                 \
320         smp_capture();                                                          \
321         local_irq_save(flags);                                                  \
322         tmp = *srcdest_reg;                                                     \
323         do_integer_load(srcdest_reg, 4, mem, 0, errh);                          \
324         store_common(mem, 4, &tmp, errh);                                       \
325         local_irq_restore(flags);                                               \
326         smp_release();                                                          \
327 })
328
329 static inline void advance(struct pt_regs *regs)
330 {
331         regs->pc   = regs->npc;
332         regs->npc += 4;
333 }
334
335 static inline int floating_point_load_or_store_p(unsigned int insn)
336 {
337         return (insn >> 24) & 1;
338 }
339
340 static inline int ok_for_kernel(unsigned int insn)
341 {
342         return !floating_point_load_or_store_p(insn);
343 }
344
345 void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn) __asm__ ("kernel_mna_trap_fault");
346
347 void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn)
348 {
349         unsigned long g2 = regs->u_regs [UREG_G2];
350         unsigned long fixup = search_extables_range(regs->pc, &g2);
351
352         if (!fixup) {
353                 unsigned long address = compute_effective_address(regs, insn);
354                 if(address < PAGE_SIZE) {
355                         printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference in mna handler");
356                 } else
357                         printk(KERN_ALERT "Unable to handle kernel paging request in mna handler");
358                 printk(KERN_ALERT " at virtual address %08lx\n",address);
359                 printk(KERN_ALERT "current->{mm,active_mm}->context = %08lx\n",
360                         (current->mm ? current->mm->context :
361                         current->active_mm->context));
362                 printk(KERN_ALERT "current->{mm,active_mm}->pgd = %08lx\n",
363                         (current->mm ? (unsigned long) current->mm->pgd :
364                         (unsigned long) current->active_mm->pgd));
365                 die_if_kernel("Oops", regs);
366                 /* Not reached */
367         }
368         regs->pc = fixup;
369         regs->npc = regs->pc + 4;
370         regs->u_regs [UREG_G2] = g2;
371 }
372
373 asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn)
374 {
375         enum direction dir = decode_direction(insn);
376         int size = decode_access_size(insn);
377
378         if(!ok_for_kernel(insn) || dir == both) {
379                 printk("Unsupported unaligned load/store trap for kernel at <%08lx>.\n",
380                        regs->pc);
381                 unaligned_panic("Wheee. Kernel does fpu/atomic unaligned load/store.");
382
383                 __asm__ __volatile__ ("\n"
384 "kernel_unaligned_trap_fault:\n\t"
385                 "mov    %0, %%o0\n\t"
386                 "call   kernel_mna_trap_fault\n\t"
387                 " mov   %1, %%o1\n\t"
388                 :
389                 : "r" (regs), "r" (insn)
390                 : "o0", "o1", "o2", "o3", "o4", "o5", "o7",
391                   "g1", "g2", "g3", "g4", "g5", "g7", "cc");
392         } else {
393                 unsigned long addr = compute_effective_address(regs, insn);
394
395 #ifdef DEBUG_MNA
396                 printk("KMNA: pc=%08lx [dir=%s addr=%08lx size=%d] retpc[%08lx]\n",
397                        regs->pc, dirstrings[dir], addr, size, regs->u_regs[UREG_RETPC]);
398 #endif
399                 switch(dir) {
400                 case load:
401                         do_integer_load(fetch_reg_addr(((insn>>25)&0x1f), regs),
402                                         size, (unsigned long *) addr,
403                                         decode_signedness(insn),
404                                         kernel_unaligned_trap_fault);
405                         break;
406
407                 case store:
408                         do_integer_store(((insn>>25)&0x1f), size,
409                                          (unsigned long *) addr, regs,
410                                          kernel_unaligned_trap_fault);
411                         break;
412 #if 0 /* unsupported */
413                 case both:
414                         do_atomic(fetch_reg_addr(((insn>>25)&0x1f), regs),
415                                   (unsigned long *) addr,
416                                   kernel_unaligned_trap_fault);
417                         break;
418 #endif
419                 default:
420                         panic("Impossible kernel unaligned trap.");
421                         /* Not reached... */
422                 }
423                 advance(regs);
424         }
425 }
426
427 static inline int ok_for_user(struct pt_regs *regs, unsigned int insn,
428                               enum direction dir)
429 {
430         unsigned int reg;
431         int check = (dir == load) ? VERIFY_READ : VERIFY_WRITE;
432         int size = ((insn >> 19) & 3) == 3 ? 8 : 4;
433
434         if ((regs->pc | regs->npc) & 3)
435                 return 0;
436
437         /* Must access_ok() in all the necessary places. */
438 #define WINREG_ADDR(regnum) \
439         ((void __user *)(((unsigned long *)regs->u_regs[UREG_FP])+(regnum)))
440
441         reg = (insn >> 25) & 0x1f;
442         if (reg >= 16) {
443                 if (!access_ok(check, WINREG_ADDR(reg - 16), size))
444                         return -EFAULT;
445         }
446         reg = (insn >> 14) & 0x1f;
447         if (reg >= 16) {
448                 if (!access_ok(check, WINREG_ADDR(reg - 16), size))
449                         return -EFAULT;
450         }
451         if (!(insn & 0x2000)) {
452                 reg = (insn & 0x1f);
453                 if (reg >= 16) {
454                         if (!access_ok(check, WINREG_ADDR(reg - 16), size))
455                                 return -EFAULT;
456                 }
457         }
458 #undef WINREG_ADDR
459         return 0;
460 }
461
462 void user_mna_trap_fault(struct pt_regs *regs, unsigned int insn) __asm__ ("user_mna_trap_fault");
463
464 void user_mna_trap_fault(struct pt_regs *regs, unsigned int insn)
465 {
466         siginfo_t info;
467
468         info.si_signo = SIGBUS;
469         info.si_errno = 0;
470         info.si_code = BUS_ADRALN;
471         info.si_addr = (void __user *)safe_compute_effective_address(regs, insn);
472         info.si_trapno = 0;
473         send_sig_info(SIGBUS, &info, current);
474 }
475
476 asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn)
477 {
478         enum direction dir;
479
480         lock_kernel();
481         if(!(current->thread.flags & SPARC_FLAG_UNALIGNED) ||
482            (((insn >> 30) & 3) != 3))
483                 goto kill_user;
484         dir = decode_direction(insn);
485         if(!ok_for_user(regs, insn, dir)) {
486                 goto kill_user;
487         } else {
488                 int size = decode_access_size(insn);
489                 unsigned long addr;
490
491                 if(floating_point_load_or_store_p(insn)) {
492                         printk("User FPU load/store unaligned unsupported.\n");
493                         goto kill_user;
494                 }
495
496                 addr = compute_effective_address(regs, insn);
497                 switch(dir) {
498                 case load:
499                         do_integer_load(fetch_reg_addr(((insn>>25)&0x1f), regs),
500                                         size, (unsigned long *) addr,
501                                         decode_signedness(insn),
502                                         user_unaligned_trap_fault);
503                         break;
504
505                 case store:
506                         do_integer_store(((insn>>25)&0x1f), size,
507                                          (unsigned long *) addr, regs,
508                                          user_unaligned_trap_fault);
509                         break;
510
511                 case both:
512 #if 0 /* unsupported */
513                         do_atomic(fetch_reg_addr(((insn>>25)&0x1f), regs),
514                                   (unsigned long *) addr,
515                                   user_unaligned_trap_fault);
516 #else
517                         /*
518                          * This was supported in 2.4. However, we question
519                          * the value of SWAP instruction across word boundaries.
520                          */
521                         printk("Unaligned SWAP unsupported.\n");
522                         goto kill_user;
523 #endif
524                         break;
525
526                 default:
527                         unaligned_panic("Impossible user unaligned trap.");
528
529                         __asm__ __volatile__ ("\n"
530 "user_unaligned_trap_fault:\n\t"
531                         "mov    %0, %%o0\n\t"
532                         "call   user_mna_trap_fault\n\t"
533                         " mov   %1, %%o1\n\t"
534                         :
535                         : "r" (regs), "r" (insn)
536                         : "o0", "o1", "o2", "o3", "o4", "o5", "o7",
537                           "g1", "g2", "g3", "g4", "g5", "g7", "cc");
538                         goto out;
539                 }
540                 advance(regs);
541                 goto out;
542         }
543
544 kill_user:
545         user_mna_trap_fault(regs, insn);
546 out:
547         unlock_kernel();
548 }