Merge branch 'for-2.6.30' into for-2.6.31
[linux-2.6] / arch / powerpc / lib / copyuser_64.S
1 /*
2  * Copyright (C) 2002 Paul Mackerras, IBM Corp.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version
7  * 2 of the License, or (at your option) any later version.
8  */
9 #include <asm/processor.h>
10 #include <asm/ppc_asm.h>
11
12         .align  7
13 _GLOBAL(__copy_tofrom_user)
14         /* first check for a whole page copy on a page boundary */
15         cmpldi  cr1,r5,16
16         cmpdi   cr6,r5,4096
17         or      r0,r3,r4
18         neg     r6,r3           /* LS 3 bits = # bytes to 8-byte dest bdry */
19         andi.   r0,r0,4095
20         std     r3,-24(r1)
21         crand   cr0*4+2,cr0*4+2,cr6*4+2
22         std     r4,-16(r1)
23         std     r5,-8(r1)
24         dcbt    0,r4
25         beq     .Lcopy_page_4K
26         andi.   r6,r6,7
27         PPC_MTOCRF      0x01,r5
28         blt     cr1,.Lshort_copy
29 /* Below we want to nop out the bne if we're on a CPU that has the
30  * CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
31  * cleared.
32  * At the time of writing the only CPU that has this combination of bits
33  * set is Power6.
34  */
35 BEGIN_FTR_SECTION
36         nop
37 FTR_SECTION_ELSE
38         bne     .Ldst_unaligned
39 ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \
40                     CPU_FTR_UNALIGNED_LD_STD)
41 .Ldst_aligned:
42         addi    r3,r3,-16
43 BEGIN_FTR_SECTION
44         andi.   r0,r4,7
45         bne     .Lsrc_unaligned
46 END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
47         srdi    r7,r5,4
48 20:     ld      r9,0(r4)
49         addi    r4,r4,-8
50         mtctr   r7
51         andi.   r5,r5,7
52         bf      cr7*4+0,22f
53         addi    r3,r3,8
54         addi    r4,r4,8
55         mr      r8,r9
56         blt     cr1,72f
57 21:     ld      r9,8(r4)
58 70:     std     r8,8(r3)
59 22:     ldu     r8,16(r4)
60 71:     stdu    r9,16(r3)
61         bdnz    21b
62 72:     std     r8,8(r3)
63         beq+    3f
64         addi    r3,r3,16
65 .Ldo_tail:
66         bf      cr7*4+1,1f
67 23:     lwz     r9,8(r4)
68         addi    r4,r4,4
69 73:     stw     r9,0(r3)
70         addi    r3,r3,4
71 1:      bf      cr7*4+2,2f
72 44:     lhz     r9,8(r4)
73         addi    r4,r4,2
74 74:     sth     r9,0(r3)
75         addi    r3,r3,2
76 2:      bf      cr7*4+3,3f
77 45:     lbz     r9,8(r4)
78 75:     stb     r9,0(r3)
79 3:      li      r3,0
80         blr
81
82 .Lsrc_unaligned:
83         srdi    r6,r5,3
84         addi    r5,r5,-16
85         subf    r4,r0,r4
86         srdi    r7,r5,4
87         sldi    r10,r0,3
88         cmpldi  cr6,r6,3
89         andi.   r5,r5,7
90         mtctr   r7
91         subfic  r11,r10,64
92         add     r5,r5,r0
93         bt      cr7*4+0,28f
94
95 24:     ld      r9,0(r4)        /* 3+2n loads, 2+2n stores */
96 25:     ld      r0,8(r4)
97         sld     r6,r9,r10
98 26:     ldu     r9,16(r4)
99         srd     r7,r0,r11
100         sld     r8,r0,r10
101         or      r7,r7,r6
102         blt     cr6,79f
103 27:     ld      r0,8(r4)
104         b       2f
105
106 28:     ld      r0,0(r4)        /* 4+2n loads, 3+2n stores */
107 29:     ldu     r9,8(r4)
108         sld     r8,r0,r10
109         addi    r3,r3,-8
110         blt     cr6,5f
111 30:     ld      r0,8(r4)
112         srd     r12,r9,r11
113         sld     r6,r9,r10
114 31:     ldu     r9,16(r4)
115         or      r12,r8,r12
116         srd     r7,r0,r11
117         sld     r8,r0,r10
118         addi    r3,r3,16
119         beq     cr6,78f
120
121 1:      or      r7,r7,r6
122 32:     ld      r0,8(r4)
123 76:     std     r12,8(r3)
124 2:      srd     r12,r9,r11
125         sld     r6,r9,r10
126 33:     ldu     r9,16(r4)
127         or      r12,r8,r12
128 77:     stdu    r7,16(r3)
129         srd     r7,r0,r11
130         sld     r8,r0,r10
131         bdnz    1b
132
133 78:     std     r12,8(r3)
134         or      r7,r7,r6
135 79:     std     r7,16(r3)
136 5:      srd     r12,r9,r11
137         or      r12,r8,r12
138 80:     std     r12,24(r3)
139         bne     6f
140         li      r3,0
141         blr
142 6:      cmpwi   cr1,r5,8
143         addi    r3,r3,32
144         sld     r9,r9,r10
145         ble     cr1,7f
146 34:     ld      r0,8(r4)
147         srd     r7,r0,r11
148         or      r9,r7,r9
149 7:
150         bf      cr7*4+1,1f
151         rotldi  r9,r9,32
152 94:     stw     r9,0(r3)
153         addi    r3,r3,4
154 1:      bf      cr7*4+2,2f
155         rotldi  r9,r9,16
156 95:     sth     r9,0(r3)
157         addi    r3,r3,2
158 2:      bf      cr7*4+3,3f
159         rotldi  r9,r9,8
160 96:     stb     r9,0(r3)
161 3:      li      r3,0
162         blr
163
164 .Ldst_unaligned:
165         PPC_MTOCRF      0x01,r6         /* put #bytes to 8B bdry into cr7 */
166         subf    r5,r6,r5
167         li      r7,0
168         cmpldi  cr1,r5,16
169         bf      cr7*4+3,1f
170 35:     lbz     r0,0(r4)
171 81:     stb     r0,0(r3)
172         addi    r7,r7,1
173 1:      bf      cr7*4+2,2f
174 36:     lhzx    r0,r7,r4
175 82:     sthx    r0,r7,r3
176         addi    r7,r7,2
177 2:      bf      cr7*4+1,3f
178 37:     lwzx    r0,r7,r4
179 83:     stwx    r0,r7,r3
180 3:      PPC_MTOCRF      0x01,r5
181         add     r4,r6,r4
182         add     r3,r6,r3
183         b       .Ldst_aligned
184
185 .Lshort_copy:
186         bf      cr7*4+0,1f
187 38:     lwz     r0,0(r4)
188 39:     lwz     r9,4(r4)
189         addi    r4,r4,8
190 84:     stw     r0,0(r3)
191 85:     stw     r9,4(r3)
192         addi    r3,r3,8
193 1:      bf      cr7*4+1,2f
194 40:     lwz     r0,0(r4)
195         addi    r4,r4,4
196 86:     stw     r0,0(r3)
197         addi    r3,r3,4
198 2:      bf      cr7*4+2,3f
199 41:     lhz     r0,0(r4)
200         addi    r4,r4,2
201 87:     sth     r0,0(r3)
202         addi    r3,r3,2
203 3:      bf      cr7*4+3,4f
204 42:     lbz     r0,0(r4)
205 88:     stb     r0,0(r3)
206 4:      li      r3,0
207         blr
208
209 /*
210  * exception handlers follow
211  * we have to return the number of bytes not copied
212  * for an exception on a load, we set the rest of the destination to 0
213  */
214
215 136:
216 137:
217         add     r3,r3,r7
218         b       1f
219 130:
220 131:
221         addi    r3,r3,8
222 120:
223 122:
224 124:
225 125:
226 126:
227 127:
228 128:
229 129:
230 133:
231         addi    r3,r3,8
232 121:
233 132:
234         addi    r3,r3,8
235 134:
236 135:
237 138:
238 139:
239 140:
240 141:
241 142:
242 123:
243 144:
244 145:
245
246 /*
247  * here we have had a fault on a load and r3 points to the first
248  * unmodified byte of the destination
249  */
250 1:      ld      r6,-24(r1)
251         ld      r4,-16(r1)
252         ld      r5,-8(r1)
253         subf    r6,r6,r3
254         add     r4,r4,r6
255         subf    r5,r6,r5        /* #bytes left to go */
256
257 /*
258  * first see if we can copy any more bytes before hitting another exception
259  */
260         mtctr   r5
261 43:     lbz     r0,0(r4)
262         addi    r4,r4,1
263 89:     stb     r0,0(r3)
264         addi    r3,r3,1
265         bdnz    43b
266         li      r3,0            /* huh? all copied successfully this time? */
267         blr
268
269 /*
270  * here we have trapped again, need to clear ctr bytes starting at r3
271  */
272 143:    mfctr   r5
273         li      r0,0
274         mr      r4,r3
275         mr      r3,r5           /* return the number of bytes not copied */
276 1:      andi.   r9,r4,7
277         beq     3f
278 90:     stb     r0,0(r4)
279         addic.  r5,r5,-1
280         addi    r4,r4,1
281         bne     1b
282         blr
283 3:      cmpldi  cr1,r5,8
284         srdi    r9,r5,3
285         andi.   r5,r5,7
286         blt     cr1,93f
287         mtctr   r9
288 91:     std     r0,0(r4)
289         addi    r4,r4,8
290         bdnz    91b
291 93:     beqlr
292         mtctr   r5      
293 92:     stb     r0,0(r4)
294         addi    r4,r4,1
295         bdnz    92b
296         blr
297
298 /*
299  * exception handlers for stores: we just need to work
300  * out how many bytes weren't copied
301  */
302 182:
303 183:
304         add     r3,r3,r7
305         b       1f
306 180:
307         addi    r3,r3,8
308 171:
309 177:
310         addi    r3,r3,8
311 170:
312 172:
313 176:
314 178:
315         addi    r3,r3,4
316 185:
317         addi    r3,r3,4
318 173:
319 174:
320 175:
321 179:
322 181:
323 184:
324 186:
325 187:
326 188:
327 189:    
328 194:
329 195:
330 196:
331 1:
332         ld      r6,-24(r1)
333         ld      r5,-8(r1)
334         add     r6,r6,r5
335         subf    r3,r3,r6        /* #bytes not copied */
336 190:
337 191:
338 192:
339         blr                     /* #bytes not copied in r3 */
340
341         .section __ex_table,"a"
342         .align  3
343         .llong  20b,120b
344         .llong  21b,121b
345         .llong  70b,170b
346         .llong  22b,122b
347         .llong  71b,171b
348         .llong  72b,172b
349         .llong  23b,123b
350         .llong  73b,173b
351         .llong  44b,144b
352         .llong  74b,174b
353         .llong  45b,145b
354         .llong  75b,175b
355         .llong  24b,124b
356         .llong  25b,125b
357         .llong  26b,126b
358         .llong  27b,127b
359         .llong  28b,128b
360         .llong  29b,129b
361         .llong  30b,130b
362         .llong  31b,131b
363         .llong  32b,132b
364         .llong  76b,176b
365         .llong  33b,133b
366         .llong  77b,177b
367         .llong  78b,178b
368         .llong  79b,179b
369         .llong  80b,180b
370         .llong  34b,134b
371         .llong  94b,194b
372         .llong  95b,195b
373         .llong  96b,196b
374         .llong  35b,135b
375         .llong  81b,181b
376         .llong  36b,136b
377         .llong  82b,182b
378         .llong  37b,137b
379         .llong  83b,183b
380         .llong  38b,138b
381         .llong  39b,139b
382         .llong  84b,184b
383         .llong  85b,185b
384         .llong  40b,140b
385         .llong  86b,186b
386         .llong  41b,141b
387         .llong  87b,187b
388         .llong  42b,142b
389         .llong  88b,188b
390         .llong  43b,143b
391         .llong  89b,189b
392         .llong  90b,190b
393         .llong  91b,191b
394         .llong  92b,192b
395         
396         .text
397
398 /*
399  * Routine to copy a whole page of data, optimized for POWER4.
400  * On POWER4 it is more than 50% faster than the simple loop
401  * above (following the .Ldst_aligned label) but it runs slightly
402  * slower on POWER3.
403  */
404 .Lcopy_page_4K:
405         std     r31,-32(1)
406         std     r30,-40(1)
407         std     r29,-48(1)
408         std     r28,-56(1)
409         std     r27,-64(1)
410         std     r26,-72(1)
411         std     r25,-80(1)
412         std     r24,-88(1)
413         std     r23,-96(1)
414         std     r22,-104(1)
415         std     r21,-112(1)
416         std     r20,-120(1)
417         li      r5,4096/32 - 1
418         addi    r3,r3,-8
419         li      r0,5
420 0:      addi    r5,r5,-24
421         mtctr   r0
422 20:     ld      r22,640(4)
423 21:     ld      r21,512(4)
424 22:     ld      r20,384(4)
425 23:     ld      r11,256(4)
426 24:     ld      r9,128(4)
427 25:     ld      r7,0(4)
428 26:     ld      r25,648(4)
429 27:     ld      r24,520(4)
430 28:     ld      r23,392(4)
431 29:     ld      r10,264(4)
432 30:     ld      r8,136(4)
433 31:     ldu     r6,8(4)
434         cmpwi   r5,24
435 1:
436 32:     std     r22,648(3)
437 33:     std     r21,520(3)
438 34:     std     r20,392(3)
439 35:     std     r11,264(3)
440 36:     std     r9,136(3)
441 37:     std     r7,8(3)
442 38:     ld      r28,648(4)
443 39:     ld      r27,520(4)
444 40:     ld      r26,392(4)
445 41:     ld      r31,264(4)
446 42:     ld      r30,136(4)
447 43:     ld      r29,8(4)
448 44:     std     r25,656(3)
449 45:     std     r24,528(3)
450 46:     std     r23,400(3)
451 47:     std     r10,272(3)
452 48:     std     r8,144(3)
453 49:     std     r6,16(3)
454 50:     ld      r22,656(4)
455 51:     ld      r21,528(4)
456 52:     ld      r20,400(4)
457 53:     ld      r11,272(4)
458 54:     ld      r9,144(4)
459 55:     ld      r7,16(4)
460 56:     std     r28,664(3)
461 57:     std     r27,536(3)
462 58:     std     r26,408(3)
463 59:     std     r31,280(3)
464 60:     std     r30,152(3)
465 61:     stdu    r29,24(3)
466 62:     ld      r25,664(4)
467 63:     ld      r24,536(4)
468 64:     ld      r23,408(4)
469 65:     ld      r10,280(4)
470 66:     ld      r8,152(4)
471 67:     ldu     r6,24(4)
472         bdnz    1b
473 68:     std     r22,648(3)
474 69:     std     r21,520(3)
475 70:     std     r20,392(3)
476 71:     std     r11,264(3)
477 72:     std     r9,136(3)
478 73:     std     r7,8(3)
479 74:     addi    r4,r4,640
480 75:     addi    r3,r3,648
481         bge     0b
482         mtctr   r5
483 76:     ld      r7,0(4)
484 77:     ld      r8,8(4)
485 78:     ldu     r9,16(4)
486 3:
487 79:     ld      r10,8(4)
488 80:     std     r7,8(3)
489 81:     ld      r7,16(4)
490 82:     std     r8,16(3)
491 83:     ld      r8,24(4)
492 84:     std     r9,24(3)
493 85:     ldu     r9,32(4)
494 86:     stdu    r10,32(3)
495         bdnz    3b
496 4:
497 87:     ld      r10,8(4)
498 88:     std     r7,8(3)
499 89:     std     r8,16(3)
500 90:     std     r9,24(3)
501 91:     std     r10,32(3)
502 9:      ld      r20,-120(1)
503         ld      r21,-112(1)
504         ld      r22,-104(1)
505         ld      r23,-96(1)
506         ld      r24,-88(1)
507         ld      r25,-80(1)
508         ld      r26,-72(1)
509         ld      r27,-64(1)
510         ld      r28,-56(1)
511         ld      r29,-48(1)
512         ld      r30,-40(1)
513         ld      r31,-32(1)
514         li      r3,0
515         blr
516
517 /*
518  * on an exception, reset to the beginning and jump back into the
519  * standard __copy_tofrom_user
520  */
521 100:    ld      r20,-120(1)
522         ld      r21,-112(1)
523         ld      r22,-104(1)
524         ld      r23,-96(1)
525         ld      r24,-88(1)
526         ld      r25,-80(1)
527         ld      r26,-72(1)
528         ld      r27,-64(1)
529         ld      r28,-56(1)
530         ld      r29,-48(1)
531         ld      r30,-40(1)
532         ld      r31,-32(1)
533         ld      r3,-24(r1)
534         ld      r4,-16(r1)
535         li      r5,4096
536         b       .Ldst_aligned
537
538         .section __ex_table,"a"
539         .align  3
540         .llong  20b,100b
541         .llong  21b,100b
542         .llong  22b,100b
543         .llong  23b,100b
544         .llong  24b,100b
545         .llong  25b,100b
546         .llong  26b,100b
547         .llong  27b,100b
548         .llong  28b,100b
549         .llong  29b,100b
550         .llong  30b,100b
551         .llong  31b,100b
552         .llong  32b,100b
553         .llong  33b,100b
554         .llong  34b,100b
555         .llong  35b,100b
556         .llong  36b,100b
557         .llong  37b,100b
558         .llong  38b,100b
559         .llong  39b,100b
560         .llong  40b,100b
561         .llong  41b,100b
562         .llong  42b,100b
563         .llong  43b,100b
564         .llong  44b,100b
565         .llong  45b,100b
566         .llong  46b,100b
567         .llong  47b,100b
568         .llong  48b,100b
569         .llong  49b,100b
570         .llong  50b,100b
571         .llong  51b,100b
572         .llong  52b,100b
573         .llong  53b,100b
574         .llong  54b,100b
575         .llong  55b,100b
576         .llong  56b,100b
577         .llong  57b,100b
578         .llong  58b,100b
579         .llong  59b,100b
580         .llong  60b,100b
581         .llong  61b,100b
582         .llong  62b,100b
583         .llong  63b,100b
584         .llong  64b,100b
585         .llong  65b,100b
586         .llong  66b,100b
587         .llong  67b,100b
588         .llong  68b,100b
589         .llong  69b,100b
590         .llong  70b,100b
591         .llong  71b,100b
592         .llong  72b,100b
593         .llong  73b,100b
594         .llong  74b,100b
595         .llong  75b,100b
596         .llong  76b,100b
597         .llong  77b,100b
598         .llong  78b,100b
599         .llong  79b,100b
600         .llong  80b,100b
601         .llong  81b,100b
602         .llong  82b,100b
603         .llong  83b,100b
604         .llong  84b,100b
605         .llong  85b,100b
606         .llong  86b,100b
607         .llong  87b,100b
608         .llong  88b,100b
609         .llong  89b,100b
610         .llong  90b,100b
611         .llong  91b,100b