1 /* Copyright 2002 Andi Kleen, SuSE Labs.
2 * Subject to the GNU Public License v2.
4 * Functions to copy from and to user space.
7 #include <linux/linkage.h>
8 #include <asm/dwarf2.h>
10 #define FIX_ALIGNMENT 1
12 #include <asm/current.h>
13 #include <asm/asm-offsets.h>
14 #include <asm/thread_info.h>
15 #include <asm/cpufeature.h>
17 .macro ALTERNATIVE_JUMP feature,orig,alt
19 .byte 0xe9 /* 32bit jump */
20 .long \orig-1f /* by default jump to orig */
22 .section .altinstr_replacement,"ax"
23 2: .byte 0xe9 /* near jump with 32bit immediate */
24 .long \alt-1b /* offset */ /* or alternatively to alt */
26 .section .altinstructions,"a"
30 .byte \feature /* when feature is set */
36 /* Standard copy_to_user with segment limit checking */
43 cmpq threadinfo_addr_limit(%rax),%rcx
45 xorl %eax,%eax /* clear zero flag */
46 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
49 ENTRY(copy_user_generic)
51 movl $1,%ecx /* set zero flag */
52 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
55 ENTRY(__copy_from_user_inatomic)
57 xorl %ecx,%ecx /* clear zero flag */
58 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
61 /* Standard copy_from_user with segment limit checking */
68 cmpq threadinfo_addr_limit(%rax),%rcx
70 movl $1,%ecx /* set zero flag */
71 ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
73 ENDPROC(copy_from_user)
92 * copy_user_generic_unrolled - memory copy with exception handling.
93 * This version is for CPUs like P4 that don't have efficient micro code for rep movsq
99 * ecx zero flag -- if true zero destination on error
102 * eax uncopied bytes or 0 if successful.
104 ENTRY(copy_user_generic_unrolled)
107 CFI_ADJUST_CFA_OFFSET 8
108 CFI_REL_OFFSET rbx, 0
110 CFI_ADJUST_CFA_OFFSET 8
111 CFI_REL_OFFSET rcx, 0
112 xorl %eax,%eax /*zero for the exception handler */
115 /* check for bad alignment of destination */
119 .Lafter_bad_alignment:
131 .Ls1: movq (%rsi),%r11
132 .Ls2: movq 1*8(%rsi),%r8
133 .Ls3: movq 2*8(%rsi),%r9
134 .Ls4: movq 3*8(%rsi),%r10
135 .Ld1: movq %r11,(%rdi)
136 .Ld2: movq %r8,1*8(%rdi)
137 .Ld3: movq %r9,2*8(%rdi)
138 .Ld4: movq %r10,3*8(%rdi)
140 .Ls5: movq 4*8(%rsi),%r11
141 .Ls6: movq 5*8(%rsi),%r8
142 .Ls7: movq 6*8(%rsi),%r9
143 .Ls8: movq 7*8(%rsi),%r10
144 .Ld5: movq %r11,4*8(%rdi)
145 .Ld6: movq %r8,5*8(%rdi)
146 .Ld7: movq %r9,6*8(%rdi)
147 .Ld8: movq %r10,7*8(%rdi)
165 .Ls9: movq (%rsi),%r8
166 .Ld9: movq %r8,(%rdi)
178 .Ls10: movb (%rsi),%bl
179 .Ld10: movb %bl,(%rdi)
188 CFI_ADJUST_CFA_OFFSET -8
191 CFI_ADJUST_CFA_OFFSET -8
197 /* align destination */
207 .Ls11: movb (%rsi),%bl
208 .Ld11: movb %bl,(%rdi)
214 jmp .Lafter_bad_alignment
217 /* table sorted by exception address */
218 .section __ex_table,"a"
220 .quad .Ls1,.Ls1e /* Ls1-Ls4 have copied zero bytes */
224 .quad .Ld1,.Ls1e /* Ld1-Ld4 have copied 0-24 bytes */
228 .quad .Ls5,.Ls5e /* Ls5-Ls8 have copied 32 bytes */
232 .quad .Ld5,.Ls5e /* Ld5-Ld8 have copied 32-56 bytes */
241 .quad .Ls11,.Lzero_rest
242 .quad .Ld11,.Lzero_rest
247 /* eax: zero, ebx: 64 */
248 .Ls1e: addl $8,%eax /* eax is bytes left uncopied within the loop (Ls1e: 64 .. Ls8e: 8) */
256 addq %rbx,%rdi /* +64 */
257 subq %rax,%rdi /* correct destination with computed offset */
259 shlq $6,%rdx /* loop counter * 64 (stride length) */
260 addq %rax,%rdx /* add offset to loopcnt */
261 andl $63,%ecx /* remaining bytes */
262 addq %rcx,%rdx /* add them */
265 /* exception on quad word loop in tail handling */
266 /* ecx: loopcnt/8, %edx: length, rdi: correct */
271 /* edx: bytes to zero, rdi: dest, eax:zero */
280 /* when there is another exception while zeroing the rest just return */
285 ENDPROC(copy_user_generic)
288 /* Some CPUs run faster using the string copy instructions.
289 This is also a lot simpler. Use them when possible.
290 Patch in jmps to this code instead of copying it fully
291 to avoid unwanted aliasing in the exception tables. */
299 * eax uncopied bytes or 0 if successfull.
301 * Only 4GB of copy is supported. This shouldn't be a problem
302 * because the kernel normally only writes from/to page sized chunks
303 * even if user space passed a longer buffer.
304 * And more would be dangerous because both Intel and AMD have
305 * errata with rep movsq > 4GB. If someone feels the need to fix
306 * this please consider this.
308 ENTRY(copy_user_generic_string)
310 movl %ecx,%r8d /* save zero flag */
323 /* multiple of 8 byte */
329 /* exception handling */
330 3: lea (%rdx,%rcx,8),%rax /* exception on quad loop */
332 5: movl %ecx,%eax /* exception on byte loop */
333 /* eax: left over bytes */
334 6: testl %r8d,%r8d /* zero flag set? */
336 movl %eax,%ecx /* initialize x86 loop counter */
340 stosb /* zero the rest */
344 END(copy_user_generic_c)
346 .section __ex_table,"a"