1 #ifndef __i386_UACCESS_H
2 #define __i386_UACCESS_H
5 * User space memory access functions
7 #include <linux/errno.h>
8 #include <linux/thread_info.h>
9 #include <linux/prefetch.h>
10 #include <linux/string.h>
15 * movsl can be slow when source and dest are not both 8-byte aligned
17 #ifdef CONFIG_X86_INTEL_USERCOPY
18 extern struct movsl_mask {
20 } ____cacheline_aligned_in_smp movsl_mask;
23 #define __addr_ok(addr) \
24 ((unsigned long __force)(addr) < \
25 (current_thread_info()->addr_limit.seg))
27 /* Careful: we have to cast the result to the type of the pointer
31 * get_user: - Get a simple variable from user space.
32 * @x: Variable to store result.
33 * @ptr: Source address, in user space.
35 * Context: User context only. This function may sleep.
37 * This macro copies a single simple variable from user space to kernel
38 * space. It supports simple types like char and int, but not larger
39 * data types like structures or arrays.
41 * @ptr must have pointer-to-simple-variable type, and the result of
42 * dereferencing @ptr must be assignable to @x without a cast.
44 * Returns zero on success, or -EFAULT on error.
45 * On error, the variable @x is set to zero.
47 #define get_user(x, ptr) \
50 unsigned long __val_gu; \
51 __chk_user_ptr(ptr); \
52 switch (sizeof(*(ptr))) { \
54 __get_user_x(1, __ret_gu, __val_gu, ptr); \
57 __get_user_x(2, __ret_gu, __val_gu, ptr); \
60 __get_user_x(4, __ret_gu, __val_gu, ptr); \
63 __get_user_x(X, __ret_gu, __val_gu, ptr); \
66 (x) = (__typeof__(*(ptr)))__val_gu; \
70 extern void __put_user_bad(void);
73 * Strange magic calling convention: pointer in %ecx,
74 * value in %eax(:%edx), return value in %eax, no clobbers.
76 extern void __put_user_1(void);
77 extern void __put_user_2(void);
78 extern void __put_user_4(void);
79 extern void __put_user_8(void);
81 #define __put_user_x(size, x, ptr) \
82 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
83 :"0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
85 #define __put_user_8(x, ptr) \
86 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
87 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
91 * put_user: - Write a simple value into user space.
92 * @x: Value to copy to user space.
93 * @ptr: Destination address, in user space.
95 * Context: User context only. This function may sleep.
97 * This macro copies a single simple value from kernel space to user
98 * space. It supports simple types like char and int, but not larger
99 * data types like structures or arrays.
101 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
102 * to the result of dereferencing @ptr.
104 * Returns zero on success, or -EFAULT on error.
106 #ifdef CONFIG_X86_WP_WORKS_OK
108 #define put_user(x, ptr) \
111 __typeof__(*(ptr)) __pu_val; \
112 __chk_user_ptr(ptr); \
114 switch (sizeof(*(ptr))) { \
116 __put_user_x(1, __pu_val, ptr); \
119 __put_user_x(2, __pu_val, ptr); \
122 __put_user_x(4, __pu_val, ptr); \
125 __put_user_8(__pu_val, ptr); \
128 __put_user_x(X, __pu_val, ptr); \
135 #define put_user(x, ptr) \
138 __typeof__(*(ptr))__pus_tmp = x; \
140 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \
141 sizeof(*(ptr))) != 0)) \
142 __ret_pu = -EFAULT; \
150 * __get_user: - Get a simple variable from user space, with less checking.
151 * @x: Variable to store result.
152 * @ptr: Source address, in user space.
154 * Context: User context only. This function may sleep.
156 * This macro copies a single simple variable from user space to kernel
157 * space. It supports simple types like char and int, but not larger
158 * data types like structures or arrays.
160 * @ptr must have pointer-to-simple-variable type, and the result of
161 * dereferencing @ptr must be assignable to @x without a cast.
163 * Caller must check the pointer with access_ok() before calling this
166 * Returns zero on success, or -EFAULT on error.
167 * On error, the variable @x is set to zero.
169 #define __get_user(x, ptr) \
170 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
174 * __put_user: - Write a simple value into user space, with less checking.
175 * @x: Value to copy to user space.
176 * @ptr: Destination address, in user space.
178 * Context: User context only. This function may sleep.
180 * This macro copies a single simple value from kernel space to user
181 * space. It supports simple types like char and int, but not larger
182 * data types like structures or arrays.
184 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
185 * to the result of dereferencing @ptr.
187 * Caller must check the pointer with access_ok() before calling this
190 * Returns zero on success, or -EFAULT on error.
192 #define __put_user(x, ptr) \
193 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
195 #define __put_user_nocheck(x, ptr, size) \
198 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
203 #define __put_user_u64(x, addr, err) \
204 asm volatile("1: movl %%eax,0(%2)\n" \
205 "2: movl %%edx,4(%2)\n" \
207 ".section .fixup,\"ax\"\n" \
211 _ASM_EXTABLE(1b, 4b) \
212 _ASM_EXTABLE(2b, 4b) \
214 : "A" (x), "r" (addr), "i" (-EFAULT), "0" (err))
216 #ifdef CONFIG_X86_WP_WORKS_OK
218 #define __put_user_size(x, ptr, size, retval, errret) \
221 __chk_user_ptr(ptr); \
224 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
227 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
230 __put_user_asm(x, ptr, retval, "l", "", "ir", errret); \
233 __put_user_u64((__typeof__(*ptr))(x), ptr, retval); \
242 #define __put_user_size(x, ptr, size, retval, errret) \
244 __typeof__(*(ptr))__pus_tmp = x; \
247 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \
252 struct __large_struct { unsigned long buf[100]; };
253 #define __m(x) (*(struct __large_struct __user *)(x))
256 * Tell gcc we read from memory instead of writing: this is because
257 * we do not write to any memory gcc knows about, so there are no
260 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
261 asm volatile("1: mov"itype" %"rtype"1,%2\n" \
263 ".section .fixup,\"ax\"\n" \
267 _ASM_EXTABLE(1b, 3b) \
269 : ltype (x), "m" (__m(addr)), "i" (errret), "0" (err))
272 #define __get_user_nocheck(x, ptr, size) \
275 unsigned long __gu_val; \
276 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
277 (x) = (__typeof__(*(ptr)))__gu_val; \
281 #define __get_user_size(x, ptr, size, retval, errret) \
284 __chk_user_ptr(ptr); \
287 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
290 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
293 __get_user_asm(x, ptr, retval, "l", "", "=r", errret); \
296 (x) = __get_user_bad(); \
300 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
301 asm volatile("1: mov"itype" %2,%"rtype"1\n" \
303 ".section .fixup,\"ax\"\n" \
305 " xor"itype" %"rtype"1,%"rtype"1\n" \
308 _ASM_EXTABLE(1b, 3b) \
309 : "=r" (err), ltype (x) \
310 : "m" (__m(addr)), "i" (errret), "0" (err))
313 unsigned long __must_check __copy_to_user_ll
314 (void __user *to, const void *from, unsigned long n);
315 unsigned long __must_check __copy_from_user_ll
316 (void *to, const void __user *from, unsigned long n);
317 unsigned long __must_check __copy_from_user_ll_nozero
318 (void *to, const void __user *from, unsigned long n);
319 unsigned long __must_check __copy_from_user_ll_nocache
320 (void *to, const void __user *from, unsigned long n);
321 unsigned long __must_check __copy_from_user_ll_nocache_nozero
322 (void *to, const void __user *from, unsigned long n);
325 * __copy_to_user_inatomic: - Copy a block of data into user space, with less checking.
326 * @to: Destination address, in user space.
327 * @from: Source address, in kernel space.
328 * @n: Number of bytes to copy.
330 * Context: User context only.
332 * Copy data from kernel space to user space. Caller must check
333 * the specified block with access_ok() before calling this function.
334 * The caller should also make sure he pins the user space address
335 * so that the we don't result in page fault and sleep.
337 * Here we special-case 1, 2 and 4-byte copy_*_user invocations. On a fault
338 * we return the initial request size (1, 2 or 4), as copy_*_user should do.
339 * If a store crosses a page boundary and gets a fault, the x86 will not write
340 * anything, so this is accurate.
343 static __always_inline unsigned long __must_check
344 __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
346 if (__builtin_constant_p(n)) {
351 __put_user_size(*(u8 *)from, (u8 __user *)to,
355 __put_user_size(*(u16 *)from, (u16 __user *)to,
359 __put_user_size(*(u32 *)from, (u32 __user *)to,
364 return __copy_to_user_ll(to, from, n);
368 * __copy_to_user: - Copy a block of data into user space, with less checking.
369 * @to: Destination address, in user space.
370 * @from: Source address, in kernel space.
371 * @n: Number of bytes to copy.
373 * Context: User context only. This function may sleep.
375 * Copy data from kernel space to user space. Caller must check
376 * the specified block with access_ok() before calling this function.
378 * Returns number of bytes that could not be copied.
379 * On success, this will be zero.
381 static __always_inline unsigned long __must_check
382 __copy_to_user(void __user *to, const void *from, unsigned long n)
385 return __copy_to_user_inatomic(to, from, n);
388 static __always_inline unsigned long
389 __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
391 /* Avoid zeroing the tail if the copy fails..
392 * If 'n' is constant and 1, 2, or 4, we do still zero on a failure,
393 * but as the zeroing behaviour is only significant when n is not
394 * constant, that shouldn't be a problem.
396 if (__builtin_constant_p(n)) {
401 __get_user_size(*(u8 *)to, from, 1, ret, 1);
404 __get_user_size(*(u16 *)to, from, 2, ret, 2);
407 __get_user_size(*(u32 *)to, from, 4, ret, 4);
411 return __copy_from_user_ll_nozero(to, from, n);
415 * __copy_from_user: - Copy a block of data from user space, with less checking.
416 * @to: Destination address, in kernel space.
417 * @from: Source address, in user space.
418 * @n: Number of bytes to copy.
420 * Context: User context only. This function may sleep.
422 * Copy data from user space to kernel space. Caller must check
423 * the specified block with access_ok() before calling this function.
425 * Returns number of bytes that could not be copied.
426 * On success, this will be zero.
428 * If some data could not be copied, this function will pad the copied
429 * data to the requested size using zero bytes.
431 * An alternate version - __copy_from_user_inatomic() - may be called from
432 * atomic context and will fail rather than sleep. In this case the
433 * uncopied bytes will *NOT* be padded with zeros. See fs/filemap.h
434 * for explanation of why this is needed.
436 static __always_inline unsigned long
437 __copy_from_user(void *to, const void __user *from, unsigned long n)
440 if (__builtin_constant_p(n)) {
445 __get_user_size(*(u8 *)to, from, 1, ret, 1);
448 __get_user_size(*(u16 *)to, from, 2, ret, 2);
451 __get_user_size(*(u32 *)to, from, 4, ret, 4);
455 return __copy_from_user_ll(to, from, n);
458 #define ARCH_HAS_NOCACHE_UACCESS
460 static __always_inline unsigned long __copy_from_user_nocache(void *to,
461 const void __user *from, unsigned long n)
464 if (__builtin_constant_p(n)) {
469 __get_user_size(*(u8 *)to, from, 1, ret, 1);
472 __get_user_size(*(u16 *)to, from, 2, ret, 2);
475 __get_user_size(*(u32 *)to, from, 4, ret, 4);
479 return __copy_from_user_ll_nocache(to, from, n);
482 static __always_inline unsigned long
483 __copy_from_user_inatomic_nocache(void *to, const void __user *from,
486 return __copy_from_user_ll_nocache_nozero(to, from, n);
489 unsigned long __must_check copy_to_user(void __user *to,
490 const void *from, unsigned long n);
491 unsigned long __must_check copy_from_user(void *to,
492 const void __user *from,
494 long __must_check strncpy_from_user(char *dst, const char __user *src,
496 long __must_check __strncpy_from_user(char *dst,
497 const char __user *src, long count);
500 * strlen_user: - Get the size of a string in user space.
501 * @str: The string to measure.
503 * Context: User context only. This function may sleep.
505 * Get the size of a NUL-terminated string in user space.
507 * Returns the size of the string INCLUDING the terminating NUL.
508 * On exception, returns 0.
510 * If there is a limit on the length of a valid string, you may wish to
511 * consider using strnlen_user() instead.
513 #define strlen_user(str) strnlen_user(str, LONG_MAX)
515 long strnlen_user(const char __user *str, long n);
516 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
517 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
519 #endif /* __i386_UACCESS_H */