1 #ifndef __X86_64_UACCESS_H
2 #define __X86_64_UACCESS_H
5 * User space memory access functions
7 #include <linux/compiler.h>
8 #include <linux/errno.h>
9 #include <linux/prefetch.h>
12 #define ARCH_HAS_SEARCH_EXTABLE
14 extern void __put_user_1(void);
15 extern void __put_user_2(void);
16 extern void __put_user_4(void);
17 extern void __put_user_8(void);
18 extern void __put_user_bad(void);
20 #define __put_user_x(size, ret, x, ptr) \
21 asm volatile("call __put_user_" #size \
26 #define put_user(x, ptr) \
27 __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
29 #define __get_user(x, ptr) \
30 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
31 #define __put_user(x, ptr) \
32 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
34 #define __get_user_unaligned __get_user
35 #define __put_user_unaligned __put_user
37 #define __put_user_nocheck(x, ptr, size) \
40 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
45 #define __put_user_check(x, ptr, size) \
48 typeof(*(ptr)) __user *__pu_addr = (ptr); \
51 __put_user_x(1, __pu_err, x, __pu_addr); \
54 __put_user_x(2, __pu_err, x, __pu_addr); \
57 __put_user_x(4, __pu_err, x, __pu_addr); \
60 __put_user_x(8, __pu_err, x, __pu_addr); \
68 #define __put_user_size(x, ptr, size, retval, errret) \
71 __chk_user_ptr(ptr); \
74 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret);\
77 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret);\
80 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret);\
83 __put_user_asm(x, ptr, retval, "q", "", "Zr", errret); \
90 /* FIXME: this hack is definitely wrong -AK */
91 struct __large_struct { unsigned long buf[100]; };
92 #define __m(x) (*(struct __large_struct __user *)(x))
95 * Tell gcc we read from memory instead of writing: this is because
96 * we do not write to any memory gcc knows about, so there are no
99 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \
100 asm volatile("1: mov"itype" %"rtype"1,%2\n" \
102 ".section .fixup, \"ax\"\n" \
106 _ASM_EXTABLE(1b, 3b) \
108 : ltype (x), "m" (__m(addr)), "i" (errno), "0" (err))
111 #define __get_user_nocheck(x, ptr, size) \
114 unsigned long __gu_val; \
115 __get_user_size(__gu_val, (ptr), (size), __gu_err); \
116 (x) = (__force typeof(*(ptr)))__gu_val; \
120 #define __get_user_size(x, ptr, size, retval) \
123 __chk_user_ptr(ptr); \
126 __get_user_asm(x, ptr, retval, "b", "b", "=q", -EFAULT);\
129 __get_user_asm(x, ptr, retval, "w", "w", "=r", -EFAULT);\
132 __get_user_asm(x, ptr, retval, "l", "k", "=r", -EFAULT);\
135 __get_user_asm(x, ptr, retval, "q", "", "=r", -EFAULT); \
138 (x) = __get_user_bad(); \
142 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \
143 asm volatile("1: mov"itype" %2,%"rtype"1\n" \
145 ".section .fixup, \"ax\"\n" \
147 " xor"itype" %"rtype"1,%"rtype"1\n" \
150 _ASM_EXTABLE(1b, 3b) \
151 : "=r" (err), ltype (x) \
152 : "m" (__m(addr)), "i"(errno), "0"(err))
155 * Copy To/From Userspace
158 /* Handles exceptions in both to and from, but doesn't do access_ok */
159 __must_check unsigned long
160 copy_user_generic(void *to, const void *from, unsigned len);
162 __must_check unsigned long
163 copy_to_user(void __user *to, const void *from, unsigned len);
164 __must_check unsigned long
165 copy_from_user(void *to, const void __user *from, unsigned len);
166 __must_check unsigned long
167 copy_in_user(void __user *to, const void __user *from, unsigned len);
169 static __always_inline __must_check
170 int __copy_from_user(void *dst, const void __user *src, unsigned size)
173 if (!__builtin_constant_p(size))
174 return copy_user_generic(dst, (__force void *)src, size);
176 case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
177 ret, "b", "b", "=q", 1);
179 case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
180 ret, "w", "w", "=r", 2);
182 case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
183 ret, "l", "k", "=r", 4);
185 case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
186 ret, "q", "", "=r", 8);
189 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
190 ret, "q", "", "=r", 16);
193 __get_user_asm(*(u16 *)(8 + (char *)dst),
194 (u16 __user *)(8 + (char __user *)src),
195 ret, "w", "w", "=r", 2);
198 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
199 ret, "q", "", "=r", 16);
202 __get_user_asm(*(u64 *)(8 + (char *)dst),
203 (u64 __user *)(8 + (char __user *)src),
204 ret, "q", "", "=r", 8);
207 return copy_user_generic(dst, (__force void *)src, size);
211 static __always_inline __must_check
212 int __copy_to_user(void __user *dst, const void *src, unsigned size)
215 if (!__builtin_constant_p(size))
216 return copy_user_generic((__force void *)dst, src, size);
218 case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
219 ret, "b", "b", "iq", 1);
221 case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
222 ret, "w", "w", "ir", 2);
224 case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
225 ret, "l", "k", "ir", 4);
227 case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
228 ret, "q", "", "ir", 8);
231 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
232 ret, "q", "", "ir", 10);
236 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
237 ret, "w", "w", "ir", 2);
240 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
241 ret, "q", "", "ir", 16);
245 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
246 ret, "q", "", "ir", 8);
249 return copy_user_generic((__force void *)dst, src, size);
253 static __always_inline __must_check
254 int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
257 if (!__builtin_constant_p(size))
258 return copy_user_generic((__force void *)dst,
259 (__force void *)src, size);
263 __get_user_asm(tmp, (u8 __user *)src,
264 ret, "b", "b", "=q", 1);
266 __put_user_asm(tmp, (u8 __user *)dst,
267 ret, "b", "b", "iq", 1);
272 __get_user_asm(tmp, (u16 __user *)src,
273 ret, "w", "w", "=r", 2);
275 __put_user_asm(tmp, (u16 __user *)dst,
276 ret, "w", "w", "ir", 2);
282 __get_user_asm(tmp, (u32 __user *)src,
283 ret, "l", "k", "=r", 4);
285 __put_user_asm(tmp, (u32 __user *)dst,
286 ret, "l", "k", "ir", 4);
291 __get_user_asm(tmp, (u64 __user *)src,
292 ret, "q", "", "=r", 8);
294 __put_user_asm(tmp, (u64 __user *)dst,
295 ret, "q", "", "ir", 8);
299 return copy_user_generic((__force void *)dst,
300 (__force void *)src, size);
305 strncpy_from_user(char *dst, const char __user *src, long count);
307 __strncpy_from_user(char *dst, const char __user *src, long count);
308 __must_check long strnlen_user(const char __user *str, long n);
309 __must_check long __strnlen_user(const char __user *str, long n);
310 __must_check long strlen_user(const char __user *str);
311 __must_check unsigned long clear_user(void __user *mem, unsigned long len);
312 __must_check unsigned long __clear_user(void __user *mem, unsigned long len);
314 __must_check long __copy_from_user_inatomic(void *dst, const void __user *src,
317 static __must_check __always_inline int
318 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
320 return copy_user_generic((__force void *)dst, src, size);
323 #define ARCH_HAS_NOCACHE_UACCESS 1
324 extern long __copy_user_nocache(void *dst, const void __user *src,
325 unsigned size, int zerorest);
327 static inline int __copy_from_user_nocache(void *dst, const void __user *src,
331 return __copy_user_nocache(dst, src, size, 1);
334 static inline int __copy_from_user_inatomic_nocache(void *dst,
335 const void __user *src,
338 return __copy_user_nocache(dst, src, size, 0);
341 #endif /* __X86_64_UACCESS_H */