SG: Move functions to lib/scatterlist.c and add sg chaining allocator helpers
[linux-2.6] / arch / alpha / lib / csum_partial_copy.c
1 /*
2  * csum_partial_copy - do IP checksumming and copy
3  *
4  * (C) Copyright 1996 Linus Torvalds
5  * accelerated versions (and 21264 assembly versions ) contributed by
6  *      Rick Gorton     <rick.gorton@alpha-processor.com>
7  *
8  * Don't look at this too closely - you'll go mad. The things
9  * we do for performance..
10  */
11
12 #include <linux/types.h>
13 #include <linux/string.h>
14 #include <asm/uaccess.h>
15
16
17 #define ldq_u(x,y) \
18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
19
20 #define stq_u(x,y) \
21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
22
23 #define extql(x,y,z) \
24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
25
26 #define extqh(x,y,z) \
27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
28
29 #define mskql(x,y,z) \
30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
31
32 #define mskqh(x,y,z) \
33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
34
35 #define insql(x,y,z) \
36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
37
38 #define insqh(x,y,z) \
39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
40
41
42 #define __get_user_u(x,ptr)                             \
43 ({                                                      \
44         long __guu_err;                                 \
45         __asm__ __volatile__(                           \
46         "1:     ldq_u %0,%2\n"                          \
47         "2:\n"                                          \
48         ".section __ex_table,\"a\"\n"                   \
49         "       .long 1b - .\n"                         \
50         "       lda %0,2b-1b(%1)\n"                     \
51         ".previous"                                     \
52                 : "=r"(x), "=r"(__guu_err)              \
53                 : "m"(__m(ptr)), "1"(0));               \
54         __guu_err;                                      \
55 })
56
57 #define __put_user_u(x,ptr)                             \
58 ({                                                      \
59         long __puu_err;                                 \
60         __asm__ __volatile__(                           \
61         "1:     stq_u %2,%1\n"                          \
62         "2:\n"                                          \
63         ".section __ex_table,\"a\"\n"                   \
64         "       .long 1b - ."                           \
65         "       lda $31,2b-1b(%0)\n"                    \
66         ".previous"                                     \
67                 : "=r"(__puu_err)                       \
68                 : "m"(__m(addr)), "rJ"(x), "0"(0));     \
69         __puu_err;                                      \
70 })
71
72
73 static inline unsigned short from64to16(unsigned long x)
74 {
75         /* Using extract instructions is a bit more efficient
76            than the original shift/bitmask version.  */
77
78         union {
79                 unsigned long   ul;
80                 unsigned int    ui[2];
81                 unsigned short  us[4];
82         } in_v, tmp_v, out_v;
83
84         in_v.ul = x;
85         tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
86
87         /* Since the bits of tmp_v.sh[3] are going to always be zero,
88            we don't have to bother to add that in.  */
89         out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
90                         + (unsigned long) tmp_v.us[2];
91
92         /* Similarly, out_v.us[2] is always zero for the final add.  */
93         return out_v.us[0] + out_v.us[1];
94 }
95
96
97
98 /*
99  * Ok. This isn't fun, but this is the EASY case.
100  */
101 static inline unsigned long
102 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
103                          long len, unsigned long checksum,
104                          int *errp)
105 {
106         unsigned long carry = 0;
107         int err = 0;
108
109         while (len >= 0) {
110                 unsigned long word;
111                 err |= __get_user(word, src);
112                 checksum += carry;
113                 src++;
114                 checksum += word;
115                 len -= 8;
116                 carry = checksum < word;
117                 *dst = word;
118                 dst++;
119         }
120         len += 8;
121         checksum += carry;
122         if (len) {
123                 unsigned long word, tmp;
124                 err |= __get_user(word, src);
125                 tmp = *dst;
126                 mskql(word, len, word);
127                 checksum += word;
128                 mskqh(tmp, len, tmp);
129                 carry = checksum < word;
130                 *dst = word | tmp;
131                 checksum += carry;
132         }
133         if (err) *errp = err;
134         return checksum;
135 }
136
137 /*
138  * This is even less fun, but this is still reasonably
139  * easy.
140  */
141 static inline unsigned long
142 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
143                               unsigned long *dst,
144                               unsigned long soff,
145                               long len, unsigned long checksum,
146                               int *errp)
147 {
148         unsigned long first;
149         unsigned long word, carry;
150         unsigned long lastsrc = 7+len+(unsigned long)src;
151         int err = 0;
152
153         err |= __get_user_u(first,src);
154         carry = 0;
155         while (len >= 0) {
156                 unsigned long second;
157
158                 err |= __get_user_u(second, src+1);
159                 extql(first, soff, word);
160                 len -= 8;
161                 src++;
162                 extqh(second, soff, first);
163                 checksum += carry;
164                 word |= first;
165                 first = second;
166                 checksum += word;
167                 *dst = word;
168                 dst++;
169                 carry = checksum < word;
170         }
171         len += 8;
172         checksum += carry;
173         if (len) {
174                 unsigned long tmp;
175                 unsigned long second;
176                 err |= __get_user_u(second, lastsrc);
177                 tmp = *dst;
178                 extql(first, soff, word);
179                 extqh(second, soff, first);
180                 word |= first;
181                 mskql(word, len, word);
182                 checksum += word;
183                 mskqh(tmp, len, tmp);
184                 carry = checksum < word;
185                 *dst = word | tmp;
186                 checksum += carry;
187         }
188         if (err) *errp = err;
189         return checksum;
190 }
191
192 /*
193  * This is slightly less fun than the above..
194  */
195 static inline unsigned long
196 csum_partial_cfu_src_aligned(const unsigned long __user *src,
197                              unsigned long *dst,
198                              unsigned long doff,
199                              long len, unsigned long checksum,
200                              unsigned long partial_dest,
201                              int *errp)
202 {
203         unsigned long carry = 0;
204         unsigned long word;
205         unsigned long second_dest;
206         int err = 0;
207
208         mskql(partial_dest, doff, partial_dest);
209         while (len >= 0) {
210                 err |= __get_user(word, src);
211                 len -= 8;
212                 insql(word, doff, second_dest);
213                 checksum += carry;
214                 stq_u(partial_dest | second_dest, dst);
215                 src++;
216                 checksum += word;
217                 insqh(word, doff, partial_dest);
218                 carry = checksum < word;
219                 dst++;
220         }
221         len += 8;
222         if (len) {
223                 checksum += carry;
224                 err |= __get_user(word, src);
225                 mskql(word, len, word);
226                 len -= 8;
227                 checksum += word;
228                 insql(word, doff, second_dest);
229                 len += doff;
230                 carry = checksum < word;
231                 partial_dest |= second_dest;
232                 if (len >= 0) {
233                         stq_u(partial_dest, dst);
234                         if (!len) goto out;
235                         dst++;
236                         insqh(word, doff, partial_dest);
237                 }
238                 doff = len;
239         }
240         ldq_u(second_dest, dst);
241         mskqh(second_dest, doff, second_dest);
242         stq_u(partial_dest | second_dest, dst);
243 out:
244         checksum += carry;
245         if (err) *errp = err;
246         return checksum;
247 }
248
249 /*
250  * This is so totally un-fun that it's frightening. Don't
251  * look at this too closely, you'll go blind.
252  */
253 static inline unsigned long
254 csum_partial_cfu_unaligned(const unsigned long __user * src,
255                            unsigned long * dst,
256                            unsigned long soff, unsigned long doff,
257                            long len, unsigned long checksum,
258                            unsigned long partial_dest,
259                            int *errp)
260 {
261         unsigned long carry = 0;
262         unsigned long first;
263         unsigned long lastsrc;
264         int err = 0;
265
266         err |= __get_user_u(first, src);
267         lastsrc = 7+len+(unsigned long)src;
268         mskql(partial_dest, doff, partial_dest);
269         while (len >= 0) {
270                 unsigned long second, word;
271                 unsigned long second_dest;
272
273                 err |= __get_user_u(second, src+1);
274                 extql(first, soff, word);
275                 checksum += carry;
276                 len -= 8;
277                 extqh(second, soff, first);
278                 src++;
279                 word |= first;
280                 first = second;
281                 insql(word, doff, second_dest);
282                 checksum += word;
283                 stq_u(partial_dest | second_dest, dst);
284                 carry = checksum < word;
285                 insqh(word, doff, partial_dest);
286                 dst++;
287         }
288         len += doff;
289         checksum += carry;
290         if (len >= 0) {
291                 unsigned long second, word;
292                 unsigned long second_dest;
293
294                 err |= __get_user_u(second, lastsrc);
295                 extql(first, soff, word);
296                 extqh(second, soff, first);
297                 word |= first;
298                 first = second;
299                 mskql(word, len-doff, word);
300                 checksum += word;
301                 insql(word, doff, second_dest);
302                 carry = checksum < word;
303                 stq_u(partial_dest | second_dest, dst);
304                 if (len) {
305                         ldq_u(second_dest, dst+1);
306                         insqh(word, doff, partial_dest);
307                         mskqh(second_dest, len, second_dest);
308                         stq_u(partial_dest | second_dest, dst+1);
309                 }
310                 checksum += carry;
311         } else {
312                 unsigned long second, word;
313                 unsigned long second_dest;
314
315                 err |= __get_user_u(second, lastsrc);
316                 extql(first, soff, word);
317                 extqh(second, soff, first);
318                 word |= first;
319                 ldq_u(second_dest, dst);
320                 mskql(word, len-doff, word);
321                 checksum += word;
322                 mskqh(second_dest, len, second_dest);
323                 carry = checksum < word;
324                 insql(word, doff, word);
325                 stq_u(partial_dest | word | second_dest, dst);
326                 checksum += carry;
327         }
328         if (err) *errp = err;
329         return checksum;
330 }
331
332 __wsum
333 csum_partial_copy_from_user(const void __user *src, void *dst, int len,
334                                __wsum sum, int *errp)
335 {
336         unsigned long checksum = (__force u32) sum;
337         unsigned long soff = 7 & (unsigned long) src;
338         unsigned long doff = 7 & (unsigned long) dst;
339
340         if (len) {
341                 if (!doff) {
342                         if (!soff)
343                                 checksum = csum_partial_cfu_aligned(
344                                         (const unsigned long __user *) src,
345                                         (unsigned long *) dst,
346                                         len-8, checksum, errp);
347                         else
348                                 checksum = csum_partial_cfu_dest_aligned(
349                                         (const unsigned long __user *) src,
350                                         (unsigned long *) dst,
351                                         soff, len-8, checksum, errp);
352                 } else {
353                         unsigned long partial_dest;
354                         ldq_u(partial_dest, dst);
355                         if (!soff)
356                                 checksum = csum_partial_cfu_src_aligned(
357                                         (const unsigned long __user *) src,
358                                         (unsigned long *) dst,
359                                         doff, len-8, checksum,
360                                         partial_dest, errp);
361                         else
362                                 checksum = csum_partial_cfu_unaligned(
363                                         (const unsigned long __user *) src,
364                                         (unsigned long *) dst,
365                                         soff, doff, len-8, checksum,
366                                         partial_dest, errp);
367                 }
368                 checksum = from64to16 (checksum);
369         }
370         return (__force __wsum)checksum;
371 }
372
373 __wsum
374 csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum)
375 {
376         return csum_partial_copy_from_user((__force const void __user *)src,
377                         dst, len, sum, NULL);
378 }