Automatic merge with /usr/src/ntfs-2.6.git.
[linux-2.6] / include / asm-mips / atomic.h
1 /*
2  * Atomic operations that C can't guarantee us.  Useful for
3  * resource counting etc..
4  *
5  * But use these as seldom as possible since they are much more slower
6  * than regular operations.
7  *
8  * This file is subject to the terms and conditions of the GNU General Public
9  * License.  See the file "COPYING" in the main directory of this archive
10  * for more details.
11  *
12  * Copyright (C) 1996, 97, 99, 2000, 03, 04 by Ralf Baechle
13  */
14
15 /*
16  * As workaround for the ATOMIC_DEC_AND_LOCK / atomic_dec_and_lock mess in
17  * <linux/spinlock.h> we have to include <linux/spinlock.h> outside the
18  * main big wrapper ...
19  */
20 #include <linux/config.h>
21 #include <linux/spinlock.h>
22
23 #ifndef _ASM_ATOMIC_H
24 #define _ASM_ATOMIC_H
25
26 #include <asm/cpu-features.h>
27 #include <asm/war.h>
28
29 extern spinlock_t atomic_lock;
30
31 typedef struct { volatile int counter; } atomic_t;
32
33 #define ATOMIC_INIT(i)    { (i) }
34
35 /*
36  * atomic_read - read atomic variable
37  * @v: pointer of type atomic_t
38  *
39  * Atomically reads the value of @v.
40  */
41 #define atomic_read(v)          ((v)->counter)
42
43 /*
44  * atomic_set - set atomic variable
45  * @v: pointer of type atomic_t
46  * @i: required value
47  *
48  * Atomically sets the value of @v to @i.
49  */
50 #define atomic_set(v,i)         ((v)->counter = (i))
51
52 /*
53  * atomic_add - add integer to atomic variable
54  * @i: integer value to add
55  * @v: pointer of type atomic_t
56  *
57  * Atomically adds @i to @v.
58  */
59 static __inline__ void atomic_add(int i, atomic_t * v)
60 {
61         if (cpu_has_llsc && R10000_LLSC_WAR) {
62                 unsigned long temp;
63
64                 __asm__ __volatile__(
65                 "1:     ll      %0, %1          # atomic_add            \n"
66                 "       addu    %0, %2                                  \n"
67                 "       sc      %0, %1                                  \n"
68                 "       beqzl   %0, 1b                                  \n"
69                 : "=&r" (temp), "=m" (v->counter)
70                 : "Ir" (i), "m" (v->counter));
71         } else if (cpu_has_llsc) {
72                 unsigned long temp;
73
74                 __asm__ __volatile__(
75                 "1:     ll      %0, %1          # atomic_add            \n"
76                 "       addu    %0, %2                                  \n"
77                 "       sc      %0, %1                                  \n"
78                 "       beqz    %0, 1b                                  \n"
79                 : "=&r" (temp), "=m" (v->counter)
80                 : "Ir" (i), "m" (v->counter));
81         } else {
82                 unsigned long flags;
83
84                 spin_lock_irqsave(&atomic_lock, flags);
85                 v->counter += i;
86                 spin_unlock_irqrestore(&atomic_lock, flags);
87         }
88 }
89
90 /*
91  * atomic_sub - subtract the atomic variable
92  * @i: integer value to subtract
93  * @v: pointer of type atomic_t
94  *
95  * Atomically subtracts @i from @v.
96  */
97 static __inline__ void atomic_sub(int i, atomic_t * v)
98 {
99         if (cpu_has_llsc && R10000_LLSC_WAR) {
100                 unsigned long temp;
101
102                 __asm__ __volatile__(
103                 "1:     ll      %0, %1          # atomic_sub            \n"
104                 "       subu    %0, %2                                  \n"
105                 "       sc      %0, %1                                  \n"
106                 "       beqzl   %0, 1b                                  \n"
107                 : "=&r" (temp), "=m" (v->counter)
108                 : "Ir" (i), "m" (v->counter));
109         } else if (cpu_has_llsc) {
110                 unsigned long temp;
111
112                 __asm__ __volatile__(
113                 "1:     ll      %0, %1          # atomic_sub            \n"
114                 "       subu    %0, %2                                  \n"
115                 "       sc      %0, %1                                  \n"
116                 "       beqz    %0, 1b                                  \n"
117                 : "=&r" (temp), "=m" (v->counter)
118                 : "Ir" (i), "m" (v->counter));
119         } else {
120                 unsigned long flags;
121
122                 spin_lock_irqsave(&atomic_lock, flags);
123                 v->counter -= i;
124                 spin_unlock_irqrestore(&atomic_lock, flags);
125         }
126 }
127
128 /*
129  * Same as above, but return the result value
130  */
131 static __inline__ int atomic_add_return(int i, atomic_t * v)
132 {
133         unsigned long result;
134
135         if (cpu_has_llsc && R10000_LLSC_WAR) {
136                 unsigned long temp;
137
138                 __asm__ __volatile__(
139                 "1:     ll      %1, %2          # atomic_add_return     \n"
140                 "       addu    %0, %1, %3                              \n"
141                 "       sc      %0, %2                                  \n"
142                 "       beqzl   %0, 1b                                  \n"
143                 "       addu    %0, %1, %3                              \n"
144                 "       sync                                            \n"
145                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
146                 : "Ir" (i), "m" (v->counter)
147                 : "memory");
148         } else if (cpu_has_llsc) {
149                 unsigned long temp;
150
151                 __asm__ __volatile__(
152                 "1:     ll      %1, %2          # atomic_add_return     \n"
153                 "       addu    %0, %1, %3                              \n"
154                 "       sc      %0, %2                                  \n"
155                 "       beqz    %0, 1b                                  \n"
156                 "       addu    %0, %1, %3                              \n"
157                 "       sync                                            \n"
158                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
159                 : "Ir" (i), "m" (v->counter)
160                 : "memory");
161         } else {
162                 unsigned long flags;
163
164                 spin_lock_irqsave(&atomic_lock, flags);
165                 result = v->counter;
166                 result += i;
167                 v->counter = result;
168                 spin_unlock_irqrestore(&atomic_lock, flags);
169         }
170
171         return result;
172 }
173
174 static __inline__ int atomic_sub_return(int i, atomic_t * v)
175 {
176         unsigned long result;
177
178         if (cpu_has_llsc && R10000_LLSC_WAR) {
179                 unsigned long temp;
180
181                 __asm__ __volatile__(
182                 "1:     ll      %1, %2          # atomic_sub_return     \n"
183                 "       subu    %0, %1, %3                              \n"
184                 "       sc      %0, %2                                  \n"
185                 "       beqzl   %0, 1b                                  \n"
186                 "       subu    %0, %1, %3                              \n"
187                 "       sync                                            \n"
188                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
189                 : "Ir" (i), "m" (v->counter)
190                 : "memory");
191         } else if (cpu_has_llsc) {
192                 unsigned long temp;
193
194                 __asm__ __volatile__(
195                 "1:     ll      %1, %2          # atomic_sub_return     \n"
196                 "       subu    %0, %1, %3                              \n"
197                 "       sc      %0, %2                                  \n"
198                 "       beqz    %0, 1b                                  \n"
199                 "       subu    %0, %1, %3                              \n"
200                 "       sync                                            \n"
201                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
202                 : "Ir" (i), "m" (v->counter)
203                 : "memory");
204         } else {
205                 unsigned long flags;
206
207                 spin_lock_irqsave(&atomic_lock, flags);
208                 result = v->counter;
209                 result -= i;
210                 v->counter = result;
211                 spin_unlock_irqrestore(&atomic_lock, flags);
212         }
213
214         return result;
215 }
216
217 /*
218  * atomic_sub_if_positive - add integer to atomic variable
219  * @v: pointer of type atomic_t
220  *
221  * Atomically test @v and decrement if it is greater than 0.
222  * The function returns the old value of @v minus 1.
223  */
224 static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
225 {
226         unsigned long result;
227
228         if (cpu_has_llsc && R10000_LLSC_WAR) {
229                 unsigned long temp;
230
231                 __asm__ __volatile__(
232                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
233                 "       subu    %0, %1, %3                              \n"
234                 "       bltz    %0, 1f                                  \n"
235                 "       sc      %0, %2                                  \n"
236                 "       beqzl   %0, 1b                                  \n"
237                 "       sync                                            \n"
238                 "1:                                                     \n"
239                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
240                 : "Ir" (i), "m" (v->counter)
241                 : "memory");
242         } else if (cpu_has_llsc) {
243                 unsigned long temp;
244
245                 __asm__ __volatile__(
246                 "1:     ll      %1, %2          # atomic_sub_if_positive\n"
247                 "       subu    %0, %1, %3                              \n"
248                 "       bltz    %0, 1f                                  \n"
249                 "       sc      %0, %2                                  \n"
250                 "       beqz    %0, 1b                                  \n"
251                 "       sync                                            \n"
252                 "1:                                                     \n"
253                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
254                 : "Ir" (i), "m" (v->counter)
255                 : "memory");
256         } else {
257                 unsigned long flags;
258
259                 spin_lock_irqsave(&atomic_lock, flags);
260                 result = v->counter;
261                 result -= i;
262                 if (result >= 0)
263                         v->counter = result;
264                 spin_unlock_irqrestore(&atomic_lock, flags);
265         }
266
267         return result;
268 }
269
270 #define atomic_dec_return(v) atomic_sub_return(1,(v))
271 #define atomic_inc_return(v) atomic_add_return(1,(v))
272
273 /*
274  * atomic_sub_and_test - subtract value from variable and test result
275  * @i: integer value to subtract
276  * @v: pointer of type atomic_t
277  *
278  * Atomically subtracts @i from @v and returns
279  * true if the result is zero, or false for all
280  * other cases.
281  */
282 #define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0)
283
284 /*
285  * atomic_inc_and_test - increment and test
286  * @v: pointer of type atomic_t
287  *
288  * Atomically increments @v by 1
289  * and returns true if the result is zero, or false for all
290  * other cases.
291  */
292 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
293
294 /*
295  * atomic_dec_and_test - decrement by 1 and test
296  * @v: pointer of type atomic_t
297  *
298  * Atomically decrements @v by 1 and
299  * returns true if the result is 0, or false for all other
300  * cases.
301  */
302 #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0)
303
304 /*
305  * atomic_dec_if_positive - decrement by 1 if old value positive
306  * @v: pointer of type atomic_t
307  */
308 #define atomic_dec_if_positive(v)       atomic_sub_if_positive(1, v)
309
310 /*
311  * atomic_inc - increment atomic variable
312  * @v: pointer of type atomic_t
313  *
314  * Atomically increments @v by 1.
315  */
316 #define atomic_inc(v) atomic_add(1,(v))
317
318 /*
319  * atomic_dec - decrement and test
320  * @v: pointer of type atomic_t
321  *
322  * Atomically decrements @v by 1.
323  */
324 #define atomic_dec(v) atomic_sub(1,(v))
325
326 /*
327  * atomic_add_negative - add and test if negative
328  * @v: pointer of type atomic_t
329  * @i: integer value to add
330  *
331  * Atomically adds @i to @v and returns true
332  * if the result is negative, or false when
333  * result is greater than or equal to zero.
334  */
335 #define atomic_add_negative(i,v) (atomic_add_return(i, (v)) < 0)
336
337 #ifdef CONFIG_MIPS64
338
339 typedef struct { volatile __s64 counter; } atomic64_t;
340
341 #define ATOMIC64_INIT(i)    { (i) }
342
343 /*
344  * atomic64_read - read atomic variable
345  * @v: pointer of type atomic64_t
346  *
347  */
348 #define atomic64_read(v)        ((v)->counter)
349
350 /*
351  * atomic64_set - set atomic variable
352  * @v: pointer of type atomic64_t
353  * @i: required value
354  */
355 #define atomic64_set(v,i)       ((v)->counter = (i))
356
357 /*
358  * atomic64_add - add integer to atomic variable
359  * @i: integer value to add
360  * @v: pointer of type atomic64_t
361  *
362  * Atomically adds @i to @v.
363  */
364 static __inline__ void atomic64_add(long i, atomic64_t * v)
365 {
366         if (cpu_has_llsc && R10000_LLSC_WAR) {
367                 unsigned long temp;
368
369                 __asm__ __volatile__(
370                 "1:     lld     %0, %1          # atomic64_add          \n"
371                 "       addu    %0, %2                                  \n"
372                 "       scd     %0, %1                                  \n"
373                 "       beqzl   %0, 1b                                  \n"
374                 : "=&r" (temp), "=m" (v->counter)
375                 : "Ir" (i), "m" (v->counter));
376         } else if (cpu_has_llsc) {
377                 unsigned long temp;
378
379                 __asm__ __volatile__(
380                 "1:     lld     %0, %1          # atomic64_add          \n"
381                 "       addu    %0, %2                                  \n"
382                 "       scd     %0, %1                                  \n"
383                 "       beqz    %0, 1b                                  \n"
384                 : "=&r" (temp), "=m" (v->counter)
385                 : "Ir" (i), "m" (v->counter));
386         } else {
387                 unsigned long flags;
388
389                 spin_lock_irqsave(&atomic_lock, flags);
390                 v->counter += i;
391                 spin_unlock_irqrestore(&atomic_lock, flags);
392         }
393 }
394
395 /*
396  * atomic64_sub - subtract the atomic variable
397  * @i: integer value to subtract
398  * @v: pointer of type atomic64_t
399  *
400  * Atomically subtracts @i from @v.
401  */
402 static __inline__ void atomic64_sub(long i, atomic64_t * v)
403 {
404         if (cpu_has_llsc && R10000_LLSC_WAR) {
405                 unsigned long temp;
406
407                 __asm__ __volatile__(
408                 "1:     lld     %0, %1          # atomic64_sub          \n"
409                 "       subu    %0, %2                                  \n"
410                 "       scd     %0, %1                                  \n"
411                 "       beqzl   %0, 1b                                  \n"
412                 : "=&r" (temp), "=m" (v->counter)
413                 : "Ir" (i), "m" (v->counter));
414         } else if (cpu_has_llsc) {
415                 unsigned long temp;
416
417                 __asm__ __volatile__(
418                 "1:     lld     %0, %1          # atomic64_sub          \n"
419                 "       subu    %0, %2                                  \n"
420                 "       scd     %0, %1                                  \n"
421                 "       beqz    %0, 1b                                  \n"
422                 : "=&r" (temp), "=m" (v->counter)
423                 : "Ir" (i), "m" (v->counter));
424         } else {
425                 unsigned long flags;
426
427                 spin_lock_irqsave(&atomic_lock, flags);
428                 v->counter -= i;
429                 spin_unlock_irqrestore(&atomic_lock, flags);
430         }
431 }
432
433 /*
434  * Same as above, but return the result value
435  */
436 static __inline__ long atomic64_add_return(long i, atomic64_t * v)
437 {
438         unsigned long result;
439
440         if (cpu_has_llsc && R10000_LLSC_WAR) {
441                 unsigned long temp;
442
443                 __asm__ __volatile__(
444                 "1:     lld     %1, %2          # atomic64_add_return   \n"
445                 "       addu    %0, %1, %3                              \n"
446                 "       scd     %0, %2                                  \n"
447                 "       beqzl   %0, 1b                                  \n"
448                 "       addu    %0, %1, %3                              \n"
449                 "       sync                                            \n"
450                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
451                 : "Ir" (i), "m" (v->counter)
452                 : "memory");
453         } else if (cpu_has_llsc) {
454                 unsigned long temp;
455
456                 __asm__ __volatile__(
457                 "1:     lld     %1, %2          # atomic64_add_return   \n"
458                 "       addu    %0, %1, %3                              \n"
459                 "       scd     %0, %2                                  \n"
460                 "       beqz    %0, 1b                                  \n"
461                 "       addu    %0, %1, %3                              \n"
462                 "       sync                                            \n"
463                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
464                 : "Ir" (i), "m" (v->counter)
465                 : "memory");
466         } else {
467                 unsigned long flags;
468
469                 spin_lock_irqsave(&atomic_lock, flags);
470                 result = v->counter;
471                 result += i;
472                 v->counter = result;
473                 spin_unlock_irqrestore(&atomic_lock, flags);
474         }
475
476         return result;
477 }
478
479 static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
480 {
481         unsigned long result;
482
483         if (cpu_has_llsc && R10000_LLSC_WAR) {
484                 unsigned long temp;
485
486                 __asm__ __volatile__(
487                 "1:     lld     %1, %2          # atomic64_sub_return   \n"
488                 "       subu    %0, %1, %3                              \n"
489                 "       scd     %0, %2                                  \n"
490                 "       beqzl   %0, 1b                                  \n"
491                 "       subu    %0, %1, %3                              \n"
492                 "       sync                                            \n"
493                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
494                 : "Ir" (i), "m" (v->counter)
495                 : "memory");
496         } else if (cpu_has_llsc) {
497                 unsigned long temp;
498
499                 __asm__ __volatile__(
500                 "1:     lld     %1, %2          # atomic64_sub_return   \n"
501                 "       subu    %0, %1, %3                              \n"
502                 "       scd     %0, %2                                  \n"
503                 "       beqz    %0, 1b                                  \n"
504                 "       subu    %0, %1, %3                              \n"
505                 "       sync                                            \n"
506                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
507                 : "Ir" (i), "m" (v->counter)
508                 : "memory");
509         } else {
510                 unsigned long flags;
511
512                 spin_lock_irqsave(&atomic_lock, flags);
513                 result = v->counter;
514                 result -= i;
515                 v->counter = result;
516                 spin_unlock_irqrestore(&atomic_lock, flags);
517         }
518
519         return result;
520 }
521
522 /*
523  * atomic64_sub_if_positive - add integer to atomic variable
524  * @v: pointer of type atomic64_t
525  *
526  * Atomically test @v and decrement if it is greater than 0.
527  * The function returns the old value of @v minus 1.
528  */
529 static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
530 {
531         unsigned long result;
532
533         if (cpu_has_llsc && R10000_LLSC_WAR) {
534                 unsigned long temp;
535
536                 __asm__ __volatile__(
537                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
538                 "       dsubu   %0, %1, %3                              \n"
539                 "       bltz    %0, 1f                                  \n"
540                 "       scd     %0, %2                                  \n"
541                 "       beqzl   %0, 1b                                  \n"
542                 "       sync                                            \n"
543                 "1:                                                     \n"
544                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
545                 : "Ir" (i), "m" (v->counter)
546                 : "memory");
547         } else if (cpu_has_llsc) {
548                 unsigned long temp;
549
550                 __asm__ __volatile__(
551                 "1:     lld     %1, %2          # atomic64_sub_if_positive\n"
552                 "       dsubu   %0, %1, %3                              \n"
553                 "       bltz    %0, 1f                                  \n"
554                 "       scd     %0, %2                                  \n"
555                 "       beqz    %0, 1b                                  \n"
556                 "       sync                                            \n"
557                 "1:                                                     \n"
558                 : "=&r" (result), "=&r" (temp), "=m" (v->counter)
559                 : "Ir" (i), "m" (v->counter)
560                 : "memory");
561         } else {
562                 unsigned long flags;
563
564                 spin_lock_irqsave(&atomic_lock, flags);
565                 result = v->counter;
566                 result -= i;
567                 if (result >= 0)
568                         v->counter = result;
569                 spin_unlock_irqrestore(&atomic_lock, flags);
570         }
571
572         return result;
573 }
574
575 #define atomic64_dec_return(v) atomic64_sub_return(1,(v))
576 #define atomic64_inc_return(v) atomic64_add_return(1,(v))
577
578 /*
579  * atomic64_sub_and_test - subtract value from variable and test result
580  * @i: integer value to subtract
581  * @v: pointer of type atomic64_t
582  *
583  * Atomically subtracts @i from @v and returns
584  * true if the result is zero, or false for all
585  * other cases.
586  */
587 #define atomic64_sub_and_test(i,v) (atomic64_sub_return((i), (v)) == 0)
588
589 /*
590  * atomic64_inc_and_test - increment and test
591  * @v: pointer of type atomic64_t
592  *
593  * Atomically increments @v by 1
594  * and returns true if the result is zero, or false for all
595  * other cases.
596  */
597 #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
598
599 /*
600  * atomic64_dec_and_test - decrement by 1 and test
601  * @v: pointer of type atomic64_t
602  *
603  * Atomically decrements @v by 1 and
604  * returns true if the result is 0, or false for all other
605  * cases.
606  */
607 #define atomic64_dec_and_test(v) (atomic64_sub_return(1, (v)) == 0)
608
609 /*
610  * atomic64_dec_if_positive - decrement by 1 if old value positive
611  * @v: pointer of type atomic64_t
612  */
613 #define atomic64_dec_if_positive(v)     atomic64_sub_if_positive(1, v)
614
615 /*
616  * atomic64_inc - increment atomic variable
617  * @v: pointer of type atomic64_t
618  *
619  * Atomically increments @v by 1.
620  */
621 #define atomic64_inc(v) atomic64_add(1,(v))
622
623 /*
624  * atomic64_dec - decrement and test
625  * @v: pointer of type atomic64_t
626  *
627  * Atomically decrements @v by 1.
628  */
629 #define atomic64_dec(v) atomic64_sub(1,(v))
630
631 /*
632  * atomic64_add_negative - add and test if negative
633  * @v: pointer of type atomic64_t
634  * @i: integer value to add
635  *
636  * Atomically adds @i to @v and returns true
637  * if the result is negative, or false when
638  * result is greater than or equal to zero.
639  */
640 #define atomic64_add_negative(i,v) (atomic64_add_return(i, (v)) < 0)
641
642 #endif /* CONFIG_MIPS64 */
643
644 /*
645  * atomic*_return operations are serializing but not the non-*_return
646  * versions.
647  */
648 #define smp_mb__before_atomic_dec()     smp_mb()
649 #define smp_mb__after_atomic_dec()      smp_mb()
650 #define smp_mb__before_atomic_inc()     smp_mb()
651 #define smp_mb__after_atomic_inc()      smp_mb()
652
653 #endif /* _ASM_ATOMIC_H */