sched: debug: update exec_clock only when SCHED_DEBUG
[linux-2.6] / kernel / spinlock.c
1 /*
2  * Copyright (2004) Linus Torvalds
3  *
4  * Author: Zwane Mwaikambo <zwane@fsmlabs.com>
5  *
6  * Copyright (2004, 2005) Ingo Molnar
7  *
8  * This file contains the spinlock/rwlock implementations for the
9  * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them)
10  *
11  * Note that some architectures have special knowledge about the
12  * stack frames of these functions in their profile_pc. If you
13  * change anything significant here that could change the stack
14  * frame contact the architecture maintainers.
15  */
16
17 #include <linux/linkage.h>
18 #include <linux/preempt.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/debug_locks.h>
22 #include <linux/module.h>
23
24 int __lockfunc _spin_trylock(spinlock_t *lock)
25 {
26         preempt_disable();
27         if (_raw_spin_trylock(lock)) {
28                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
29                 return 1;
30         }
31         
32         preempt_enable();
33         return 0;
34 }
35 EXPORT_SYMBOL(_spin_trylock);
36
37 int __lockfunc _read_trylock(rwlock_t *lock)
38 {
39         preempt_disable();
40         if (_raw_read_trylock(lock)) {
41                 rwlock_acquire_read(&lock->dep_map, 0, 1, _RET_IP_);
42                 return 1;
43         }
44
45         preempt_enable();
46         return 0;
47 }
48 EXPORT_SYMBOL(_read_trylock);
49
50 int __lockfunc _write_trylock(rwlock_t *lock)
51 {
52         preempt_disable();
53         if (_raw_write_trylock(lock)) {
54                 rwlock_acquire(&lock->dep_map, 0, 1, _RET_IP_);
55                 return 1;
56         }
57
58         preempt_enable();
59         return 0;
60 }
61 EXPORT_SYMBOL(_write_trylock);
62
63 /*
64  * If lockdep is enabled then we use the non-preemption spin-ops
65  * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are
66  * not re-enabled during lock-acquire (which the preempt-spin-ops do):
67  */
68 #if !defined(CONFIG_PREEMPT) || !defined(CONFIG_SMP) || \
69         defined(CONFIG_DEBUG_LOCK_ALLOC)
70
71 void __lockfunc _read_lock(rwlock_t *lock)
72 {
73         preempt_disable();
74         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
75         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
76 }
77 EXPORT_SYMBOL(_read_lock);
78
79 unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
80 {
81         unsigned long flags;
82
83         local_irq_save(flags);
84         preempt_disable();
85         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
86         /*
87          * On lockdep we dont want the hand-coded irq-enable of
88          * _raw_spin_lock_flags() code, because lockdep assumes
89          * that interrupts are not re-enabled during lock-acquire:
90          */
91 #ifdef CONFIG_LOCKDEP
92         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
93 #else
94         _raw_spin_lock_flags(lock, &flags);
95 #endif
96         return flags;
97 }
98 EXPORT_SYMBOL(_spin_lock_irqsave);
99
100 void __lockfunc _spin_lock_irq(spinlock_t *lock)
101 {
102         local_irq_disable();
103         preempt_disable();
104         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
105         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
106 }
107 EXPORT_SYMBOL(_spin_lock_irq);
108
109 void __lockfunc _spin_lock_bh(spinlock_t *lock)
110 {
111         local_bh_disable();
112         preempt_disable();
113         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
114         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
115 }
116 EXPORT_SYMBOL(_spin_lock_bh);
117
118 unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
119 {
120         unsigned long flags;
121
122         local_irq_save(flags);
123         preempt_disable();
124         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
125         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
126         return flags;
127 }
128 EXPORT_SYMBOL(_read_lock_irqsave);
129
130 void __lockfunc _read_lock_irq(rwlock_t *lock)
131 {
132         local_irq_disable();
133         preempt_disable();
134         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
135         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
136 }
137 EXPORT_SYMBOL(_read_lock_irq);
138
139 void __lockfunc _read_lock_bh(rwlock_t *lock)
140 {
141         local_bh_disable();
142         preempt_disable();
143         rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_);
144         LOCK_CONTENDED(lock, _raw_read_trylock, _raw_read_lock);
145 }
146 EXPORT_SYMBOL(_read_lock_bh);
147
148 unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
149 {
150         unsigned long flags;
151
152         local_irq_save(flags);
153         preempt_disable();
154         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
155         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
156         return flags;
157 }
158 EXPORT_SYMBOL(_write_lock_irqsave);
159
160 void __lockfunc _write_lock_irq(rwlock_t *lock)
161 {
162         local_irq_disable();
163         preempt_disable();
164         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
165         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
166 }
167 EXPORT_SYMBOL(_write_lock_irq);
168
169 void __lockfunc _write_lock_bh(rwlock_t *lock)
170 {
171         local_bh_disable();
172         preempt_disable();
173         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
174         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
175 }
176 EXPORT_SYMBOL(_write_lock_bh);
177
178 void __lockfunc _spin_lock(spinlock_t *lock)
179 {
180         preempt_disable();
181         spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
182         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
183 }
184
185 EXPORT_SYMBOL(_spin_lock);
186
187 void __lockfunc _write_lock(rwlock_t *lock)
188 {
189         preempt_disable();
190         rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_);
191         LOCK_CONTENDED(lock, _raw_write_trylock, _raw_write_lock);
192 }
193
194 EXPORT_SYMBOL(_write_lock);
195
196 #else /* CONFIG_PREEMPT: */
197
198 /*
199  * This could be a long-held lock. We both prepare to spin for a long
200  * time (making _this_ CPU preemptable if possible), and we also signal
201  * towards that other CPU that it should break the lock ASAP.
202  *
203  * (We do this in a function because inlining it would be excessive.)
204  */
205
206 #define BUILD_LOCK_OPS(op, locktype)                                    \
207 void __lockfunc _##op##_lock(locktype##_t *lock)                        \
208 {                                                                       \
209         for (;;) {                                                      \
210                 preempt_disable();                                      \
211                 if (likely(_raw_##op##_trylock(lock)))                  \
212                         break;                                          \
213                 preempt_enable();                                       \
214                                                                         \
215                 if (!(lock)->break_lock)                                \
216                         (lock)->break_lock = 1;                         \
217                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
218                         _raw_##op##_relax(&lock->raw_lock);             \
219         }                                                               \
220         (lock)->break_lock = 0;                                         \
221 }                                                                       \
222                                                                         \
223 EXPORT_SYMBOL(_##op##_lock);                                            \
224                                                                         \
225 unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock)       \
226 {                                                                       \
227         unsigned long flags;                                            \
228                                                                         \
229         for (;;) {                                                      \
230                 preempt_disable();                                      \
231                 local_irq_save(flags);                                  \
232                 if (likely(_raw_##op##_trylock(lock)))                  \
233                         break;                                          \
234                 local_irq_restore(flags);                               \
235                 preempt_enable();                                       \
236                                                                         \
237                 if (!(lock)->break_lock)                                \
238                         (lock)->break_lock = 1;                         \
239                 while (!op##_can_lock(lock) && (lock)->break_lock)      \
240                         _raw_##op##_relax(&lock->raw_lock);             \
241         }                                                               \
242         (lock)->break_lock = 0;                                         \
243         return flags;                                                   \
244 }                                                                       \
245                                                                         \
246 EXPORT_SYMBOL(_##op##_lock_irqsave);                                    \
247                                                                         \
248 void __lockfunc _##op##_lock_irq(locktype##_t *lock)                    \
249 {                                                                       \
250         _##op##_lock_irqsave(lock);                                     \
251 }                                                                       \
252                                                                         \
253 EXPORT_SYMBOL(_##op##_lock_irq);                                        \
254                                                                         \
255 void __lockfunc _##op##_lock_bh(locktype##_t *lock)                     \
256 {                                                                       \
257         unsigned long flags;                                            \
258                                                                         \
259         /*                                                      */      \
260         /* Careful: we must exclude softirqs too, hence the     */      \
261         /* irq-disabling. We use the generic preemption-aware   */      \
262         /* function:                                            */      \
263         /**/                                                            \
264         flags = _##op##_lock_irqsave(lock);                             \
265         local_bh_disable();                                             \
266         local_irq_restore(flags);                                       \
267 }                                                                       \
268                                                                         \
269 EXPORT_SYMBOL(_##op##_lock_bh)
270
271 /*
272  * Build preemption-friendly versions of the following
273  * lock-spinning functions:
274  *
275  *         _[spin|read|write]_lock()
276  *         _[spin|read|write]_lock_irq()
277  *         _[spin|read|write]_lock_irqsave()
278  *         _[spin|read|write]_lock_bh()
279  */
280 BUILD_LOCK_OPS(spin, spinlock);
281 BUILD_LOCK_OPS(read, rwlock);
282 BUILD_LOCK_OPS(write, rwlock);
283
284 #endif /* CONFIG_PREEMPT */
285
286 #ifdef CONFIG_DEBUG_LOCK_ALLOC
287
288 void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass)
289 {
290         preempt_disable();
291         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
292         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
293 }
294
295 EXPORT_SYMBOL(_spin_lock_nested);
296 unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass)
297 {
298         unsigned long flags;
299
300         local_irq_save(flags);
301         preempt_disable();
302         spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
303         /*
304          * On lockdep we dont want the hand-coded irq-enable of
305          * _raw_spin_lock_flags() code, because lockdep assumes
306          * that interrupts are not re-enabled during lock-acquire:
307          */
308 #ifdef CONFIG_LOCKDEP
309         LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
310 #else
311         _raw_spin_lock_flags(lock, &flags);
312 #endif
313         return flags;
314 }
315
316 EXPORT_SYMBOL(_spin_lock_irqsave_nested);
317
318 #endif
319
320 void __lockfunc _spin_unlock(spinlock_t *lock)
321 {
322         spin_release(&lock->dep_map, 1, _RET_IP_);
323         _raw_spin_unlock(lock);
324         preempt_enable();
325 }
326 EXPORT_SYMBOL(_spin_unlock);
327
328 void __lockfunc _write_unlock(rwlock_t *lock)
329 {
330         rwlock_release(&lock->dep_map, 1, _RET_IP_);
331         _raw_write_unlock(lock);
332         preempt_enable();
333 }
334 EXPORT_SYMBOL(_write_unlock);
335
336 void __lockfunc _read_unlock(rwlock_t *lock)
337 {
338         rwlock_release(&lock->dep_map, 1, _RET_IP_);
339         _raw_read_unlock(lock);
340         preempt_enable();
341 }
342 EXPORT_SYMBOL(_read_unlock);
343
344 void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
345 {
346         spin_release(&lock->dep_map, 1, _RET_IP_);
347         _raw_spin_unlock(lock);
348         local_irq_restore(flags);
349         preempt_enable();
350 }
351 EXPORT_SYMBOL(_spin_unlock_irqrestore);
352
353 void __lockfunc _spin_unlock_irq(spinlock_t *lock)
354 {
355         spin_release(&lock->dep_map, 1, _RET_IP_);
356         _raw_spin_unlock(lock);
357         local_irq_enable();
358         preempt_enable();
359 }
360 EXPORT_SYMBOL(_spin_unlock_irq);
361
362 void __lockfunc _spin_unlock_bh(spinlock_t *lock)
363 {
364         spin_release(&lock->dep_map, 1, _RET_IP_);
365         _raw_spin_unlock(lock);
366         preempt_enable_no_resched();
367         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
368 }
369 EXPORT_SYMBOL(_spin_unlock_bh);
370
371 void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
372 {
373         rwlock_release(&lock->dep_map, 1, _RET_IP_);
374         _raw_read_unlock(lock);
375         local_irq_restore(flags);
376         preempt_enable();
377 }
378 EXPORT_SYMBOL(_read_unlock_irqrestore);
379
380 void __lockfunc _read_unlock_irq(rwlock_t *lock)
381 {
382         rwlock_release(&lock->dep_map, 1, _RET_IP_);
383         _raw_read_unlock(lock);
384         local_irq_enable();
385         preempt_enable();
386 }
387 EXPORT_SYMBOL(_read_unlock_irq);
388
389 void __lockfunc _read_unlock_bh(rwlock_t *lock)
390 {
391         rwlock_release(&lock->dep_map, 1, _RET_IP_);
392         _raw_read_unlock(lock);
393         preempt_enable_no_resched();
394         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
395 }
396 EXPORT_SYMBOL(_read_unlock_bh);
397
398 void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
399 {
400         rwlock_release(&lock->dep_map, 1, _RET_IP_);
401         _raw_write_unlock(lock);
402         local_irq_restore(flags);
403         preempt_enable();
404 }
405 EXPORT_SYMBOL(_write_unlock_irqrestore);
406
407 void __lockfunc _write_unlock_irq(rwlock_t *lock)
408 {
409         rwlock_release(&lock->dep_map, 1, _RET_IP_);
410         _raw_write_unlock(lock);
411         local_irq_enable();
412         preempt_enable();
413 }
414 EXPORT_SYMBOL(_write_unlock_irq);
415
416 void __lockfunc _write_unlock_bh(rwlock_t *lock)
417 {
418         rwlock_release(&lock->dep_map, 1, _RET_IP_);
419         _raw_write_unlock(lock);
420         preempt_enable_no_resched();
421         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
422 }
423 EXPORT_SYMBOL(_write_unlock_bh);
424
425 int __lockfunc _spin_trylock_bh(spinlock_t *lock)
426 {
427         local_bh_disable();
428         preempt_disable();
429         if (_raw_spin_trylock(lock)) {
430                 spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
431                 return 1;
432         }
433
434         preempt_enable_no_resched();
435         local_bh_enable_ip((unsigned long)__builtin_return_address(0));
436         return 0;
437 }
438 EXPORT_SYMBOL(_spin_trylock_bh);
439
440 int in_lock_functions(unsigned long addr)
441 {
442         /* Linker adds these: start and end of __lockfunc functions */
443         extern char __lock_text_start[], __lock_text_end[];
444
445         return addr >= (unsigned long)__lock_text_start
446         && addr < (unsigned long)__lock_text_end;
447 }
448 EXPORT_SYMBOL(in_lock_functions);