Merge branch 'linus' into x86/cleanups
[linux-2.6] / arch / x86 / xen / spinlock.c
1 /*
2  * Split spinlock implementation out into its own file, so it can be
3  * compiled in a FTRACE-compatible way.
4  */
5 #include <linux/kernel_stat.h>
6 #include <linux/spinlock.h>
7 #include <linux/debugfs.h>
8 #include <linux/log2.h>
9
10 #include <asm/paravirt.h>
11
12 #include <xen/interface/xen.h>
13 #include <xen/events.h>
14
15 #include "xen-ops.h"
16 #include "debugfs.h"
17
18 #ifdef CONFIG_XEN_DEBUG_FS
19 static struct xen_spinlock_stats
20 {
21         u64 taken;
22         u32 taken_slow;
23         u32 taken_slow_nested;
24         u32 taken_slow_pickup;
25         u32 taken_slow_spurious;
26         u32 taken_slow_irqenable;
27
28         u64 released;
29         u32 released_slow;
30         u32 released_slow_kicked;
31
32 #define HISTO_BUCKETS   30
33         u32 histo_spin_total[HISTO_BUCKETS+1];
34         u32 histo_spin_spinning[HISTO_BUCKETS+1];
35         u32 histo_spin_blocked[HISTO_BUCKETS+1];
36
37         u64 time_total;
38         u64 time_spinning;
39         u64 time_blocked;
40 } spinlock_stats;
41
42 static u8 zero_stats;
43
44 static unsigned lock_timeout = 1 << 10;
45 #define TIMEOUT lock_timeout
46
47 static inline void check_zero(void)
48 {
49         if (unlikely(zero_stats)) {
50                 memset(&spinlock_stats, 0, sizeof(spinlock_stats));
51                 zero_stats = 0;
52         }
53 }
54
55 #define ADD_STATS(elem, val)                    \
56         do { check_zero(); spinlock_stats.elem += (val); } while(0)
57
58 static inline u64 spin_time_start(void)
59 {
60         return xen_clocksource_read();
61 }
62
63 static void __spin_time_accum(u64 delta, u32 *array)
64 {
65         unsigned index = ilog2(delta);
66
67         check_zero();
68
69         if (index < HISTO_BUCKETS)
70                 array[index]++;
71         else
72                 array[HISTO_BUCKETS]++;
73 }
74
75 static inline void spin_time_accum_spinning(u64 start)
76 {
77         u32 delta = xen_clocksource_read() - start;
78
79         __spin_time_accum(delta, spinlock_stats.histo_spin_spinning);
80         spinlock_stats.time_spinning += delta;
81 }
82
83 static inline void spin_time_accum_total(u64 start)
84 {
85         u32 delta = xen_clocksource_read() - start;
86
87         __spin_time_accum(delta, spinlock_stats.histo_spin_total);
88         spinlock_stats.time_total += delta;
89 }
90
91 static inline void spin_time_accum_blocked(u64 start)
92 {
93         u32 delta = xen_clocksource_read() - start;
94
95         __spin_time_accum(delta, spinlock_stats.histo_spin_blocked);
96         spinlock_stats.time_blocked += delta;
97 }
98 #else  /* !CONFIG_XEN_DEBUG_FS */
99 #define TIMEOUT                 (1 << 10)
100 #define ADD_STATS(elem, val)    do { (void)(val); } while(0)
101
102 static inline u64 spin_time_start(void)
103 {
104         return 0;
105 }
106
107 static inline void spin_time_accum_total(u64 start)
108 {
109 }
110 static inline void spin_time_accum_spinning(u64 start)
111 {
112 }
113 static inline void spin_time_accum_blocked(u64 start)
114 {
115 }
116 #endif  /* CONFIG_XEN_DEBUG_FS */
117
118 struct xen_spinlock {
119         unsigned char lock;             /* 0 -> free; 1 -> locked */
120         unsigned short spinners;        /* count of waiting cpus */
121 };
122
123 static int xen_spin_is_locked(struct raw_spinlock *lock)
124 {
125         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
126
127         return xl->lock != 0;
128 }
129
130 static int xen_spin_is_contended(struct raw_spinlock *lock)
131 {
132         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
133
134         /* Not strictly true; this is only the count of contended
135            lock-takers entering the slow path. */
136         return xl->spinners != 0;
137 }
138
139 static int xen_spin_trylock(struct raw_spinlock *lock)
140 {
141         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
142         u8 old = 1;
143
144         asm("xchgb %b0,%1"
145             : "+q" (old), "+m" (xl->lock) : : "memory");
146
147         return old == 0;
148 }
149
150 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
151 static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners);
152
153 /*
154  * Mark a cpu as interested in a lock.  Returns the CPU's previous
155  * lock of interest, in case we got preempted by an interrupt.
156  */
157 static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl)
158 {
159         struct xen_spinlock *prev;
160
161         prev = __get_cpu_var(lock_spinners);
162         __get_cpu_var(lock_spinners) = xl;
163
164         wmb();                  /* set lock of interest before count */
165
166         asm(LOCK_PREFIX " incw %0"
167             : "+m" (xl->spinners) : : "memory");
168
169         return prev;
170 }
171
172 /*
173  * Mark a cpu as no longer interested in a lock.  Restores previous
174  * lock of interest (NULL for none).
175  */
176 static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev)
177 {
178         asm(LOCK_PREFIX " decw %0"
179             : "+m" (xl->spinners) : : "memory");
180         wmb();                  /* decrement count before restoring lock */
181         __get_cpu_var(lock_spinners) = prev;
182 }
183
184 static noinline int xen_spin_lock_slow(struct raw_spinlock *lock, bool irq_enable)
185 {
186         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
187         struct xen_spinlock *prev;
188         int irq = __get_cpu_var(lock_kicker_irq);
189         int ret;
190         unsigned long flags;
191         u64 start;
192
193         /* If kicker interrupts not initialized yet, just spin */
194         if (irq == -1)
195                 return 0;
196
197         start = spin_time_start();
198
199         /* announce we're spinning */
200         prev = spinning_lock(xl);
201
202         flags = __raw_local_save_flags();
203         if (irq_enable) {
204                 ADD_STATS(taken_slow_irqenable, 1);
205                 raw_local_irq_enable();
206         }
207
208         ADD_STATS(taken_slow, 1);
209         ADD_STATS(taken_slow_nested, prev != NULL);
210
211         do {
212                 /* clear pending */
213                 xen_clear_irq_pending(irq);
214
215                 /* check again make sure it didn't become free while
216                    we weren't looking  */
217                 ret = xen_spin_trylock(lock);
218                 if (ret) {
219                         ADD_STATS(taken_slow_pickup, 1);
220
221                         /*
222                          * If we interrupted another spinlock while it
223                          * was blocking, make sure it doesn't block
224                          * without rechecking the lock.
225                          */
226                         if (prev != NULL)
227                                 xen_set_irq_pending(irq);
228                         goto out;
229                 }
230
231                 /*
232                  * Block until irq becomes pending.  If we're
233                  * interrupted at this point (after the trylock but
234                  * before entering the block), then the nested lock
235                  * handler guarantees that the irq will be left
236                  * pending if there's any chance the lock became free;
237                  * xen_poll_irq() returns immediately if the irq is
238                  * pending.
239                  */
240                 xen_poll_irq(irq);
241                 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq));
242         } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */
243
244         kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
245
246 out:
247         raw_local_irq_restore(flags);
248         unspinning_lock(xl, prev);
249         spin_time_accum_blocked(start);
250
251         return ret;
252 }
253
254 static inline void __xen_spin_lock(struct raw_spinlock *lock, bool irq_enable)
255 {
256         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
257         unsigned timeout;
258         u8 oldval;
259         u64 start_spin;
260
261         ADD_STATS(taken, 1);
262
263         start_spin = spin_time_start();
264
265         do {
266                 u64 start_spin_fast = spin_time_start();
267
268                 timeout = TIMEOUT;
269
270                 asm("1: xchgb %1,%0\n"
271                     "   testb %1,%1\n"
272                     "   jz 3f\n"
273                     "2: rep;nop\n"
274                     "   cmpb $0,%0\n"
275                     "   je 1b\n"
276                     "   dec %2\n"
277                     "   jnz 2b\n"
278                     "3:\n"
279                     : "+m" (xl->lock), "=q" (oldval), "+r" (timeout)
280                     : "1" (1)
281                     : "memory");
282
283                 spin_time_accum_spinning(start_spin_fast);
284
285         } while (unlikely(oldval != 0 &&
286                           (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable))));
287
288         spin_time_accum_total(start_spin);
289 }
290
291 static void xen_spin_lock(struct raw_spinlock *lock)
292 {
293         __xen_spin_lock(lock, false);
294 }
295
296 static void xen_spin_lock_flags(struct raw_spinlock *lock, unsigned long flags)
297 {
298         __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags));
299 }
300
301 static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl)
302 {
303         int cpu;
304
305         ADD_STATS(released_slow, 1);
306
307         for_each_online_cpu(cpu) {
308                 /* XXX should mix up next cpu selection */
309                 if (per_cpu(lock_spinners, cpu) == xl) {
310                         ADD_STATS(released_slow_kicked, 1);
311                         xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
312                         break;
313                 }
314         }
315 }
316
317 static void xen_spin_unlock(struct raw_spinlock *lock)
318 {
319         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
320
321         ADD_STATS(released, 1);
322
323         smp_wmb();              /* make sure no writes get moved after unlock */
324         xl->lock = 0;           /* release lock */
325
326         /* make sure unlock happens before kick */
327         barrier();
328
329         if (unlikely(xl->spinners))
330                 xen_spin_unlock_slow(xl);
331 }
332
333 static irqreturn_t dummy_handler(int irq, void *dev_id)
334 {
335         BUG();
336         return IRQ_HANDLED;
337 }
338
339 void __cpuinit xen_init_lock_cpu(int cpu)
340 {
341         int irq;
342         const char *name;
343
344         name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
345         irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
346                                      cpu,
347                                      dummy_handler,
348                                      IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
349                                      name,
350                                      NULL);
351
352         if (irq >= 0) {
353                 disable_irq(irq); /* make sure it's never delivered */
354                 per_cpu(lock_kicker_irq, cpu) = irq;
355         }
356
357         printk("cpu %d spinlock event irq %d\n", cpu, irq);
358 }
359
360 void xen_uninit_lock_cpu(int cpu)
361 {
362         unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL);
363 }
364
365 void __init xen_init_spinlocks(void)
366 {
367         pv_lock_ops.spin_is_locked = xen_spin_is_locked;
368         pv_lock_ops.spin_is_contended = xen_spin_is_contended;
369         pv_lock_ops.spin_lock = xen_spin_lock;
370         pv_lock_ops.spin_lock_flags = xen_spin_lock_flags;
371         pv_lock_ops.spin_trylock = xen_spin_trylock;
372         pv_lock_ops.spin_unlock = xen_spin_unlock;
373 }
374
375 #ifdef CONFIG_XEN_DEBUG_FS
376
377 static struct dentry *d_spin_debug;
378
379 static int __init xen_spinlock_debugfs(void)
380 {
381         struct dentry *d_xen = xen_init_debugfs();
382
383         if (d_xen == NULL)
384                 return -ENOMEM;
385
386         d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
387
388         debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
389
390         debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout);
391
392         debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken);
393         debugfs_create_u32("taken_slow", 0444, d_spin_debug,
394                            &spinlock_stats.taken_slow);
395         debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug,
396                            &spinlock_stats.taken_slow_nested);
397         debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
398                            &spinlock_stats.taken_slow_pickup);
399         debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug,
400                            &spinlock_stats.taken_slow_spurious);
401         debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug,
402                            &spinlock_stats.taken_slow_irqenable);
403
404         debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released);
405         debugfs_create_u32("released_slow", 0444, d_spin_debug,
406                            &spinlock_stats.released_slow);
407         debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
408                            &spinlock_stats.released_slow_kicked);
409
410         debugfs_create_u64("time_spinning", 0444, d_spin_debug,
411                            &spinlock_stats.time_spinning);
412         debugfs_create_u64("time_blocked", 0444, d_spin_debug,
413                            &spinlock_stats.time_blocked);
414         debugfs_create_u64("time_total", 0444, d_spin_debug,
415                            &spinlock_stats.time_total);
416
417         xen_debugfs_create_u32_array("histo_total", 0444, d_spin_debug,
418                                      spinlock_stats.histo_spin_total, HISTO_BUCKETS + 1);
419         xen_debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug,
420                                      spinlock_stats.histo_spin_spinning, HISTO_BUCKETS + 1);
421         xen_debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
422                                      spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
423
424         return 0;
425 }
426 fs_initcall(xen_spinlock_debugfs);
427
428 #endif  /* CONFIG_XEN_DEBUG_FS */