Pull sbs into release branch
[linux-2.6] / kernel / softirq.c
1 /*
2  *      linux/kernel/softirq.c
3  *
4  *      Copyright (C) 1992 Linus Torvalds
5  *
6  * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
7  */
8
9 #include <linux/module.h>
10 #include <linux/kernel_stat.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
13 #include <linux/mm.h>
14 #include <linux/notifier.h>
15 #include <linux/percpu.h>
16 #include <linux/cpu.h>
17 #include <linux/freezer.h>
18 #include <linux/kthread.h>
19 #include <linux/rcupdate.h>
20 #include <linux/smp.h>
21 #include <linux/tick.h>
22
23 #include <asm/irq.h>
24 /*
25    - No shared variables, all the data are CPU local.
26    - If a softirq needs serialization, let it serialize itself
27      by its own spinlocks.
28    - Even if softirq is serialized, only local cpu is marked for
29      execution. Hence, we get something sort of weak cpu binding.
30      Though it is still not clear, will it result in better locality
31      or will not.
32
33    Examples:
34    - NET RX softirq. It is multithreaded and does not require
35      any global serialization.
36    - NET TX softirq. It kicks software netdevice queues, hence
37      it is logically serialized per device, but this serialization
38      is invisible to common code.
39    - Tasklets: serialized wrt itself.
40  */
41
42 #ifndef __ARCH_IRQ_STAT
43 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
44 EXPORT_SYMBOL(irq_stat);
45 #endif
46
47 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
48
49 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
50
51 /*
52  * we cannot loop indefinitely here to avoid userspace starvation,
53  * but we also don't want to introduce a worst case 1/HZ latency
54  * to the pending events, so lets the scheduler to balance
55  * the softirq load for us.
56  */
57 static inline void wakeup_softirqd(void)
58 {
59         /* Interrupts are disabled: no need to stop preemption */
60         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
61
62         if (tsk && tsk->state != TASK_RUNNING)
63                 wake_up_process(tsk);
64 }
65
66 /*
67  * This one is for softirq.c-internal use,
68  * where hardirqs are disabled legitimately:
69  */
70 #ifdef CONFIG_TRACE_IRQFLAGS
71 static void __local_bh_disable(unsigned long ip)
72 {
73         unsigned long flags;
74
75         WARN_ON_ONCE(in_irq());
76
77         raw_local_irq_save(flags);
78         add_preempt_count(SOFTIRQ_OFFSET);
79         /*
80          * Were softirqs turned off above:
81          */
82         if (softirq_count() == SOFTIRQ_OFFSET)
83                 trace_softirqs_off(ip);
84         raw_local_irq_restore(flags);
85 }
86 #else /* !CONFIG_TRACE_IRQFLAGS */
87 static inline void __local_bh_disable(unsigned long ip)
88 {
89         add_preempt_count(SOFTIRQ_OFFSET);
90         barrier();
91 }
92 #endif /* CONFIG_TRACE_IRQFLAGS */
93
94 void local_bh_disable(void)
95 {
96         __local_bh_disable((unsigned long)__builtin_return_address(0));
97 }
98
99 EXPORT_SYMBOL(local_bh_disable);
100
101 void __local_bh_enable(void)
102 {
103         WARN_ON_ONCE(in_irq());
104
105         /*
106          * softirqs should never be enabled by __local_bh_enable(),
107          * it always nests inside local_bh_enable() sections:
108          */
109         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
110
111         sub_preempt_count(SOFTIRQ_OFFSET);
112 }
113 EXPORT_SYMBOL_GPL(__local_bh_enable);
114
115 /*
116  * Special-case - softirqs can safely be enabled in
117  * cond_resched_softirq(), or by __do_softirq(),
118  * without processing still-pending softirqs:
119  */
120 void _local_bh_enable(void)
121 {
122         WARN_ON_ONCE(in_irq());
123         WARN_ON_ONCE(!irqs_disabled());
124
125         if (softirq_count() == SOFTIRQ_OFFSET)
126                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
127         sub_preempt_count(SOFTIRQ_OFFSET);
128 }
129
130 EXPORT_SYMBOL(_local_bh_enable);
131
132 void local_bh_enable(void)
133 {
134 #ifdef CONFIG_TRACE_IRQFLAGS
135         unsigned long flags;
136
137         WARN_ON_ONCE(in_irq());
138 #endif
139         WARN_ON_ONCE(irqs_disabled());
140
141 #ifdef CONFIG_TRACE_IRQFLAGS
142         local_irq_save(flags);
143 #endif
144         /*
145          * Are softirqs going to be turned on now:
146          */
147         if (softirq_count() == SOFTIRQ_OFFSET)
148                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
149         /*
150          * Keep preemption disabled until we are done with
151          * softirq processing:
152          */
153         sub_preempt_count(SOFTIRQ_OFFSET - 1);
154
155         if (unlikely(!in_interrupt() && local_softirq_pending()))
156                 do_softirq();
157
158         dec_preempt_count();
159 #ifdef CONFIG_TRACE_IRQFLAGS
160         local_irq_restore(flags);
161 #endif
162         preempt_check_resched();
163 }
164 EXPORT_SYMBOL(local_bh_enable);
165
166 void local_bh_enable_ip(unsigned long ip)
167 {
168 #ifdef CONFIG_TRACE_IRQFLAGS
169         unsigned long flags;
170
171         WARN_ON_ONCE(in_irq());
172
173         local_irq_save(flags);
174 #endif
175         /*
176          * Are softirqs going to be turned on now:
177          */
178         if (softirq_count() == SOFTIRQ_OFFSET)
179                 trace_softirqs_on(ip);
180         /*
181          * Keep preemption disabled until we are done with
182          * softirq processing:
183          */
184         sub_preempt_count(SOFTIRQ_OFFSET - 1);
185
186         if (unlikely(!in_interrupt() && local_softirq_pending()))
187                 do_softirq();
188
189         dec_preempt_count();
190 #ifdef CONFIG_TRACE_IRQFLAGS
191         local_irq_restore(flags);
192 #endif
193         preempt_check_resched();
194 }
195 EXPORT_SYMBOL(local_bh_enable_ip);
196
197 /*
198  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
199  * and we fall back to softirqd after that.
200  *
201  * This number has been established via experimentation.
202  * The two things to balance is latency against fairness -
203  * we want to handle softirqs as soon as possible, but they
204  * should not be able to lock up the box.
205  */
206 #define MAX_SOFTIRQ_RESTART 10
207
208 asmlinkage void __do_softirq(void)
209 {
210         struct softirq_action *h;
211         __u32 pending;
212         int max_restart = MAX_SOFTIRQ_RESTART;
213         int cpu;
214
215         pending = local_softirq_pending();
216         account_system_vtime(current);
217
218         __local_bh_disable((unsigned long)__builtin_return_address(0));
219         trace_softirq_enter();
220
221         cpu = smp_processor_id();
222 restart:
223         /* Reset the pending bitmask before enabling irqs */
224         set_softirq_pending(0);
225
226         local_irq_enable();
227
228         h = softirq_vec;
229
230         do {
231                 if (pending & 1) {
232                         h->action(h);
233                         rcu_bh_qsctr_inc(cpu);
234                 }
235                 h++;
236                 pending >>= 1;
237         } while (pending);
238
239         local_irq_disable();
240
241         pending = local_softirq_pending();
242         if (pending && --max_restart)
243                 goto restart;
244
245         if (pending)
246                 wakeup_softirqd();
247
248         trace_softirq_exit();
249
250         account_system_vtime(current);
251         _local_bh_enable();
252 }
253
254 #ifndef __ARCH_HAS_DO_SOFTIRQ
255
256 asmlinkage void do_softirq(void)
257 {
258         __u32 pending;
259         unsigned long flags;
260
261         if (in_interrupt())
262                 return;
263
264         local_irq_save(flags);
265
266         pending = local_softirq_pending();
267
268         if (pending)
269                 __do_softirq();
270
271         local_irq_restore(flags);
272 }
273
274 EXPORT_SYMBOL(do_softirq);
275
276 #endif
277
278 /*
279  * Enter an interrupt context.
280  */
281 void irq_enter(void)
282 {
283         __irq_enter();
284 #ifdef CONFIG_NO_HZ
285         if (idle_cpu(smp_processor_id()))
286                 tick_nohz_update_jiffies();
287 #endif
288 }
289
290 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
291 # define invoke_softirq()       __do_softirq()
292 #else
293 # define invoke_softirq()       do_softirq()
294 #endif
295
296 /*
297  * Exit an interrupt context. Process softirqs if needed and possible:
298  */
299 void irq_exit(void)
300 {
301         account_system_vtime(current);
302         trace_hardirq_exit();
303         sub_preempt_count(IRQ_EXIT_OFFSET);
304         if (!in_interrupt() && local_softirq_pending())
305                 invoke_softirq();
306
307 #ifdef CONFIG_NO_HZ
308         /* Make sure that timer wheel updates are propagated */
309         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
310                 tick_nohz_stop_sched_tick();
311 #endif
312         preempt_enable_no_resched();
313 }
314
315 /*
316  * This function must run with irqs disabled!
317  */
318 inline fastcall void raise_softirq_irqoff(unsigned int nr)
319 {
320         __raise_softirq_irqoff(nr);
321
322         /*
323          * If we're in an interrupt or softirq, we're done
324          * (this also catches softirq-disabled code). We will
325          * actually run the softirq once we return from
326          * the irq or softirq.
327          *
328          * Otherwise we wake up ksoftirqd to make sure we
329          * schedule the softirq soon.
330          */
331         if (!in_interrupt())
332                 wakeup_softirqd();
333 }
334
335 EXPORT_SYMBOL(raise_softirq_irqoff);
336
337 void fastcall raise_softirq(unsigned int nr)
338 {
339         unsigned long flags;
340
341         local_irq_save(flags);
342         raise_softirq_irqoff(nr);
343         local_irq_restore(flags);
344 }
345
346 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
347 {
348         softirq_vec[nr].data = data;
349         softirq_vec[nr].action = action;
350 }
351
352 /* Tasklets */
353 struct tasklet_head
354 {
355         struct tasklet_struct *list;
356 };
357
358 /* Some compilers disobey section attribute on statics when not
359    initialized -- RR */
360 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
361 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
362
363 void fastcall __tasklet_schedule(struct tasklet_struct *t)
364 {
365         unsigned long flags;
366
367         local_irq_save(flags);
368         t->next = __get_cpu_var(tasklet_vec).list;
369         __get_cpu_var(tasklet_vec).list = t;
370         raise_softirq_irqoff(TASKLET_SOFTIRQ);
371         local_irq_restore(flags);
372 }
373
374 EXPORT_SYMBOL(__tasklet_schedule);
375
376 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
377 {
378         unsigned long flags;
379
380         local_irq_save(flags);
381         t->next = __get_cpu_var(tasklet_hi_vec).list;
382         __get_cpu_var(tasklet_hi_vec).list = t;
383         raise_softirq_irqoff(HI_SOFTIRQ);
384         local_irq_restore(flags);
385 }
386
387 EXPORT_SYMBOL(__tasklet_hi_schedule);
388
389 static void tasklet_action(struct softirq_action *a)
390 {
391         struct tasklet_struct *list;
392
393         local_irq_disable();
394         list = __get_cpu_var(tasklet_vec).list;
395         __get_cpu_var(tasklet_vec).list = NULL;
396         local_irq_enable();
397
398         while (list) {
399                 struct tasklet_struct *t = list;
400
401                 list = list->next;
402
403                 if (tasklet_trylock(t)) {
404                         if (!atomic_read(&t->count)) {
405                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
406                                         BUG();
407                                 t->func(t->data);
408                                 tasklet_unlock(t);
409                                 continue;
410                         }
411                         tasklet_unlock(t);
412                 }
413
414                 local_irq_disable();
415                 t->next = __get_cpu_var(tasklet_vec).list;
416                 __get_cpu_var(tasklet_vec).list = t;
417                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
418                 local_irq_enable();
419         }
420 }
421
422 static void tasklet_hi_action(struct softirq_action *a)
423 {
424         struct tasklet_struct *list;
425
426         local_irq_disable();
427         list = __get_cpu_var(tasklet_hi_vec).list;
428         __get_cpu_var(tasklet_hi_vec).list = NULL;
429         local_irq_enable();
430
431         while (list) {
432                 struct tasklet_struct *t = list;
433
434                 list = list->next;
435
436                 if (tasklet_trylock(t)) {
437                         if (!atomic_read(&t->count)) {
438                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
439                                         BUG();
440                                 t->func(t->data);
441                                 tasklet_unlock(t);
442                                 continue;
443                         }
444                         tasklet_unlock(t);
445                 }
446
447                 local_irq_disable();
448                 t->next = __get_cpu_var(tasklet_hi_vec).list;
449                 __get_cpu_var(tasklet_hi_vec).list = t;
450                 __raise_softirq_irqoff(HI_SOFTIRQ);
451                 local_irq_enable();
452         }
453 }
454
455
456 void tasklet_init(struct tasklet_struct *t,
457                   void (*func)(unsigned long), unsigned long data)
458 {
459         t->next = NULL;
460         t->state = 0;
461         atomic_set(&t->count, 0);
462         t->func = func;
463         t->data = data;
464 }
465
466 EXPORT_SYMBOL(tasklet_init);
467
468 void tasklet_kill(struct tasklet_struct *t)
469 {
470         if (in_interrupt())
471                 printk("Attempt to kill tasklet from interrupt\n");
472
473         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
474                 do
475                         yield();
476                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
477         }
478         tasklet_unlock_wait(t);
479         clear_bit(TASKLET_STATE_SCHED, &t->state);
480 }
481
482 EXPORT_SYMBOL(tasklet_kill);
483
484 void __init softirq_init(void)
485 {
486         open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
487         open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
488 }
489
490 static int ksoftirqd(void * __bind_cpu)
491 {
492         set_current_state(TASK_INTERRUPTIBLE);
493
494         while (!kthread_should_stop()) {
495                 preempt_disable();
496                 if (!local_softirq_pending()) {
497                         preempt_enable_no_resched();
498                         schedule();
499                         preempt_disable();
500                 }
501
502                 __set_current_state(TASK_RUNNING);
503
504                 while (local_softirq_pending()) {
505                         /* Preempt disable stops cpu going offline.
506                            If already offline, we'll be on wrong CPU:
507                            don't process */
508                         if (cpu_is_offline((long)__bind_cpu))
509                                 goto wait_to_die;
510                         do_softirq();
511                         preempt_enable_no_resched();
512                         cond_resched();
513                         preempt_disable();
514                 }
515                 preempt_enable();
516                 set_current_state(TASK_INTERRUPTIBLE);
517         }
518         __set_current_state(TASK_RUNNING);
519         return 0;
520
521 wait_to_die:
522         preempt_enable();
523         /* Wait for kthread_stop */
524         set_current_state(TASK_INTERRUPTIBLE);
525         while (!kthread_should_stop()) {
526                 schedule();
527                 set_current_state(TASK_INTERRUPTIBLE);
528         }
529         __set_current_state(TASK_RUNNING);
530         return 0;
531 }
532
533 #ifdef CONFIG_HOTPLUG_CPU
534 /*
535  * tasklet_kill_immediate is called to remove a tasklet which can already be
536  * scheduled for execution on @cpu.
537  *
538  * Unlike tasklet_kill, this function removes the tasklet
539  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
540  *
541  * When this function is called, @cpu must be in the CPU_DEAD state.
542  */
543 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
544 {
545         struct tasklet_struct **i;
546
547         BUG_ON(cpu_online(cpu));
548         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
549
550         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
551                 return;
552
553         /* CPU is dead, so no lock needed. */
554         for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
555                 if (*i == t) {
556                         *i = t->next;
557                         return;
558                 }
559         }
560         BUG();
561 }
562
563 static void takeover_tasklets(unsigned int cpu)
564 {
565         struct tasklet_struct **i;
566
567         /* CPU is dead, so no lock needed. */
568         local_irq_disable();
569
570         /* Find end, append list for that CPU. */
571         for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
572         *i = per_cpu(tasklet_vec, cpu).list;
573         per_cpu(tasklet_vec, cpu).list = NULL;
574         raise_softirq_irqoff(TASKLET_SOFTIRQ);
575
576         for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
577         *i = per_cpu(tasklet_hi_vec, cpu).list;
578         per_cpu(tasklet_hi_vec, cpu).list = NULL;
579         raise_softirq_irqoff(HI_SOFTIRQ);
580
581         local_irq_enable();
582 }
583 #endif /* CONFIG_HOTPLUG_CPU */
584
585 static int __cpuinit cpu_callback(struct notifier_block *nfb,
586                                   unsigned long action,
587                                   void *hcpu)
588 {
589         int hotcpu = (unsigned long)hcpu;
590         struct task_struct *p;
591
592         switch (action) {
593         case CPU_UP_PREPARE:
594         case CPU_UP_PREPARE_FROZEN:
595                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
596                 if (IS_ERR(p)) {
597                         printk("ksoftirqd for %i failed\n", hotcpu);
598                         return NOTIFY_BAD;
599                 }
600                 kthread_bind(p, hotcpu);
601                 per_cpu(ksoftirqd, hotcpu) = p;
602                 break;
603         case CPU_ONLINE:
604         case CPU_ONLINE_FROZEN:
605                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
606                 break;
607 #ifdef CONFIG_HOTPLUG_CPU
608         case CPU_UP_CANCELED:
609         case CPU_UP_CANCELED_FROZEN:
610                 if (!per_cpu(ksoftirqd, hotcpu))
611                         break;
612                 /* Unbind so it can run.  Fall thru. */
613                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
614                              any_online_cpu(cpu_online_map));
615         case CPU_DEAD:
616         case CPU_DEAD_FROZEN: {
617                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
618
619                 p = per_cpu(ksoftirqd, hotcpu);
620                 per_cpu(ksoftirqd, hotcpu) = NULL;
621                 sched_setscheduler(p, SCHED_FIFO, &param);
622                 kthread_stop(p);
623                 takeover_tasklets(hotcpu);
624                 break;
625         }
626 #endif /* CONFIG_HOTPLUG_CPU */
627         }
628         return NOTIFY_OK;
629 }
630
631 static struct notifier_block __cpuinitdata cpu_nfb = {
632         .notifier_call = cpu_callback
633 };
634
635 __init int spawn_ksoftirqd(void)
636 {
637         void *cpu = (void *)(long)smp_processor_id();
638         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
639
640         BUG_ON(err == NOTIFY_BAD);
641         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
642         register_cpu_notifier(&cpu_nfb);
643         return 0;
644 }
645
646 #ifdef CONFIG_SMP
647 /*
648  * Call a function on all processors
649  */
650 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
651 {
652         int ret = 0;
653
654         preempt_disable();
655         ret = smp_call_function(func, info, retry, wait);
656         local_irq_disable();
657         func(info);
658         local_irq_enable();
659         preempt_enable();
660         return ret;
661 }
662 EXPORT_SYMBOL(on_each_cpu);
663 #endif