cgroups: fix invalid cgrp->dentry before cgroup has been completely removed
[linux-2.6] / kernel / softirq.c
1 /*
2  *      linux/kernel/softirq.c
3  *
4  *      Copyright (C) 1992 Linus Torvalds
5  *
6  *      Distribute under GPLv2.
7  *
8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9  *
10  *      Remote softirq infrastructure is by Jens Axboe.
11  */
12
13 #include <linux/module.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/init.h>
17 #include <linux/mm.h>
18 #include <linux/notifier.h>
19 #include <linux/percpu.h>
20 #include <linux/cpu.h>
21 #include <linux/freezer.h>
22 #include <linux/kthread.h>
23 #include <linux/rcupdate.h>
24 #include <linux/smp.h>
25 #include <linux/tick.h>
26
27 #include <asm/irq.h>
28 /*
29    - No shared variables, all the data are CPU local.
30    - If a softirq needs serialization, let it serialize itself
31      by its own spinlocks.
32    - Even if softirq is serialized, only local cpu is marked for
33      execution. Hence, we get something sort of weak cpu binding.
34      Though it is still not clear, will it result in better locality
35      or will not.
36
37    Examples:
38    - NET RX softirq. It is multithreaded and does not require
39      any global serialization.
40    - NET TX softirq. It kicks software netdevice queues, hence
41      it is logically serialized per device, but this serialization
42      is invisible to common code.
43    - Tasklets: serialized wrt itself.
44  */
45
46 #ifndef __ARCH_IRQ_STAT
47 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
48 EXPORT_SYMBOL(irq_stat);
49 #endif
50
51 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
52
53 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
54
55 /*
56  * we cannot loop indefinitely here to avoid userspace starvation,
57  * but we also don't want to introduce a worst case 1/HZ latency
58  * to the pending events, so lets the scheduler to balance
59  * the softirq load for us.
60  */
61 static inline void wakeup_softirqd(void)
62 {
63         /* Interrupts are disabled: no need to stop preemption */
64         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
65
66         if (tsk && tsk->state != TASK_RUNNING)
67                 wake_up_process(tsk);
68 }
69
70 /*
71  * This one is for softirq.c-internal use,
72  * where hardirqs are disabled legitimately:
73  */
74 #ifdef CONFIG_TRACE_IRQFLAGS
75 static void __local_bh_disable(unsigned long ip)
76 {
77         unsigned long flags;
78
79         WARN_ON_ONCE(in_irq());
80
81         raw_local_irq_save(flags);
82         add_preempt_count(SOFTIRQ_OFFSET);
83         /*
84          * Were softirqs turned off above:
85          */
86         if (softirq_count() == SOFTIRQ_OFFSET)
87                 trace_softirqs_off(ip);
88         raw_local_irq_restore(flags);
89 }
90 #else /* !CONFIG_TRACE_IRQFLAGS */
91 static inline void __local_bh_disable(unsigned long ip)
92 {
93         add_preempt_count(SOFTIRQ_OFFSET);
94         barrier();
95 }
96 #endif /* CONFIG_TRACE_IRQFLAGS */
97
98 void local_bh_disable(void)
99 {
100         __local_bh_disable((unsigned long)__builtin_return_address(0));
101 }
102
103 EXPORT_SYMBOL(local_bh_disable);
104
105 void __local_bh_enable(void)
106 {
107         WARN_ON_ONCE(in_irq());
108
109         /*
110          * softirqs should never be enabled by __local_bh_enable(),
111          * it always nests inside local_bh_enable() sections:
112          */
113         WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
114
115         sub_preempt_count(SOFTIRQ_OFFSET);
116 }
117 EXPORT_SYMBOL_GPL(__local_bh_enable);
118
119 /*
120  * Special-case - softirqs can safely be enabled in
121  * cond_resched_softirq(), or by __do_softirq(),
122  * without processing still-pending softirqs:
123  */
124 void _local_bh_enable(void)
125 {
126         WARN_ON_ONCE(in_irq());
127         WARN_ON_ONCE(!irqs_disabled());
128
129         if (softirq_count() == SOFTIRQ_OFFSET)
130                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
131         sub_preempt_count(SOFTIRQ_OFFSET);
132 }
133
134 EXPORT_SYMBOL(_local_bh_enable);
135
136 static inline void _local_bh_enable_ip(unsigned long ip)
137 {
138         WARN_ON_ONCE(in_irq() || irqs_disabled());
139 #ifdef CONFIG_TRACE_IRQFLAGS
140         local_irq_disable();
141 #endif
142         /*
143          * Are softirqs going to be turned on now:
144          */
145         if (softirq_count() == SOFTIRQ_OFFSET)
146                 trace_softirqs_on(ip);
147         /*
148          * Keep preemption disabled until we are done with
149          * softirq processing:
150          */
151         sub_preempt_count(SOFTIRQ_OFFSET - 1);
152
153         if (unlikely(!in_interrupt() && local_softirq_pending()))
154                 do_softirq();
155
156         dec_preempt_count();
157 #ifdef CONFIG_TRACE_IRQFLAGS
158         local_irq_enable();
159 #endif
160         preempt_check_resched();
161 }
162
163 void local_bh_enable(void)
164 {
165         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
166 }
167 EXPORT_SYMBOL(local_bh_enable);
168
169 void local_bh_enable_ip(unsigned long ip)
170 {
171         _local_bh_enable_ip(ip);
172 }
173 EXPORT_SYMBOL(local_bh_enable_ip);
174
175 /*
176  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
177  * and we fall back to softirqd after that.
178  *
179  * This number has been established via experimentation.
180  * The two things to balance is latency against fairness -
181  * we want to handle softirqs as soon as possible, but they
182  * should not be able to lock up the box.
183  */
184 #define MAX_SOFTIRQ_RESTART 10
185
186 asmlinkage void __do_softirq(void)
187 {
188         struct softirq_action *h;
189         __u32 pending;
190         int max_restart = MAX_SOFTIRQ_RESTART;
191         int cpu;
192
193         pending = local_softirq_pending();
194         account_system_vtime(current);
195
196         __local_bh_disable((unsigned long)__builtin_return_address(0));
197         trace_softirq_enter();
198
199         cpu = smp_processor_id();
200 restart:
201         /* Reset the pending bitmask before enabling irqs */
202         set_softirq_pending(0);
203
204         local_irq_enable();
205
206         h = softirq_vec;
207
208         do {
209                 if (pending & 1) {
210                         int prev_count = preempt_count();
211
212                         h->action(h);
213
214                         if (unlikely(prev_count != preempt_count())) {
215                                 printk(KERN_ERR "huh, entered softirq %td %p"
216                                        "with preempt_count %08x,"
217                                        " exited with %08x?\n", h - softirq_vec,
218                                        h->action, prev_count, preempt_count());
219                                 preempt_count() = prev_count;
220                         }
221
222                         rcu_bh_qsctr_inc(cpu);
223                 }
224                 h++;
225                 pending >>= 1;
226         } while (pending);
227
228         local_irq_disable();
229
230         pending = local_softirq_pending();
231         if (pending && --max_restart)
232                 goto restart;
233
234         if (pending)
235                 wakeup_softirqd();
236
237         trace_softirq_exit();
238
239         account_system_vtime(current);
240         _local_bh_enable();
241 }
242
243 #ifndef __ARCH_HAS_DO_SOFTIRQ
244
245 asmlinkage void do_softirq(void)
246 {
247         __u32 pending;
248         unsigned long flags;
249
250         if (in_interrupt())
251                 return;
252
253         local_irq_save(flags);
254
255         pending = local_softirq_pending();
256
257         if (pending)
258                 __do_softirq();
259
260         local_irq_restore(flags);
261 }
262
263 #endif
264
265 /*
266  * Enter an interrupt context.
267  */
268 void irq_enter(void)
269 {
270         int cpu = smp_processor_id();
271
272         if (idle_cpu(cpu) && !in_interrupt())
273                 tick_check_idle(cpu);
274
275         __irq_enter();
276 }
277
278 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
279 # define invoke_softirq()       __do_softirq()
280 #else
281 # define invoke_softirq()       do_softirq()
282 #endif
283
284 /*
285  * Exit an interrupt context. Process softirqs if needed and possible:
286  */
287 void irq_exit(void)
288 {
289         account_system_vtime(current);
290         trace_hardirq_exit();
291         sub_preempt_count(IRQ_EXIT_OFFSET);
292         if (!in_interrupt() && local_softirq_pending())
293                 invoke_softirq();
294
295 #ifdef CONFIG_NO_HZ
296         /* Make sure that timer wheel updates are propagated */
297         if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
298                 tick_nohz_stop_sched_tick(0);
299         rcu_irq_exit();
300 #endif
301         preempt_enable_no_resched();
302 }
303
304 /*
305  * This function must run with irqs disabled!
306  */
307 inline void raise_softirq_irqoff(unsigned int nr)
308 {
309         __raise_softirq_irqoff(nr);
310
311         /*
312          * If we're in an interrupt or softirq, we're done
313          * (this also catches softirq-disabled code). We will
314          * actually run the softirq once we return from
315          * the irq or softirq.
316          *
317          * Otherwise we wake up ksoftirqd to make sure we
318          * schedule the softirq soon.
319          */
320         if (!in_interrupt())
321                 wakeup_softirqd();
322 }
323
324 void raise_softirq(unsigned int nr)
325 {
326         unsigned long flags;
327
328         local_irq_save(flags);
329         raise_softirq_irqoff(nr);
330         local_irq_restore(flags);
331 }
332
333 void open_softirq(int nr, void (*action)(struct softirq_action *))
334 {
335         softirq_vec[nr].action = action;
336 }
337
338 /* Tasklets */
339 struct tasklet_head
340 {
341         struct tasklet_struct *head;
342         struct tasklet_struct **tail;
343 };
344
345 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
346 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
347
348 void __tasklet_schedule(struct tasklet_struct *t)
349 {
350         unsigned long flags;
351
352         local_irq_save(flags);
353         t->next = NULL;
354         *__get_cpu_var(tasklet_vec).tail = t;
355         __get_cpu_var(tasklet_vec).tail = &(t->next);
356         raise_softirq_irqoff(TASKLET_SOFTIRQ);
357         local_irq_restore(flags);
358 }
359
360 EXPORT_SYMBOL(__tasklet_schedule);
361
362 void __tasklet_hi_schedule(struct tasklet_struct *t)
363 {
364         unsigned long flags;
365
366         local_irq_save(flags);
367         t->next = NULL;
368         *__get_cpu_var(tasklet_hi_vec).tail = t;
369         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
370         raise_softirq_irqoff(HI_SOFTIRQ);
371         local_irq_restore(flags);
372 }
373
374 EXPORT_SYMBOL(__tasklet_hi_schedule);
375
376 static void tasklet_action(struct softirq_action *a)
377 {
378         struct tasklet_struct *list;
379
380         local_irq_disable();
381         list = __get_cpu_var(tasklet_vec).head;
382         __get_cpu_var(tasklet_vec).head = NULL;
383         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
384         local_irq_enable();
385
386         while (list) {
387                 struct tasklet_struct *t = list;
388
389                 list = list->next;
390
391                 if (tasklet_trylock(t)) {
392                         if (!atomic_read(&t->count)) {
393                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
394                                         BUG();
395                                 t->func(t->data);
396                                 tasklet_unlock(t);
397                                 continue;
398                         }
399                         tasklet_unlock(t);
400                 }
401
402                 local_irq_disable();
403                 t->next = NULL;
404                 *__get_cpu_var(tasklet_vec).tail = t;
405                 __get_cpu_var(tasklet_vec).tail = &(t->next);
406                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
407                 local_irq_enable();
408         }
409 }
410
411 static void tasklet_hi_action(struct softirq_action *a)
412 {
413         struct tasklet_struct *list;
414
415         local_irq_disable();
416         list = __get_cpu_var(tasklet_hi_vec).head;
417         __get_cpu_var(tasklet_hi_vec).head = NULL;
418         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
419         local_irq_enable();
420
421         while (list) {
422                 struct tasklet_struct *t = list;
423
424                 list = list->next;
425
426                 if (tasklet_trylock(t)) {
427                         if (!atomic_read(&t->count)) {
428                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
429                                         BUG();
430                                 t->func(t->data);
431                                 tasklet_unlock(t);
432                                 continue;
433                         }
434                         tasklet_unlock(t);
435                 }
436
437                 local_irq_disable();
438                 t->next = NULL;
439                 *__get_cpu_var(tasklet_hi_vec).tail = t;
440                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
441                 __raise_softirq_irqoff(HI_SOFTIRQ);
442                 local_irq_enable();
443         }
444 }
445
446
447 void tasklet_init(struct tasklet_struct *t,
448                   void (*func)(unsigned long), unsigned long data)
449 {
450         t->next = NULL;
451         t->state = 0;
452         atomic_set(&t->count, 0);
453         t->func = func;
454         t->data = data;
455 }
456
457 EXPORT_SYMBOL(tasklet_init);
458
459 void tasklet_kill(struct tasklet_struct *t)
460 {
461         if (in_interrupt())
462                 printk("Attempt to kill tasklet from interrupt\n");
463
464         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
465                 do
466                         yield();
467                 while (test_bit(TASKLET_STATE_SCHED, &t->state));
468         }
469         tasklet_unlock_wait(t);
470         clear_bit(TASKLET_STATE_SCHED, &t->state);
471 }
472
473 EXPORT_SYMBOL(tasklet_kill);
474
475 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
476 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
477
478 static void __local_trigger(struct call_single_data *cp, int softirq)
479 {
480         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
481
482         list_add_tail(&cp->list, head);
483
484         /* Trigger the softirq only if the list was previously empty.  */
485         if (head->next == &cp->list)
486                 raise_softirq_irqoff(softirq);
487 }
488
489 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
490 static void remote_softirq_receive(void *data)
491 {
492         struct call_single_data *cp = data;
493         unsigned long flags;
494         int softirq;
495
496         softirq = cp->priv;
497
498         local_irq_save(flags);
499         __local_trigger(cp, softirq);
500         local_irq_restore(flags);
501 }
502
503 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
504 {
505         if (cpu_online(cpu)) {
506                 cp->func = remote_softirq_receive;
507                 cp->info = cp;
508                 cp->flags = 0;
509                 cp->priv = softirq;
510
511                 __smp_call_function_single(cpu, cp);
512                 return 0;
513         }
514         return 1;
515 }
516 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
517 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
518 {
519         return 1;
520 }
521 #endif
522
523 /**
524  * __send_remote_softirq - try to schedule softirq work on a remote cpu
525  * @cp: private SMP call function data area
526  * @cpu: the remote cpu
527  * @this_cpu: the currently executing cpu
528  * @softirq: the softirq for the work
529  *
530  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
531  * done, the work is instead queued up on the local cpu.
532  *
533  * Interrupts must be disabled.
534  */
535 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
536 {
537         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
538                 __local_trigger(cp, softirq);
539 }
540 EXPORT_SYMBOL(__send_remote_softirq);
541
542 /**
543  * send_remote_softirq - try to schedule softirq work on a remote cpu
544  * @cp: private SMP call function data area
545  * @cpu: the remote cpu
546  * @softirq: the softirq for the work
547  *
548  * Like __send_remote_softirq except that disabling interrupts and
549  * computing the current cpu is done for the caller.
550  */
551 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
552 {
553         unsigned long flags;
554         int this_cpu;
555
556         local_irq_save(flags);
557         this_cpu = smp_processor_id();
558         __send_remote_softirq(cp, cpu, this_cpu, softirq);
559         local_irq_restore(flags);
560 }
561 EXPORT_SYMBOL(send_remote_softirq);
562
563 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
564                                                unsigned long action, void *hcpu)
565 {
566         /*
567          * If a CPU goes away, splice its entries to the current CPU
568          * and trigger a run of the softirq
569          */
570         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
571                 int cpu = (unsigned long) hcpu;
572                 int i;
573
574                 local_irq_disable();
575                 for (i = 0; i < NR_SOFTIRQS; i++) {
576                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
577                         struct list_head *local_head;
578
579                         if (list_empty(head))
580                                 continue;
581
582                         local_head = &__get_cpu_var(softirq_work_list[i]);
583                         list_splice_init(head, local_head);
584                         raise_softirq_irqoff(i);
585                 }
586                 local_irq_enable();
587         }
588
589         return NOTIFY_OK;
590 }
591
592 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
593         .notifier_call  = remote_softirq_cpu_notify,
594 };
595
596 void __init softirq_init(void)
597 {
598         int cpu;
599
600         for_each_possible_cpu(cpu) {
601                 int i;
602
603                 per_cpu(tasklet_vec, cpu).tail =
604                         &per_cpu(tasklet_vec, cpu).head;
605                 per_cpu(tasklet_hi_vec, cpu).tail =
606                         &per_cpu(tasklet_hi_vec, cpu).head;
607                 for (i = 0; i < NR_SOFTIRQS; i++)
608                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
609         }
610
611         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
612
613         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
614         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
615 }
616
617 static int ksoftirqd(void * __bind_cpu)
618 {
619         set_current_state(TASK_INTERRUPTIBLE);
620
621         while (!kthread_should_stop()) {
622                 preempt_disable();
623                 if (!local_softirq_pending()) {
624                         preempt_enable_no_resched();
625                         schedule();
626                         preempt_disable();
627                 }
628
629                 __set_current_state(TASK_RUNNING);
630
631                 while (local_softirq_pending()) {
632                         /* Preempt disable stops cpu going offline.
633                            If already offline, we'll be on wrong CPU:
634                            don't process */
635                         if (cpu_is_offline((long)__bind_cpu))
636                                 goto wait_to_die;
637                         do_softirq();
638                         preempt_enable_no_resched();
639                         cond_resched();
640                         preempt_disable();
641                 }
642                 preempt_enable();
643                 set_current_state(TASK_INTERRUPTIBLE);
644         }
645         __set_current_state(TASK_RUNNING);
646         return 0;
647
648 wait_to_die:
649         preempt_enable();
650         /* Wait for kthread_stop */
651         set_current_state(TASK_INTERRUPTIBLE);
652         while (!kthread_should_stop()) {
653                 schedule();
654                 set_current_state(TASK_INTERRUPTIBLE);
655         }
656         __set_current_state(TASK_RUNNING);
657         return 0;
658 }
659
660 #ifdef CONFIG_HOTPLUG_CPU
661 /*
662  * tasklet_kill_immediate is called to remove a tasklet which can already be
663  * scheduled for execution on @cpu.
664  *
665  * Unlike tasklet_kill, this function removes the tasklet
666  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
667  *
668  * When this function is called, @cpu must be in the CPU_DEAD state.
669  */
670 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
671 {
672         struct tasklet_struct **i;
673
674         BUG_ON(cpu_online(cpu));
675         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
676
677         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
678                 return;
679
680         /* CPU is dead, so no lock needed. */
681         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
682                 if (*i == t) {
683                         *i = t->next;
684                         /* If this was the tail element, move the tail ptr */
685                         if (*i == NULL)
686                                 per_cpu(tasklet_vec, cpu).tail = i;
687                         return;
688                 }
689         }
690         BUG();
691 }
692
693 static void takeover_tasklets(unsigned int cpu)
694 {
695         /* CPU is dead, so no lock needed. */
696         local_irq_disable();
697
698         /* Find end, append list for that CPU. */
699         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
700                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
701                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
702                 per_cpu(tasklet_vec, cpu).head = NULL;
703                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
704         }
705         raise_softirq_irqoff(TASKLET_SOFTIRQ);
706
707         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
708                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
709                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
710                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
711                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
712         }
713         raise_softirq_irqoff(HI_SOFTIRQ);
714
715         local_irq_enable();
716 }
717 #endif /* CONFIG_HOTPLUG_CPU */
718
719 static int __cpuinit cpu_callback(struct notifier_block *nfb,
720                                   unsigned long action,
721                                   void *hcpu)
722 {
723         int hotcpu = (unsigned long)hcpu;
724         struct task_struct *p;
725
726         switch (action) {
727         case CPU_UP_PREPARE:
728         case CPU_UP_PREPARE_FROZEN:
729                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
730                 if (IS_ERR(p)) {
731                         printk("ksoftirqd for %i failed\n", hotcpu);
732                         return NOTIFY_BAD;
733                 }
734                 kthread_bind(p, hotcpu);
735                 per_cpu(ksoftirqd, hotcpu) = p;
736                 break;
737         case CPU_ONLINE:
738         case CPU_ONLINE_FROZEN:
739                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
740                 break;
741 #ifdef CONFIG_HOTPLUG_CPU
742         case CPU_UP_CANCELED:
743         case CPU_UP_CANCELED_FROZEN:
744                 if (!per_cpu(ksoftirqd, hotcpu))
745                         break;
746                 /* Unbind so it can run.  Fall thru. */
747                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
748                              any_online_cpu(cpu_online_map));
749         case CPU_DEAD:
750         case CPU_DEAD_FROZEN: {
751                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
752
753                 p = per_cpu(ksoftirqd, hotcpu);
754                 per_cpu(ksoftirqd, hotcpu) = NULL;
755                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
756                 kthread_stop(p);
757                 takeover_tasklets(hotcpu);
758                 break;
759         }
760 #endif /* CONFIG_HOTPLUG_CPU */
761         }
762         return NOTIFY_OK;
763 }
764
765 static struct notifier_block __cpuinitdata cpu_nfb = {
766         .notifier_call = cpu_callback
767 };
768
769 static __init int spawn_ksoftirqd(void)
770 {
771         void *cpu = (void *)(long)smp_processor_id();
772         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
773
774         BUG_ON(err == NOTIFY_BAD);
775         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
776         register_cpu_notifier(&cpu_nfb);
777         return 0;
778 }
779 early_initcall(spawn_ksoftirqd);
780
781 #ifdef CONFIG_SMP
782 /*
783  * Call a function on all processors
784  */
785 int on_each_cpu(void (*func) (void *info), void *info, int wait)
786 {
787         int ret = 0;
788
789         preempt_disable();
790         ret = smp_call_function(func, info, wait);
791         local_irq_disable();
792         func(info);
793         local_irq_enable();
794         preempt_enable();
795         return ret;
796 }
797 EXPORT_SYMBOL(on_each_cpu);
798 #endif