2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 #include <linux/module.h>
10 #include <linux/kernel_stat.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
14 #include <linux/notifier.h>
15 #include <linux/percpu.h>
16 #include <linux/cpu.h>
17 #include <linux/freezer.h>
18 #include <linux/kthread.h>
19 #include <linux/rcupdate.h>
20 #include <linux/smp.h>
21 #include <linux/tick.h>
25 - No shared variables, all the data are CPU local.
26 - If a softirq needs serialization, let it serialize itself
28 - Even if softirq is serialized, only local cpu is marked for
29 execution. Hence, we get something sort of weak cpu binding.
30 Though it is still not clear, will it result in better locality
34 - NET RX softirq. It is multithreaded and does not require
35 any global serialization.
36 - NET TX softirq. It kicks software netdevice queues, hence
37 it is logically serialized per device, but this serialization
38 is invisible to common code.
39 - Tasklets: serialized wrt itself.
42 #ifndef __ARCH_IRQ_STAT
43 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
44 EXPORT_SYMBOL(irq_stat);
47 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
49 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
52 * we cannot loop indefinitely here to avoid userspace starvation,
53 * but we also don't want to introduce a worst case 1/HZ latency
54 * to the pending events, so lets the scheduler to balance
55 * the softirq load for us.
57 static inline void wakeup_softirqd(void)
59 /* Interrupts are disabled: no need to stop preemption */
60 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
62 if (tsk && tsk->state != TASK_RUNNING)
67 * This one is for softirq.c-internal use,
68 * where hardirqs are disabled legitimately:
70 #ifdef CONFIG_TRACE_IRQFLAGS
71 static void __local_bh_disable(unsigned long ip)
75 WARN_ON_ONCE(in_irq());
77 raw_local_irq_save(flags);
78 add_preempt_count(SOFTIRQ_OFFSET);
80 * Were softirqs turned off above:
82 if (softirq_count() == SOFTIRQ_OFFSET)
83 trace_softirqs_off(ip);
84 raw_local_irq_restore(flags);
86 #else /* !CONFIG_TRACE_IRQFLAGS */
87 static inline void __local_bh_disable(unsigned long ip)
89 add_preempt_count(SOFTIRQ_OFFSET);
92 #endif /* CONFIG_TRACE_IRQFLAGS */
94 void local_bh_disable(void)
96 __local_bh_disable((unsigned long)__builtin_return_address(0));
99 EXPORT_SYMBOL(local_bh_disable);
101 void __local_bh_enable(void)
103 WARN_ON_ONCE(in_irq());
106 * softirqs should never be enabled by __local_bh_enable(),
107 * it always nests inside local_bh_enable() sections:
109 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
111 sub_preempt_count(SOFTIRQ_OFFSET);
113 EXPORT_SYMBOL_GPL(__local_bh_enable);
116 * Special-case - softirqs can safely be enabled in
117 * cond_resched_softirq(), or by __do_softirq(),
118 * without processing still-pending softirqs:
120 void _local_bh_enable(void)
122 WARN_ON_ONCE(in_irq());
123 WARN_ON_ONCE(!irqs_disabled());
125 if (softirq_count() == SOFTIRQ_OFFSET)
126 trace_softirqs_on((unsigned long)__builtin_return_address(0));
127 sub_preempt_count(SOFTIRQ_OFFSET);
130 EXPORT_SYMBOL(_local_bh_enable);
132 void local_bh_enable(void)
134 #ifdef CONFIG_TRACE_IRQFLAGS
137 WARN_ON_ONCE(in_irq());
139 WARN_ON_ONCE(irqs_disabled());
141 #ifdef CONFIG_TRACE_IRQFLAGS
142 local_irq_save(flags);
145 * Are softirqs going to be turned on now:
147 if (softirq_count() == SOFTIRQ_OFFSET)
148 trace_softirqs_on((unsigned long)__builtin_return_address(0));
150 * Keep preemption disabled until we are done with
151 * softirq processing:
153 sub_preempt_count(SOFTIRQ_OFFSET - 1);
155 if (unlikely(!in_interrupt() && local_softirq_pending()))
159 #ifdef CONFIG_TRACE_IRQFLAGS
160 local_irq_restore(flags);
162 preempt_check_resched();
164 EXPORT_SYMBOL(local_bh_enable);
166 void local_bh_enable_ip(unsigned long ip)
168 #ifdef CONFIG_TRACE_IRQFLAGS
171 WARN_ON_ONCE(in_irq());
173 local_irq_save(flags);
176 * Are softirqs going to be turned on now:
178 if (softirq_count() == SOFTIRQ_OFFSET)
179 trace_softirqs_on(ip);
181 * Keep preemption disabled until we are done with
182 * softirq processing:
184 sub_preempt_count(SOFTIRQ_OFFSET - 1);
186 if (unlikely(!in_interrupt() && local_softirq_pending()))
190 #ifdef CONFIG_TRACE_IRQFLAGS
191 local_irq_restore(flags);
193 preempt_check_resched();
195 EXPORT_SYMBOL(local_bh_enable_ip);
198 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
199 * and we fall back to softirqd after that.
201 * This number has been established via experimentation.
202 * The two things to balance is latency against fairness -
203 * we want to handle softirqs as soon as possible, but they
204 * should not be able to lock up the box.
206 #define MAX_SOFTIRQ_RESTART 10
208 asmlinkage void __do_softirq(void)
210 struct softirq_action *h;
212 int max_restart = MAX_SOFTIRQ_RESTART;
215 pending = local_softirq_pending();
216 account_system_vtime(current);
218 __local_bh_disable((unsigned long)__builtin_return_address(0));
219 trace_softirq_enter();
221 cpu = smp_processor_id();
223 /* Reset the pending bitmask before enabling irqs */
224 set_softirq_pending(0);
233 rcu_bh_qsctr_inc(cpu);
241 pending = local_softirq_pending();
242 if (pending && --max_restart)
248 trace_softirq_exit();
250 account_system_vtime(current);
254 #ifndef __ARCH_HAS_DO_SOFTIRQ
256 asmlinkage void do_softirq(void)
264 local_irq_save(flags);
266 pending = local_softirq_pending();
271 local_irq_restore(flags);
274 EXPORT_SYMBOL(do_softirq);
279 * Enter an interrupt context.
285 if (idle_cpu(smp_processor_id()))
286 tick_nohz_update_jiffies();
290 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
291 # define invoke_softirq() __do_softirq()
293 # define invoke_softirq() do_softirq()
297 * Exit an interrupt context. Process softirqs if needed and possible:
301 account_system_vtime(current);
302 trace_hardirq_exit();
303 sub_preempt_count(IRQ_EXIT_OFFSET);
304 if (!in_interrupt() && local_softirq_pending())
308 /* Make sure that timer wheel updates are propagated */
309 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
310 tick_nohz_stop_sched_tick();
312 preempt_enable_no_resched();
316 * This function must run with irqs disabled!
318 inline fastcall void raise_softirq_irqoff(unsigned int nr)
320 __raise_softirq_irqoff(nr);
323 * If we're in an interrupt or softirq, we're done
324 * (this also catches softirq-disabled code). We will
325 * actually run the softirq once we return from
326 * the irq or softirq.
328 * Otherwise we wake up ksoftirqd to make sure we
329 * schedule the softirq soon.
335 EXPORT_SYMBOL(raise_softirq_irqoff);
337 void fastcall raise_softirq(unsigned int nr)
341 local_irq_save(flags);
342 raise_softirq_irqoff(nr);
343 local_irq_restore(flags);
346 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
348 softirq_vec[nr].data = data;
349 softirq_vec[nr].action = action;
355 struct tasklet_struct *list;
358 /* Some compilers disobey section attribute on statics when not
360 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
361 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
363 void fastcall __tasklet_schedule(struct tasklet_struct *t)
367 local_irq_save(flags);
368 t->next = __get_cpu_var(tasklet_vec).list;
369 __get_cpu_var(tasklet_vec).list = t;
370 raise_softirq_irqoff(TASKLET_SOFTIRQ);
371 local_irq_restore(flags);
374 EXPORT_SYMBOL(__tasklet_schedule);
376 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
380 local_irq_save(flags);
381 t->next = __get_cpu_var(tasklet_hi_vec).list;
382 __get_cpu_var(tasklet_hi_vec).list = t;
383 raise_softirq_irqoff(HI_SOFTIRQ);
384 local_irq_restore(flags);
387 EXPORT_SYMBOL(__tasklet_hi_schedule);
389 static void tasklet_action(struct softirq_action *a)
391 struct tasklet_struct *list;
394 list = __get_cpu_var(tasklet_vec).list;
395 __get_cpu_var(tasklet_vec).list = NULL;
399 struct tasklet_struct *t = list;
403 if (tasklet_trylock(t)) {
404 if (!atomic_read(&t->count)) {
405 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
415 t->next = __get_cpu_var(tasklet_vec).list;
416 __get_cpu_var(tasklet_vec).list = t;
417 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
422 static void tasklet_hi_action(struct softirq_action *a)
424 struct tasklet_struct *list;
427 list = __get_cpu_var(tasklet_hi_vec).list;
428 __get_cpu_var(tasklet_hi_vec).list = NULL;
432 struct tasklet_struct *t = list;
436 if (tasklet_trylock(t)) {
437 if (!atomic_read(&t->count)) {
438 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
448 t->next = __get_cpu_var(tasklet_hi_vec).list;
449 __get_cpu_var(tasklet_hi_vec).list = t;
450 __raise_softirq_irqoff(HI_SOFTIRQ);
456 void tasklet_init(struct tasklet_struct *t,
457 void (*func)(unsigned long), unsigned long data)
461 atomic_set(&t->count, 0);
466 EXPORT_SYMBOL(tasklet_init);
468 void tasklet_kill(struct tasklet_struct *t)
471 printk("Attempt to kill tasklet from interrupt\n");
473 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
476 while (test_bit(TASKLET_STATE_SCHED, &t->state));
478 tasklet_unlock_wait(t);
479 clear_bit(TASKLET_STATE_SCHED, &t->state);
482 EXPORT_SYMBOL(tasklet_kill);
484 void __init softirq_init(void)
486 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
487 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
490 static int ksoftirqd(void * __bind_cpu)
492 set_current_state(TASK_INTERRUPTIBLE);
494 while (!kthread_should_stop()) {
496 if (!local_softirq_pending()) {
497 preempt_enable_no_resched();
502 __set_current_state(TASK_RUNNING);
504 while (local_softirq_pending()) {
505 /* Preempt disable stops cpu going offline.
506 If already offline, we'll be on wrong CPU:
508 if (cpu_is_offline((long)__bind_cpu))
511 preempt_enable_no_resched();
516 set_current_state(TASK_INTERRUPTIBLE);
518 __set_current_state(TASK_RUNNING);
523 /* Wait for kthread_stop */
524 set_current_state(TASK_INTERRUPTIBLE);
525 while (!kthread_should_stop()) {
527 set_current_state(TASK_INTERRUPTIBLE);
529 __set_current_state(TASK_RUNNING);
533 #ifdef CONFIG_HOTPLUG_CPU
535 * tasklet_kill_immediate is called to remove a tasklet which can already be
536 * scheduled for execution on @cpu.
538 * Unlike tasklet_kill, this function removes the tasklet
539 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
541 * When this function is called, @cpu must be in the CPU_DEAD state.
543 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
545 struct tasklet_struct **i;
547 BUG_ON(cpu_online(cpu));
548 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
550 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
553 /* CPU is dead, so no lock needed. */
554 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
563 static void takeover_tasklets(unsigned int cpu)
565 struct tasklet_struct **i;
567 /* CPU is dead, so no lock needed. */
570 /* Find end, append list for that CPU. */
571 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
572 *i = per_cpu(tasklet_vec, cpu).list;
573 per_cpu(tasklet_vec, cpu).list = NULL;
574 raise_softirq_irqoff(TASKLET_SOFTIRQ);
576 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
577 *i = per_cpu(tasklet_hi_vec, cpu).list;
578 per_cpu(tasklet_hi_vec, cpu).list = NULL;
579 raise_softirq_irqoff(HI_SOFTIRQ);
583 #endif /* CONFIG_HOTPLUG_CPU */
585 static int __cpuinit cpu_callback(struct notifier_block *nfb,
586 unsigned long action,
589 int hotcpu = (unsigned long)hcpu;
590 struct task_struct *p;
594 case CPU_UP_PREPARE_FROZEN:
595 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
597 printk("ksoftirqd for %i failed\n", hotcpu);
600 kthread_bind(p, hotcpu);
601 per_cpu(ksoftirqd, hotcpu) = p;
604 case CPU_ONLINE_FROZEN:
605 wake_up_process(per_cpu(ksoftirqd, hotcpu));
607 #ifdef CONFIG_HOTPLUG_CPU
608 case CPU_UP_CANCELED:
609 case CPU_UP_CANCELED_FROZEN:
610 if (!per_cpu(ksoftirqd, hotcpu))
612 /* Unbind so it can run. Fall thru. */
613 kthread_bind(per_cpu(ksoftirqd, hotcpu),
614 any_online_cpu(cpu_online_map));
616 case CPU_DEAD_FROZEN: {
617 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
619 p = per_cpu(ksoftirqd, hotcpu);
620 per_cpu(ksoftirqd, hotcpu) = NULL;
621 sched_setscheduler(p, SCHED_FIFO, ¶m);
623 takeover_tasklets(hotcpu);
626 #endif /* CONFIG_HOTPLUG_CPU */
631 static struct notifier_block __cpuinitdata cpu_nfb = {
632 .notifier_call = cpu_callback
635 __init int spawn_ksoftirqd(void)
637 void *cpu = (void *)(long)smp_processor_id();
638 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
640 BUG_ON(err == NOTIFY_BAD);
641 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
642 register_cpu_notifier(&cpu_nfb);
648 * Call a function on all processors
650 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
655 ret = smp_call_function(func, info, retry, wait);
662 EXPORT_SYMBOL(on_each_cpu);