2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 #include <linux/module.h>
10 #include <linux/kernel_stat.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
14 #include <linux/notifier.h>
15 #include <linux/percpu.h>
16 #include <linux/cpu.h>
17 #include <linux/freezer.h>
18 #include <linux/kthread.h>
19 #include <linux/rcupdate.h>
20 #include <linux/smp.h>
21 #include <linux/tick.h>
25 - No shared variables, all the data are CPU local.
26 - If a softirq needs serialization, let it serialize itself
28 - Even if softirq is serialized, only local cpu is marked for
29 execution. Hence, we get something sort of weak cpu binding.
30 Though it is still not clear, will it result in better locality
34 - NET RX softirq. It is multithreaded and does not require
35 any global serialization.
36 - NET TX softirq. It kicks software netdevice queues, hence
37 it is logically serialized per device, but this serialization
38 is invisible to common code.
39 - Tasklets: serialized wrt itself.
42 #ifndef __ARCH_IRQ_STAT
43 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
44 EXPORT_SYMBOL(irq_stat);
47 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
49 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
52 * we cannot loop indefinitely here to avoid userspace starvation,
53 * but we also don't want to introduce a worst case 1/HZ latency
54 * to the pending events, so lets the scheduler to balance
55 * the softirq load for us.
57 static inline void wakeup_softirqd(void)
59 /* Interrupts are disabled: no need to stop preemption */
60 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
62 if (tsk && tsk->state != TASK_RUNNING)
67 * This one is for softirq.c-internal use,
68 * where hardirqs are disabled legitimately:
70 #ifdef CONFIG_TRACE_IRQFLAGS
71 static void __local_bh_disable(unsigned long ip)
75 WARN_ON_ONCE(in_irq());
77 raw_local_irq_save(flags);
78 add_preempt_count(SOFTIRQ_OFFSET);
80 * Were softirqs turned off above:
82 if (softirq_count() == SOFTIRQ_OFFSET)
83 trace_softirqs_off(ip);
84 raw_local_irq_restore(flags);
86 #else /* !CONFIG_TRACE_IRQFLAGS */
87 static inline void __local_bh_disable(unsigned long ip)
89 add_preempt_count(SOFTIRQ_OFFSET);
92 #endif /* CONFIG_TRACE_IRQFLAGS */
94 void local_bh_disable(void)
96 __local_bh_disable((unsigned long)__builtin_return_address(0));
99 EXPORT_SYMBOL(local_bh_disable);
101 void __local_bh_enable(void)
103 WARN_ON_ONCE(in_irq());
106 * softirqs should never be enabled by __local_bh_enable(),
107 * it always nests inside local_bh_enable() sections:
109 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
111 sub_preempt_count(SOFTIRQ_OFFSET);
113 EXPORT_SYMBOL_GPL(__local_bh_enable);
116 * Special-case - softirqs can safely be enabled in
117 * cond_resched_softirq(), or by __do_softirq(),
118 * without processing still-pending softirqs:
120 void _local_bh_enable(void)
122 WARN_ON_ONCE(in_irq());
123 WARN_ON_ONCE(!irqs_disabled());
125 if (softirq_count() == SOFTIRQ_OFFSET)
126 trace_softirqs_on((unsigned long)__builtin_return_address(0));
127 sub_preempt_count(SOFTIRQ_OFFSET);
130 EXPORT_SYMBOL(_local_bh_enable);
132 void local_bh_enable(void)
134 #ifdef CONFIG_TRACE_IRQFLAGS
137 WARN_ON_ONCE(in_irq());
139 WARN_ON_ONCE(irqs_disabled());
141 #ifdef CONFIG_TRACE_IRQFLAGS
142 local_irq_save(flags);
145 * Are softirqs going to be turned on now:
147 if (softirq_count() == SOFTIRQ_OFFSET)
148 trace_softirqs_on((unsigned long)__builtin_return_address(0));
150 * Keep preemption disabled until we are done with
151 * softirq processing:
153 sub_preempt_count(SOFTIRQ_OFFSET - 1);
155 if (unlikely(!in_interrupt() && local_softirq_pending()))
159 #ifdef CONFIG_TRACE_IRQFLAGS
160 local_irq_restore(flags);
162 preempt_check_resched();
164 EXPORT_SYMBOL(local_bh_enable);
166 void local_bh_enable_ip(unsigned long ip)
168 #ifdef CONFIG_TRACE_IRQFLAGS
171 WARN_ON_ONCE(in_irq());
173 local_irq_save(flags);
176 * Are softirqs going to be turned on now:
178 if (softirq_count() == SOFTIRQ_OFFSET)
179 trace_softirqs_on(ip);
181 * Keep preemption disabled until we are done with
182 * softirq processing:
184 sub_preempt_count(SOFTIRQ_OFFSET - 1);
186 if (unlikely(!in_interrupt() && local_softirq_pending()))
190 #ifdef CONFIG_TRACE_IRQFLAGS
191 local_irq_restore(flags);
193 preempt_check_resched();
195 EXPORT_SYMBOL(local_bh_enable_ip);
198 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
199 * and we fall back to softirqd after that.
201 * This number has been established via experimentation.
202 * The two things to balance is latency against fairness -
203 * we want to handle softirqs as soon as possible, but they
204 * should not be able to lock up the box.
206 #define MAX_SOFTIRQ_RESTART 10
208 asmlinkage void __do_softirq(void)
210 struct softirq_action *h;
212 int max_restart = MAX_SOFTIRQ_RESTART;
215 pending = local_softirq_pending();
216 account_system_vtime(current);
218 __local_bh_disable((unsigned long)__builtin_return_address(0));
219 trace_softirq_enter();
221 cpu = smp_processor_id();
223 /* Reset the pending bitmask before enabling irqs */
224 set_softirq_pending(0);
233 rcu_bh_qsctr_inc(cpu);
241 pending = local_softirq_pending();
242 if (pending && --max_restart)
248 trace_softirq_exit();
250 account_system_vtime(current);
254 #ifndef __ARCH_HAS_DO_SOFTIRQ
256 asmlinkage void do_softirq(void)
264 local_irq_save(flags);
266 pending = local_softirq_pending();
271 local_irq_restore(flags);
277 * Enter an interrupt context.
283 if (idle_cpu(smp_processor_id()))
284 tick_nohz_update_jiffies();
288 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
289 # define invoke_softirq() __do_softirq()
291 # define invoke_softirq() do_softirq()
295 * Exit an interrupt context. Process softirqs if needed and possible:
299 account_system_vtime(current);
300 trace_hardirq_exit();
301 sub_preempt_count(IRQ_EXIT_OFFSET);
302 if (!in_interrupt() && local_softirq_pending())
306 /* Make sure that timer wheel updates are propagated */
307 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
308 tick_nohz_stop_sched_tick();
310 preempt_enable_no_resched();
314 * This function must run with irqs disabled!
316 inline fastcall void raise_softirq_irqoff(unsigned int nr)
318 __raise_softirq_irqoff(nr);
321 * If we're in an interrupt or softirq, we're done
322 * (this also catches softirq-disabled code). We will
323 * actually run the softirq once we return from
324 * the irq or softirq.
326 * Otherwise we wake up ksoftirqd to make sure we
327 * schedule the softirq soon.
333 void fastcall raise_softirq(unsigned int nr)
337 local_irq_save(flags);
338 raise_softirq_irqoff(nr);
339 local_irq_restore(flags);
342 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
344 softirq_vec[nr].data = data;
345 softirq_vec[nr].action = action;
351 struct tasklet_struct *list;
354 /* Some compilers disobey section attribute on statics when not
356 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
357 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
359 void fastcall __tasklet_schedule(struct tasklet_struct *t)
363 local_irq_save(flags);
364 t->next = __get_cpu_var(tasklet_vec).list;
365 __get_cpu_var(tasklet_vec).list = t;
366 raise_softirq_irqoff(TASKLET_SOFTIRQ);
367 local_irq_restore(flags);
370 EXPORT_SYMBOL(__tasklet_schedule);
372 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
376 local_irq_save(flags);
377 t->next = __get_cpu_var(tasklet_hi_vec).list;
378 __get_cpu_var(tasklet_hi_vec).list = t;
379 raise_softirq_irqoff(HI_SOFTIRQ);
380 local_irq_restore(flags);
383 EXPORT_SYMBOL(__tasklet_hi_schedule);
385 static void tasklet_action(struct softirq_action *a)
387 struct tasklet_struct *list;
390 list = __get_cpu_var(tasklet_vec).list;
391 __get_cpu_var(tasklet_vec).list = NULL;
395 struct tasklet_struct *t = list;
399 if (tasklet_trylock(t)) {
400 if (!atomic_read(&t->count)) {
401 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
411 t->next = __get_cpu_var(tasklet_vec).list;
412 __get_cpu_var(tasklet_vec).list = t;
413 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
418 static void tasklet_hi_action(struct softirq_action *a)
420 struct tasklet_struct *list;
423 list = __get_cpu_var(tasklet_hi_vec).list;
424 __get_cpu_var(tasklet_hi_vec).list = NULL;
428 struct tasklet_struct *t = list;
432 if (tasklet_trylock(t)) {
433 if (!atomic_read(&t->count)) {
434 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
444 t->next = __get_cpu_var(tasklet_hi_vec).list;
445 __get_cpu_var(tasklet_hi_vec).list = t;
446 __raise_softirq_irqoff(HI_SOFTIRQ);
452 void tasklet_init(struct tasklet_struct *t,
453 void (*func)(unsigned long), unsigned long data)
457 atomic_set(&t->count, 0);
462 EXPORT_SYMBOL(tasklet_init);
464 void tasklet_kill(struct tasklet_struct *t)
467 printk("Attempt to kill tasklet from interrupt\n");
469 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
472 while (test_bit(TASKLET_STATE_SCHED, &t->state));
474 tasklet_unlock_wait(t);
475 clear_bit(TASKLET_STATE_SCHED, &t->state);
478 EXPORT_SYMBOL(tasklet_kill);
480 void __init softirq_init(void)
482 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
483 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
486 static int ksoftirqd(void * __bind_cpu)
488 set_current_state(TASK_INTERRUPTIBLE);
490 while (!kthread_should_stop()) {
492 if (!local_softirq_pending()) {
493 preempt_enable_no_resched();
498 __set_current_state(TASK_RUNNING);
500 while (local_softirq_pending()) {
501 /* Preempt disable stops cpu going offline.
502 If already offline, we'll be on wrong CPU:
504 if (cpu_is_offline((long)__bind_cpu))
507 preempt_enable_no_resched();
512 set_current_state(TASK_INTERRUPTIBLE);
514 __set_current_state(TASK_RUNNING);
519 /* Wait for kthread_stop */
520 set_current_state(TASK_INTERRUPTIBLE);
521 while (!kthread_should_stop()) {
523 set_current_state(TASK_INTERRUPTIBLE);
525 __set_current_state(TASK_RUNNING);
529 #ifdef CONFIG_HOTPLUG_CPU
531 * tasklet_kill_immediate is called to remove a tasklet which can already be
532 * scheduled for execution on @cpu.
534 * Unlike tasklet_kill, this function removes the tasklet
535 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
537 * When this function is called, @cpu must be in the CPU_DEAD state.
539 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
541 struct tasklet_struct **i;
543 BUG_ON(cpu_online(cpu));
544 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
546 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
549 /* CPU is dead, so no lock needed. */
550 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
559 static void takeover_tasklets(unsigned int cpu)
561 struct tasklet_struct **i;
563 /* CPU is dead, so no lock needed. */
566 /* Find end, append list for that CPU. */
567 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
568 *i = per_cpu(tasklet_vec, cpu).list;
569 per_cpu(tasklet_vec, cpu).list = NULL;
570 raise_softirq_irqoff(TASKLET_SOFTIRQ);
572 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
573 *i = per_cpu(tasklet_hi_vec, cpu).list;
574 per_cpu(tasklet_hi_vec, cpu).list = NULL;
575 raise_softirq_irqoff(HI_SOFTIRQ);
579 #endif /* CONFIG_HOTPLUG_CPU */
581 static int __cpuinit cpu_callback(struct notifier_block *nfb,
582 unsigned long action,
585 int hotcpu = (unsigned long)hcpu;
586 struct task_struct *p;
590 case CPU_UP_PREPARE_FROZEN:
591 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
593 printk("ksoftirqd for %i failed\n", hotcpu);
596 kthread_bind(p, hotcpu);
597 per_cpu(ksoftirqd, hotcpu) = p;
600 case CPU_ONLINE_FROZEN:
601 wake_up_process(per_cpu(ksoftirqd, hotcpu));
603 #ifdef CONFIG_HOTPLUG_CPU
604 case CPU_UP_CANCELED:
605 case CPU_UP_CANCELED_FROZEN:
606 if (!per_cpu(ksoftirqd, hotcpu))
608 /* Unbind so it can run. Fall thru. */
609 kthread_bind(per_cpu(ksoftirqd, hotcpu),
610 any_online_cpu(cpu_online_map));
612 case CPU_DEAD_FROZEN: {
613 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
615 p = per_cpu(ksoftirqd, hotcpu);
616 per_cpu(ksoftirqd, hotcpu) = NULL;
617 sched_setscheduler(p, SCHED_FIFO, ¶m);
619 takeover_tasklets(hotcpu);
622 #endif /* CONFIG_HOTPLUG_CPU */
627 static struct notifier_block __cpuinitdata cpu_nfb = {
628 .notifier_call = cpu_callback
631 __init int spawn_ksoftirqd(void)
633 void *cpu = (void *)(long)smp_processor_id();
634 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
636 BUG_ON(err == NOTIFY_BAD);
637 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
638 register_cpu_notifier(&cpu_nfb);
644 * Call a function on all processors
646 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
651 ret = smp_call_function(func, info, retry, wait);
658 EXPORT_SYMBOL(on_each_cpu);