2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 #include <linux/module.h>
10 #include <linux/kernel_stat.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
14 #include <linux/notifier.h>
15 #include <linux/percpu.h>
16 #include <linux/cpu.h>
17 #include <linux/kthread.h>
18 #include <linux/rcupdate.h>
19 #include <linux/smp.h>
20 #include <linux/tick.h>
24 - No shared variables, all the data are CPU local.
25 - If a softirq needs serialization, let it serialize itself
27 - Even if softirq is serialized, only local cpu is marked for
28 execution. Hence, we get something sort of weak cpu binding.
29 Though it is still not clear, will it result in better locality
33 - NET RX softirq. It is multithreaded and does not require
34 any global serialization.
35 - NET TX softirq. It kicks software netdevice queues, hence
36 it is logically serialized per device, but this serialization
37 is invisible to common code.
38 - Tasklets: serialized wrt itself.
41 #ifndef __ARCH_IRQ_STAT
42 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
43 EXPORT_SYMBOL(irq_stat);
46 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
48 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
51 * we cannot loop indefinitely here to avoid userspace starvation,
52 * but we also don't want to introduce a worst case 1/HZ latency
53 * to the pending events, so lets the scheduler to balance
54 * the softirq load for us.
56 static inline void wakeup_softirqd(void)
58 /* Interrupts are disabled: no need to stop preemption */
59 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
61 if (tsk && tsk->state != TASK_RUNNING)
66 * This one is for softirq.c-internal use,
67 * where hardirqs are disabled legitimately:
69 #ifdef CONFIG_TRACE_IRQFLAGS
70 static void __local_bh_disable(unsigned long ip)
74 WARN_ON_ONCE(in_irq());
76 raw_local_irq_save(flags);
77 add_preempt_count(SOFTIRQ_OFFSET);
79 * Were softirqs turned off above:
81 if (softirq_count() == SOFTIRQ_OFFSET)
82 trace_softirqs_off(ip);
83 raw_local_irq_restore(flags);
85 #else /* !CONFIG_TRACE_IRQFLAGS */
86 static inline void __local_bh_disable(unsigned long ip)
88 add_preempt_count(SOFTIRQ_OFFSET);
91 #endif /* CONFIG_TRACE_IRQFLAGS */
93 void local_bh_disable(void)
95 __local_bh_disable((unsigned long)__builtin_return_address(0));
98 EXPORT_SYMBOL(local_bh_disable);
100 void __local_bh_enable(void)
102 WARN_ON_ONCE(in_irq());
105 * softirqs should never be enabled by __local_bh_enable(),
106 * it always nests inside local_bh_enable() sections:
108 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
110 sub_preempt_count(SOFTIRQ_OFFSET);
112 EXPORT_SYMBOL_GPL(__local_bh_enable);
115 * Special-case - softirqs can safely be enabled in
116 * cond_resched_softirq(), or by __do_softirq(),
117 * without processing still-pending softirqs:
119 void _local_bh_enable(void)
121 WARN_ON_ONCE(in_irq());
122 WARN_ON_ONCE(!irqs_disabled());
124 if (softirq_count() == SOFTIRQ_OFFSET)
125 trace_softirqs_on((unsigned long)__builtin_return_address(0));
126 sub_preempt_count(SOFTIRQ_OFFSET);
129 EXPORT_SYMBOL(_local_bh_enable);
131 void local_bh_enable(void)
133 #ifdef CONFIG_TRACE_IRQFLAGS
136 WARN_ON_ONCE(in_irq());
138 WARN_ON_ONCE(irqs_disabled());
140 #ifdef CONFIG_TRACE_IRQFLAGS
141 local_irq_save(flags);
144 * Are softirqs going to be turned on now:
146 if (softirq_count() == SOFTIRQ_OFFSET)
147 trace_softirqs_on((unsigned long)__builtin_return_address(0));
149 * Keep preemption disabled until we are done with
150 * softirq processing:
152 sub_preempt_count(SOFTIRQ_OFFSET - 1);
154 if (unlikely(!in_interrupt() && local_softirq_pending()))
158 #ifdef CONFIG_TRACE_IRQFLAGS
159 local_irq_restore(flags);
161 preempt_check_resched();
163 EXPORT_SYMBOL(local_bh_enable);
165 void local_bh_enable_ip(unsigned long ip)
167 #ifdef CONFIG_TRACE_IRQFLAGS
170 WARN_ON_ONCE(in_irq());
172 local_irq_save(flags);
175 * Are softirqs going to be turned on now:
177 if (softirq_count() == SOFTIRQ_OFFSET)
178 trace_softirqs_on(ip);
180 * Keep preemption disabled until we are done with
181 * softirq processing:
183 sub_preempt_count(SOFTIRQ_OFFSET - 1);
185 if (unlikely(!in_interrupt() && local_softirq_pending()))
189 #ifdef CONFIG_TRACE_IRQFLAGS
190 local_irq_restore(flags);
192 preempt_check_resched();
194 EXPORT_SYMBOL(local_bh_enable_ip);
197 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
198 * and we fall back to softirqd after that.
200 * This number has been established via experimentation.
201 * The two things to balance is latency against fairness -
202 * we want to handle softirqs as soon as possible, but they
203 * should not be able to lock up the box.
205 #define MAX_SOFTIRQ_RESTART 10
207 asmlinkage void __do_softirq(void)
209 struct softirq_action *h;
211 int max_restart = MAX_SOFTIRQ_RESTART;
214 pending = local_softirq_pending();
215 account_system_vtime(current);
217 __local_bh_disable((unsigned long)__builtin_return_address(0));
218 trace_softirq_enter();
220 cpu = smp_processor_id();
222 /* Reset the pending bitmask before enabling irqs */
223 set_softirq_pending(0);
232 rcu_bh_qsctr_inc(cpu);
240 pending = local_softirq_pending();
241 if (pending && --max_restart)
247 trace_softirq_exit();
249 account_system_vtime(current);
253 #ifndef __ARCH_HAS_DO_SOFTIRQ
255 asmlinkage void do_softirq(void)
263 local_irq_save(flags);
265 pending = local_softirq_pending();
270 local_irq_restore(flags);
273 EXPORT_SYMBOL(do_softirq);
278 * Enter an interrupt context.
284 if (idle_cpu(smp_processor_id()))
285 tick_nohz_update_jiffies();
289 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
290 # define invoke_softirq() __do_softirq()
292 # define invoke_softirq() do_softirq()
296 * Exit an interrupt context. Process softirqs if needed and possible:
300 account_system_vtime(current);
301 trace_hardirq_exit();
302 sub_preempt_count(IRQ_EXIT_OFFSET);
303 if (!in_interrupt() && local_softirq_pending())
307 /* Make sure that timer wheel updates are propagated */
308 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
309 tick_nohz_stop_sched_tick();
311 preempt_enable_no_resched();
315 * This function must run with irqs disabled!
317 inline fastcall void raise_softirq_irqoff(unsigned int nr)
319 __raise_softirq_irqoff(nr);
322 * If we're in an interrupt or softirq, we're done
323 * (this also catches softirq-disabled code). We will
324 * actually run the softirq once we return from
325 * the irq or softirq.
327 * Otherwise we wake up ksoftirqd to make sure we
328 * schedule the softirq soon.
334 EXPORT_SYMBOL(raise_softirq_irqoff);
336 void fastcall raise_softirq(unsigned int nr)
340 local_irq_save(flags);
341 raise_softirq_irqoff(nr);
342 local_irq_restore(flags);
345 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
347 softirq_vec[nr].data = data;
348 softirq_vec[nr].action = action;
354 struct tasklet_struct *list;
357 /* Some compilers disobey section attribute on statics when not
359 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
360 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
362 void fastcall __tasklet_schedule(struct tasklet_struct *t)
366 local_irq_save(flags);
367 t->next = __get_cpu_var(tasklet_vec).list;
368 __get_cpu_var(tasklet_vec).list = t;
369 raise_softirq_irqoff(TASKLET_SOFTIRQ);
370 local_irq_restore(flags);
373 EXPORT_SYMBOL(__tasklet_schedule);
375 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
379 local_irq_save(flags);
380 t->next = __get_cpu_var(tasklet_hi_vec).list;
381 __get_cpu_var(tasklet_hi_vec).list = t;
382 raise_softirq_irqoff(HI_SOFTIRQ);
383 local_irq_restore(flags);
386 EXPORT_SYMBOL(__tasklet_hi_schedule);
388 static void tasklet_action(struct softirq_action *a)
390 struct tasklet_struct *list;
393 list = __get_cpu_var(tasklet_vec).list;
394 __get_cpu_var(tasklet_vec).list = NULL;
398 struct tasklet_struct *t = list;
402 if (tasklet_trylock(t)) {
403 if (!atomic_read(&t->count)) {
404 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
414 t->next = __get_cpu_var(tasklet_vec).list;
415 __get_cpu_var(tasklet_vec).list = t;
416 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
421 static void tasklet_hi_action(struct softirq_action *a)
423 struct tasklet_struct *list;
426 list = __get_cpu_var(tasklet_hi_vec).list;
427 __get_cpu_var(tasklet_hi_vec).list = NULL;
431 struct tasklet_struct *t = list;
435 if (tasklet_trylock(t)) {
436 if (!atomic_read(&t->count)) {
437 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
447 t->next = __get_cpu_var(tasklet_hi_vec).list;
448 __get_cpu_var(tasklet_hi_vec).list = t;
449 __raise_softirq_irqoff(HI_SOFTIRQ);
455 void tasklet_init(struct tasklet_struct *t,
456 void (*func)(unsigned long), unsigned long data)
460 atomic_set(&t->count, 0);
465 EXPORT_SYMBOL(tasklet_init);
467 void tasklet_kill(struct tasklet_struct *t)
470 printk("Attempt to kill tasklet from interrupt\n");
472 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
475 while (test_bit(TASKLET_STATE_SCHED, &t->state));
477 tasklet_unlock_wait(t);
478 clear_bit(TASKLET_STATE_SCHED, &t->state);
481 EXPORT_SYMBOL(tasklet_kill);
483 void __init softirq_init(void)
485 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
486 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
489 static int ksoftirqd(void * __bind_cpu)
491 set_user_nice(current, 19);
492 current->flags |= PF_NOFREEZE;
494 set_current_state(TASK_INTERRUPTIBLE);
496 while (!kthread_should_stop()) {
498 if (!local_softirq_pending()) {
499 preempt_enable_no_resched();
504 __set_current_state(TASK_RUNNING);
506 while (local_softirq_pending()) {
507 /* Preempt disable stops cpu going offline.
508 If already offline, we'll be on wrong CPU:
510 if (cpu_is_offline((long)__bind_cpu))
513 preempt_enable_no_resched();
518 set_current_state(TASK_INTERRUPTIBLE);
520 __set_current_state(TASK_RUNNING);
525 /* Wait for kthread_stop */
526 set_current_state(TASK_INTERRUPTIBLE);
527 while (!kthread_should_stop()) {
529 set_current_state(TASK_INTERRUPTIBLE);
531 __set_current_state(TASK_RUNNING);
535 #ifdef CONFIG_HOTPLUG_CPU
537 * tasklet_kill_immediate is called to remove a tasklet which can already be
538 * scheduled for execution on @cpu.
540 * Unlike tasklet_kill, this function removes the tasklet
541 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
543 * When this function is called, @cpu must be in the CPU_DEAD state.
545 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
547 struct tasklet_struct **i;
549 BUG_ON(cpu_online(cpu));
550 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
552 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
555 /* CPU is dead, so no lock needed. */
556 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
565 static void takeover_tasklets(unsigned int cpu)
567 struct tasklet_struct **i;
569 /* CPU is dead, so no lock needed. */
572 /* Find end, append list for that CPU. */
573 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
574 *i = per_cpu(tasklet_vec, cpu).list;
575 per_cpu(tasklet_vec, cpu).list = NULL;
576 raise_softirq_irqoff(TASKLET_SOFTIRQ);
578 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
579 *i = per_cpu(tasklet_hi_vec, cpu).list;
580 per_cpu(tasklet_hi_vec, cpu).list = NULL;
581 raise_softirq_irqoff(HI_SOFTIRQ);
585 #endif /* CONFIG_HOTPLUG_CPU */
587 static int __cpuinit cpu_callback(struct notifier_block *nfb,
588 unsigned long action,
591 int hotcpu = (unsigned long)hcpu;
592 struct task_struct *p;
596 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
598 printk("ksoftirqd for %i failed\n", hotcpu);
601 kthread_bind(p, hotcpu);
602 per_cpu(ksoftirqd, hotcpu) = p;
605 wake_up_process(per_cpu(ksoftirqd, hotcpu));
607 #ifdef CONFIG_HOTPLUG_CPU
608 case CPU_UP_CANCELED:
609 if (!per_cpu(ksoftirqd, hotcpu))
611 /* Unbind so it can run. Fall thru. */
612 kthread_bind(per_cpu(ksoftirqd, hotcpu),
613 any_online_cpu(cpu_online_map));
615 p = per_cpu(ksoftirqd, hotcpu);
616 per_cpu(ksoftirqd, hotcpu) = NULL;
618 takeover_tasklets(hotcpu);
620 #endif /* CONFIG_HOTPLUG_CPU */
625 static struct notifier_block __cpuinitdata cpu_nfb = {
626 .notifier_call = cpu_callback
629 __init int spawn_ksoftirqd(void)
631 void *cpu = (void *)(long)smp_processor_id();
632 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
634 BUG_ON(err == NOTIFY_BAD);
635 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
636 register_cpu_notifier(&cpu_nfb);
642 * Call a function on all processors
644 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
649 ret = smp_call_function(func, info, retry, wait);
656 EXPORT_SYMBOL(on_each_cpu);