2 * linux/kernel/softirq.c
4 * Copyright (C) 1992 Linus Torvalds
6 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
9 #include <linux/module.h>
10 #include <linux/kernel_stat.h>
11 #include <linux/interrupt.h>
12 #include <linux/init.h>
14 #include <linux/notifier.h>
15 #include <linux/percpu.h>
16 #include <linux/cpu.h>
17 #include <linux/kthread.h>
18 #include <linux/rcupdate.h>
19 #include <linux/smp.h>
23 - No shared variables, all the data are CPU local.
24 - If a softirq needs serialization, let it serialize itself
26 - Even if softirq is serialized, only local cpu is marked for
27 execution. Hence, we get something sort of weak cpu binding.
28 Though it is still not clear, will it result in better locality
32 - NET RX softirq. It is multithreaded and does not require
33 any global serialization.
34 - NET TX softirq. It kicks software netdevice queues, hence
35 it is logically serialized per device, but this serialization
36 is invisible to common code.
37 - Tasklets: serialized wrt itself.
40 #ifndef __ARCH_IRQ_STAT
41 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
42 EXPORT_SYMBOL(irq_stat);
45 static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
47 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
50 * we cannot loop indefinitely here to avoid userspace starvation,
51 * but we also don't want to introduce a worst case 1/HZ latency
52 * to the pending events, so lets the scheduler to balance
53 * the softirq load for us.
55 static inline void wakeup_softirqd(void)
57 /* Interrupts are disabled: no need to stop preemption */
58 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
60 if (tsk && tsk->state != TASK_RUNNING)
65 * This one is for softirq.c-internal use,
66 * where hardirqs are disabled legitimately:
68 static void __local_bh_disable(unsigned long ip)
72 WARN_ON_ONCE(in_irq());
74 raw_local_irq_save(flags);
75 add_preempt_count(SOFTIRQ_OFFSET);
77 * Were softirqs turned off above:
79 if (softirq_count() == SOFTIRQ_OFFSET)
80 trace_softirqs_off(ip);
81 raw_local_irq_restore(flags);
84 void local_bh_disable(void)
86 __local_bh_disable((unsigned long)__builtin_return_address(0));
89 EXPORT_SYMBOL(local_bh_disable);
91 void __local_bh_enable(void)
93 WARN_ON_ONCE(in_irq());
96 * softirqs should never be enabled by __local_bh_enable(),
97 * it always nests inside local_bh_enable() sections:
99 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
101 sub_preempt_count(SOFTIRQ_OFFSET);
103 EXPORT_SYMBOL_GPL(__local_bh_enable);
106 * Special-case - softirqs can safely be enabled in
107 * cond_resched_softirq(), or by __do_softirq(),
108 * without processing still-pending softirqs:
110 void _local_bh_enable(void)
112 WARN_ON_ONCE(in_irq());
113 WARN_ON_ONCE(!irqs_disabled());
115 if (softirq_count() == SOFTIRQ_OFFSET)
116 trace_softirqs_on((unsigned long)__builtin_return_address(0));
117 sub_preempt_count(SOFTIRQ_OFFSET);
120 EXPORT_SYMBOL(_local_bh_enable);
122 void local_bh_enable(void)
126 WARN_ON_ONCE(in_irq());
127 WARN_ON_ONCE(irqs_disabled());
129 local_irq_save(flags);
131 * Are softirqs going to be turned on now:
133 if (softirq_count() == SOFTIRQ_OFFSET)
134 trace_softirqs_on((unsigned long)__builtin_return_address(0));
136 * Keep preemption disabled until we are done with
137 * softirq processing:
139 sub_preempt_count(SOFTIRQ_OFFSET - 1);
141 if (unlikely(!in_interrupt() && local_softirq_pending()))
145 local_irq_restore(flags);
146 preempt_check_resched();
148 EXPORT_SYMBOL(local_bh_enable);
150 void local_bh_enable_ip(unsigned long ip)
154 WARN_ON_ONCE(in_irq());
156 local_irq_save(flags);
158 * Are softirqs going to be turned on now:
160 if (softirq_count() == SOFTIRQ_OFFSET)
161 trace_softirqs_on(ip);
163 * Keep preemption disabled until we are done with
164 * softirq processing:
166 sub_preempt_count(SOFTIRQ_OFFSET - 1);
168 if (unlikely(!in_interrupt() && local_softirq_pending()))
172 local_irq_restore(flags);
173 preempt_check_resched();
175 EXPORT_SYMBOL(local_bh_enable_ip);
178 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
179 * and we fall back to softirqd after that.
181 * This number has been established via experimentation.
182 * The two things to balance is latency against fairness -
183 * we want to handle softirqs as soon as possible, but they
184 * should not be able to lock up the box.
186 #define MAX_SOFTIRQ_RESTART 10
188 asmlinkage void __do_softirq(void)
190 struct softirq_action *h;
192 int max_restart = MAX_SOFTIRQ_RESTART;
195 pending = local_softirq_pending();
196 __local_bh_disable((unsigned long)__builtin_return_address(0));
197 trace_softirq_enter();
199 cpu = smp_processor_id();
201 /* Reset the pending bitmask before enabling irqs */
202 set_softirq_pending(0);
211 rcu_bh_qsctr_inc(cpu);
219 pending = local_softirq_pending();
220 if (pending && --max_restart)
226 trace_softirq_exit();
230 #ifndef __ARCH_HAS_DO_SOFTIRQ
232 asmlinkage void do_softirq(void)
240 local_irq_save(flags);
242 pending = local_softirq_pending();
247 local_irq_restore(flags);
250 EXPORT_SYMBOL(do_softirq);
254 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
255 # define invoke_softirq() __do_softirq()
257 # define invoke_softirq() do_softirq()
261 * Exit an interrupt context. Process softirqs if needed and possible:
265 account_system_vtime(current);
266 trace_hardirq_exit();
267 sub_preempt_count(IRQ_EXIT_OFFSET);
268 if (!in_interrupt() && local_softirq_pending())
270 preempt_enable_no_resched();
274 * This function must run with irqs disabled!
276 inline fastcall void raise_softirq_irqoff(unsigned int nr)
278 __raise_softirq_irqoff(nr);
281 * If we're in an interrupt or softirq, we're done
282 * (this also catches softirq-disabled code). We will
283 * actually run the softirq once we return from
284 * the irq or softirq.
286 * Otherwise we wake up ksoftirqd to make sure we
287 * schedule the softirq soon.
293 EXPORT_SYMBOL(raise_softirq_irqoff);
295 void fastcall raise_softirq(unsigned int nr)
299 local_irq_save(flags);
300 raise_softirq_irqoff(nr);
301 local_irq_restore(flags);
304 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
306 softirq_vec[nr].data = data;
307 softirq_vec[nr].action = action;
310 EXPORT_SYMBOL(open_softirq);
315 struct tasklet_struct *list;
318 /* Some compilers disobey section attribute on statics when not
320 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
321 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
323 void fastcall __tasklet_schedule(struct tasklet_struct *t)
327 local_irq_save(flags);
328 t->next = __get_cpu_var(tasklet_vec).list;
329 __get_cpu_var(tasklet_vec).list = t;
330 raise_softirq_irqoff(TASKLET_SOFTIRQ);
331 local_irq_restore(flags);
334 EXPORT_SYMBOL(__tasklet_schedule);
336 void fastcall __tasklet_hi_schedule(struct tasklet_struct *t)
340 local_irq_save(flags);
341 t->next = __get_cpu_var(tasklet_hi_vec).list;
342 __get_cpu_var(tasklet_hi_vec).list = t;
343 raise_softirq_irqoff(HI_SOFTIRQ);
344 local_irq_restore(flags);
347 EXPORT_SYMBOL(__tasklet_hi_schedule);
349 static void tasklet_action(struct softirq_action *a)
351 struct tasklet_struct *list;
354 list = __get_cpu_var(tasklet_vec).list;
355 __get_cpu_var(tasklet_vec).list = NULL;
359 struct tasklet_struct *t = list;
363 if (tasklet_trylock(t)) {
364 if (!atomic_read(&t->count)) {
365 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
375 t->next = __get_cpu_var(tasklet_vec).list;
376 __get_cpu_var(tasklet_vec).list = t;
377 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
382 static void tasklet_hi_action(struct softirq_action *a)
384 struct tasklet_struct *list;
387 list = __get_cpu_var(tasklet_hi_vec).list;
388 __get_cpu_var(tasklet_hi_vec).list = NULL;
392 struct tasklet_struct *t = list;
396 if (tasklet_trylock(t)) {
397 if (!atomic_read(&t->count)) {
398 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
408 t->next = __get_cpu_var(tasklet_hi_vec).list;
409 __get_cpu_var(tasklet_hi_vec).list = t;
410 __raise_softirq_irqoff(HI_SOFTIRQ);
416 void tasklet_init(struct tasklet_struct *t,
417 void (*func)(unsigned long), unsigned long data)
421 atomic_set(&t->count, 0);
426 EXPORT_SYMBOL(tasklet_init);
428 void tasklet_kill(struct tasklet_struct *t)
431 printk("Attempt to kill tasklet from interrupt\n");
433 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
436 while (test_bit(TASKLET_STATE_SCHED, &t->state));
438 tasklet_unlock_wait(t);
439 clear_bit(TASKLET_STATE_SCHED, &t->state);
442 EXPORT_SYMBOL(tasklet_kill);
444 void __init softirq_init(void)
446 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
447 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
450 static int ksoftirqd(void * __bind_cpu)
452 set_user_nice(current, 19);
453 current->flags |= PF_NOFREEZE;
455 set_current_state(TASK_INTERRUPTIBLE);
457 while (!kthread_should_stop()) {
459 if (!local_softirq_pending()) {
460 preempt_enable_no_resched();
465 __set_current_state(TASK_RUNNING);
467 while (local_softirq_pending()) {
468 /* Preempt disable stops cpu going offline.
469 If already offline, we'll be on wrong CPU:
471 if (cpu_is_offline((long)__bind_cpu))
474 preempt_enable_no_resched();
479 set_current_state(TASK_INTERRUPTIBLE);
481 __set_current_state(TASK_RUNNING);
486 /* Wait for kthread_stop */
487 set_current_state(TASK_INTERRUPTIBLE);
488 while (!kthread_should_stop()) {
490 set_current_state(TASK_INTERRUPTIBLE);
492 __set_current_state(TASK_RUNNING);
496 #ifdef CONFIG_HOTPLUG_CPU
498 * tasklet_kill_immediate is called to remove a tasklet which can already be
499 * scheduled for execution on @cpu.
501 * Unlike tasklet_kill, this function removes the tasklet
502 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
504 * When this function is called, @cpu must be in the CPU_DEAD state.
506 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
508 struct tasklet_struct **i;
510 BUG_ON(cpu_online(cpu));
511 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
513 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
516 /* CPU is dead, so no lock needed. */
517 for (i = &per_cpu(tasklet_vec, cpu).list; *i; i = &(*i)->next) {
526 static void takeover_tasklets(unsigned int cpu)
528 struct tasklet_struct **i;
530 /* CPU is dead, so no lock needed. */
533 /* Find end, append list for that CPU. */
534 for (i = &__get_cpu_var(tasklet_vec).list; *i; i = &(*i)->next);
535 *i = per_cpu(tasklet_vec, cpu).list;
536 per_cpu(tasklet_vec, cpu).list = NULL;
537 raise_softirq_irqoff(TASKLET_SOFTIRQ);
539 for (i = &__get_cpu_var(tasklet_hi_vec).list; *i; i = &(*i)->next);
540 *i = per_cpu(tasklet_hi_vec, cpu).list;
541 per_cpu(tasklet_hi_vec, cpu).list = NULL;
542 raise_softirq_irqoff(HI_SOFTIRQ);
546 #endif /* CONFIG_HOTPLUG_CPU */
548 static int __devinit cpu_callback(struct notifier_block *nfb,
549 unsigned long action,
552 int hotcpu = (unsigned long)hcpu;
553 struct task_struct *p;
557 BUG_ON(per_cpu(tasklet_vec, hotcpu).list);
558 BUG_ON(per_cpu(tasklet_hi_vec, hotcpu).list);
559 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
561 printk("ksoftirqd for %i failed\n", hotcpu);
564 kthread_bind(p, hotcpu);
565 per_cpu(ksoftirqd, hotcpu) = p;
568 wake_up_process(per_cpu(ksoftirqd, hotcpu));
570 #ifdef CONFIG_HOTPLUG_CPU
571 case CPU_UP_CANCELED:
572 if (!per_cpu(ksoftirqd, hotcpu))
574 /* Unbind so it can run. Fall thru. */
575 kthread_bind(per_cpu(ksoftirqd, hotcpu),
576 any_online_cpu(cpu_online_map));
578 p = per_cpu(ksoftirqd, hotcpu);
579 per_cpu(ksoftirqd, hotcpu) = NULL;
581 takeover_tasklets(hotcpu);
583 #endif /* CONFIG_HOTPLUG_CPU */
588 static struct notifier_block __devinitdata cpu_nfb = {
589 .notifier_call = cpu_callback
592 __init int spawn_ksoftirqd(void)
594 void *cpu = (void *)(long)smp_processor_id();
595 cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
596 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
597 register_cpu_notifier(&cpu_nfb);
603 * Call a function on all processors
605 int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait)
610 ret = smp_call_function(func, info, retry, wait);
617 EXPORT_SYMBOL(on_each_cpu);