kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/tick.h>
  27
  28 #define CREATE_TRACE_POINTS
  29 #include <trace/events/irq.h>
  30
  31 #include <asm/irq.h>
  32 /*
  33    - No shared variables, all the data are CPU local.
  34    - If a softirq needs serialization, let it serialize itself
  35      by its own spinlocks.
  36    - Even if softirq is serialized, only local cpu is marked for
  37      execution. Hence, we get something sort of weak cpu binding.
  38      Though it is still not clear, will it result in better locality
  39      or will not.
  40
  41    Examples:
  42    - NET RX softirq. It is multithreaded and does not require
  43      any global serialization.
  44    - NET TX softirq. It kicks software netdevice queues, hence
  45      it is logically serialized per device, but this serialization
  46      is invisible to common code.
  47    - Tasklets: serialized wrt itself.
  48  */
  49
  50 #ifndef __ARCH_IRQ_STAT
  51 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  52 EXPORT_SYMBOL(irq_stat);
  53 #endif
  54
  55 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  56
  57 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  58
  59 char *softirq_to_name[NR_SOFTIRQS] = {
  60         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  61         "TASKLET", "SCHED", "HRTIMER",  "RCU"
  62 };
  63
  64 /*
  65  * we cannot loop indefinitely here to avoid userspace starvation,
  66  * but we also don't want to introduce a worst case 1/HZ latency
  67  * to the pending events, so lets the scheduler to balance
  68  * the softirq load for us.
  69  */
  70 void wakeup_softirqd(void)
  71 {
  72         /* Interrupts are disabled: no need to stop preemption */
  73         struct task_struct *tsk = __get_cpu_var(ksoftirqd);
  74
  75         if (tsk && tsk->state != TASK_RUNNING)
  76                 wake_up_process(tsk);
  77 }
  78
  79 /*
  80  * preempt_count and SOFTIRQ_OFFSET usage:
  81  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  82  *   softirq processing.
  83  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  84  *   on local_bh_disable or local_bh_enable.
  85  * This lets us distinguish between whether we are currently processing
  86  * softirq and whether we just have bh disabled.
  87  */
  88
  89 /*
  90  * This one is for softirq.c-internal use,
  91  * where hardirqs are disabled legitimately:
  92  */
  93 #ifdef CONFIG_TRACE_IRQFLAGS
  94 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  95 {
  96         unsigned long flags;
  97
  98         WARN_ON_ONCE(in_irq());
  99
 100         raw_local_irq_save(flags);
 101         /*
 102          * The preempt tracer hooks into add_preempt_count and will break
 103          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 104          * is set and before current->softirq_enabled is cleared.
 105          * We must manually increment preempt_count here and manually
 106          * call the trace_preempt_off later.
 107          */
 108         preempt_count() += cnt;
 109         /*
 110          * Were softirqs turned off above:
 111          */
 112         if (softirq_count() == cnt)
 113                 trace_softirqs_off(ip);
 114         raw_local_irq_restore(flags);
 115
 116         if (preempt_count() == cnt)
 117                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 118 }
 119 #else /* !CONFIG_TRACE_IRQFLAGS */
 120 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 121 {
 122         add_preempt_count(cnt);
 123         barrier();
 124 }
 125 #endif /* CONFIG_TRACE_IRQFLAGS */
 126
 127 void local_bh_disable(void)
 128 {
 129         __local_bh_disable((unsigned long)__builtin_return_address(0),
 130                                 SOFTIRQ_DISABLE_OFFSET);
 131 }
 132
 133 EXPORT_SYMBOL(local_bh_disable);
 134
 135 static void __local_bh_enable(unsigned int cnt)
 136 {
 137         WARN_ON_ONCE(in_irq());
 138         WARN_ON_ONCE(!irqs_disabled());
 139
 140         if (softirq_count() == cnt)
 141                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 142         sub_preempt_count(cnt);
 143 }
 144
 145 /*
 146  * Special-case - softirqs can safely be enabled in
 147  * cond_resched_softirq(), or by __do_softirq(),
 148  * without processing still-pending softirqs:
 149  */
 150 void _local_bh_enable(void)
 151 {
 152         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 153 }
 154
 155 EXPORT_SYMBOL(_local_bh_enable);
 156
 157 static inline void _local_bh_enable_ip(unsigned long ip)
 158 {
 159         WARN_ON_ONCE(in_irq() || irqs_disabled());
 160 #ifdef CONFIG_TRACE_IRQFLAGS
 161         local_irq_disable();
 162 #endif
 163         /*
 164          * Are softirqs going to be turned on now:
 165          */
 166         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 167                 trace_softirqs_on(ip);
 168         /*
 169          * Keep preemption disabled until we are done with
 170          * softirq processing:
 171          */
 172         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 173
 174         if (unlikely(!in_interrupt() && local_softirq_pending()))
 175                 do_softirq();
 176
 177         dec_preempt_count();
 178 #ifdef CONFIG_TRACE_IRQFLAGS
 179         local_irq_enable();
 180 #endif
 181         preempt_check_resched();
 182 }
 183
 184 void local_bh_enable(void)
 185 {
 186         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 187 }
 188 EXPORT_SYMBOL(local_bh_enable);
 189
 190 void local_bh_enable_ip(unsigned long ip)
 191 {
 192         _local_bh_enable_ip(ip);
 193 }
 194 EXPORT_SYMBOL(local_bh_enable_ip);
 195
 196 /*
 197  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
 198  * and we fall back to softirqd after that.
 199  *
 200  * This number has been established via experimentation.
 201  * The two things to balance is latency against fairness -
 202  * we want to handle softirqs as soon as possible, but they
 203  * should not be able to lock up the box.
 204  */
 205 #define MAX_SOFTIRQ_RESTART 10
 206
 207 asmlinkage void __do_softirq(void)
 208 {
 209         struct softirq_action *h;
 210         __u32 pending;
 211         int max_restart = MAX_SOFTIRQ_RESTART;
 212         int cpu;
 213
 214         pending = local_softirq_pending();
 215         account_system_vtime(current);
 216
 217         __local_bh_disable((unsigned long)__builtin_return_address(0),
 218                                 SOFTIRQ_OFFSET);
 219         lockdep_softirq_enter();
 220
 221         cpu = smp_processor_id();
 222 restart:
 223         /* Reset the pending bitmask before enabling irqs */
 224         set_softirq_pending(0);
 225
 226         local_irq_enable();
 227
 228         h = softirq_vec;
 229
 230         do {
 231                 if (pending & 1) {
 232                         int prev_count = preempt_count();
 233                         kstat_incr_softirqs_this_cpu(h - softirq_vec);
 234
 235                         trace_softirq_entry(h, softirq_vec);
 236                         h->action(h);
 237                         trace_softirq_exit(h, softirq_vec);
 238                         if (unlikely(prev_count != preempt_count())) {
 239                                 printk(KERN_ERR "huh, entered softirq %td %s %p"
 240                                        "with preempt_count %08x,"
 241                                        " exited with %08x?\n", h - softirq_vec,
 242                                        softirq_to_name[h - softirq_vec],
 243                                        h->action, prev_count, preempt_count());
 244                                 preempt_count() = prev_count;
 245                         }
 246
 247                         rcu_bh_qs(cpu);
 248                 }
 249                 h++;
 250                 pending >>= 1;
 251         } while (pending);
 252
 253         local_irq_disable();
 254
 255         pending = local_softirq_pending();
 256         if (pending && --max_restart)
 257                 goto restart;
 258
 259         if (pending)
 260                 wakeup_softirqd();
 261
 262         lockdep_softirq_exit();
 263
 264         account_system_vtime(current);
 265         __local_bh_enable(SOFTIRQ_OFFSET);
 266 }
 267
 268 #ifndef __ARCH_HAS_DO_SOFTIRQ
 269
 270 asmlinkage void do_softirq(void)
 271 {
 272         __u32 pending;
 273         unsigned long flags;
 274
 275         if (in_interrupt())
 276                 return;
 277
 278         local_irq_save(flags);
 279
 280         pending = local_softirq_pending();
 281
 282         if (pending)
 283                 __do_softirq();
 284
 285         local_irq_restore(flags);
 286 }
 287
 288 #endif
 289
 290 /*
 291  * Enter an interrupt context.
 292  */
 293 void irq_enter(void)
 294 {
 295         int cpu = smp_processor_id();
 296
 297         rcu_irq_enter();
 298         if (idle_cpu(cpu) && !in_interrupt()) {
 299                 /*
 300                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 301                  * here, as softirq will be serviced on return from interrupt.
 302                  */
 303                 local_bh_disable();
 304                 tick_check_idle(cpu);
 305                 _local_bh_enable();
 306         }
 307
 308         __irq_enter();
 309 }
 310
 311 #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
 312 # define invoke_softirq()       __do_softirq()
 313 #else
 314 # define invoke_softirq()       do_softirq()
 315 #endif
 316
 317 /*
 318  * Exit an interrupt context. Process softirqs if needed and possible:
 319  */
 320 void irq_exit(void)
 321 {
 322         account_system_vtime(current);
 323         trace_hardirq_exit();
 324         sub_preempt_count(IRQ_EXIT_OFFSET);
 325         if (!in_interrupt() && local_softirq_pending())
 326                 invoke_softirq();
 327
 328 #ifdef CONFIG_NO_HZ
 329         /* Make sure that timer wheel updates are propagated */
 330         rcu_irq_exit();
 331         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 332                 tick_nohz_stop_sched_tick(0);
 333 #endif
 334         preempt_enable_no_resched();
 335 }
 336
 337 /*
 338  * This function must run with irqs disabled!
 339  */
 340 inline void raise_softirq_irqoff(unsigned int nr)
 341 {
 342         __raise_softirq_irqoff(nr);
 343
 344         /*
 345          * If we're in an interrupt or softirq, we're done
 346          * (this also catches softirq-disabled code). We will
 347          * actually run the softirq once we return from
 348          * the irq or softirq.
 349          *
 350          * Otherwise we wake up ksoftirqd to make sure we
 351          * schedule the softirq soon.
 352          */
 353         if (!in_interrupt())
 354                 wakeup_softirqd();
 355 }
 356
 357 void raise_softirq(unsigned int nr)
 358 {
 359         unsigned long flags;
 360
 361         local_irq_save(flags);
 362         raise_softirq_irqoff(nr);
 363         local_irq_restore(flags);
 364 }
 365
 366 void open_softirq(int nr, void (*action)(struct softirq_action *))
 367 {
 368         softirq_vec[nr].action = action;
 369 }
 370
 371 /*
 372  * Tasklets
 373  */
 374 struct tasklet_head
 375 {
 376         struct tasklet_struct *head;
 377         struct tasklet_struct **tail;
 378 };
 379
 380 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 381 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 382
 383 void __tasklet_schedule(struct tasklet_struct *t)
 384 {
 385         unsigned long flags;
 386
 387         local_irq_save(flags);
 388         t->next = NULL;
 389         *__get_cpu_var(tasklet_vec).tail = t;
 390         __get_cpu_var(tasklet_vec).tail = &(t->next);
 391         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 392         local_irq_restore(flags);
 393 }
 394
 395 EXPORT_SYMBOL(__tasklet_schedule);
 396
 397 void __tasklet_hi_schedule(struct tasklet_struct *t)
 398 {
 399         unsigned long flags;
 400
 401         local_irq_save(flags);
 402         t->next = NULL;
 403         *__get_cpu_var(tasklet_hi_vec).tail = t;
 404         __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 405         raise_softirq_irqoff(HI_SOFTIRQ);
 406         local_irq_restore(flags);
 407 }
 408
 409 EXPORT_SYMBOL(__tasklet_hi_schedule);
 410
 411 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 412 {
 413         BUG_ON(!irqs_disabled());
 414
 415         t->next = __get_cpu_var(tasklet_hi_vec).head;
 416         __get_cpu_var(tasklet_hi_vec).head = t;
 417         __raise_softirq_irqoff(HI_SOFTIRQ);
 418 }
 419
 420 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 421
 422 static void tasklet_action(struct softirq_action *a)
 423 {
 424         struct tasklet_struct *list;
 425
 426         local_irq_disable();
 427         list = __get_cpu_var(tasklet_vec).head;
 428         __get_cpu_var(tasklet_vec).head = NULL;
 429         __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
 430         local_irq_enable();
 431
 432         while (list) {
 433                 struct tasklet_struct *t = list;
 434
 435                 list = list->next;
 436
 437                 if (tasklet_trylock(t)) {
 438                         if (!atomic_read(&t->count)) {
 439                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 440                                         BUG();
 441                                 t->func(t->data);
 442                                 tasklet_unlock(t);
 443                                 continue;
 444                         }
 445                         tasklet_unlock(t);
 446                 }
 447
 448                 local_irq_disable();
 449                 t->next = NULL;
 450                 *__get_cpu_var(tasklet_vec).tail = t;
 451                 __get_cpu_var(tasklet_vec).tail = &(t->next);
 452                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 453                 local_irq_enable();
 454         }
 455 }
 456
 457 static void tasklet_hi_action(struct softirq_action *a)
 458 {
 459         struct tasklet_struct *list;
 460
 461         local_irq_disable();
 462         list = __get_cpu_var(tasklet_hi_vec).head;
 463         __get_cpu_var(tasklet_hi_vec).head = NULL;
 464         __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
 465         local_irq_enable();
 466
 467         while (list) {
 468                 struct tasklet_struct *t = list;
 469
 470                 list = list->next;
 471
 472                 if (tasklet_trylock(t)) {
 473                         if (!atomic_read(&t->count)) {
 474                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 475                                         BUG();
 476                                 t->func(t->data);
 477                                 tasklet_unlock(t);
 478                                 continue;
 479                         }
 480                         tasklet_unlock(t);
 481                 }
 482
 483                 local_irq_disable();
 484                 t->next = NULL;
 485                 *__get_cpu_var(tasklet_hi_vec).tail = t;
 486                 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
 487                 __raise_softirq_irqoff(HI_SOFTIRQ);
 488                 local_irq_enable();
 489         }
 490 }
 491
 492
 493 void tasklet_init(struct tasklet_struct *t,
 494                   void (*func)(unsigned long), unsigned long data)
 495 {
 496         t->next = NULL;
 497         t->state = 0;
 498         atomic_set(&t->count, 0);
 499         t->func = func;
 500         t->data = data;
 501 }
 502
 503 EXPORT_SYMBOL(tasklet_init);
 504
 505 void tasklet_kill(struct tasklet_struct *t)
 506 {
 507         if (in_interrupt())
 508                 printk("Attempt to kill tasklet from interrupt\n");
 509
 510         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 511                 do {
 512                         yield();
 513                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 514         }
 515         tasklet_unlock_wait(t);
 516         clear_bit(TASKLET_STATE_SCHED, &t->state);
 517 }
 518
 519 EXPORT_SYMBOL(tasklet_kill);
 520
 521 /*
 522  * tasklet_hrtimer
 523  */
 524
 525 /*
 526  * The trampoline is called when the hrtimer expires. If this is
 527  * called from the hrtimer interrupt then we schedule the tasklet as
 528  * the timer callback function expects to run in softirq context. If
 529  * it's called in softirq context anyway (i.e. high resolution timers
 530  * disabled) then the hrtimer callback is called right away.
 531  */
 532 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 533 {
 534         struct tasklet_hrtimer *ttimer =
 535                 container_of(timer, struct tasklet_hrtimer, timer);
 536
 537         if (hrtimer_is_hres_active(timer)) {
 538                 tasklet_hi_schedule(&ttimer->tasklet);
 539                 return HRTIMER_NORESTART;
 540         }
 541         return ttimer->function(timer);
 542 }
 543
 544 /*
 545  * Helper function which calls the hrtimer callback from
 546  * tasklet/softirq context
 547  */
 548 static void __tasklet_hrtimer_trampoline(unsigned long data)
 549 {
 550         struct tasklet_hrtimer *ttimer = (void *)data;
 551         enum hrtimer_restart restart;
 552
 553         restart = ttimer->function(&ttimer->timer);
 554         if (restart != HRTIMER_NORESTART)
 555                 hrtimer_restart(&ttimer->timer);
 556 }
 557
 558 /**
 559  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 560  * @ttimer:      tasklet_hrtimer which is initialized
 561  * @function:    hrtimer callback funtion which gets called from softirq context
 562  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 563  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 564  */
 565 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 566                           enum hrtimer_restart (*function)(struct hrtimer *),
 567                           clockid_t which_clock, enum hrtimer_mode mode)
 568 {
 569         hrtimer_init(&ttimer->timer, which_clock, mode);
 570         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 571         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 572                      (unsigned long)ttimer);
 573         ttimer->function = function;
 574 }
 575 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 576
 577 /*
 578  * Remote softirq bits
 579  */
 580
 581 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 582 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 583
 584 static void __local_trigger(struct call_single_data *cp, int softirq)
 585 {
 586         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 587
 588         list_add_tail(&cp->list, head);
 589
 590         /* Trigger the softirq only if the list was previously empty.  */
 591         if (head->next == &cp->list)
 592                 raise_softirq_irqoff(softirq);
 593 }
 594
 595 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 596 static void remote_softirq_receive(void *data)
 597 {
 598         struct call_single_data *cp = data;
 599         unsigned long flags;
 600         int softirq;
 601
 602         softirq = cp->priv;
 603
 604         local_irq_save(flags);
 605         __local_trigger(cp, softirq);
 606         local_irq_restore(flags);
 607 }
 608
 609 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 610 {
 611         if (cpu_online(cpu)) {
 612                 cp->func = remote_softirq_receive;
 613                 cp->info = cp;
 614                 cp->flags = 0;
 615                 cp->priv = softirq;
 616
 617                 __smp_call_function_single(cpu, cp, 0);
 618                 return 0;
 619         }
 620         return 1;
 621 }
 622 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 623 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 624 {
 625         return 1;
 626 }
 627 #endif
 628
 629 /**
 630  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 631  * @cp: private SMP call function data area
 632  * @cpu: the remote cpu
 633  * @this_cpu: the currently executing cpu
 634  * @softirq: the softirq for the work
 635  *
 636  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 637  * done, the work is instead queued up on the local cpu.
 638  *
 639  * Interrupts must be disabled.
 640  */
 641 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 642 {
 643         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 644                 __local_trigger(cp, softirq);
 645 }
 646 EXPORT_SYMBOL(__send_remote_softirq);
 647
 648 /**
 649  * send_remote_softirq - try to schedule softirq work on a remote cpu
 650  * @cp: private SMP call function data area
 651  * @cpu: the remote cpu
 652  * @softirq: the softirq for the work
 653  *
 654  * Like __send_remote_softirq except that disabling interrupts and
 655  * computing the current cpu is done for the caller.
 656  */
 657 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 658 {
 659         unsigned long flags;
 660         int this_cpu;
 661
 662         local_irq_save(flags);
 663         this_cpu = smp_processor_id();
 664         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 665         local_irq_restore(flags);
 666 }
 667 EXPORT_SYMBOL(send_remote_softirq);
 668
 669 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 670                                                unsigned long action, void *hcpu)
 671 {
 672         /*
 673          * If a CPU goes away, splice its entries to the current CPU
 674          * and trigger a run of the softirq
 675          */
 676         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 677                 int cpu = (unsigned long) hcpu;
 678                 int i;
 679
 680                 local_irq_disable();
 681                 for (i = 0; i < NR_SOFTIRQS; i++) {
 682                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 683                         struct list_head *local_head;
 684
 685                         if (list_empty(head))
 686                                 continue;
 687
 688                         local_head = &__get_cpu_var(softirq_work_list[i]);
 689                         list_splice_init(head, local_head);
 690                         raise_softirq_irqoff(i);
 691                 }
 692                 local_irq_enable();
 693         }
 694
 695         return NOTIFY_OK;
 696 }
 697
 698 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 699         .notifier_call  = remote_softirq_cpu_notify,
 700 };
 701
 702 void __init softirq_init(void)
 703 {
 704         int cpu;
 705
 706         for_each_possible_cpu(cpu) {
 707                 int i;
 708
 709                 per_cpu(tasklet_vec, cpu).tail =
 710                         &per_cpu(tasklet_vec, cpu).head;
 711                 per_cpu(tasklet_hi_vec, cpu).tail =
 712                         &per_cpu(tasklet_hi_vec, cpu).head;
 713                 for (i = 0; i < NR_SOFTIRQS; i++)
 714                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 715         }
 716
 717         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 718
 719         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 720         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 721 }
 722
 723 static int ksoftirqd(void * __bind_cpu)
 724 {
 725         set_current_state(TASK_INTERRUPTIBLE);
 726
 727         current->flags |= PF_KSOFTIRQD;
 728         while (!kthread_should_stop()) {
 729                 preempt_disable();
 730                 if (!local_softirq_pending()) {
 731                         preempt_enable_no_resched();
 732                         schedule();
 733                         preempt_disable();
 734                 }
 735
 736                 __set_current_state(TASK_RUNNING);
 737
 738                 while (local_softirq_pending()) {
 739                         /* Preempt disable stops cpu going offline.
 740                            If already offline, we'll be on wrong CPU:
 741                            don't process */
 742                         if (cpu_is_offline((long)__bind_cpu))
 743                                 goto wait_to_die;
 744                         do_softirq();
 745                         preempt_enable_no_resched();
 746                         cond_resched();
 747                         preempt_disable();
 748                         rcu_sched_qs((long)__bind_cpu);
 749                 }
 750                 preempt_enable();
 751                 set_current_state(TASK_INTERRUPTIBLE);
 752         }
 753         __set_current_state(TASK_RUNNING);
 754         return 0;
 755
 756 wait_to_die:
 757         preempt_enable();
 758         /* Wait for kthread_stop */
 759         set_current_state(TASK_INTERRUPTIBLE);
 760         while (!kthread_should_stop()) {
 761                 schedule();
 762                 set_current_state(TASK_INTERRUPTIBLE);
 763         }
 764         __set_current_state(TASK_RUNNING);
 765         return 0;
 766 }
 767
 768 #ifdef CONFIG_HOTPLUG_CPU
 769 /*
 770  * tasklet_kill_immediate is called to remove a tasklet which can already be
 771  * scheduled for execution on @cpu.
 772  *
 773  * Unlike tasklet_kill, this function removes the tasklet
 774  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 775  *
 776  * When this function is called, @cpu must be in the CPU_DEAD state.
 777  */
 778 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 779 {
 780         struct tasklet_struct **i;
 781
 782         BUG_ON(cpu_online(cpu));
 783         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 784
 785         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 786                 return;
 787
 788         /* CPU is dead, so no lock needed. */
 789         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 790                 if (*i == t) {
 791                         *i = t->next;
 792                         /* If this was the tail element, move the tail ptr */
 793                         if (*i == NULL)
 794                                 per_cpu(tasklet_vec, cpu).tail = i;
 795                         return;
 796                 }
 797         }
 798         BUG();
 799 }
 800
 801 static void takeover_tasklets(unsigned int cpu)
 802 {
 803         /* CPU is dead, so no lock needed. */
 804         local_irq_disable();
 805
 806         /* Find end, append list for that CPU. */
 807         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 808                 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
 809                 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
 810                 per_cpu(tasklet_vec, cpu).head = NULL;
 811                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 812         }
 813         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 814
 815         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 816                 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
 817                 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
 818                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 819                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 820         }
 821         raise_softirq_irqoff(HI_SOFTIRQ);
 822
 823         local_irq_enable();
 824 }
 825 #endif /* CONFIG_HOTPLUG_CPU */
 826
 827 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 828                                   unsigned long action,
 829                                   void *hcpu)
 830 {
 831         int hotcpu = (unsigned long)hcpu;
 832         struct task_struct *p;
 833
 834         switch (action) {
 835         case CPU_UP_PREPARE:
 836         case CPU_UP_PREPARE_FROZEN:
 837                 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
 838                 if (IS_ERR(p)) {
 839                         printk("ksoftirqd for %i failed\n", hotcpu);
 840                         return NOTIFY_BAD;
 841                 }
 842                 kthread_bind(p, hotcpu);
 843                 per_cpu(ksoftirqd, hotcpu) = p;
 844                 break;
 845         case CPU_ONLINE:
 846         case CPU_ONLINE_FROZEN:
 847                 wake_up_process(per_cpu(ksoftirqd, hotcpu));
 848                 break;
 849 #ifdef CONFIG_HOTPLUG_CPU
 850         case CPU_UP_CANCELED:
 851         case CPU_UP_CANCELED_FROZEN:
 852                 if (!per_cpu(ksoftirqd, hotcpu))
 853                         break;
 854                 /* Unbind so it can run.  Fall thru. */
 855                 kthread_bind(per_cpu(ksoftirqd, hotcpu),
 856                              cpumask_any(cpu_online_mask));
 857         case CPU_DEAD:
 858         case CPU_DEAD_FROZEN: {
 859                 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 860
 861                 p = per_cpu(ksoftirqd, hotcpu);
 862                 per_cpu(ksoftirqd, hotcpu) = NULL;
 863                 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
 864                 kthread_stop(p);
 865                 takeover_tasklets(hotcpu);
 866                 break;
 867         }
 868 #endif /* CONFIG_HOTPLUG_CPU */
 869         }
 870         return NOTIFY_OK;
 871 }
 872
 873 static struct notifier_block __cpuinitdata cpu_nfb = {
 874         .notifier_call = cpu_callback
 875 };
 876
 877 static __init int spawn_ksoftirqd(void)
 878 {
 879         void *cpu = (void *)(long)smp_processor_id();
 880         int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
 881
 882         BUG_ON(err == NOTIFY_BAD);
 883         cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
 884         register_cpu_notifier(&cpu_nfb);
 885         return 0;
 886 }
 887 early_initcall(spawn_ksoftirqd);
 888
 889 #ifdef CONFIG_SMP
 890 /*
 891  * Call a function on all processors
 892  */
 893 int on_each_cpu(void (*func) (void *info), void *info, int wait)
 894 {
 895         int ret = 0;
 896
 897         preempt_disable();
 898         ret = smp_call_function(func, info, wait);
 899         local_irq_disable();
 900         func(info);
 901         local_irq_enable();
 902         preempt_enable();
 903         return ret;
 904 }
 905 EXPORT_SYMBOL(on_each_cpu);
 906 #endif
 907
 908 /*
 909  * [ These __weak aliases are kept in a separate compilation unit, so that
 910  *   GCC does not inline them incorrectly. ]
 911  */
 912
 913 int __init __weak early_irq_init(void)
 914 {
 915         return 0;
 916 }
 917
 918 int __init __weak arch_probe_nr_irqs(void)
 919 {
 920         return 0;
 921 }
 922
 923 int __init __weak arch_early_irq_init(void)
 924 {
 925         return 0;
 926 }
 927
 928 int __weak arch_init_chip_data(struct irq_desc *desc, int node)
 929 {
 930         return 0;
 931 }