kernel/cpu.c

   1 /* CPU control.
   2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3  *
   4  * This code is licenced under the GPL.
   5  */
   6 #include <linux/proc_fs.h>
   7 #include <linux/smp.h>
   8 #include <linux/init.h>
   9 #include <linux/notifier.h>
  10 #include <linux/sched/signal.h>
  11 #include <linux/sched/hotplug.h>
  12 #include <linux/sched/task.h>
  13 #include <linux/unistd.h>
  14 #include <linux/cpu.h>
  15 #include <linux/oom.h>
  16 #include <linux/rcupdate.h>
  17 #include <linux/export.h>
  18 #include <linux/bug.h>
  19 #include <linux/kthread.h>
  20 #include <linux/stop_machine.h>
  21 #include <linux/mutex.h>
  22 #include <linux/gfp.h>
  23 #include <linux/suspend.h>
  24 #include <linux/lockdep.h>
  25 #include <linux/tick.h>
  26 #include <linux/irq.h>
  27 #include <linux/smpboot.h>
  28 #include <linux/relay.h>
  29 #include <linux/slab.h>
  30 #include <linux/percpu-rwsem.h>
  31
  32 #include <trace/events/power.h>
  33 #define CREATE_TRACE_POINTS
  34 #include <trace/events/cpuhp.h>
  35
  36 #include "smpboot.h"
  37
  38 /**
  39  * cpuhp_cpu_state - Per cpu hotplug state storage
  40  * @state:      The current cpu state
  41  * @target:     The target state
  42  * @thread:     Pointer to the hotplug thread
  43  * @should_run: Thread should execute
  44  * @rollback:   Perform a rollback
  45  * @single:     Single callback invocation
  46  * @bringup:    Single callback bringup or teardown selector
  47  * @cb_state:   The state for a single callback (install/uninstall)
  48  * @result:     Result of the operation
  49  * @done:       Signal completion to the issuer of the task
  50  */
  51 struct cpuhp_cpu_state {
  52         enum cpuhp_state        state;
  53         enum cpuhp_state        target;
  54 #ifdef CONFIG_SMP
  55         struct task_struct      *thread;
  56         bool                    should_run;
  57         bool                    rollback;
  58         bool                    single;
  59         bool                    bringup;
  60         struct hlist_node       *node;
  61         enum cpuhp_state        cb_state;
  62         int                     result;
  63         struct completion       done;
  64 #endif
  65 };
  66
  67 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state);
  68
  69 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  70 static struct lock_class_key cpuhp_state_key;
  71 static struct lockdep_map cpuhp_state_lock_map =
  72         STATIC_LOCKDEP_MAP_INIT("cpuhp_state", &cpuhp_state_key);
  73 #endif
  74
  75 /**
  76  * cpuhp_step - Hotplug state machine step
  77  * @name:       Name of the step
  78  * @startup:    Startup function of the step
  79  * @teardown:   Teardown function of the step
  80  * @skip_onerr: Do not invoke the functions on error rollback
  81  *              Will go away once the notifiers are gone
  82  * @cant_stop:  Bringup/teardown can't be stopped at this step
  83  */
  84 struct cpuhp_step {
  85         const char              *name;
  86         union {
  87                 int             (*single)(unsigned int cpu);
  88                 int             (*multi)(unsigned int cpu,
  89                                          struct hlist_node *node);
  90         } startup;
  91         union {
  92                 int             (*single)(unsigned int cpu);
  93                 int             (*multi)(unsigned int cpu,
  94                                          struct hlist_node *node);
  95         } teardown;
  96         struct hlist_head       list;
  97         bool                    skip_onerr;
  98         bool                    cant_stop;
  99         bool                    multi_instance;
 100 };
 101
 102 static DEFINE_MUTEX(cpuhp_state_mutex);
 103 static struct cpuhp_step cpuhp_bp_states[];
 104 static struct cpuhp_step cpuhp_ap_states[];
 105
 106 static bool cpuhp_is_ap_state(enum cpuhp_state state)
 107 {
 108         /*
 109          * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 110          * purposes as that state is handled explicitly in cpu_down.
 111          */
 112         return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 113 }
 114
 115 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 116 {
 117         struct cpuhp_step *sp;
 118
 119         sp = cpuhp_is_ap_state(state) ? cpuhp_ap_states : cpuhp_bp_states;
 120         return sp + state;
 121 }
 122
 123 /**
 124  * cpuhp_invoke_callback _ Invoke the callbacks for a given state
 125  * @cpu:        The cpu for which the callback should be invoked
 126  * @step:       The step in the state machine
 127  * @bringup:    True if the bringup callback should be invoked
 128  *
 129  * Called from cpu hotplug and from the state register machinery.
 130  */
 131 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 132                                  bool bringup, struct hlist_node *node)
 133 {
 134         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 135         struct cpuhp_step *step = cpuhp_get_step(state);
 136         int (*cbm)(unsigned int cpu, struct hlist_node *node);
 137         int (*cb)(unsigned int cpu);
 138         int ret, cnt;
 139
 140         if (!step->multi_instance) {
 141                 cb = bringup ? step->startup.single : step->teardown.single;
 142                 if (!cb)
 143                         return 0;
 144                 trace_cpuhp_enter(cpu, st->target, state, cb);
 145                 ret = cb(cpu);
 146                 trace_cpuhp_exit(cpu, st->state, state, ret);
 147                 return ret;
 148         }
 149         cbm = bringup ? step->startup.multi : step->teardown.multi;
 150         if (!cbm)
 151                 return 0;
 152
 153         /* Single invocation for instance add/remove */
 154         if (node) {
 155                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 156                 ret = cbm(cpu, node);
 157                 trace_cpuhp_exit(cpu, st->state, state, ret);
 158                 return ret;
 159         }
 160
 161         /* State transition. Invoke on all instances */
 162         cnt = 0;
 163         hlist_for_each(node, &step->list) {
 164                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 165                 ret = cbm(cpu, node);
 166                 trace_cpuhp_exit(cpu, st->state, state, ret);
 167                 if (ret)
 168                         goto err;
 169                 cnt++;
 170         }
 171         return 0;
 172 err:
 173         /* Rollback the instances if one failed */
 174         cbm = !bringup ? step->startup.multi : step->teardown.multi;
 175         if (!cbm)
 176                 return ret;
 177
 178         hlist_for_each(node, &step->list) {
 179                 if (!cnt--)
 180                         break;
 181                 cbm(cpu, node);
 182         }
 183         return ret;
 184 }
 185
 186 #ifdef CONFIG_SMP
 187 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 188 static DEFINE_MUTEX(cpu_add_remove_lock);
 189 bool cpuhp_tasks_frozen;
 190 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 191
 192 /*
 193  * The following two APIs (cpu_maps_update_begin/done) must be used when
 194  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 195  */
 196 void cpu_maps_update_begin(void)
 197 {
 198         mutex_lock(&cpu_add_remove_lock);
 199 }
 200
 201 void cpu_maps_update_done(void)
 202 {
 203         mutex_unlock(&cpu_add_remove_lock);
 204 }
 205
 206 /*
 207  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 208  * Should always be manipulated under cpu_add_remove_lock
 209  */
 210 static int cpu_hotplug_disabled;
 211
 212 #ifdef CONFIG_HOTPLUG_CPU
 213
 214 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 215
 216 void cpus_read_lock(void)
 217 {
 218         percpu_down_read(&cpu_hotplug_lock);
 219 }
 220 EXPORT_SYMBOL_GPL(cpus_read_lock);
 221
 222 void cpus_read_unlock(void)
 223 {
 224         percpu_up_read(&cpu_hotplug_lock);
 225 }
 226 EXPORT_SYMBOL_GPL(cpus_read_unlock);
 227
 228 void cpus_write_lock(void)
 229 {
 230         percpu_down_write(&cpu_hotplug_lock);
 231 }
 232
 233 void cpus_write_unlock(void)
 234 {
 235         percpu_up_write(&cpu_hotplug_lock);
 236 }
 237
 238 void lockdep_assert_cpus_held(void)
 239 {
 240         percpu_rwsem_assert_held(&cpu_hotplug_lock);
 241 }
 242
 243 /*
 244  * Wait for currently running CPU hotplug operations to complete (if any) and
 245  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 246  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 247  * hotplug path before performing hotplug operations. So acquiring that lock
 248  * guarantees mutual exclusion from any currently running hotplug operations.
 249  */
 250 void cpu_hotplug_disable(void)
 251 {
 252         cpu_maps_update_begin();
 253         cpu_hotplug_disabled++;
 254         cpu_maps_update_done();
 255 }
 256 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 257
 258 static void __cpu_hotplug_enable(void)
 259 {
 260         if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 261                 return;
 262         cpu_hotplug_disabled--;
 263 }
 264
 265 void cpu_hotplug_enable(void)
 266 {
 267         cpu_maps_update_begin();
 268         __cpu_hotplug_enable();
 269         cpu_maps_update_done();
 270 }
 271 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 272 #endif  /* CONFIG_HOTPLUG_CPU */
 273
 274 static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st);
 275
 276 static int bringup_wait_for_ap(unsigned int cpu)
 277 {
 278         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 279
 280         /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 281         wait_for_completion(&st->done);
 282         BUG_ON(!cpu_online(cpu));
 283
 284         /* Unpark the stopper thread and the hotplug thread of the target cpu */
 285         stop_machine_unpark(cpu);
 286         kthread_unpark(st->thread);
 287
 288         /* Should we go further up ? */
 289         if (st->target > CPUHP_AP_ONLINE_IDLE) {
 290                 __cpuhp_kick_ap_work(st);
 291                 wait_for_completion(&st->done);
 292         }
 293         return st->result;
 294 }
 295
 296 static int bringup_cpu(unsigned int cpu)
 297 {
 298         struct task_struct *idle = idle_thread_get(cpu);
 299         int ret;
 300
 301         /*
 302          * Some architectures have to walk the irq descriptors to
 303          * setup the vector space for the cpu which comes online.
 304          * Prevent irq alloc/free across the bringup.
 305          */
 306         irq_lock_sparse();
 307
 308         /* Arch-specific enabling code. */
 309         ret = __cpu_up(cpu, idle);
 310         irq_unlock_sparse();
 311         if (ret)
 312                 return ret;
 313         return bringup_wait_for_ap(cpu);
 314 }
 315
 316 /*
 317  * Hotplug state machine related functions
 318  */
 319 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
 320 {
 321         for (st->state++; st->state < st->target; st->state++) {
 322                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 323
 324                 if (!step->skip_onerr)
 325                         cpuhp_invoke_callback(cpu, st->state, true, NULL);
 326         }
 327 }
 328
 329 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 330                                 enum cpuhp_state target)
 331 {
 332         enum cpuhp_state prev_state = st->state;
 333         int ret = 0;
 334
 335         for (; st->state > target; st->state--) {
 336                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL);
 337                 if (ret) {
 338                         st->target = prev_state;
 339                         undo_cpu_down(cpu, st);
 340                         break;
 341                 }
 342         }
 343         return ret;
 344 }
 345
 346 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
 347 {
 348         for (st->state--; st->state > st->target; st->state--) {
 349                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 350
 351                 if (!step->skip_onerr)
 352                         cpuhp_invoke_callback(cpu, st->state, false, NULL);
 353         }
 354 }
 355
 356 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 357                               enum cpuhp_state target)
 358 {
 359         enum cpuhp_state prev_state = st->state;
 360         int ret = 0;
 361
 362         while (st->state < target) {
 363                 st->state++;
 364                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL);
 365                 if (ret) {
 366                         st->target = prev_state;
 367                         undo_cpu_up(cpu, st);
 368                         break;
 369                 }
 370         }
 371         return ret;
 372 }
 373
 374 /*
 375  * The cpu hotplug threads manage the bringup and teardown of the cpus
 376  */
 377 static void cpuhp_create(unsigned int cpu)
 378 {
 379         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 380
 381         init_completion(&st->done);
 382 }
 383
 384 static int cpuhp_should_run(unsigned int cpu)
 385 {
 386         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 387
 388         return st->should_run;
 389 }
 390
 391 /* Execute the teardown callbacks. Used to be CPU_DOWN_PREPARE */
 392 static int cpuhp_ap_offline(unsigned int cpu, struct cpuhp_cpu_state *st)
 393 {
 394         enum cpuhp_state target = max((int)st->target, CPUHP_TEARDOWN_CPU);
 395
 396         return cpuhp_down_callbacks(cpu, st, target);
 397 }
 398
 399 /* Execute the online startup callbacks. Used to be CPU_ONLINE */
 400 static int cpuhp_ap_online(unsigned int cpu, struct cpuhp_cpu_state *st)
 401 {
 402         return cpuhp_up_callbacks(cpu, st, st->target);
 403 }
 404
 405 /*
 406  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 407  * callbacks when a state gets [un]installed at runtime.
 408  */
 409 static void cpuhp_thread_fun(unsigned int cpu)
 410 {
 411         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 412         int ret = 0;
 413
 414         /*
 415          * Paired with the mb() in cpuhp_kick_ap_work and
 416          * cpuhp_invoke_ap_callback, so the work set is consistent visible.
 417          */
 418         smp_mb();
 419         if (!st->should_run)
 420                 return;
 421
 422         st->should_run = false;
 423
 424         lock_map_acquire(&cpuhp_state_lock_map);
 425         /* Single callback invocation for [un]install ? */
 426         if (st->single) {
 427                 if (st->cb_state < CPUHP_AP_ONLINE) {
 428                         local_irq_disable();
 429                         ret = cpuhp_invoke_callback(cpu, st->cb_state,
 430                                                     st->bringup, st->node);
 431                         local_irq_enable();
 432                 } else {
 433                         ret = cpuhp_invoke_callback(cpu, st->cb_state,
 434                                                     st->bringup, st->node);
 435                 }
 436         } else if (st->rollback) {
 437                 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
 438
 439                 undo_cpu_down(cpu, st);
 440                 st->rollback = false;
 441         } else {
 442                 /* Cannot happen .... */
 443                 BUG_ON(st->state < CPUHP_AP_ONLINE_IDLE);
 444
 445                 /* Regular hotplug work */
 446                 if (st->state < st->target)
 447                         ret = cpuhp_ap_online(cpu, st);
 448                 else if (st->state > st->target)
 449                         ret = cpuhp_ap_offline(cpu, st);
 450         }
 451         lock_map_release(&cpuhp_state_lock_map);
 452         st->result = ret;
 453         complete(&st->done);
 454 }
 455
 456 /* Invoke a single callback on a remote cpu */
 457 static int
 458 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
 459                          struct hlist_node *node)
 460 {
 461         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 462
 463         if (!cpu_online(cpu))
 464                 return 0;
 465
 466         lock_map_acquire(&cpuhp_state_lock_map);
 467         lock_map_release(&cpuhp_state_lock_map);
 468
 469         /*
 470          * If we are up and running, use the hotplug thread. For early calls
 471          * we invoke the thread function directly.
 472          */
 473         if (!st->thread)
 474                 return cpuhp_invoke_callback(cpu, state, bringup, node);
 475
 476         st->cb_state = state;
 477         st->single = true;
 478         st->bringup = bringup;
 479         st->node = node;
 480
 481         /*
 482          * Make sure the above stores are visible before should_run becomes
 483          * true. Paired with the mb() above in cpuhp_thread_fun()
 484          */
 485         smp_mb();
 486         st->should_run = true;
 487         wake_up_process(st->thread);
 488         wait_for_completion(&st->done);
 489         return st->result;
 490 }
 491
 492 /* Regular hotplug invocation of the AP hotplug thread */
 493 static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st)
 494 {
 495         st->result = 0;
 496         st->single = false;
 497         /*
 498          * Make sure the above stores are visible before should_run becomes
 499          * true. Paired with the mb() above in cpuhp_thread_fun()
 500          */
 501         smp_mb();
 502         st->should_run = true;
 503         wake_up_process(st->thread);
 504 }
 505
 506 static int cpuhp_kick_ap_work(unsigned int cpu)
 507 {
 508         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 509         enum cpuhp_state state = st->state;
 510
 511         trace_cpuhp_enter(cpu, st->target, state, cpuhp_kick_ap_work);
 512         lock_map_acquire(&cpuhp_state_lock_map);
 513         lock_map_release(&cpuhp_state_lock_map);
 514         __cpuhp_kick_ap_work(st);
 515         wait_for_completion(&st->done);
 516         trace_cpuhp_exit(cpu, st->state, state, st->result);
 517         return st->result;
 518 }
 519
 520 static struct smp_hotplug_thread cpuhp_threads = {
 521         .store                  = &cpuhp_state.thread,
 522         .create                 = &cpuhp_create,
 523         .thread_should_run      = cpuhp_should_run,
 524         .thread_fn              = cpuhp_thread_fun,
 525         .thread_comm            = "cpuhp/%u",
 526         .selfparking            = true,
 527 };
 528
 529 void __init cpuhp_threads_init(void)
 530 {
 531         BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
 532         kthread_unpark(this_cpu_read(cpuhp_state.thread));
 533 }
 534
 535 #ifdef CONFIG_HOTPLUG_CPU
 536 /**
 537  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 538  * @cpu: a CPU id
 539  *
 540  * This function walks all processes, finds a valid mm struct for each one and
 541  * then clears a corresponding bit in mm's cpumask.  While this all sounds
 542  * trivial, there are various non-obvious corner cases, which this function
 543  * tries to solve in a safe manner.
 544  *
 545  * Also note that the function uses a somewhat relaxed locking scheme, so it may
 546  * be called only for an already offlined CPU.
 547  */
 548 void clear_tasks_mm_cpumask(int cpu)
 549 {
 550         struct task_struct *p;
 551
 552         /*
 553          * This function is called after the cpu is taken down and marked
 554          * offline, so its not like new tasks will ever get this cpu set in
 555          * their mm mask. -- Peter Zijlstra
 556          * Thus, we may use rcu_read_lock() here, instead of grabbing
 557          * full-fledged tasklist_lock.
 558          */
 559         WARN_ON(cpu_online(cpu));
 560         rcu_read_lock();
 561         for_each_process(p) {
 562                 struct task_struct *t;
 563
 564                 /*
 565                  * Main thread might exit, but other threads may still have
 566                  * a valid mm. Find one.
 567                  */
 568                 t = find_lock_task_mm(p);
 569                 if (!t)
 570                         continue;
 571                 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
 572                 task_unlock(t);
 573         }
 574         rcu_read_unlock();
 575 }
 576
 577 /* Take this CPU down. */
 578 static int take_cpu_down(void *_param)
 579 {
 580         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 581         enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
 582         int err, cpu = smp_processor_id();
 583
 584         /* Ensure this CPU doesn't handle any more interrupts. */
 585         err = __cpu_disable();
 586         if (err < 0)
 587                 return err;
 588
 589         /*
 590          * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
 591          * do this step again.
 592          */
 593         WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
 594         st->state--;
 595         /* Invoke the former CPU_DYING callbacks */
 596         for (; st->state > target; st->state--)
 597                 cpuhp_invoke_callback(cpu, st->state, false, NULL);
 598
 599         /* Give up timekeeping duties */
 600         tick_handover_do_timer();
 601         /* Park the stopper thread */
 602         stop_machine_park(cpu);
 603         return 0;
 604 }
 605
 606 static int takedown_cpu(unsigned int cpu)
 607 {
 608         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 609         int err;
 610
 611         /* Park the smpboot threads */
 612         kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 613         smpboot_park_threads(cpu);
 614
 615         /*
 616          * Prevent irq alloc/free while the dying cpu reorganizes the
 617          * interrupt affinities.
 618          */
 619         irq_lock_sparse();
 620
 621         /*
 622          * So now all preempt/rcu users must observe !cpu_active().
 623          */
 624         err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
 625         if (err) {
 626                 /* CPU refused to die */
 627                 irq_unlock_sparse();
 628                 /* Unpark the hotplug thread so we can rollback there */
 629                 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 630                 return err;
 631         }
 632         BUG_ON(cpu_online(cpu));
 633
 634         /*
 635          * The CPUHP_AP_SCHED_MIGRATE_DYING callback will have removed all
 636          * runnable tasks from the cpu, there's only the idle task left now
 637          * that the migration thread is done doing the stop_machine thing.
 638          *
 639          * Wait for the stop thread to go away.
 640          */
 641         wait_for_completion(&st->done);
 642         BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
 643
 644         /* Interrupts are moved away from the dying cpu, reenable alloc/free */
 645         irq_unlock_sparse();
 646
 647         hotplug_cpu__broadcast_tick_pull(cpu);
 648         /* This actually kills the CPU. */
 649         __cpu_die(cpu);
 650
 651         tick_cleanup_dead_cpu(cpu);
 652         return 0;
 653 }
 654
 655 static void cpuhp_complete_idle_dead(void *arg)
 656 {
 657         struct cpuhp_cpu_state *st = arg;
 658
 659         complete(&st->done);
 660 }
 661
 662 void cpuhp_report_idle_dead(void)
 663 {
 664         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 665
 666         BUG_ON(st->state != CPUHP_AP_OFFLINE);
 667         rcu_report_dead(smp_processor_id());
 668         st->state = CPUHP_AP_IDLE_DEAD;
 669         /*
 670          * We cannot call complete after rcu_report_dead() so we delegate it
 671          * to an online cpu.
 672          */
 673         smp_call_function_single(cpumask_first(cpu_online_mask),
 674                                  cpuhp_complete_idle_dead, st, 0);
 675 }
 676
 677 #else
 678 #define takedown_cpu            NULL
 679 #endif
 680
 681 #ifdef CONFIG_HOTPLUG_CPU
 682
 683 /* Requires cpu_add_remove_lock to be held */
 684 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 685                            enum cpuhp_state target)
 686 {
 687         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 688         int prev_state, ret = 0;
 689
 690         if (num_online_cpus() == 1)
 691                 return -EBUSY;
 692
 693         if (!cpu_present(cpu))
 694                 return -EINVAL;
 695
 696         cpus_write_lock();
 697
 698         cpuhp_tasks_frozen = tasks_frozen;
 699
 700         prev_state = st->state;
 701         st->target = target;
 702         /*
 703          * If the current CPU state is in the range of the AP hotplug thread,
 704          * then we need to kick the thread.
 705          */
 706         if (st->state > CPUHP_TEARDOWN_CPU) {
 707                 ret = cpuhp_kick_ap_work(cpu);
 708                 /*
 709                  * The AP side has done the error rollback already. Just
 710                  * return the error code..
 711                  */
 712                 if (ret)
 713                         goto out;
 714
 715                 /*
 716                  * We might have stopped still in the range of the AP hotplug
 717                  * thread. Nothing to do anymore.
 718                  */
 719                 if (st->state > CPUHP_TEARDOWN_CPU)
 720                         goto out;
 721         }
 722         /*
 723          * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
 724          * to do the further cleanups.
 725          */
 726         ret = cpuhp_down_callbacks(cpu, st, target);
 727         if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) {
 728                 st->target = prev_state;
 729                 st->rollback = true;
 730                 cpuhp_kick_ap_work(cpu);
 731         }
 732
 733 out:
 734         cpus_write_unlock();
 735         return ret;
 736 }
 737
 738 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
 739 {
 740         int err;
 741
 742         cpu_maps_update_begin();
 743
 744         if (cpu_hotplug_disabled) {
 745                 err = -EBUSY;
 746                 goto out;
 747         }
 748
 749         err = _cpu_down(cpu, 0, target);
 750
 751 out:
 752         cpu_maps_update_done();
 753         return err;
 754 }
 755 int cpu_down(unsigned int cpu)
 756 {
 757         return do_cpu_down(cpu, CPUHP_OFFLINE);
 758 }
 759 EXPORT_SYMBOL(cpu_down);
 760 #endif /*CONFIG_HOTPLUG_CPU*/
 761
 762 /**
 763  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
 764  * @cpu: cpu that just started
 765  *
 766  * It must be called by the arch code on the new cpu, before the new cpu
 767  * enables interrupts and before the "boot" cpu returns from __cpu_up().
 768  */
 769 void notify_cpu_starting(unsigned int cpu)
 770 {
 771         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 772         enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
 773
 774         rcu_cpu_starting(cpu);  /* Enables RCU usage on this CPU. */
 775         while (st->state < target) {
 776                 st->state++;
 777                 cpuhp_invoke_callback(cpu, st->state, true, NULL);
 778         }
 779 }
 780
 781 /*
 782  * Called from the idle task. Wake up the controlling task which brings the
 783  * stopper and the hotplug thread of the upcoming CPU up and then delegates
 784  * the rest of the online bringup to the hotplug thread.
 785  */
 786 void cpuhp_online_idle(enum cpuhp_state state)
 787 {
 788         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 789
 790         /* Happens for the boot cpu */
 791         if (state != CPUHP_AP_ONLINE_IDLE)
 792                 return;
 793
 794         st->state = CPUHP_AP_ONLINE_IDLE;
 795         complete(&st->done);
 796 }
 797
 798 /* Requires cpu_add_remove_lock to be held */
 799 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 800 {
 801         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 802         struct task_struct *idle;
 803         int ret = 0;
 804
 805         cpus_write_lock();
 806
 807         if (!cpu_present(cpu)) {
 808                 ret = -EINVAL;
 809                 goto out;
 810         }
 811
 812         /*
 813          * The caller of do_cpu_up might have raced with another
 814          * caller. Ignore it for now.
 815          */
 816         if (st->state >= target)
 817                 goto out;
 818
 819         if (st->state == CPUHP_OFFLINE) {
 820                 /* Let it fail before we try to bring the cpu up */
 821                 idle = idle_thread_get(cpu);
 822                 if (IS_ERR(idle)) {
 823                         ret = PTR_ERR(idle);
 824                         goto out;
 825                 }
 826         }
 827
 828         cpuhp_tasks_frozen = tasks_frozen;
 829
 830         st->target = target;
 831         /*
 832          * If the current CPU state is in the range of the AP hotplug thread,
 833          * then we need to kick the thread once more.
 834          */
 835         if (st->state > CPUHP_BRINGUP_CPU) {
 836                 ret = cpuhp_kick_ap_work(cpu);
 837                 /*
 838                  * The AP side has done the error rollback already. Just
 839                  * return the error code..
 840                  */
 841                 if (ret)
 842                         goto out;
 843         }
 844
 845         /*
 846          * Try to reach the target state. We max out on the BP at
 847          * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
 848          * responsible for bringing it up to the target state.
 849          */
 850         target = min((int)target, CPUHP_BRINGUP_CPU);
 851         ret = cpuhp_up_callbacks(cpu, st, target);
 852 out:
 853         cpus_write_unlock();
 854         return ret;
 855 }
 856
 857 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
 858 {
 859         int err = 0;
 860
 861         if (!cpu_possible(cpu)) {
 862                 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
 863                        cpu);
 864 #if defined(CONFIG_IA64)
 865                 pr_err("please check additional_cpus= boot parameter\n");
 866 #endif
 867                 return -EINVAL;
 868         }
 869
 870         err = try_online_node(cpu_to_node(cpu));
 871         if (err)
 872                 return err;
 873
 874         cpu_maps_update_begin();
 875
 876         if (cpu_hotplug_disabled) {
 877                 err = -EBUSY;
 878                 goto out;
 879         }
 880
 881         err = _cpu_up(cpu, 0, target);
 882 out:
 883         cpu_maps_update_done();
 884         return err;
 885 }
 886
 887 int cpu_up(unsigned int cpu)
 888 {
 889         return do_cpu_up(cpu, CPUHP_ONLINE);
 890 }
 891 EXPORT_SYMBOL_GPL(cpu_up);
 892
 893 #ifdef CONFIG_PM_SLEEP_SMP
 894 static cpumask_var_t frozen_cpus;
 895
 896 int freeze_secondary_cpus(int primary)
 897 {
 898         int cpu, error = 0;
 899
 900         cpu_maps_update_begin();
 901         if (!cpu_online(primary))
 902                 primary = cpumask_first(cpu_online_mask);
 903         /*
 904          * We take down all of the non-boot CPUs in one shot to avoid races
 905          * with the userspace trying to use the CPU hotplug at the same time
 906          */
 907         cpumask_clear(frozen_cpus);
 908
 909         pr_info("Disabling non-boot CPUs ...\n");
 910         for_each_online_cpu(cpu) {
 911                 if (cpu == primary)
 912                         continue;
 913                 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
 914                 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
 915                 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
 916                 if (!error)
 917                         cpumask_set_cpu(cpu, frozen_cpus);
 918                 else {
 919                         pr_err("Error taking CPU%d down: %d\n", cpu, error);
 920                         break;
 921                 }
 922         }
 923
 924         if (!error)
 925                 BUG_ON(num_online_cpus() > 1);
 926         else
 927                 pr_err("Non-boot CPUs are not disabled\n");
 928
 929         /*
 930          * Make sure the CPUs won't be enabled by someone else. We need to do
 931          * this even in case of failure as all disable_nonboot_cpus() users are
 932          * supposed to do enable_nonboot_cpus() on the failure path.
 933          */
 934         cpu_hotplug_disabled++;
 935
 936         cpu_maps_update_done();
 937         return error;
 938 }
 939
 940 void __weak arch_enable_nonboot_cpus_begin(void)
 941 {
 942 }
 943
 944 void __weak arch_enable_nonboot_cpus_end(void)
 945 {
 946 }
 947
 948 void enable_nonboot_cpus(void)
 949 {
 950         int cpu, error;
 951
 952         /* Allow everyone to use the CPU hotplug again */
 953         cpu_maps_update_begin();
 954         __cpu_hotplug_enable();
 955         if (cpumask_empty(frozen_cpus))
 956                 goto out;
 957
 958         pr_info("Enabling non-boot CPUs ...\n");
 959
 960         arch_enable_nonboot_cpus_begin();
 961
 962         for_each_cpu(cpu, frozen_cpus) {
 963                 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
 964                 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
 965                 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
 966                 if (!error) {
 967                         pr_info("CPU%d is up\n", cpu);
 968                         continue;
 969                 }
 970                 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
 971         }
 972
 973         arch_enable_nonboot_cpus_end();
 974
 975         cpumask_clear(frozen_cpus);
 976 out:
 977         cpu_maps_update_done();
 978 }
 979
 980 static int __init alloc_frozen_cpus(void)
 981 {
 982         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
 983                 return -ENOMEM;
 984         return 0;
 985 }
 986 core_initcall(alloc_frozen_cpus);
 987
 988 /*
 989  * When callbacks for CPU hotplug notifications are being executed, we must
 990  * ensure that the state of the system with respect to the tasks being frozen
 991  * or not, as reported by the notification, remains unchanged *throughout the
 992  * duration* of the execution of the callbacks.
 993  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
 994  *
 995  * This synchronization is implemented by mutually excluding regular CPU
 996  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
 997  * Hibernate notifications.
 998  */
 999 static int
1000 cpu_hotplug_pm_callback(struct notifier_block *nb,
1001                         unsigned long action, void *ptr)
1002 {
1003         switch (action) {
1004
1005         case PM_SUSPEND_PREPARE:
1006         case PM_HIBERNATION_PREPARE:
1007                 cpu_hotplug_disable();
1008                 break;
1009
1010         case PM_POST_SUSPEND:
1011         case PM_POST_HIBERNATION:
1012                 cpu_hotplug_enable();
1013                 break;
1014
1015         default:
1016                 return NOTIFY_DONE;
1017         }
1018
1019         return NOTIFY_OK;
1020 }
1021
1022
1023 static int __init cpu_hotplug_pm_sync_init(void)
1024 {
1025         /*
1026          * cpu_hotplug_pm_callback has higher priority than x86
1027          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1028          * to disable cpu hotplug to avoid cpu hotplug race.
1029          */
1030         pm_notifier(cpu_hotplug_pm_callback, 0);
1031         return 0;
1032 }
1033 core_initcall(cpu_hotplug_pm_sync_init);
1034
1035 #endif /* CONFIG_PM_SLEEP_SMP */
1036
1037 int __boot_cpu_id;
1038
1039 #endif /* CONFIG_SMP */
1040
1041 /* Boot processor state steps */
1042 static struct cpuhp_step cpuhp_bp_states[] = {
1043         [CPUHP_OFFLINE] = {
1044                 .name                   = "offline",
1045                 .startup.single         = NULL,
1046                 .teardown.single        = NULL,
1047         },
1048 #ifdef CONFIG_SMP
1049         [CPUHP_CREATE_THREADS]= {
1050                 .name                   = "threads:prepare",
1051                 .startup.single         = smpboot_create_threads,
1052                 .teardown.single        = NULL,
1053                 .cant_stop              = true,
1054         },
1055         [CPUHP_PERF_PREPARE] = {
1056                 .name                   = "perf:prepare",
1057                 .startup.single         = perf_event_init_cpu,
1058                 .teardown.single        = perf_event_exit_cpu,
1059         },
1060         [CPUHP_WORKQUEUE_PREP] = {
1061                 .name                   = "workqueue:prepare",
1062                 .startup.single         = workqueue_prepare_cpu,
1063                 .teardown.single        = NULL,
1064         },
1065         [CPUHP_HRTIMERS_PREPARE] = {
1066                 .name                   = "hrtimers:prepare",
1067                 .startup.single         = hrtimers_prepare_cpu,
1068                 .teardown.single        = hrtimers_dead_cpu,
1069         },
1070         [CPUHP_SMPCFD_PREPARE] = {
1071                 .name                   = "smpcfd:prepare",
1072                 .startup.single         = smpcfd_prepare_cpu,
1073                 .teardown.single        = smpcfd_dead_cpu,
1074         },
1075         [CPUHP_RELAY_PREPARE] = {
1076                 .name                   = "relay:prepare",
1077                 .startup.single         = relay_prepare_cpu,
1078                 .teardown.single        = NULL,
1079         },
1080         [CPUHP_SLAB_PREPARE] = {
1081                 .name                   = "slab:prepare",
1082                 .startup.single         = slab_prepare_cpu,
1083                 .teardown.single        = slab_dead_cpu,
1084         },
1085         [CPUHP_RCUTREE_PREP] = {
1086                 .name                   = "RCU/tree:prepare",
1087                 .startup.single         = rcutree_prepare_cpu,
1088                 .teardown.single        = rcutree_dead_cpu,
1089         },
1090         /*
1091          * On the tear-down path, timers_dead_cpu() must be invoked
1092          * before blk_mq_queue_reinit_notify() from notify_dead(),
1093          * otherwise a RCU stall occurs.
1094          */
1095         [CPUHP_TIMERS_DEAD] = {
1096                 .name                   = "timers:dead",
1097                 .startup.single         = NULL,
1098                 .teardown.single        = timers_dead_cpu,
1099         },
1100         /* Kicks the plugged cpu into life */
1101         [CPUHP_BRINGUP_CPU] = {
1102                 .name                   = "cpu:bringup",
1103                 .startup.single         = bringup_cpu,
1104                 .teardown.single        = NULL,
1105                 .cant_stop              = true,
1106         },
1107         [CPUHP_AP_SMPCFD_DYING] = {
1108                 .name                   = "smpcfd:dying",
1109                 .startup.single         = NULL,
1110                 .teardown.single        = smpcfd_dying_cpu,
1111         },
1112         /*
1113          * Handled on controll processor until the plugged processor manages
1114          * this itself.
1115          */
1116         [CPUHP_TEARDOWN_CPU] = {
1117                 .name                   = "cpu:teardown",
1118                 .startup.single         = NULL,
1119                 .teardown.single        = takedown_cpu,
1120                 .cant_stop              = true,
1121         },
1122 #else
1123         [CPUHP_BRINGUP_CPU] = { },
1124 #endif
1125 };
1126
1127 /* Application processor state steps */
1128 static struct cpuhp_step cpuhp_ap_states[] = {
1129 #ifdef CONFIG_SMP
1130         /* Final state before CPU kills itself */
1131         [CPUHP_AP_IDLE_DEAD] = {
1132                 .name                   = "idle:dead",
1133         },
1134         /*
1135          * Last state before CPU enters the idle loop to die. Transient state
1136          * for synchronization.
1137          */
1138         [CPUHP_AP_OFFLINE] = {
1139                 .name                   = "ap:offline",
1140                 .cant_stop              = true,
1141         },
1142         /* First state is scheduler control. Interrupts are disabled */
1143         [CPUHP_AP_SCHED_STARTING] = {
1144                 .name                   = "sched:starting",
1145                 .startup.single         = sched_cpu_starting,
1146                 .teardown.single        = sched_cpu_dying,
1147         },
1148         [CPUHP_AP_RCUTREE_DYING] = {
1149                 .name                   = "RCU/tree:dying",
1150                 .startup.single         = NULL,
1151                 .teardown.single        = rcutree_dying_cpu,
1152         },
1153         /* Entry state on starting. Interrupts enabled from here on. Transient
1154          * state for synchronsization */
1155         [CPUHP_AP_ONLINE] = {
1156                 .name                   = "ap:online",
1157         },
1158         /* Handle smpboot threads park/unpark */
1159         [CPUHP_AP_SMPBOOT_THREADS] = {
1160                 .name                   = "smpboot/threads:online",
1161                 .startup.single         = smpboot_unpark_threads,
1162                 .teardown.single        = NULL,
1163         },
1164         [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1165                 .name                   = "irq/affinity:online",
1166                 .startup.single         = irq_affinity_online_cpu,
1167                 .teardown.single        = NULL,
1168         },
1169         [CPUHP_AP_PERF_ONLINE] = {
1170                 .name                   = "perf:online",
1171                 .startup.single         = perf_event_init_cpu,
1172                 .teardown.single        = perf_event_exit_cpu,
1173         },
1174         [CPUHP_AP_WORKQUEUE_ONLINE] = {
1175                 .name                   = "workqueue:online",
1176                 .startup.single         = workqueue_online_cpu,
1177                 .teardown.single        = workqueue_offline_cpu,
1178         },
1179         [CPUHP_AP_RCUTREE_ONLINE] = {
1180                 .name                   = "RCU/tree:online",
1181                 .startup.single         = rcutree_online_cpu,
1182                 .teardown.single        = rcutree_offline_cpu,
1183         },
1184 #endif
1185         /*
1186          * The dynamically registered state space is here
1187          */
1188
1189 #ifdef CONFIG_SMP
1190         /* Last state is scheduler control setting the cpu active */
1191         [CPUHP_AP_ACTIVE] = {
1192                 .name                   = "sched:active",
1193                 .startup.single         = sched_cpu_activate,
1194                 .teardown.single        = sched_cpu_deactivate,
1195         },
1196 #endif
1197
1198         /* CPU is fully up and running. */
1199         [CPUHP_ONLINE] = {
1200                 .name                   = "online",
1201                 .startup.single         = NULL,
1202                 .teardown.single        = NULL,
1203         },
1204 };
1205
1206 /* Sanity check for callbacks */
1207 static int cpuhp_cb_check(enum cpuhp_state state)
1208 {
1209         if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1210                 return -EINVAL;
1211         return 0;
1212 }
1213
1214 /*
1215  * Returns a free for dynamic slot assignment of the Online state. The states
1216  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1217  * by having no name assigned.
1218  */
1219 static int cpuhp_reserve_state(enum cpuhp_state state)
1220 {
1221         enum cpuhp_state i, end;
1222         struct cpuhp_step *step;
1223
1224         switch (state) {
1225         case CPUHP_AP_ONLINE_DYN:
1226                 step = cpuhp_ap_states + CPUHP_AP_ONLINE_DYN;
1227                 end = CPUHP_AP_ONLINE_DYN_END;
1228                 break;
1229         case CPUHP_BP_PREPARE_DYN:
1230                 step = cpuhp_bp_states + CPUHP_BP_PREPARE_DYN;
1231                 end = CPUHP_BP_PREPARE_DYN_END;
1232                 break;
1233         default:
1234                 return -EINVAL;
1235         }
1236
1237         for (i = state; i <= end; i++, step++) {
1238                 if (!step->name)
1239                         return i;
1240         }
1241         WARN(1, "No more dynamic states available for CPU hotplug\n");
1242         return -ENOSPC;
1243 }
1244
1245 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1246                                  int (*startup)(unsigned int cpu),
1247                                  int (*teardown)(unsigned int cpu),
1248                                  bool multi_instance)
1249 {
1250         /* (Un)Install the callbacks for further cpu hotplug operations */
1251         struct cpuhp_step *sp;
1252         int ret = 0;
1253
1254         if (state == CPUHP_AP_ONLINE_DYN || state == CPUHP_BP_PREPARE_DYN) {
1255                 ret = cpuhp_reserve_state(state);
1256                 if (ret < 0)
1257                         return ret;
1258                 state = ret;
1259         }
1260         sp = cpuhp_get_step(state);
1261         if (name && sp->name)
1262                 return -EBUSY;
1263
1264         sp->startup.single = startup;
1265         sp->teardown.single = teardown;
1266         sp->name = name;
1267         sp->multi_instance = multi_instance;
1268         INIT_HLIST_HEAD(&sp->list);
1269         return ret;
1270 }
1271
1272 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1273 {
1274         return cpuhp_get_step(state)->teardown.single;
1275 }
1276
1277 /*
1278  * Call the startup/teardown function for a step either on the AP or
1279  * on the current CPU.
1280  */
1281 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1282                             struct hlist_node *node)
1283 {
1284         struct cpuhp_step *sp = cpuhp_get_step(state);
1285         int ret;
1286
1287         if ((bringup && !sp->startup.single) ||
1288             (!bringup && !sp->teardown.single))
1289                 return 0;
1290         /*
1291          * The non AP bound callbacks can fail on bringup. On teardown
1292          * e.g. module removal we crash for now.
1293          */
1294 #ifdef CONFIG_SMP
1295         if (cpuhp_is_ap_state(state))
1296                 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1297         else
1298                 ret = cpuhp_invoke_callback(cpu, state, bringup, node);
1299 #else
1300         ret = cpuhp_invoke_callback(cpu, state, bringup, node);
1301 #endif
1302         BUG_ON(ret && !bringup);
1303         return ret;
1304 }
1305
1306 /*
1307  * Called from __cpuhp_setup_state on a recoverable failure.
1308  *
1309  * Note: The teardown callbacks for rollback are not allowed to fail!
1310  */
1311 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1312                                    struct hlist_node *node)
1313 {
1314         int cpu;
1315
1316         /* Roll back the already executed steps on the other cpus */
1317         for_each_present_cpu(cpu) {
1318                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1319                 int cpustate = st->state;
1320
1321                 if (cpu >= failedcpu)
1322                         break;
1323
1324                 /* Did we invoke the startup call on that cpu ? */
1325                 if (cpustate >= state)
1326                         cpuhp_issue_call(cpu, state, false, node);
1327         }
1328 }
1329
1330 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1331                                           struct hlist_node *node,
1332                                           bool invoke)
1333 {
1334         struct cpuhp_step *sp;
1335         int cpu;
1336         int ret;
1337
1338         lockdep_assert_cpus_held();
1339
1340         sp = cpuhp_get_step(state);
1341         if (sp->multi_instance == false)
1342                 return -EINVAL;
1343
1344         mutex_lock(&cpuhp_state_mutex);
1345
1346         if (!invoke || !sp->startup.multi)
1347                 goto add_node;
1348
1349         /*
1350          * Try to call the startup callback for each present cpu
1351          * depending on the hotplug state of the cpu.
1352          */
1353         for_each_present_cpu(cpu) {
1354                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1355                 int cpustate = st->state;
1356
1357                 if (cpustate < state)
1358                         continue;
1359
1360                 ret = cpuhp_issue_call(cpu, state, true, node);
1361                 if (ret) {
1362                         if (sp->teardown.multi)
1363                                 cpuhp_rollback_install(cpu, state, node);
1364                         goto unlock;
1365                 }
1366         }
1367 add_node:
1368         ret = 0;
1369         hlist_add_head(node, &sp->list);
1370 unlock:
1371         mutex_unlock(&cpuhp_state_mutex);
1372         return ret;
1373 }
1374
1375 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1376                                bool invoke)
1377 {
1378         int ret;
1379
1380         cpus_read_lock();
1381         ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1382         cpus_read_unlock();
1383         return ret;
1384 }
1385 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1386
1387 /**
1388  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1389  * @state:              The state to setup
1390  * @invoke:             If true, the startup function is invoked for cpus where
1391  *                      cpu state >= @state
1392  * @startup:            startup callback function
1393  * @teardown:           teardown callback function
1394  * @multi_instance:     State is set up for multiple instances which get
1395  *                      added afterwards.
1396  *
1397  * The caller needs to hold cpus read locked while calling this function.
1398  * Returns:
1399  *   On success:
1400  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
1401  *      0 for all other states
1402  *   On failure: proper (negative) error code
1403  */
1404 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1405                                    const char *name, bool invoke,
1406                                    int (*startup)(unsigned int cpu),
1407                                    int (*teardown)(unsigned int cpu),
1408                                    bool multi_instance)
1409 {
1410         int cpu, ret = 0;
1411         bool dynstate;
1412
1413         lockdep_assert_cpus_held();
1414
1415         if (cpuhp_cb_check(state) || !name)
1416                 return -EINVAL;
1417
1418         mutex_lock(&cpuhp_state_mutex);
1419
1420         ret = cpuhp_store_callbacks(state, name, startup, teardown,
1421                                     multi_instance);
1422
1423         dynstate = state == CPUHP_AP_ONLINE_DYN;
1424         if (ret > 0 && dynstate) {
1425                 state = ret;
1426                 ret = 0;
1427         }
1428
1429         if (ret || !invoke || !startup)
1430                 goto out;
1431
1432         /*
1433          * Try to call the startup callback for each present cpu
1434          * depending on the hotplug state of the cpu.
1435          */
1436         for_each_present_cpu(cpu) {
1437                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1438                 int cpustate = st->state;
1439
1440                 if (cpustate < state)
1441                         continue;
1442
1443                 ret = cpuhp_issue_call(cpu, state, true, NULL);
1444                 if (ret) {
1445                         if (teardown)
1446                                 cpuhp_rollback_install(cpu, state, NULL);
1447                         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1448                         goto out;
1449                 }
1450         }
1451 out:
1452         mutex_unlock(&cpuhp_state_mutex);
1453         /*
1454          * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1455          * dynamically allocated state in case of success.
1456          */
1457         if (!ret && dynstate)
1458                 return state;
1459         return ret;
1460 }
1461 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1462
1463 int __cpuhp_setup_state(enum cpuhp_state state,
1464                         const char *name, bool invoke,
1465                         int (*startup)(unsigned int cpu),
1466                         int (*teardown)(unsigned int cpu),
1467                         bool multi_instance)
1468 {
1469         int ret;
1470
1471         cpus_read_lock();
1472         ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1473                                              teardown, multi_instance);
1474         cpus_read_unlock();
1475         return ret;
1476 }
1477 EXPORT_SYMBOL(__cpuhp_setup_state);
1478
1479 int __cpuhp_state_remove_instance(enum cpuhp_state state,
1480                                   struct hlist_node *node, bool invoke)
1481 {
1482         struct cpuhp_step *sp = cpuhp_get_step(state);
1483         int cpu;
1484
1485         BUG_ON(cpuhp_cb_check(state));
1486
1487         if (!sp->multi_instance)
1488                 return -EINVAL;
1489
1490         cpus_read_lock();
1491         mutex_lock(&cpuhp_state_mutex);
1492
1493         if (!invoke || !cpuhp_get_teardown_cb(state))
1494                 goto remove;
1495         /*
1496          * Call the teardown callback for each present cpu depending
1497          * on the hotplug state of the cpu. This function is not
1498          * allowed to fail currently!
1499          */
1500         for_each_present_cpu(cpu) {
1501                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1502                 int cpustate = st->state;
1503
1504                 if (cpustate >= state)
1505                         cpuhp_issue_call(cpu, state, false, node);
1506         }
1507
1508 remove:
1509         hlist_del(node);
1510         mutex_unlock(&cpuhp_state_mutex);
1511         cpus_read_unlock();
1512
1513         return 0;
1514 }
1515 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1516
1517 /**
1518  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1519  * @state:      The state to remove
1520  * @invoke:     If true, the teardown function is invoked for cpus where
1521  *              cpu state >= @state
1522  *
1523  * The caller needs to hold cpus read locked while calling this function.
1524  * The teardown callback is currently not allowed to fail. Think
1525  * about module removal!
1526  */
1527 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1528 {
1529         struct cpuhp_step *sp = cpuhp_get_step(state);
1530         int cpu;
1531
1532         BUG_ON(cpuhp_cb_check(state));
1533
1534         lockdep_assert_cpus_held();
1535
1536         mutex_lock(&cpuhp_state_mutex);
1537         if (sp->multi_instance) {
1538                 WARN(!hlist_empty(&sp->list),
1539                      "Error: Removing state %d which has instances left.\n",
1540                      state);
1541                 goto remove;
1542         }
1543
1544         if (!invoke || !cpuhp_get_teardown_cb(state))
1545                 goto remove;
1546
1547         /*
1548          * Call the teardown callback for each present cpu depending
1549          * on the hotplug state of the cpu. This function is not
1550          * allowed to fail currently!
1551          */
1552         for_each_present_cpu(cpu) {
1553                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1554                 int cpustate = st->state;
1555
1556                 if (cpustate >= state)
1557                         cpuhp_issue_call(cpu, state, false, NULL);
1558         }
1559 remove:
1560         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1561         mutex_unlock(&cpuhp_state_mutex);
1562 }
1563 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1564
1565 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1566 {
1567         cpus_read_lock();
1568         __cpuhp_remove_state_cpuslocked(state, invoke);
1569         cpus_read_unlock();
1570 }
1571 EXPORT_SYMBOL(__cpuhp_remove_state);
1572
1573 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1574 static ssize_t show_cpuhp_state(struct device *dev,
1575                                 struct device_attribute *attr, char *buf)
1576 {
1577         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1578
1579         return sprintf(buf, "%d\n", st->state);
1580 }
1581 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1582
1583 static ssize_t write_cpuhp_target(struct device *dev,
1584                                   struct device_attribute *attr,
1585                                   const char *buf, size_t count)
1586 {
1587         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1588         struct cpuhp_step *sp;
1589         int target, ret;
1590
1591         ret = kstrtoint(buf, 10, &target);
1592         if (ret)
1593                 return ret;
1594
1595 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1596         if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1597                 return -EINVAL;
1598 #else
1599         if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1600                 return -EINVAL;
1601 #endif
1602
1603         ret = lock_device_hotplug_sysfs();
1604         if (ret)
1605                 return ret;
1606
1607         mutex_lock(&cpuhp_state_mutex);
1608         sp = cpuhp_get_step(target);
1609         ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1610         mutex_unlock(&cpuhp_state_mutex);
1611         if (ret)
1612                 goto out;
1613
1614         if (st->state < target)
1615                 ret = do_cpu_up(dev->id, target);
1616         else
1617                 ret = do_cpu_down(dev->id, target);
1618 out:
1619         unlock_device_hotplug();
1620         return ret ? ret : count;
1621 }
1622
1623 static ssize_t show_cpuhp_target(struct device *dev,
1624                                  struct device_attribute *attr, char *buf)
1625 {
1626         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1627
1628         return sprintf(buf, "%d\n", st->target);
1629 }
1630 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1631
1632 static struct attribute *cpuhp_cpu_attrs[] = {
1633         &dev_attr_state.attr,
1634         &dev_attr_target.attr,
1635         NULL
1636 };
1637
1638 static const struct attribute_group cpuhp_cpu_attr_group = {
1639         .attrs = cpuhp_cpu_attrs,
1640         .name = "hotplug",
1641         NULL
1642 };
1643
1644 static ssize_t show_cpuhp_states(struct device *dev,
1645                                  struct device_attribute *attr, char *buf)
1646 {
1647         ssize_t cur, res = 0;
1648         int i;
1649
1650         mutex_lock(&cpuhp_state_mutex);
1651         for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
1652                 struct cpuhp_step *sp = cpuhp_get_step(i);
1653
1654                 if (sp->name) {
1655                         cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1656                         buf += cur;
1657                         res += cur;
1658                 }
1659         }
1660         mutex_unlock(&cpuhp_state_mutex);
1661         return res;
1662 }
1663 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1664
1665 static struct attribute *cpuhp_cpu_root_attrs[] = {
1666         &dev_attr_states.attr,
1667         NULL
1668 };
1669
1670 static const struct attribute_group cpuhp_cpu_root_attr_group = {
1671         .attrs = cpuhp_cpu_root_attrs,
1672         .name = "hotplug",
1673         NULL
1674 };
1675
1676 static int __init cpuhp_sysfs_init(void)
1677 {
1678         int cpu, ret;
1679
1680         ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
1681                                  &cpuhp_cpu_root_attr_group);
1682         if (ret)
1683                 return ret;
1684
1685         for_each_possible_cpu(cpu) {
1686                 struct device *dev = get_cpu_device(cpu);
1687
1688                 if (!dev)
1689                         continue;
1690                 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
1691                 if (ret)
1692                         return ret;
1693         }
1694         return 0;
1695 }
1696 device_initcall(cpuhp_sysfs_init);
1697 #endif
1698
1699 /*
1700  * cpu_bit_bitmap[] is a special, "compressed" data structure that
1701  * represents all NR_CPUS bits binary values of 1<<nr.
1702  *
1703  * It is used by cpumask_of() to get a constant address to a CPU
1704  * mask value that has a single bit set only.
1705  */
1706
1707 /* cpu_bit_bitmap[0] is empty - so we can back into it */
1708 #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
1709 #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
1710 #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
1711 #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
1712
1713 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
1714
1715         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
1716         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
1717 #if BITS_PER_LONG > 32
1718         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
1719         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
1720 #endif
1721 };
1722 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
1723
1724 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
1725 EXPORT_SYMBOL(cpu_all_bits);
1726
1727 #ifdef CONFIG_INIT_ALL_POSSIBLE
1728 struct cpumask __cpu_possible_mask __read_mostly
1729         = {CPU_BITS_ALL};
1730 #else
1731 struct cpumask __cpu_possible_mask __read_mostly;
1732 #endif
1733 EXPORT_SYMBOL(__cpu_possible_mask);
1734
1735 struct cpumask __cpu_online_mask __read_mostly;
1736 EXPORT_SYMBOL(__cpu_online_mask);
1737
1738 struct cpumask __cpu_present_mask __read_mostly;
1739 EXPORT_SYMBOL(__cpu_present_mask);
1740
1741 struct cpumask __cpu_active_mask __read_mostly;
1742 EXPORT_SYMBOL(__cpu_active_mask);
1743
1744 void init_cpu_present(const struct cpumask *src)
1745 {
1746         cpumask_copy(&__cpu_present_mask, src);
1747 }
1748
1749 void init_cpu_possible(const struct cpumask *src)
1750 {
1751         cpumask_copy(&__cpu_possible_mask, src);
1752 }
1753
1754 void init_cpu_online(const struct cpumask *src)
1755 {
1756         cpumask_copy(&__cpu_online_mask, src);
1757 }
1758
1759 /*
1760  * Activate the first processor.
1761  */
1762 void __init boot_cpu_init(void)
1763 {
1764         int cpu = smp_processor_id();
1765
1766         /* Mark the boot cpu "present", "online" etc for SMP and UP case */
1767         set_cpu_online(cpu, true);
1768         set_cpu_active(cpu, true);
1769         set_cpu_present(cpu, true);
1770         set_cpu_possible(cpu, true);
1771
1772 #ifdef CONFIG_SMP
1773         __boot_cpu_id = cpu;
1774 #endif
1775 }
1776
1777 /*
1778  * Must be called _AFTER_ setting up the per_cpu areas
1779  */
1780 void __init boot_cpu_state_init(void)
1781 {
1782         per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE;
1783 }