1 /* Copyright 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
2 * GPL v2 and any later version.
6 #include <linux/kthread.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/stop_machine.h>
10 #include <linux/syscalls.h>
11 #include <asm/atomic.h>
12 #include <asm/semaphore.h>
13 #include <asm/uaccess.h>
15 /* Since we effect priority and affinity (both of which are visible
16 * to, and settable by outside processes) we do indirection via a
19 /* Thread to stop each CPU in user context. */
20 enum stopmachine_state {
23 STOPMACHINE_DISABLE_IRQ,
27 static enum stopmachine_state stopmachine_state;
28 static unsigned int stopmachine_num_threads;
29 static atomic_t stopmachine_thread_ack;
30 static DECLARE_MUTEX(stopmachine_mutex);
32 static int stopmachine(void *cpu)
34 int irqs_disabled = 0;
37 set_cpus_allowed(current, cpumask_of_cpu((int)(long)cpu));
39 /* Ack: we are alive */
40 smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */
41 atomic_inc(&stopmachine_thread_ack);
43 /* Simple state machine */
44 while (stopmachine_state != STOPMACHINE_EXIT) {
45 if (stopmachine_state == STOPMACHINE_DISABLE_IRQ
49 /* Ack: irqs disabled. */
50 smp_mb(); /* Must read state first. */
51 atomic_inc(&stopmachine_thread_ack);
52 } else if (stopmachine_state == STOPMACHINE_PREPARE
54 /* Everyone is in place, hold CPU. */
57 smp_mb(); /* Must read state first. */
58 atomic_inc(&stopmachine_thread_ack);
60 /* Yield in first stage: migration threads need to
61 * help our sisters onto their CPUs. */
62 if (!prepared && !irqs_disabled)
68 /* Ack: we are exiting. */
69 smp_mb(); /* Must read state first. */
70 atomic_inc(&stopmachine_thread_ack);
80 /* Change the thread state */
81 static void stopmachine_set_state(enum stopmachine_state state)
83 atomic_set(&stopmachine_thread_ack, 0);
85 stopmachine_state = state;
86 while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads)
90 static int stop_machine(void)
93 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
95 /* One high-prio thread per cpu. We'll do this one. */
96 sched_setscheduler(current, SCHED_FIFO, ¶m);
98 atomic_set(&stopmachine_thread_ack, 0);
99 stopmachine_num_threads = 0;
100 stopmachine_state = STOPMACHINE_WAIT;
102 for_each_online_cpu(i) {
103 if (i == raw_smp_processor_id())
105 ret = kernel_thread(stopmachine, (void *)(long)i,CLONE_KERNEL);
108 stopmachine_num_threads++;
111 /* Wait for them all to come to life. */
112 while (atomic_read(&stopmachine_thread_ack) != stopmachine_num_threads)
115 /* If some failed, kill them all. */
117 stopmachine_set_state(STOPMACHINE_EXIT);
121 /* Now they are all started, make them hold the CPUs, ready. */
123 stopmachine_set_state(STOPMACHINE_PREPARE);
125 /* Make them disable irqs. */
127 stopmachine_set_state(STOPMACHINE_DISABLE_IRQ);
132 static void restart_machine(void)
134 stopmachine_set_state(STOPMACHINE_EXIT);
136 preempt_enable_no_resched();
139 struct stop_machine_data
143 struct completion done;
146 static int do_stop(void *_smdata)
148 struct stop_machine_data *smdata = _smdata;
151 ret = stop_machine();
153 ret = smdata->fn(smdata->data);
157 /* We're done: you can kthread_stop us now */
158 complete(&smdata->done);
160 /* Wait for kthread_stop */
161 set_current_state(TASK_INTERRUPTIBLE);
162 while (!kthread_should_stop()) {
164 set_current_state(TASK_INTERRUPTIBLE);
166 __set_current_state(TASK_RUNNING);
170 struct task_struct *__stop_machine_run(int (*fn)(void *), void *data,
173 struct stop_machine_data smdata;
174 struct task_struct *p;
178 init_completion(&smdata.done);
180 down(&stopmachine_mutex);
182 /* If they don't care which CPU fn runs on, bind to any online one. */
184 cpu = raw_smp_processor_id();
186 p = kthread_create(do_stop, &smdata, "kstopmachine");
188 kthread_bind(p, cpu);
190 wait_for_completion(&smdata.done);
192 up(&stopmachine_mutex);
196 int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu)
198 struct task_struct *p;
201 /* No CPUs can come up or down during this. */
203 p = __stop_machine_run(fn, data, cpu);
205 ret = kthread_stop(p);
208 unlock_cpu_hotplug();
212 EXPORT_SYMBOL_GPL(stop_machine_run);