1 /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
2 * GPL v2 and any later version.
6 #include <linux/kthread.h>
7 #include <linux/module.h>
8 #include <linux/sched.h>
9 #include <linux/stop_machine.h>
10 #include <linux/syscalls.h>
11 #include <linux/interrupt.h>
13 #include <asm/atomic.h>
14 #include <asm/uaccess.h>
16 /* This controls the threads on each CPU. */
17 enum stopmachine_state {
18 /* Dummy starting state for thread. */
20 /* Awaiting everyone to be scheduled. */
22 /* Disable interrupts. */
23 STOPMACHINE_DISABLE_IRQ,
24 /* Run the function */
29 static enum stopmachine_state state;
31 struct stop_machine_data {
37 /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
38 static unsigned int num_threads;
39 static atomic_t thread_ack;
40 static struct completion finished;
41 static DEFINE_MUTEX(lock);
43 static void set_state(enum stopmachine_state newstate)
45 /* Reset ack counter. */
46 atomic_set(&thread_ack, num_threads);
51 /* Last one to ack a state moves to the next state. */
52 static void ack_state(void)
54 if (atomic_dec_and_test(&thread_ack)) {
55 /* If we're the last one to ack the EXIT, we're finished. */
56 if (state == STOPMACHINE_EXIT)
63 /* This is the actual thread which stops the CPU. It exits by itself rather
64 * than waiting for kthread_stop(), because it's easier for hotplug CPU. */
65 static int stop_cpu(struct stop_machine_data *smdata)
67 enum stopmachine_state curstate = STOPMACHINE_NONE;
68 int uninitialized_var(ret);
70 /* Simple state machine */
72 /* Chill out and ensure we re-read stopmachine_state. */
74 if (state != curstate) {
77 case STOPMACHINE_DISABLE_IRQ:
82 /* |= allows error detection if functions on
84 smdata->fnret |= smdata->fn(smdata->data);
91 } while (curstate != STOPMACHINE_EXIT);
97 /* Callback for CPUs which aren't supposed to do anything. */
98 static int chill(void *unused)
103 int __stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu)
106 struct stop_machine_data active, idle;
107 struct task_struct **threads;
115 /* If they don't care which cpu fn runs on, just pick one. */
117 cpu = any_online_cpu(cpu_online_map);
119 /* This could be too big for stack on large machines. */
120 threads = kcalloc(NR_CPUS, sizeof(threads[0]), GFP_KERNEL);
124 /* Set up initial state. */
126 init_completion(&finished);
127 num_threads = num_online_cpus();
128 set_state(STOPMACHINE_PREPARE);
130 for_each_online_cpu(i) {
131 struct stop_machine_data *smdata;
132 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
134 if (cpu == ALL_CPUS || i == cpu)
139 threads[i] = kthread_create((void *)stop_cpu, smdata, "kstop%u",
141 if (IS_ERR(threads[i])) {
142 err = PTR_ERR(threads[i]);
147 /* Place it onto correct cpu. */
148 kthread_bind(threads[i], i);
150 /* Make it highest prio. */
151 if (sched_setscheduler_nocheck(threads[i], SCHED_FIFO, ¶m))
155 /* We've created all the threads. Wake them all: hold this CPU so one
156 * doesn't hit this CPU until we're ready. */
158 for_each_online_cpu(i)
159 wake_up_process(threads[i]);
161 /* This will release the thread on our CPU. */
163 wait_for_completion(&finished);
171 for_each_online_cpu(i)
173 kthread_stop(threads[i]);
180 int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu)
184 /* No CPUs can come up or down during this. */
186 ret = __stop_machine_run(fn, data, cpu);
191 EXPORT_SYMBOL_GPL(stop_machine_run);