1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
/* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation.
* GPL v2 and any later version.
*/
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/stop_machine.h>
#include <linux/syscalls.h>
#include <linux/interrupt.h>
#include <asm/atomic.h>
#include <asm/uaccess.h>
/* This controls the threads on each CPU. */
enum stopmachine_state {
/* Dummy starting state for thread. */
STOPMACHINE_NONE,
/* Awaiting everyone to be scheduled. */
STOPMACHINE_PREPARE,
/* Disable interrupts. */
STOPMACHINE_DISABLE_IRQ,
/* Run the function */
STOPMACHINE_RUN,
/* Exit */
STOPMACHINE_EXIT,
};
static enum stopmachine_state state;
struct stop_machine_data {
int (*fn)(void *);
void *data;
int fnret;
};
/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
static unsigned int num_threads;
static atomic_t thread_ack;
static struct completion finished;
static DEFINE_MUTEX(lock);
static void set_state(enum stopmachine_state newstate)
{
/* Reset ack counter. */
atomic_set(&thread_ack, num_threads);
smp_wmb();
state = newstate;
}
/* Last one to ack a state moves to the next state. */
static void ack_state(void)
{
if (atomic_dec_and_test(&thread_ack)) {
/* If we're the last one to ack the EXIT, we're finished. */
if (state == STOPMACHINE_EXIT)
complete(&finished);
else
set_state(state + 1);
}
}
/* This is the actual thread which stops the CPU. It exits by itself rather
* than waiting for kthread_stop(), because it's easier for hotplug CPU. */
static int stop_cpu(struct stop_machine_data *smdata)
{
enum stopmachine_state curstate = STOPMACHINE_NONE;
int uninitialized_var(ret);
/* Simple state machine */
do {
/* Chill out and ensure we re-read stopmachine_state. */
cpu_relax();
if (state != curstate) {
curstate = state;
switch (curstate) {
case STOPMACHINE_DISABLE_IRQ:
local_irq_disable();
hard_irq_disable();
break;
case STOPMACHINE_RUN:
/* |= allows error detection if functions on
* multiple CPUs. */
smdata->fnret |= smdata->fn(smdata->data);
break;
default:
break;
}
ack_state();
}
} while (curstate != STOPMACHINE_EXIT);
local_irq_enable();
do_exit(0);
}
/* Callback for CPUs which aren't supposed to do anything. */
static int chill(void *unused)
{
return 0;
}
int __stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu)
{
int i, err;
struct stop_machine_data active, idle;
struct task_struct **threads;
active.fn = fn;
active.data = data;
active.fnret = 0;
idle.fn = chill;
idle.data = NULL;
/* If they don't care which cpu fn runs on, just pick one. */
if (cpu == NR_CPUS)
cpu = any_online_cpu(cpu_online_map);
/* This could be too big for stack on large machines. */
threads = kcalloc(NR_CPUS, sizeof(threads[0]), GFP_KERNEL);
if (!threads)
return -ENOMEM;
/* Set up initial state. */
mutex_lock(&lock);
init_completion(&finished);
num_threads = num_online_cpus();
set_state(STOPMACHINE_PREPARE);
for_each_online_cpu(i) {
struct stop_machine_data *smdata;
struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
if (cpu == ALL_CPUS || i == cpu)
smdata = &active;
else
smdata = &idle;
threads[i] = kthread_create((void *)stop_cpu, smdata, "kstop%u",
i);
if (IS_ERR(threads[i])) {
err = PTR_ERR(threads[i]);
threads[i] = NULL;
goto kill_threads;
}
/* Place it onto correct cpu. */
kthread_bind(threads[i], i);
/* Make it highest prio. */
if (sched_setscheduler_nocheck(threads[i], SCHED_FIFO, ¶m))
BUG();
}
/* We've created all the threads. Wake them all: hold this CPU so one
* doesn't hit this CPU until we're ready. */
cpu = get_cpu();
for_each_online_cpu(i)
wake_up_process(threads[i]);
/* This will release the thread on our CPU. */
put_cpu();
wait_for_completion(&finished);
mutex_unlock(&lock);
kfree(threads);
return active.fnret;
kill_threads:
for_each_online_cpu(i)
if (threads[i])
kthread_stop(threads[i]);
mutex_unlock(&lock);
kfree(threads);
return err;
}
int stop_machine_run(int (*fn)(void *), void *data, unsigned int cpu)
{
int ret;
/* No CPUs can come up or down during this. */
get_online_cpus();
ret = __stop_machine_run(fn, data, cpu);
put_online_cpus();
return ret;
}
EXPORT_SYMBOL_GPL(stop_machine_run);
|
