1 #include <linux/module.h>
2 #include <linux/spinlock.h>
3 #include <linux/list.h>
4 #include <asm/alternative.h>
5 #include <asm/sections.h>
9 # define DPRINTK(fmt, args...) printk(fmt, args)
11 # define DPRINTK(fmt, args...)
14 /* Use inline assembly to define this because the nops are defined
15 as inline assembly strings in the include files and we cannot
16 get them easily into strings. */
17 asm("\t.data\nintelnops: "
18 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
19 GENERIC_NOP7 GENERIC_NOP8);
20 asm("\t.data\nk8nops: "
21 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
23 asm("\t.data\nk7nops: "
24 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
27 extern unsigned char intelnops[], k8nops[], k7nops[];
28 static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
33 intelnops + 1 + 2 + 3,
34 intelnops + 1 + 2 + 3 + 4,
35 intelnops + 1 + 2 + 3 + 4 + 5,
36 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
37 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
39 static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
45 k8nops + 1 + 2 + 3 + 4,
46 k8nops + 1 + 2 + 3 + 4 + 5,
47 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
48 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
50 static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
56 k7nops + 1 + 2 + 3 + 4,
57 k7nops + 1 + 2 + 3 + 4 + 5,
58 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
59 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
63 unsigned char **noptable;
65 { X86_FEATURE_K8, k8_nops },
66 { X86_FEATURE_K7, k7_nops },
71 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
72 extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
73 extern u8 *__smp_locks[], *__smp_locks_end[];
75 extern u8 __smp_alt_begin[], __smp_alt_end[];
78 static unsigned char** find_nop_table(void)
80 unsigned char **noptable = intel_nops;
83 for (i = 0; noptypes[i].cpuid >= 0; i++) {
84 if (boot_cpu_has(noptypes[i].cpuid)) {
85 noptable = noptypes[i].noptable;
92 /* Replace instructions with better alternatives for this CPU type.
93 This runs before SMP is initialized to avoid SMP problems with
94 self modifying code. This implies that assymetric systems where
95 APs have less capabilities than the boot processor are not handled.
96 Tough. Make sure you disable such features by hand. */
98 void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
100 unsigned char **noptable = find_nop_table();
104 DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
105 for (a = start; a < end; a++) {
106 BUG_ON(a->replacementlen > a->instrlen);
107 if (!boot_cpu_has(a->cpuid))
109 memcpy(a->instr, a->replacement, a->replacementlen);
110 diff = a->instrlen - a->replacementlen;
111 /* Pad the rest with nops */
112 for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
116 memcpy(a->instr + i, noptable[k], k);
121 static void alternatives_smp_save(struct alt_instr *start, struct alt_instr *end)
125 DPRINTK("%s: alt table %p-%p\n", __FUNCTION__, start, end);
126 for (a = start; a < end; a++) {
127 memcpy(a->replacement + a->replacementlen,
133 static void alternatives_smp_apply(struct alt_instr *start, struct alt_instr *end)
137 for (a = start; a < end; a++) {
139 a->replacement + a->replacementlen,
144 static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
148 for (ptr = start; ptr < end; ptr++) {
153 **ptr = 0xf0; /* lock prefix */
157 static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
159 unsigned char **noptable = find_nop_table();
162 for (ptr = start; ptr < end; ptr++) {
167 **ptr = noptable[1][0];
171 struct smp_alt_module {
172 /* what is this ??? */
176 /* ptrs to lock prefixes */
180 /* .text segment, needed to avoid patching init code ;) */
184 struct list_head next;
186 static LIST_HEAD(smp_alt_modules);
187 static DEFINE_SPINLOCK(smp_alt);
189 static int smp_alt_once = 0;
190 static int __init bootonly(char *str)
195 __setup("smp-alt-boot", bootonly);
197 void alternatives_smp_module_add(struct module *mod, char *name,
198 void *locks, void *locks_end,
199 void *text, void *text_end)
201 struct smp_alt_module *smp;
205 if (boot_cpu_has(X86_FEATURE_UP))
206 alternatives_smp_unlock(locks, locks_end,
211 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
213 return; /* we'll run the (safe but slow) SMP code then ... */
218 smp->locks_end = locks_end;
220 smp->text_end = text_end;
221 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
222 __FUNCTION__, smp->locks, smp->locks_end,
223 smp->text, smp->text_end, smp->name);
225 spin_lock_irqsave(&smp_alt, flags);
226 list_add_tail(&smp->next, &smp_alt_modules);
227 if (boot_cpu_has(X86_FEATURE_UP))
228 alternatives_smp_unlock(smp->locks, smp->locks_end,
229 smp->text, smp->text_end);
230 spin_unlock_irqrestore(&smp_alt, flags);
233 void alternatives_smp_module_del(struct module *mod)
235 struct smp_alt_module *item;
241 spin_lock_irqsave(&smp_alt, flags);
242 list_for_each_entry(item, &smp_alt_modules, next) {
243 if (mod != item->mod)
245 list_del(&item->next);
246 spin_unlock_irqrestore(&smp_alt, flags);
247 DPRINTK("%s: %s\n", __FUNCTION__, item->name);
251 spin_unlock_irqrestore(&smp_alt, flags);
254 void alternatives_smp_switch(int smp)
256 struct smp_alt_module *mod;
261 BUG_ON(!smp && (num_online_cpus() > 1));
263 spin_lock_irqsave(&smp_alt, flags);
265 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
266 clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
267 clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
268 alternatives_smp_apply(__smp_alt_instructions,
269 __smp_alt_instructions_end);
270 list_for_each_entry(mod, &smp_alt_modules, next)
271 alternatives_smp_lock(mod->locks, mod->locks_end,
272 mod->text, mod->text_end);
274 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
275 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
276 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
277 apply_alternatives(__smp_alt_instructions,
278 __smp_alt_instructions_end);
279 list_for_each_entry(mod, &smp_alt_modules, next)
280 alternatives_smp_unlock(mod->locks, mod->locks_end,
281 mod->text, mod->text_end);
283 spin_unlock_irqrestore(&smp_alt, flags);
286 void __init alternative_instructions(void)
288 apply_alternatives(__alt_instructions, __alt_instructions_end);
290 /* switch to patch-once-at-boottime-only mode and free the
291 * tables in case we know the number of CPUs will never ever
293 #ifdef CONFIG_HOTPLUG_CPU
294 if (num_possible_cpus() < 2)
301 if (1 == num_possible_cpus()) {
302 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
303 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
304 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
305 apply_alternatives(__smp_alt_instructions,
306 __smp_alt_instructions_end);
307 alternatives_smp_unlock(__smp_locks, __smp_locks_end,
310 free_init_pages("SMP alternatives",
311 (unsigned long)__smp_alt_begin,
312 (unsigned long)__smp_alt_end);
314 alternatives_smp_save(__smp_alt_instructions,
315 __smp_alt_instructions_end);
316 alternatives_smp_module_add(NULL, "core kernel",
317 __smp_locks, __smp_locks_end,
319 alternatives_smp_switch(0);