1 #include <linux/module.h>
2 #include <linux/sched.h>
3 #include <linux/spinlock.h>
4 #include <linux/list.h>
5 #include <linux/kprobes.h>
7 #include <linux/vmalloc.h>
8 #include <asm/alternative.h>
9 #include <asm/sections.h>
10 #include <asm/pgtable.h>
14 #ifdef CONFIG_HOTPLUG_CPU
15 static int smp_alt_once;
17 static int __init bootonly(char *str)
22 __setup("smp-alt-boot", bootonly);
24 #define smp_alt_once 1
27 static int debug_alternative;
29 static int __init debug_alt(char *str)
31 debug_alternative = 1;
34 __setup("debug-alternative", debug_alt);
36 static int noreplace_smp;
38 static int __init setup_noreplace_smp(char *str)
43 __setup("noreplace-smp", setup_noreplace_smp);
45 #ifdef CONFIG_PARAVIRT
46 static int noreplace_paravirt = 0;
48 static int __init setup_noreplace_paravirt(char *str)
50 noreplace_paravirt = 1;
53 __setup("noreplace-paravirt", setup_noreplace_paravirt);
56 #define DPRINTK(fmt, args...) if (debug_alternative) \
57 printk(KERN_DEBUG fmt, args)
60 /* Use inline assembly to define this because the nops are defined
61 as inline assembly strings in the include files and we cannot
62 get them easily into strings. */
63 asm("\t.data\nintelnops: "
64 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
65 GENERIC_NOP7 GENERIC_NOP8);
66 extern unsigned char intelnops[];
67 static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
72 intelnops + 1 + 2 + 3,
73 intelnops + 1 + 2 + 3 + 4,
74 intelnops + 1 + 2 + 3 + 4 + 5,
75 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
76 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
81 asm("\t.data\nk8nops: "
82 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
84 extern unsigned char k8nops[];
85 static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
91 k8nops + 1 + 2 + 3 + 4,
92 k8nops + 1 + 2 + 3 + 4 + 5,
93 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
94 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
99 asm("\t.data\nk7nops: "
100 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
102 extern unsigned char k7nops[];
103 static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
109 k7nops + 1 + 2 + 3 + 4,
110 k7nops + 1 + 2 + 3 + 4 + 5,
111 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
112 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
118 extern char __vsyscall_0;
119 static inline unsigned char** find_nop_table(void)
124 #else /* CONFIG_X86_64 */
128 unsigned char **noptable;
130 { X86_FEATURE_K8, k8_nops },
131 { X86_FEATURE_K7, k7_nops },
135 static unsigned char** find_nop_table(void)
137 unsigned char **noptable = intel_nops;
140 for (i = 0; noptypes[i].cpuid >= 0; i++) {
141 if (boot_cpu_has(noptypes[i].cpuid)) {
142 noptable = noptypes[i].noptable;
149 #endif /* CONFIG_X86_64 */
151 static void nop_out(void *insns, unsigned int len)
153 unsigned char **noptable = find_nop_table();
156 unsigned int noplen = len;
157 if (noplen > ASM_NOP_MAX)
158 noplen = ASM_NOP_MAX;
159 text_poke(insns, noptable[noplen], noplen);
165 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
166 extern u8 *__smp_locks[], *__smp_locks_end[];
168 /* Replace instructions with better alternatives for this CPU type.
169 This runs before SMP is initialized to avoid SMP problems with
170 self modifying code. This implies that assymetric systems where
171 APs have less capabilities than the boot processor are not handled.
172 Tough. Make sure you disable such features by hand. */
174 void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
180 DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
181 for (a = start; a < end; a++) {
182 BUG_ON(a->replacementlen > a->instrlen);
183 if (!boot_cpu_has(a->cpuid))
187 /* vsyscall code is not mapped yet. resolve it manually. */
188 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
189 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
190 DPRINTK("%s: vsyscall fixup: %p => %p\n",
191 __FUNCTION__, a->instr, instr);
194 memcpy(instr, a->replacement, a->replacementlen);
195 diff = a->instrlen - a->replacementlen;
196 nop_out(instr + a->replacementlen, diff);
202 static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
206 for (ptr = start; ptr < end; ptr++) {
211 text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */
215 static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
222 for (ptr = start; ptr < end; ptr++) {
231 struct smp_alt_module {
232 /* what is this ??? */
236 /* ptrs to lock prefixes */
240 /* .text segment, needed to avoid patching init code ;) */
244 struct list_head next;
246 static LIST_HEAD(smp_alt_modules);
247 static DEFINE_SPINLOCK(smp_alt);
249 void alternatives_smp_module_add(struct module *mod, char *name,
250 void *locks, void *locks_end,
251 void *text, void *text_end)
253 struct smp_alt_module *smp;
260 if (boot_cpu_has(X86_FEATURE_UP))
261 alternatives_smp_unlock(locks, locks_end,
266 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
268 return; /* we'll run the (safe but slow) SMP code then ... */
273 smp->locks_end = locks_end;
275 smp->text_end = text_end;
276 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
277 __FUNCTION__, smp->locks, smp->locks_end,
278 smp->text, smp->text_end, smp->name);
280 spin_lock_irqsave(&smp_alt, flags);
281 list_add_tail(&smp->next, &smp_alt_modules);
282 if (boot_cpu_has(X86_FEATURE_UP))
283 alternatives_smp_unlock(smp->locks, smp->locks_end,
284 smp->text, smp->text_end);
285 spin_unlock_irqrestore(&smp_alt, flags);
288 void alternatives_smp_module_del(struct module *mod)
290 struct smp_alt_module *item;
293 if (smp_alt_once || noreplace_smp)
296 spin_lock_irqsave(&smp_alt, flags);
297 list_for_each_entry(item, &smp_alt_modules, next) {
298 if (mod != item->mod)
300 list_del(&item->next);
301 spin_unlock_irqrestore(&smp_alt, flags);
302 DPRINTK("%s: %s\n", __FUNCTION__, item->name);
306 spin_unlock_irqrestore(&smp_alt, flags);
309 void alternatives_smp_switch(int smp)
311 struct smp_alt_module *mod;
314 #ifdef CONFIG_LOCKDEP
316 * A not yet fixed binutils section handling bug prevents
317 * alternatives-replacement from working reliably, so turn
320 printk("lockdep: not fixing up alternatives.\n");
324 if (noreplace_smp || smp_alt_once)
326 BUG_ON(!smp && (num_online_cpus() > 1));
328 spin_lock_irqsave(&smp_alt, flags);
330 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
331 clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
332 clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
333 list_for_each_entry(mod, &smp_alt_modules, next)
334 alternatives_smp_lock(mod->locks, mod->locks_end,
335 mod->text, mod->text_end);
337 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
338 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
339 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
340 list_for_each_entry(mod, &smp_alt_modules, next)
341 alternatives_smp_unlock(mod->locks, mod->locks_end,
342 mod->text, mod->text_end);
344 spin_unlock_irqrestore(&smp_alt, flags);
349 #ifdef CONFIG_PARAVIRT
350 void apply_paravirt(struct paravirt_patch_site *start,
351 struct paravirt_patch_site *end)
353 struct paravirt_patch_site *p;
355 if (noreplace_paravirt)
358 for (p = start; p < end; p++) {
361 used = paravirt_ops.patch(p->instrtype, p->clobbers, p->instr,
364 BUG_ON(used > p->len);
366 /* Pad the rest with nops */
367 nop_out(p->instr + used, p->len - used);
370 extern struct paravirt_patch_site __start_parainstructions[],
371 __stop_parainstructions[];
372 #endif /* CONFIG_PARAVIRT */
374 void __init alternative_instructions(void)
378 /* The patching is not fully atomic, so try to avoid local interruptions
379 that might execute the to be patched code.
380 Other CPUs are not running. */
386 local_irq_save(flags);
387 apply_alternatives(__alt_instructions, __alt_instructions_end);
389 /* switch to patch-once-at-boottime-only mode and free the
390 * tables in case we know the number of CPUs will never ever
392 #ifdef CONFIG_HOTPLUG_CPU
393 if (num_possible_cpus() < 2)
399 if (1 == num_possible_cpus()) {
400 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
401 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
402 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
403 alternatives_smp_unlock(__smp_locks, __smp_locks_end,
406 free_init_pages("SMP alternatives",
407 (unsigned long)__smp_locks,
408 (unsigned long)__smp_locks_end);
410 alternatives_smp_module_add(NULL, "core kernel",
411 __smp_locks, __smp_locks_end,
413 alternatives_smp_switch(0);
416 apply_paravirt(__parainstructions, __parainstructions_end);
417 local_irq_restore(flags);
427 * When you use this code to patch more than one byte of an instruction
428 * you need to make sure that other CPUs cannot execute this code in parallel.
429 * Also no thread must be currently preempted in the middle of these instructions.
430 * And on the local CPU you need to be protected again NMI or MCE handlers
431 * seeing an inconsistent instruction while you patch.
433 void __kprobes text_poke(void *oaddr, unsigned char *opcode, int len)
436 if (!pte_write(*lookup_address((unsigned long)addr))) {
437 struct page *p[2] = { virt_to_page(addr), virt_to_page(addr+PAGE_SIZE) };
438 addr = vmap(p, 2, VM_MAP, PAGE_KERNEL);
441 addr += ((unsigned long)oaddr) % PAGE_SIZE;
443 memcpy(addr, opcode, len);
445 /* Not strictly needed, but can speed CPU recovery up. Ignore cross cacheline
448 asm("clflush (%0) " :: "r" (oaddr) : "memory");