1 #define pr_fmt(fmt) "SMP alternatives: " fmt 2 3 #include <linux/module.h> 4 #include <linux/sched.h> 5 #include <linux/mutex.h> 6 #include <linux/list.h> 7 #include <linux/stringify.h> 8 #include <linux/kprobes.h> 9 #include <linux/mm.h> 10 #include <linux/vmalloc.h> 11 #include <linux/memory.h> 12 #include <linux/stop_machine.h> 13 #include <linux/slab.h> 14 #include <linux/kdebug.h> 15 #include <asm/alternative.h> 16 #include <asm/sections.h> 17 #include <asm/pgtable.h> 18 #include <asm/mce.h> 19 #include <asm/nmi.h> 20 #include <asm/cacheflush.h> 21 #include <asm/tlbflush.h> 22 #include <asm/io.h> 23 #include <asm/fixmap.h> 24 25 #define MAX_PATCH_LEN (255-1) 26 27 static int __initdata_or_module debug_alternative; 28 29 static int __init debug_alt(char *str) 30 { 31 debug_alternative = 1; 32 return 1; 33 } 34 __setup("debug-alternative", debug_alt); 35 36 static int noreplace_smp; 37 38 static int __init setup_noreplace_smp(char *str) 39 { 40 noreplace_smp = 1; 41 return 1; 42 } 43 __setup("noreplace-smp", setup_noreplace_smp); 44 45 #ifdef CONFIG_PARAVIRT 46 static int __initdata_or_module noreplace_paravirt = 0; 47 48 static int __init setup_noreplace_paravirt(char *str) 49 { 50 noreplace_paravirt = 1; 51 return 1; 52 } 53 __setup("noreplace-paravirt", setup_noreplace_paravirt); 54 #endif 55 56 #define DPRINTK(fmt, ...) \ 57 do { \ 58 if (debug_alternative) \ 59 printk(KERN_DEBUG fmt, ##__VA_ARGS__); \ 60 } while (0) 61 62 /* 63 * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes 64 * that correspond to that nop. Getting from one nop to the next, we 65 * add to the array the offset that is equal to the sum of all sizes of 66 * nops preceding the one we are after. 67 * 68 * Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the 69 * nice symmetry of sizes of the previous nops. 70 */ 71 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64) 72 static const unsigned char intelnops[] = 73 { 74 GENERIC_NOP1, 75 GENERIC_NOP2, 76 GENERIC_NOP3, 77 GENERIC_NOP4, 78 GENERIC_NOP5, 79 GENERIC_NOP6, 80 GENERIC_NOP7, 81 GENERIC_NOP8, 82 GENERIC_NOP5_ATOMIC 83 }; 84 static const unsigned char * const intel_nops[ASM_NOP_MAX+2] = 85 { 86 NULL, 87 intelnops, 88 intelnops + 1, 89 intelnops + 1 + 2, 90 intelnops + 1 + 2 + 3, 91 intelnops + 1 + 2 + 3 + 4, 92 intelnops + 1 + 2 + 3 + 4 + 5, 93 intelnops + 1 + 2 + 3 + 4 + 5 + 6, 94 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 95 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 96 }; 97 #endif 98 99 #ifdef K8_NOP1 100 static const unsigned char k8nops[] = 101 { 102 K8_NOP1, 103 K8_NOP2, 104 K8_NOP3, 105 K8_NOP4, 106 K8_NOP5, 107 K8_NOP6, 108 K8_NOP7, 109 K8_NOP8, 110 K8_NOP5_ATOMIC 111 }; 112 static const unsigned char * const k8_nops[ASM_NOP_MAX+2] = 113 { 114 NULL, 115 k8nops, 116 k8nops + 1, 117 k8nops + 1 + 2, 118 k8nops + 1 + 2 + 3, 119 k8nops + 1 + 2 + 3 + 4, 120 k8nops + 1 + 2 + 3 + 4 + 5, 121 k8nops + 1 + 2 + 3 + 4 + 5 + 6, 122 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 123 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 124 }; 125 #endif 126 127 #if defined(K7_NOP1) && !defined(CONFIG_X86_64) 128 static const unsigned char k7nops[] = 129 { 130 K7_NOP1, 131 K7_NOP2, 132 K7_NOP3, 133 K7_NOP4, 134 K7_NOP5, 135 K7_NOP6, 136 K7_NOP7, 137 K7_NOP8, 138 K7_NOP5_ATOMIC 139 }; 140 static const unsigned char * const k7_nops[ASM_NOP_MAX+2] = 141 { 142 NULL, 143 k7nops, 144 k7nops + 1, 145 k7nops + 1 + 2, 146 k7nops + 1 + 2 + 3, 147 k7nops + 1 + 2 + 3 + 4, 148 k7nops + 1 + 2 + 3 + 4 + 5, 149 k7nops + 1 + 2 + 3 + 4 + 5 + 6, 150 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 151 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 152 }; 153 #endif 154 155 #ifdef P6_NOP1 156 static const unsigned char p6nops[] = 157 { 158 P6_NOP1, 159 P6_NOP2, 160 P6_NOP3, 161 P6_NOP4, 162 P6_NOP5, 163 P6_NOP6, 164 P6_NOP7, 165 P6_NOP8, 166 P6_NOP5_ATOMIC 167 }; 168 static const unsigned char * const p6_nops[ASM_NOP_MAX+2] = 169 { 170 NULL, 171 p6nops, 172 p6nops + 1, 173 p6nops + 1 + 2, 174 p6nops + 1 + 2 + 3, 175 p6nops + 1 + 2 + 3 + 4, 176 p6nops + 1 + 2 + 3 + 4 + 5, 177 p6nops + 1 + 2 + 3 + 4 + 5 + 6, 178 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7, 179 p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8, 180 }; 181 #endif 182 183 /* Initialize these to a safe default */ 184 #ifdef CONFIG_X86_64 185 const unsigned char * const *ideal_nops = p6_nops; 186 #else 187 const unsigned char * const *ideal_nops = intel_nops; 188 #endif 189 190 void __init arch_init_ideal_nops(void) 191 { 192 switch (boot_cpu_data.x86_vendor) { 193 case X86_VENDOR_INTEL: 194 /* 195 * Due to a decoder implementation quirk, some 196 * specific Intel CPUs actually perform better with 197 * the "k8_nops" than with the SDM-recommended NOPs. 198 */ 199 if (boot_cpu_data.x86 == 6 && 200 boot_cpu_data.x86_model >= 0x0f && 201 boot_cpu_data.x86_model != 0x1c && 202 boot_cpu_data.x86_model != 0x26 && 203 boot_cpu_data.x86_model != 0x27 && 204 boot_cpu_data.x86_model < 0x30) { 205 ideal_nops = k8_nops; 206 } else if (boot_cpu_has(X86_FEATURE_NOPL)) { 207 ideal_nops = p6_nops; 208 } else { 209 #ifdef CONFIG_X86_64 210 ideal_nops = k8_nops; 211 #else 212 ideal_nops = intel_nops; 213 #endif 214 } 215 break; 216 default: 217 #ifdef CONFIG_X86_64 218 ideal_nops = k8_nops; 219 #else 220 if (boot_cpu_has(X86_FEATURE_K8)) 221 ideal_nops = k8_nops; 222 else if (boot_cpu_has(X86_FEATURE_K7)) 223 ideal_nops = k7_nops; 224 else 225 ideal_nops = intel_nops; 226 #endif 227 } 228 } 229 230 /* Use this to add nops to a buffer, then text_poke the whole buffer. */ 231 static void __init_or_module add_nops(void *insns, unsigned int len) 232 { 233 while (len > 0) { 234 unsigned int noplen = len; 235 if (noplen > ASM_NOP_MAX) 236 noplen = ASM_NOP_MAX; 237 memcpy(insns, ideal_nops[noplen], noplen); 238 insns += noplen; 239 len -= noplen; 240 } 241 } 242 243 extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; 244 extern s32 __smp_locks[], __smp_locks_end[]; 245 void *text_poke_early(void *addr, const void *opcode, size_t len); 246 247 /* Replace instructions with better alternatives for this CPU type. 248 This runs before SMP is initialized to avoid SMP problems with 249 self modifying code. This implies that asymmetric systems where 250 APs have less capabilities than the boot processor are not handled. 251 Tough. Make sure you disable such features by hand. */ 252 253 void __init_or_module apply_alternatives(struct alt_instr *start, 254 struct alt_instr *end) 255 { 256 struct alt_instr *a; 257 u8 *instr, *replacement; 258 u8 insnbuf[MAX_PATCH_LEN]; 259 260 DPRINTK("%s: alt table %p -> %p\n", __func__, start, end); 261 /* 262 * The scan order should be from start to end. A later scanned 263 * alternative code can overwrite a previous scanned alternative code. 264 * Some kernel functions (e.g. memcpy, memset, etc) use this order to 265 * patch code. 266 * 267 * So be careful if you want to change the scan order to any other 268 * order. 269 */ 270 for (a = start; a < end; a++) { 271 instr = (u8 *)&a->instr_offset + a->instr_offset; 272 replacement = (u8 *)&a->repl_offset + a->repl_offset; 273 BUG_ON(a->replacementlen > a->instrlen); 274 BUG_ON(a->instrlen > sizeof(insnbuf)); 275 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); 276 if (!boot_cpu_has(a->cpuid)) 277 continue; 278 279 memcpy(insnbuf, replacement, a->replacementlen); 280 281 /* 0xe8 is a relative jump; fix the offset. */ 282 if (*insnbuf == 0xe8 && a->replacementlen == 5) 283 *(s32 *)(insnbuf + 1) += replacement - instr; 284 285 add_nops(insnbuf + a->replacementlen, 286 a->instrlen - a->replacementlen); 287 288 text_poke_early(instr, insnbuf, a->instrlen); 289 } 290 } 291 292 #ifdef CONFIG_SMP 293 294 static void alternatives_smp_lock(const s32 *start, const s32 *end, 295 u8 *text, u8 *text_end) 296 { 297 const s32 *poff; 298 299 mutex_lock(&text_mutex); 300 for (poff = start; poff < end; poff++) { 301 u8 *ptr = (u8 *)poff + *poff; 302 303 if (!*poff || ptr < text || ptr >= text_end) 304 continue; 305 /* turn DS segment override prefix into lock prefix */ 306 if (*ptr == 0x3e) 307 text_poke(ptr, ((unsigned char []){0xf0}), 1); 308 } 309 mutex_unlock(&text_mutex); 310 } 311 312 static void alternatives_smp_unlock(const s32 *start, const s32 *end, 313 u8 *text, u8 *text_end) 314 { 315 const s32 *poff; 316 317 mutex_lock(&text_mutex); 318 for (poff = start; poff < end; poff++) { 319 u8 *ptr = (u8 *)poff + *poff; 320 321 if (!*poff || ptr < text || ptr >= text_end) 322 continue; 323 /* turn lock prefix into DS segment override prefix */ 324 if (*ptr == 0xf0) 325 text_poke(ptr, ((unsigned char []){0x3E}), 1); 326 } 327 mutex_unlock(&text_mutex); 328 } 329 330 struct smp_alt_module { 331 /* what is this ??? */ 332 struct module *mod; 333 char *name; 334 335 /* ptrs to lock prefixes */ 336 const s32 *locks; 337 const s32 *locks_end; 338 339 /* .text segment, needed to avoid patching init code ;) */ 340 u8 *text; 341 u8 *text_end; 342 343 struct list_head next; 344 }; 345 static LIST_HEAD(smp_alt_modules); 346 static DEFINE_MUTEX(smp_alt); 347 static bool uniproc_patched = false; /* protected by smp_alt */ 348 349 void __init_or_module alternatives_smp_module_add(struct module *mod, 350 char *name, 351 void *locks, void *locks_end, 352 void *text, void *text_end) 353 { 354 struct smp_alt_module *smp; 355 356 mutex_lock(&smp_alt); 357 if (!uniproc_patched) 358 goto unlock; 359 360 if (num_possible_cpus() == 1) 361 /* Don't bother remembering, we'll never have to undo it. */ 362 goto smp_unlock; 363 364 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 365 if (NULL == smp) 366 /* we'll run the (safe but slow) SMP code then ... */ 367 goto unlock; 368 369 smp->mod = mod; 370 smp->name = name; 371 smp->locks = locks; 372 smp->locks_end = locks_end; 373 smp->text = text; 374 smp->text_end = text_end; 375 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n", 376 __func__, smp->locks, smp->locks_end, 377 smp->text, smp->text_end, smp->name); 378 379 list_add_tail(&smp->next, &smp_alt_modules); 380 smp_unlock: 381 alternatives_smp_unlock(locks, locks_end, text, text_end); 382 unlock: 383 mutex_unlock(&smp_alt); 384 } 385 386 void __init_or_module alternatives_smp_module_del(struct module *mod) 387 { 388 struct smp_alt_module *item; 389 390 mutex_lock(&smp_alt); 391 list_for_each_entry(item, &smp_alt_modules, next) { 392 if (mod != item->mod) 393 continue; 394 list_del(&item->next); 395 kfree(item); 396 break; 397 } 398 mutex_unlock(&smp_alt); 399 } 400 401 void alternatives_enable_smp(void) 402 { 403 struct smp_alt_module *mod; 404 405 #ifdef CONFIG_LOCKDEP 406 /* 407 * Older binutils section handling bug prevented 408 * alternatives-replacement from working reliably. 409 * 410 * If this still occurs then you should see a hang 411 * or crash shortly after this line: 412 */ 413 pr_info("lockdep: fixing up alternatives\n"); 414 #endif 415 416 /* Why bother if there are no other CPUs? */ 417 BUG_ON(num_possible_cpus() == 1); 418 419 mutex_lock(&smp_alt); 420 421 if (uniproc_patched) { 422 pr_info("switching to SMP code\n"); 423 BUG_ON(num_online_cpus() != 1); 424 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 425 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 426 list_for_each_entry(mod, &smp_alt_modules, next) 427 alternatives_smp_lock(mod->locks, mod->locks_end, 428 mod->text, mod->text_end); 429 uniproc_patched = false; 430 } 431 mutex_unlock(&smp_alt); 432 } 433 434 /* Return 1 if the address range is reserved for smp-alternatives */ 435 int alternatives_text_reserved(void *start, void *end) 436 { 437 struct smp_alt_module *mod; 438 const s32 *poff; 439 u8 *text_start = start; 440 u8 *text_end = end; 441 442 list_for_each_entry(mod, &smp_alt_modules, next) { 443 if (mod->text > text_end || mod->text_end < text_start) 444 continue; 445 for (poff = mod->locks; poff < mod->locks_end; poff++) { 446 const u8 *ptr = (const u8 *)poff + *poff; 447 448 if (text_start <= ptr && text_end > ptr) 449 return 1; 450 } 451 } 452 453 return 0; 454 } 455 #endif 456 457 #ifdef CONFIG_PARAVIRT 458 void __init_or_module apply_paravirt(struct paravirt_patch_site *start, 459 struct paravirt_patch_site *end) 460 { 461 struct paravirt_patch_site *p; 462 char insnbuf[MAX_PATCH_LEN]; 463 464 if (noreplace_paravirt) 465 return; 466 467 for (p = start; p < end; p++) { 468 unsigned int used; 469 470 BUG_ON(p->len > MAX_PATCH_LEN); 471 /* prep the buffer with the original instructions */ 472 memcpy(insnbuf, p->instr, p->len); 473 used = pv_init_ops.patch(p->instrtype, p->clobbers, insnbuf, 474 (unsigned long)p->instr, p->len); 475 476 BUG_ON(used > p->len); 477 478 /* Pad the rest with nops */ 479 add_nops(insnbuf + used, p->len - used); 480 text_poke_early(p->instr, insnbuf, p->len); 481 } 482 } 483 extern struct paravirt_patch_site __start_parainstructions[], 484 __stop_parainstructions[]; 485 #endif /* CONFIG_PARAVIRT */ 486 487 void __init alternative_instructions(void) 488 { 489 /* The patching is not fully atomic, so try to avoid local interruptions 490 that might execute the to be patched code. 491 Other CPUs are not running. */ 492 stop_nmi(); 493 494 /* 495 * Don't stop machine check exceptions while patching. 496 * MCEs only happen when something got corrupted and in this 497 * case we must do something about the corruption. 498 * Ignoring it is worse than a unlikely patching race. 499 * Also machine checks tend to be broadcast and if one CPU 500 * goes into machine check the others follow quickly, so we don't 501 * expect a machine check to cause undue problems during to code 502 * patching. 503 */ 504 505 apply_alternatives(__alt_instructions, __alt_instructions_end); 506 507 #ifdef CONFIG_SMP 508 /* Patch to UP if other cpus not imminent. */ 509 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) { 510 uniproc_patched = true; 511 alternatives_smp_module_add(NULL, "core kernel", 512 __smp_locks, __smp_locks_end, 513 _text, _etext); 514 } 515 516 if (!uniproc_patched || num_possible_cpus() == 1) 517 free_init_pages("SMP alternatives", 518 (unsigned long)__smp_locks, 519 (unsigned long)__smp_locks_end); 520 #endif 521 522 apply_paravirt(__parainstructions, __parainstructions_end); 523 524 restart_nmi(); 525 } 526 527 /** 528 * text_poke_early - Update instructions on a live kernel at boot time 529 * @addr: address to modify 530 * @opcode: source of the copy 531 * @len: length to copy 532 * 533 * When you use this code to patch more than one byte of an instruction 534 * you need to make sure that other CPUs cannot execute this code in parallel. 535 * Also no thread must be currently preempted in the middle of these 536 * instructions. And on the local CPU you need to be protected again NMI or MCE 537 * handlers seeing an inconsistent instruction while you patch. 538 */ 539 void *__init_or_module text_poke_early(void *addr, const void *opcode, 540 size_t len) 541 { 542 unsigned long flags; 543 local_irq_save(flags); 544 memcpy(addr, opcode, len); 545 sync_core(); 546 local_irq_restore(flags); 547 /* Could also do a CLFLUSH here to speed up CPU recovery; but 548 that causes hangs on some VIA CPUs. */ 549 return addr; 550 } 551 552 /** 553 * text_poke - Update instructions on a live kernel 554 * @addr: address to modify 555 * @opcode: source of the copy 556 * @len: length to copy 557 * 558 * Only atomic text poke/set should be allowed when not doing early patching. 559 * It means the size must be writable atomically and the address must be aligned 560 * in a way that permits an atomic write. It also makes sure we fit on a single 561 * page. 562 * 563 * Note: Must be called under text_mutex. 564 */ 565 void *__kprobes text_poke(void *addr, const void *opcode, size_t len) 566 { 567 unsigned long flags; 568 char *vaddr; 569 struct page *pages[2]; 570 int i; 571 572 if (!core_kernel_text((unsigned long)addr)) { 573 pages[0] = vmalloc_to_page(addr); 574 pages[1] = vmalloc_to_page(addr + PAGE_SIZE); 575 } else { 576 pages[0] = virt_to_page(addr); 577 WARN_ON(!PageReserved(pages[0])); 578 pages[1] = virt_to_page(addr + PAGE_SIZE); 579 } 580 BUG_ON(!pages[0]); 581 local_irq_save(flags); 582 set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0])); 583 if (pages[1]) 584 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1])); 585 vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0); 586 memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len); 587 clear_fixmap(FIX_TEXT_POKE0); 588 if (pages[1]) 589 clear_fixmap(FIX_TEXT_POKE1); 590 local_flush_tlb(); 591 sync_core(); 592 /* Could also do a CLFLUSH here to speed up CPU recovery; but 593 that causes hangs on some VIA CPUs. */ 594 for (i = 0; i < len; i++) 595 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]); 596 local_irq_restore(flags); 597 return addr; 598 } 599 600 static void do_sync_core(void *info) 601 { 602 sync_core(); 603 } 604 605 static bool bp_patching_in_progress; 606 static void *bp_int3_handler, *bp_int3_addr; 607 608 int poke_int3_handler(struct pt_regs *regs) 609 { 610 /* bp_patching_in_progress */ 611 smp_rmb(); 612 613 if (likely(!bp_patching_in_progress)) 614 return 0; 615 616 if (user_mode_vm(regs) || regs->ip != (unsigned long)bp_int3_addr) 617 return 0; 618 619 /* set up the specified breakpoint handler */ 620 regs->ip = (unsigned long) bp_int3_handler; 621 622 return 1; 623 624 } 625 626 /** 627 * text_poke_bp() -- update instructions on live kernel on SMP 628 * @addr: address to patch 629 * @opcode: opcode of new instruction 630 * @len: length to copy 631 * @handler: address to jump to when the temporary breakpoint is hit 632 * 633 * Modify multi-byte instruction by using int3 breakpoint on SMP. 634 * We completely avoid stop_machine() here, and achieve the 635 * synchronization using int3 breakpoint. 636 * 637 * The way it is done: 638 * - add a int3 trap to the address that will be patched 639 * - sync cores 640 * - update all but the first byte of the patched range 641 * - sync cores 642 * - replace the first byte (int3) by the first byte of 643 * replacing opcode 644 * - sync cores 645 * 646 * Note: must be called under text_mutex. 647 */ 648 void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler) 649 { 650 unsigned char int3 = 0xcc; 651 652 bp_int3_handler = handler; 653 bp_int3_addr = (u8 *)addr + sizeof(int3); 654 bp_patching_in_progress = true; 655 /* 656 * Corresponding read barrier in int3 notifier for 657 * making sure the in_progress flags is correctly ordered wrt. 658 * patching 659 */ 660 smp_wmb(); 661 662 text_poke(addr, &int3, sizeof(int3)); 663 664 on_each_cpu(do_sync_core, NULL, 1); 665 666 if (len - sizeof(int3) > 0) { 667 /* patch all but the first byte */ 668 text_poke((char *)addr + sizeof(int3), 669 (const char *) opcode + sizeof(int3), 670 len - sizeof(int3)); 671 /* 672 * According to Intel, this core syncing is very likely 673 * not necessary and we'd be safe even without it. But 674 * better safe than sorry (plus there's not only Intel). 675 */ 676 on_each_cpu(do_sync_core, NULL, 1); 677 } 678 679 /* patch the first byte */ 680 text_poke(addr, opcode, sizeof(int3)); 681 682 on_each_cpu(do_sync_core, NULL, 1); 683 684 bp_patching_in_progress = false; 685 smp_wmb(); 686 687 return addr; 688 } 689 690