1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) 4 * 5 * Modifications for ppc64: 6 * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> 7 * 8 * Copyright 2008 Michael Ellerman, IBM Corporation. 9 */ 10 11 #include <linux/types.h> 12 #include <linux/jump_label.h> 13 #include <linux/kernel.h> 14 #include <linux/string.h> 15 #include <linux/init.h> 16 #include <linux/sched/mm.h> 17 #include <linux/stop_machine.h> 18 #include <asm/cputable.h> 19 #include <asm/code-patching.h> 20 #include <asm/interrupt.h> 21 #include <asm/page.h> 22 #include <asm/sections.h> 23 #include <asm/setup.h> 24 #include <asm/security_features.h> 25 #include <asm/firmware.h> 26 #include <asm/inst.h> 27 28 struct fixup_entry { 29 unsigned long mask; 30 unsigned long value; 31 long start_off; 32 long end_off; 33 long alt_start_off; 34 long alt_end_off; 35 }; 36 37 static u32 *calc_addr(struct fixup_entry *fcur, long offset) 38 { 39 /* 40 * We store the offset to the code as a negative offset from 41 * the start of the alt_entry, to support the VDSO. This 42 * routine converts that back into an actual address. 43 */ 44 return (u32 *)((unsigned long)fcur + offset); 45 } 46 47 static int patch_alt_instruction(u32 *src, u32 *dest, u32 *alt_start, u32 *alt_end) 48 { 49 int err; 50 ppc_inst_t instr; 51 52 instr = ppc_inst_read(src); 53 54 if (instr_is_relative_branch(ppc_inst_read(src))) { 55 u32 *target = (u32 *)branch_target(src); 56 57 /* Branch within the section doesn't need translating */ 58 if (target < alt_start || target > alt_end) { 59 err = translate_branch(&instr, dest, src); 60 if (err) 61 return 1; 62 } 63 } 64 65 raw_patch_instruction(dest, instr); 66 67 return 0; 68 } 69 70 static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) 71 { 72 u32 *start, *end, *alt_start, *alt_end, *src, *dest; 73 74 start = calc_addr(fcur, fcur->start_off); 75 end = calc_addr(fcur, fcur->end_off); 76 alt_start = calc_addr(fcur, fcur->alt_start_off); 77 alt_end = calc_addr(fcur, fcur->alt_end_off); 78 79 if ((alt_end - alt_start) > (end - start)) 80 return 1; 81 82 if ((value & fcur->mask) == fcur->value) 83 return 0; 84 85 src = alt_start; 86 dest = start; 87 88 for (; src < alt_end; src = ppc_inst_next(src, src), 89 dest = ppc_inst_next(dest, dest)) { 90 if (patch_alt_instruction(src, dest, alt_start, alt_end)) 91 return 1; 92 } 93 94 for (; dest < end; dest++) 95 raw_patch_instruction(dest, ppc_inst(PPC_RAW_NOP())); 96 97 return 0; 98 } 99 100 void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) 101 { 102 struct fixup_entry *fcur, *fend; 103 104 fcur = fixup_start; 105 fend = fixup_end; 106 107 for (; fcur < fend; fcur++) { 108 if (patch_feature_section(value, fcur)) { 109 WARN_ON(1); 110 printk("Unable to patch feature section at %p - %p" \ 111 " with %p - %p\n", 112 calc_addr(fcur, fcur->start_off), 113 calc_addr(fcur, fcur->end_off), 114 calc_addr(fcur, fcur->alt_start_off), 115 calc_addr(fcur, fcur->alt_end_off)); 116 } 117 } 118 } 119 120 #ifdef CONFIG_PPC_BARRIER_NOSPEC 121 static bool is_fixup_addr_valid(void *dest, size_t size) 122 { 123 return system_state < SYSTEM_FREEING_INITMEM || 124 !init_section_contains(dest, size); 125 } 126 127 static int do_patch_fixups(long *start, long *end, unsigned int *instrs, int num) 128 { 129 int i; 130 131 for (i = 0; start < end; start++, i++) { 132 int j; 133 unsigned int *dest = (void *)start + *start; 134 135 if (!is_fixup_addr_valid(dest, sizeof(*instrs) * num)) 136 continue; 137 138 pr_devel("patching dest %lx\n", (unsigned long)dest); 139 140 for (j = 0; j < num; j++) 141 patch_instruction(dest + j, ppc_inst(instrs[j])); 142 } 143 return i; 144 } 145 #endif 146 147 #ifdef CONFIG_PPC_BOOK3S_64 148 static int do_patch_entry_fixups(long *start, long *end, unsigned int *instrs, 149 bool do_fallback, void *fallback) 150 { 151 int i; 152 153 for (i = 0; start < end; start++, i++) { 154 unsigned int *dest = (void *)start + *start; 155 156 if (!is_fixup_addr_valid(dest, sizeof(*instrs) * 3)) 157 continue; 158 159 pr_devel("patching dest %lx\n", (unsigned long)dest); 160 161 // See comment in do_entry_flush_fixups() RE order of patching 162 if (do_fallback) { 163 patch_instruction(dest, ppc_inst(instrs[0])); 164 patch_instruction(dest + 2, ppc_inst(instrs[2])); 165 patch_branch(dest + 1, (unsigned long)fallback, BRANCH_SET_LINK); 166 } else { 167 patch_instruction(dest + 1, ppc_inst(instrs[1])); 168 patch_instruction(dest + 2, ppc_inst(instrs[2])); 169 patch_instruction(dest, ppc_inst(instrs[0])); 170 } 171 } 172 return i; 173 } 174 175 static void do_stf_entry_barrier_fixups(enum stf_barrier_type types) 176 { 177 unsigned int instrs[3]; 178 long *start, *end; 179 int i; 180 181 start = PTRRELOC(&__start___stf_entry_barrier_fixup); 182 end = PTRRELOC(&__stop___stf_entry_barrier_fixup); 183 184 instrs[0] = PPC_RAW_NOP(); 185 instrs[1] = PPC_RAW_NOP(); 186 instrs[2] = PPC_RAW_NOP(); 187 188 i = 0; 189 if (types & STF_BARRIER_FALLBACK) { 190 instrs[i++] = PPC_RAW_MFLR(_R10); 191 instrs[i++] = PPC_RAW_NOP(); /* branch patched below */ 192 instrs[i++] = PPC_RAW_MTLR(_R10); 193 } else if (types & STF_BARRIER_EIEIO) { 194 instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */ 195 } else if (types & STF_BARRIER_SYNC_ORI) { 196 instrs[i++] = PPC_RAW_SYNC(); 197 instrs[i++] = PPC_RAW_LD(_R10, _R13, 0); 198 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 199 } 200 201 i = do_patch_entry_fixups(start, end, instrs, types & STF_BARRIER_FALLBACK, 202 &stf_barrier_fallback); 203 204 printk(KERN_DEBUG "stf-barrier: patched %d entry locations (%s barrier)\n", i, 205 (types == STF_BARRIER_NONE) ? "no" : 206 (types == STF_BARRIER_FALLBACK) ? "fallback" : 207 (types == STF_BARRIER_EIEIO) ? "eieio" : 208 (types == (STF_BARRIER_SYNC_ORI)) ? "hwsync" 209 : "unknown"); 210 } 211 212 static void do_stf_exit_barrier_fixups(enum stf_barrier_type types) 213 { 214 unsigned int instrs[6]; 215 long *start, *end; 216 int i; 217 218 start = PTRRELOC(&__start___stf_exit_barrier_fixup); 219 end = PTRRELOC(&__stop___stf_exit_barrier_fixup); 220 221 instrs[0] = PPC_RAW_NOP(); 222 instrs[1] = PPC_RAW_NOP(); 223 instrs[2] = PPC_RAW_NOP(); 224 instrs[3] = PPC_RAW_NOP(); 225 instrs[4] = PPC_RAW_NOP(); 226 instrs[5] = PPC_RAW_NOP(); 227 228 i = 0; 229 if (types & STF_BARRIER_FALLBACK || types & STF_BARRIER_SYNC_ORI) { 230 if (cpu_has_feature(CPU_FTR_HVMODE)) { 231 instrs[i++] = PPC_RAW_MTSPR(SPRN_HSPRG1, _R13); 232 instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_HSPRG0); 233 } else { 234 instrs[i++] = PPC_RAW_MTSPR(SPRN_SPRG2, _R13); 235 instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_SPRG1); 236 } 237 instrs[i++] = PPC_RAW_SYNC(); 238 instrs[i++] = PPC_RAW_LD(_R13, _R13, 0); 239 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 240 if (cpu_has_feature(CPU_FTR_HVMODE)) 241 instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_HSPRG1); 242 else 243 instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_SPRG2); 244 } else if (types & STF_BARRIER_EIEIO) { 245 instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */ 246 } 247 248 i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs)); 249 250 printk(KERN_DEBUG "stf-barrier: patched %d exit locations (%s barrier)\n", i, 251 (types == STF_BARRIER_NONE) ? "no" : 252 (types == STF_BARRIER_FALLBACK) ? "fallback" : 253 (types == STF_BARRIER_EIEIO) ? "eieio" : 254 (types == (STF_BARRIER_SYNC_ORI)) ? "hwsync" 255 : "unknown"); 256 } 257 258 static bool stf_exit_reentrant = false; 259 static bool rfi_exit_reentrant = false; 260 static DEFINE_MUTEX(exit_flush_lock); 261 262 static int __do_stf_barrier_fixups(void *data) 263 { 264 enum stf_barrier_type *types = data; 265 266 do_stf_entry_barrier_fixups(*types); 267 do_stf_exit_barrier_fixups(*types); 268 269 return 0; 270 } 271 272 void do_stf_barrier_fixups(enum stf_barrier_type types) 273 { 274 /* 275 * The call to the fallback entry flush, and the fallback/sync-ori exit 276 * flush can not be safely patched in/out while other CPUs are 277 * executing them. So call __do_stf_barrier_fixups() on one CPU while 278 * all other CPUs spin in the stop machine core with interrupts hard 279 * disabled. 280 * 281 * The branch to mark interrupt exits non-reentrant is enabled first, 282 * then stop_machine runs which will ensure all CPUs are out of the 283 * low level interrupt exit code before patching. After the patching, 284 * if allowed, then flip the branch to allow fast exits. 285 */ 286 287 // Prevent static key update races with do_rfi_flush_fixups() 288 mutex_lock(&exit_flush_lock); 289 static_branch_enable(&interrupt_exit_not_reentrant); 290 291 stop_machine(__do_stf_barrier_fixups, &types, NULL); 292 293 if ((types & STF_BARRIER_FALLBACK) || (types & STF_BARRIER_SYNC_ORI)) 294 stf_exit_reentrant = false; 295 else 296 stf_exit_reentrant = true; 297 298 if (stf_exit_reentrant && rfi_exit_reentrant) 299 static_branch_disable(&interrupt_exit_not_reentrant); 300 301 mutex_unlock(&exit_flush_lock); 302 } 303 304 void do_uaccess_flush_fixups(enum l1d_flush_type types) 305 { 306 unsigned int instrs[4]; 307 long *start, *end; 308 int i; 309 310 start = PTRRELOC(&__start___uaccess_flush_fixup); 311 end = PTRRELOC(&__stop___uaccess_flush_fixup); 312 313 instrs[0] = PPC_RAW_NOP(); 314 instrs[1] = PPC_RAW_NOP(); 315 instrs[2] = PPC_RAW_NOP(); 316 instrs[3] = PPC_RAW_BLR(); 317 318 i = 0; 319 if (types == L1D_FLUSH_FALLBACK) { 320 instrs[3] = PPC_RAW_NOP(); 321 /* fallthrough to fallback flush */ 322 } 323 324 if (types & L1D_FLUSH_ORI) { 325 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 326 instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ 327 } 328 329 if (types & L1D_FLUSH_MTTRIG) 330 instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); 331 332 i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs)); 333 334 printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i, 335 (types == L1D_FLUSH_NONE) ? "no" : 336 (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : 337 (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) 338 ? "ori+mttrig type" 339 : "ori type" : 340 (types & L1D_FLUSH_MTTRIG) ? "mttrig type" 341 : "unknown"); 342 } 343 344 static int __do_entry_flush_fixups(void *data) 345 { 346 enum l1d_flush_type types = *(enum l1d_flush_type *)data; 347 unsigned int instrs[3]; 348 long *start, *end; 349 int i; 350 351 instrs[0] = PPC_RAW_NOP(); 352 instrs[1] = PPC_RAW_NOP(); 353 instrs[2] = PPC_RAW_NOP(); 354 355 i = 0; 356 if (types == L1D_FLUSH_FALLBACK) { 357 instrs[i++] = PPC_RAW_MFLR(_R10); 358 instrs[i++] = PPC_RAW_NOP(); /* branch patched below */ 359 instrs[i++] = PPC_RAW_MTLR(_R10); 360 } 361 362 if (types & L1D_FLUSH_ORI) { 363 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 364 instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ 365 } 366 367 if (types & L1D_FLUSH_MTTRIG) 368 instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); 369 370 /* 371 * If we're patching in or out the fallback flush we need to be careful about the 372 * order in which we patch instructions. That's because it's possible we could 373 * take a page fault after patching one instruction, so the sequence of 374 * instructions must be safe even in a half patched state. 375 * 376 * To make that work, when patching in the fallback flush we patch in this order: 377 * - the mflr (dest) 378 * - the mtlr (dest + 2) 379 * - the branch (dest + 1) 380 * 381 * That ensures the sequence is safe to execute at any point. In contrast if we 382 * patch the mtlr last, it's possible we could return from the branch and not 383 * restore LR, leading to a crash later. 384 * 385 * When patching out the fallback flush (either with nops or another flush type), 386 * we patch in this order: 387 * - the branch (dest + 1) 388 * - the mtlr (dest + 2) 389 * - the mflr (dest) 390 * 391 * Note we are protected by stop_machine() from other CPUs executing the code in a 392 * semi-patched state. 393 */ 394 395 start = PTRRELOC(&__start___entry_flush_fixup); 396 end = PTRRELOC(&__stop___entry_flush_fixup); 397 i = do_patch_entry_fixups(start, end, instrs, types == L1D_FLUSH_FALLBACK, 398 &entry_flush_fallback); 399 400 start = PTRRELOC(&__start___scv_entry_flush_fixup); 401 end = PTRRELOC(&__stop___scv_entry_flush_fixup); 402 i += do_patch_entry_fixups(start, end, instrs, types == L1D_FLUSH_FALLBACK, 403 &scv_entry_flush_fallback); 404 405 printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i, 406 (types == L1D_FLUSH_NONE) ? "no" : 407 (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : 408 (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) 409 ? "ori+mttrig type" 410 : "ori type" : 411 (types & L1D_FLUSH_MTTRIG) ? "mttrig type" 412 : "unknown"); 413 414 return 0; 415 } 416 417 void do_entry_flush_fixups(enum l1d_flush_type types) 418 { 419 /* 420 * The call to the fallback flush can not be safely patched in/out while 421 * other CPUs are executing it. So call __do_entry_flush_fixups() on one 422 * CPU while all other CPUs spin in the stop machine core with interrupts 423 * hard disabled. 424 */ 425 stop_machine(__do_entry_flush_fixups, &types, NULL); 426 } 427 428 static int __do_rfi_flush_fixups(void *data) 429 { 430 enum l1d_flush_type types = *(enum l1d_flush_type *)data; 431 unsigned int instrs[3]; 432 long *start, *end; 433 int i; 434 435 start = PTRRELOC(&__start___rfi_flush_fixup); 436 end = PTRRELOC(&__stop___rfi_flush_fixup); 437 438 instrs[0] = PPC_RAW_NOP(); 439 instrs[1] = PPC_RAW_NOP(); 440 instrs[2] = PPC_RAW_NOP(); 441 442 if (types & L1D_FLUSH_FALLBACK) 443 /* b .+16 to fallback flush */ 444 instrs[0] = PPC_RAW_BRANCH(16); 445 446 i = 0; 447 if (types & L1D_FLUSH_ORI) { 448 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 449 instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ 450 } 451 452 if (types & L1D_FLUSH_MTTRIG) 453 instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); 454 455 i = do_patch_fixups(start, end, instrs, ARRAY_SIZE(instrs)); 456 457 printk(KERN_DEBUG "rfi-flush: patched %d locations (%s flush)\n", i, 458 (types == L1D_FLUSH_NONE) ? "no" : 459 (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : 460 (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) 461 ? "ori+mttrig type" 462 : "ori type" : 463 (types & L1D_FLUSH_MTTRIG) ? "mttrig type" 464 : "unknown"); 465 466 return 0; 467 } 468 469 void do_rfi_flush_fixups(enum l1d_flush_type types) 470 { 471 /* 472 * stop_machine gets all CPUs out of the interrupt exit handler same 473 * as do_stf_barrier_fixups. do_rfi_flush_fixups patching can run 474 * without stop_machine, so this could be achieved with a broadcast 475 * IPI instead, but this matches the stf sequence. 476 */ 477 478 // Prevent static key update races with do_stf_barrier_fixups() 479 mutex_lock(&exit_flush_lock); 480 static_branch_enable(&interrupt_exit_not_reentrant); 481 482 stop_machine(__do_rfi_flush_fixups, &types, NULL); 483 484 if (types & L1D_FLUSH_FALLBACK) 485 rfi_exit_reentrant = false; 486 else 487 rfi_exit_reentrant = true; 488 489 if (stf_exit_reentrant && rfi_exit_reentrant) 490 static_branch_disable(&interrupt_exit_not_reentrant); 491 492 mutex_unlock(&exit_flush_lock); 493 } 494 495 void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end) 496 { 497 unsigned int instr; 498 long *start, *end; 499 int i; 500 501 start = fixup_start; 502 end = fixup_end; 503 504 instr = PPC_RAW_NOP(); 505 506 if (enable) { 507 pr_info("barrier-nospec: using ORI speculation barrier\n"); 508 instr = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 509 } 510 511 i = do_patch_fixups(start, end, &instr, 1); 512 513 printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i); 514 } 515 516 #endif /* CONFIG_PPC_BOOK3S_64 */ 517 518 #ifdef CONFIG_PPC_BARRIER_NOSPEC 519 void do_barrier_nospec_fixups(bool enable) 520 { 521 void *start, *end; 522 523 start = PTRRELOC(&__start___barrier_nospec_fixup); 524 end = PTRRELOC(&__stop___barrier_nospec_fixup); 525 526 do_barrier_nospec_fixups_range(enable, start, end); 527 } 528 #endif /* CONFIG_PPC_BARRIER_NOSPEC */ 529 530 #ifdef CONFIG_PPC_E500 531 void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end) 532 { 533 unsigned int instr[2]; 534 long *start, *end; 535 int i; 536 537 start = fixup_start; 538 end = fixup_end; 539 540 instr[0] = PPC_RAW_NOP(); 541 instr[1] = PPC_RAW_NOP(); 542 543 if (enable) { 544 pr_info("barrier-nospec: using isync; sync as speculation barrier\n"); 545 instr[0] = PPC_RAW_ISYNC(); 546 instr[1] = PPC_RAW_SYNC(); 547 } 548 549 i = do_patch_fixups(start, end, instr, ARRAY_SIZE(instr)); 550 551 printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i); 552 } 553 554 static void __init patch_btb_flush_section(long *curr) 555 { 556 unsigned int *start, *end; 557 558 start = (void *)curr + *curr; 559 end = (void *)curr + *(curr + 1); 560 for (; start < end; start++) { 561 pr_devel("patching dest %lx\n", (unsigned long)start); 562 patch_instruction(start, ppc_inst(PPC_RAW_NOP())); 563 } 564 } 565 566 void __init do_btb_flush_fixups(void) 567 { 568 long *start, *end; 569 570 start = PTRRELOC(&__start__btb_flush_fixup); 571 end = PTRRELOC(&__stop__btb_flush_fixup); 572 573 for (; start < end; start += 2) 574 patch_btb_flush_section(start); 575 } 576 #endif /* CONFIG_PPC_E500 */ 577 578 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) 579 { 580 long *start, *end; 581 u32 *dest; 582 583 if (!(value & CPU_FTR_LWSYNC)) 584 return ; 585 586 start = fixup_start; 587 end = fixup_end; 588 589 for (; start < end; start++) { 590 dest = (void *)start + *start; 591 raw_patch_instruction(dest, ppc_inst(PPC_INST_LWSYNC)); 592 } 593 } 594 595 static void __init do_final_fixups(void) 596 { 597 #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) 598 ppc_inst_t inst; 599 u32 *src, *dest, *end; 600 601 if (PHYSICAL_START == 0) 602 return; 603 604 src = (u32 *)(KERNELBASE + PHYSICAL_START); 605 dest = (u32 *)KERNELBASE; 606 end = (void *)src + (__end_interrupts - _stext); 607 608 while (src < end) { 609 inst = ppc_inst_read(src); 610 raw_patch_instruction(dest, inst); 611 src = ppc_inst_next(src, src); 612 dest = ppc_inst_next(dest, dest); 613 } 614 #endif 615 } 616 617 static unsigned long __initdata saved_cpu_features; 618 static unsigned int __initdata saved_mmu_features; 619 #ifdef CONFIG_PPC64 620 static unsigned long __initdata saved_firmware_features; 621 #endif 622 623 void __init apply_feature_fixups(void) 624 { 625 struct cpu_spec *spec = PTRRELOC(*PTRRELOC(&cur_cpu_spec)); 626 627 *PTRRELOC(&saved_cpu_features) = spec->cpu_features; 628 *PTRRELOC(&saved_mmu_features) = spec->mmu_features; 629 630 /* 631 * Apply the CPU-specific and firmware specific fixups to kernel text 632 * (nop out sections not relevant to this CPU or this firmware). 633 */ 634 do_feature_fixups(spec->cpu_features, 635 PTRRELOC(&__start___ftr_fixup), 636 PTRRELOC(&__stop___ftr_fixup)); 637 638 do_feature_fixups(spec->mmu_features, 639 PTRRELOC(&__start___mmu_ftr_fixup), 640 PTRRELOC(&__stop___mmu_ftr_fixup)); 641 642 do_lwsync_fixups(spec->cpu_features, 643 PTRRELOC(&__start___lwsync_fixup), 644 PTRRELOC(&__stop___lwsync_fixup)); 645 646 #ifdef CONFIG_PPC64 647 saved_firmware_features = powerpc_firmware_features; 648 do_feature_fixups(powerpc_firmware_features, 649 &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); 650 #endif 651 do_final_fixups(); 652 } 653 654 void __init setup_feature_keys(void) 655 { 656 /* 657 * Initialise jump label. This causes all the cpu/mmu_has_feature() 658 * checks to take on their correct polarity based on the current set of 659 * CPU/MMU features. 660 */ 661 jump_label_init(); 662 cpu_feature_keys_init(); 663 mmu_feature_keys_init(); 664 } 665 666 static int __init check_features(void) 667 { 668 WARN(saved_cpu_features != cur_cpu_spec->cpu_features, 669 "CPU features changed after feature patching!\n"); 670 WARN(saved_mmu_features != cur_cpu_spec->mmu_features, 671 "MMU features changed after feature patching!\n"); 672 #ifdef CONFIG_PPC64 673 WARN(saved_firmware_features != powerpc_firmware_features, 674 "Firmware features changed after feature patching!\n"); 675 #endif 676 677 return 0; 678 } 679 late_initcall(check_features); 680 681 #ifdef CONFIG_FTR_FIXUP_SELFTEST 682 683 #define check(x) \ 684 if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); 685 686 /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ 687 static struct fixup_entry fixup; 688 689 static long __init calc_offset(struct fixup_entry *entry, unsigned int *p) 690 { 691 return (unsigned long)p - (unsigned long)entry; 692 } 693 694 static void __init test_basic_patching(void) 695 { 696 extern unsigned int ftr_fixup_test1[]; 697 extern unsigned int end_ftr_fixup_test1[]; 698 extern unsigned int ftr_fixup_test1_orig[]; 699 extern unsigned int ftr_fixup_test1_expected[]; 700 int size = 4 * (end_ftr_fixup_test1 - ftr_fixup_test1); 701 702 fixup.value = fixup.mask = 8; 703 fixup.start_off = calc_offset(&fixup, ftr_fixup_test1 + 1); 704 fixup.end_off = calc_offset(&fixup, ftr_fixup_test1 + 2); 705 fixup.alt_start_off = fixup.alt_end_off = 0; 706 707 /* Sanity check */ 708 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 709 710 /* Check we don't patch if the value matches */ 711 patch_feature_section(8, &fixup); 712 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 713 714 /* Check we do patch if the value doesn't match */ 715 patch_feature_section(0, &fixup); 716 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); 717 718 /* Check we do patch if the mask doesn't match */ 719 memcpy(ftr_fixup_test1, ftr_fixup_test1_orig, size); 720 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 721 patch_feature_section(~8, &fixup); 722 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); 723 } 724 725 static void __init test_alternative_patching(void) 726 { 727 extern unsigned int ftr_fixup_test2[]; 728 extern unsigned int end_ftr_fixup_test2[]; 729 extern unsigned int ftr_fixup_test2_orig[]; 730 extern unsigned int ftr_fixup_test2_alt[]; 731 extern unsigned int ftr_fixup_test2_expected[]; 732 int size = 4 * (end_ftr_fixup_test2 - ftr_fixup_test2); 733 734 fixup.value = fixup.mask = 0xF; 735 fixup.start_off = calc_offset(&fixup, ftr_fixup_test2 + 1); 736 fixup.end_off = calc_offset(&fixup, ftr_fixup_test2 + 2); 737 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test2_alt); 738 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test2_alt + 1); 739 740 /* Sanity check */ 741 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 742 743 /* Check we don't patch if the value matches */ 744 patch_feature_section(0xF, &fixup); 745 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 746 747 /* Check we do patch if the value doesn't match */ 748 patch_feature_section(0, &fixup); 749 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); 750 751 /* Check we do patch if the mask doesn't match */ 752 memcpy(ftr_fixup_test2, ftr_fixup_test2_orig, size); 753 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 754 patch_feature_section(~0xF, &fixup); 755 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); 756 } 757 758 static void __init test_alternative_case_too_big(void) 759 { 760 extern unsigned int ftr_fixup_test3[]; 761 extern unsigned int end_ftr_fixup_test3[]; 762 extern unsigned int ftr_fixup_test3_orig[]; 763 extern unsigned int ftr_fixup_test3_alt[]; 764 int size = 4 * (end_ftr_fixup_test3 - ftr_fixup_test3); 765 766 fixup.value = fixup.mask = 0xC; 767 fixup.start_off = calc_offset(&fixup, ftr_fixup_test3 + 1); 768 fixup.end_off = calc_offset(&fixup, ftr_fixup_test3 + 2); 769 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test3_alt); 770 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test3_alt + 2); 771 772 /* Sanity check */ 773 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 774 775 /* Expect nothing to be patched, and the error returned to us */ 776 check(patch_feature_section(0xF, &fixup) == 1); 777 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 778 check(patch_feature_section(0, &fixup) == 1); 779 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 780 check(patch_feature_section(~0xF, &fixup) == 1); 781 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 782 } 783 784 static void __init test_alternative_case_too_small(void) 785 { 786 extern unsigned int ftr_fixup_test4[]; 787 extern unsigned int end_ftr_fixup_test4[]; 788 extern unsigned int ftr_fixup_test4_orig[]; 789 extern unsigned int ftr_fixup_test4_alt[]; 790 extern unsigned int ftr_fixup_test4_expected[]; 791 int size = 4 * (end_ftr_fixup_test4 - ftr_fixup_test4); 792 unsigned long flag; 793 794 /* Check a high-bit flag */ 795 flag = 1UL << ((sizeof(unsigned long) - 1) * 8); 796 fixup.value = fixup.mask = flag; 797 fixup.start_off = calc_offset(&fixup, ftr_fixup_test4 + 1); 798 fixup.end_off = calc_offset(&fixup, ftr_fixup_test4 + 5); 799 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test4_alt); 800 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test4_alt + 2); 801 802 /* Sanity check */ 803 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 804 805 /* Check we don't patch if the value matches */ 806 patch_feature_section(flag, &fixup); 807 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 808 809 /* Check we do patch if the value doesn't match */ 810 patch_feature_section(0, &fixup); 811 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); 812 813 /* Check we do patch if the mask doesn't match */ 814 memcpy(ftr_fixup_test4, ftr_fixup_test4_orig, size); 815 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 816 patch_feature_section(~flag, &fixup); 817 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); 818 } 819 820 static void test_alternative_case_with_branch(void) 821 { 822 extern unsigned int ftr_fixup_test5[]; 823 extern unsigned int end_ftr_fixup_test5[]; 824 extern unsigned int ftr_fixup_test5_expected[]; 825 int size = 4 * (end_ftr_fixup_test5 - ftr_fixup_test5); 826 827 check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0); 828 } 829 830 static void __init test_alternative_case_with_external_branch(void) 831 { 832 extern unsigned int ftr_fixup_test6[]; 833 extern unsigned int end_ftr_fixup_test6[]; 834 extern unsigned int ftr_fixup_test6_expected[]; 835 int size = 4 * (end_ftr_fixup_test6 - ftr_fixup_test6); 836 837 check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0); 838 } 839 840 static void __init test_alternative_case_with_branch_to_end(void) 841 { 842 extern unsigned int ftr_fixup_test7[]; 843 extern unsigned int end_ftr_fixup_test7[]; 844 extern unsigned int ftr_fixup_test7_expected[]; 845 int size = 4 * (end_ftr_fixup_test7 - ftr_fixup_test7); 846 847 check(memcmp(ftr_fixup_test7, ftr_fixup_test7_expected, size) == 0); 848 } 849 850 static void __init test_cpu_macros(void) 851 { 852 extern u8 ftr_fixup_test_FTR_macros[]; 853 extern u8 ftr_fixup_test_FTR_macros_expected[]; 854 unsigned long size = ftr_fixup_test_FTR_macros_expected - 855 ftr_fixup_test_FTR_macros; 856 857 /* The fixups have already been done for us during boot */ 858 check(memcmp(ftr_fixup_test_FTR_macros, 859 ftr_fixup_test_FTR_macros_expected, size) == 0); 860 } 861 862 static void __init test_fw_macros(void) 863 { 864 #ifdef CONFIG_PPC64 865 extern u8 ftr_fixup_test_FW_FTR_macros[]; 866 extern u8 ftr_fixup_test_FW_FTR_macros_expected[]; 867 unsigned long size = ftr_fixup_test_FW_FTR_macros_expected - 868 ftr_fixup_test_FW_FTR_macros; 869 870 /* The fixups have already been done for us during boot */ 871 check(memcmp(ftr_fixup_test_FW_FTR_macros, 872 ftr_fixup_test_FW_FTR_macros_expected, size) == 0); 873 #endif 874 } 875 876 static void __init test_lwsync_macros(void) 877 { 878 extern u8 lwsync_fixup_test[]; 879 extern u8 end_lwsync_fixup_test[]; 880 extern u8 lwsync_fixup_test_expected_LWSYNC[]; 881 extern u8 lwsync_fixup_test_expected_SYNC[]; 882 unsigned long size = end_lwsync_fixup_test - 883 lwsync_fixup_test; 884 885 /* The fixups have already been done for us during boot */ 886 if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { 887 check(memcmp(lwsync_fixup_test, 888 lwsync_fixup_test_expected_LWSYNC, size) == 0); 889 } else { 890 check(memcmp(lwsync_fixup_test, 891 lwsync_fixup_test_expected_SYNC, size) == 0); 892 } 893 } 894 895 #ifdef CONFIG_PPC64 896 static void __init test_prefix_patching(void) 897 { 898 extern unsigned int ftr_fixup_prefix1[]; 899 extern unsigned int end_ftr_fixup_prefix1[]; 900 extern unsigned int ftr_fixup_prefix1_orig[]; 901 extern unsigned int ftr_fixup_prefix1_expected[]; 902 int size = sizeof(unsigned int) * (end_ftr_fixup_prefix1 - ftr_fixup_prefix1); 903 904 fixup.value = fixup.mask = 8; 905 fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix1 + 1); 906 fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix1 + 3); 907 fixup.alt_start_off = fixup.alt_end_off = 0; 908 909 /* Sanity check */ 910 check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) == 0); 911 912 patch_feature_section(0, &fixup); 913 check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_expected, size) == 0); 914 check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) != 0); 915 } 916 917 static void __init test_prefix_alt_patching(void) 918 { 919 extern unsigned int ftr_fixup_prefix2[]; 920 extern unsigned int end_ftr_fixup_prefix2[]; 921 extern unsigned int ftr_fixup_prefix2_orig[]; 922 extern unsigned int ftr_fixup_prefix2_expected[]; 923 extern unsigned int ftr_fixup_prefix2_alt[]; 924 int size = sizeof(unsigned int) * (end_ftr_fixup_prefix2 - ftr_fixup_prefix2); 925 926 fixup.value = fixup.mask = 8; 927 fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix2 + 1); 928 fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix2 + 3); 929 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_prefix2_alt); 930 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_prefix2_alt + 2); 931 /* Sanity check */ 932 check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) == 0); 933 934 patch_feature_section(0, &fixup); 935 check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_expected, size) == 0); 936 check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) != 0); 937 } 938 939 static void __init test_prefix_word_alt_patching(void) 940 { 941 extern unsigned int ftr_fixup_prefix3[]; 942 extern unsigned int end_ftr_fixup_prefix3[]; 943 extern unsigned int ftr_fixup_prefix3_orig[]; 944 extern unsigned int ftr_fixup_prefix3_expected[]; 945 extern unsigned int ftr_fixup_prefix3_alt[]; 946 int size = sizeof(unsigned int) * (end_ftr_fixup_prefix3 - ftr_fixup_prefix3); 947 948 fixup.value = fixup.mask = 8; 949 fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix3 + 1); 950 fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix3 + 4); 951 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_prefix3_alt); 952 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_prefix3_alt + 3); 953 /* Sanity check */ 954 check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) == 0); 955 956 patch_feature_section(0, &fixup); 957 check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_expected, size) == 0); 958 patch_feature_section(0, &fixup); 959 check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) != 0); 960 } 961 #else 962 static inline void test_prefix_patching(void) {} 963 static inline void test_prefix_alt_patching(void) {} 964 static inline void test_prefix_word_alt_patching(void) {} 965 #endif /* CONFIG_PPC64 */ 966 967 static int __init test_feature_fixups(void) 968 { 969 printk(KERN_DEBUG "Running feature fixup self-tests ...\n"); 970 971 test_basic_patching(); 972 test_alternative_patching(); 973 test_alternative_case_too_big(); 974 test_alternative_case_too_small(); 975 test_alternative_case_with_branch(); 976 test_alternative_case_with_external_branch(); 977 test_alternative_case_with_branch_to_end(); 978 test_cpu_macros(); 979 test_fw_macros(); 980 test_lwsync_macros(); 981 test_prefix_patching(); 982 test_prefix_alt_patching(); 983 test_prefix_word_alt_patching(); 984 985 return 0; 986 } 987 late_initcall(test_feature_fixups); 988 989 #endif /* CONFIG_FTR_FIXUP_SELFTEST */ 990