1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) 4 * 5 * Modifications for ppc64: 6 * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> 7 * 8 * Copyright 2008 Michael Ellerman, IBM Corporation. 9 */ 10 11 #include <linux/types.h> 12 #include <linux/jump_label.h> 13 #include <linux/kernel.h> 14 #include <linux/string.h> 15 #include <linux/init.h> 16 #include <linux/sched/mm.h> 17 #include <linux/stop_machine.h> 18 #include <asm/cputable.h> 19 #include <asm/code-patching.h> 20 #include <asm/interrupt.h> 21 #include <asm/page.h> 22 #include <asm/sections.h> 23 #include <asm/setup.h> 24 #include <asm/security_features.h> 25 #include <asm/firmware.h> 26 #include <asm/inst.h> 27 28 struct fixup_entry { 29 unsigned long mask; 30 unsigned long value; 31 long start_off; 32 long end_off; 33 long alt_start_off; 34 long alt_end_off; 35 }; 36 37 static u32 *calc_addr(struct fixup_entry *fcur, long offset) 38 { 39 /* 40 * We store the offset to the code as a negative offset from 41 * the start of the alt_entry, to support the VDSO. This 42 * routine converts that back into an actual address. 43 */ 44 return (u32 *)((unsigned long)fcur + offset); 45 } 46 47 static int patch_alt_instruction(u32 *src, u32 *dest, u32 *alt_start, u32 *alt_end) 48 { 49 int err; 50 struct ppc_inst instr; 51 52 instr = ppc_inst_read(src); 53 54 if (instr_is_relative_branch(ppc_inst_read(src))) { 55 u32 *target = (u32 *)branch_target(src); 56 57 /* Branch within the section doesn't need translating */ 58 if (target < alt_start || target > alt_end) { 59 err = translate_branch(&instr, dest, src); 60 if (err) 61 return 1; 62 } 63 } 64 65 raw_patch_instruction(dest, instr); 66 67 return 0; 68 } 69 70 static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) 71 { 72 u32 *start, *end, *alt_start, *alt_end, *src, *dest; 73 74 start = calc_addr(fcur, fcur->start_off); 75 end = calc_addr(fcur, fcur->end_off); 76 alt_start = calc_addr(fcur, fcur->alt_start_off); 77 alt_end = calc_addr(fcur, fcur->alt_end_off); 78 79 if ((alt_end - alt_start) > (end - start)) 80 return 1; 81 82 if ((value & fcur->mask) == fcur->value) 83 return 0; 84 85 src = alt_start; 86 dest = start; 87 88 for (; src < alt_end; src = ppc_inst_next(src, src), 89 dest = ppc_inst_next(dest, dest)) { 90 if (patch_alt_instruction(src, dest, alt_start, alt_end)) 91 return 1; 92 } 93 94 for (; dest < end; dest++) 95 raw_patch_instruction(dest, ppc_inst(PPC_RAW_NOP())); 96 97 return 0; 98 } 99 100 void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) 101 { 102 struct fixup_entry *fcur, *fend; 103 104 fcur = fixup_start; 105 fend = fixup_end; 106 107 for (; fcur < fend; fcur++) { 108 if (patch_feature_section(value, fcur)) { 109 WARN_ON(1); 110 printk("Unable to patch feature section at %p - %p" \ 111 " with %p - %p\n", 112 calc_addr(fcur, fcur->start_off), 113 calc_addr(fcur, fcur->end_off), 114 calc_addr(fcur, fcur->alt_start_off), 115 calc_addr(fcur, fcur->alt_end_off)); 116 } 117 } 118 } 119 120 #ifdef CONFIG_PPC_BOOK3S_64 121 static void do_stf_entry_barrier_fixups(enum stf_barrier_type types) 122 { 123 unsigned int instrs[3], *dest; 124 long *start, *end; 125 int i; 126 127 start = PTRRELOC(&__start___stf_entry_barrier_fixup); 128 end = PTRRELOC(&__stop___stf_entry_barrier_fixup); 129 130 instrs[0] = PPC_RAW_NOP(); 131 instrs[1] = PPC_RAW_NOP(); 132 instrs[2] = PPC_RAW_NOP(); 133 134 i = 0; 135 if (types & STF_BARRIER_FALLBACK) { 136 instrs[i++] = PPC_RAW_MFLR(_R10); 137 instrs[i++] = PPC_RAW_NOP(); /* branch patched below */ 138 instrs[i++] = PPC_RAW_MTLR(_R10); 139 } else if (types & STF_BARRIER_EIEIO) { 140 instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */ 141 } else if (types & STF_BARRIER_SYNC_ORI) { 142 instrs[i++] = PPC_RAW_SYNC(); 143 instrs[i++] = PPC_RAW_LD(_R10, _R13, 0); 144 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 145 } 146 147 for (i = 0; start < end; start++, i++) { 148 dest = (void *)start + *start; 149 150 pr_devel("patching dest %lx\n", (unsigned long)dest); 151 152 // See comment in do_entry_flush_fixups() RE order of patching 153 if (types & STF_BARRIER_FALLBACK) { 154 patch_instruction(dest, ppc_inst(instrs[0])); 155 patch_instruction(dest + 2, ppc_inst(instrs[2])); 156 patch_branch(dest + 1, 157 (unsigned long)&stf_barrier_fallback, BRANCH_SET_LINK); 158 } else { 159 patch_instruction(dest + 1, ppc_inst(instrs[1])); 160 patch_instruction(dest + 2, ppc_inst(instrs[2])); 161 patch_instruction(dest, ppc_inst(instrs[0])); 162 } 163 } 164 165 printk(KERN_DEBUG "stf-barrier: patched %d entry locations (%s barrier)\n", i, 166 (types == STF_BARRIER_NONE) ? "no" : 167 (types == STF_BARRIER_FALLBACK) ? "fallback" : 168 (types == STF_BARRIER_EIEIO) ? "eieio" : 169 (types == (STF_BARRIER_SYNC_ORI)) ? "hwsync" 170 : "unknown"); 171 } 172 173 static void do_stf_exit_barrier_fixups(enum stf_barrier_type types) 174 { 175 unsigned int instrs[6], *dest; 176 long *start, *end; 177 int i; 178 179 start = PTRRELOC(&__start___stf_exit_barrier_fixup); 180 end = PTRRELOC(&__stop___stf_exit_barrier_fixup); 181 182 instrs[0] = PPC_RAW_NOP(); 183 instrs[1] = PPC_RAW_NOP(); 184 instrs[2] = PPC_RAW_NOP(); 185 instrs[3] = PPC_RAW_NOP(); 186 instrs[4] = PPC_RAW_NOP(); 187 instrs[5] = PPC_RAW_NOP(); 188 189 i = 0; 190 if (types & STF_BARRIER_FALLBACK || types & STF_BARRIER_SYNC_ORI) { 191 if (cpu_has_feature(CPU_FTR_HVMODE)) { 192 instrs[i++] = PPC_RAW_MTSPR(SPRN_HSPRG1, _R13); 193 instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_HSPRG0); 194 } else { 195 instrs[i++] = PPC_RAW_MTSPR(SPRN_SPRG2, _R13); 196 instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_SPRG1); 197 } 198 instrs[i++] = PPC_RAW_SYNC(); 199 instrs[i++] = PPC_RAW_LD(_R13, _R13, 0); 200 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 201 if (cpu_has_feature(CPU_FTR_HVMODE)) 202 instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_HSPRG1); 203 else 204 instrs[i++] = PPC_RAW_MFSPR(_R13, SPRN_SPRG2); 205 } else if (types & STF_BARRIER_EIEIO) { 206 instrs[i++] = PPC_RAW_EIEIO() | 0x02000000; /* eieio + bit 6 hint */ 207 } 208 209 for (i = 0; start < end; start++, i++) { 210 dest = (void *)start + *start; 211 212 pr_devel("patching dest %lx\n", (unsigned long)dest); 213 214 patch_instruction(dest, ppc_inst(instrs[0])); 215 patch_instruction(dest + 1, ppc_inst(instrs[1])); 216 patch_instruction(dest + 2, ppc_inst(instrs[2])); 217 patch_instruction(dest + 3, ppc_inst(instrs[3])); 218 patch_instruction(dest + 4, ppc_inst(instrs[4])); 219 patch_instruction(dest + 5, ppc_inst(instrs[5])); 220 } 221 printk(KERN_DEBUG "stf-barrier: patched %d exit locations (%s barrier)\n", i, 222 (types == STF_BARRIER_NONE) ? "no" : 223 (types == STF_BARRIER_FALLBACK) ? "fallback" : 224 (types == STF_BARRIER_EIEIO) ? "eieio" : 225 (types == (STF_BARRIER_SYNC_ORI)) ? "hwsync" 226 : "unknown"); 227 } 228 229 static bool stf_exit_reentrant = false; 230 static bool rfi_exit_reentrant = false; 231 static DEFINE_MUTEX(exit_flush_lock); 232 233 static int __do_stf_barrier_fixups(void *data) 234 { 235 enum stf_barrier_type *types = data; 236 237 do_stf_entry_barrier_fixups(*types); 238 do_stf_exit_barrier_fixups(*types); 239 240 return 0; 241 } 242 243 void do_stf_barrier_fixups(enum stf_barrier_type types) 244 { 245 /* 246 * The call to the fallback entry flush, and the fallback/sync-ori exit 247 * flush can not be safely patched in/out while other CPUs are 248 * executing them. So call __do_stf_barrier_fixups() on one CPU while 249 * all other CPUs spin in the stop machine core with interrupts hard 250 * disabled. 251 * 252 * The branch to mark interrupt exits non-reentrant is enabled first, 253 * then stop_machine runs which will ensure all CPUs are out of the 254 * low level interrupt exit code before patching. After the patching, 255 * if allowed, then flip the branch to allow fast exits. 256 */ 257 258 // Prevent static key update races with do_rfi_flush_fixups() 259 mutex_lock(&exit_flush_lock); 260 static_branch_enable(&interrupt_exit_not_reentrant); 261 262 stop_machine(__do_stf_barrier_fixups, &types, NULL); 263 264 if ((types & STF_BARRIER_FALLBACK) || (types & STF_BARRIER_SYNC_ORI)) 265 stf_exit_reentrant = false; 266 else 267 stf_exit_reentrant = true; 268 269 if (stf_exit_reentrant && rfi_exit_reentrant) 270 static_branch_disable(&interrupt_exit_not_reentrant); 271 272 mutex_unlock(&exit_flush_lock); 273 } 274 275 void do_uaccess_flush_fixups(enum l1d_flush_type types) 276 { 277 unsigned int instrs[4], *dest; 278 long *start, *end; 279 int i; 280 281 start = PTRRELOC(&__start___uaccess_flush_fixup); 282 end = PTRRELOC(&__stop___uaccess_flush_fixup); 283 284 instrs[0] = PPC_RAW_NOP(); 285 instrs[1] = PPC_RAW_NOP(); 286 instrs[2] = PPC_RAW_NOP(); 287 instrs[3] = PPC_RAW_BLR(); 288 289 i = 0; 290 if (types == L1D_FLUSH_FALLBACK) { 291 instrs[3] = PPC_RAW_NOP(); 292 /* fallthrough to fallback flush */ 293 } 294 295 if (types & L1D_FLUSH_ORI) { 296 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 297 instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ 298 } 299 300 if (types & L1D_FLUSH_MTTRIG) 301 instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); 302 303 for (i = 0; start < end; start++, i++) { 304 dest = (void *)start + *start; 305 306 pr_devel("patching dest %lx\n", (unsigned long)dest); 307 308 patch_instruction(dest, ppc_inst(instrs[0])); 309 310 patch_instruction(dest + 1, ppc_inst(instrs[1])); 311 patch_instruction(dest + 2, ppc_inst(instrs[2])); 312 patch_instruction(dest + 3, ppc_inst(instrs[3])); 313 } 314 315 printk(KERN_DEBUG "uaccess-flush: patched %d locations (%s flush)\n", i, 316 (types == L1D_FLUSH_NONE) ? "no" : 317 (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : 318 (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) 319 ? "ori+mttrig type" 320 : "ori type" : 321 (types & L1D_FLUSH_MTTRIG) ? "mttrig type" 322 : "unknown"); 323 } 324 325 static int __do_entry_flush_fixups(void *data) 326 { 327 enum l1d_flush_type types = *(enum l1d_flush_type *)data; 328 unsigned int instrs[3], *dest; 329 long *start, *end; 330 int i; 331 332 instrs[0] = PPC_RAW_NOP(); 333 instrs[1] = PPC_RAW_NOP(); 334 instrs[2] = PPC_RAW_NOP(); 335 336 i = 0; 337 if (types == L1D_FLUSH_FALLBACK) { 338 instrs[i++] = PPC_RAW_MFLR(_R10); 339 instrs[i++] = PPC_RAW_NOP(); /* branch patched below */ 340 instrs[i++] = PPC_RAW_MTLR(_R10); 341 } 342 343 if (types & L1D_FLUSH_ORI) { 344 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 345 instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ 346 } 347 348 if (types & L1D_FLUSH_MTTRIG) 349 instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); 350 351 /* 352 * If we're patching in or out the fallback flush we need to be careful about the 353 * order in which we patch instructions. That's because it's possible we could 354 * take a page fault after patching one instruction, so the sequence of 355 * instructions must be safe even in a half patched state. 356 * 357 * To make that work, when patching in the fallback flush we patch in this order: 358 * - the mflr (dest) 359 * - the mtlr (dest + 2) 360 * - the branch (dest + 1) 361 * 362 * That ensures the sequence is safe to execute at any point. In contrast if we 363 * patch the mtlr last, it's possible we could return from the branch and not 364 * restore LR, leading to a crash later. 365 * 366 * When patching out the fallback flush (either with nops or another flush type), 367 * we patch in this order: 368 * - the branch (dest + 1) 369 * - the mtlr (dest + 2) 370 * - the mflr (dest) 371 * 372 * Note we are protected by stop_machine() from other CPUs executing the code in a 373 * semi-patched state. 374 */ 375 376 start = PTRRELOC(&__start___entry_flush_fixup); 377 end = PTRRELOC(&__stop___entry_flush_fixup); 378 for (i = 0; start < end; start++, i++) { 379 dest = (void *)start + *start; 380 381 pr_devel("patching dest %lx\n", (unsigned long)dest); 382 383 if (types == L1D_FLUSH_FALLBACK) { 384 patch_instruction(dest, ppc_inst(instrs[0])); 385 patch_instruction(dest + 2, ppc_inst(instrs[2])); 386 patch_branch(dest + 1, 387 (unsigned long)&entry_flush_fallback, BRANCH_SET_LINK); 388 } else { 389 patch_instruction(dest + 1, ppc_inst(instrs[1])); 390 patch_instruction(dest + 2, ppc_inst(instrs[2])); 391 patch_instruction(dest, ppc_inst(instrs[0])); 392 } 393 } 394 395 start = PTRRELOC(&__start___scv_entry_flush_fixup); 396 end = PTRRELOC(&__stop___scv_entry_flush_fixup); 397 for (; start < end; start++, i++) { 398 dest = (void *)start + *start; 399 400 pr_devel("patching dest %lx\n", (unsigned long)dest); 401 402 if (types == L1D_FLUSH_FALLBACK) { 403 patch_instruction(dest, ppc_inst(instrs[0])); 404 patch_instruction(dest + 2, ppc_inst(instrs[2])); 405 patch_branch(dest + 1, 406 (unsigned long)&scv_entry_flush_fallback, BRANCH_SET_LINK); 407 } else { 408 patch_instruction(dest + 1, ppc_inst(instrs[1])); 409 patch_instruction(dest + 2, ppc_inst(instrs[2])); 410 patch_instruction(dest, ppc_inst(instrs[0])); 411 } 412 } 413 414 415 printk(KERN_DEBUG "entry-flush: patched %d locations (%s flush)\n", i, 416 (types == L1D_FLUSH_NONE) ? "no" : 417 (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : 418 (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) 419 ? "ori+mttrig type" 420 : "ori type" : 421 (types & L1D_FLUSH_MTTRIG) ? "mttrig type" 422 : "unknown"); 423 424 return 0; 425 } 426 427 void do_entry_flush_fixups(enum l1d_flush_type types) 428 { 429 /* 430 * The call to the fallback flush can not be safely patched in/out while 431 * other CPUs are executing it. So call __do_entry_flush_fixups() on one 432 * CPU while all other CPUs spin in the stop machine core with interrupts 433 * hard disabled. 434 */ 435 stop_machine(__do_entry_flush_fixups, &types, NULL); 436 } 437 438 static int __do_rfi_flush_fixups(void *data) 439 { 440 enum l1d_flush_type types = *(enum l1d_flush_type *)data; 441 unsigned int instrs[3], *dest; 442 long *start, *end; 443 int i; 444 445 start = PTRRELOC(&__start___rfi_flush_fixup); 446 end = PTRRELOC(&__stop___rfi_flush_fixup); 447 448 instrs[0] = PPC_RAW_NOP(); 449 instrs[1] = PPC_RAW_NOP(); 450 instrs[2] = PPC_RAW_NOP(); 451 452 if (types & L1D_FLUSH_FALLBACK) 453 /* b .+16 to fallback flush */ 454 instrs[0] = PPC_INST_BRANCH | 16; 455 456 i = 0; 457 if (types & L1D_FLUSH_ORI) { 458 instrs[i++] = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 459 instrs[i++] = PPC_RAW_ORI(_R30, _R30, 0); /* L1d flush */ 460 } 461 462 if (types & L1D_FLUSH_MTTRIG) 463 instrs[i++] = PPC_RAW_MTSPR(SPRN_TRIG2, _R0); 464 465 for (i = 0; start < end; start++, i++) { 466 dest = (void *)start + *start; 467 468 pr_devel("patching dest %lx\n", (unsigned long)dest); 469 470 patch_instruction(dest, ppc_inst(instrs[0])); 471 patch_instruction(dest + 1, ppc_inst(instrs[1])); 472 patch_instruction(dest + 2, ppc_inst(instrs[2])); 473 } 474 475 printk(KERN_DEBUG "rfi-flush: patched %d locations (%s flush)\n", i, 476 (types == L1D_FLUSH_NONE) ? "no" : 477 (types == L1D_FLUSH_FALLBACK) ? "fallback displacement" : 478 (types & L1D_FLUSH_ORI) ? (types & L1D_FLUSH_MTTRIG) 479 ? "ori+mttrig type" 480 : "ori type" : 481 (types & L1D_FLUSH_MTTRIG) ? "mttrig type" 482 : "unknown"); 483 484 return 0; 485 } 486 487 void do_rfi_flush_fixups(enum l1d_flush_type types) 488 { 489 /* 490 * stop_machine gets all CPUs out of the interrupt exit handler same 491 * as do_stf_barrier_fixups. do_rfi_flush_fixups patching can run 492 * without stop_machine, so this could be achieved with a broadcast 493 * IPI instead, but this matches the stf sequence. 494 */ 495 496 // Prevent static key update races with do_stf_barrier_fixups() 497 mutex_lock(&exit_flush_lock); 498 static_branch_enable(&interrupt_exit_not_reentrant); 499 500 stop_machine(__do_rfi_flush_fixups, &types, NULL); 501 502 if (types & L1D_FLUSH_FALLBACK) 503 rfi_exit_reentrant = false; 504 else 505 rfi_exit_reentrant = true; 506 507 if (stf_exit_reentrant && rfi_exit_reentrant) 508 static_branch_disable(&interrupt_exit_not_reentrant); 509 510 mutex_unlock(&exit_flush_lock); 511 } 512 513 void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end) 514 { 515 unsigned int instr, *dest; 516 long *start, *end; 517 int i; 518 519 start = fixup_start; 520 end = fixup_end; 521 522 instr = PPC_RAW_NOP(); 523 524 if (enable) { 525 pr_info("barrier-nospec: using ORI speculation barrier\n"); 526 instr = PPC_RAW_ORI(_R31, _R31, 0); /* speculation barrier */ 527 } 528 529 for (i = 0; start < end; start++, i++) { 530 dest = (void *)start + *start; 531 532 pr_devel("patching dest %lx\n", (unsigned long)dest); 533 patch_instruction(dest, ppc_inst(instr)); 534 } 535 536 printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i); 537 } 538 539 #endif /* CONFIG_PPC_BOOK3S_64 */ 540 541 #ifdef CONFIG_PPC_BARRIER_NOSPEC 542 void do_barrier_nospec_fixups(bool enable) 543 { 544 void *start, *end; 545 546 start = PTRRELOC(&__start___barrier_nospec_fixup); 547 end = PTRRELOC(&__stop___barrier_nospec_fixup); 548 549 do_barrier_nospec_fixups_range(enable, start, end); 550 } 551 #endif /* CONFIG_PPC_BARRIER_NOSPEC */ 552 553 #ifdef CONFIG_PPC_FSL_BOOK3E 554 void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end) 555 { 556 unsigned int instr[2], *dest; 557 long *start, *end; 558 int i; 559 560 start = fixup_start; 561 end = fixup_end; 562 563 instr[0] = PPC_RAW_NOP(); 564 instr[1] = PPC_RAW_NOP(); 565 566 if (enable) { 567 pr_info("barrier-nospec: using isync; sync as speculation barrier\n"); 568 instr[0] = PPC_RAW_ISYNC(); 569 instr[1] = PPC_RAW_SYNC(); 570 } 571 572 for (i = 0; start < end; start++, i++) { 573 dest = (void *)start + *start; 574 575 pr_devel("patching dest %lx\n", (unsigned long)dest); 576 patch_instruction(dest, ppc_inst(instr[0])); 577 patch_instruction(dest + 1, ppc_inst(instr[1])); 578 } 579 580 printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i); 581 } 582 583 static void patch_btb_flush_section(long *curr) 584 { 585 unsigned int *start, *end; 586 587 start = (void *)curr + *curr; 588 end = (void *)curr + *(curr + 1); 589 for (; start < end; start++) { 590 pr_devel("patching dest %lx\n", (unsigned long)start); 591 patch_instruction(start, ppc_inst(PPC_RAW_NOP())); 592 } 593 } 594 595 void do_btb_flush_fixups(void) 596 { 597 long *start, *end; 598 599 start = PTRRELOC(&__start__btb_flush_fixup); 600 end = PTRRELOC(&__stop__btb_flush_fixup); 601 602 for (; start < end; start += 2) 603 patch_btb_flush_section(start); 604 } 605 #endif /* CONFIG_PPC_FSL_BOOK3E */ 606 607 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) 608 { 609 long *start, *end; 610 u32 *dest; 611 612 if (!(value & CPU_FTR_LWSYNC)) 613 return ; 614 615 start = fixup_start; 616 end = fixup_end; 617 618 for (; start < end; start++) { 619 dest = (void *)start + *start; 620 raw_patch_instruction(dest, ppc_inst(PPC_INST_LWSYNC)); 621 } 622 } 623 624 static void do_final_fixups(void) 625 { 626 #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) 627 struct ppc_inst inst; 628 u32 *src, *dest, *end; 629 630 if (PHYSICAL_START == 0) 631 return; 632 633 src = (u32 *)(KERNELBASE + PHYSICAL_START); 634 dest = (u32 *)KERNELBASE; 635 end = (void *)src + (__end_interrupts - _stext); 636 637 while (src < end) { 638 inst = ppc_inst_read(src); 639 raw_patch_instruction(dest, inst); 640 src = ppc_inst_next(src, src); 641 dest = ppc_inst_next(dest, dest); 642 } 643 #endif 644 } 645 646 static unsigned long __initdata saved_cpu_features; 647 static unsigned int __initdata saved_mmu_features; 648 #ifdef CONFIG_PPC64 649 static unsigned long __initdata saved_firmware_features; 650 #endif 651 652 void __init apply_feature_fixups(void) 653 { 654 struct cpu_spec *spec = PTRRELOC(*PTRRELOC(&cur_cpu_spec)); 655 656 *PTRRELOC(&saved_cpu_features) = spec->cpu_features; 657 *PTRRELOC(&saved_mmu_features) = spec->mmu_features; 658 659 /* 660 * Apply the CPU-specific and firmware specific fixups to kernel text 661 * (nop out sections not relevant to this CPU or this firmware). 662 */ 663 do_feature_fixups(spec->cpu_features, 664 PTRRELOC(&__start___ftr_fixup), 665 PTRRELOC(&__stop___ftr_fixup)); 666 667 do_feature_fixups(spec->mmu_features, 668 PTRRELOC(&__start___mmu_ftr_fixup), 669 PTRRELOC(&__stop___mmu_ftr_fixup)); 670 671 do_lwsync_fixups(spec->cpu_features, 672 PTRRELOC(&__start___lwsync_fixup), 673 PTRRELOC(&__stop___lwsync_fixup)); 674 675 #ifdef CONFIG_PPC64 676 saved_firmware_features = powerpc_firmware_features; 677 do_feature_fixups(powerpc_firmware_features, 678 &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); 679 #endif 680 do_final_fixups(); 681 } 682 683 void __init setup_feature_keys(void) 684 { 685 /* 686 * Initialise jump label. This causes all the cpu/mmu_has_feature() 687 * checks to take on their correct polarity based on the current set of 688 * CPU/MMU features. 689 */ 690 jump_label_init(); 691 cpu_feature_keys_init(); 692 mmu_feature_keys_init(); 693 } 694 695 static int __init check_features(void) 696 { 697 WARN(saved_cpu_features != cur_cpu_spec->cpu_features, 698 "CPU features changed after feature patching!\n"); 699 WARN(saved_mmu_features != cur_cpu_spec->mmu_features, 700 "MMU features changed after feature patching!\n"); 701 #ifdef CONFIG_PPC64 702 WARN(saved_firmware_features != powerpc_firmware_features, 703 "Firmware features changed after feature patching!\n"); 704 #endif 705 706 return 0; 707 } 708 late_initcall(check_features); 709 710 #ifdef CONFIG_FTR_FIXUP_SELFTEST 711 712 #define check(x) \ 713 if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); 714 715 /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ 716 static struct fixup_entry fixup; 717 718 static long calc_offset(struct fixup_entry *entry, unsigned int *p) 719 { 720 return (unsigned long)p - (unsigned long)entry; 721 } 722 723 static void test_basic_patching(void) 724 { 725 extern unsigned int ftr_fixup_test1[]; 726 extern unsigned int end_ftr_fixup_test1[]; 727 extern unsigned int ftr_fixup_test1_orig[]; 728 extern unsigned int ftr_fixup_test1_expected[]; 729 int size = 4 * (end_ftr_fixup_test1 - ftr_fixup_test1); 730 731 fixup.value = fixup.mask = 8; 732 fixup.start_off = calc_offset(&fixup, ftr_fixup_test1 + 1); 733 fixup.end_off = calc_offset(&fixup, ftr_fixup_test1 + 2); 734 fixup.alt_start_off = fixup.alt_end_off = 0; 735 736 /* Sanity check */ 737 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 738 739 /* Check we don't patch if the value matches */ 740 patch_feature_section(8, &fixup); 741 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 742 743 /* Check we do patch if the value doesn't match */ 744 patch_feature_section(0, &fixup); 745 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); 746 747 /* Check we do patch if the mask doesn't match */ 748 memcpy(ftr_fixup_test1, ftr_fixup_test1_orig, size); 749 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_orig, size) == 0); 750 patch_feature_section(~8, &fixup); 751 check(memcmp(ftr_fixup_test1, ftr_fixup_test1_expected, size) == 0); 752 } 753 754 static void test_alternative_patching(void) 755 { 756 extern unsigned int ftr_fixup_test2[]; 757 extern unsigned int end_ftr_fixup_test2[]; 758 extern unsigned int ftr_fixup_test2_orig[]; 759 extern unsigned int ftr_fixup_test2_alt[]; 760 extern unsigned int ftr_fixup_test2_expected[]; 761 int size = 4 * (end_ftr_fixup_test2 - ftr_fixup_test2); 762 763 fixup.value = fixup.mask = 0xF; 764 fixup.start_off = calc_offset(&fixup, ftr_fixup_test2 + 1); 765 fixup.end_off = calc_offset(&fixup, ftr_fixup_test2 + 2); 766 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test2_alt); 767 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test2_alt + 1); 768 769 /* Sanity check */ 770 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 771 772 /* Check we don't patch if the value matches */ 773 patch_feature_section(0xF, &fixup); 774 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 775 776 /* Check we do patch if the value doesn't match */ 777 patch_feature_section(0, &fixup); 778 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); 779 780 /* Check we do patch if the mask doesn't match */ 781 memcpy(ftr_fixup_test2, ftr_fixup_test2_orig, size); 782 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_orig, size) == 0); 783 patch_feature_section(~0xF, &fixup); 784 check(memcmp(ftr_fixup_test2, ftr_fixup_test2_expected, size) == 0); 785 } 786 787 static void test_alternative_case_too_big(void) 788 { 789 extern unsigned int ftr_fixup_test3[]; 790 extern unsigned int end_ftr_fixup_test3[]; 791 extern unsigned int ftr_fixup_test3_orig[]; 792 extern unsigned int ftr_fixup_test3_alt[]; 793 int size = 4 * (end_ftr_fixup_test3 - ftr_fixup_test3); 794 795 fixup.value = fixup.mask = 0xC; 796 fixup.start_off = calc_offset(&fixup, ftr_fixup_test3 + 1); 797 fixup.end_off = calc_offset(&fixup, ftr_fixup_test3 + 2); 798 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test3_alt); 799 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test3_alt + 2); 800 801 /* Sanity check */ 802 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 803 804 /* Expect nothing to be patched, and the error returned to us */ 805 check(patch_feature_section(0xF, &fixup) == 1); 806 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 807 check(patch_feature_section(0, &fixup) == 1); 808 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 809 check(patch_feature_section(~0xF, &fixup) == 1); 810 check(memcmp(ftr_fixup_test3, ftr_fixup_test3_orig, size) == 0); 811 } 812 813 static void test_alternative_case_too_small(void) 814 { 815 extern unsigned int ftr_fixup_test4[]; 816 extern unsigned int end_ftr_fixup_test4[]; 817 extern unsigned int ftr_fixup_test4_orig[]; 818 extern unsigned int ftr_fixup_test4_alt[]; 819 extern unsigned int ftr_fixup_test4_expected[]; 820 int size = 4 * (end_ftr_fixup_test4 - ftr_fixup_test4); 821 unsigned long flag; 822 823 /* Check a high-bit flag */ 824 flag = 1UL << ((sizeof(unsigned long) - 1) * 8); 825 fixup.value = fixup.mask = flag; 826 fixup.start_off = calc_offset(&fixup, ftr_fixup_test4 + 1); 827 fixup.end_off = calc_offset(&fixup, ftr_fixup_test4 + 5); 828 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_test4_alt); 829 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_test4_alt + 2); 830 831 /* Sanity check */ 832 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 833 834 /* Check we don't patch if the value matches */ 835 patch_feature_section(flag, &fixup); 836 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 837 838 /* Check we do patch if the value doesn't match */ 839 patch_feature_section(0, &fixup); 840 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); 841 842 /* Check we do patch if the mask doesn't match */ 843 memcpy(ftr_fixup_test4, ftr_fixup_test4_orig, size); 844 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_orig, size) == 0); 845 patch_feature_section(~flag, &fixup); 846 check(memcmp(ftr_fixup_test4, ftr_fixup_test4_expected, size) == 0); 847 } 848 849 static void test_alternative_case_with_branch(void) 850 { 851 extern unsigned int ftr_fixup_test5[]; 852 extern unsigned int end_ftr_fixup_test5[]; 853 extern unsigned int ftr_fixup_test5_expected[]; 854 int size = 4 * (end_ftr_fixup_test5 - ftr_fixup_test5); 855 856 check(memcmp(ftr_fixup_test5, ftr_fixup_test5_expected, size) == 0); 857 } 858 859 static void test_alternative_case_with_external_branch(void) 860 { 861 extern unsigned int ftr_fixup_test6[]; 862 extern unsigned int end_ftr_fixup_test6[]; 863 extern unsigned int ftr_fixup_test6_expected[]; 864 int size = 4 * (end_ftr_fixup_test6 - ftr_fixup_test6); 865 866 check(memcmp(ftr_fixup_test6, ftr_fixup_test6_expected, size) == 0); 867 } 868 869 static void test_alternative_case_with_branch_to_end(void) 870 { 871 extern unsigned int ftr_fixup_test7[]; 872 extern unsigned int end_ftr_fixup_test7[]; 873 extern unsigned int ftr_fixup_test7_expected[]; 874 int size = 4 * (end_ftr_fixup_test7 - ftr_fixup_test7); 875 876 check(memcmp(ftr_fixup_test7, ftr_fixup_test7_expected, size) == 0); 877 } 878 879 static void test_cpu_macros(void) 880 { 881 extern u8 ftr_fixup_test_FTR_macros[]; 882 extern u8 ftr_fixup_test_FTR_macros_expected[]; 883 unsigned long size = ftr_fixup_test_FTR_macros_expected - 884 ftr_fixup_test_FTR_macros; 885 886 /* The fixups have already been done for us during boot */ 887 check(memcmp(ftr_fixup_test_FTR_macros, 888 ftr_fixup_test_FTR_macros_expected, size) == 0); 889 } 890 891 static void test_fw_macros(void) 892 { 893 #ifdef CONFIG_PPC64 894 extern u8 ftr_fixup_test_FW_FTR_macros[]; 895 extern u8 ftr_fixup_test_FW_FTR_macros_expected[]; 896 unsigned long size = ftr_fixup_test_FW_FTR_macros_expected - 897 ftr_fixup_test_FW_FTR_macros; 898 899 /* The fixups have already been done for us during boot */ 900 check(memcmp(ftr_fixup_test_FW_FTR_macros, 901 ftr_fixup_test_FW_FTR_macros_expected, size) == 0); 902 #endif 903 } 904 905 static void test_lwsync_macros(void) 906 { 907 extern u8 lwsync_fixup_test[]; 908 extern u8 end_lwsync_fixup_test[]; 909 extern u8 lwsync_fixup_test_expected_LWSYNC[]; 910 extern u8 lwsync_fixup_test_expected_SYNC[]; 911 unsigned long size = end_lwsync_fixup_test - 912 lwsync_fixup_test; 913 914 /* The fixups have already been done for us during boot */ 915 if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { 916 check(memcmp(lwsync_fixup_test, 917 lwsync_fixup_test_expected_LWSYNC, size) == 0); 918 } else { 919 check(memcmp(lwsync_fixup_test, 920 lwsync_fixup_test_expected_SYNC, size) == 0); 921 } 922 } 923 924 #ifdef CONFIG_PPC64 925 static void __init test_prefix_patching(void) 926 { 927 extern unsigned int ftr_fixup_prefix1[]; 928 extern unsigned int end_ftr_fixup_prefix1[]; 929 extern unsigned int ftr_fixup_prefix1_orig[]; 930 extern unsigned int ftr_fixup_prefix1_expected[]; 931 int size = sizeof(unsigned int) * (end_ftr_fixup_prefix1 - ftr_fixup_prefix1); 932 933 fixup.value = fixup.mask = 8; 934 fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix1 + 1); 935 fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix1 + 3); 936 fixup.alt_start_off = fixup.alt_end_off = 0; 937 938 /* Sanity check */ 939 check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) == 0); 940 941 patch_feature_section(0, &fixup); 942 check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_expected, size) == 0); 943 check(memcmp(ftr_fixup_prefix1, ftr_fixup_prefix1_orig, size) != 0); 944 } 945 946 static void __init test_prefix_alt_patching(void) 947 { 948 extern unsigned int ftr_fixup_prefix2[]; 949 extern unsigned int end_ftr_fixup_prefix2[]; 950 extern unsigned int ftr_fixup_prefix2_orig[]; 951 extern unsigned int ftr_fixup_prefix2_expected[]; 952 extern unsigned int ftr_fixup_prefix2_alt[]; 953 int size = sizeof(unsigned int) * (end_ftr_fixup_prefix2 - ftr_fixup_prefix2); 954 955 fixup.value = fixup.mask = 8; 956 fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix2 + 1); 957 fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix2 + 3); 958 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_prefix2_alt); 959 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_prefix2_alt + 2); 960 /* Sanity check */ 961 check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) == 0); 962 963 patch_feature_section(0, &fixup); 964 check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_expected, size) == 0); 965 check(memcmp(ftr_fixup_prefix2, ftr_fixup_prefix2_orig, size) != 0); 966 } 967 968 static void __init test_prefix_word_alt_patching(void) 969 { 970 extern unsigned int ftr_fixup_prefix3[]; 971 extern unsigned int end_ftr_fixup_prefix3[]; 972 extern unsigned int ftr_fixup_prefix3_orig[]; 973 extern unsigned int ftr_fixup_prefix3_expected[]; 974 extern unsigned int ftr_fixup_prefix3_alt[]; 975 int size = sizeof(unsigned int) * (end_ftr_fixup_prefix3 - ftr_fixup_prefix3); 976 977 fixup.value = fixup.mask = 8; 978 fixup.start_off = calc_offset(&fixup, ftr_fixup_prefix3 + 1); 979 fixup.end_off = calc_offset(&fixup, ftr_fixup_prefix3 + 4); 980 fixup.alt_start_off = calc_offset(&fixup, ftr_fixup_prefix3_alt); 981 fixup.alt_end_off = calc_offset(&fixup, ftr_fixup_prefix3_alt + 3); 982 /* Sanity check */ 983 check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) == 0); 984 985 patch_feature_section(0, &fixup); 986 check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_expected, size) == 0); 987 patch_feature_section(0, &fixup); 988 check(memcmp(ftr_fixup_prefix3, ftr_fixup_prefix3_orig, size) != 0); 989 } 990 #else 991 static inline void test_prefix_patching(void) {} 992 static inline void test_prefix_alt_patching(void) {} 993 static inline void test_prefix_word_alt_patching(void) {} 994 #endif /* CONFIG_PPC64 */ 995 996 static int __init test_feature_fixups(void) 997 { 998 printk(KERN_DEBUG "Running feature fixup self-tests ...\n"); 999 1000 test_basic_patching(); 1001 test_alternative_patching(); 1002 test_alternative_case_too_big(); 1003 test_alternative_case_too_small(); 1004 test_alternative_case_with_branch(); 1005 test_alternative_case_with_external_branch(); 1006 test_alternative_case_with_branch_to_end(); 1007 test_cpu_macros(); 1008 test_fw_macros(); 1009 test_lwsync_macros(); 1010 test_prefix_patching(); 1011 test_prefix_alt_patching(); 1012 test_prefix_word_alt_patching(); 1013 1014 return 0; 1015 } 1016 late_initcall(test_feature_fixups); 1017 1018 #endif /* CONFIG_FTR_FIXUP_SELFTEST */ 1019