1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2008 Michael Ellerman, IBM Corporation. 4 */ 5 6 #include <linux/kernel.h> 7 #include <linux/kprobes.h> 8 #include <linux/vmalloc.h> 9 #include <linux/init.h> 10 #include <linux/mm.h> 11 #include <linux/cpuhotplug.h> 12 #include <linux/slab.h> 13 #include <linux/uaccess.h> 14 15 #include <asm/pgtable.h> 16 #include <asm/tlbflush.h> 17 #include <asm/page.h> 18 #include <asm/code-patching.h> 19 #include <asm/setup.h> 20 21 static int __patch_instruction(unsigned int *exec_addr, unsigned int instr, 22 unsigned int *patch_addr) 23 { 24 int err = 0; 25 26 __put_user_asm(instr, patch_addr, err, "stw"); 27 if (err) 28 return err; 29 30 asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr), 31 "r" (exec_addr)); 32 33 return 0; 34 } 35 36 int raw_patch_instruction(unsigned int *addr, unsigned int instr) 37 { 38 return __patch_instruction(addr, instr, addr); 39 } 40 41 #ifdef CONFIG_STRICT_KERNEL_RWX 42 static DEFINE_PER_CPU(struct vm_struct *, text_poke_area); 43 44 static int text_area_cpu_up(unsigned int cpu) 45 { 46 struct vm_struct *area; 47 48 area = get_vm_area(PAGE_SIZE, VM_ALLOC); 49 if (!area) { 50 WARN_ONCE(1, "Failed to create text area for cpu %d\n", 51 cpu); 52 return -1; 53 } 54 this_cpu_write(text_poke_area, area); 55 56 return 0; 57 } 58 59 static int text_area_cpu_down(unsigned int cpu) 60 { 61 free_vm_area(this_cpu_read(text_poke_area)); 62 return 0; 63 } 64 65 /* 66 * Run as a late init call. This allows all the boot time patching to be done 67 * simply by patching the code, and then we're called here prior to 68 * mark_rodata_ro(), which happens after all init calls are run. Although 69 * BUG_ON() is rude, in this case it should only happen if ENOMEM, and we judge 70 * it as being preferable to a kernel that will crash later when someone tries 71 * to use patch_instruction(). 72 */ 73 static int __init setup_text_poke_area(void) 74 { 75 BUG_ON(!cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, 76 "powerpc/text_poke:online", text_area_cpu_up, 77 text_area_cpu_down)); 78 79 return 0; 80 } 81 late_initcall(setup_text_poke_area); 82 83 /* 84 * This can be called for kernel text or a module. 85 */ 86 static int map_patch_area(void *addr, unsigned long text_poke_addr) 87 { 88 unsigned long pfn; 89 int err; 90 91 if (is_vmalloc_addr(addr)) 92 pfn = vmalloc_to_pfn(addr); 93 else 94 pfn = __pa_symbol(addr) >> PAGE_SHIFT; 95 96 err = map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL); 97 98 pr_devel("Mapped addr %lx with pfn %lx:%d\n", text_poke_addr, pfn, err); 99 if (err) 100 return -1; 101 102 return 0; 103 } 104 105 static inline int unmap_patch_area(unsigned long addr) 106 { 107 pte_t *ptep; 108 pmd_t *pmdp; 109 pud_t *pudp; 110 pgd_t *pgdp; 111 112 pgdp = pgd_offset_k(addr); 113 if (unlikely(!pgdp)) 114 return -EINVAL; 115 116 pudp = pud_offset(pgdp, addr); 117 if (unlikely(!pudp)) 118 return -EINVAL; 119 120 pmdp = pmd_offset(pudp, addr); 121 if (unlikely(!pmdp)) 122 return -EINVAL; 123 124 ptep = pte_offset_kernel(pmdp, addr); 125 if (unlikely(!ptep)) 126 return -EINVAL; 127 128 pr_devel("clearing mm %p, pte %p, addr %lx\n", &init_mm, ptep, addr); 129 130 /* 131 * In hash, pte_clear flushes the tlb, in radix, we have to 132 */ 133 pte_clear(&init_mm, addr, ptep); 134 flush_tlb_kernel_range(addr, addr + PAGE_SIZE); 135 136 return 0; 137 } 138 139 static int do_patch_instruction(unsigned int *addr, unsigned int instr) 140 { 141 int err; 142 unsigned int *patch_addr = NULL; 143 unsigned long flags; 144 unsigned long text_poke_addr; 145 unsigned long kaddr = (unsigned long)addr; 146 147 /* 148 * During early early boot patch_instruction is called 149 * when text_poke_area is not ready, but we still need 150 * to allow patching. We just do the plain old patching 151 */ 152 if (!this_cpu_read(text_poke_area)) 153 return raw_patch_instruction(addr, instr); 154 155 local_irq_save(flags); 156 157 text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr; 158 if (map_patch_area(addr, text_poke_addr)) { 159 err = -1; 160 goto out; 161 } 162 163 patch_addr = (unsigned int *)(text_poke_addr) + 164 ((kaddr & ~PAGE_MASK) / sizeof(unsigned int)); 165 166 __patch_instruction(addr, instr, patch_addr); 167 168 err = unmap_patch_area(text_poke_addr); 169 if (err) 170 pr_warn("failed to unmap %lx\n", text_poke_addr); 171 172 out: 173 local_irq_restore(flags); 174 175 return err; 176 } 177 #else /* !CONFIG_STRICT_KERNEL_RWX */ 178 179 static int do_patch_instruction(unsigned int *addr, unsigned int instr) 180 { 181 return raw_patch_instruction(addr, instr); 182 } 183 184 #endif /* CONFIG_STRICT_KERNEL_RWX */ 185 186 int patch_instruction(unsigned int *addr, unsigned int instr) 187 { 188 /* Make sure we aren't patching a freed init section */ 189 if (init_mem_is_free && init_section_contains(addr, 4)) { 190 pr_debug("Skipping init section patching addr: 0x%px\n", addr); 191 return 0; 192 } 193 return do_patch_instruction(addr, instr); 194 } 195 NOKPROBE_SYMBOL(patch_instruction); 196 197 int patch_branch(unsigned int *addr, unsigned long target, int flags) 198 { 199 return patch_instruction(addr, create_branch(addr, target, flags)); 200 } 201 202 bool is_offset_in_branch_range(long offset) 203 { 204 /* 205 * Powerpc branch instruction is : 206 * 207 * 0 6 30 31 208 * +---------+----------------+---+---+ 209 * | opcode | LI |AA |LK | 210 * +---------+----------------+---+---+ 211 * Where AA = 0 and LK = 0 212 * 213 * LI is a signed 24 bits integer. The real branch offset is computed 214 * by: imm32 = SignExtend(LI:'0b00', 32); 215 * 216 * So the maximum forward branch should be: 217 * (0x007fffff << 2) = 0x01fffffc = 0x1fffffc 218 * The maximum backward branch should be: 219 * (0xff800000 << 2) = 0xfe000000 = -0x2000000 220 */ 221 return (offset >= -0x2000000 && offset <= 0x1fffffc && !(offset & 0x3)); 222 } 223 224 /* 225 * Helper to check if a given instruction is a conditional branch 226 * Derived from the conditional checks in analyse_instr() 227 */ 228 bool is_conditional_branch(unsigned int instr) 229 { 230 unsigned int opcode = instr >> 26; 231 232 if (opcode == 16) /* bc, bca, bcl, bcla */ 233 return true; 234 if (opcode == 19) { 235 switch ((instr >> 1) & 0x3ff) { 236 case 16: /* bclr, bclrl */ 237 case 528: /* bcctr, bcctrl */ 238 case 560: /* bctar, bctarl */ 239 return true; 240 } 241 } 242 return false; 243 } 244 NOKPROBE_SYMBOL(is_conditional_branch); 245 246 unsigned int create_branch(const unsigned int *addr, 247 unsigned long target, int flags) 248 { 249 unsigned int instruction; 250 long offset; 251 252 offset = target; 253 if (! (flags & BRANCH_ABSOLUTE)) 254 offset = offset - (unsigned long)addr; 255 256 /* Check we can represent the target in the instruction format */ 257 if (!is_offset_in_branch_range(offset)) 258 return 0; 259 260 /* Mask out the flags and target, so they don't step on each other. */ 261 instruction = 0x48000000 | (flags & 0x3) | (offset & 0x03FFFFFC); 262 263 return instruction; 264 } 265 266 unsigned int create_cond_branch(const unsigned int *addr, 267 unsigned long target, int flags) 268 { 269 unsigned int instruction; 270 long offset; 271 272 offset = target; 273 if (! (flags & BRANCH_ABSOLUTE)) 274 offset = offset - (unsigned long)addr; 275 276 /* Check we can represent the target in the instruction format */ 277 if (offset < -0x8000 || offset > 0x7FFF || offset & 0x3) 278 return 0; 279 280 /* Mask out the flags and target, so they don't step on each other. */ 281 instruction = 0x40000000 | (flags & 0x3FF0003) | (offset & 0xFFFC); 282 283 return instruction; 284 } 285 286 static unsigned int branch_opcode(unsigned int instr) 287 { 288 return (instr >> 26) & 0x3F; 289 } 290 291 static int instr_is_branch_iform(unsigned int instr) 292 { 293 return branch_opcode(instr) == 18; 294 } 295 296 static int instr_is_branch_bform(unsigned int instr) 297 { 298 return branch_opcode(instr) == 16; 299 } 300 301 int instr_is_relative_branch(unsigned int instr) 302 { 303 if (instr & BRANCH_ABSOLUTE) 304 return 0; 305 306 return instr_is_branch_iform(instr) || instr_is_branch_bform(instr); 307 } 308 309 int instr_is_relative_link_branch(unsigned int instr) 310 { 311 return instr_is_relative_branch(instr) && (instr & BRANCH_SET_LINK); 312 } 313 314 static unsigned long branch_iform_target(const unsigned int *instr) 315 { 316 signed long imm; 317 318 imm = *instr & 0x3FFFFFC; 319 320 /* If the top bit of the immediate value is set this is negative */ 321 if (imm & 0x2000000) 322 imm -= 0x4000000; 323 324 if ((*instr & BRANCH_ABSOLUTE) == 0) 325 imm += (unsigned long)instr; 326 327 return (unsigned long)imm; 328 } 329 330 static unsigned long branch_bform_target(const unsigned int *instr) 331 { 332 signed long imm; 333 334 imm = *instr & 0xFFFC; 335 336 /* If the top bit of the immediate value is set this is negative */ 337 if (imm & 0x8000) 338 imm -= 0x10000; 339 340 if ((*instr & BRANCH_ABSOLUTE) == 0) 341 imm += (unsigned long)instr; 342 343 return (unsigned long)imm; 344 } 345 346 unsigned long branch_target(const unsigned int *instr) 347 { 348 if (instr_is_branch_iform(*instr)) 349 return branch_iform_target(instr); 350 else if (instr_is_branch_bform(*instr)) 351 return branch_bform_target(instr); 352 353 return 0; 354 } 355 356 int instr_is_branch_to_addr(const unsigned int *instr, unsigned long addr) 357 { 358 if (instr_is_branch_iform(*instr) || instr_is_branch_bform(*instr)) 359 return branch_target(instr) == addr; 360 361 return 0; 362 } 363 364 unsigned int translate_branch(const unsigned int *dest, const unsigned int *src) 365 { 366 unsigned long target; 367 368 target = branch_target(src); 369 370 if (instr_is_branch_iform(*src)) 371 return create_branch(dest, target, *src); 372 else if (instr_is_branch_bform(*src)) 373 return create_cond_branch(dest, target, *src); 374 375 return 0; 376 } 377 378 #ifdef CONFIG_PPC_BOOK3E_64 379 void __patch_exception(int exc, unsigned long addr) 380 { 381 extern unsigned int interrupt_base_book3e; 382 unsigned int *ibase = &interrupt_base_book3e; 383 384 /* Our exceptions vectors start with a NOP and -then- a branch 385 * to deal with single stepping from userspace which stops on 386 * the second instruction. Thus we need to patch the second 387 * instruction of the exception, not the first one 388 */ 389 390 patch_branch(ibase + (exc / 4) + 1, addr, 0); 391 } 392 #endif 393 394 #ifdef CONFIG_CODE_PATCHING_SELFTEST 395 396 static void __init test_trampoline(void) 397 { 398 asm ("nop;\n"); 399 } 400 401 #define check(x) \ 402 if (!(x)) printk("code-patching: test failed at line %d\n", __LINE__); 403 404 static void __init test_branch_iform(void) 405 { 406 unsigned int instr; 407 unsigned long addr; 408 409 addr = (unsigned long)&instr; 410 411 /* The simplest case, branch to self, no flags */ 412 check(instr_is_branch_iform(0x48000000)); 413 /* All bits of target set, and flags */ 414 check(instr_is_branch_iform(0x4bffffff)); 415 /* High bit of opcode set, which is wrong */ 416 check(!instr_is_branch_iform(0xcbffffff)); 417 /* Middle bits of opcode set, which is wrong */ 418 check(!instr_is_branch_iform(0x7bffffff)); 419 420 /* Simplest case, branch to self with link */ 421 check(instr_is_branch_iform(0x48000001)); 422 /* All bits of targets set */ 423 check(instr_is_branch_iform(0x4bfffffd)); 424 /* Some bits of targets set */ 425 check(instr_is_branch_iform(0x4bff00fd)); 426 /* Must be a valid branch to start with */ 427 check(!instr_is_branch_iform(0x7bfffffd)); 428 429 /* Absolute branch to 0x100 */ 430 instr = 0x48000103; 431 check(instr_is_branch_to_addr(&instr, 0x100)); 432 /* Absolute branch to 0x420fc */ 433 instr = 0x480420ff; 434 check(instr_is_branch_to_addr(&instr, 0x420fc)); 435 /* Maximum positive relative branch, + 20MB - 4B */ 436 instr = 0x49fffffc; 437 check(instr_is_branch_to_addr(&instr, addr + 0x1FFFFFC)); 438 /* Smallest negative relative branch, - 4B */ 439 instr = 0x4bfffffc; 440 check(instr_is_branch_to_addr(&instr, addr - 4)); 441 /* Largest negative relative branch, - 32 MB */ 442 instr = 0x4a000000; 443 check(instr_is_branch_to_addr(&instr, addr - 0x2000000)); 444 445 /* Branch to self, with link */ 446 instr = create_branch(&instr, addr, BRANCH_SET_LINK); 447 check(instr_is_branch_to_addr(&instr, addr)); 448 449 /* Branch to self - 0x100, with link */ 450 instr = create_branch(&instr, addr - 0x100, BRANCH_SET_LINK); 451 check(instr_is_branch_to_addr(&instr, addr - 0x100)); 452 453 /* Branch to self + 0x100, no link */ 454 instr = create_branch(&instr, addr + 0x100, 0); 455 check(instr_is_branch_to_addr(&instr, addr + 0x100)); 456 457 /* Maximum relative negative offset, - 32 MB */ 458 instr = create_branch(&instr, addr - 0x2000000, BRANCH_SET_LINK); 459 check(instr_is_branch_to_addr(&instr, addr - 0x2000000)); 460 461 /* Out of range relative negative offset, - 32 MB + 4*/ 462 instr = create_branch(&instr, addr - 0x2000004, BRANCH_SET_LINK); 463 check(instr == 0); 464 465 /* Out of range relative positive offset, + 32 MB */ 466 instr = create_branch(&instr, addr + 0x2000000, BRANCH_SET_LINK); 467 check(instr == 0); 468 469 /* Unaligned target */ 470 instr = create_branch(&instr, addr + 3, BRANCH_SET_LINK); 471 check(instr == 0); 472 473 /* Check flags are masked correctly */ 474 instr = create_branch(&instr, addr, 0xFFFFFFFC); 475 check(instr_is_branch_to_addr(&instr, addr)); 476 check(instr == 0x48000000); 477 } 478 479 static void __init test_create_function_call(void) 480 { 481 unsigned int *iptr; 482 unsigned long dest; 483 484 /* Check we can create a function call */ 485 iptr = (unsigned int *)ppc_function_entry(test_trampoline); 486 dest = ppc_function_entry(test_create_function_call); 487 patch_instruction(iptr, create_branch(iptr, dest, BRANCH_SET_LINK)); 488 check(instr_is_branch_to_addr(iptr, dest)); 489 } 490 491 static void __init test_branch_bform(void) 492 { 493 unsigned long addr; 494 unsigned int *iptr, instr, flags; 495 496 iptr = &instr; 497 addr = (unsigned long)iptr; 498 499 /* The simplest case, branch to self, no flags */ 500 check(instr_is_branch_bform(0x40000000)); 501 /* All bits of target set, and flags */ 502 check(instr_is_branch_bform(0x43ffffff)); 503 /* High bit of opcode set, which is wrong */ 504 check(!instr_is_branch_bform(0xc3ffffff)); 505 /* Middle bits of opcode set, which is wrong */ 506 check(!instr_is_branch_bform(0x7bffffff)); 507 508 /* Absolute conditional branch to 0x100 */ 509 instr = 0x43ff0103; 510 check(instr_is_branch_to_addr(&instr, 0x100)); 511 /* Absolute conditional branch to 0x20fc */ 512 instr = 0x43ff20ff; 513 check(instr_is_branch_to_addr(&instr, 0x20fc)); 514 /* Maximum positive relative conditional branch, + 32 KB - 4B */ 515 instr = 0x43ff7ffc; 516 check(instr_is_branch_to_addr(&instr, addr + 0x7FFC)); 517 /* Smallest negative relative conditional branch, - 4B */ 518 instr = 0x43fffffc; 519 check(instr_is_branch_to_addr(&instr, addr - 4)); 520 /* Largest negative relative conditional branch, - 32 KB */ 521 instr = 0x43ff8000; 522 check(instr_is_branch_to_addr(&instr, addr - 0x8000)); 523 524 /* All condition code bits set & link */ 525 flags = 0x3ff000 | BRANCH_SET_LINK; 526 527 /* Branch to self */ 528 instr = create_cond_branch(iptr, addr, flags); 529 check(instr_is_branch_to_addr(&instr, addr)); 530 531 /* Branch to self - 0x100 */ 532 instr = create_cond_branch(iptr, addr - 0x100, flags); 533 check(instr_is_branch_to_addr(&instr, addr - 0x100)); 534 535 /* Branch to self + 0x100 */ 536 instr = create_cond_branch(iptr, addr + 0x100, flags); 537 check(instr_is_branch_to_addr(&instr, addr + 0x100)); 538 539 /* Maximum relative negative offset, - 32 KB */ 540 instr = create_cond_branch(iptr, addr - 0x8000, flags); 541 check(instr_is_branch_to_addr(&instr, addr - 0x8000)); 542 543 /* Out of range relative negative offset, - 32 KB + 4*/ 544 instr = create_cond_branch(iptr, addr - 0x8004, flags); 545 check(instr == 0); 546 547 /* Out of range relative positive offset, + 32 KB */ 548 instr = create_cond_branch(iptr, addr + 0x8000, flags); 549 check(instr == 0); 550 551 /* Unaligned target */ 552 instr = create_cond_branch(iptr, addr + 3, flags); 553 check(instr == 0); 554 555 /* Check flags are masked correctly */ 556 instr = create_cond_branch(iptr, addr, 0xFFFFFFFC); 557 check(instr_is_branch_to_addr(&instr, addr)); 558 check(instr == 0x43FF0000); 559 } 560 561 static void __init test_translate_branch(void) 562 { 563 unsigned long addr; 564 unsigned int *p, *q; 565 void *buf; 566 567 buf = vmalloc(PAGE_ALIGN(0x2000000 + 1)); 568 check(buf); 569 if (!buf) 570 return; 571 572 /* Simple case, branch to self moved a little */ 573 p = buf; 574 addr = (unsigned long)p; 575 patch_branch(p, addr, 0); 576 check(instr_is_branch_to_addr(p, addr)); 577 q = p + 1; 578 patch_instruction(q, translate_branch(q, p)); 579 check(instr_is_branch_to_addr(q, addr)); 580 581 /* Maximum negative case, move b . to addr + 32 MB */ 582 p = buf; 583 addr = (unsigned long)p; 584 patch_branch(p, addr, 0); 585 q = buf + 0x2000000; 586 patch_instruction(q, translate_branch(q, p)); 587 check(instr_is_branch_to_addr(p, addr)); 588 check(instr_is_branch_to_addr(q, addr)); 589 check(*q == 0x4a000000); 590 591 /* Maximum positive case, move x to x - 32 MB + 4 */ 592 p = buf + 0x2000000; 593 addr = (unsigned long)p; 594 patch_branch(p, addr, 0); 595 q = buf + 4; 596 patch_instruction(q, translate_branch(q, p)); 597 check(instr_is_branch_to_addr(p, addr)); 598 check(instr_is_branch_to_addr(q, addr)); 599 check(*q == 0x49fffffc); 600 601 /* Jump to x + 16 MB moved to x + 20 MB */ 602 p = buf; 603 addr = 0x1000000 + (unsigned long)buf; 604 patch_branch(p, addr, BRANCH_SET_LINK); 605 q = buf + 0x1400000; 606 patch_instruction(q, translate_branch(q, p)); 607 check(instr_is_branch_to_addr(p, addr)); 608 check(instr_is_branch_to_addr(q, addr)); 609 610 /* Jump to x + 16 MB moved to x - 16 MB + 4 */ 611 p = buf + 0x1000000; 612 addr = 0x2000000 + (unsigned long)buf; 613 patch_branch(p, addr, 0); 614 q = buf + 4; 615 patch_instruction(q, translate_branch(q, p)); 616 check(instr_is_branch_to_addr(p, addr)); 617 check(instr_is_branch_to_addr(q, addr)); 618 619 620 /* Conditional branch tests */ 621 622 /* Simple case, branch to self moved a little */ 623 p = buf; 624 addr = (unsigned long)p; 625 patch_instruction(p, create_cond_branch(p, addr, 0)); 626 check(instr_is_branch_to_addr(p, addr)); 627 q = p + 1; 628 patch_instruction(q, translate_branch(q, p)); 629 check(instr_is_branch_to_addr(q, addr)); 630 631 /* Maximum negative case, move b . to addr + 32 KB */ 632 p = buf; 633 addr = (unsigned long)p; 634 patch_instruction(p, create_cond_branch(p, addr, 0xFFFFFFFC)); 635 q = buf + 0x8000; 636 patch_instruction(q, translate_branch(q, p)); 637 check(instr_is_branch_to_addr(p, addr)); 638 check(instr_is_branch_to_addr(q, addr)); 639 check(*q == 0x43ff8000); 640 641 /* Maximum positive case, move x to x - 32 KB + 4 */ 642 p = buf + 0x8000; 643 addr = (unsigned long)p; 644 patch_instruction(p, create_cond_branch(p, addr, 0xFFFFFFFC)); 645 q = buf + 4; 646 patch_instruction(q, translate_branch(q, p)); 647 check(instr_is_branch_to_addr(p, addr)); 648 check(instr_is_branch_to_addr(q, addr)); 649 check(*q == 0x43ff7ffc); 650 651 /* Jump to x + 12 KB moved to x + 20 KB */ 652 p = buf; 653 addr = 0x3000 + (unsigned long)buf; 654 patch_instruction(p, create_cond_branch(p, addr, BRANCH_SET_LINK)); 655 q = buf + 0x5000; 656 patch_instruction(q, translate_branch(q, p)); 657 check(instr_is_branch_to_addr(p, addr)); 658 check(instr_is_branch_to_addr(q, addr)); 659 660 /* Jump to x + 8 KB moved to x - 8 KB + 4 */ 661 p = buf + 0x2000; 662 addr = 0x4000 + (unsigned long)buf; 663 patch_instruction(p, create_cond_branch(p, addr, 0)); 664 q = buf + 4; 665 patch_instruction(q, translate_branch(q, p)); 666 check(instr_is_branch_to_addr(p, addr)); 667 check(instr_is_branch_to_addr(q, addr)); 668 669 /* Free the buffer we were using */ 670 vfree(buf); 671 } 672 673 static int __init test_code_patching(void) 674 { 675 printk(KERN_DEBUG "Running code patching self-tests ...\n"); 676 677 test_branch_iform(); 678 test_branch_bform(); 679 test_create_function_call(); 680 test_translate_branch(); 681 682 return 0; 683 } 684 late_initcall(test_code_patching); 685 686 #endif /* CONFIG_CODE_PATCHING_SELFTEST */ 687