1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * 7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 8 * 9 * Added function graph tracer code, taken from x86 that was written 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 11 * 12 */ 13 14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt 15 16 #include <linux/spinlock.h> 17 #include <linux/hardirq.h> 18 #include <linux/uaccess.h> 19 #include <linux/module.h> 20 #include <linux/ftrace.h> 21 #include <linux/percpu.h> 22 #include <linux/init.h> 23 #include <linux/list.h> 24 25 #include <asm/asm-prototypes.h> 26 #include <asm/cacheflush.h> 27 #include <asm/code-patching.h> 28 #include <asm/ftrace.h> 29 #include <asm/syscall.h> 30 #include <asm/inst.h> 31 32 33 #ifdef CONFIG_DYNAMIC_FTRACE 34 35 /* 36 * We generally only have a single long_branch tramp and at most 2 or 3 plt 37 * tramps generated. But, we don't use the plt tramps currently. We also allot 38 * 2 tramps after .text and .init.text. So, we only end up with around 3 usable 39 * tramps in total. Set aside 8 just to be sure. 40 */ 41 #define NUM_FTRACE_TRAMPS 8 42 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; 43 44 static struct ppc_inst 45 ftrace_call_replace(unsigned long ip, unsigned long addr, int link) 46 { 47 struct ppc_inst op; 48 49 addr = ppc_function_entry((void *)addr); 50 51 /* if (link) set op to 'bl' else 'b' */ 52 create_branch(&op, (struct ppc_inst *)ip, addr, link ? 1 : 0); 53 54 return op; 55 } 56 57 static int 58 ftrace_modify_code(unsigned long ip, struct ppc_inst old, struct ppc_inst new) 59 { 60 struct ppc_inst replaced; 61 62 /* 63 * Note: 64 * We are paranoid about modifying text, as if a bug was to happen, it 65 * could cause us to read or write to someplace that could cause harm. 66 * Carefully read and modify the code with probe_kernel_*(), and make 67 * sure what we read is what we expected it to be before modifying it. 68 */ 69 70 /* read the text we want to modify */ 71 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE)) 72 return -EFAULT; 73 74 /* Make sure it is what we expect it to be */ 75 if (!ppc_inst_equal(replaced, old)) { 76 pr_err("%p: replaced (%#x) != old (%#x)", 77 (void *)ip, ppc_inst_val(replaced), ppc_inst_val(old)); 78 return -EINVAL; 79 } 80 81 /* replace the text with the new text */ 82 if (patch_instruction((struct ppc_inst *)ip, new)) 83 return -EPERM; 84 85 return 0; 86 } 87 88 /* 89 * Helper functions that are the same for both PPC64 and PPC32. 90 */ 91 static int test_24bit_addr(unsigned long ip, unsigned long addr) 92 { 93 struct ppc_inst op; 94 addr = ppc_function_entry((void *)addr); 95 96 /* use the create_branch to verify that this offset can be branched */ 97 return create_branch(&op, (struct ppc_inst *)ip, addr, 0) == 0; 98 } 99 100 static int is_bl_op(struct ppc_inst op) 101 { 102 return (ppc_inst_val(op) & 0xfc000003) == 0x48000001; 103 } 104 105 static int is_b_op(struct ppc_inst op) 106 { 107 return (ppc_inst_val(op) & 0xfc000003) == 0x48000000; 108 } 109 110 static unsigned long find_bl_target(unsigned long ip, struct ppc_inst op) 111 { 112 int offset; 113 114 offset = (ppc_inst_val(op) & 0x03fffffc); 115 /* make it signed */ 116 if (offset & 0x02000000) 117 offset |= 0xfe000000; 118 119 return ip + (long)offset; 120 } 121 122 #ifdef CONFIG_MODULES 123 #ifdef CONFIG_PPC64 124 static int 125 __ftrace_make_nop(struct module *mod, 126 struct dyn_ftrace *rec, unsigned long addr) 127 { 128 unsigned long entry, ptr, tramp; 129 unsigned long ip = rec->ip; 130 struct ppc_inst op, pop; 131 132 /* read where this goes */ 133 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 134 pr_err("Fetching opcode failed.\n"); 135 return -EFAULT; 136 } 137 138 /* Make sure that that this is still a 24bit jump */ 139 if (!is_bl_op(op)) { 140 pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op)); 141 return -EINVAL; 142 } 143 144 /* lets find where the pointer goes */ 145 tramp = find_bl_target(ip, op); 146 147 pr_devel("ip:%lx jumps to %lx", ip, tramp); 148 149 if (module_trampoline_target(mod, tramp, &ptr)) { 150 pr_err("Failed to get trampoline target\n"); 151 return -EFAULT; 152 } 153 154 pr_devel("trampoline target %lx", ptr); 155 156 entry = ppc_global_function_entry((void *)addr); 157 /* This should match what was called */ 158 if (ptr != entry) { 159 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 160 return -EINVAL; 161 } 162 163 #ifdef CONFIG_MPROFILE_KERNEL 164 /* When using -mkernel_profile there is no load to jump over */ 165 pop = ppc_inst(PPC_INST_NOP); 166 167 if (probe_kernel_read(&op, (void *)(ip - 4), 4)) { 168 pr_err("Fetching instruction at %lx failed.\n", ip - 4); 169 return -EFAULT; 170 } 171 172 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */ 173 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_MFLR)) && 174 !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) { 175 pr_err("Unexpected instruction %08x around bl _mcount\n", 176 ppc_inst_val(op)); 177 return -EINVAL; 178 } 179 #else 180 /* 181 * Our original call site looks like: 182 * 183 * bl <tramp> 184 * ld r2,XX(r1) 185 * 186 * Milton Miller pointed out that we can not simply nop the branch. 187 * If a task was preempted when calling a trace function, the nops 188 * will remove the way to restore the TOC in r2 and the r2 TOC will 189 * get corrupted. 190 * 191 * Use a b +8 to jump over the load. 192 */ 193 194 pop = ppc_inst(PPC_INST_BRANCH | 8); /* b +8 */ 195 196 /* 197 * Check what is in the next instruction. We can see ld r2,40(r1), but 198 * on first pass after boot we will see mflr r0. 199 */ 200 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) { 201 pr_err("Fetching op failed.\n"); 202 return -EFAULT; 203 } 204 205 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_LD_TOC))) { 206 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, ppc_inst_val(op)); 207 return -EINVAL; 208 } 209 #endif /* CONFIG_MPROFILE_KERNEL */ 210 211 if (patch_instruction((struct ppc_inst *)ip, pop)) { 212 pr_err("Patching NOP failed.\n"); 213 return -EPERM; 214 } 215 216 return 0; 217 } 218 219 #else /* !PPC64 */ 220 static int 221 __ftrace_make_nop(struct module *mod, 222 struct dyn_ftrace *rec, unsigned long addr) 223 { 224 struct ppc_inst op; 225 unsigned int jmp[4]; 226 unsigned long ip = rec->ip; 227 unsigned long tramp; 228 229 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 230 return -EFAULT; 231 232 /* Make sure that that this is still a 24bit jump */ 233 if (!is_bl_op(op)) { 234 pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op)); 235 return -EINVAL; 236 } 237 238 /* lets find where the pointer goes */ 239 tramp = find_bl_target(ip, op); 240 241 /* 242 * On PPC32 the trampoline looks like: 243 * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha 244 * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l 245 * 0x7d, 0x89, 0x03, 0xa6 mtctr r12 246 * 0x4e, 0x80, 0x04, 0x20 bctr 247 */ 248 249 pr_devel("ip:%lx jumps to %lx", ip, tramp); 250 251 /* Find where the trampoline jumps to */ 252 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) { 253 pr_err("Failed to read %lx\n", tramp); 254 return -EFAULT; 255 } 256 257 pr_devel(" %08x %08x ", jmp[0], jmp[1]); 258 259 /* verify that this is what we expect it to be */ 260 if (((jmp[0] & 0xffff0000) != 0x3d800000) || 261 ((jmp[1] & 0xffff0000) != 0x398c0000) || 262 (jmp[2] != 0x7d8903a6) || 263 (jmp[3] != 0x4e800420)) { 264 pr_err("Not a trampoline\n"); 265 return -EINVAL; 266 } 267 268 tramp = (jmp[1] & 0xffff) | 269 ((jmp[0] & 0xffff) << 16); 270 if (tramp & 0x8000) 271 tramp -= 0x10000; 272 273 pr_devel(" %lx ", tramp); 274 275 if (tramp != addr) { 276 pr_err("Trampoline location %08lx does not match addr\n", 277 tramp); 278 return -EINVAL; 279 } 280 281 op = ppc_inst(PPC_INST_NOP); 282 283 if (patch_instruction((struct ppc_inst *)ip, op)) 284 return -EPERM; 285 286 return 0; 287 } 288 #endif /* PPC64 */ 289 #endif /* CONFIG_MODULES */ 290 291 static unsigned long find_ftrace_tramp(unsigned long ip) 292 { 293 int i; 294 struct ppc_inst instr; 295 296 /* 297 * We have the compiler generated long_branch tramps at the end 298 * and we prefer those 299 */ 300 for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--) 301 if (!ftrace_tramps[i]) 302 continue; 303 else if (create_branch(&instr, (void *)ip, 304 ftrace_tramps[i], 0) == 0) 305 return ftrace_tramps[i]; 306 307 return 0; 308 } 309 310 static int add_ftrace_tramp(unsigned long tramp) 311 { 312 int i; 313 314 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 315 if (!ftrace_tramps[i]) { 316 ftrace_tramps[i] = tramp; 317 return 0; 318 } 319 320 return -1; 321 } 322 323 /* 324 * If this is a compiler generated long_branch trampoline (essentially, a 325 * trampoline that has a branch to _mcount()), we re-write the branch to 326 * instead go to ftrace_[regs_]caller() and note down the location of this 327 * trampoline. 328 */ 329 static int setup_mcount_compiler_tramp(unsigned long tramp) 330 { 331 int i; 332 struct ppc_inst op; 333 unsigned long ptr; 334 struct ppc_inst instr; 335 static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS]; 336 337 /* Is this a known long jump tramp? */ 338 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 339 if (!ftrace_tramps[i]) 340 break; 341 else if (ftrace_tramps[i] == tramp) 342 return 0; 343 344 /* Is this a known plt tramp? */ 345 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 346 if (!ftrace_plt_tramps[i]) 347 break; 348 else if (ftrace_plt_tramps[i] == tramp) 349 return -1; 350 351 /* New trampoline -- read where this goes */ 352 if (probe_kernel_read(&op, (void *)tramp, sizeof(int))) { 353 pr_debug("Fetching opcode failed.\n"); 354 return -1; 355 } 356 357 /* Is this a 24 bit branch? */ 358 if (!is_b_op(op)) { 359 pr_debug("Trampoline is not a long branch tramp.\n"); 360 return -1; 361 } 362 363 /* lets find where the pointer goes */ 364 ptr = find_bl_target(tramp, op); 365 366 if (ptr != ppc_global_function_entry((void *)_mcount)) { 367 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr); 368 return -1; 369 } 370 371 /* Let's re-write the tramp to go to ftrace_[regs_]caller */ 372 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 373 ptr = ppc_global_function_entry((void *)ftrace_regs_caller); 374 #else 375 ptr = ppc_global_function_entry((void *)ftrace_caller); 376 #endif 377 if (create_branch(&instr, (void *)tramp, ptr, 0)) { 378 pr_debug("%ps is not reachable from existing mcount tramp\n", 379 (void *)ptr); 380 return -1; 381 } 382 383 if (patch_branch((struct ppc_inst *)tramp, ptr, 0)) { 384 pr_debug("REL24 out of range!\n"); 385 return -1; 386 } 387 388 if (add_ftrace_tramp(tramp)) { 389 pr_debug("No tramp locations left\n"); 390 return -1; 391 } 392 393 return 0; 394 } 395 396 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) 397 { 398 unsigned long tramp, ip = rec->ip; 399 struct ppc_inst op; 400 401 /* Read where this goes */ 402 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 403 pr_err("Fetching opcode failed.\n"); 404 return -EFAULT; 405 } 406 407 /* Make sure that that this is still a 24bit jump */ 408 if (!is_bl_op(op)) { 409 pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op)); 410 return -EINVAL; 411 } 412 413 /* Let's find where the pointer goes */ 414 tramp = find_bl_target(ip, op); 415 416 pr_devel("ip:%lx jumps to %lx", ip, tramp); 417 418 if (setup_mcount_compiler_tramp(tramp)) { 419 /* Are other trampolines reachable? */ 420 if (!find_ftrace_tramp(ip)) { 421 pr_err("No ftrace trampolines reachable from %ps\n", 422 (void *)ip); 423 return -EINVAL; 424 } 425 } 426 427 if (patch_instruction((struct ppc_inst *)ip, ppc_inst(PPC_INST_NOP))) { 428 pr_err("Patching NOP failed.\n"); 429 return -EPERM; 430 } 431 432 return 0; 433 } 434 435 int ftrace_make_nop(struct module *mod, 436 struct dyn_ftrace *rec, unsigned long addr) 437 { 438 unsigned long ip = rec->ip; 439 struct ppc_inst old, new; 440 441 /* 442 * If the calling address is more that 24 bits away, 443 * then we had to use a trampoline to make the call. 444 * Otherwise just update the call site. 445 */ 446 if (test_24bit_addr(ip, addr)) { 447 /* within range */ 448 old = ftrace_call_replace(ip, addr, 1); 449 new = ppc_inst(PPC_INST_NOP); 450 return ftrace_modify_code(ip, old, new); 451 } else if (core_kernel_text(ip)) 452 return __ftrace_make_nop_kernel(rec, addr); 453 454 #ifdef CONFIG_MODULES 455 /* 456 * Out of range jumps are called from modules. 457 * We should either already have a pointer to the module 458 * or it has been passed in. 459 */ 460 if (!rec->arch.mod) { 461 if (!mod) { 462 pr_err("No module loaded addr=%lx\n", addr); 463 return -EFAULT; 464 } 465 rec->arch.mod = mod; 466 } else if (mod) { 467 if (mod != rec->arch.mod) { 468 pr_err("Record mod %p not equal to passed in mod %p\n", 469 rec->arch.mod, mod); 470 return -EINVAL; 471 } 472 /* nothing to do if mod == rec->arch.mod */ 473 } else 474 mod = rec->arch.mod; 475 476 return __ftrace_make_nop(mod, rec, addr); 477 #else 478 /* We should not get here without modules */ 479 return -EINVAL; 480 #endif /* CONFIG_MODULES */ 481 } 482 483 #ifdef CONFIG_MODULES 484 #ifdef CONFIG_PPC64 485 /* 486 * Examine the existing instructions for __ftrace_make_call. 487 * They should effectively be a NOP, and follow formal constraints, 488 * depending on the ABI. Return false if they don't. 489 */ 490 #ifndef CONFIG_MPROFILE_KERNEL 491 static int 492 expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1) 493 { 494 /* 495 * We expect to see: 496 * 497 * b +8 498 * ld r2,XX(r1) 499 * 500 * The load offset is different depending on the ABI. For simplicity 501 * just mask it out when doing the compare. 502 */ 503 if (!ppc_inst_equal(op0, ppc_inst(0x48000008)) || 504 (ppc_inst_val(op1) & 0xffff0000) != 0xe8410000) 505 return 0; 506 return 1; 507 } 508 #else 509 static int 510 expected_nop_sequence(void *ip, struct ppc_inst op0, struct ppc_inst op1) 511 { 512 /* look for patched "NOP" on ppc64 with -mprofile-kernel */ 513 if (!ppc_inst_equal(op0, ppc_inst(PPC_INST_NOP))) 514 return 0; 515 return 1; 516 } 517 #endif 518 519 static int 520 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 521 { 522 struct ppc_inst op[2]; 523 struct ppc_inst instr; 524 void *ip = (void *)rec->ip; 525 unsigned long entry, ptr, tramp; 526 struct module *mod = rec->arch.mod; 527 528 /* read where this goes */ 529 if (probe_kernel_read(op, ip, sizeof(op))) 530 return -EFAULT; 531 532 if (!expected_nop_sequence(ip, op[0], op[1])) { 533 pr_err("Unexpected call sequence at %p: %x %x\n", 534 ip, ppc_inst_val(op[0]), ppc_inst_val(op[1])); 535 return -EINVAL; 536 } 537 538 /* If we never set up ftrace trampoline(s), then bail */ 539 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 540 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 541 #else 542 if (!mod->arch.tramp) { 543 #endif 544 pr_err("No ftrace trampoline\n"); 545 return -EINVAL; 546 } 547 548 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 549 if (rec->flags & FTRACE_FL_REGS) 550 tramp = mod->arch.tramp_regs; 551 else 552 #endif 553 tramp = mod->arch.tramp; 554 555 if (module_trampoline_target(mod, tramp, &ptr)) { 556 pr_err("Failed to get trampoline target\n"); 557 return -EFAULT; 558 } 559 560 pr_devel("trampoline target %lx", ptr); 561 562 entry = ppc_global_function_entry((void *)addr); 563 /* This should match what was called */ 564 if (ptr != entry) { 565 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 566 return -EINVAL; 567 } 568 569 /* Ensure branch is within 24 bits */ 570 if (create_branch(&instr, ip, tramp, BRANCH_SET_LINK)) { 571 pr_err("Branch out of range\n"); 572 return -EINVAL; 573 } 574 575 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 576 pr_err("REL24 out of range!\n"); 577 return -EINVAL; 578 } 579 580 return 0; 581 } 582 583 #else /* !CONFIG_PPC64: */ 584 static int 585 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 586 { 587 int err; 588 struct ppc_inst op; 589 unsigned long ip = rec->ip; 590 591 /* read where this goes */ 592 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 593 return -EFAULT; 594 595 /* It should be pointing to a nop */ 596 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_NOP))) { 597 pr_err("Expected NOP but have %x\n", ppc_inst_val(op)); 598 return -EINVAL; 599 } 600 601 /* If we never set up a trampoline to ftrace_caller, then bail */ 602 if (!rec->arch.mod->arch.tramp) { 603 pr_err("No ftrace trampoline\n"); 604 return -EINVAL; 605 } 606 607 /* create the branch to the trampoline */ 608 err = create_branch(&op, (struct ppc_inst *)ip, 609 rec->arch.mod->arch.tramp, BRANCH_SET_LINK); 610 if (err) { 611 pr_err("REL24 out of range!\n"); 612 return -EINVAL; 613 } 614 615 pr_devel("write to %lx\n", rec->ip); 616 617 if (patch_instruction((struct ppc_inst *)ip, op)) 618 return -EPERM; 619 620 return 0; 621 } 622 #endif /* CONFIG_PPC64 */ 623 #endif /* CONFIG_MODULES */ 624 625 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) 626 { 627 struct ppc_inst op; 628 void *ip = (void *)rec->ip; 629 unsigned long tramp, entry, ptr; 630 631 /* Make sure we're being asked to patch branch to a known ftrace addr */ 632 entry = ppc_global_function_entry((void *)ftrace_caller); 633 ptr = ppc_global_function_entry((void *)addr); 634 635 if (ptr != entry) { 636 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 637 entry = ppc_global_function_entry((void *)ftrace_regs_caller); 638 if (ptr != entry) { 639 #endif 640 pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr); 641 return -EINVAL; 642 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 643 } 644 #endif 645 } 646 647 /* Make sure we have a nop */ 648 if (probe_kernel_read(&op, ip, sizeof(op))) { 649 pr_err("Unable to read ftrace location %p\n", ip); 650 return -EFAULT; 651 } 652 653 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_NOP))) { 654 pr_err("Unexpected call sequence at %p: %x\n", ip, ppc_inst_val(op)); 655 return -EINVAL; 656 } 657 658 tramp = find_ftrace_tramp((unsigned long)ip); 659 if (!tramp) { 660 pr_err("No ftrace trampolines reachable from %ps\n", ip); 661 return -EINVAL; 662 } 663 664 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 665 pr_err("Error patching branch to ftrace tramp!\n"); 666 return -EINVAL; 667 } 668 669 return 0; 670 } 671 672 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 673 { 674 unsigned long ip = rec->ip; 675 struct ppc_inst old, new; 676 677 /* 678 * If the calling address is more that 24 bits away, 679 * then we had to use a trampoline to make the call. 680 * Otherwise just update the call site. 681 */ 682 if (test_24bit_addr(ip, addr)) { 683 /* within range */ 684 old = ppc_inst(PPC_INST_NOP); 685 new = ftrace_call_replace(ip, addr, 1); 686 return ftrace_modify_code(ip, old, new); 687 } else if (core_kernel_text(ip)) 688 return __ftrace_make_call_kernel(rec, addr); 689 690 #ifdef CONFIG_MODULES 691 /* 692 * Out of range jumps are called from modules. 693 * Being that we are converting from nop, it had better 694 * already have a module defined. 695 */ 696 if (!rec->arch.mod) { 697 pr_err("No module loaded\n"); 698 return -EINVAL; 699 } 700 701 return __ftrace_make_call(rec, addr); 702 #else 703 /* We should not get here without modules */ 704 return -EINVAL; 705 #endif /* CONFIG_MODULES */ 706 } 707 708 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 709 #ifdef CONFIG_MODULES 710 static int 711 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 712 unsigned long addr) 713 { 714 struct ppc_inst op; 715 unsigned long ip = rec->ip; 716 unsigned long entry, ptr, tramp; 717 struct module *mod = rec->arch.mod; 718 719 /* If we never set up ftrace trampolines, then bail */ 720 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 721 pr_err("No ftrace trampoline\n"); 722 return -EINVAL; 723 } 724 725 /* read where this goes */ 726 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 727 pr_err("Fetching opcode failed.\n"); 728 return -EFAULT; 729 } 730 731 /* Make sure that that this is still a 24bit jump */ 732 if (!is_bl_op(op)) { 733 pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op)); 734 return -EINVAL; 735 } 736 737 /* lets find where the pointer goes */ 738 tramp = find_bl_target(ip, op); 739 entry = ppc_global_function_entry((void *)old_addr); 740 741 pr_devel("ip:%lx jumps to %lx", ip, tramp); 742 743 if (tramp != entry) { 744 /* old_addr is not within range, so we must have used a trampoline */ 745 if (module_trampoline_target(mod, tramp, &ptr)) { 746 pr_err("Failed to get trampoline target\n"); 747 return -EFAULT; 748 } 749 750 pr_devel("trampoline target %lx", ptr); 751 752 /* This should match what was called */ 753 if (ptr != entry) { 754 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 755 return -EINVAL; 756 } 757 } 758 759 /* The new target may be within range */ 760 if (test_24bit_addr(ip, addr)) { 761 /* within range */ 762 if (patch_branch((struct ppc_inst *)ip, addr, BRANCH_SET_LINK)) { 763 pr_err("REL24 out of range!\n"); 764 return -EINVAL; 765 } 766 767 return 0; 768 } 769 770 if (rec->flags & FTRACE_FL_REGS) 771 tramp = mod->arch.tramp_regs; 772 else 773 tramp = mod->arch.tramp; 774 775 if (module_trampoline_target(mod, tramp, &ptr)) { 776 pr_err("Failed to get trampoline target\n"); 777 return -EFAULT; 778 } 779 780 pr_devel("trampoline target %lx", ptr); 781 782 entry = ppc_global_function_entry((void *)addr); 783 /* This should match what was called */ 784 if (ptr != entry) { 785 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 786 return -EINVAL; 787 } 788 789 /* Ensure branch is within 24 bits */ 790 if (create_branch(&op, (struct ppc_inst *)ip, tramp, BRANCH_SET_LINK)) { 791 pr_err("Branch out of range\n"); 792 return -EINVAL; 793 } 794 795 if (patch_branch((struct ppc_inst *)ip, tramp, BRANCH_SET_LINK)) { 796 pr_err("REL24 out of range!\n"); 797 return -EINVAL; 798 } 799 800 return 0; 801 } 802 #endif 803 804 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 805 unsigned long addr) 806 { 807 unsigned long ip = rec->ip; 808 struct ppc_inst old, new; 809 810 /* 811 * If the calling address is more that 24 bits away, 812 * then we had to use a trampoline to make the call. 813 * Otherwise just update the call site. 814 */ 815 if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) { 816 /* within range */ 817 old = ftrace_call_replace(ip, old_addr, 1); 818 new = ftrace_call_replace(ip, addr, 1); 819 return ftrace_modify_code(ip, old, new); 820 } else if (core_kernel_text(ip)) { 821 /* 822 * We always patch out of range locations to go to the regs 823 * variant, so there is nothing to do here 824 */ 825 return 0; 826 } 827 828 #ifdef CONFIG_MODULES 829 /* 830 * Out of range jumps are called from modules. 831 */ 832 if (!rec->arch.mod) { 833 pr_err("No module loaded\n"); 834 return -EINVAL; 835 } 836 837 return __ftrace_modify_call(rec, old_addr, addr); 838 #else 839 /* We should not get here without modules */ 840 return -EINVAL; 841 #endif /* CONFIG_MODULES */ 842 } 843 #endif 844 845 int ftrace_update_ftrace_func(ftrace_func_t func) 846 { 847 unsigned long ip = (unsigned long)(&ftrace_call); 848 struct ppc_inst old, new; 849 int ret; 850 851 old = ppc_inst_read((struct ppc_inst *)&ftrace_call); 852 new = ftrace_call_replace(ip, (unsigned long)func, 1); 853 ret = ftrace_modify_code(ip, old, new); 854 855 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 856 /* Also update the regs callback function */ 857 if (!ret) { 858 ip = (unsigned long)(&ftrace_regs_call); 859 old = ppc_inst_read((struct ppc_inst *)&ftrace_regs_call); 860 new = ftrace_call_replace(ip, (unsigned long)func, 1); 861 ret = ftrace_modify_code(ip, old, new); 862 } 863 #endif 864 865 return ret; 866 } 867 868 /* 869 * Use the default ftrace_modify_all_code, but without 870 * stop_machine(). 871 */ 872 void arch_ftrace_update_code(int command) 873 { 874 ftrace_modify_all_code(command); 875 } 876 877 #ifdef CONFIG_PPC64 878 #define PACATOC offsetof(struct paca_struct, kernel_toc) 879 880 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; 881 882 int __init ftrace_dyn_arch_init(void) 883 { 884 int i; 885 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; 886 u32 stub_insns[] = { 887 0xe98d0000 | PACATOC, /* ld r12,PACATOC(r13) */ 888 0x3d8c0000, /* addis r12,r12,<high> */ 889 0x398c0000, /* addi r12,r12,<low> */ 890 0x7d8903a6, /* mtctr r12 */ 891 0x4e800420, /* bctr */ 892 }; 893 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 894 unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller); 895 #else 896 unsigned long addr = ppc_global_function_entry((void *)ftrace_caller); 897 #endif 898 long reladdr = addr - kernel_toc_addr(); 899 900 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { 901 pr_err("Address of %ps out of range of kernel_toc.\n", 902 (void *)addr); 903 return -1; 904 } 905 906 for (i = 0; i < 2; i++) { 907 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 908 tramp[i][1] |= PPC_HA(reladdr); 909 tramp[i][2] |= PPC_LO(reladdr); 910 add_ftrace_tramp((unsigned long)tramp[i]); 911 } 912 913 return 0; 914 } 915 #else 916 int __init ftrace_dyn_arch_init(void) 917 { 918 return 0; 919 } 920 #endif 921 #endif /* CONFIG_DYNAMIC_FTRACE */ 922 923 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 924 925 extern void ftrace_graph_call(void); 926 extern void ftrace_graph_stub(void); 927 928 int ftrace_enable_ftrace_graph_caller(void) 929 { 930 unsigned long ip = (unsigned long)(&ftrace_graph_call); 931 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 932 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 933 struct ppc_inst old, new; 934 935 old = ftrace_call_replace(ip, stub, 0); 936 new = ftrace_call_replace(ip, addr, 0); 937 938 return ftrace_modify_code(ip, old, new); 939 } 940 941 int ftrace_disable_ftrace_graph_caller(void) 942 { 943 unsigned long ip = (unsigned long)(&ftrace_graph_call); 944 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 945 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 946 struct ppc_inst old, new; 947 948 old = ftrace_call_replace(ip, addr, 0); 949 new = ftrace_call_replace(ip, stub, 0); 950 951 return ftrace_modify_code(ip, old, new); 952 } 953 954 /* 955 * Hook the return address and push it in the stack of return addrs 956 * in current thread info. Return the address we want to divert to. 957 */ 958 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, 959 unsigned long sp) 960 { 961 unsigned long return_hooker; 962 963 if (unlikely(ftrace_graph_is_dead())) 964 goto out; 965 966 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 967 goto out; 968 969 return_hooker = ppc_function_entry(return_to_handler); 970 971 if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp)) 972 parent = return_hooker; 973 out: 974 return parent; 975 } 976 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 977 978 #ifdef PPC64_ELF_ABI_v1 979 char *arch_ftrace_match_adjust(char *str, const char *search) 980 { 981 if (str[0] == '.' && search[0] != '.') 982 return str + 1; 983 else 984 return str; 985 } 986 #endif /* PPC64_ELF_ABI_v1 */ 987