1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * 7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 8 * 9 * Added function graph tracer code, taken from x86 that was written 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 11 * 12 */ 13 14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt 15 16 #include <linux/spinlock.h> 17 #include <linux/hardirq.h> 18 #include <linux/uaccess.h> 19 #include <linux/module.h> 20 #include <linux/ftrace.h> 21 #include <linux/percpu.h> 22 #include <linux/init.h> 23 #include <linux/list.h> 24 25 #include <asm/cacheflush.h> 26 #include <asm/code-patching.h> 27 #include <asm/ftrace.h> 28 #include <asm/syscall.h> 29 #include <asm/inst.h> 30 31 /* 32 * We generally only have a single long_branch tramp and at most 2 or 3 plt 33 * tramps generated. But, we don't use the plt tramps currently. We also allot 34 * 2 tramps after .text and .init.text. So, we only end up with around 3 usable 35 * tramps in total. Set aside 8 just to be sure. 36 */ 37 #define NUM_FTRACE_TRAMPS 8 38 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; 39 40 static ppc_inst_t 41 ftrace_call_replace(unsigned long ip, unsigned long addr, int link) 42 { 43 ppc_inst_t op; 44 45 addr = ppc_function_entry((void *)addr); 46 47 /* if (link) set op to 'bl' else 'b' */ 48 create_branch(&op, (u32 *)ip, addr, link ? BRANCH_SET_LINK : 0); 49 50 return op; 51 } 52 53 static inline int 54 ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new) 55 { 56 ppc_inst_t replaced; 57 58 /* 59 * Note: 60 * We are paranoid about modifying text, as if a bug was to happen, it 61 * could cause us to read or write to someplace that could cause harm. 62 * Carefully read and modify the code with probe_kernel_*(), and make 63 * sure what we read is what we expected it to be before modifying it. 64 */ 65 66 /* read the text we want to modify */ 67 if (copy_inst_from_kernel_nofault(&replaced, (void *)ip)) 68 return -EFAULT; 69 70 /* Make sure it is what we expect it to be */ 71 if (!ppc_inst_equal(replaced, old)) { 72 pr_err("%p: replaced (%s) != old (%s)", 73 (void *)ip, ppc_inst_as_str(replaced), ppc_inst_as_str(old)); 74 return -EINVAL; 75 } 76 77 /* replace the text with the new text */ 78 return patch_instruction((u32 *)ip, new); 79 } 80 81 /* 82 * Helper functions that are the same for both PPC64 and PPC32. 83 */ 84 static int test_24bit_addr(unsigned long ip, unsigned long addr) 85 { 86 addr = ppc_function_entry((void *)addr); 87 88 return is_offset_in_branch_range(addr - ip); 89 } 90 91 static int is_bl_op(ppc_inst_t op) 92 { 93 return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BL(0); 94 } 95 96 static int is_b_op(ppc_inst_t op) 97 { 98 return (ppc_inst_val(op) & ~PPC_LI_MASK) == PPC_RAW_BRANCH(0); 99 } 100 101 static unsigned long find_bl_target(unsigned long ip, ppc_inst_t op) 102 { 103 int offset; 104 105 offset = PPC_LI(ppc_inst_val(op)); 106 /* make it signed */ 107 if (offset & 0x02000000) 108 offset |= 0xfe000000; 109 110 return ip + (long)offset; 111 } 112 113 #ifdef CONFIG_MODULES 114 static int 115 __ftrace_make_nop(struct module *mod, 116 struct dyn_ftrace *rec, unsigned long addr) 117 { 118 unsigned long entry, ptr, tramp; 119 unsigned long ip = rec->ip; 120 ppc_inst_t op, pop; 121 122 /* read where this goes */ 123 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 124 pr_err("Fetching opcode failed.\n"); 125 return -EFAULT; 126 } 127 128 /* Make sure that that this is still a 24bit jump */ 129 if (!is_bl_op(op)) { 130 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 131 return -EINVAL; 132 } 133 134 /* lets find where the pointer goes */ 135 tramp = find_bl_target(ip, op); 136 137 pr_devel("ip:%lx jumps to %lx", ip, tramp); 138 139 if (module_trampoline_target(mod, tramp, &ptr)) { 140 pr_err("Failed to get trampoline target\n"); 141 return -EFAULT; 142 } 143 144 pr_devel("trampoline target %lx", ptr); 145 146 entry = ppc_global_function_entry((void *)addr); 147 /* This should match what was called */ 148 if (ptr != entry) { 149 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 150 return -EINVAL; 151 } 152 153 /* When using -mprofile-kernel or PPC32 there is no load to jump over */ 154 pop = ppc_inst(PPC_RAW_NOP()); 155 156 #ifdef CONFIG_PPC64 157 #ifdef CONFIG_MPROFILE_KERNEL 158 if (copy_inst_from_kernel_nofault(&op, (void *)(ip - 4))) { 159 pr_err("Fetching instruction at %lx failed.\n", ip - 4); 160 return -EFAULT; 161 } 162 163 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */ 164 if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_MFLR(_R0))) && 165 !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) { 166 pr_err("Unexpected instruction %s around bl _mcount\n", 167 ppc_inst_as_str(op)); 168 return -EINVAL; 169 } 170 #else 171 /* 172 * Our original call site looks like: 173 * 174 * bl <tramp> 175 * ld r2,XX(r1) 176 * 177 * Milton Miller pointed out that we can not simply nop the branch. 178 * If a task was preempted when calling a trace function, the nops 179 * will remove the way to restore the TOC in r2 and the r2 TOC will 180 * get corrupted. 181 * 182 * Use a b +8 to jump over the load. 183 */ 184 185 pop = ppc_inst(PPC_RAW_BRANCH(8)); /* b +8 */ 186 187 /* 188 * Check what is in the next instruction. We can see ld r2,40(r1), but 189 * on first pass after boot we will see mflr r0. 190 */ 191 if (copy_inst_from_kernel_nofault(&op, (void *)(ip + 4))) { 192 pr_err("Fetching op failed.\n"); 193 return -EFAULT; 194 } 195 196 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_LD_TOC))) { 197 pr_err("Expected %08lx found %s\n", PPC_INST_LD_TOC, ppc_inst_as_str(op)); 198 return -EINVAL; 199 } 200 #endif /* CONFIG_MPROFILE_KERNEL */ 201 #endif /* PPC64 */ 202 203 if (patch_instruction((u32 *)ip, pop)) { 204 pr_err("Patching NOP failed.\n"); 205 return -EPERM; 206 } 207 208 return 0; 209 } 210 #endif /* CONFIG_MODULES */ 211 212 static unsigned long find_ftrace_tramp(unsigned long ip) 213 { 214 int i; 215 216 /* 217 * We have the compiler generated long_branch tramps at the end 218 * and we prefer those 219 */ 220 for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--) 221 if (!ftrace_tramps[i]) 222 continue; 223 else if (is_offset_in_branch_range(ftrace_tramps[i] - ip)) 224 return ftrace_tramps[i]; 225 226 return 0; 227 } 228 229 static int add_ftrace_tramp(unsigned long tramp) 230 { 231 int i; 232 233 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 234 if (!ftrace_tramps[i]) { 235 ftrace_tramps[i] = tramp; 236 return 0; 237 } 238 239 return -1; 240 } 241 242 /* 243 * If this is a compiler generated long_branch trampoline (essentially, a 244 * trampoline that has a branch to _mcount()), we re-write the branch to 245 * instead go to ftrace_[regs_]caller() and note down the location of this 246 * trampoline. 247 */ 248 static int setup_mcount_compiler_tramp(unsigned long tramp) 249 { 250 int i; 251 ppc_inst_t op; 252 unsigned long ptr; 253 254 /* Is this a known long jump tramp? */ 255 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 256 if (!ftrace_tramps[i]) 257 break; 258 else if (ftrace_tramps[i] == tramp) 259 return 0; 260 261 /* New trampoline -- read where this goes */ 262 if (copy_inst_from_kernel_nofault(&op, (void *)tramp)) { 263 pr_debug("Fetching opcode failed.\n"); 264 return -1; 265 } 266 267 /* Is this a 24 bit branch? */ 268 if (!is_b_op(op)) { 269 pr_debug("Trampoline is not a long branch tramp.\n"); 270 return -1; 271 } 272 273 /* lets find where the pointer goes */ 274 ptr = find_bl_target(tramp, op); 275 276 if (ptr != ppc_global_function_entry((void *)_mcount)) { 277 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr); 278 return -1; 279 } 280 281 /* Let's re-write the tramp to go to ftrace_[regs_]caller */ 282 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 283 ptr = ppc_global_function_entry((void *)ftrace_regs_caller); 284 #else 285 ptr = ppc_global_function_entry((void *)ftrace_caller); 286 #endif 287 if (patch_branch((u32 *)tramp, ptr, 0)) { 288 pr_debug("REL24 out of range!\n"); 289 return -1; 290 } 291 292 if (add_ftrace_tramp(tramp)) { 293 pr_debug("No tramp locations left\n"); 294 return -1; 295 } 296 297 return 0; 298 } 299 300 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) 301 { 302 unsigned long tramp, ip = rec->ip; 303 ppc_inst_t op; 304 305 /* Read where this goes */ 306 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 307 pr_err("Fetching opcode failed.\n"); 308 return -EFAULT; 309 } 310 311 /* Make sure that that this is still a 24bit jump */ 312 if (!is_bl_op(op)) { 313 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 314 return -EINVAL; 315 } 316 317 /* Let's find where the pointer goes */ 318 tramp = find_bl_target(ip, op); 319 320 pr_devel("ip:%lx jumps to %lx", ip, tramp); 321 322 if (setup_mcount_compiler_tramp(tramp)) { 323 /* Are other trampolines reachable? */ 324 if (!find_ftrace_tramp(ip)) { 325 pr_err("No ftrace trampolines reachable from %ps\n", 326 (void *)ip); 327 return -EINVAL; 328 } 329 } 330 331 if (patch_instruction((u32 *)ip, ppc_inst(PPC_RAW_NOP()))) { 332 pr_err("Patching NOP failed.\n"); 333 return -EPERM; 334 } 335 336 return 0; 337 } 338 339 int ftrace_make_nop(struct module *mod, 340 struct dyn_ftrace *rec, unsigned long addr) 341 { 342 unsigned long ip = rec->ip; 343 ppc_inst_t old, new; 344 345 /* 346 * If the calling address is more that 24 bits away, 347 * then we had to use a trampoline to make the call. 348 * Otherwise just update the call site. 349 */ 350 if (test_24bit_addr(ip, addr)) { 351 /* within range */ 352 old = ftrace_call_replace(ip, addr, 1); 353 new = ppc_inst(PPC_RAW_NOP()); 354 return ftrace_modify_code(ip, old, new); 355 } else if (core_kernel_text(ip)) 356 return __ftrace_make_nop_kernel(rec, addr); 357 358 #ifdef CONFIG_MODULES 359 /* 360 * Out of range jumps are called from modules. 361 * We should either already have a pointer to the module 362 * or it has been passed in. 363 */ 364 if (!rec->arch.mod) { 365 if (!mod) { 366 pr_err("No module loaded addr=%lx\n", addr); 367 return -EFAULT; 368 } 369 rec->arch.mod = mod; 370 } else if (mod) { 371 if (mod != rec->arch.mod) { 372 pr_err("Record mod %p not equal to passed in mod %p\n", 373 rec->arch.mod, mod); 374 return -EINVAL; 375 } 376 /* nothing to do if mod == rec->arch.mod */ 377 } else 378 mod = rec->arch.mod; 379 380 return __ftrace_make_nop(mod, rec, addr); 381 #else 382 /* We should not get here without modules */ 383 return -EINVAL; 384 #endif /* CONFIG_MODULES */ 385 } 386 387 #ifdef CONFIG_MODULES 388 /* 389 * Examine the existing instructions for __ftrace_make_call. 390 * They should effectively be a NOP, and follow formal constraints, 391 * depending on the ABI. Return false if they don't. 392 */ 393 static bool expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1) 394 { 395 if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1)) 396 return ppc_inst_equal(op0, ppc_inst(PPC_RAW_BRANCH(8))) && 397 ppc_inst_equal(op1, ppc_inst(PPC_INST_LD_TOC)); 398 else 399 return ppc_inst_equal(op0, ppc_inst(PPC_RAW_NOP())); 400 } 401 402 static int 403 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 404 { 405 ppc_inst_t op[2]; 406 void *ip = (void *)rec->ip; 407 unsigned long entry, ptr, tramp; 408 struct module *mod = rec->arch.mod; 409 410 /* read where this goes */ 411 if (copy_inst_from_kernel_nofault(op, ip)) 412 return -EFAULT; 413 414 #ifdef CONFIG_PPC64_ELF_ABI_V1 415 if (copy_inst_from_kernel_nofault(op + 1, ip + 4)) 416 return -EFAULT; 417 #endif 418 419 if (!expected_nop_sequence(ip, op[0], op[1])) { 420 pr_err("Unexpected call sequence at %p: %s %s\n", 421 ip, ppc_inst_as_str(op[0]), ppc_inst_as_str(op[1])); 422 return -EINVAL; 423 } 424 425 /* If we never set up ftrace trampoline(s), then bail */ 426 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 427 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 428 #else 429 if (!mod->arch.tramp) { 430 #endif 431 pr_err("No ftrace trampoline\n"); 432 return -EINVAL; 433 } 434 435 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 436 if (rec->flags & FTRACE_FL_REGS) 437 tramp = mod->arch.tramp_regs; 438 else 439 #endif 440 tramp = mod->arch.tramp; 441 442 if (module_trampoline_target(mod, tramp, &ptr)) { 443 pr_err("Failed to get trampoline target\n"); 444 return -EFAULT; 445 } 446 447 pr_devel("trampoline target %lx", ptr); 448 449 entry = ppc_global_function_entry((void *)addr); 450 /* This should match what was called */ 451 if (ptr != entry) { 452 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 453 return -EINVAL; 454 } 455 456 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 457 pr_err("REL24 out of range!\n"); 458 return -EINVAL; 459 } 460 461 return 0; 462 } 463 #endif /* CONFIG_MODULES */ 464 465 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) 466 { 467 ppc_inst_t op; 468 void *ip = (void *)rec->ip; 469 unsigned long tramp, entry, ptr; 470 471 /* Make sure we're being asked to patch branch to a known ftrace addr */ 472 entry = ppc_global_function_entry((void *)ftrace_caller); 473 ptr = ppc_global_function_entry((void *)addr); 474 475 if (ptr != entry) { 476 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 477 entry = ppc_global_function_entry((void *)ftrace_regs_caller); 478 if (ptr != entry) { 479 #endif 480 pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr); 481 return -EINVAL; 482 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 483 } 484 #endif 485 } 486 487 /* Make sure we have a nop */ 488 if (copy_inst_from_kernel_nofault(&op, ip)) { 489 pr_err("Unable to read ftrace location %p\n", ip); 490 return -EFAULT; 491 } 492 493 if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_NOP()))) { 494 pr_err("Unexpected call sequence at %p: %s\n", ip, ppc_inst_as_str(op)); 495 return -EINVAL; 496 } 497 498 tramp = find_ftrace_tramp((unsigned long)ip); 499 if (!tramp) { 500 pr_err("No ftrace trampolines reachable from %ps\n", ip); 501 return -EINVAL; 502 } 503 504 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 505 pr_err("Error patching branch to ftrace tramp!\n"); 506 return -EINVAL; 507 } 508 509 return 0; 510 } 511 512 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 513 { 514 unsigned long ip = rec->ip; 515 ppc_inst_t old, new; 516 517 /* 518 * If the calling address is more that 24 bits away, 519 * then we had to use a trampoline to make the call. 520 * Otherwise just update the call site. 521 */ 522 if (test_24bit_addr(ip, addr)) { 523 /* within range */ 524 old = ppc_inst(PPC_RAW_NOP()); 525 new = ftrace_call_replace(ip, addr, 1); 526 return ftrace_modify_code(ip, old, new); 527 } else if (core_kernel_text(ip)) 528 return __ftrace_make_call_kernel(rec, addr); 529 530 #ifdef CONFIG_MODULES 531 /* 532 * Out of range jumps are called from modules. 533 * Being that we are converting from nop, it had better 534 * already have a module defined. 535 */ 536 if (!rec->arch.mod) { 537 pr_err("No module loaded\n"); 538 return -EINVAL; 539 } 540 541 return __ftrace_make_call(rec, addr); 542 #else 543 /* We should not get here without modules */ 544 return -EINVAL; 545 #endif /* CONFIG_MODULES */ 546 } 547 548 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 549 #ifdef CONFIG_MODULES 550 static int 551 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 552 unsigned long addr) 553 { 554 ppc_inst_t op; 555 unsigned long ip = rec->ip; 556 unsigned long entry, ptr, tramp; 557 struct module *mod = rec->arch.mod; 558 559 /* If we never set up ftrace trampolines, then bail */ 560 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 561 pr_err("No ftrace trampoline\n"); 562 return -EINVAL; 563 } 564 565 /* read where this goes */ 566 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 567 pr_err("Fetching opcode failed.\n"); 568 return -EFAULT; 569 } 570 571 /* Make sure that that this is still a 24bit jump */ 572 if (!is_bl_op(op)) { 573 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 574 return -EINVAL; 575 } 576 577 /* lets find where the pointer goes */ 578 tramp = find_bl_target(ip, op); 579 entry = ppc_global_function_entry((void *)old_addr); 580 581 pr_devel("ip:%lx jumps to %lx", ip, tramp); 582 583 if (tramp != entry) { 584 /* old_addr is not within range, so we must have used a trampoline */ 585 if (module_trampoline_target(mod, tramp, &ptr)) { 586 pr_err("Failed to get trampoline target\n"); 587 return -EFAULT; 588 } 589 590 pr_devel("trampoline target %lx", ptr); 591 592 /* This should match what was called */ 593 if (ptr != entry) { 594 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 595 return -EINVAL; 596 } 597 } 598 599 /* The new target may be within range */ 600 if (test_24bit_addr(ip, addr)) { 601 /* within range */ 602 if (patch_branch((u32 *)ip, addr, BRANCH_SET_LINK)) { 603 pr_err("REL24 out of range!\n"); 604 return -EINVAL; 605 } 606 607 return 0; 608 } 609 610 if (rec->flags & FTRACE_FL_REGS) 611 tramp = mod->arch.tramp_regs; 612 else 613 tramp = mod->arch.tramp; 614 615 if (module_trampoline_target(mod, tramp, &ptr)) { 616 pr_err("Failed to get trampoline target\n"); 617 return -EFAULT; 618 } 619 620 pr_devel("trampoline target %lx", ptr); 621 622 entry = ppc_global_function_entry((void *)addr); 623 /* This should match what was called */ 624 if (ptr != entry) { 625 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 626 return -EINVAL; 627 } 628 629 if (patch_branch((u32 *)ip, tramp, BRANCH_SET_LINK)) { 630 pr_err("REL24 out of range!\n"); 631 return -EINVAL; 632 } 633 634 return 0; 635 } 636 #endif 637 638 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 639 unsigned long addr) 640 { 641 unsigned long ip = rec->ip; 642 ppc_inst_t old, new; 643 644 /* 645 * If the calling address is more that 24 bits away, 646 * then we had to use a trampoline to make the call. 647 * Otherwise just update the call site. 648 */ 649 if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) { 650 /* within range */ 651 old = ftrace_call_replace(ip, old_addr, 1); 652 new = ftrace_call_replace(ip, addr, 1); 653 return ftrace_modify_code(ip, old, new); 654 } else if (core_kernel_text(ip)) { 655 /* 656 * We always patch out of range locations to go to the regs 657 * variant, so there is nothing to do here 658 */ 659 return 0; 660 } 661 662 #ifdef CONFIG_MODULES 663 /* 664 * Out of range jumps are called from modules. 665 */ 666 if (!rec->arch.mod) { 667 pr_err("No module loaded\n"); 668 return -EINVAL; 669 } 670 671 return __ftrace_modify_call(rec, old_addr, addr); 672 #else 673 /* We should not get here without modules */ 674 return -EINVAL; 675 #endif /* CONFIG_MODULES */ 676 } 677 #endif 678 679 int ftrace_update_ftrace_func(ftrace_func_t func) 680 { 681 unsigned long ip = (unsigned long)(&ftrace_call); 682 ppc_inst_t old, new; 683 int ret; 684 685 old = ppc_inst_read((u32 *)&ftrace_call); 686 new = ftrace_call_replace(ip, (unsigned long)func, 1); 687 ret = ftrace_modify_code(ip, old, new); 688 689 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 690 /* Also update the regs callback function */ 691 if (!ret) { 692 ip = (unsigned long)(&ftrace_regs_call); 693 old = ppc_inst_read((u32 *)&ftrace_regs_call); 694 new = ftrace_call_replace(ip, (unsigned long)func, 1); 695 ret = ftrace_modify_code(ip, old, new); 696 } 697 #endif 698 699 return ret; 700 } 701 702 /* 703 * Use the default ftrace_modify_all_code, but without 704 * stop_machine(). 705 */ 706 void arch_ftrace_update_code(int command) 707 { 708 ftrace_modify_all_code(command); 709 } 710 711 #ifdef CONFIG_PPC64 712 #define PACATOC offsetof(struct paca_struct, kernel_toc) 713 714 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; 715 716 int __init ftrace_dyn_arch_init(void) 717 { 718 int i; 719 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; 720 u32 stub_insns[] = { 721 PPC_RAW_LD(_R12, _R13, PACATOC), 722 PPC_RAW_ADDIS(_R12, _R12, 0), 723 PPC_RAW_ADDI(_R12, _R12, 0), 724 PPC_RAW_MTCTR(_R12), 725 PPC_RAW_BCTR() 726 }; 727 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 728 unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller); 729 #else 730 unsigned long addr = ppc_global_function_entry((void *)ftrace_caller); 731 #endif 732 long reladdr = addr - kernel_toc_addr(); 733 734 if (reladdr >= SZ_2G || reladdr < -(long)SZ_2G) { 735 pr_err("Address of %ps out of range of kernel_toc.\n", 736 (void *)addr); 737 return -1; 738 } 739 740 for (i = 0; i < 2; i++) { 741 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 742 tramp[i][1] |= PPC_HA(reladdr); 743 tramp[i][2] |= PPC_LO(reladdr); 744 add_ftrace_tramp((unsigned long)tramp[i]); 745 } 746 747 return 0; 748 } 749 #else 750 int __init ftrace_dyn_arch_init(void) 751 { 752 return 0; 753 } 754 #endif 755 756 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 757 758 extern void ftrace_graph_call(void); 759 extern void ftrace_graph_stub(void); 760 761 static int ftrace_modify_ftrace_graph_caller(bool enable) 762 { 763 unsigned long ip = (unsigned long)(&ftrace_graph_call); 764 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 765 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 766 ppc_inst_t old, new; 767 768 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS)) 769 return 0; 770 771 old = ftrace_call_replace(ip, enable ? stub : addr, 0); 772 new = ftrace_call_replace(ip, enable ? addr : stub, 0); 773 774 return ftrace_modify_code(ip, old, new); 775 } 776 777 int ftrace_enable_ftrace_graph_caller(void) 778 { 779 return ftrace_modify_ftrace_graph_caller(true); 780 } 781 782 int ftrace_disable_ftrace_graph_caller(void) 783 { 784 return ftrace_modify_ftrace_graph_caller(false); 785 } 786 787 /* 788 * Hook the return address and push it in the stack of return addrs 789 * in current thread info. Return the address we want to divert to. 790 */ 791 static unsigned long 792 __prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp) 793 { 794 unsigned long return_hooker; 795 int bit; 796 797 if (unlikely(ftrace_graph_is_dead())) 798 goto out; 799 800 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 801 goto out; 802 803 bit = ftrace_test_recursion_trylock(ip, parent); 804 if (bit < 0) 805 goto out; 806 807 return_hooker = ppc_function_entry(return_to_handler); 808 809 if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp)) 810 parent = return_hooker; 811 812 ftrace_test_recursion_unlock(bit); 813 out: 814 return parent; 815 } 816 817 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS 818 void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, 819 struct ftrace_ops *op, struct ftrace_regs *fregs) 820 { 821 fregs->regs.link = __prepare_ftrace_return(parent_ip, ip, fregs->regs.gpr[1]); 822 } 823 #else 824 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, 825 unsigned long sp) 826 { 827 return __prepare_ftrace_return(parent, ip, sp); 828 } 829 #endif 830 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 831 832 #ifdef CONFIG_PPC64_ELF_ABI_V1 833 char *arch_ftrace_match_adjust(char *str, const char *search) 834 { 835 if (str[0] == '.' && search[0] != '.') 836 return str + 1; 837 else 838 return str; 839 } 840 #endif /* CONFIG_PPC64_ELF_ABI_V1 */ 841