1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * 7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 8 * 9 * Added function graph tracer code, taken from x86 that was written 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 11 * 12 */ 13 14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt 15 16 #include <linux/spinlock.h> 17 #include <linux/hardirq.h> 18 #include <linux/uaccess.h> 19 #include <linux/module.h> 20 #include <linux/ftrace.h> 21 #include <linux/percpu.h> 22 #include <linux/init.h> 23 #include <linux/list.h> 24 25 #include <asm/cacheflush.h> 26 #include <asm/code-patching.h> 27 #include <asm/ftrace.h> 28 #include <asm/syscall.h> 29 #include <asm/inst.h> 30 31 /* 32 * We generally only have a single long_branch tramp and at most 2 or 3 plt 33 * tramps generated. But, we don't use the plt tramps currently. We also allot 34 * 2 tramps after .text and .init.text. So, we only end up with around 3 usable 35 * tramps in total. Set aside 8 just to be sure. 36 */ 37 #define NUM_FTRACE_TRAMPS 8 38 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; 39 40 static ppc_inst_t 41 ftrace_call_replace(unsigned long ip, unsigned long addr, int link) 42 { 43 ppc_inst_t op; 44 45 addr = ppc_function_entry((void *)addr); 46 47 /* if (link) set op to 'bl' else 'b' */ 48 create_branch(&op, (u32 *)ip, addr, link ? 1 : 0); 49 50 return op; 51 } 52 53 static inline int 54 ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new) 55 { 56 ppc_inst_t replaced; 57 58 /* 59 * Note: 60 * We are paranoid about modifying text, as if a bug was to happen, it 61 * could cause us to read or write to someplace that could cause harm. 62 * Carefully read and modify the code with probe_kernel_*(), and make 63 * sure what we read is what we expected it to be before modifying it. 64 */ 65 66 /* read the text we want to modify */ 67 if (copy_inst_from_kernel_nofault(&replaced, (void *)ip)) 68 return -EFAULT; 69 70 /* Make sure it is what we expect it to be */ 71 if (!ppc_inst_equal(replaced, old)) { 72 pr_err("%p: replaced (%s) != old (%s)", 73 (void *)ip, ppc_inst_as_str(replaced), ppc_inst_as_str(old)); 74 return -EINVAL; 75 } 76 77 /* replace the text with the new text */ 78 return patch_instruction((u32 *)ip, new); 79 } 80 81 /* 82 * Helper functions that are the same for both PPC64 and PPC32. 83 */ 84 static int test_24bit_addr(unsigned long ip, unsigned long addr) 85 { 86 addr = ppc_function_entry((void *)addr); 87 88 return is_offset_in_branch_range(addr - ip); 89 } 90 91 static int is_bl_op(ppc_inst_t op) 92 { 93 return (ppc_inst_val(op) & 0xfc000003) == 0x48000001; 94 } 95 96 static int is_b_op(ppc_inst_t op) 97 { 98 return (ppc_inst_val(op) & 0xfc000003) == 0x48000000; 99 } 100 101 static unsigned long find_bl_target(unsigned long ip, ppc_inst_t op) 102 { 103 int offset; 104 105 offset = (ppc_inst_val(op) & 0x03fffffc); 106 /* make it signed */ 107 if (offset & 0x02000000) 108 offset |= 0xfe000000; 109 110 return ip + (long)offset; 111 } 112 113 #ifdef CONFIG_MODULES 114 static int 115 __ftrace_make_nop(struct module *mod, 116 struct dyn_ftrace *rec, unsigned long addr) 117 { 118 unsigned long entry, ptr, tramp; 119 unsigned long ip = rec->ip; 120 ppc_inst_t op, pop; 121 122 /* read where this goes */ 123 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 124 pr_err("Fetching opcode failed.\n"); 125 return -EFAULT; 126 } 127 128 /* Make sure that that this is still a 24bit jump */ 129 if (!is_bl_op(op)) { 130 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 131 return -EINVAL; 132 } 133 134 /* lets find where the pointer goes */ 135 tramp = find_bl_target(ip, op); 136 137 pr_devel("ip:%lx jumps to %lx", ip, tramp); 138 139 if (module_trampoline_target(mod, tramp, &ptr)) { 140 pr_err("Failed to get trampoline target\n"); 141 return -EFAULT; 142 } 143 144 pr_devel("trampoline target %lx", ptr); 145 146 entry = ppc_global_function_entry((void *)addr); 147 /* This should match what was called */ 148 if (ptr != entry) { 149 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 150 return -EINVAL; 151 } 152 153 /* When using -mprofile-kernel or PPC32 there is no load to jump over */ 154 pop = ppc_inst(PPC_RAW_NOP()); 155 156 #ifdef CONFIG_PPC64 157 #ifdef CONFIG_MPROFILE_KERNEL 158 if (copy_inst_from_kernel_nofault(&op, (void *)(ip - 4))) { 159 pr_err("Fetching instruction at %lx failed.\n", ip - 4); 160 return -EFAULT; 161 } 162 163 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */ 164 if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_MFLR(_R0))) && 165 !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) { 166 pr_err("Unexpected instruction %s around bl _mcount\n", 167 ppc_inst_as_str(op)); 168 return -EINVAL; 169 } 170 #else 171 /* 172 * Our original call site looks like: 173 * 174 * bl <tramp> 175 * ld r2,XX(r1) 176 * 177 * Milton Miller pointed out that we can not simply nop the branch. 178 * If a task was preempted when calling a trace function, the nops 179 * will remove the way to restore the TOC in r2 and the r2 TOC will 180 * get corrupted. 181 * 182 * Use a b +8 to jump over the load. 183 */ 184 185 pop = ppc_inst(PPC_INST_BRANCH | 8); /* b +8 */ 186 187 /* 188 * Check what is in the next instruction. We can see ld r2,40(r1), but 189 * on first pass after boot we will see mflr r0. 190 */ 191 if (copy_inst_from_kernel_nofault(&op, (void *)(ip + 4))) { 192 pr_err("Fetching op failed.\n"); 193 return -EFAULT; 194 } 195 196 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_LD_TOC))) { 197 pr_err("Expected %08lx found %s\n", PPC_INST_LD_TOC, ppc_inst_as_str(op)); 198 return -EINVAL; 199 } 200 #endif /* CONFIG_MPROFILE_KERNEL */ 201 #endif /* PPC64 */ 202 203 if (patch_instruction((u32 *)ip, pop)) { 204 pr_err("Patching NOP failed.\n"); 205 return -EPERM; 206 } 207 208 return 0; 209 } 210 #endif /* CONFIG_MODULES */ 211 212 static unsigned long find_ftrace_tramp(unsigned long ip) 213 { 214 int i; 215 216 /* 217 * We have the compiler generated long_branch tramps at the end 218 * and we prefer those 219 */ 220 for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--) 221 if (!ftrace_tramps[i]) 222 continue; 223 else if (is_offset_in_branch_range(ftrace_tramps[i] - ip)) 224 return ftrace_tramps[i]; 225 226 return 0; 227 } 228 229 static int add_ftrace_tramp(unsigned long tramp) 230 { 231 int i; 232 233 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 234 if (!ftrace_tramps[i]) { 235 ftrace_tramps[i] = tramp; 236 return 0; 237 } 238 239 return -1; 240 } 241 242 /* 243 * If this is a compiler generated long_branch trampoline (essentially, a 244 * trampoline that has a branch to _mcount()), we re-write the branch to 245 * instead go to ftrace_[regs_]caller() and note down the location of this 246 * trampoline. 247 */ 248 static int setup_mcount_compiler_tramp(unsigned long tramp) 249 { 250 int i; 251 ppc_inst_t op; 252 unsigned long ptr; 253 static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS]; 254 255 /* Is this a known long jump tramp? */ 256 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 257 if (!ftrace_tramps[i]) 258 break; 259 else if (ftrace_tramps[i] == tramp) 260 return 0; 261 262 /* Is this a known plt tramp? */ 263 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 264 if (!ftrace_plt_tramps[i]) 265 break; 266 else if (ftrace_plt_tramps[i] == tramp) 267 return -1; 268 269 /* New trampoline -- read where this goes */ 270 if (copy_inst_from_kernel_nofault(&op, (void *)tramp)) { 271 pr_debug("Fetching opcode failed.\n"); 272 return -1; 273 } 274 275 /* Is this a 24 bit branch? */ 276 if (!is_b_op(op)) { 277 pr_debug("Trampoline is not a long branch tramp.\n"); 278 return -1; 279 } 280 281 /* lets find where the pointer goes */ 282 ptr = find_bl_target(tramp, op); 283 284 if (ptr != ppc_global_function_entry((void *)_mcount)) { 285 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr); 286 return -1; 287 } 288 289 /* Let's re-write the tramp to go to ftrace_[regs_]caller */ 290 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 291 ptr = ppc_global_function_entry((void *)ftrace_regs_caller); 292 #else 293 ptr = ppc_global_function_entry((void *)ftrace_caller); 294 #endif 295 if (patch_branch((u32 *)tramp, ptr, 0)) { 296 pr_debug("REL24 out of range!\n"); 297 return -1; 298 } 299 300 if (add_ftrace_tramp(tramp)) { 301 pr_debug("No tramp locations left\n"); 302 return -1; 303 } 304 305 return 0; 306 } 307 308 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) 309 { 310 unsigned long tramp, ip = rec->ip; 311 ppc_inst_t op; 312 313 /* Read where this goes */ 314 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 315 pr_err("Fetching opcode failed.\n"); 316 return -EFAULT; 317 } 318 319 /* Make sure that that this is still a 24bit jump */ 320 if (!is_bl_op(op)) { 321 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 322 return -EINVAL; 323 } 324 325 /* Let's find where the pointer goes */ 326 tramp = find_bl_target(ip, op); 327 328 pr_devel("ip:%lx jumps to %lx", ip, tramp); 329 330 if (setup_mcount_compiler_tramp(tramp)) { 331 /* Are other trampolines reachable? */ 332 if (!find_ftrace_tramp(ip)) { 333 pr_err("No ftrace trampolines reachable from %ps\n", 334 (void *)ip); 335 return -EINVAL; 336 } 337 } 338 339 if (patch_instruction((u32 *)ip, ppc_inst(PPC_RAW_NOP()))) { 340 pr_err("Patching NOP failed.\n"); 341 return -EPERM; 342 } 343 344 return 0; 345 } 346 347 int ftrace_make_nop(struct module *mod, 348 struct dyn_ftrace *rec, unsigned long addr) 349 { 350 unsigned long ip = rec->ip; 351 ppc_inst_t old, new; 352 353 /* 354 * If the calling address is more that 24 bits away, 355 * then we had to use a trampoline to make the call. 356 * Otherwise just update the call site. 357 */ 358 if (test_24bit_addr(ip, addr)) { 359 /* within range */ 360 old = ftrace_call_replace(ip, addr, 1); 361 new = ppc_inst(PPC_RAW_NOP()); 362 return ftrace_modify_code(ip, old, new); 363 } else if (core_kernel_text(ip)) 364 return __ftrace_make_nop_kernel(rec, addr); 365 366 #ifdef CONFIG_MODULES 367 /* 368 * Out of range jumps are called from modules. 369 * We should either already have a pointer to the module 370 * or it has been passed in. 371 */ 372 if (!rec->arch.mod) { 373 if (!mod) { 374 pr_err("No module loaded addr=%lx\n", addr); 375 return -EFAULT; 376 } 377 rec->arch.mod = mod; 378 } else if (mod) { 379 if (mod != rec->arch.mod) { 380 pr_err("Record mod %p not equal to passed in mod %p\n", 381 rec->arch.mod, mod); 382 return -EINVAL; 383 } 384 /* nothing to do if mod == rec->arch.mod */ 385 } else 386 mod = rec->arch.mod; 387 388 return __ftrace_make_nop(mod, rec, addr); 389 #else 390 /* We should not get here without modules */ 391 return -EINVAL; 392 #endif /* CONFIG_MODULES */ 393 } 394 395 #ifdef CONFIG_MODULES 396 /* 397 * Examine the existing instructions for __ftrace_make_call. 398 * They should effectively be a NOP, and follow formal constraints, 399 * depending on the ABI. Return false if they don't. 400 */ 401 #ifdef CONFIG_PPC64_ELF_ABI_V1 402 static int 403 expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1) 404 { 405 /* 406 * We expect to see: 407 * 408 * b +8 409 * ld r2,XX(r1) 410 * 411 * The load offset is different depending on the ABI. For simplicity 412 * just mask it out when doing the compare. 413 */ 414 if (!ppc_inst_equal(op0, ppc_inst(0x48000008)) || 415 (ppc_inst_val(op1) & 0xffff0000) != 0xe8410000) 416 return 0; 417 return 1; 418 } 419 #else 420 static int 421 expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1) 422 { 423 /* look for patched "NOP" on ppc64 with -mprofile-kernel or ppc32 */ 424 if (!ppc_inst_equal(op0, ppc_inst(PPC_RAW_NOP()))) 425 return 0; 426 return 1; 427 } 428 #endif 429 430 static int 431 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 432 { 433 ppc_inst_t op[2]; 434 void *ip = (void *)rec->ip; 435 unsigned long entry, ptr, tramp; 436 struct module *mod = rec->arch.mod; 437 438 /* read where this goes */ 439 if (copy_inst_from_kernel_nofault(op, ip)) 440 return -EFAULT; 441 442 #ifdef CONFIG_PPC64_ELF_ABI_V1 443 if (copy_inst_from_kernel_nofault(op + 1, ip + 4)) 444 return -EFAULT; 445 #endif 446 447 if (!expected_nop_sequence(ip, op[0], op[1])) { 448 pr_err("Unexpected call sequence at %p: %s %s\n", 449 ip, ppc_inst_as_str(op[0]), ppc_inst_as_str(op[1])); 450 return -EINVAL; 451 } 452 453 /* If we never set up ftrace trampoline(s), then bail */ 454 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 455 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 456 #else 457 if (!mod->arch.tramp) { 458 #endif 459 pr_err("No ftrace trampoline\n"); 460 return -EINVAL; 461 } 462 463 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 464 if (rec->flags & FTRACE_FL_REGS) 465 tramp = mod->arch.tramp_regs; 466 else 467 #endif 468 tramp = mod->arch.tramp; 469 470 if (module_trampoline_target(mod, tramp, &ptr)) { 471 pr_err("Failed to get trampoline target\n"); 472 return -EFAULT; 473 } 474 475 pr_devel("trampoline target %lx", ptr); 476 477 entry = ppc_global_function_entry((void *)addr); 478 /* This should match what was called */ 479 if (ptr != entry) { 480 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 481 return -EINVAL; 482 } 483 484 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 485 pr_err("REL24 out of range!\n"); 486 return -EINVAL; 487 } 488 489 return 0; 490 } 491 #endif /* CONFIG_MODULES */ 492 493 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) 494 { 495 ppc_inst_t op; 496 void *ip = (void *)rec->ip; 497 unsigned long tramp, entry, ptr; 498 499 /* Make sure we're being asked to patch branch to a known ftrace addr */ 500 entry = ppc_global_function_entry((void *)ftrace_caller); 501 ptr = ppc_global_function_entry((void *)addr); 502 503 if (ptr != entry) { 504 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 505 entry = ppc_global_function_entry((void *)ftrace_regs_caller); 506 if (ptr != entry) { 507 #endif 508 pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr); 509 return -EINVAL; 510 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 511 } 512 #endif 513 } 514 515 /* Make sure we have a nop */ 516 if (copy_inst_from_kernel_nofault(&op, ip)) { 517 pr_err("Unable to read ftrace location %p\n", ip); 518 return -EFAULT; 519 } 520 521 if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_NOP()))) { 522 pr_err("Unexpected call sequence at %p: %s\n", ip, ppc_inst_as_str(op)); 523 return -EINVAL; 524 } 525 526 tramp = find_ftrace_tramp((unsigned long)ip); 527 if (!tramp) { 528 pr_err("No ftrace trampolines reachable from %ps\n", ip); 529 return -EINVAL; 530 } 531 532 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 533 pr_err("Error patching branch to ftrace tramp!\n"); 534 return -EINVAL; 535 } 536 537 return 0; 538 } 539 540 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 541 { 542 unsigned long ip = rec->ip; 543 ppc_inst_t old, new; 544 545 /* 546 * If the calling address is more that 24 bits away, 547 * then we had to use a trampoline to make the call. 548 * Otherwise just update the call site. 549 */ 550 if (test_24bit_addr(ip, addr)) { 551 /* within range */ 552 old = ppc_inst(PPC_RAW_NOP()); 553 new = ftrace_call_replace(ip, addr, 1); 554 return ftrace_modify_code(ip, old, new); 555 } else if (core_kernel_text(ip)) 556 return __ftrace_make_call_kernel(rec, addr); 557 558 #ifdef CONFIG_MODULES 559 /* 560 * Out of range jumps are called from modules. 561 * Being that we are converting from nop, it had better 562 * already have a module defined. 563 */ 564 if (!rec->arch.mod) { 565 pr_err("No module loaded\n"); 566 return -EINVAL; 567 } 568 569 return __ftrace_make_call(rec, addr); 570 #else 571 /* We should not get here without modules */ 572 return -EINVAL; 573 #endif /* CONFIG_MODULES */ 574 } 575 576 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 577 #ifdef CONFIG_MODULES 578 static int 579 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 580 unsigned long addr) 581 { 582 ppc_inst_t op; 583 unsigned long ip = rec->ip; 584 unsigned long entry, ptr, tramp; 585 struct module *mod = rec->arch.mod; 586 587 /* If we never set up ftrace trampolines, then bail */ 588 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 589 pr_err("No ftrace trampoline\n"); 590 return -EINVAL; 591 } 592 593 /* read where this goes */ 594 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 595 pr_err("Fetching opcode failed.\n"); 596 return -EFAULT; 597 } 598 599 /* Make sure that that this is still a 24bit jump */ 600 if (!is_bl_op(op)) { 601 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 602 return -EINVAL; 603 } 604 605 /* lets find where the pointer goes */ 606 tramp = find_bl_target(ip, op); 607 entry = ppc_global_function_entry((void *)old_addr); 608 609 pr_devel("ip:%lx jumps to %lx", ip, tramp); 610 611 if (tramp != entry) { 612 /* old_addr is not within range, so we must have used a trampoline */ 613 if (module_trampoline_target(mod, tramp, &ptr)) { 614 pr_err("Failed to get trampoline target\n"); 615 return -EFAULT; 616 } 617 618 pr_devel("trampoline target %lx", ptr); 619 620 /* This should match what was called */ 621 if (ptr != entry) { 622 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 623 return -EINVAL; 624 } 625 } 626 627 /* The new target may be within range */ 628 if (test_24bit_addr(ip, addr)) { 629 /* within range */ 630 if (patch_branch((u32 *)ip, addr, BRANCH_SET_LINK)) { 631 pr_err("REL24 out of range!\n"); 632 return -EINVAL; 633 } 634 635 return 0; 636 } 637 638 if (rec->flags & FTRACE_FL_REGS) 639 tramp = mod->arch.tramp_regs; 640 else 641 tramp = mod->arch.tramp; 642 643 if (module_trampoline_target(mod, tramp, &ptr)) { 644 pr_err("Failed to get trampoline target\n"); 645 return -EFAULT; 646 } 647 648 pr_devel("trampoline target %lx", ptr); 649 650 entry = ppc_global_function_entry((void *)addr); 651 /* This should match what was called */ 652 if (ptr != entry) { 653 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 654 return -EINVAL; 655 } 656 657 if (patch_branch((u32 *)ip, tramp, BRANCH_SET_LINK)) { 658 pr_err("REL24 out of range!\n"); 659 return -EINVAL; 660 } 661 662 return 0; 663 } 664 #endif 665 666 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 667 unsigned long addr) 668 { 669 unsigned long ip = rec->ip; 670 ppc_inst_t old, new; 671 672 /* 673 * If the calling address is more that 24 bits away, 674 * then we had to use a trampoline to make the call. 675 * Otherwise just update the call site. 676 */ 677 if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) { 678 /* within range */ 679 old = ftrace_call_replace(ip, old_addr, 1); 680 new = ftrace_call_replace(ip, addr, 1); 681 return ftrace_modify_code(ip, old, new); 682 } else if (core_kernel_text(ip)) { 683 /* 684 * We always patch out of range locations to go to the regs 685 * variant, so there is nothing to do here 686 */ 687 return 0; 688 } 689 690 #ifdef CONFIG_MODULES 691 /* 692 * Out of range jumps are called from modules. 693 */ 694 if (!rec->arch.mod) { 695 pr_err("No module loaded\n"); 696 return -EINVAL; 697 } 698 699 return __ftrace_modify_call(rec, old_addr, addr); 700 #else 701 /* We should not get here without modules */ 702 return -EINVAL; 703 #endif /* CONFIG_MODULES */ 704 } 705 #endif 706 707 int ftrace_update_ftrace_func(ftrace_func_t func) 708 { 709 unsigned long ip = (unsigned long)(&ftrace_call); 710 ppc_inst_t old, new; 711 int ret; 712 713 old = ppc_inst_read((u32 *)&ftrace_call); 714 new = ftrace_call_replace(ip, (unsigned long)func, 1); 715 ret = ftrace_modify_code(ip, old, new); 716 717 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 718 /* Also update the regs callback function */ 719 if (!ret) { 720 ip = (unsigned long)(&ftrace_regs_call); 721 old = ppc_inst_read((u32 *)&ftrace_regs_call); 722 new = ftrace_call_replace(ip, (unsigned long)func, 1); 723 ret = ftrace_modify_code(ip, old, new); 724 } 725 #endif 726 727 return ret; 728 } 729 730 /* 731 * Use the default ftrace_modify_all_code, but without 732 * stop_machine(). 733 */ 734 void arch_ftrace_update_code(int command) 735 { 736 ftrace_modify_all_code(command); 737 } 738 739 #ifdef CONFIG_PPC64 740 #define PACATOC offsetof(struct paca_struct, kernel_toc) 741 742 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; 743 744 int __init ftrace_dyn_arch_init(void) 745 { 746 int i; 747 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; 748 u32 stub_insns[] = { 749 0xe98d0000 | PACATOC, /* ld r12,PACATOC(r13) */ 750 0x3d8c0000, /* addis r12,r12,<high> */ 751 0x398c0000, /* addi r12,r12,<low> */ 752 0x7d8903a6, /* mtctr r12 */ 753 0x4e800420, /* bctr */ 754 }; 755 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 756 unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller); 757 #else 758 unsigned long addr = ppc_global_function_entry((void *)ftrace_caller); 759 #endif 760 long reladdr = addr - kernel_toc_addr(); 761 762 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { 763 pr_err("Address of %ps out of range of kernel_toc.\n", 764 (void *)addr); 765 return -1; 766 } 767 768 for (i = 0; i < 2; i++) { 769 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 770 tramp[i][1] |= PPC_HA(reladdr); 771 tramp[i][2] |= PPC_LO(reladdr); 772 add_ftrace_tramp((unsigned long)tramp[i]); 773 } 774 775 return 0; 776 } 777 #else 778 int __init ftrace_dyn_arch_init(void) 779 { 780 return 0; 781 } 782 #endif 783 784 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 785 786 extern void ftrace_graph_call(void); 787 extern void ftrace_graph_stub(void); 788 789 static int ftrace_modify_ftrace_graph_caller(bool enable) 790 { 791 unsigned long ip = (unsigned long)(&ftrace_graph_call); 792 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 793 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 794 ppc_inst_t old, new; 795 796 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS)) 797 return 0; 798 799 old = ftrace_call_replace(ip, enable ? stub : addr, 0); 800 new = ftrace_call_replace(ip, enable ? addr : stub, 0); 801 802 return ftrace_modify_code(ip, old, new); 803 } 804 805 int ftrace_enable_ftrace_graph_caller(void) 806 { 807 return ftrace_modify_ftrace_graph_caller(true); 808 } 809 810 int ftrace_disable_ftrace_graph_caller(void) 811 { 812 return ftrace_modify_ftrace_graph_caller(false); 813 } 814 815 /* 816 * Hook the return address and push it in the stack of return addrs 817 * in current thread info. Return the address we want to divert to. 818 */ 819 static unsigned long 820 __prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp) 821 { 822 unsigned long return_hooker; 823 int bit; 824 825 if (unlikely(ftrace_graph_is_dead())) 826 goto out; 827 828 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 829 goto out; 830 831 bit = ftrace_test_recursion_trylock(ip, parent); 832 if (bit < 0) 833 goto out; 834 835 return_hooker = ppc_function_entry(return_to_handler); 836 837 if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp)) 838 parent = return_hooker; 839 840 ftrace_test_recursion_unlock(bit); 841 out: 842 return parent; 843 } 844 845 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS 846 void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, 847 struct ftrace_ops *op, struct ftrace_regs *fregs) 848 { 849 fregs->regs.link = __prepare_ftrace_return(parent_ip, ip, fregs->regs.gpr[1]); 850 } 851 #else 852 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, 853 unsigned long sp) 854 { 855 return __prepare_ftrace_return(parent, ip, sp); 856 } 857 #endif 858 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 859 860 #ifdef CONFIG_PPC64_ELF_ABI_V1 861 char *arch_ftrace_match_adjust(char *str, const char *search) 862 { 863 if (str[0] == '.' && search[0] != '.') 864 return str + 1; 865 else 866 return str; 867 } 868 #endif /* CONFIG_PPC64_ELF_ABI_V1 */ 869