1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * 7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 8 * 9 * Added function graph tracer code, taken from x86 that was written 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 11 * 12 */ 13 14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt 15 16 #include <linux/spinlock.h> 17 #include <linux/hardirq.h> 18 #include <linux/uaccess.h> 19 #include <linux/module.h> 20 #include <linux/ftrace.h> 21 #include <linux/percpu.h> 22 #include <linux/init.h> 23 #include <linux/list.h> 24 25 #include <asm/cacheflush.h> 26 #include <asm/code-patching.h> 27 #include <asm/ftrace.h> 28 #include <asm/syscall.h> 29 #include <asm/inst.h> 30 31 /* 32 * We generally only have a single long_branch tramp and at most 2 or 3 plt 33 * tramps generated. But, we don't use the plt tramps currently. We also allot 34 * 2 tramps after .text and .init.text. So, we only end up with around 3 usable 35 * tramps in total. Set aside 8 just to be sure. 36 */ 37 #define NUM_FTRACE_TRAMPS 8 38 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; 39 40 static ppc_inst_t 41 ftrace_call_replace(unsigned long ip, unsigned long addr, int link) 42 { 43 ppc_inst_t op; 44 45 addr = ppc_function_entry((void *)addr); 46 47 /* if (link) set op to 'bl' else 'b' */ 48 create_branch(&op, (u32 *)ip, addr, link ? BRANCH_SET_LINK : 0); 49 50 return op; 51 } 52 53 static inline int 54 ftrace_modify_code(unsigned long ip, ppc_inst_t old, ppc_inst_t new) 55 { 56 ppc_inst_t replaced; 57 58 /* 59 * Note: 60 * We are paranoid about modifying text, as if a bug was to happen, it 61 * could cause us to read or write to someplace that could cause harm. 62 * Carefully read and modify the code with probe_kernel_*(), and make 63 * sure what we read is what we expected it to be before modifying it. 64 */ 65 66 /* read the text we want to modify */ 67 if (copy_inst_from_kernel_nofault(&replaced, (void *)ip)) 68 return -EFAULT; 69 70 /* Make sure it is what we expect it to be */ 71 if (!ppc_inst_equal(replaced, old)) { 72 pr_err("%p: replaced (%s) != old (%s)", 73 (void *)ip, ppc_inst_as_str(replaced), ppc_inst_as_str(old)); 74 return -EINVAL; 75 } 76 77 /* replace the text with the new text */ 78 return patch_instruction((u32 *)ip, new); 79 } 80 81 /* 82 * Helper functions that are the same for both PPC64 and PPC32. 83 */ 84 static int test_24bit_addr(unsigned long ip, unsigned long addr) 85 { 86 addr = ppc_function_entry((void *)addr); 87 88 return is_offset_in_branch_range(addr - ip); 89 } 90 91 static int is_bl_op(ppc_inst_t op) 92 { 93 return (ppc_inst_val(op) & 0xfc000003) == 0x48000001; 94 } 95 96 static int is_b_op(ppc_inst_t op) 97 { 98 return (ppc_inst_val(op) & 0xfc000003) == 0x48000000; 99 } 100 101 static unsigned long find_bl_target(unsigned long ip, ppc_inst_t op) 102 { 103 int offset; 104 105 offset = (ppc_inst_val(op) & 0x03fffffc); 106 /* make it signed */ 107 if (offset & 0x02000000) 108 offset |= 0xfe000000; 109 110 return ip + (long)offset; 111 } 112 113 #ifdef CONFIG_MODULES 114 static int 115 __ftrace_make_nop(struct module *mod, 116 struct dyn_ftrace *rec, unsigned long addr) 117 { 118 unsigned long entry, ptr, tramp; 119 unsigned long ip = rec->ip; 120 ppc_inst_t op, pop; 121 122 /* read where this goes */ 123 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 124 pr_err("Fetching opcode failed.\n"); 125 return -EFAULT; 126 } 127 128 /* Make sure that that this is still a 24bit jump */ 129 if (!is_bl_op(op)) { 130 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 131 return -EINVAL; 132 } 133 134 /* lets find where the pointer goes */ 135 tramp = find_bl_target(ip, op); 136 137 pr_devel("ip:%lx jumps to %lx", ip, tramp); 138 139 if (module_trampoline_target(mod, tramp, &ptr)) { 140 pr_err("Failed to get trampoline target\n"); 141 return -EFAULT; 142 } 143 144 pr_devel("trampoline target %lx", ptr); 145 146 entry = ppc_global_function_entry((void *)addr); 147 /* This should match what was called */ 148 if (ptr != entry) { 149 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 150 return -EINVAL; 151 } 152 153 /* When using -mprofile-kernel or PPC32 there is no load to jump over */ 154 pop = ppc_inst(PPC_RAW_NOP()); 155 156 #ifdef CONFIG_PPC64 157 #ifdef CONFIG_MPROFILE_KERNEL 158 if (copy_inst_from_kernel_nofault(&op, (void *)(ip - 4))) { 159 pr_err("Fetching instruction at %lx failed.\n", ip - 4); 160 return -EFAULT; 161 } 162 163 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */ 164 if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_MFLR(_R0))) && 165 !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) { 166 pr_err("Unexpected instruction %s around bl _mcount\n", 167 ppc_inst_as_str(op)); 168 return -EINVAL; 169 } 170 #else 171 /* 172 * Our original call site looks like: 173 * 174 * bl <tramp> 175 * ld r2,XX(r1) 176 * 177 * Milton Miller pointed out that we can not simply nop the branch. 178 * If a task was preempted when calling a trace function, the nops 179 * will remove the way to restore the TOC in r2 and the r2 TOC will 180 * get corrupted. 181 * 182 * Use a b +8 to jump over the load. 183 */ 184 185 pop = ppc_inst(PPC_INST_BRANCH | 8); /* b +8 */ 186 187 /* 188 * Check what is in the next instruction. We can see ld r2,40(r1), but 189 * on first pass after boot we will see mflr r0. 190 */ 191 if (copy_inst_from_kernel_nofault(&op, (void *)(ip + 4))) { 192 pr_err("Fetching op failed.\n"); 193 return -EFAULT; 194 } 195 196 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_LD_TOC))) { 197 pr_err("Expected %08lx found %s\n", PPC_INST_LD_TOC, ppc_inst_as_str(op)); 198 return -EINVAL; 199 } 200 #endif /* CONFIG_MPROFILE_KERNEL */ 201 #endif /* PPC64 */ 202 203 if (patch_instruction((u32 *)ip, pop)) { 204 pr_err("Patching NOP failed.\n"); 205 return -EPERM; 206 } 207 208 return 0; 209 } 210 #endif /* CONFIG_MODULES */ 211 212 static unsigned long find_ftrace_tramp(unsigned long ip) 213 { 214 int i; 215 216 /* 217 * We have the compiler generated long_branch tramps at the end 218 * and we prefer those 219 */ 220 for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--) 221 if (!ftrace_tramps[i]) 222 continue; 223 else if (is_offset_in_branch_range(ftrace_tramps[i] - ip)) 224 return ftrace_tramps[i]; 225 226 return 0; 227 } 228 229 static int add_ftrace_tramp(unsigned long tramp) 230 { 231 int i; 232 233 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 234 if (!ftrace_tramps[i]) { 235 ftrace_tramps[i] = tramp; 236 return 0; 237 } 238 239 return -1; 240 } 241 242 /* 243 * If this is a compiler generated long_branch trampoline (essentially, a 244 * trampoline that has a branch to _mcount()), we re-write the branch to 245 * instead go to ftrace_[regs_]caller() and note down the location of this 246 * trampoline. 247 */ 248 static int setup_mcount_compiler_tramp(unsigned long tramp) 249 { 250 int i; 251 ppc_inst_t op; 252 unsigned long ptr; 253 254 /* Is this a known long jump tramp? */ 255 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 256 if (!ftrace_tramps[i]) 257 break; 258 else if (ftrace_tramps[i] == tramp) 259 return 0; 260 261 /* New trampoline -- read where this goes */ 262 if (copy_inst_from_kernel_nofault(&op, (void *)tramp)) { 263 pr_debug("Fetching opcode failed.\n"); 264 return -1; 265 } 266 267 /* Is this a 24 bit branch? */ 268 if (!is_b_op(op)) { 269 pr_debug("Trampoline is not a long branch tramp.\n"); 270 return -1; 271 } 272 273 /* lets find where the pointer goes */ 274 ptr = find_bl_target(tramp, op); 275 276 if (ptr != ppc_global_function_entry((void *)_mcount)) { 277 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr); 278 return -1; 279 } 280 281 /* Let's re-write the tramp to go to ftrace_[regs_]caller */ 282 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 283 ptr = ppc_global_function_entry((void *)ftrace_regs_caller); 284 #else 285 ptr = ppc_global_function_entry((void *)ftrace_caller); 286 #endif 287 if (patch_branch((u32 *)tramp, ptr, 0)) { 288 pr_debug("REL24 out of range!\n"); 289 return -1; 290 } 291 292 if (add_ftrace_tramp(tramp)) { 293 pr_debug("No tramp locations left\n"); 294 return -1; 295 } 296 297 return 0; 298 } 299 300 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) 301 { 302 unsigned long tramp, ip = rec->ip; 303 ppc_inst_t op; 304 305 /* Read where this goes */ 306 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 307 pr_err("Fetching opcode failed.\n"); 308 return -EFAULT; 309 } 310 311 /* Make sure that that this is still a 24bit jump */ 312 if (!is_bl_op(op)) { 313 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 314 return -EINVAL; 315 } 316 317 /* Let's find where the pointer goes */ 318 tramp = find_bl_target(ip, op); 319 320 pr_devel("ip:%lx jumps to %lx", ip, tramp); 321 322 if (setup_mcount_compiler_tramp(tramp)) { 323 /* Are other trampolines reachable? */ 324 if (!find_ftrace_tramp(ip)) { 325 pr_err("No ftrace trampolines reachable from %ps\n", 326 (void *)ip); 327 return -EINVAL; 328 } 329 } 330 331 if (patch_instruction((u32 *)ip, ppc_inst(PPC_RAW_NOP()))) { 332 pr_err("Patching NOP failed.\n"); 333 return -EPERM; 334 } 335 336 return 0; 337 } 338 339 int ftrace_make_nop(struct module *mod, 340 struct dyn_ftrace *rec, unsigned long addr) 341 { 342 unsigned long ip = rec->ip; 343 ppc_inst_t old, new; 344 345 /* 346 * If the calling address is more that 24 bits away, 347 * then we had to use a trampoline to make the call. 348 * Otherwise just update the call site. 349 */ 350 if (test_24bit_addr(ip, addr)) { 351 /* within range */ 352 old = ftrace_call_replace(ip, addr, 1); 353 new = ppc_inst(PPC_RAW_NOP()); 354 return ftrace_modify_code(ip, old, new); 355 } else if (core_kernel_text(ip)) 356 return __ftrace_make_nop_kernel(rec, addr); 357 358 #ifdef CONFIG_MODULES 359 /* 360 * Out of range jumps are called from modules. 361 * We should either already have a pointer to the module 362 * or it has been passed in. 363 */ 364 if (!rec->arch.mod) { 365 if (!mod) { 366 pr_err("No module loaded addr=%lx\n", addr); 367 return -EFAULT; 368 } 369 rec->arch.mod = mod; 370 } else if (mod) { 371 if (mod != rec->arch.mod) { 372 pr_err("Record mod %p not equal to passed in mod %p\n", 373 rec->arch.mod, mod); 374 return -EINVAL; 375 } 376 /* nothing to do if mod == rec->arch.mod */ 377 } else 378 mod = rec->arch.mod; 379 380 return __ftrace_make_nop(mod, rec, addr); 381 #else 382 /* We should not get here without modules */ 383 return -EINVAL; 384 #endif /* CONFIG_MODULES */ 385 } 386 387 #ifdef CONFIG_MODULES 388 /* 389 * Examine the existing instructions for __ftrace_make_call. 390 * They should effectively be a NOP, and follow formal constraints, 391 * depending on the ABI. Return false if they don't. 392 */ 393 #ifdef CONFIG_PPC64_ELF_ABI_V1 394 static int 395 expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1) 396 { 397 /* 398 * We expect to see: 399 * 400 * b +8 401 * ld r2,XX(r1) 402 * 403 * The load offset is different depending on the ABI. For simplicity 404 * just mask it out when doing the compare. 405 */ 406 if (!ppc_inst_equal(op0, ppc_inst(0x48000008)) || 407 (ppc_inst_val(op1) & 0xffff0000) != 0xe8410000) 408 return 0; 409 return 1; 410 } 411 #else 412 static int 413 expected_nop_sequence(void *ip, ppc_inst_t op0, ppc_inst_t op1) 414 { 415 /* look for patched "NOP" on ppc64 with -mprofile-kernel or ppc32 */ 416 if (!ppc_inst_equal(op0, ppc_inst(PPC_RAW_NOP()))) 417 return 0; 418 return 1; 419 } 420 #endif 421 422 static int 423 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 424 { 425 ppc_inst_t op[2]; 426 void *ip = (void *)rec->ip; 427 unsigned long entry, ptr, tramp; 428 struct module *mod = rec->arch.mod; 429 430 /* read where this goes */ 431 if (copy_inst_from_kernel_nofault(op, ip)) 432 return -EFAULT; 433 434 #ifdef CONFIG_PPC64_ELF_ABI_V1 435 if (copy_inst_from_kernel_nofault(op + 1, ip + 4)) 436 return -EFAULT; 437 #endif 438 439 if (!expected_nop_sequence(ip, op[0], op[1])) { 440 pr_err("Unexpected call sequence at %p: %s %s\n", 441 ip, ppc_inst_as_str(op[0]), ppc_inst_as_str(op[1])); 442 return -EINVAL; 443 } 444 445 /* If we never set up ftrace trampoline(s), then bail */ 446 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 447 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 448 #else 449 if (!mod->arch.tramp) { 450 #endif 451 pr_err("No ftrace trampoline\n"); 452 return -EINVAL; 453 } 454 455 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 456 if (rec->flags & FTRACE_FL_REGS) 457 tramp = mod->arch.tramp_regs; 458 else 459 #endif 460 tramp = mod->arch.tramp; 461 462 if (module_trampoline_target(mod, tramp, &ptr)) { 463 pr_err("Failed to get trampoline target\n"); 464 return -EFAULT; 465 } 466 467 pr_devel("trampoline target %lx", ptr); 468 469 entry = ppc_global_function_entry((void *)addr); 470 /* This should match what was called */ 471 if (ptr != entry) { 472 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 473 return -EINVAL; 474 } 475 476 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 477 pr_err("REL24 out of range!\n"); 478 return -EINVAL; 479 } 480 481 return 0; 482 } 483 #endif /* CONFIG_MODULES */ 484 485 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) 486 { 487 ppc_inst_t op; 488 void *ip = (void *)rec->ip; 489 unsigned long tramp, entry, ptr; 490 491 /* Make sure we're being asked to patch branch to a known ftrace addr */ 492 entry = ppc_global_function_entry((void *)ftrace_caller); 493 ptr = ppc_global_function_entry((void *)addr); 494 495 if (ptr != entry) { 496 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 497 entry = ppc_global_function_entry((void *)ftrace_regs_caller); 498 if (ptr != entry) { 499 #endif 500 pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr); 501 return -EINVAL; 502 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 503 } 504 #endif 505 } 506 507 /* Make sure we have a nop */ 508 if (copy_inst_from_kernel_nofault(&op, ip)) { 509 pr_err("Unable to read ftrace location %p\n", ip); 510 return -EFAULT; 511 } 512 513 if (!ppc_inst_equal(op, ppc_inst(PPC_RAW_NOP()))) { 514 pr_err("Unexpected call sequence at %p: %s\n", ip, ppc_inst_as_str(op)); 515 return -EINVAL; 516 } 517 518 tramp = find_ftrace_tramp((unsigned long)ip); 519 if (!tramp) { 520 pr_err("No ftrace trampolines reachable from %ps\n", ip); 521 return -EINVAL; 522 } 523 524 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 525 pr_err("Error patching branch to ftrace tramp!\n"); 526 return -EINVAL; 527 } 528 529 return 0; 530 } 531 532 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 533 { 534 unsigned long ip = rec->ip; 535 ppc_inst_t old, new; 536 537 /* 538 * If the calling address is more that 24 bits away, 539 * then we had to use a trampoline to make the call. 540 * Otherwise just update the call site. 541 */ 542 if (test_24bit_addr(ip, addr)) { 543 /* within range */ 544 old = ppc_inst(PPC_RAW_NOP()); 545 new = ftrace_call_replace(ip, addr, 1); 546 return ftrace_modify_code(ip, old, new); 547 } else if (core_kernel_text(ip)) 548 return __ftrace_make_call_kernel(rec, addr); 549 550 #ifdef CONFIG_MODULES 551 /* 552 * Out of range jumps are called from modules. 553 * Being that we are converting from nop, it had better 554 * already have a module defined. 555 */ 556 if (!rec->arch.mod) { 557 pr_err("No module loaded\n"); 558 return -EINVAL; 559 } 560 561 return __ftrace_make_call(rec, addr); 562 #else 563 /* We should not get here without modules */ 564 return -EINVAL; 565 #endif /* CONFIG_MODULES */ 566 } 567 568 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 569 #ifdef CONFIG_MODULES 570 static int 571 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 572 unsigned long addr) 573 { 574 ppc_inst_t op; 575 unsigned long ip = rec->ip; 576 unsigned long entry, ptr, tramp; 577 struct module *mod = rec->arch.mod; 578 579 /* If we never set up ftrace trampolines, then bail */ 580 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 581 pr_err("No ftrace trampoline\n"); 582 return -EINVAL; 583 } 584 585 /* read where this goes */ 586 if (copy_inst_from_kernel_nofault(&op, (void *)ip)) { 587 pr_err("Fetching opcode failed.\n"); 588 return -EFAULT; 589 } 590 591 /* Make sure that that this is still a 24bit jump */ 592 if (!is_bl_op(op)) { 593 pr_err("Not expected bl: opcode is %s\n", ppc_inst_as_str(op)); 594 return -EINVAL; 595 } 596 597 /* lets find where the pointer goes */ 598 tramp = find_bl_target(ip, op); 599 entry = ppc_global_function_entry((void *)old_addr); 600 601 pr_devel("ip:%lx jumps to %lx", ip, tramp); 602 603 if (tramp != entry) { 604 /* old_addr is not within range, so we must have used a trampoline */ 605 if (module_trampoline_target(mod, tramp, &ptr)) { 606 pr_err("Failed to get trampoline target\n"); 607 return -EFAULT; 608 } 609 610 pr_devel("trampoline target %lx", ptr); 611 612 /* This should match what was called */ 613 if (ptr != entry) { 614 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 615 return -EINVAL; 616 } 617 } 618 619 /* The new target may be within range */ 620 if (test_24bit_addr(ip, addr)) { 621 /* within range */ 622 if (patch_branch((u32 *)ip, addr, BRANCH_SET_LINK)) { 623 pr_err("REL24 out of range!\n"); 624 return -EINVAL; 625 } 626 627 return 0; 628 } 629 630 if (rec->flags & FTRACE_FL_REGS) 631 tramp = mod->arch.tramp_regs; 632 else 633 tramp = mod->arch.tramp; 634 635 if (module_trampoline_target(mod, tramp, &ptr)) { 636 pr_err("Failed to get trampoline target\n"); 637 return -EFAULT; 638 } 639 640 pr_devel("trampoline target %lx", ptr); 641 642 entry = ppc_global_function_entry((void *)addr); 643 /* This should match what was called */ 644 if (ptr != entry) { 645 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 646 return -EINVAL; 647 } 648 649 if (patch_branch((u32 *)ip, tramp, BRANCH_SET_LINK)) { 650 pr_err("REL24 out of range!\n"); 651 return -EINVAL; 652 } 653 654 return 0; 655 } 656 #endif 657 658 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 659 unsigned long addr) 660 { 661 unsigned long ip = rec->ip; 662 ppc_inst_t old, new; 663 664 /* 665 * If the calling address is more that 24 bits away, 666 * then we had to use a trampoline to make the call. 667 * Otherwise just update the call site. 668 */ 669 if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) { 670 /* within range */ 671 old = ftrace_call_replace(ip, old_addr, 1); 672 new = ftrace_call_replace(ip, addr, 1); 673 return ftrace_modify_code(ip, old, new); 674 } else if (core_kernel_text(ip)) { 675 /* 676 * We always patch out of range locations to go to the regs 677 * variant, so there is nothing to do here 678 */ 679 return 0; 680 } 681 682 #ifdef CONFIG_MODULES 683 /* 684 * Out of range jumps are called from modules. 685 */ 686 if (!rec->arch.mod) { 687 pr_err("No module loaded\n"); 688 return -EINVAL; 689 } 690 691 return __ftrace_modify_call(rec, old_addr, addr); 692 #else 693 /* We should not get here without modules */ 694 return -EINVAL; 695 #endif /* CONFIG_MODULES */ 696 } 697 #endif 698 699 int ftrace_update_ftrace_func(ftrace_func_t func) 700 { 701 unsigned long ip = (unsigned long)(&ftrace_call); 702 ppc_inst_t old, new; 703 int ret; 704 705 old = ppc_inst_read((u32 *)&ftrace_call); 706 new = ftrace_call_replace(ip, (unsigned long)func, 1); 707 ret = ftrace_modify_code(ip, old, new); 708 709 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 710 /* Also update the regs callback function */ 711 if (!ret) { 712 ip = (unsigned long)(&ftrace_regs_call); 713 old = ppc_inst_read((u32 *)&ftrace_regs_call); 714 new = ftrace_call_replace(ip, (unsigned long)func, 1); 715 ret = ftrace_modify_code(ip, old, new); 716 } 717 #endif 718 719 return ret; 720 } 721 722 /* 723 * Use the default ftrace_modify_all_code, but without 724 * stop_machine(). 725 */ 726 void arch_ftrace_update_code(int command) 727 { 728 ftrace_modify_all_code(command); 729 } 730 731 #ifdef CONFIG_PPC64 732 #define PACATOC offsetof(struct paca_struct, kernel_toc) 733 734 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; 735 736 int __init ftrace_dyn_arch_init(void) 737 { 738 int i; 739 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; 740 u32 stub_insns[] = { 741 0xe98d0000 | PACATOC, /* ld r12,PACATOC(r13) */ 742 0x3d8c0000, /* addis r12,r12,<high> */ 743 0x398c0000, /* addi r12,r12,<low> */ 744 0x7d8903a6, /* mtctr r12 */ 745 0x4e800420, /* bctr */ 746 }; 747 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 748 unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller); 749 #else 750 unsigned long addr = ppc_global_function_entry((void *)ftrace_caller); 751 #endif 752 long reladdr = addr - kernel_toc_addr(); 753 754 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { 755 pr_err("Address of %ps out of range of kernel_toc.\n", 756 (void *)addr); 757 return -1; 758 } 759 760 for (i = 0; i < 2; i++) { 761 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 762 tramp[i][1] |= PPC_HA(reladdr); 763 tramp[i][2] |= PPC_LO(reladdr); 764 add_ftrace_tramp((unsigned long)tramp[i]); 765 } 766 767 return 0; 768 } 769 #else 770 int __init ftrace_dyn_arch_init(void) 771 { 772 return 0; 773 } 774 #endif 775 776 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 777 778 extern void ftrace_graph_call(void); 779 extern void ftrace_graph_stub(void); 780 781 static int ftrace_modify_ftrace_graph_caller(bool enable) 782 { 783 unsigned long ip = (unsigned long)(&ftrace_graph_call); 784 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 785 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 786 ppc_inst_t old, new; 787 788 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_ARGS)) 789 return 0; 790 791 old = ftrace_call_replace(ip, enable ? stub : addr, 0); 792 new = ftrace_call_replace(ip, enable ? addr : stub, 0); 793 794 return ftrace_modify_code(ip, old, new); 795 } 796 797 int ftrace_enable_ftrace_graph_caller(void) 798 { 799 return ftrace_modify_ftrace_graph_caller(true); 800 } 801 802 int ftrace_disable_ftrace_graph_caller(void) 803 { 804 return ftrace_modify_ftrace_graph_caller(false); 805 } 806 807 /* 808 * Hook the return address and push it in the stack of return addrs 809 * in current thread info. Return the address we want to divert to. 810 */ 811 static unsigned long 812 __prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp) 813 { 814 unsigned long return_hooker; 815 int bit; 816 817 if (unlikely(ftrace_graph_is_dead())) 818 goto out; 819 820 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 821 goto out; 822 823 bit = ftrace_test_recursion_trylock(ip, parent); 824 if (bit < 0) 825 goto out; 826 827 return_hooker = ppc_function_entry(return_to_handler); 828 829 if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp)) 830 parent = return_hooker; 831 832 ftrace_test_recursion_unlock(bit); 833 out: 834 return parent; 835 } 836 837 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS 838 void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, 839 struct ftrace_ops *op, struct ftrace_regs *fregs) 840 { 841 fregs->regs.link = __prepare_ftrace_return(parent_ip, ip, fregs->regs.gpr[1]); 842 } 843 #else 844 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, 845 unsigned long sp) 846 { 847 return __prepare_ftrace_return(parent, ip, sp); 848 } 849 #endif 850 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 851 852 #ifdef CONFIG_PPC64_ELF_ABI_V1 853 char *arch_ftrace_match_adjust(char *str, const char *search) 854 { 855 if (str[0] == '.' && search[0] != '.') 856 return str + 1; 857 else 858 return str; 859 } 860 #endif /* CONFIG_PPC64_ELF_ABI_V1 */ 861