1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * 7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 8 * 9 * Added function graph tracer code, taken from x86 that was written 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 11 * 12 */ 13 14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt 15 16 #include <linux/spinlock.h> 17 #include <linux/hardirq.h> 18 #include <linux/uaccess.h> 19 #include <linux/module.h> 20 #include <linux/ftrace.h> 21 #include <linux/percpu.h> 22 #include <linux/init.h> 23 #include <linux/list.h> 24 25 #include <asm/asm-prototypes.h> 26 #include <asm/cacheflush.h> 27 #include <asm/code-patching.h> 28 #include <asm/ftrace.h> 29 #include <asm/syscall.h> 30 31 32 #ifdef CONFIG_DYNAMIC_FTRACE 33 34 /* 35 * We generally only have a single long_branch tramp and at most 2 or 3 plt 36 * tramps generated. But, we don't use the plt tramps currently. We also allot 37 * 2 tramps after .text and .init.text. So, we only end up with around 3 usable 38 * tramps in total. Set aside 8 just to be sure. 39 */ 40 #define NUM_FTRACE_TRAMPS 8 41 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; 42 43 static unsigned int 44 ftrace_call_replace(unsigned long ip, unsigned long addr, int link) 45 { 46 unsigned int op; 47 48 addr = ppc_function_entry((void *)addr); 49 50 /* if (link) set op to 'bl' else 'b' */ 51 op = create_branch((unsigned int *)ip, addr, link ? 1 : 0); 52 53 return op; 54 } 55 56 static int 57 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new) 58 { 59 unsigned int replaced; 60 61 /* 62 * Note: 63 * We are paranoid about modifying text, as if a bug was to happen, it 64 * could cause us to read or write to someplace that could cause harm. 65 * Carefully read and modify the code with probe_kernel_*(), and make 66 * sure what we read is what we expected it to be before modifying it. 67 */ 68 69 /* read the text we want to modify */ 70 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE)) 71 return -EFAULT; 72 73 /* Make sure it is what we expect it to be */ 74 if (replaced != old) { 75 pr_err("%p: replaced (%#x) != old (%#x)", 76 (void *)ip, replaced, old); 77 return -EINVAL; 78 } 79 80 /* replace the text with the new text */ 81 if (patch_instruction((unsigned int *)ip, new)) 82 return -EPERM; 83 84 return 0; 85 } 86 87 /* 88 * Helper functions that are the same for both PPC64 and PPC32. 89 */ 90 static int test_24bit_addr(unsigned long ip, unsigned long addr) 91 { 92 addr = ppc_function_entry((void *)addr); 93 94 /* use the create_branch to verify that this offset can be branched */ 95 return create_branch((unsigned int *)ip, addr, 0); 96 } 97 98 static int is_bl_op(unsigned int op) 99 { 100 return (op & 0xfc000003) == 0x48000001; 101 } 102 103 static int is_b_op(unsigned int op) 104 { 105 return (op & 0xfc000003) == 0x48000000; 106 } 107 108 static unsigned long find_bl_target(unsigned long ip, unsigned int op) 109 { 110 static int offset; 111 112 offset = (op & 0x03fffffc); 113 /* make it signed */ 114 if (offset & 0x02000000) 115 offset |= 0xfe000000; 116 117 return ip + (long)offset; 118 } 119 120 #ifdef CONFIG_MODULES 121 #ifdef CONFIG_PPC64 122 static int 123 __ftrace_make_nop(struct module *mod, 124 struct dyn_ftrace *rec, unsigned long addr) 125 { 126 unsigned long entry, ptr, tramp; 127 unsigned long ip = rec->ip; 128 unsigned int op, pop; 129 130 /* read where this goes */ 131 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 132 pr_err("Fetching opcode failed.\n"); 133 return -EFAULT; 134 } 135 136 /* Make sure that that this is still a 24bit jump */ 137 if (!is_bl_op(op)) { 138 pr_err("Not expected bl: opcode is %x\n", op); 139 return -EINVAL; 140 } 141 142 /* lets find where the pointer goes */ 143 tramp = find_bl_target(ip, op); 144 145 pr_devel("ip:%lx jumps to %lx", ip, tramp); 146 147 if (module_trampoline_target(mod, tramp, &ptr)) { 148 pr_err("Failed to get trampoline target\n"); 149 return -EFAULT; 150 } 151 152 pr_devel("trampoline target %lx", ptr); 153 154 entry = ppc_global_function_entry((void *)addr); 155 /* This should match what was called */ 156 if (ptr != entry) { 157 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 158 return -EINVAL; 159 } 160 161 #ifdef CONFIG_MPROFILE_KERNEL 162 /* When using -mkernel_profile there is no load to jump over */ 163 pop = PPC_INST_NOP; 164 165 if (probe_kernel_read(&op, (void *)(ip - 4), 4)) { 166 pr_err("Fetching instruction at %lx failed.\n", ip - 4); 167 return -EFAULT; 168 } 169 170 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */ 171 if (op != PPC_INST_MFLR && op != PPC_INST_STD_LR) { 172 pr_err("Unexpected instruction %08x around bl _mcount\n", op); 173 return -EINVAL; 174 } 175 #else 176 /* 177 * Our original call site looks like: 178 * 179 * bl <tramp> 180 * ld r2,XX(r1) 181 * 182 * Milton Miller pointed out that we can not simply nop the branch. 183 * If a task was preempted when calling a trace function, the nops 184 * will remove the way to restore the TOC in r2 and the r2 TOC will 185 * get corrupted. 186 * 187 * Use a b +8 to jump over the load. 188 */ 189 190 pop = PPC_INST_BRANCH | 8; /* b +8 */ 191 192 /* 193 * Check what is in the next instruction. We can see ld r2,40(r1), but 194 * on first pass after boot we will see mflr r0. 195 */ 196 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) { 197 pr_err("Fetching op failed.\n"); 198 return -EFAULT; 199 } 200 201 if (op != PPC_INST_LD_TOC) { 202 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, op); 203 return -EINVAL; 204 } 205 #endif /* CONFIG_MPROFILE_KERNEL */ 206 207 if (patch_instruction((unsigned int *)ip, pop)) { 208 pr_err("Patching NOP failed.\n"); 209 return -EPERM; 210 } 211 212 return 0; 213 } 214 215 #else /* !PPC64 */ 216 static int 217 __ftrace_make_nop(struct module *mod, 218 struct dyn_ftrace *rec, unsigned long addr) 219 { 220 unsigned int op; 221 unsigned int jmp[4]; 222 unsigned long ip = rec->ip; 223 unsigned long tramp; 224 225 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 226 return -EFAULT; 227 228 /* Make sure that that this is still a 24bit jump */ 229 if (!is_bl_op(op)) { 230 pr_err("Not expected bl: opcode is %x\n", op); 231 return -EINVAL; 232 } 233 234 /* lets find where the pointer goes */ 235 tramp = find_bl_target(ip, op); 236 237 /* 238 * On PPC32 the trampoline looks like: 239 * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha 240 * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l 241 * 0x7d, 0x89, 0x03, 0xa6 mtctr r12 242 * 0x4e, 0x80, 0x04, 0x20 bctr 243 */ 244 245 pr_devel("ip:%lx jumps to %lx", ip, tramp); 246 247 /* Find where the trampoline jumps to */ 248 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) { 249 pr_err("Failed to read %lx\n", tramp); 250 return -EFAULT; 251 } 252 253 pr_devel(" %08x %08x ", jmp[0], jmp[1]); 254 255 /* verify that this is what we expect it to be */ 256 if (((jmp[0] & 0xffff0000) != 0x3d800000) || 257 ((jmp[1] & 0xffff0000) != 0x398c0000) || 258 (jmp[2] != 0x7d8903a6) || 259 (jmp[3] != 0x4e800420)) { 260 pr_err("Not a trampoline\n"); 261 return -EINVAL; 262 } 263 264 tramp = (jmp[1] & 0xffff) | 265 ((jmp[0] & 0xffff) << 16); 266 if (tramp & 0x8000) 267 tramp -= 0x10000; 268 269 pr_devel(" %lx ", tramp); 270 271 if (tramp != addr) { 272 pr_err("Trampoline location %08lx does not match addr\n", 273 tramp); 274 return -EINVAL; 275 } 276 277 op = PPC_INST_NOP; 278 279 if (patch_instruction((unsigned int *)ip, op)) 280 return -EPERM; 281 282 return 0; 283 } 284 #endif /* PPC64 */ 285 #endif /* CONFIG_MODULES */ 286 287 static unsigned long find_ftrace_tramp(unsigned long ip) 288 { 289 int i; 290 291 /* 292 * We have the compiler generated long_branch tramps at the end 293 * and we prefer those 294 */ 295 for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--) 296 if (!ftrace_tramps[i]) 297 continue; 298 else if (create_branch((void *)ip, ftrace_tramps[i], 0)) 299 return ftrace_tramps[i]; 300 301 return 0; 302 } 303 304 static int add_ftrace_tramp(unsigned long tramp) 305 { 306 int i; 307 308 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 309 if (!ftrace_tramps[i]) { 310 ftrace_tramps[i] = tramp; 311 return 0; 312 } 313 314 return -1; 315 } 316 317 /* 318 * If this is a compiler generated long_branch trampoline (essentially, a 319 * trampoline that has a branch to _mcount()), we re-write the branch to 320 * instead go to ftrace_[regs_]caller() and note down the location of this 321 * trampoline. 322 */ 323 static int setup_mcount_compiler_tramp(unsigned long tramp) 324 { 325 int i, op; 326 unsigned long ptr; 327 static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS]; 328 329 /* Is this a known long jump tramp? */ 330 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 331 if (!ftrace_tramps[i]) 332 break; 333 else if (ftrace_tramps[i] == tramp) 334 return 0; 335 336 /* Is this a known plt tramp? */ 337 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 338 if (!ftrace_plt_tramps[i]) 339 break; 340 else if (ftrace_plt_tramps[i] == tramp) 341 return -1; 342 343 /* New trampoline -- read where this goes */ 344 if (probe_kernel_read(&op, (void *)tramp, sizeof(int))) { 345 pr_debug("Fetching opcode failed.\n"); 346 return -1; 347 } 348 349 /* Is this a 24 bit branch? */ 350 if (!is_b_op(op)) { 351 pr_debug("Trampoline is not a long branch tramp.\n"); 352 return -1; 353 } 354 355 /* lets find where the pointer goes */ 356 ptr = find_bl_target(tramp, op); 357 358 if (ptr != ppc_global_function_entry((void *)_mcount)) { 359 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr); 360 return -1; 361 } 362 363 /* Let's re-write the tramp to go to ftrace_[regs_]caller */ 364 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 365 ptr = ppc_global_function_entry((void *)ftrace_regs_caller); 366 #else 367 ptr = ppc_global_function_entry((void *)ftrace_caller); 368 #endif 369 if (!create_branch((void *)tramp, ptr, 0)) { 370 pr_debug("%ps is not reachable from existing mcount tramp\n", 371 (void *)ptr); 372 return -1; 373 } 374 375 if (patch_branch((unsigned int *)tramp, ptr, 0)) { 376 pr_debug("REL24 out of range!\n"); 377 return -1; 378 } 379 380 if (add_ftrace_tramp(tramp)) { 381 pr_debug("No tramp locations left\n"); 382 return -1; 383 } 384 385 return 0; 386 } 387 388 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) 389 { 390 unsigned long tramp, ip = rec->ip; 391 unsigned int op; 392 393 /* Read where this goes */ 394 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 395 pr_err("Fetching opcode failed.\n"); 396 return -EFAULT; 397 } 398 399 /* Make sure that that this is still a 24bit jump */ 400 if (!is_bl_op(op)) { 401 pr_err("Not expected bl: opcode is %x\n", op); 402 return -EINVAL; 403 } 404 405 /* Let's find where the pointer goes */ 406 tramp = find_bl_target(ip, op); 407 408 pr_devel("ip:%lx jumps to %lx", ip, tramp); 409 410 if (setup_mcount_compiler_tramp(tramp)) { 411 /* Are other trampolines reachable? */ 412 if (!find_ftrace_tramp(ip)) { 413 pr_err("No ftrace trampolines reachable from %ps\n", 414 (void *)ip); 415 return -EINVAL; 416 } 417 } 418 419 if (patch_instruction((unsigned int *)ip, PPC_INST_NOP)) { 420 pr_err("Patching NOP failed.\n"); 421 return -EPERM; 422 } 423 424 return 0; 425 } 426 427 int ftrace_make_nop(struct module *mod, 428 struct dyn_ftrace *rec, unsigned long addr) 429 { 430 unsigned long ip = rec->ip; 431 unsigned int old, new; 432 433 /* 434 * If the calling address is more that 24 bits away, 435 * then we had to use a trampoline to make the call. 436 * Otherwise just update the call site. 437 */ 438 if (test_24bit_addr(ip, addr)) { 439 /* within range */ 440 old = ftrace_call_replace(ip, addr, 1); 441 new = PPC_INST_NOP; 442 return ftrace_modify_code(ip, old, new); 443 } else if (core_kernel_text(ip)) 444 return __ftrace_make_nop_kernel(rec, addr); 445 446 #ifdef CONFIG_MODULES 447 /* 448 * Out of range jumps are called from modules. 449 * We should either already have a pointer to the module 450 * or it has been passed in. 451 */ 452 if (!rec->arch.mod) { 453 if (!mod) { 454 pr_err("No module loaded addr=%lx\n", addr); 455 return -EFAULT; 456 } 457 rec->arch.mod = mod; 458 } else if (mod) { 459 if (mod != rec->arch.mod) { 460 pr_err("Record mod %p not equal to passed in mod %p\n", 461 rec->arch.mod, mod); 462 return -EINVAL; 463 } 464 /* nothing to do if mod == rec->arch.mod */ 465 } else 466 mod = rec->arch.mod; 467 468 return __ftrace_make_nop(mod, rec, addr); 469 #else 470 /* We should not get here without modules */ 471 return -EINVAL; 472 #endif /* CONFIG_MODULES */ 473 } 474 475 #ifdef CONFIG_MODULES 476 #ifdef CONFIG_PPC64 477 /* 478 * Examine the existing instructions for __ftrace_make_call. 479 * They should effectively be a NOP, and follow formal constraints, 480 * depending on the ABI. Return false if they don't. 481 */ 482 #ifndef CONFIG_MPROFILE_KERNEL 483 static int 484 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1) 485 { 486 /* 487 * We expect to see: 488 * 489 * b +8 490 * ld r2,XX(r1) 491 * 492 * The load offset is different depending on the ABI. For simplicity 493 * just mask it out when doing the compare. 494 */ 495 if ((op0 != 0x48000008) || ((op1 & 0xffff0000) != 0xe8410000)) 496 return 0; 497 return 1; 498 } 499 #else 500 static int 501 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1) 502 { 503 /* look for patched "NOP" on ppc64 with -mprofile-kernel */ 504 if (op0 != PPC_INST_NOP) 505 return 0; 506 return 1; 507 } 508 #endif 509 510 static int 511 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 512 { 513 unsigned int op[2]; 514 void *ip = (void *)rec->ip; 515 unsigned long entry, ptr, tramp; 516 struct module *mod = rec->arch.mod; 517 518 /* read where this goes */ 519 if (probe_kernel_read(op, ip, sizeof(op))) 520 return -EFAULT; 521 522 if (!expected_nop_sequence(ip, op[0], op[1])) { 523 pr_err("Unexpected call sequence at %p: %x %x\n", 524 ip, op[0], op[1]); 525 return -EINVAL; 526 } 527 528 /* If we never set up ftrace trampoline(s), then bail */ 529 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 530 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 531 #else 532 if (!mod->arch.tramp) { 533 #endif 534 pr_err("No ftrace trampoline\n"); 535 return -EINVAL; 536 } 537 538 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 539 if (rec->flags & FTRACE_FL_REGS) 540 tramp = mod->arch.tramp_regs; 541 else 542 #endif 543 tramp = mod->arch.tramp; 544 545 if (module_trampoline_target(mod, tramp, &ptr)) { 546 pr_err("Failed to get trampoline target\n"); 547 return -EFAULT; 548 } 549 550 pr_devel("trampoline target %lx", ptr); 551 552 entry = ppc_global_function_entry((void *)addr); 553 /* This should match what was called */ 554 if (ptr != entry) { 555 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 556 return -EINVAL; 557 } 558 559 /* Ensure branch is within 24 bits */ 560 if (!create_branch(ip, tramp, BRANCH_SET_LINK)) { 561 pr_err("Branch out of range\n"); 562 return -EINVAL; 563 } 564 565 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 566 pr_err("REL24 out of range!\n"); 567 return -EINVAL; 568 } 569 570 return 0; 571 } 572 573 #else /* !CONFIG_PPC64: */ 574 static int 575 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 576 { 577 unsigned int op; 578 unsigned long ip = rec->ip; 579 580 /* read where this goes */ 581 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 582 return -EFAULT; 583 584 /* It should be pointing to a nop */ 585 if (op != PPC_INST_NOP) { 586 pr_err("Expected NOP but have %x\n", op); 587 return -EINVAL; 588 } 589 590 /* If we never set up a trampoline to ftrace_caller, then bail */ 591 if (!rec->arch.mod->arch.tramp) { 592 pr_err("No ftrace trampoline\n"); 593 return -EINVAL; 594 } 595 596 /* create the branch to the trampoline */ 597 op = create_branch((unsigned int *)ip, 598 rec->arch.mod->arch.tramp, BRANCH_SET_LINK); 599 if (!op) { 600 pr_err("REL24 out of range!\n"); 601 return -EINVAL; 602 } 603 604 pr_devel("write to %lx\n", rec->ip); 605 606 if (patch_instruction((unsigned int *)ip, op)) 607 return -EPERM; 608 609 return 0; 610 } 611 #endif /* CONFIG_PPC64 */ 612 #endif /* CONFIG_MODULES */ 613 614 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) 615 { 616 unsigned int op; 617 void *ip = (void *)rec->ip; 618 unsigned long tramp, entry, ptr; 619 620 /* Make sure we're being asked to patch branch to a known ftrace addr */ 621 entry = ppc_global_function_entry((void *)ftrace_caller); 622 ptr = ppc_global_function_entry((void *)addr); 623 624 if (ptr != entry) { 625 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 626 entry = ppc_global_function_entry((void *)ftrace_regs_caller); 627 if (ptr != entry) { 628 #endif 629 pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr); 630 return -EINVAL; 631 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 632 } 633 #endif 634 } 635 636 /* Make sure we have a nop */ 637 if (probe_kernel_read(&op, ip, sizeof(op))) { 638 pr_err("Unable to read ftrace location %p\n", ip); 639 return -EFAULT; 640 } 641 642 if (op != PPC_INST_NOP) { 643 pr_err("Unexpected call sequence at %p: %x\n", ip, op); 644 return -EINVAL; 645 } 646 647 tramp = find_ftrace_tramp((unsigned long)ip); 648 if (!tramp) { 649 pr_err("No ftrace trampolines reachable from %ps\n", ip); 650 return -EINVAL; 651 } 652 653 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 654 pr_err("Error patching branch to ftrace tramp!\n"); 655 return -EINVAL; 656 } 657 658 return 0; 659 } 660 661 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 662 { 663 unsigned long ip = rec->ip; 664 unsigned int old, new; 665 666 /* 667 * If the calling address is more that 24 bits away, 668 * then we had to use a trampoline to make the call. 669 * Otherwise just update the call site. 670 */ 671 if (test_24bit_addr(ip, addr)) { 672 /* within range */ 673 old = PPC_INST_NOP; 674 new = ftrace_call_replace(ip, addr, 1); 675 return ftrace_modify_code(ip, old, new); 676 } else if (core_kernel_text(ip)) 677 return __ftrace_make_call_kernel(rec, addr); 678 679 #ifdef CONFIG_MODULES 680 /* 681 * Out of range jumps are called from modules. 682 * Being that we are converting from nop, it had better 683 * already have a module defined. 684 */ 685 if (!rec->arch.mod) { 686 pr_err("No module loaded\n"); 687 return -EINVAL; 688 } 689 690 return __ftrace_make_call(rec, addr); 691 #else 692 /* We should not get here without modules */ 693 return -EINVAL; 694 #endif /* CONFIG_MODULES */ 695 } 696 697 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 698 #ifdef CONFIG_MODULES 699 static int 700 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 701 unsigned long addr) 702 { 703 unsigned int op; 704 unsigned long ip = rec->ip; 705 unsigned long entry, ptr, tramp; 706 struct module *mod = rec->arch.mod; 707 708 /* If we never set up ftrace trampolines, then bail */ 709 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 710 pr_err("No ftrace trampoline\n"); 711 return -EINVAL; 712 } 713 714 /* read where this goes */ 715 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 716 pr_err("Fetching opcode failed.\n"); 717 return -EFAULT; 718 } 719 720 /* Make sure that that this is still a 24bit jump */ 721 if (!is_bl_op(op)) { 722 pr_err("Not expected bl: opcode is %x\n", op); 723 return -EINVAL; 724 } 725 726 /* lets find where the pointer goes */ 727 tramp = find_bl_target(ip, op); 728 entry = ppc_global_function_entry((void *)old_addr); 729 730 pr_devel("ip:%lx jumps to %lx", ip, tramp); 731 732 if (tramp != entry) { 733 /* old_addr is not within range, so we must have used a trampoline */ 734 if (module_trampoline_target(mod, tramp, &ptr)) { 735 pr_err("Failed to get trampoline target\n"); 736 return -EFAULT; 737 } 738 739 pr_devel("trampoline target %lx", ptr); 740 741 /* This should match what was called */ 742 if (ptr != entry) { 743 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 744 return -EINVAL; 745 } 746 } 747 748 /* The new target may be within range */ 749 if (test_24bit_addr(ip, addr)) { 750 /* within range */ 751 if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) { 752 pr_err("REL24 out of range!\n"); 753 return -EINVAL; 754 } 755 756 return 0; 757 } 758 759 if (rec->flags & FTRACE_FL_REGS) 760 tramp = mod->arch.tramp_regs; 761 else 762 tramp = mod->arch.tramp; 763 764 if (module_trampoline_target(mod, tramp, &ptr)) { 765 pr_err("Failed to get trampoline target\n"); 766 return -EFAULT; 767 } 768 769 pr_devel("trampoline target %lx", ptr); 770 771 entry = ppc_global_function_entry((void *)addr); 772 /* This should match what was called */ 773 if (ptr != entry) { 774 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 775 return -EINVAL; 776 } 777 778 /* Ensure branch is within 24 bits */ 779 if (!create_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) { 780 pr_err("Branch out of range\n"); 781 return -EINVAL; 782 } 783 784 if (patch_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) { 785 pr_err("REL24 out of range!\n"); 786 return -EINVAL; 787 } 788 789 return 0; 790 } 791 #endif 792 793 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 794 unsigned long addr) 795 { 796 unsigned long ip = rec->ip; 797 unsigned int old, new; 798 799 /* 800 * If the calling address is more that 24 bits away, 801 * then we had to use a trampoline to make the call. 802 * Otherwise just update the call site. 803 */ 804 if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) { 805 /* within range */ 806 old = ftrace_call_replace(ip, old_addr, 1); 807 new = ftrace_call_replace(ip, addr, 1); 808 return ftrace_modify_code(ip, old, new); 809 } else if (core_kernel_text(ip)) { 810 /* 811 * We always patch out of range locations to go to the regs 812 * variant, so there is nothing to do here 813 */ 814 return 0; 815 } 816 817 #ifdef CONFIG_MODULES 818 /* 819 * Out of range jumps are called from modules. 820 */ 821 if (!rec->arch.mod) { 822 pr_err("No module loaded\n"); 823 return -EINVAL; 824 } 825 826 return __ftrace_modify_call(rec, old_addr, addr); 827 #else 828 /* We should not get here without modules */ 829 return -EINVAL; 830 #endif /* CONFIG_MODULES */ 831 } 832 #endif 833 834 int ftrace_update_ftrace_func(ftrace_func_t func) 835 { 836 unsigned long ip = (unsigned long)(&ftrace_call); 837 unsigned int old, new; 838 int ret; 839 840 old = *(unsigned int *)&ftrace_call; 841 new = ftrace_call_replace(ip, (unsigned long)func, 1); 842 ret = ftrace_modify_code(ip, old, new); 843 844 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 845 /* Also update the regs callback function */ 846 if (!ret) { 847 ip = (unsigned long)(&ftrace_regs_call); 848 old = *(unsigned int *)&ftrace_regs_call; 849 new = ftrace_call_replace(ip, (unsigned long)func, 1); 850 ret = ftrace_modify_code(ip, old, new); 851 } 852 #endif 853 854 return ret; 855 } 856 857 /* 858 * Use the default ftrace_modify_all_code, but without 859 * stop_machine(). 860 */ 861 void arch_ftrace_update_code(int command) 862 { 863 ftrace_modify_all_code(command); 864 } 865 866 #ifdef CONFIG_PPC64 867 #define PACATOC offsetof(struct paca_struct, kernel_toc) 868 869 #define PPC_LO(v) ((v) & 0xffff) 870 #define PPC_HI(v) (((v) >> 16) & 0xffff) 871 #define PPC_HA(v) PPC_HI ((v) + 0x8000) 872 873 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; 874 875 int __init ftrace_dyn_arch_init(void) 876 { 877 int i; 878 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; 879 u32 stub_insns[] = { 880 0xe98d0000 | PACATOC, /* ld r12,PACATOC(r13) */ 881 0x3d8c0000, /* addis r12,r12,<high> */ 882 0x398c0000, /* addi r12,r12,<low> */ 883 0x7d8903a6, /* mtctr r12 */ 884 0x4e800420, /* bctr */ 885 }; 886 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 887 unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller); 888 #else 889 unsigned long addr = ppc_global_function_entry((void *)ftrace_caller); 890 #endif 891 long reladdr = addr - kernel_toc_addr(); 892 893 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { 894 pr_err("Address of %ps out of range of kernel_toc.\n", 895 (void *)addr); 896 return -1; 897 } 898 899 for (i = 0; i < 2; i++) { 900 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 901 tramp[i][1] |= PPC_HA(reladdr); 902 tramp[i][2] |= PPC_LO(reladdr); 903 add_ftrace_tramp((unsigned long)tramp[i]); 904 } 905 906 return 0; 907 } 908 #else 909 int __init ftrace_dyn_arch_init(void) 910 { 911 return 0; 912 } 913 #endif 914 #endif /* CONFIG_DYNAMIC_FTRACE */ 915 916 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 917 918 extern void ftrace_graph_call(void); 919 extern void ftrace_graph_stub(void); 920 921 int ftrace_enable_ftrace_graph_caller(void) 922 { 923 unsigned long ip = (unsigned long)(&ftrace_graph_call); 924 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 925 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 926 unsigned int old, new; 927 928 old = ftrace_call_replace(ip, stub, 0); 929 new = ftrace_call_replace(ip, addr, 0); 930 931 return ftrace_modify_code(ip, old, new); 932 } 933 934 int ftrace_disable_ftrace_graph_caller(void) 935 { 936 unsigned long ip = (unsigned long)(&ftrace_graph_call); 937 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 938 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 939 unsigned int old, new; 940 941 old = ftrace_call_replace(ip, addr, 0); 942 new = ftrace_call_replace(ip, stub, 0); 943 944 return ftrace_modify_code(ip, old, new); 945 } 946 947 /* 948 * Hook the return address and push it in the stack of return addrs 949 * in current thread info. Return the address we want to divert to. 950 */ 951 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip) 952 { 953 unsigned long return_hooker; 954 955 if (unlikely(ftrace_graph_is_dead())) 956 goto out; 957 958 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 959 goto out; 960 961 return_hooker = ppc_function_entry(return_to_handler); 962 963 if (!function_graph_enter(parent, ip, 0, NULL)) 964 parent = return_hooker; 965 out: 966 return parent; 967 } 968 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 969 970 #if defined(CONFIG_FTRACE_SYSCALLS) && defined(CONFIG_PPC64) 971 unsigned long __init arch_syscall_addr(int nr) 972 { 973 return sys_call_table[nr*2]; 974 } 975 #endif /* CONFIG_FTRACE_SYSCALLS && CONFIG_PPC64 */ 976 977 #ifdef PPC64_ELF_ABI_v1 978 char *arch_ftrace_match_adjust(char *str, const char *search) 979 { 980 if (str[0] == '.' && search[0] != '.') 981 return str + 1; 982 else 983 return str; 984 } 985 #endif /* PPC64_ELF_ABI_v1 */ 986