1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Code for replacing ftrace calls with jumps. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * 7 * Thanks goes out to P.A. Semi, Inc for supplying me with a PPC64 box. 8 * 9 * Added function graph tracer code, taken from x86 that was written 10 * by Frederic Weisbecker, and ported to PPC by Steven Rostedt. 11 * 12 */ 13 14 #define pr_fmt(fmt) "ftrace-powerpc: " fmt 15 16 #include <linux/spinlock.h> 17 #include <linux/hardirq.h> 18 #include <linux/uaccess.h> 19 #include <linux/module.h> 20 #include <linux/ftrace.h> 21 #include <linux/percpu.h> 22 #include <linux/init.h> 23 #include <linux/list.h> 24 25 #include <asm/asm-prototypes.h> 26 #include <asm/cacheflush.h> 27 #include <asm/code-patching.h> 28 #include <asm/ftrace.h> 29 #include <asm/syscall.h> 30 #include <asm/inst.h> 31 32 33 #ifdef CONFIG_DYNAMIC_FTRACE 34 35 /* 36 * We generally only have a single long_branch tramp and at most 2 or 3 plt 37 * tramps generated. But, we don't use the plt tramps currently. We also allot 38 * 2 tramps after .text and .init.text. So, we only end up with around 3 usable 39 * tramps in total. Set aside 8 just to be sure. 40 */ 41 #define NUM_FTRACE_TRAMPS 8 42 static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS]; 43 44 static unsigned int 45 ftrace_call_replace(unsigned long ip, unsigned long addr, int link) 46 { 47 unsigned int op; 48 49 addr = ppc_function_entry((void *)addr); 50 51 /* if (link) set op to 'bl' else 'b' */ 52 create_branch(&op, (unsigned int *)ip, addr, link ? 1 : 0); 53 54 return op; 55 } 56 57 static int 58 ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new) 59 { 60 unsigned int replaced; 61 62 /* 63 * Note: 64 * We are paranoid about modifying text, as if a bug was to happen, it 65 * could cause us to read or write to someplace that could cause harm. 66 * Carefully read and modify the code with probe_kernel_*(), and make 67 * sure what we read is what we expected it to be before modifying it. 68 */ 69 70 /* read the text we want to modify */ 71 if (probe_kernel_read(&replaced, (void *)ip, MCOUNT_INSN_SIZE)) 72 return -EFAULT; 73 74 /* Make sure it is what we expect it to be */ 75 if (!ppc_inst_equal(replaced, old)) { 76 pr_err("%p: replaced (%#x) != old (%#x)", 77 (void *)ip, ppc_inst_val(replaced), ppc_inst_val(old)); 78 return -EINVAL; 79 } 80 81 /* replace the text with the new text */ 82 if (patch_instruction((unsigned int *)ip, new)) 83 return -EPERM; 84 85 return 0; 86 } 87 88 /* 89 * Helper functions that are the same for both PPC64 and PPC32. 90 */ 91 static int test_24bit_addr(unsigned long ip, unsigned long addr) 92 { 93 unsigned int op; 94 addr = ppc_function_entry((void *)addr); 95 96 /* use the create_branch to verify that this offset can be branched */ 97 return create_branch(&op, (unsigned int *)ip, addr, 0) == 0; 98 } 99 100 static int is_bl_op(unsigned int op) 101 { 102 return (ppc_inst_val(op) & 0xfc000003) == 0x48000001; 103 } 104 105 static int is_b_op(unsigned int op) 106 { 107 return (ppc_inst_val(op) & 0xfc000003) == 0x48000000; 108 } 109 110 static unsigned long find_bl_target(unsigned long ip, unsigned int op) 111 { 112 int offset; 113 114 offset = (ppc_inst_val(op) & 0x03fffffc); 115 /* make it signed */ 116 if (offset & 0x02000000) 117 offset |= 0xfe000000; 118 119 return ip + (long)offset; 120 } 121 122 #ifdef CONFIG_MODULES 123 #ifdef CONFIG_PPC64 124 static int 125 __ftrace_make_nop(struct module *mod, 126 struct dyn_ftrace *rec, unsigned long addr) 127 { 128 unsigned long entry, ptr, tramp; 129 unsigned long ip = rec->ip; 130 unsigned int op, pop; 131 132 /* read where this goes */ 133 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 134 pr_err("Fetching opcode failed.\n"); 135 return -EFAULT; 136 } 137 138 /* Make sure that that this is still a 24bit jump */ 139 if (!is_bl_op(op)) { 140 pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op)); 141 return -EINVAL; 142 } 143 144 /* lets find where the pointer goes */ 145 tramp = find_bl_target(ip, op); 146 147 pr_devel("ip:%lx jumps to %lx", ip, tramp); 148 149 if (module_trampoline_target(mod, tramp, &ptr)) { 150 pr_err("Failed to get trampoline target\n"); 151 return -EFAULT; 152 } 153 154 pr_devel("trampoline target %lx", ptr); 155 156 entry = ppc_global_function_entry((void *)addr); 157 /* This should match what was called */ 158 if (ptr != entry) { 159 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 160 return -EINVAL; 161 } 162 163 #ifdef CONFIG_MPROFILE_KERNEL 164 /* When using -mkernel_profile there is no load to jump over */ 165 pop = ppc_inst(PPC_INST_NOP); 166 167 if (probe_kernel_read(&op, (void *)(ip - 4), 4)) { 168 pr_err("Fetching instruction at %lx failed.\n", ip - 4); 169 return -EFAULT; 170 } 171 172 /* We expect either a mflr r0, or a std r0, LRSAVE(r1) */ 173 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_MFLR)) && 174 !ppc_inst_equal(op, ppc_inst(PPC_INST_STD_LR))) { 175 pr_err("Unexpected instruction %08x around bl _mcount\n", 176 ppc_inst_val(op)); 177 return -EINVAL; 178 } 179 #else 180 /* 181 * Our original call site looks like: 182 * 183 * bl <tramp> 184 * ld r2,XX(r1) 185 * 186 * Milton Miller pointed out that we can not simply nop the branch. 187 * If a task was preempted when calling a trace function, the nops 188 * will remove the way to restore the TOC in r2 and the r2 TOC will 189 * get corrupted. 190 * 191 * Use a b +8 to jump over the load. 192 */ 193 194 pop = ppc_inst(PPC_INST_BRANCH | 8); /* b +8 */ 195 196 /* 197 * Check what is in the next instruction. We can see ld r2,40(r1), but 198 * on first pass after boot we will see mflr r0. 199 */ 200 if (probe_kernel_read(&op, (void *)(ip+4), MCOUNT_INSN_SIZE)) { 201 pr_err("Fetching op failed.\n"); 202 return -EFAULT; 203 } 204 205 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_LD_TOC))) { 206 pr_err("Expected %08x found %08x\n", PPC_INST_LD_TOC, ppc_inst_val(op)); 207 return -EINVAL; 208 } 209 #endif /* CONFIG_MPROFILE_KERNEL */ 210 211 if (patch_instruction((unsigned int *)ip, pop)) { 212 pr_err("Patching NOP failed.\n"); 213 return -EPERM; 214 } 215 216 return 0; 217 } 218 219 #else /* !PPC64 */ 220 static int 221 __ftrace_make_nop(struct module *mod, 222 struct dyn_ftrace *rec, unsigned long addr) 223 { 224 unsigned int op; 225 unsigned int jmp[4]; 226 unsigned long ip = rec->ip; 227 unsigned long tramp; 228 229 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 230 return -EFAULT; 231 232 /* Make sure that that this is still a 24bit jump */ 233 if (!is_bl_op(op)) { 234 pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op)); 235 return -EINVAL; 236 } 237 238 /* lets find where the pointer goes */ 239 tramp = find_bl_target(ip, op); 240 241 /* 242 * On PPC32 the trampoline looks like: 243 * 0x3d, 0x80, 0x00, 0x00 lis r12,sym@ha 244 * 0x39, 0x8c, 0x00, 0x00 addi r12,r12,sym@l 245 * 0x7d, 0x89, 0x03, 0xa6 mtctr r12 246 * 0x4e, 0x80, 0x04, 0x20 bctr 247 */ 248 249 pr_devel("ip:%lx jumps to %lx", ip, tramp); 250 251 /* Find where the trampoline jumps to */ 252 if (probe_kernel_read(jmp, (void *)tramp, sizeof(jmp))) { 253 pr_err("Failed to read %lx\n", tramp); 254 return -EFAULT; 255 } 256 257 pr_devel(" %08x %08x ", jmp[0], jmp[1]); 258 259 /* verify that this is what we expect it to be */ 260 if (((jmp[0] & 0xffff0000) != 0x3d800000) || 261 ((jmp[1] & 0xffff0000) != 0x398c0000) || 262 (jmp[2] != 0x7d8903a6) || 263 (jmp[3] != 0x4e800420)) { 264 pr_err("Not a trampoline\n"); 265 return -EINVAL; 266 } 267 268 tramp = (jmp[1] & 0xffff) | 269 ((jmp[0] & 0xffff) << 16); 270 if (tramp & 0x8000) 271 tramp -= 0x10000; 272 273 pr_devel(" %lx ", tramp); 274 275 if (tramp != addr) { 276 pr_err("Trampoline location %08lx does not match addr\n", 277 tramp); 278 return -EINVAL; 279 } 280 281 op = ppc_inst(PPC_INST_NOP); 282 283 if (patch_instruction((unsigned int *)ip, op)) 284 return -EPERM; 285 286 return 0; 287 } 288 #endif /* PPC64 */ 289 #endif /* CONFIG_MODULES */ 290 291 static unsigned long find_ftrace_tramp(unsigned long ip) 292 { 293 int i; 294 unsigned int instr; 295 296 /* 297 * We have the compiler generated long_branch tramps at the end 298 * and we prefer those 299 */ 300 for (i = NUM_FTRACE_TRAMPS - 1; i >= 0; i--) 301 if (!ftrace_tramps[i]) 302 continue; 303 else if (create_branch(&instr, (void *)ip, 304 ftrace_tramps[i], 0) == 0) 305 return ftrace_tramps[i]; 306 307 return 0; 308 } 309 310 static int add_ftrace_tramp(unsigned long tramp) 311 { 312 int i; 313 314 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 315 if (!ftrace_tramps[i]) { 316 ftrace_tramps[i] = tramp; 317 return 0; 318 } 319 320 return -1; 321 } 322 323 /* 324 * If this is a compiler generated long_branch trampoline (essentially, a 325 * trampoline that has a branch to _mcount()), we re-write the branch to 326 * instead go to ftrace_[regs_]caller() and note down the location of this 327 * trampoline. 328 */ 329 static int setup_mcount_compiler_tramp(unsigned long tramp) 330 { 331 int i, op; 332 unsigned long ptr; 333 unsigned int instr; 334 static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS]; 335 336 /* Is this a known long jump tramp? */ 337 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 338 if (!ftrace_tramps[i]) 339 break; 340 else if (ftrace_tramps[i] == tramp) 341 return 0; 342 343 /* Is this a known plt tramp? */ 344 for (i = 0; i < NUM_FTRACE_TRAMPS; i++) 345 if (!ftrace_plt_tramps[i]) 346 break; 347 else if (ftrace_plt_tramps[i] == tramp) 348 return -1; 349 350 /* New trampoline -- read where this goes */ 351 if (probe_kernel_read(&op, (void *)tramp, sizeof(int))) { 352 pr_debug("Fetching opcode failed.\n"); 353 return -1; 354 } 355 356 /* Is this a 24 bit branch? */ 357 if (!is_b_op(op)) { 358 pr_debug("Trampoline is not a long branch tramp.\n"); 359 return -1; 360 } 361 362 /* lets find where the pointer goes */ 363 ptr = find_bl_target(tramp, op); 364 365 if (ptr != ppc_global_function_entry((void *)_mcount)) { 366 pr_debug("Trampoline target %p is not _mcount\n", (void *)ptr); 367 return -1; 368 } 369 370 /* Let's re-write the tramp to go to ftrace_[regs_]caller */ 371 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 372 ptr = ppc_global_function_entry((void *)ftrace_regs_caller); 373 #else 374 ptr = ppc_global_function_entry((void *)ftrace_caller); 375 #endif 376 if (create_branch(&instr, (void *)tramp, ptr, 0)) { 377 pr_debug("%ps is not reachable from existing mcount tramp\n", 378 (void *)ptr); 379 return -1; 380 } 381 382 if (patch_branch((unsigned int *)tramp, ptr, 0)) { 383 pr_debug("REL24 out of range!\n"); 384 return -1; 385 } 386 387 if (add_ftrace_tramp(tramp)) { 388 pr_debug("No tramp locations left\n"); 389 return -1; 390 } 391 392 return 0; 393 } 394 395 static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long addr) 396 { 397 unsigned long tramp, ip = rec->ip; 398 unsigned int op; 399 400 /* Read where this goes */ 401 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 402 pr_err("Fetching opcode failed.\n"); 403 return -EFAULT; 404 } 405 406 /* Make sure that that this is still a 24bit jump */ 407 if (!is_bl_op(op)) { 408 pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op)); 409 return -EINVAL; 410 } 411 412 /* Let's find where the pointer goes */ 413 tramp = find_bl_target(ip, op); 414 415 pr_devel("ip:%lx jumps to %lx", ip, tramp); 416 417 if (setup_mcount_compiler_tramp(tramp)) { 418 /* Are other trampolines reachable? */ 419 if (!find_ftrace_tramp(ip)) { 420 pr_err("No ftrace trampolines reachable from %ps\n", 421 (void *)ip); 422 return -EINVAL; 423 } 424 } 425 426 if (patch_instruction((unsigned int *)ip, ppc_inst(PPC_INST_NOP))) { 427 pr_err("Patching NOP failed.\n"); 428 return -EPERM; 429 } 430 431 return 0; 432 } 433 434 int ftrace_make_nop(struct module *mod, 435 struct dyn_ftrace *rec, unsigned long addr) 436 { 437 unsigned long ip = rec->ip; 438 unsigned int old, new; 439 440 /* 441 * If the calling address is more that 24 bits away, 442 * then we had to use a trampoline to make the call. 443 * Otherwise just update the call site. 444 */ 445 if (test_24bit_addr(ip, addr)) { 446 /* within range */ 447 old = ftrace_call_replace(ip, addr, 1); 448 new = ppc_inst(PPC_INST_NOP); 449 return ftrace_modify_code(ip, old, new); 450 } else if (core_kernel_text(ip)) 451 return __ftrace_make_nop_kernel(rec, addr); 452 453 #ifdef CONFIG_MODULES 454 /* 455 * Out of range jumps are called from modules. 456 * We should either already have a pointer to the module 457 * or it has been passed in. 458 */ 459 if (!rec->arch.mod) { 460 if (!mod) { 461 pr_err("No module loaded addr=%lx\n", addr); 462 return -EFAULT; 463 } 464 rec->arch.mod = mod; 465 } else if (mod) { 466 if (mod != rec->arch.mod) { 467 pr_err("Record mod %p not equal to passed in mod %p\n", 468 rec->arch.mod, mod); 469 return -EINVAL; 470 } 471 /* nothing to do if mod == rec->arch.mod */ 472 } else 473 mod = rec->arch.mod; 474 475 return __ftrace_make_nop(mod, rec, addr); 476 #else 477 /* We should not get here without modules */ 478 return -EINVAL; 479 #endif /* CONFIG_MODULES */ 480 } 481 482 #ifdef CONFIG_MODULES 483 #ifdef CONFIG_PPC64 484 /* 485 * Examine the existing instructions for __ftrace_make_call. 486 * They should effectively be a NOP, and follow formal constraints, 487 * depending on the ABI. Return false if they don't. 488 */ 489 #ifndef CONFIG_MPROFILE_KERNEL 490 static int 491 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1) 492 { 493 /* 494 * We expect to see: 495 * 496 * b +8 497 * ld r2,XX(r1) 498 * 499 * The load offset is different depending on the ABI. For simplicity 500 * just mask it out when doing the compare. 501 */ 502 if (!ppc_inst_equal(op0, ppc_inst(0x48000008)) || 503 (ppc_inst_val(op1) & 0xffff0000) != 0xe8410000) 504 return 0; 505 return 1; 506 } 507 #else 508 static int 509 expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1) 510 { 511 /* look for patched "NOP" on ppc64 with -mprofile-kernel */ 512 if (!ppc_inst_equal(op0, ppc_inst(PPC_INST_NOP))) 513 return 0; 514 return 1; 515 } 516 #endif 517 518 static int 519 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 520 { 521 unsigned int op[2]; 522 unsigned int instr; 523 void *ip = (void *)rec->ip; 524 unsigned long entry, ptr, tramp; 525 struct module *mod = rec->arch.mod; 526 527 /* read where this goes */ 528 if (probe_kernel_read(op, ip, sizeof(op))) 529 return -EFAULT; 530 531 if (!expected_nop_sequence(ip, op[0], op[1])) { 532 pr_err("Unexpected call sequence at %p: %x %x\n", 533 ip, ppc_inst_val(op[0]), ppc_inst_val(op[1])); 534 return -EINVAL; 535 } 536 537 /* If we never set up ftrace trampoline(s), then bail */ 538 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 539 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 540 #else 541 if (!mod->arch.tramp) { 542 #endif 543 pr_err("No ftrace trampoline\n"); 544 return -EINVAL; 545 } 546 547 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 548 if (rec->flags & FTRACE_FL_REGS) 549 tramp = mod->arch.tramp_regs; 550 else 551 #endif 552 tramp = mod->arch.tramp; 553 554 if (module_trampoline_target(mod, tramp, &ptr)) { 555 pr_err("Failed to get trampoline target\n"); 556 return -EFAULT; 557 } 558 559 pr_devel("trampoline target %lx", ptr); 560 561 entry = ppc_global_function_entry((void *)addr); 562 /* This should match what was called */ 563 if (ptr != entry) { 564 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 565 return -EINVAL; 566 } 567 568 /* Ensure branch is within 24 bits */ 569 if (create_branch(&instr, ip, tramp, BRANCH_SET_LINK)) { 570 pr_err("Branch out of range\n"); 571 return -EINVAL; 572 } 573 574 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 575 pr_err("REL24 out of range!\n"); 576 return -EINVAL; 577 } 578 579 return 0; 580 } 581 582 #else /* !CONFIG_PPC64: */ 583 static int 584 __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 585 { 586 int err; 587 unsigned int op; 588 unsigned long ip = rec->ip; 589 590 /* read where this goes */ 591 if (probe_kernel_read(&op, (void *)ip, MCOUNT_INSN_SIZE)) 592 return -EFAULT; 593 594 /* It should be pointing to a nop */ 595 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_NOP))) { 596 pr_err("Expected NOP but have %x\n", ppc_inst_val(op)); 597 return -EINVAL; 598 } 599 600 /* If we never set up a trampoline to ftrace_caller, then bail */ 601 if (!rec->arch.mod->arch.tramp) { 602 pr_err("No ftrace trampoline\n"); 603 return -EINVAL; 604 } 605 606 /* create the branch to the trampoline */ 607 err = create_branch(&op, (unsigned int *)ip, 608 rec->arch.mod->arch.tramp, BRANCH_SET_LINK); 609 if (err) { 610 pr_err("REL24 out of range!\n"); 611 return -EINVAL; 612 } 613 614 pr_devel("write to %lx\n", rec->ip); 615 616 if (patch_instruction((unsigned int *)ip, op)) 617 return -EPERM; 618 619 return 0; 620 } 621 #endif /* CONFIG_PPC64 */ 622 #endif /* CONFIG_MODULES */ 623 624 static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long addr) 625 { 626 unsigned int op; 627 void *ip = (void *)rec->ip; 628 unsigned long tramp, entry, ptr; 629 630 /* Make sure we're being asked to patch branch to a known ftrace addr */ 631 entry = ppc_global_function_entry((void *)ftrace_caller); 632 ptr = ppc_global_function_entry((void *)addr); 633 634 if (ptr != entry) { 635 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 636 entry = ppc_global_function_entry((void *)ftrace_regs_caller); 637 if (ptr != entry) { 638 #endif 639 pr_err("Unknown ftrace addr to patch: %ps\n", (void *)ptr); 640 return -EINVAL; 641 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 642 } 643 #endif 644 } 645 646 /* Make sure we have a nop */ 647 if (probe_kernel_read(&op, ip, sizeof(op))) { 648 pr_err("Unable to read ftrace location %p\n", ip); 649 return -EFAULT; 650 } 651 652 if (!ppc_inst_equal(op, ppc_inst(PPC_INST_NOP))) { 653 pr_err("Unexpected call sequence at %p: %x\n", ip, ppc_inst_val(op)); 654 return -EINVAL; 655 } 656 657 tramp = find_ftrace_tramp((unsigned long)ip); 658 if (!tramp) { 659 pr_err("No ftrace trampolines reachable from %ps\n", ip); 660 return -EINVAL; 661 } 662 663 if (patch_branch(ip, tramp, BRANCH_SET_LINK)) { 664 pr_err("Error patching branch to ftrace tramp!\n"); 665 return -EINVAL; 666 } 667 668 return 0; 669 } 670 671 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 672 { 673 unsigned long ip = rec->ip; 674 unsigned int old, new; 675 676 /* 677 * If the calling address is more that 24 bits away, 678 * then we had to use a trampoline to make the call. 679 * Otherwise just update the call site. 680 */ 681 if (test_24bit_addr(ip, addr)) { 682 /* within range */ 683 old = ppc_inst(PPC_INST_NOP); 684 new = ftrace_call_replace(ip, addr, 1); 685 return ftrace_modify_code(ip, old, new); 686 } else if (core_kernel_text(ip)) 687 return __ftrace_make_call_kernel(rec, addr); 688 689 #ifdef CONFIG_MODULES 690 /* 691 * Out of range jumps are called from modules. 692 * Being that we are converting from nop, it had better 693 * already have a module defined. 694 */ 695 if (!rec->arch.mod) { 696 pr_err("No module loaded\n"); 697 return -EINVAL; 698 } 699 700 return __ftrace_make_call(rec, addr); 701 #else 702 /* We should not get here without modules */ 703 return -EINVAL; 704 #endif /* CONFIG_MODULES */ 705 } 706 707 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 708 #ifdef CONFIG_MODULES 709 static int 710 __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 711 unsigned long addr) 712 { 713 unsigned int op; 714 unsigned long ip = rec->ip; 715 unsigned long entry, ptr, tramp; 716 struct module *mod = rec->arch.mod; 717 718 /* If we never set up ftrace trampolines, then bail */ 719 if (!mod->arch.tramp || !mod->arch.tramp_regs) { 720 pr_err("No ftrace trampoline\n"); 721 return -EINVAL; 722 } 723 724 /* read where this goes */ 725 if (probe_kernel_read(&op, (void *)ip, sizeof(int))) { 726 pr_err("Fetching opcode failed.\n"); 727 return -EFAULT; 728 } 729 730 /* Make sure that that this is still a 24bit jump */ 731 if (!is_bl_op(op)) { 732 pr_err("Not expected bl: opcode is %x\n", ppc_inst_val(op)); 733 return -EINVAL; 734 } 735 736 /* lets find where the pointer goes */ 737 tramp = find_bl_target(ip, op); 738 entry = ppc_global_function_entry((void *)old_addr); 739 740 pr_devel("ip:%lx jumps to %lx", ip, tramp); 741 742 if (tramp != entry) { 743 /* old_addr is not within range, so we must have used a trampoline */ 744 if (module_trampoline_target(mod, tramp, &ptr)) { 745 pr_err("Failed to get trampoline target\n"); 746 return -EFAULT; 747 } 748 749 pr_devel("trampoline target %lx", ptr); 750 751 /* This should match what was called */ 752 if (ptr != entry) { 753 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 754 return -EINVAL; 755 } 756 } 757 758 /* The new target may be within range */ 759 if (test_24bit_addr(ip, addr)) { 760 /* within range */ 761 if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) { 762 pr_err("REL24 out of range!\n"); 763 return -EINVAL; 764 } 765 766 return 0; 767 } 768 769 if (rec->flags & FTRACE_FL_REGS) 770 tramp = mod->arch.tramp_regs; 771 else 772 tramp = mod->arch.tramp; 773 774 if (module_trampoline_target(mod, tramp, &ptr)) { 775 pr_err("Failed to get trampoline target\n"); 776 return -EFAULT; 777 } 778 779 pr_devel("trampoline target %lx", ptr); 780 781 entry = ppc_global_function_entry((void *)addr); 782 /* This should match what was called */ 783 if (ptr != entry) { 784 pr_err("addr %lx does not match expected %lx\n", ptr, entry); 785 return -EINVAL; 786 } 787 788 /* Ensure branch is within 24 bits */ 789 if (create_branch(&op, (unsigned int *)ip, tramp, BRANCH_SET_LINK)) { 790 pr_err("Branch out of range\n"); 791 return -EINVAL; 792 } 793 794 if (patch_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) { 795 pr_err("REL24 out of range!\n"); 796 return -EINVAL; 797 } 798 799 return 0; 800 } 801 #endif 802 803 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, 804 unsigned long addr) 805 { 806 unsigned long ip = rec->ip; 807 unsigned int old, new; 808 809 /* 810 * If the calling address is more that 24 bits away, 811 * then we had to use a trampoline to make the call. 812 * Otherwise just update the call site. 813 */ 814 if (test_24bit_addr(ip, addr) && test_24bit_addr(ip, old_addr)) { 815 /* within range */ 816 old = ftrace_call_replace(ip, old_addr, 1); 817 new = ftrace_call_replace(ip, addr, 1); 818 return ftrace_modify_code(ip, old, new); 819 } else if (core_kernel_text(ip)) { 820 /* 821 * We always patch out of range locations to go to the regs 822 * variant, so there is nothing to do here 823 */ 824 return 0; 825 } 826 827 #ifdef CONFIG_MODULES 828 /* 829 * Out of range jumps are called from modules. 830 */ 831 if (!rec->arch.mod) { 832 pr_err("No module loaded\n"); 833 return -EINVAL; 834 } 835 836 return __ftrace_modify_call(rec, old_addr, addr); 837 #else 838 /* We should not get here without modules */ 839 return -EINVAL; 840 #endif /* CONFIG_MODULES */ 841 } 842 #endif 843 844 int ftrace_update_ftrace_func(ftrace_func_t func) 845 { 846 unsigned long ip = (unsigned long)(&ftrace_call); 847 unsigned int old, new; 848 int ret; 849 850 old = *(unsigned int *)&ftrace_call; 851 new = ftrace_call_replace(ip, (unsigned long)func, 1); 852 ret = ftrace_modify_code(ip, old, new); 853 854 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 855 /* Also update the regs callback function */ 856 if (!ret) { 857 ip = (unsigned long)(&ftrace_regs_call); 858 old = *(unsigned int *)&ftrace_regs_call; 859 new = ftrace_call_replace(ip, (unsigned long)func, 1); 860 ret = ftrace_modify_code(ip, old, new); 861 } 862 #endif 863 864 return ret; 865 } 866 867 /* 868 * Use the default ftrace_modify_all_code, but without 869 * stop_machine(). 870 */ 871 void arch_ftrace_update_code(int command) 872 { 873 ftrace_modify_all_code(command); 874 } 875 876 #ifdef CONFIG_PPC64 877 #define PACATOC offsetof(struct paca_struct, kernel_toc) 878 879 extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; 880 881 int __init ftrace_dyn_arch_init(void) 882 { 883 int i; 884 unsigned int *tramp[] = { ftrace_tramp_text, ftrace_tramp_init }; 885 u32 stub_insns[] = { 886 0xe98d0000 | PACATOC, /* ld r12,PACATOC(r13) */ 887 0x3d8c0000, /* addis r12,r12,<high> */ 888 0x398c0000, /* addi r12,r12,<low> */ 889 0x7d8903a6, /* mtctr r12 */ 890 0x4e800420, /* bctr */ 891 }; 892 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 893 unsigned long addr = ppc_global_function_entry((void *)ftrace_regs_caller); 894 #else 895 unsigned long addr = ppc_global_function_entry((void *)ftrace_caller); 896 #endif 897 long reladdr = addr - kernel_toc_addr(); 898 899 if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) { 900 pr_err("Address of %ps out of range of kernel_toc.\n", 901 (void *)addr); 902 return -1; 903 } 904 905 for (i = 0; i < 2; i++) { 906 memcpy(tramp[i], stub_insns, sizeof(stub_insns)); 907 tramp[i][1] |= PPC_HA(reladdr); 908 tramp[i][2] |= PPC_LO(reladdr); 909 add_ftrace_tramp((unsigned long)tramp[i]); 910 } 911 912 return 0; 913 } 914 #else 915 int __init ftrace_dyn_arch_init(void) 916 { 917 return 0; 918 } 919 #endif 920 #endif /* CONFIG_DYNAMIC_FTRACE */ 921 922 #ifdef CONFIG_FUNCTION_GRAPH_TRACER 923 924 extern void ftrace_graph_call(void); 925 extern void ftrace_graph_stub(void); 926 927 int ftrace_enable_ftrace_graph_caller(void) 928 { 929 unsigned long ip = (unsigned long)(&ftrace_graph_call); 930 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 931 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 932 unsigned int old, new; 933 934 old = ftrace_call_replace(ip, stub, 0); 935 new = ftrace_call_replace(ip, addr, 0); 936 937 return ftrace_modify_code(ip, old, new); 938 } 939 940 int ftrace_disable_ftrace_graph_caller(void) 941 { 942 unsigned long ip = (unsigned long)(&ftrace_graph_call); 943 unsigned long addr = (unsigned long)(&ftrace_graph_caller); 944 unsigned long stub = (unsigned long)(&ftrace_graph_stub); 945 unsigned int old, new; 946 947 old = ftrace_call_replace(ip, addr, 0); 948 new = ftrace_call_replace(ip, stub, 0); 949 950 return ftrace_modify_code(ip, old, new); 951 } 952 953 /* 954 * Hook the return address and push it in the stack of return addrs 955 * in current thread info. Return the address we want to divert to. 956 */ 957 unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, 958 unsigned long sp) 959 { 960 unsigned long return_hooker; 961 962 if (unlikely(ftrace_graph_is_dead())) 963 goto out; 964 965 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 966 goto out; 967 968 return_hooker = ppc_function_entry(return_to_handler); 969 970 if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp)) 971 parent = return_hooker; 972 out: 973 return parent; 974 } 975 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 976 977 #ifdef PPC64_ELF_ABI_v1 978 char *arch_ftrace_match_adjust(char *str, const char *search) 979 { 980 if (str[0] == '.' && search[0] != '.') 981 return str + 1; 982 else 983 return str; 984 } 985 #endif /* PPC64_ELF_ABI_v1 */ 986