1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/kdebug.h> 3 #include <linux/kprobes.h> 4 #include <linux/preempt.h> 5 #include <asm/break.h> 6 7 #define KPROBE_BP_INSN larch_insn_gen_break(BRK_KPROBE_BP) 8 #define KPROBE_SSTEPBP_INSN larch_insn_gen_break(BRK_KPROBE_SSTEPBP) 9 10 DEFINE_PER_CPU(struct kprobe *, current_kprobe); 11 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 12 13 static void arch_prepare_ss_slot(struct kprobe *p) 14 { 15 p->ainsn.insn[0] = *p->addr; 16 p->ainsn.insn[1] = KPROBE_SSTEPBP_INSN; 17 p->ainsn.restore = (unsigned long)p->addr + LOONGARCH_INSN_SIZE; 18 } 19 NOKPROBE_SYMBOL(arch_prepare_ss_slot); 20 21 static void arch_prepare_simulate(struct kprobe *p) 22 { 23 p->ainsn.restore = 0; 24 } 25 NOKPROBE_SYMBOL(arch_prepare_simulate); 26 27 int arch_prepare_kprobe(struct kprobe *p) 28 { 29 union loongarch_instruction insn; 30 31 if ((unsigned long)p->addr & 0x3) 32 return -EILSEQ; 33 34 /* copy instruction */ 35 p->opcode = *p->addr; 36 insn.word = p->opcode; 37 38 /* decode instruction */ 39 if (insns_not_supported(insn)) 40 return -EINVAL; 41 42 if (insns_need_simulation(insn)) { 43 p->ainsn.insn = NULL; 44 } else { 45 p->ainsn.insn = get_insn_slot(); 46 if (!p->ainsn.insn) 47 return -ENOMEM; 48 } 49 50 /* prepare the instruction */ 51 if (p->ainsn.insn) 52 arch_prepare_ss_slot(p); 53 else 54 arch_prepare_simulate(p); 55 56 return 0; 57 } 58 NOKPROBE_SYMBOL(arch_prepare_kprobe); 59 60 /* Install breakpoint in text */ 61 void arch_arm_kprobe(struct kprobe *p) 62 { 63 *p->addr = KPROBE_BP_INSN; 64 flush_insn_slot(p); 65 } 66 NOKPROBE_SYMBOL(arch_arm_kprobe); 67 68 /* Remove breakpoint from text */ 69 void arch_disarm_kprobe(struct kprobe *p) 70 { 71 *p->addr = p->opcode; 72 flush_insn_slot(p); 73 } 74 NOKPROBE_SYMBOL(arch_disarm_kprobe); 75 76 void arch_remove_kprobe(struct kprobe *p) 77 { 78 if (p->ainsn.insn) { 79 free_insn_slot(p->ainsn.insn, 0); 80 p->ainsn.insn = NULL; 81 } 82 } 83 NOKPROBE_SYMBOL(arch_remove_kprobe); 84 85 static void save_previous_kprobe(struct kprobe_ctlblk *kcb) 86 { 87 kcb->prev_kprobe.kp = kprobe_running(); 88 kcb->prev_kprobe.status = kcb->kprobe_status; 89 } 90 NOKPROBE_SYMBOL(save_previous_kprobe); 91 92 static void restore_previous_kprobe(struct kprobe_ctlblk *kcb) 93 { 94 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 95 kcb->kprobe_status = kcb->prev_kprobe.status; 96 } 97 NOKPROBE_SYMBOL(restore_previous_kprobe); 98 99 static void set_current_kprobe(struct kprobe *p) 100 { 101 __this_cpu_write(current_kprobe, p); 102 } 103 NOKPROBE_SYMBOL(set_current_kprobe); 104 105 /* 106 * Interrupts need to be disabled before single-step mode is set, 107 * and not reenabled until after single-step mode ends. 108 * Without disabling interrupt on local CPU, there is a chance of 109 * interrupt occurrence in the period of exception return and start 110 * of out-of-line single-step, that result in wrongly single stepping 111 * into the interrupt handler. 112 */ 113 static void save_local_irqflag(struct kprobe_ctlblk *kcb, 114 struct pt_regs *regs) 115 { 116 kcb->saved_status = regs->csr_prmd; 117 regs->csr_prmd &= ~CSR_PRMD_PIE; 118 } 119 NOKPROBE_SYMBOL(save_local_irqflag); 120 121 static void restore_local_irqflag(struct kprobe_ctlblk *kcb, 122 struct pt_regs *regs) 123 { 124 regs->csr_prmd = kcb->saved_status; 125 } 126 NOKPROBE_SYMBOL(restore_local_irqflag); 127 128 static void post_kprobe_handler(struct kprobe *cur, struct kprobe_ctlblk *kcb, 129 struct pt_regs *regs) 130 { 131 /* return addr restore if non-branching insn */ 132 if (cur->ainsn.restore != 0) 133 instruction_pointer_set(regs, cur->ainsn.restore); 134 135 /* restore back original saved kprobe variables and continue */ 136 if (kcb->kprobe_status == KPROBE_REENTER) { 137 restore_previous_kprobe(kcb); 138 preempt_enable_no_resched(); 139 return; 140 } 141 142 /* 143 * update the kcb status even if the cur->post_handler is 144 * not set because reset_curent_kprobe() doesn't update kcb. 145 */ 146 kcb->kprobe_status = KPROBE_HIT_SSDONE; 147 if (cur->post_handler) 148 cur->post_handler(cur, regs, 0); 149 150 reset_current_kprobe(); 151 preempt_enable_no_resched(); 152 } 153 NOKPROBE_SYMBOL(post_kprobe_handler); 154 155 static void setup_singlestep(struct kprobe *p, struct pt_regs *regs, 156 struct kprobe_ctlblk *kcb, int reenter) 157 { 158 union loongarch_instruction insn; 159 160 if (reenter) { 161 save_previous_kprobe(kcb); 162 set_current_kprobe(p); 163 kcb->kprobe_status = KPROBE_REENTER; 164 } else { 165 kcb->kprobe_status = KPROBE_HIT_SS; 166 } 167 168 if (p->ainsn.insn) { 169 /* IRQs and single stepping do not mix well */ 170 save_local_irqflag(kcb, regs); 171 /* set ip register to prepare for single stepping */ 172 regs->csr_era = (unsigned long)p->ainsn.insn; 173 } else { 174 /* simulate single steping */ 175 insn.word = p->opcode; 176 arch_simulate_insn(insn, regs); 177 /* now go for post processing */ 178 post_kprobe_handler(p, kcb, regs); 179 } 180 } 181 NOKPROBE_SYMBOL(setup_singlestep); 182 183 static bool reenter_kprobe(struct kprobe *p, struct pt_regs *regs, 184 struct kprobe_ctlblk *kcb) 185 { 186 switch (kcb->kprobe_status) { 187 case KPROBE_HIT_SS: 188 case KPROBE_HIT_SSDONE: 189 case KPROBE_HIT_ACTIVE: 190 kprobes_inc_nmissed_count(p); 191 setup_singlestep(p, regs, kcb, 1); 192 break; 193 case KPROBE_REENTER: 194 pr_warn("Failed to recover from reentered kprobes.\n"); 195 dump_kprobe(p); 196 WARN_ON_ONCE(1); 197 break; 198 default: 199 WARN_ON(1); 200 return false; 201 } 202 203 return true; 204 } 205 NOKPROBE_SYMBOL(reenter_kprobe); 206 207 bool kprobe_breakpoint_handler(struct pt_regs *regs) 208 { 209 struct kprobe_ctlblk *kcb; 210 struct kprobe *p, *cur_kprobe; 211 kprobe_opcode_t *addr = (kprobe_opcode_t *)regs->csr_era; 212 213 /* 214 * We don't want to be preempted for the entire 215 * duration of kprobe processing. 216 */ 217 preempt_disable(); 218 kcb = get_kprobe_ctlblk(); 219 cur_kprobe = kprobe_running(); 220 221 p = get_kprobe(addr); 222 if (p) { 223 if (cur_kprobe) { 224 if (reenter_kprobe(p, regs, kcb)) 225 return true; 226 } else { 227 /* Probe hit */ 228 set_current_kprobe(p); 229 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 230 231 /* 232 * If we have no pre-handler or it returned 0, we 233 * continue with normal processing. If we have a 234 * pre-handler and it returned non-zero, it will 235 * modify the execution path and no need to single 236 * stepping. Let's just reset current kprobe and exit. 237 * 238 * pre_handler can hit a breakpoint and can step thru 239 * before return. 240 */ 241 if (!p->pre_handler || !p->pre_handler(p, regs)) { 242 setup_singlestep(p, regs, kcb, 0); 243 } else { 244 reset_current_kprobe(); 245 preempt_enable_no_resched(); 246 } 247 return true; 248 } 249 } 250 251 if (*addr != KPROBE_BP_INSN) { 252 /* 253 * The breakpoint instruction was removed right 254 * after we hit it. Another cpu has removed 255 * either a probepoint or a debugger breakpoint 256 * at this address. In either case, no further 257 * handling of this interrupt is appropriate. 258 * Return back to original instruction, and continue. 259 */ 260 regs->csr_era = (unsigned long)addr; 261 preempt_enable_no_resched(); 262 return true; 263 } 264 265 preempt_enable_no_resched(); 266 return false; 267 } 268 NOKPROBE_SYMBOL(kprobe_breakpoint_handler); 269 270 bool kprobe_singlestep_handler(struct pt_regs *regs) 271 { 272 struct kprobe *cur = kprobe_running(); 273 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 274 unsigned long addr = instruction_pointer(regs); 275 276 if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) && 277 ((unsigned long)&cur->ainsn.insn[1] == addr)) { 278 restore_local_irqflag(kcb, regs); 279 post_kprobe_handler(cur, kcb, regs); 280 return true; 281 } 282 283 preempt_enable_no_resched(); 284 return false; 285 } 286 NOKPROBE_SYMBOL(kprobe_singlestep_handler); 287 288 bool kprobe_fault_handler(struct pt_regs *regs, int trapnr) 289 { 290 struct kprobe *cur = kprobe_running(); 291 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 292 293 switch (kcb->kprobe_status) { 294 case KPROBE_HIT_SS: 295 case KPROBE_REENTER: 296 /* 297 * We are here because the instruction being single 298 * stepped caused a page fault. We reset the current 299 * kprobe and the ip points back to the probe address 300 * and allow the page fault handler to continue as a 301 * normal page fault. 302 */ 303 regs->csr_era = (unsigned long)cur->addr; 304 WARN_ON_ONCE(!instruction_pointer(regs)); 305 306 if (kcb->kprobe_status == KPROBE_REENTER) { 307 restore_previous_kprobe(kcb); 308 } else { 309 restore_local_irqflag(kcb, regs); 310 reset_current_kprobe(); 311 } 312 preempt_enable_no_resched(); 313 break; 314 } 315 return false; 316 } 317 NOKPROBE_SYMBOL(kprobe_fault_handler); 318 319 /* 320 * Provide a blacklist of symbols identifying ranges which cannot be kprobed. 321 * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). 322 */ 323 int __init arch_populate_kprobe_blacklist(void) 324 { 325 return kprobe_add_area_blacklist((unsigned long)__irqentry_text_start, 326 (unsigned long)__irqentry_text_end); 327 } 328 329 int __init arch_init_kprobes(void) 330 { 331 return 0; 332 } 333 334 int arch_trampoline_kprobe(struct kprobe *p) 335 { 336 return 0; 337 } 338 NOKPROBE_SYMBOL(arch_trampoline_kprobe); 339