1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
4  * using the CPU's debug registers. Derived from
5  * "arch/x86/kernel/hw_breakpoint.c"
6  *
7  * Copyright 2010 IBM Corporation
8  * Author: K.Prasad <prasad@linux.vnet.ibm.com>
9  */
10 
11 #include <linux/hw_breakpoint.h>
12 #include <linux/notifier.h>
13 #include <linux/kprobes.h>
14 #include <linux/percpu.h>
15 #include <linux/kernel.h>
16 #include <linux/sched.h>
17 #include <linux/smp.h>
18 #include <linux/spinlock.h>
19 #include <linux/debugfs.h>
20 #include <linux/init.h>
21 
22 #include <asm/hw_breakpoint.h>
23 #include <asm/processor.h>
24 #include <asm/sstep.h>
25 #include <asm/debug.h>
26 #include <asm/hvcall.h>
27 #include <asm/inst.h>
28 #include <linux/uaccess.h>
29 
30 /*
31  * Stores the breakpoints currently in use on each breakpoint address
32  * register for every cpu
33  */
34 static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM_MAX]);
35 
36 /*
37  * Returns total number of data or instruction breakpoints available.
38  */
39 int hw_breakpoint_slots(int type)
40 {
41 	if (type == TYPE_DATA)
42 		return nr_wp_slots();
43 	return 0;		/* no instruction breakpoints available */
44 }
45 
46 
47 /*
48  * Install a perf counter breakpoint.
49  *
50  * We seek a free debug address register and use it for this
51  * breakpoint.
52  *
53  * Atomic: we hold the counter->ctx->lock and we only handle variables
54  * and registers local to this cpu.
55  */
56 int arch_install_hw_breakpoint(struct perf_event *bp)
57 {
58 	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
59 	struct perf_event **slot;
60 	int i;
61 
62 	for (i = 0; i < nr_wp_slots(); i++) {
63 		slot = this_cpu_ptr(&bp_per_reg[i]);
64 		if (!*slot) {
65 			*slot = bp;
66 			break;
67 		}
68 	}
69 
70 	if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot"))
71 		return -EBUSY;
72 
73 	/*
74 	 * Do not install DABR values if the instruction must be single-stepped.
75 	 * If so, DABR will be populated in single_step_dabr_instruction().
76 	 */
77 	if (!info->perf_single_step)
78 		__set_breakpoint(i, info);
79 
80 	return 0;
81 }
82 
83 /*
84  * Uninstall the breakpoint contained in the given counter.
85  *
86  * First we search the debug address register it uses and then we disable
87  * it.
88  *
89  * Atomic: we hold the counter->ctx->lock and we only handle variables
90  * and registers local to this cpu.
91  */
92 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
93 {
94 	struct arch_hw_breakpoint null_brk = {0};
95 	struct perf_event **slot;
96 	int i;
97 
98 	for (i = 0; i < nr_wp_slots(); i++) {
99 		slot = this_cpu_ptr(&bp_per_reg[i]);
100 		if (*slot == bp) {
101 			*slot = NULL;
102 			break;
103 		}
104 	}
105 
106 	if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot"))
107 		return;
108 
109 	__set_breakpoint(i, &null_brk);
110 }
111 
112 static bool is_ptrace_bp(struct perf_event *bp)
113 {
114 	return bp->overflow_handler == ptrace_triggered;
115 }
116 
117 /*
118  * Check for virtual address in kernel space.
119  */
120 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
121 {
122 	return is_kernel_addr(hw->address);
123 }
124 
125 int arch_bp_generic_fields(int type, int *gen_bp_type)
126 {
127 	*gen_bp_type = 0;
128 	if (type & HW_BRK_TYPE_READ)
129 		*gen_bp_type |= HW_BREAKPOINT_R;
130 	if (type & HW_BRK_TYPE_WRITE)
131 		*gen_bp_type |= HW_BREAKPOINT_W;
132 	if (*gen_bp_type == 0)
133 		return -EINVAL;
134 	return 0;
135 }
136 
137 /*
138  * Watchpoint match range is always doubleword(8 bytes) aligned on
139  * powerpc. If the given range is crossing doubleword boundary, we
140  * need to increase the length such that next doubleword also get
141  * covered. Ex,
142  *
143  *          address   len = 6 bytes
144  *                |=========.
145  *   |------------v--|------v--------|
146  *   | | | | | | | | | | | | | | | | |
147  *   |---------------|---------------|
148  *    <---8 bytes--->
149  *
150  * In this case, we should configure hw as:
151  *   start_addr = address & ~(HW_BREAKPOINT_SIZE - 1)
152  *   len = 16 bytes
153  *
154  * @start_addr is inclusive but @end_addr is exclusive.
155  */
156 static int hw_breakpoint_validate_len(struct arch_hw_breakpoint *hw)
157 {
158 	u16 max_len = DABR_MAX_LEN;
159 	u16 hw_len;
160 	unsigned long start_addr, end_addr;
161 
162 	start_addr = ALIGN_DOWN(hw->address, HW_BREAKPOINT_SIZE);
163 	end_addr = ALIGN(hw->address + hw->len, HW_BREAKPOINT_SIZE);
164 	hw_len = end_addr - start_addr;
165 
166 	if (dawr_enabled()) {
167 		max_len = DAWR_MAX_LEN;
168 		/* DAWR region can't cross 512 bytes boundary on p10 predecessors */
169 		if (!cpu_has_feature(CPU_FTR_ARCH_31) &&
170 		    (ALIGN_DOWN(start_addr, SZ_512) != ALIGN_DOWN(end_addr - 1, SZ_512)))
171 			return -EINVAL;
172 	} else if (IS_ENABLED(CONFIG_PPC_8xx)) {
173 		/* 8xx can setup a range without limitation */
174 		max_len = U16_MAX;
175 	}
176 
177 	if (hw_len > max_len)
178 		return -EINVAL;
179 
180 	hw->hw_len = hw_len;
181 	return 0;
182 }
183 
184 /*
185  * Validate the arch-specific HW Breakpoint register settings
186  */
187 int hw_breakpoint_arch_parse(struct perf_event *bp,
188 			     const struct perf_event_attr *attr,
189 			     struct arch_hw_breakpoint *hw)
190 {
191 	int ret = -EINVAL;
192 
193 	if (!bp || !attr->bp_len)
194 		return ret;
195 
196 	hw->type = HW_BRK_TYPE_TRANSLATE;
197 	if (attr->bp_type & HW_BREAKPOINT_R)
198 		hw->type |= HW_BRK_TYPE_READ;
199 	if (attr->bp_type & HW_BREAKPOINT_W)
200 		hw->type |= HW_BRK_TYPE_WRITE;
201 	if (hw->type == HW_BRK_TYPE_TRANSLATE)
202 		/* must set alteast read or write */
203 		return ret;
204 	if (!attr->exclude_user)
205 		hw->type |= HW_BRK_TYPE_USER;
206 	if (!attr->exclude_kernel)
207 		hw->type |= HW_BRK_TYPE_KERNEL;
208 	if (!attr->exclude_hv)
209 		hw->type |= HW_BRK_TYPE_HYP;
210 	hw->address = attr->bp_addr;
211 	hw->len = attr->bp_len;
212 
213 	if (!ppc_breakpoint_available())
214 		return -ENODEV;
215 
216 	return hw_breakpoint_validate_len(hw);
217 }
218 
219 /*
220  * Restores the breakpoint on the debug registers.
221  * Invoke this function if it is known that the execution context is
222  * about to change to cause loss of MSR_SE settings.
223  *
224  * The perf watchpoint will simply re-trigger once the thread is started again,
225  * and the watchpoint handler will set up MSR_SE and perf_single_step as
226  * needed.
227  */
228 void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs)
229 {
230 	struct arch_hw_breakpoint *info;
231 	int i;
232 
233 	for (i = 0; i < nr_wp_slots(); i++) {
234 		struct perf_event *bp = __this_cpu_read(bp_per_reg[i]);
235 
236 		if (unlikely(bp && counter_arch_bp(bp)->perf_single_step))
237 			goto reset;
238 	}
239 	return;
240 
241 reset:
242 	regs_set_return_msr(regs, regs->msr & ~MSR_SE);
243 	for (i = 0; i < nr_wp_slots(); i++) {
244 		info = counter_arch_bp(__this_cpu_read(bp_per_reg[i]));
245 		__set_breakpoint(i, info);
246 		info->perf_single_step = false;
247 	}
248 }
249 
250 static bool is_larx_stcx_instr(int type)
251 {
252 	return type == LARX || type == STCX;
253 }
254 
255 static bool is_octword_vsx_instr(int type, int size)
256 {
257 	return ((type == LOAD_VSX || type == STORE_VSX) && size == 32);
258 }
259 
260 /*
261  * We've failed in reliably handling the hw-breakpoint. Unregister
262  * it and throw a warning message to let the user know about it.
263  */
264 static void handler_error(struct perf_event *bp)
265 {
266 	WARN(1, "Unable to handle hardware breakpoint. Breakpoint at 0x%lx will be disabled.",
267 	     counter_arch_bp(bp)->address);
268 	perf_event_disable_inatomic(bp);
269 }
270 
271 static void larx_stcx_err(struct perf_event *bp)
272 {
273 	printk_ratelimited("Breakpoint hit on instruction that can't be emulated. Breakpoint at 0x%lx will be disabled.\n",
274 			   counter_arch_bp(bp)->address);
275 	perf_event_disable_inatomic(bp);
276 }
277 
278 static bool stepping_handler(struct pt_regs *regs, struct perf_event **bp,
279 			     int *hit, ppc_inst_t instr)
280 {
281 	int i;
282 	int stepped;
283 
284 	/* Do not emulate user-space instructions, instead single-step them */
285 	if (user_mode(regs)) {
286 		for (i = 0; i < nr_wp_slots(); i++) {
287 			if (!hit[i])
288 				continue;
289 
290 			counter_arch_bp(bp[i])->perf_single_step = true;
291 			bp[i] = NULL;
292 		}
293 		regs_set_return_msr(regs, regs->msr | MSR_SE);
294 		return false;
295 	}
296 
297 	stepped = emulate_step(regs, instr);
298 	if (!stepped) {
299 		for (i = 0; i < nr_wp_slots(); i++) {
300 			if (!hit[i])
301 				continue;
302 			handler_error(bp[i]);
303 			bp[i] = NULL;
304 		}
305 		return false;
306 	}
307 	return true;
308 }
309 
310 static void handle_p10dd1_spurious_exception(struct perf_event **bp,
311 					     int *hit, unsigned long ea)
312 {
313 	int i;
314 	unsigned long hw_end_addr;
315 
316 	/*
317 	 * Handle spurious exception only when any bp_per_reg is set.
318 	 * Otherwise this might be created by xmon and not actually a
319 	 * spurious exception.
320 	 */
321 	for (i = 0; i < nr_wp_slots(); i++) {
322 		struct arch_hw_breakpoint *info;
323 
324 		if (!bp[i])
325 			continue;
326 
327 		info = counter_arch_bp(bp[i]);
328 
329 		hw_end_addr = ALIGN(info->address + info->len, HW_BREAKPOINT_SIZE);
330 
331 		/*
332 		 * Ending address of DAWR range is less than starting
333 		 * address of op.
334 		 */
335 		if ((hw_end_addr - 1) >= ea)
336 			continue;
337 
338 		/*
339 		 * Those addresses need to be in the same or in two
340 		 * consecutive 512B blocks;
341 		 */
342 		if (((hw_end_addr - 1) >> 10) != (ea >> 10))
343 			continue;
344 
345 		/*
346 		 * 'op address + 64B' generates an address that has a
347 		 * carry into bit 52 (crosses 2K boundary).
348 		 */
349 		if ((ea & 0x800) == ((ea + 64) & 0x800))
350 			continue;
351 
352 		break;
353 	}
354 
355 	if (i == nr_wp_slots())
356 		return;
357 
358 	for (i = 0; i < nr_wp_slots(); i++) {
359 		if (bp[i]) {
360 			hit[i] = 1;
361 			counter_arch_bp(bp[i])->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ;
362 		}
363 	}
364 }
365 
366 int hw_breakpoint_handler(struct die_args *args)
367 {
368 	bool err = false;
369 	int rc = NOTIFY_STOP;
370 	struct perf_event *bp[HBP_NUM_MAX] = { NULL };
371 	struct pt_regs *regs = args->regs;
372 	int i;
373 	int hit[HBP_NUM_MAX] = {0};
374 	int nr_hit = 0;
375 	bool ptrace_bp = false;
376 	ppc_inst_t instr = ppc_inst(0);
377 	int type = 0;
378 	int size = 0;
379 	unsigned long ea = 0;
380 
381 	/* Disable breakpoints during exception handling */
382 	hw_breakpoint_disable();
383 
384 	/*
385 	 * The counter may be concurrently released but that can only
386 	 * occur from a call_rcu() path. We can then safely fetch
387 	 * the breakpoint, use its callback, touch its counter
388 	 * while we are in an rcu_read_lock() path.
389 	 */
390 	rcu_read_lock();
391 
392 	if (!IS_ENABLED(CONFIG_PPC_8xx))
393 		wp_get_instr_detail(regs, &instr, &type, &size, &ea);
394 
395 	for (i = 0; i < nr_wp_slots(); i++) {
396 		struct arch_hw_breakpoint *info;
397 
398 		bp[i] = __this_cpu_read(bp_per_reg[i]);
399 		if (!bp[i])
400 			continue;
401 
402 		info = counter_arch_bp(bp[i]);
403 		info->type &= ~HW_BRK_TYPE_EXTRANEOUS_IRQ;
404 
405 		if (wp_check_constraints(regs, instr, ea, type, size, info)) {
406 			if (!IS_ENABLED(CONFIG_PPC_8xx) &&
407 			    ppc_inst_equal(instr, ppc_inst(0))) {
408 				handler_error(bp[i]);
409 				bp[i] = NULL;
410 				err = 1;
411 				continue;
412 			}
413 
414 			if (is_ptrace_bp(bp[i]))
415 				ptrace_bp = true;
416 			hit[i] = 1;
417 			nr_hit++;
418 		}
419 	}
420 
421 	if (err)
422 		goto reset;
423 
424 	if (!nr_hit) {
425 		/* Workaround for Power10 DD1 */
426 		if (!IS_ENABLED(CONFIG_PPC_8xx) && mfspr(SPRN_PVR) == 0x800100 &&
427 		    is_octword_vsx_instr(type, size)) {
428 			handle_p10dd1_spurious_exception(bp, hit, ea);
429 		} else {
430 			rc = NOTIFY_DONE;
431 			goto out;
432 		}
433 	}
434 
435 	/*
436 	 * Return early after invoking user-callback function without restoring
437 	 * DABR if the breakpoint is from ptrace which always operates in
438 	 * one-shot mode. The ptrace-ed process will receive the SIGTRAP signal
439 	 * generated in do_dabr().
440 	 */
441 	if (ptrace_bp) {
442 		for (i = 0; i < nr_wp_slots(); i++) {
443 			if (!hit[i] || !is_ptrace_bp(bp[i]))
444 				continue;
445 			perf_bp_event(bp[i], regs);
446 			bp[i] = NULL;
447 		}
448 		rc = NOTIFY_DONE;
449 		goto reset;
450 	}
451 
452 	if (!IS_ENABLED(CONFIG_PPC_8xx)) {
453 		if (is_larx_stcx_instr(type)) {
454 			for (i = 0; i < nr_wp_slots(); i++) {
455 				if (!hit[i])
456 					continue;
457 				larx_stcx_err(bp[i]);
458 				bp[i] = NULL;
459 			}
460 			goto reset;
461 		}
462 
463 		if (!stepping_handler(regs, bp, hit, instr))
464 			goto reset;
465 	}
466 
467 	/*
468 	 * As a policy, the callback is invoked in a 'trigger-after-execute'
469 	 * fashion
470 	 */
471 	for (i = 0; i < nr_wp_slots(); i++) {
472 		if (!hit[i])
473 			continue;
474 		if (!(counter_arch_bp(bp[i])->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
475 			perf_bp_event(bp[i], regs);
476 	}
477 
478 reset:
479 	for (i = 0; i < nr_wp_slots(); i++) {
480 		if (!bp[i])
481 			continue;
482 		__set_breakpoint(i, counter_arch_bp(bp[i]));
483 	}
484 
485 out:
486 	rcu_read_unlock();
487 	return rc;
488 }
489 NOKPROBE_SYMBOL(hw_breakpoint_handler);
490 
491 /*
492  * Handle single-step exceptions following a DABR hit.
493  */
494 static int single_step_dabr_instruction(struct die_args *args)
495 {
496 	struct pt_regs *regs = args->regs;
497 	bool found = false;
498 
499 	/*
500 	 * Check if we are single-stepping as a result of a
501 	 * previous HW Breakpoint exception
502 	 */
503 	for (int i = 0; i < nr_wp_slots(); i++) {
504 		struct perf_event *bp;
505 		struct arch_hw_breakpoint *info;
506 
507 		bp = __this_cpu_read(bp_per_reg[i]);
508 
509 		if (!bp)
510 			continue;
511 
512 		info = counter_arch_bp(bp);
513 
514 		if (!info->perf_single_step)
515 			continue;
516 
517 		found = true;
518 
519 		/*
520 		 * We shall invoke the user-defined callback function in the
521 		 * single stepping handler to confirm to 'trigger-after-execute'
522 		 * semantics
523 		 */
524 		if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
525 			perf_bp_event(bp, regs);
526 
527 		info->perf_single_step = false;
528 		__set_breakpoint(i, counter_arch_bp(bp));
529 	}
530 
531 	/*
532 	 * If the process was being single-stepped by ptrace, let the
533 	 * other single-step actions occur (e.g. generate SIGTRAP).
534 	 */
535 	if (!found || test_thread_flag(TIF_SINGLESTEP))
536 		return NOTIFY_DONE;
537 
538 	return NOTIFY_STOP;
539 }
540 NOKPROBE_SYMBOL(single_step_dabr_instruction);
541 
542 /*
543  * Handle debug exception notifications.
544  */
545 int hw_breakpoint_exceptions_notify(
546 		struct notifier_block *unused, unsigned long val, void *data)
547 {
548 	int ret = NOTIFY_DONE;
549 
550 	switch (val) {
551 	case DIE_DABR_MATCH:
552 		ret = hw_breakpoint_handler(data);
553 		break;
554 	case DIE_SSTEP:
555 		ret = single_step_dabr_instruction(data);
556 		break;
557 	}
558 
559 	return ret;
560 }
561 NOKPROBE_SYMBOL(hw_breakpoint_exceptions_notify);
562 
563 /*
564  * Release the user breakpoints used by ptrace
565  */
566 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
567 {
568 	int i;
569 	struct thread_struct *t = &tsk->thread;
570 
571 	for (i = 0; i < nr_wp_slots(); i++) {
572 		unregister_hw_breakpoint(t->ptrace_bps[i]);
573 		t->ptrace_bps[i] = NULL;
574 	}
575 }
576 
577 void hw_breakpoint_pmu_read(struct perf_event *bp)
578 {
579 	/* TODO */
580 }
581 
582 void ptrace_triggered(struct perf_event *bp,
583 		      struct perf_sample_data *data, struct pt_regs *regs)
584 {
585 	struct perf_event_attr attr;
586 
587 	/*
588 	 * Disable the breakpoint request here since ptrace has defined a
589 	 * one-shot behaviour for breakpoint exceptions in PPC64.
590 	 * The SIGTRAP signal is generated automatically for us in do_dabr().
591 	 * We don't have to do anything about that here
592 	 */
593 	attr = bp->attr;
594 	attr.disabled = true;
595 	modify_user_hw_breakpoint(bp, &attr);
596 }
597