1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
4  * using the CPU's debug registers. Derived from
5  * "arch/x86/kernel/hw_breakpoint.c"
6  *
7  * Copyright 2010 IBM Corporation
8  * Author: K.Prasad <prasad@linux.vnet.ibm.com>
9  */
10 
11 #include <linux/hw_breakpoint.h>
12 #include <linux/notifier.h>
13 #include <linux/kprobes.h>
14 #include <linux/percpu.h>
15 #include <linux/kernel.h>
16 #include <linux/sched.h>
17 #include <linux/smp.h>
18 #include <linux/debugfs.h>
19 #include <linux/init.h>
20 
21 #include <asm/hw_breakpoint.h>
22 #include <asm/processor.h>
23 #include <asm/sstep.h>
24 #include <asm/debug.h>
25 #include <asm/debugfs.h>
26 #include <asm/hvcall.h>
27 #include <asm/inst.h>
28 #include <linux/uaccess.h>
29 
30 /*
31  * Stores the breakpoints currently in use on each breakpoint address
32  * register for every cpu
33  */
34 static DEFINE_PER_CPU(struct perf_event *, bp_per_reg[HBP_NUM_MAX]);
35 
36 /*
37  * Returns total number of data or instruction breakpoints available.
38  */
39 int hw_breakpoint_slots(int type)
40 {
41 	if (type == TYPE_DATA)
42 		return nr_wp_slots();
43 	return 0;		/* no instruction breakpoints available */
44 }
45 
46 static bool single_step_pending(void)
47 {
48 	int i;
49 
50 	for (i = 0; i < nr_wp_slots(); i++) {
51 		if (current->thread.last_hit_ubp[i])
52 			return true;
53 	}
54 	return false;
55 }
56 
57 /*
58  * Install a perf counter breakpoint.
59  *
60  * We seek a free debug address register and use it for this
61  * breakpoint.
62  *
63  * Atomic: we hold the counter->ctx->lock and we only handle variables
64  * and registers local to this cpu.
65  */
66 int arch_install_hw_breakpoint(struct perf_event *bp)
67 {
68 	struct arch_hw_breakpoint *info = counter_arch_bp(bp);
69 	struct perf_event **slot;
70 	int i;
71 
72 	for (i = 0; i < nr_wp_slots(); i++) {
73 		slot = this_cpu_ptr(&bp_per_reg[i]);
74 		if (!*slot) {
75 			*slot = bp;
76 			break;
77 		}
78 	}
79 
80 	if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot"))
81 		return -EBUSY;
82 
83 	/*
84 	 * Do not install DABR values if the instruction must be single-stepped.
85 	 * If so, DABR will be populated in single_step_dabr_instruction().
86 	 */
87 	if (!single_step_pending())
88 		__set_breakpoint(i, info);
89 
90 	return 0;
91 }
92 
93 /*
94  * Uninstall the breakpoint contained in the given counter.
95  *
96  * First we search the debug address register it uses and then we disable
97  * it.
98  *
99  * Atomic: we hold the counter->ctx->lock and we only handle variables
100  * and registers local to this cpu.
101  */
102 void arch_uninstall_hw_breakpoint(struct perf_event *bp)
103 {
104 	struct arch_hw_breakpoint null_brk = {0};
105 	struct perf_event **slot;
106 	int i;
107 
108 	for (i = 0; i < nr_wp_slots(); i++) {
109 		slot = this_cpu_ptr(&bp_per_reg[i]);
110 		if (*slot == bp) {
111 			*slot = NULL;
112 			break;
113 		}
114 	}
115 
116 	if (WARN_ONCE(i == nr_wp_slots(), "Can't find any breakpoint slot"))
117 		return;
118 
119 	__set_breakpoint(i, &null_brk);
120 }
121 
122 static bool is_ptrace_bp(struct perf_event *bp)
123 {
124 	return bp->overflow_handler == ptrace_triggered;
125 }
126 
127 struct breakpoint {
128 	struct list_head list;
129 	struct perf_event *bp;
130 	bool ptrace_bp;
131 };
132 
133 static DEFINE_PER_CPU(struct breakpoint *, cpu_bps[HBP_NUM_MAX]);
134 static LIST_HEAD(task_bps);
135 
136 static struct breakpoint *alloc_breakpoint(struct perf_event *bp)
137 {
138 	struct breakpoint *tmp;
139 
140 	tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
141 	if (!tmp)
142 		return ERR_PTR(-ENOMEM);
143 	tmp->bp = bp;
144 	tmp->ptrace_bp = is_ptrace_bp(bp);
145 	return tmp;
146 }
147 
148 static bool bp_addr_range_overlap(struct perf_event *bp1, struct perf_event *bp2)
149 {
150 	__u64 bp1_saddr, bp1_eaddr, bp2_saddr, bp2_eaddr;
151 
152 	bp1_saddr = ALIGN_DOWN(bp1->attr.bp_addr, HW_BREAKPOINT_SIZE);
153 	bp1_eaddr = ALIGN(bp1->attr.bp_addr + bp1->attr.bp_len, HW_BREAKPOINT_SIZE);
154 	bp2_saddr = ALIGN_DOWN(bp2->attr.bp_addr, HW_BREAKPOINT_SIZE);
155 	bp2_eaddr = ALIGN(bp2->attr.bp_addr + bp2->attr.bp_len, HW_BREAKPOINT_SIZE);
156 
157 	return (bp1_saddr < bp2_eaddr && bp1_eaddr > bp2_saddr);
158 }
159 
160 static bool alternate_infra_bp(struct breakpoint *b, struct perf_event *bp)
161 {
162 	return is_ptrace_bp(bp) ? !b->ptrace_bp : b->ptrace_bp;
163 }
164 
165 static bool can_co_exist(struct breakpoint *b, struct perf_event *bp)
166 {
167 	return !(alternate_infra_bp(b, bp) && bp_addr_range_overlap(b->bp, bp));
168 }
169 
170 static int task_bps_add(struct perf_event *bp)
171 {
172 	struct breakpoint *tmp;
173 
174 	tmp = alloc_breakpoint(bp);
175 	if (IS_ERR(tmp))
176 		return PTR_ERR(tmp);
177 
178 	list_add(&tmp->list, &task_bps);
179 	return 0;
180 }
181 
182 static void task_bps_remove(struct perf_event *bp)
183 {
184 	struct list_head *pos, *q;
185 
186 	list_for_each_safe(pos, q, &task_bps) {
187 		struct breakpoint *tmp = list_entry(pos, struct breakpoint, list);
188 
189 		if (tmp->bp == bp) {
190 			list_del(&tmp->list);
191 			kfree(tmp);
192 			break;
193 		}
194 	}
195 }
196 
197 /*
198  * If any task has breakpoint from alternate infrastructure,
199  * return true. Otherwise return false.
200  */
201 static bool all_task_bps_check(struct perf_event *bp)
202 {
203 	struct breakpoint *tmp;
204 
205 	list_for_each_entry(tmp, &task_bps, list) {
206 		if (!can_co_exist(tmp, bp))
207 			return true;
208 	}
209 	return false;
210 }
211 
212 /*
213  * If same task has breakpoint from alternate infrastructure,
214  * return true. Otherwise return false.
215  */
216 static bool same_task_bps_check(struct perf_event *bp)
217 {
218 	struct breakpoint *tmp;
219 
220 	list_for_each_entry(tmp, &task_bps, list) {
221 		if (tmp->bp->hw.target == bp->hw.target &&
222 		    !can_co_exist(tmp, bp))
223 			return true;
224 	}
225 	return false;
226 }
227 
228 static int cpu_bps_add(struct perf_event *bp)
229 {
230 	struct breakpoint **cpu_bp;
231 	struct breakpoint *tmp;
232 	int i = 0;
233 
234 	tmp = alloc_breakpoint(bp);
235 	if (IS_ERR(tmp))
236 		return PTR_ERR(tmp);
237 
238 	cpu_bp = per_cpu_ptr(cpu_bps, bp->cpu);
239 	for (i = 0; i < nr_wp_slots(); i++) {
240 		if (!cpu_bp[i]) {
241 			cpu_bp[i] = tmp;
242 			break;
243 		}
244 	}
245 	return 0;
246 }
247 
248 static void cpu_bps_remove(struct perf_event *bp)
249 {
250 	struct breakpoint **cpu_bp;
251 	int i = 0;
252 
253 	cpu_bp = per_cpu_ptr(cpu_bps, bp->cpu);
254 	for (i = 0; i < nr_wp_slots(); i++) {
255 		if (!cpu_bp[i])
256 			continue;
257 
258 		if (cpu_bp[i]->bp == bp) {
259 			kfree(cpu_bp[i]);
260 			cpu_bp[i] = NULL;
261 			break;
262 		}
263 	}
264 }
265 
266 static bool cpu_bps_check(int cpu, struct perf_event *bp)
267 {
268 	struct breakpoint **cpu_bp;
269 	int i;
270 
271 	cpu_bp = per_cpu_ptr(cpu_bps, cpu);
272 	for (i = 0; i < nr_wp_slots(); i++) {
273 		if (cpu_bp[i] && !can_co_exist(cpu_bp[i], bp))
274 			return true;
275 	}
276 	return false;
277 }
278 
279 static bool all_cpu_bps_check(struct perf_event *bp)
280 {
281 	int cpu;
282 
283 	for_each_online_cpu(cpu) {
284 		if (cpu_bps_check(cpu, bp))
285 			return true;
286 	}
287 	return false;
288 }
289 
290 /*
291  * We don't use any locks to serialize accesses to cpu_bps or task_bps
292  * because are already inside nr_bp_mutex.
293  */
294 int arch_reserve_bp_slot(struct perf_event *bp)
295 {
296 	int ret;
297 
298 	/* ptrace breakpoint */
299 	if (is_ptrace_bp(bp)) {
300 		if (all_cpu_bps_check(bp))
301 			return -ENOSPC;
302 
303 		if (same_task_bps_check(bp))
304 			return -ENOSPC;
305 
306 		return task_bps_add(bp);
307 	}
308 
309 	/* perf breakpoint */
310 	if (is_kernel_addr(bp->attr.bp_addr))
311 		return 0;
312 
313 	if (bp->hw.target && bp->cpu == -1) {
314 		if (same_task_bps_check(bp))
315 			return -ENOSPC;
316 
317 		return task_bps_add(bp);
318 	} else if (!bp->hw.target && bp->cpu != -1) {
319 		if (all_task_bps_check(bp))
320 			return -ENOSPC;
321 
322 		return cpu_bps_add(bp);
323 	}
324 
325 	if (same_task_bps_check(bp))
326 		return -ENOSPC;
327 
328 	ret = cpu_bps_add(bp);
329 	if (ret)
330 		return ret;
331 	ret = task_bps_add(bp);
332 	if (ret)
333 		cpu_bps_remove(bp);
334 
335 	return ret;
336 }
337 
338 void arch_release_bp_slot(struct perf_event *bp)
339 {
340 	if (!is_kernel_addr(bp->attr.bp_addr)) {
341 		if (bp->hw.target)
342 			task_bps_remove(bp);
343 		if (bp->cpu != -1)
344 			cpu_bps_remove(bp);
345 	}
346 }
347 
348 /*
349  * Perform cleanup of arch-specific counters during unregistration
350  * of the perf-event
351  */
352 void arch_unregister_hw_breakpoint(struct perf_event *bp)
353 {
354 	/*
355 	 * If the breakpoint is unregistered between a hw_breakpoint_handler()
356 	 * and the single_step_dabr_instruction(), then cleanup the breakpoint
357 	 * restoration variables to prevent dangling pointers.
358 	 * FIXME, this should not be using bp->ctx at all! Sayeth peterz.
359 	 */
360 	if (bp->ctx && bp->ctx->task && bp->ctx->task != ((void *)-1L)) {
361 		int i;
362 
363 		for (i = 0; i < nr_wp_slots(); i++) {
364 			if (bp->ctx->task->thread.last_hit_ubp[i] == bp)
365 				bp->ctx->task->thread.last_hit_ubp[i] = NULL;
366 		}
367 	}
368 }
369 
370 /*
371  * Check for virtual address in kernel space.
372  */
373 int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw)
374 {
375 	return is_kernel_addr(hw->address);
376 }
377 
378 int arch_bp_generic_fields(int type, int *gen_bp_type)
379 {
380 	*gen_bp_type = 0;
381 	if (type & HW_BRK_TYPE_READ)
382 		*gen_bp_type |= HW_BREAKPOINT_R;
383 	if (type & HW_BRK_TYPE_WRITE)
384 		*gen_bp_type |= HW_BREAKPOINT_W;
385 	if (*gen_bp_type == 0)
386 		return -EINVAL;
387 	return 0;
388 }
389 
390 /*
391  * Watchpoint match range is always doubleword(8 bytes) aligned on
392  * powerpc. If the given range is crossing doubleword boundary, we
393  * need to increase the length such that next doubleword also get
394  * covered. Ex,
395  *
396  *          address   len = 6 bytes
397  *                |=========.
398  *   |------------v--|------v--------|
399  *   | | | | | | | | | | | | | | | | |
400  *   |---------------|---------------|
401  *    <---8 bytes--->
402  *
403  * In this case, we should configure hw as:
404  *   start_addr = address & ~(HW_BREAKPOINT_SIZE - 1)
405  *   len = 16 bytes
406  *
407  * @start_addr is inclusive but @end_addr is exclusive.
408  */
409 static int hw_breakpoint_validate_len(struct arch_hw_breakpoint *hw)
410 {
411 	u16 max_len = DABR_MAX_LEN;
412 	u16 hw_len;
413 	unsigned long start_addr, end_addr;
414 
415 	start_addr = ALIGN_DOWN(hw->address, HW_BREAKPOINT_SIZE);
416 	end_addr = ALIGN(hw->address + hw->len, HW_BREAKPOINT_SIZE);
417 	hw_len = end_addr - start_addr;
418 
419 	if (dawr_enabled()) {
420 		max_len = DAWR_MAX_LEN;
421 		/* DAWR region can't cross 512 bytes boundary on p10 predecessors */
422 		if (!cpu_has_feature(CPU_FTR_ARCH_31) &&
423 		    (ALIGN_DOWN(start_addr, SZ_512) != ALIGN_DOWN(end_addr - 1, SZ_512)))
424 			return -EINVAL;
425 	} else if (IS_ENABLED(CONFIG_PPC_8xx)) {
426 		/* 8xx can setup a range without limitation */
427 		max_len = U16_MAX;
428 	}
429 
430 	if (hw_len > max_len)
431 		return -EINVAL;
432 
433 	hw->hw_len = hw_len;
434 	return 0;
435 }
436 
437 /*
438  * Validate the arch-specific HW Breakpoint register settings
439  */
440 int hw_breakpoint_arch_parse(struct perf_event *bp,
441 			     const struct perf_event_attr *attr,
442 			     struct arch_hw_breakpoint *hw)
443 {
444 	int ret = -EINVAL;
445 
446 	if (!bp || !attr->bp_len)
447 		return ret;
448 
449 	hw->type = HW_BRK_TYPE_TRANSLATE;
450 	if (attr->bp_type & HW_BREAKPOINT_R)
451 		hw->type |= HW_BRK_TYPE_READ;
452 	if (attr->bp_type & HW_BREAKPOINT_W)
453 		hw->type |= HW_BRK_TYPE_WRITE;
454 	if (hw->type == HW_BRK_TYPE_TRANSLATE)
455 		/* must set alteast read or write */
456 		return ret;
457 	if (!attr->exclude_user)
458 		hw->type |= HW_BRK_TYPE_USER;
459 	if (!attr->exclude_kernel)
460 		hw->type |= HW_BRK_TYPE_KERNEL;
461 	if (!attr->exclude_hv)
462 		hw->type |= HW_BRK_TYPE_HYP;
463 	hw->address = attr->bp_addr;
464 	hw->len = attr->bp_len;
465 
466 	if (!ppc_breakpoint_available())
467 		return -ENODEV;
468 
469 	return hw_breakpoint_validate_len(hw);
470 }
471 
472 /*
473  * Restores the breakpoint on the debug registers.
474  * Invoke this function if it is known that the execution context is
475  * about to change to cause loss of MSR_SE settings.
476  */
477 void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs)
478 {
479 	struct arch_hw_breakpoint *info;
480 	int i;
481 
482 	for (i = 0; i < nr_wp_slots(); i++) {
483 		if (unlikely(tsk->thread.last_hit_ubp[i]))
484 			goto reset;
485 	}
486 	return;
487 
488 reset:
489 	regs->msr &= ~MSR_SE;
490 	for (i = 0; i < nr_wp_slots(); i++) {
491 		info = counter_arch_bp(__this_cpu_read(bp_per_reg[i]));
492 		__set_breakpoint(i, info);
493 		tsk->thread.last_hit_ubp[i] = NULL;
494 	}
495 }
496 
497 static bool is_larx_stcx_instr(int type)
498 {
499 	return type == LARX || type == STCX;
500 }
501 
502 static bool is_octword_vsx_instr(int type, int size)
503 {
504 	return ((type == LOAD_VSX || type == STORE_VSX) && size == 32);
505 }
506 
507 /*
508  * We've failed in reliably handling the hw-breakpoint. Unregister
509  * it and throw a warning message to let the user know about it.
510  */
511 static void handler_error(struct perf_event *bp, struct arch_hw_breakpoint *info)
512 {
513 	WARN(1, "Unable to handle hardware breakpoint. Breakpoint at 0x%lx will be disabled.",
514 	     info->address);
515 	perf_event_disable_inatomic(bp);
516 }
517 
518 static void larx_stcx_err(struct perf_event *bp, struct arch_hw_breakpoint *info)
519 {
520 	printk_ratelimited("Breakpoint hit on instruction that can't be emulated. Breakpoint at 0x%lx will be disabled.\n",
521 			   info->address);
522 	perf_event_disable_inatomic(bp);
523 }
524 
525 static bool stepping_handler(struct pt_regs *regs, struct perf_event **bp,
526 			     struct arch_hw_breakpoint **info, int *hit,
527 			     struct ppc_inst instr)
528 {
529 	int i;
530 	int stepped;
531 
532 	/* Do not emulate user-space instructions, instead single-step them */
533 	if (user_mode(regs)) {
534 		for (i = 0; i < nr_wp_slots(); i++) {
535 			if (!hit[i])
536 				continue;
537 			current->thread.last_hit_ubp[i] = bp[i];
538 			info[i] = NULL;
539 		}
540 		regs->msr |= MSR_SE;
541 		return false;
542 	}
543 
544 	stepped = emulate_step(regs, instr);
545 	if (!stepped) {
546 		for (i = 0; i < nr_wp_slots(); i++) {
547 			if (!hit[i])
548 				continue;
549 			handler_error(bp[i], info[i]);
550 			info[i] = NULL;
551 		}
552 		return false;
553 	}
554 	return true;
555 }
556 
557 static void handle_p10dd1_spurious_exception(struct arch_hw_breakpoint **info,
558 					     int *hit, unsigned long ea)
559 {
560 	int i;
561 	unsigned long hw_end_addr;
562 
563 	/*
564 	 * Handle spurious exception only when any bp_per_reg is set.
565 	 * Otherwise this might be created by xmon and not actually a
566 	 * spurious exception.
567 	 */
568 	for (i = 0; i < nr_wp_slots(); i++) {
569 		if (!info[i])
570 			continue;
571 
572 		hw_end_addr = ALIGN(info[i]->address + info[i]->len, HW_BREAKPOINT_SIZE);
573 
574 		/*
575 		 * Ending address of DAWR range is less than starting
576 		 * address of op.
577 		 */
578 		if ((hw_end_addr - 1) >= ea)
579 			continue;
580 
581 		/*
582 		 * Those addresses need to be in the same or in two
583 		 * consecutive 512B blocks;
584 		 */
585 		if (((hw_end_addr - 1) >> 10) != (ea >> 10))
586 			continue;
587 
588 		/*
589 		 * 'op address + 64B' generates an address that has a
590 		 * carry into bit 52 (crosses 2K boundary).
591 		 */
592 		if ((ea & 0x800) == ((ea + 64) & 0x800))
593 			continue;
594 
595 		break;
596 	}
597 
598 	if (i == nr_wp_slots())
599 		return;
600 
601 	for (i = 0; i < nr_wp_slots(); i++) {
602 		if (info[i]) {
603 			hit[i] = 1;
604 			info[i]->type |= HW_BRK_TYPE_EXTRANEOUS_IRQ;
605 		}
606 	}
607 }
608 
609 int hw_breakpoint_handler(struct die_args *args)
610 {
611 	bool err = false;
612 	int rc = NOTIFY_STOP;
613 	struct perf_event *bp[HBP_NUM_MAX] = { NULL };
614 	struct pt_regs *regs = args->regs;
615 	struct arch_hw_breakpoint *info[HBP_NUM_MAX] = { NULL };
616 	int i;
617 	int hit[HBP_NUM_MAX] = {0};
618 	int nr_hit = 0;
619 	bool ptrace_bp = false;
620 	struct ppc_inst instr = ppc_inst(0);
621 	int type = 0;
622 	int size = 0;
623 	unsigned long ea;
624 
625 	/* Disable breakpoints during exception handling */
626 	hw_breakpoint_disable();
627 
628 	/*
629 	 * The counter may be concurrently released but that can only
630 	 * occur from a call_rcu() path. We can then safely fetch
631 	 * the breakpoint, use its callback, touch its counter
632 	 * while we are in an rcu_read_lock() path.
633 	 */
634 	rcu_read_lock();
635 
636 	if (!IS_ENABLED(CONFIG_PPC_8xx))
637 		wp_get_instr_detail(regs, &instr, &type, &size, &ea);
638 
639 	for (i = 0; i < nr_wp_slots(); i++) {
640 		bp[i] = __this_cpu_read(bp_per_reg[i]);
641 		if (!bp[i])
642 			continue;
643 
644 		info[i] = counter_arch_bp(bp[i]);
645 		info[i]->type &= ~HW_BRK_TYPE_EXTRANEOUS_IRQ;
646 
647 		if (wp_check_constraints(regs, instr, ea, type, size, info[i])) {
648 			if (!IS_ENABLED(CONFIG_PPC_8xx) &&
649 			    ppc_inst_equal(instr, ppc_inst(0))) {
650 				handler_error(bp[i], info[i]);
651 				info[i] = NULL;
652 				err = 1;
653 				continue;
654 			}
655 
656 			if (is_ptrace_bp(bp[i]))
657 				ptrace_bp = true;
658 			hit[i] = 1;
659 			nr_hit++;
660 		}
661 	}
662 
663 	if (err)
664 		goto reset;
665 
666 	if (!nr_hit) {
667 		/* Workaround for Power10 DD1 */
668 		if (!IS_ENABLED(CONFIG_PPC_8xx) && mfspr(SPRN_PVR) == 0x800100 &&
669 		    is_octword_vsx_instr(type, size)) {
670 			handle_p10dd1_spurious_exception(info, hit, ea);
671 		} else {
672 			rc = NOTIFY_DONE;
673 			goto out;
674 		}
675 	}
676 
677 	/*
678 	 * Return early after invoking user-callback function without restoring
679 	 * DABR if the breakpoint is from ptrace which always operates in
680 	 * one-shot mode. The ptrace-ed process will receive the SIGTRAP signal
681 	 * generated in do_dabr().
682 	 */
683 	if (ptrace_bp) {
684 		for (i = 0; i < nr_wp_slots(); i++) {
685 			if (!hit[i])
686 				continue;
687 			perf_bp_event(bp[i], regs);
688 			info[i] = NULL;
689 		}
690 		rc = NOTIFY_DONE;
691 		goto reset;
692 	}
693 
694 	if (!IS_ENABLED(CONFIG_PPC_8xx)) {
695 		if (is_larx_stcx_instr(type)) {
696 			for (i = 0; i < nr_wp_slots(); i++) {
697 				if (!hit[i])
698 					continue;
699 				larx_stcx_err(bp[i], info[i]);
700 				info[i] = NULL;
701 			}
702 			goto reset;
703 		}
704 
705 		if (!stepping_handler(regs, bp, info, hit, instr))
706 			goto reset;
707 	}
708 
709 	/*
710 	 * As a policy, the callback is invoked in a 'trigger-after-execute'
711 	 * fashion
712 	 */
713 	for (i = 0; i < nr_wp_slots(); i++) {
714 		if (!hit[i])
715 			continue;
716 		if (!(info[i]->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
717 			perf_bp_event(bp[i], regs);
718 	}
719 
720 reset:
721 	for (i = 0; i < nr_wp_slots(); i++) {
722 		if (!info[i])
723 			continue;
724 		__set_breakpoint(i, info[i]);
725 	}
726 
727 out:
728 	rcu_read_unlock();
729 	return rc;
730 }
731 NOKPROBE_SYMBOL(hw_breakpoint_handler);
732 
733 /*
734  * Handle single-step exceptions following a DABR hit.
735  */
736 static int single_step_dabr_instruction(struct die_args *args)
737 {
738 	struct pt_regs *regs = args->regs;
739 	struct perf_event *bp = NULL;
740 	struct arch_hw_breakpoint *info;
741 	int i;
742 	bool found = false;
743 
744 	/*
745 	 * Check if we are single-stepping as a result of a
746 	 * previous HW Breakpoint exception
747 	 */
748 	for (i = 0; i < nr_wp_slots(); i++) {
749 		bp = current->thread.last_hit_ubp[i];
750 
751 		if (!bp)
752 			continue;
753 
754 		found = true;
755 		info = counter_arch_bp(bp);
756 
757 		/*
758 		 * We shall invoke the user-defined callback function in the
759 		 * single stepping handler to confirm to 'trigger-after-execute'
760 		 * semantics
761 		 */
762 		if (!(info->type & HW_BRK_TYPE_EXTRANEOUS_IRQ))
763 			perf_bp_event(bp, regs);
764 		current->thread.last_hit_ubp[i] = NULL;
765 	}
766 
767 	if (!found)
768 		return NOTIFY_DONE;
769 
770 	for (i = 0; i < nr_wp_slots(); i++) {
771 		bp = __this_cpu_read(bp_per_reg[i]);
772 		if (!bp)
773 			continue;
774 
775 		info = counter_arch_bp(bp);
776 		__set_breakpoint(i, info);
777 	}
778 
779 	/*
780 	 * If the process was being single-stepped by ptrace, let the
781 	 * other single-step actions occur (e.g. generate SIGTRAP).
782 	 */
783 	if (test_thread_flag(TIF_SINGLESTEP))
784 		return NOTIFY_DONE;
785 
786 	return NOTIFY_STOP;
787 }
788 NOKPROBE_SYMBOL(single_step_dabr_instruction);
789 
790 /*
791  * Handle debug exception notifications.
792  */
793 int hw_breakpoint_exceptions_notify(
794 		struct notifier_block *unused, unsigned long val, void *data)
795 {
796 	int ret = NOTIFY_DONE;
797 
798 	switch (val) {
799 	case DIE_DABR_MATCH:
800 		ret = hw_breakpoint_handler(data);
801 		break;
802 	case DIE_SSTEP:
803 		ret = single_step_dabr_instruction(data);
804 		break;
805 	}
806 
807 	return ret;
808 }
809 NOKPROBE_SYMBOL(hw_breakpoint_exceptions_notify);
810 
811 /*
812  * Release the user breakpoints used by ptrace
813  */
814 void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
815 {
816 	int i;
817 	struct thread_struct *t = &tsk->thread;
818 
819 	for (i = 0; i < nr_wp_slots(); i++) {
820 		unregister_hw_breakpoint(t->ptrace_bps[i]);
821 		t->ptrace_bps[i] = NULL;
822 	}
823 }
824 
825 void hw_breakpoint_pmu_read(struct perf_event *bp)
826 {
827 	/* TODO */
828 }
829 
830 void ptrace_triggered(struct perf_event *bp,
831 		      struct perf_sample_data *data, struct pt_regs *regs)
832 {
833 	struct perf_event_attr attr;
834 
835 	/*
836 	 * Disable the breakpoint request here since ptrace has defined a
837 	 * one-shot behaviour for breakpoint exceptions in PPC64.
838 	 * The SIGTRAP signal is generated automatically for us in do_dabr().
839 	 * We don't have to do anything about that here
840 	 */
841 	attr = bp->attr;
842 	attr.disabled = true;
843 	modify_user_hw_breakpoint(bp, &attr);
844 }
845