xref: /openbmc/linux/arch/x86/kernel/kprobes/core.c (revision e23feb16)
1 /*
2  *  Kernel Probes (KProbes)
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License as published by
6  * the Free Software Foundation; either version 2 of the License, or
7  * (at your option) any later version.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12  * GNU General Public License for more details.
13  *
14  * You should have received a copy of the GNU General Public License
15  * along with this program; if not, write to the Free Software
16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17  *
18  * Copyright (C) IBM Corporation, 2002, 2004
19  *
20  * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
21  *		Probes initial implementation ( includes contributions from
22  *		Rusty Russell).
23  * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
24  *		interface to access function arguments.
25  * 2004-Oct	Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
26  *		<prasanna@in.ibm.com> adapted for x86_64 from i386.
27  * 2005-Mar	Roland McGrath <roland@redhat.com>
28  *		Fixed to handle %rip-relative addressing mode correctly.
29  * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
30  *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
31  *		<prasanna@in.ibm.com> added function-return probes.
32  * 2005-May	Rusty Lynch <rusty.lynch@intel.com>
33  *		Added function return probes functionality
34  * 2006-Feb	Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added
35  *		kprobe-booster and kretprobe-booster for i386.
36  * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
37  *		and kretprobe-booster for x86-64
38  * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven
39  *		<arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com>
40  *		unified x86 kprobes code.
41  */
42 #include <linux/kprobes.h>
43 #include <linux/ptrace.h>
44 #include <linux/string.h>
45 #include <linux/slab.h>
46 #include <linux/hardirq.h>
47 #include <linux/preempt.h>
48 #include <linux/module.h>
49 #include <linux/kdebug.h>
50 #include <linux/kallsyms.h>
51 #include <linux/ftrace.h>
52 
53 #include <asm/cacheflush.h>
54 #include <asm/desc.h>
55 #include <asm/pgtable.h>
56 #include <asm/uaccess.h>
57 #include <asm/alternative.h>
58 #include <asm/insn.h>
59 #include <asm/debugreg.h>
60 
61 #include "common.h"
62 
63 void jprobe_return_end(void);
64 
65 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
66 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
67 
68 #define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
69 
70 #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
71 	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
72 	  (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) |   \
73 	  (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) |   \
74 	  (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf))    \
75 	 << (row % 32))
76 	/*
77 	 * Undefined/reserved opcodes, conditional jump, Opcode Extension
78 	 * Groups, and some special opcodes can not boost.
79 	 * This is non-const and volatile to keep gcc from statically
80 	 * optimizing it out, as variable_test_bit makes gcc think only
81 	 * *(unsigned long*) is used.
82 	 */
83 static volatile u32 twobyte_is_boostable[256 / 32] = {
84 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
85 	/*      ----------------------------------------------          */
86 	W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */
87 	W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 10 */
88 	W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */
89 	W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
90 	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
91 	W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
92 	W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */
93 	W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
94 	W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */
95 	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
96 	W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */
97 	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */
98 	W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
99 	W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */
100 	W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */
101 	W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0)   /* f0 */
102 	/*      -----------------------------------------------         */
103 	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
104 };
105 #undef W
106 
107 struct kretprobe_blackpoint kretprobe_blacklist[] = {
108 	{"__switch_to", }, /* This function switches only current task, but
109 			      doesn't switch kernel stack.*/
110 	{NULL, NULL}	/* Terminator */
111 };
112 
113 const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
114 
115 static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
116 {
117 	struct __arch_relative_insn {
118 		u8 op;
119 		s32 raddr;
120 	} __packed *insn;
121 
122 	insn = (struct __arch_relative_insn *)from;
123 	insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
124 	insn->op = op;
125 }
126 
127 /* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
128 void __kprobes synthesize_reljump(void *from, void *to)
129 {
130 	__synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE);
131 }
132 
133 /* Insert a call instruction at address 'from', which calls address 'to'.*/
134 void __kprobes synthesize_relcall(void *from, void *to)
135 {
136 	__synthesize_relative_insn(from, to, RELATIVECALL_OPCODE);
137 }
138 
139 /*
140  * Skip the prefixes of the instruction.
141  */
142 static kprobe_opcode_t *__kprobes skip_prefixes(kprobe_opcode_t *insn)
143 {
144 	insn_attr_t attr;
145 
146 	attr = inat_get_opcode_attribute((insn_byte_t)*insn);
147 	while (inat_is_legacy_prefix(attr)) {
148 		insn++;
149 		attr = inat_get_opcode_attribute((insn_byte_t)*insn);
150 	}
151 #ifdef CONFIG_X86_64
152 	if (inat_is_rex_prefix(attr))
153 		insn++;
154 #endif
155 	return insn;
156 }
157 
158 /*
159  * Returns non-zero if opcode is boostable.
160  * RIP relative instructions are adjusted at copying time in 64 bits mode
161  */
162 int __kprobes can_boost(kprobe_opcode_t *opcodes)
163 {
164 	kprobe_opcode_t opcode;
165 	kprobe_opcode_t *orig_opcodes = opcodes;
166 
167 	if (search_exception_tables((unsigned long)opcodes))
168 		return 0;	/* Page fault may occur on this address. */
169 
170 retry:
171 	if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
172 		return 0;
173 	opcode = *(opcodes++);
174 
175 	/* 2nd-byte opcode */
176 	if (opcode == 0x0f) {
177 		if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
178 			return 0;
179 		return test_bit(*opcodes,
180 				(unsigned long *)twobyte_is_boostable);
181 	}
182 
183 	switch (opcode & 0xf0) {
184 #ifdef CONFIG_X86_64
185 	case 0x40:
186 		goto retry; /* REX prefix is boostable */
187 #endif
188 	case 0x60:
189 		if (0x63 < opcode && opcode < 0x67)
190 			goto retry; /* prefixes */
191 		/* can't boost Address-size override and bound */
192 		return (opcode != 0x62 && opcode != 0x67);
193 	case 0x70:
194 		return 0; /* can't boost conditional jump */
195 	case 0xc0:
196 		/* can't boost software-interruptions */
197 		return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
198 	case 0xd0:
199 		/* can boost AA* and XLAT */
200 		return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
201 	case 0xe0:
202 		/* can boost in/out and absolute jmps */
203 		return ((opcode & 0x04) || opcode == 0xea);
204 	case 0xf0:
205 		if ((opcode & 0x0c) == 0 && opcode != 0xf1)
206 			goto retry; /* lock/rep(ne) prefix */
207 		/* clear and set flags are boostable */
208 		return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
209 	default:
210 		/* segment override prefixes are boostable */
211 		if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
212 			goto retry; /* prefixes */
213 		/* CS override prefix and call are not boostable */
214 		return (opcode != 0x2e && opcode != 0x9a);
215 	}
216 }
217 
218 static unsigned long
219 __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
220 {
221 	struct kprobe *kp;
222 
223 	kp = get_kprobe((void *)addr);
224 	/* There is no probe, return original address */
225 	if (!kp)
226 		return addr;
227 
228 	/*
229 	 *  Basically, kp->ainsn.insn has an original instruction.
230 	 *  However, RIP-relative instruction can not do single-stepping
231 	 *  at different place, __copy_instruction() tweaks the displacement of
232 	 *  that instruction. In that case, we can't recover the instruction
233 	 *  from the kp->ainsn.insn.
234 	 *
235 	 *  On the other hand, kp->opcode has a copy of the first byte of
236 	 *  the probed instruction, which is overwritten by int3. And
237 	 *  the instruction at kp->addr is not modified by kprobes except
238 	 *  for the first byte, we can recover the original instruction
239 	 *  from it and kp->opcode.
240 	 */
241 	memcpy(buf, kp->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
242 	buf[0] = kp->opcode;
243 	return (unsigned long)buf;
244 }
245 
246 /*
247  * Recover the probed instruction at addr for further analysis.
248  * Caller must lock kprobes by kprobe_mutex, or disable preemption
249  * for preventing to release referencing kprobes.
250  */
251 unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr)
252 {
253 	unsigned long __addr;
254 
255 	__addr = __recover_optprobed_insn(buf, addr);
256 	if (__addr != addr)
257 		return __addr;
258 
259 	return __recover_probed_insn(buf, addr);
260 }
261 
262 /* Check if paddr is at an instruction boundary */
263 static int __kprobes can_probe(unsigned long paddr)
264 {
265 	unsigned long addr, __addr, offset = 0;
266 	struct insn insn;
267 	kprobe_opcode_t buf[MAX_INSN_SIZE];
268 
269 	if (!kallsyms_lookup_size_offset(paddr, NULL, &offset))
270 		return 0;
271 
272 	/* Decode instructions */
273 	addr = paddr - offset;
274 	while (addr < paddr) {
275 		/*
276 		 * Check if the instruction has been modified by another
277 		 * kprobe, in which case we replace the breakpoint by the
278 		 * original instruction in our buffer.
279 		 * Also, jump optimization will change the breakpoint to
280 		 * relative-jump. Since the relative-jump itself is
281 		 * normally used, we just go through if there is no kprobe.
282 		 */
283 		__addr = recover_probed_instruction(buf, addr);
284 		kernel_insn_init(&insn, (void *)__addr);
285 		insn_get_length(&insn);
286 
287 		/*
288 		 * Another debugging subsystem might insert this breakpoint.
289 		 * In that case, we can't recover it.
290 		 */
291 		if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
292 			return 0;
293 		addr += insn.length;
294 	}
295 
296 	return (addr == paddr);
297 }
298 
299 /*
300  * Returns non-zero if opcode modifies the interrupt flag.
301  */
302 static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)
303 {
304 	/* Skip prefixes */
305 	insn = skip_prefixes(insn);
306 
307 	switch (*insn) {
308 	case 0xfa:		/* cli */
309 	case 0xfb:		/* sti */
310 	case 0xcf:		/* iret/iretd */
311 	case 0x9d:		/* popf/popfd */
312 		return 1;
313 	}
314 
315 	return 0;
316 }
317 
318 /*
319  * Copy an instruction and adjust the displacement if the instruction
320  * uses the %rip-relative addressing mode.
321  * If it does, Return the address of the 32-bit displacement word.
322  * If not, return null.
323  * Only applicable to 64-bit x86.
324  */
325 int __kprobes __copy_instruction(u8 *dest, u8 *src)
326 {
327 	struct insn insn;
328 	kprobe_opcode_t buf[MAX_INSN_SIZE];
329 
330 	kernel_insn_init(&insn, (void *)recover_probed_instruction(buf, (unsigned long)src));
331 	insn_get_length(&insn);
332 	/* Another subsystem puts a breakpoint, failed to recover */
333 	if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION)
334 		return 0;
335 	memcpy(dest, insn.kaddr, insn.length);
336 
337 #ifdef CONFIG_X86_64
338 	if (insn_rip_relative(&insn)) {
339 		s64 newdisp;
340 		u8 *disp;
341 		kernel_insn_init(&insn, dest);
342 		insn_get_displacement(&insn);
343 		/*
344 		 * The copied instruction uses the %rip-relative addressing
345 		 * mode.  Adjust the displacement for the difference between
346 		 * the original location of this instruction and the location
347 		 * of the copy that will actually be run.  The tricky bit here
348 		 * is making sure that the sign extension happens correctly in
349 		 * this calculation, since we need a signed 32-bit result to
350 		 * be sign-extended to 64 bits when it's added to the %rip
351 		 * value and yield the same 64-bit result that the sign-
352 		 * extension of the original signed 32-bit displacement would
353 		 * have given.
354 		 */
355 		newdisp = (u8 *) src + (s64) insn.displacement.value - (u8 *) dest;
356 		if ((s64) (s32) newdisp != newdisp) {
357 			pr_err("Kprobes error: new displacement does not fit into s32 (%llx)\n", newdisp);
358 			pr_err("\tSrc: %p, Dest: %p, old disp: %x\n", src, dest, insn.displacement.value);
359 			return 0;
360 		}
361 		disp = (u8 *) dest + insn_offset_displacement(&insn);
362 		*(s32 *) disp = (s32) newdisp;
363 	}
364 #endif
365 	return insn.length;
366 }
367 
368 static int __kprobes arch_copy_kprobe(struct kprobe *p)
369 {
370 	int ret;
371 
372 	/* Copy an instruction with recovering if other optprobe modifies it.*/
373 	ret = __copy_instruction(p->ainsn.insn, p->addr);
374 	if (!ret)
375 		return -EINVAL;
376 
377 	/*
378 	 * __copy_instruction can modify the displacement of the instruction,
379 	 * but it doesn't affect boostable check.
380 	 */
381 	if (can_boost(p->ainsn.insn))
382 		p->ainsn.boostable = 0;
383 	else
384 		p->ainsn.boostable = -1;
385 
386 	/* Check whether the instruction modifies Interrupt Flag or not */
387 	p->ainsn.if_modifier = is_IF_modifier(p->ainsn.insn);
388 
389 	/* Also, displacement change doesn't affect the first byte */
390 	p->opcode = p->ainsn.insn[0];
391 
392 	return 0;
393 }
394 
395 int __kprobes arch_prepare_kprobe(struct kprobe *p)
396 {
397 	if (alternatives_text_reserved(p->addr, p->addr))
398 		return -EINVAL;
399 
400 	if (!can_probe((unsigned long)p->addr))
401 		return -EILSEQ;
402 	/* insn: must be on special executable page on x86. */
403 	p->ainsn.insn = get_insn_slot();
404 	if (!p->ainsn.insn)
405 		return -ENOMEM;
406 
407 	return arch_copy_kprobe(p);
408 }
409 
410 void __kprobes arch_arm_kprobe(struct kprobe *p)
411 {
412 	text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
413 }
414 
415 void __kprobes arch_disarm_kprobe(struct kprobe *p)
416 {
417 	text_poke(p->addr, &p->opcode, 1);
418 }
419 
420 void __kprobes arch_remove_kprobe(struct kprobe *p)
421 {
422 	if (p->ainsn.insn) {
423 		free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
424 		p->ainsn.insn = NULL;
425 	}
426 }
427 
428 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
429 {
430 	kcb->prev_kprobe.kp = kprobe_running();
431 	kcb->prev_kprobe.status = kcb->kprobe_status;
432 	kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;
433 	kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
434 }
435 
436 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
437 {
438 	__this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
439 	kcb->kprobe_status = kcb->prev_kprobe.status;
440 	kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;
441 	kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
442 }
443 
444 static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
445 				struct kprobe_ctlblk *kcb)
446 {
447 	__this_cpu_write(current_kprobe, p);
448 	kcb->kprobe_saved_flags = kcb->kprobe_old_flags
449 		= (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
450 	if (p->ainsn.if_modifier)
451 		kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
452 }
453 
454 static void __kprobes clear_btf(void)
455 {
456 	if (test_thread_flag(TIF_BLOCKSTEP)) {
457 		unsigned long debugctl = get_debugctlmsr();
458 
459 		debugctl &= ~DEBUGCTLMSR_BTF;
460 		update_debugctlmsr(debugctl);
461 	}
462 }
463 
464 static void __kprobes restore_btf(void)
465 {
466 	if (test_thread_flag(TIF_BLOCKSTEP)) {
467 		unsigned long debugctl = get_debugctlmsr();
468 
469 		debugctl |= DEBUGCTLMSR_BTF;
470 		update_debugctlmsr(debugctl);
471 	}
472 }
473 
474 void __kprobes
475 arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
476 {
477 	unsigned long *sara = stack_addr(regs);
478 
479 	ri->ret_addr = (kprobe_opcode_t *) *sara;
480 
481 	/* Replace the return addr with trampoline addr */
482 	*sara = (unsigned long) &kretprobe_trampoline;
483 }
484 
485 static void __kprobes
486 setup_singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb, int reenter)
487 {
488 	if (setup_detour_execution(p, regs, reenter))
489 		return;
490 
491 #if !defined(CONFIG_PREEMPT)
492 	if (p->ainsn.boostable == 1 && !p->post_handler) {
493 		/* Boost up -- we can execute copied instructions directly */
494 		if (!reenter)
495 			reset_current_kprobe();
496 		/*
497 		 * Reentering boosted probe doesn't reset current_kprobe,
498 		 * nor set current_kprobe, because it doesn't use single
499 		 * stepping.
500 		 */
501 		regs->ip = (unsigned long)p->ainsn.insn;
502 		preempt_enable_no_resched();
503 		return;
504 	}
505 #endif
506 	if (reenter) {
507 		save_previous_kprobe(kcb);
508 		set_current_kprobe(p, regs, kcb);
509 		kcb->kprobe_status = KPROBE_REENTER;
510 	} else
511 		kcb->kprobe_status = KPROBE_HIT_SS;
512 	/* Prepare real single stepping */
513 	clear_btf();
514 	regs->flags |= X86_EFLAGS_TF;
515 	regs->flags &= ~X86_EFLAGS_IF;
516 	/* single step inline if the instruction is an int3 */
517 	if (p->opcode == BREAKPOINT_INSTRUCTION)
518 		regs->ip = (unsigned long)p->addr;
519 	else
520 		regs->ip = (unsigned long)p->ainsn.insn;
521 }
522 
523 /*
524  * We have reentered the kprobe_handler(), since another probe was hit while
525  * within the handler. We save the original kprobes variables and just single
526  * step on the instruction of the new probe without calling any user handlers.
527  */
528 static int __kprobes
529 reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
530 {
531 	switch (kcb->kprobe_status) {
532 	case KPROBE_HIT_SSDONE:
533 	case KPROBE_HIT_ACTIVE:
534 		kprobes_inc_nmissed_count(p);
535 		setup_singlestep(p, regs, kcb, 1);
536 		break;
537 	case KPROBE_HIT_SS:
538 		/* A probe has been hit in the codepath leading up to, or just
539 		 * after, single-stepping of a probed instruction. This entire
540 		 * codepath should strictly reside in .kprobes.text section.
541 		 * Raise a BUG or we'll continue in an endless reentering loop
542 		 * and eventually a stack overflow.
543 		 */
544 		printk(KERN_WARNING "Unrecoverable kprobe detected at %p.\n",
545 		       p->addr);
546 		dump_kprobe(p);
547 		BUG();
548 	default:
549 		/* impossible cases */
550 		WARN_ON(1);
551 		return 0;
552 	}
553 
554 	return 1;
555 }
556 
557 /*
558  * Interrupts are disabled on entry as trap3 is an interrupt gate and they
559  * remain disabled throughout this function.
560  */
561 static int __kprobes kprobe_handler(struct pt_regs *regs)
562 {
563 	kprobe_opcode_t *addr;
564 	struct kprobe *p;
565 	struct kprobe_ctlblk *kcb;
566 
567 	addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
568 	/*
569 	 * We don't want to be preempted for the entire
570 	 * duration of kprobe processing. We conditionally
571 	 * re-enable preemption at the end of this function,
572 	 * and also in reenter_kprobe() and setup_singlestep().
573 	 */
574 	preempt_disable();
575 
576 	kcb = get_kprobe_ctlblk();
577 	p = get_kprobe(addr);
578 
579 	if (p) {
580 		if (kprobe_running()) {
581 			if (reenter_kprobe(p, regs, kcb))
582 				return 1;
583 		} else {
584 			set_current_kprobe(p, regs, kcb);
585 			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
586 
587 			/*
588 			 * If we have no pre-handler or it returned 0, we
589 			 * continue with normal processing.  If we have a
590 			 * pre-handler and it returned non-zero, it prepped
591 			 * for calling the break_handler below on re-entry
592 			 * for jprobe processing, so get out doing nothing
593 			 * more here.
594 			 */
595 			if (!p->pre_handler || !p->pre_handler(p, regs))
596 				setup_singlestep(p, regs, kcb, 0);
597 			return 1;
598 		}
599 	} else if (*addr != BREAKPOINT_INSTRUCTION) {
600 		/*
601 		 * The breakpoint instruction was removed right
602 		 * after we hit it.  Another cpu has removed
603 		 * either a probepoint or a debugger breakpoint
604 		 * at this address.  In either case, no further
605 		 * handling of this interrupt is appropriate.
606 		 * Back up over the (now missing) int3 and run
607 		 * the original instruction.
608 		 */
609 		regs->ip = (unsigned long)addr;
610 		preempt_enable_no_resched();
611 		return 1;
612 	} else if (kprobe_running()) {
613 		p = __this_cpu_read(current_kprobe);
614 		if (p->break_handler && p->break_handler(p, regs)) {
615 			if (!skip_singlestep(p, regs, kcb))
616 				setup_singlestep(p, regs, kcb, 0);
617 			return 1;
618 		}
619 	} /* else: not a kprobe fault; let the kernel handle it */
620 
621 	preempt_enable_no_resched();
622 	return 0;
623 }
624 
625 /*
626  * When a retprobed function returns, this code saves registers and
627  * calls trampoline_handler() runs, which calls the kretprobe's handler.
628  */
629 static void __used __kprobes kretprobe_trampoline_holder(void)
630 {
631 	asm volatile (
632 			".global kretprobe_trampoline\n"
633 			"kretprobe_trampoline: \n"
634 #ifdef CONFIG_X86_64
635 			/* We don't bother saving the ss register */
636 			"	pushq %rsp\n"
637 			"	pushfq\n"
638 			SAVE_REGS_STRING
639 			"	movq %rsp, %rdi\n"
640 			"	call trampoline_handler\n"
641 			/* Replace saved sp with true return address. */
642 			"	movq %rax, 152(%rsp)\n"
643 			RESTORE_REGS_STRING
644 			"	popfq\n"
645 #else
646 			"	pushf\n"
647 			SAVE_REGS_STRING
648 			"	movl %esp, %eax\n"
649 			"	call trampoline_handler\n"
650 			/* Move flags to cs */
651 			"	movl 56(%esp), %edx\n"
652 			"	movl %edx, 52(%esp)\n"
653 			/* Replace saved flags with true return address. */
654 			"	movl %eax, 56(%esp)\n"
655 			RESTORE_REGS_STRING
656 			"	popf\n"
657 #endif
658 			"	ret\n");
659 }
660 
661 /*
662  * Called from kretprobe_trampoline
663  */
664 __visible __used __kprobes void *trampoline_handler(struct pt_regs *regs)
665 {
666 	struct kretprobe_instance *ri = NULL;
667 	struct hlist_head *head, empty_rp;
668 	struct hlist_node *tmp;
669 	unsigned long flags, orig_ret_address = 0;
670 	unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
671 	kprobe_opcode_t *correct_ret_addr = NULL;
672 
673 	INIT_HLIST_HEAD(&empty_rp);
674 	kretprobe_hash_lock(current, &head, &flags);
675 	/* fixup registers */
676 #ifdef CONFIG_X86_64
677 	regs->cs = __KERNEL_CS;
678 #else
679 	regs->cs = __KERNEL_CS | get_kernel_rpl();
680 	regs->gs = 0;
681 #endif
682 	regs->ip = trampoline_address;
683 	regs->orig_ax = ~0UL;
684 
685 	/*
686 	 * It is possible to have multiple instances associated with a given
687 	 * task either because multiple functions in the call path have
688 	 * return probes installed on them, and/or more than one
689 	 * return probe was registered for a target function.
690 	 *
691 	 * We can handle this because:
692 	 *     - instances are always pushed into the head of the list
693 	 *     - when multiple return probes are registered for the same
694 	 *	 function, the (chronologically) first instance's ret_addr
695 	 *	 will be the real return address, and all the rest will
696 	 *	 point to kretprobe_trampoline.
697 	 */
698 	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
699 		if (ri->task != current)
700 			/* another task is sharing our hash bucket */
701 			continue;
702 
703 		orig_ret_address = (unsigned long)ri->ret_addr;
704 
705 		if (orig_ret_address != trampoline_address)
706 			/*
707 			 * This is the real return address. Any other
708 			 * instances associated with this task are for
709 			 * other calls deeper on the call stack
710 			 */
711 			break;
712 	}
713 
714 	kretprobe_assert(ri, orig_ret_address, trampoline_address);
715 
716 	correct_ret_addr = ri->ret_addr;
717 	hlist_for_each_entry_safe(ri, tmp, head, hlist) {
718 		if (ri->task != current)
719 			/* another task is sharing our hash bucket */
720 			continue;
721 
722 		orig_ret_address = (unsigned long)ri->ret_addr;
723 		if (ri->rp && ri->rp->handler) {
724 			__this_cpu_write(current_kprobe, &ri->rp->kp);
725 			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
726 			ri->ret_addr = correct_ret_addr;
727 			ri->rp->handler(ri, regs);
728 			__this_cpu_write(current_kprobe, NULL);
729 		}
730 
731 		recycle_rp_inst(ri, &empty_rp);
732 
733 		if (orig_ret_address != trampoline_address)
734 			/*
735 			 * This is the real return address. Any other
736 			 * instances associated with this task are for
737 			 * other calls deeper on the call stack
738 			 */
739 			break;
740 	}
741 
742 	kretprobe_hash_unlock(current, &flags);
743 
744 	hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
745 		hlist_del(&ri->hlist);
746 		kfree(ri);
747 	}
748 	return (void *)orig_ret_address;
749 }
750 
751 /*
752  * Called after single-stepping.  p->addr is the address of the
753  * instruction whose first byte has been replaced by the "int 3"
754  * instruction.  To avoid the SMP problems that can occur when we
755  * temporarily put back the original opcode to single-step, we
756  * single-stepped a copy of the instruction.  The address of this
757  * copy is p->ainsn.insn.
758  *
759  * This function prepares to return from the post-single-step
760  * interrupt.  We have to fix up the stack as follows:
761  *
762  * 0) Except in the case of absolute or indirect jump or call instructions,
763  * the new ip is relative to the copied instruction.  We need to make
764  * it relative to the original instruction.
765  *
766  * 1) If the single-stepped instruction was pushfl, then the TF and IF
767  * flags are set in the just-pushed flags, and may need to be cleared.
768  *
769  * 2) If the single-stepped instruction was a call, the return address
770  * that is atop the stack is the address following the copied instruction.
771  * We need to make it the address following the original instruction.
772  *
773  * If this is the first time we've single-stepped the instruction at
774  * this probepoint, and the instruction is boostable, boost it: add a
775  * jump instruction after the copied instruction, that jumps to the next
776  * instruction after the probepoint.
777  */
778 static void __kprobes
779 resume_execution(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb)
780 {
781 	unsigned long *tos = stack_addr(regs);
782 	unsigned long copy_ip = (unsigned long)p->ainsn.insn;
783 	unsigned long orig_ip = (unsigned long)p->addr;
784 	kprobe_opcode_t *insn = p->ainsn.insn;
785 
786 	/* Skip prefixes */
787 	insn = skip_prefixes(insn);
788 
789 	regs->flags &= ~X86_EFLAGS_TF;
790 	switch (*insn) {
791 	case 0x9c:	/* pushfl */
792 		*tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF);
793 		*tos |= kcb->kprobe_old_flags;
794 		break;
795 	case 0xc2:	/* iret/ret/lret */
796 	case 0xc3:
797 	case 0xca:
798 	case 0xcb:
799 	case 0xcf:
800 	case 0xea:	/* jmp absolute -- ip is correct */
801 		/* ip is already adjusted, no more changes required */
802 		p->ainsn.boostable = 1;
803 		goto no_change;
804 	case 0xe8:	/* call relative - Fix return addr */
805 		*tos = orig_ip + (*tos - copy_ip);
806 		break;
807 #ifdef CONFIG_X86_32
808 	case 0x9a:	/* call absolute -- same as call absolute, indirect */
809 		*tos = orig_ip + (*tos - copy_ip);
810 		goto no_change;
811 #endif
812 	case 0xff:
813 		if ((insn[1] & 0x30) == 0x10) {
814 			/*
815 			 * call absolute, indirect
816 			 * Fix return addr; ip is correct.
817 			 * But this is not boostable
818 			 */
819 			*tos = orig_ip + (*tos - copy_ip);
820 			goto no_change;
821 		} else if (((insn[1] & 0x31) == 0x20) ||
822 			   ((insn[1] & 0x31) == 0x21)) {
823 			/*
824 			 * jmp near and far, absolute indirect
825 			 * ip is correct. And this is boostable
826 			 */
827 			p->ainsn.boostable = 1;
828 			goto no_change;
829 		}
830 	default:
831 		break;
832 	}
833 
834 	if (p->ainsn.boostable == 0) {
835 		if ((regs->ip > copy_ip) &&
836 		    (regs->ip - copy_ip) + 5 < MAX_INSN_SIZE) {
837 			/*
838 			 * These instructions can be executed directly if it
839 			 * jumps back to correct address.
840 			 */
841 			synthesize_reljump((void *)regs->ip,
842 				(void *)orig_ip + (regs->ip - copy_ip));
843 			p->ainsn.boostable = 1;
844 		} else {
845 			p->ainsn.boostable = -1;
846 		}
847 	}
848 
849 	regs->ip += orig_ip - copy_ip;
850 
851 no_change:
852 	restore_btf();
853 }
854 
855 /*
856  * Interrupts are disabled on entry as trap1 is an interrupt gate and they
857  * remain disabled throughout this function.
858  */
859 static int __kprobes post_kprobe_handler(struct pt_regs *regs)
860 {
861 	struct kprobe *cur = kprobe_running();
862 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
863 
864 	if (!cur)
865 		return 0;
866 
867 	resume_execution(cur, regs, kcb);
868 	regs->flags |= kcb->kprobe_saved_flags;
869 
870 	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
871 		kcb->kprobe_status = KPROBE_HIT_SSDONE;
872 		cur->post_handler(cur, regs, 0);
873 	}
874 
875 	/* Restore back the original saved kprobes variables and continue. */
876 	if (kcb->kprobe_status == KPROBE_REENTER) {
877 		restore_previous_kprobe(kcb);
878 		goto out;
879 	}
880 	reset_current_kprobe();
881 out:
882 	preempt_enable_no_resched();
883 
884 	/*
885 	 * if somebody else is singlestepping across a probe point, flags
886 	 * will have TF set, in which case, continue the remaining processing
887 	 * of do_debug, as if this is not a probe hit.
888 	 */
889 	if (regs->flags & X86_EFLAGS_TF)
890 		return 0;
891 
892 	return 1;
893 }
894 
895 int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
896 {
897 	struct kprobe *cur = kprobe_running();
898 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
899 
900 	switch (kcb->kprobe_status) {
901 	case KPROBE_HIT_SS:
902 	case KPROBE_REENTER:
903 		/*
904 		 * We are here because the instruction being single
905 		 * stepped caused a page fault. We reset the current
906 		 * kprobe and the ip points back to the probe address
907 		 * and allow the page fault handler to continue as a
908 		 * normal page fault.
909 		 */
910 		regs->ip = (unsigned long)cur->addr;
911 		regs->flags |= kcb->kprobe_old_flags;
912 		if (kcb->kprobe_status == KPROBE_REENTER)
913 			restore_previous_kprobe(kcb);
914 		else
915 			reset_current_kprobe();
916 		preempt_enable_no_resched();
917 		break;
918 	case KPROBE_HIT_ACTIVE:
919 	case KPROBE_HIT_SSDONE:
920 		/*
921 		 * We increment the nmissed count for accounting,
922 		 * we can also use npre/npostfault count for accounting
923 		 * these specific fault cases.
924 		 */
925 		kprobes_inc_nmissed_count(cur);
926 
927 		/*
928 		 * We come here because instructions in the pre/post
929 		 * handler caused the page_fault, this could happen
930 		 * if handler tries to access user space by
931 		 * copy_from_user(), get_user() etc. Let the
932 		 * user-specified handler try to fix it first.
933 		 */
934 		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
935 			return 1;
936 
937 		/*
938 		 * In case the user-specified fault handler returned
939 		 * zero, try to fix up.
940 		 */
941 		if (fixup_exception(regs))
942 			return 1;
943 
944 		/*
945 		 * fixup routine could not handle it,
946 		 * Let do_page_fault() fix it.
947 		 */
948 		break;
949 	default:
950 		break;
951 	}
952 	return 0;
953 }
954 
955 /*
956  * Wrapper routine for handling exceptions.
957  */
958 int __kprobes
959 kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data)
960 {
961 	struct die_args *args = data;
962 	int ret = NOTIFY_DONE;
963 
964 	if (args->regs && user_mode_vm(args->regs))
965 		return ret;
966 
967 	switch (val) {
968 	case DIE_INT3:
969 		if (kprobe_handler(args->regs))
970 			ret = NOTIFY_STOP;
971 		break;
972 	case DIE_DEBUG:
973 		if (post_kprobe_handler(args->regs)) {
974 			/*
975 			 * Reset the BS bit in dr6 (pointed by args->err) to
976 			 * denote completion of processing
977 			 */
978 			(*(unsigned long *)ERR_PTR(args->err)) &= ~DR_STEP;
979 			ret = NOTIFY_STOP;
980 		}
981 		break;
982 	case DIE_GPF:
983 		/*
984 		 * To be potentially processing a kprobe fault and to
985 		 * trust the result from kprobe_running(), we have
986 		 * be non-preemptible.
987 		 */
988 		if (!preemptible() && kprobe_running() &&
989 		    kprobe_fault_handler(args->regs, args->trapnr))
990 			ret = NOTIFY_STOP;
991 		break;
992 	default:
993 		break;
994 	}
995 	return ret;
996 }
997 
998 int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
999 {
1000 	struct jprobe *jp = container_of(p, struct jprobe, kp);
1001 	unsigned long addr;
1002 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1003 
1004 	kcb->jprobe_saved_regs = *regs;
1005 	kcb->jprobe_saved_sp = stack_addr(regs);
1006 	addr = (unsigned long)(kcb->jprobe_saved_sp);
1007 
1008 	/*
1009 	 * As Linus pointed out, gcc assumes that the callee
1010 	 * owns the argument space and could overwrite it, e.g.
1011 	 * tailcall optimization. So, to be absolutely safe
1012 	 * we also save and restore enough stack bytes to cover
1013 	 * the argument area.
1014 	 */
1015 	memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
1016 	       MIN_STACK_SIZE(addr));
1017 	regs->flags &= ~X86_EFLAGS_IF;
1018 	trace_hardirqs_off();
1019 	regs->ip = (unsigned long)(jp->entry);
1020 	return 1;
1021 }
1022 
1023 void __kprobes jprobe_return(void)
1024 {
1025 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1026 
1027 	asm volatile (
1028 #ifdef CONFIG_X86_64
1029 			"       xchg   %%rbx,%%rsp	\n"
1030 #else
1031 			"       xchgl   %%ebx,%%esp	\n"
1032 #endif
1033 			"       int3			\n"
1034 			"       .globl jprobe_return_end\n"
1035 			"       jprobe_return_end:	\n"
1036 			"       nop			\n"::"b"
1037 			(kcb->jprobe_saved_sp):"memory");
1038 }
1039 
1040 int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
1041 {
1042 	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1043 	u8 *addr = (u8 *) (regs->ip - 1);
1044 	struct jprobe *jp = container_of(p, struct jprobe, kp);
1045 
1046 	if ((addr > (u8 *) jprobe_return) &&
1047 	    (addr < (u8 *) jprobe_return_end)) {
1048 		if (stack_addr(regs) != kcb->jprobe_saved_sp) {
1049 			struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;
1050 			printk(KERN_ERR
1051 			       "current sp %p does not match saved sp %p\n",
1052 			       stack_addr(regs), kcb->jprobe_saved_sp);
1053 			printk(KERN_ERR "Saved registers for jprobe %p\n", jp);
1054 			show_regs(saved_regs);
1055 			printk(KERN_ERR "Current registers\n");
1056 			show_regs(regs);
1057 			BUG();
1058 		}
1059 		*regs = kcb->jprobe_saved_regs;
1060 		memcpy((kprobe_opcode_t *)(kcb->jprobe_saved_sp),
1061 		       kcb->jprobes_stack,
1062 		       MIN_STACK_SIZE(kcb->jprobe_saved_sp));
1063 		preempt_enable_no_resched();
1064 		return 1;
1065 	}
1066 	return 0;
1067 }
1068 
1069 int __init arch_init_kprobes(void)
1070 {
1071 	return 0;
1072 }
1073 
1074 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
1075 {
1076 	return 0;
1077 }
1078