427 static int prepare_singlestep(kprobe_opcode_t *buf, struct kprobe *p,  in prepare_singlestep()  argument
433 	    !p->post_handler && can_boost(insn, p->addr) &&  in prepare_singlestep()
439 		synthesize_reljump(buf + len, p->ainsn.insn + len,  in prepare_singlestep()
440 				   p->addr + insn->length);  in prepare_singlestep()
442 		p->ainsn.boostable = 1;  in prepare_singlestep()
475 static void kprobe_emulate_ifmodifiers(struct kprobe *p, struct pt_regs *regs)  in kprobe_emulate_ifmodifiers()  argument
477 	switch (p->ainsn.opcode) {  in kprobe_emulate_ifmodifiers()
491 	regs->ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;  in kprobe_emulate_ifmodifiers()
495 static void kprobe_emulate_ret(struct kprobe *p, struct pt_regs *regs)  in kprobe_emulate_ret()  argument
501 static void kprobe_emulate_call(struct kprobe *p, struct pt_regs *regs)  in kprobe_emulate_call()  argument
503 	unsigned long func = regs->ip - INT3_INSN_SIZE + p->ainsn.size;  in kprobe_emulate_call()
505 	func += p->ainsn.rel32;  in kprobe_emulate_call()
510 static void kprobe_emulate_jmp(struct kprobe *p, struct pt_regs *regs)  in kprobe_emulate_jmp()  argument
512 	unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;  in kprobe_emulate_jmp()
514 	ip += p->ainsn.rel32;  in kprobe_emulate_jmp()
519 static void kprobe_emulate_jcc(struct kprobe *p, struct pt_regs *regs)  in kprobe_emulate_jcc()  argument
521 	unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;  in kprobe_emulate_jcc()
523 	int3_emulate_jcc(regs, p->ainsn.jcc.type, ip, p->ainsn.rel32);  in kprobe_emulate_jcc()
527 static void kprobe_emulate_loop(struct kprobe *p, struct pt_regs *regs)  in kprobe_emulate_loop()  argument
529 	unsigned long ip = regs->ip - INT3_INSN_SIZE + p->ainsn.size;  in kprobe_emulate_loop()
532 	if (p->ainsn.loop.type != 3) {	/* LOOP* */  in kprobe_emulate_loop()
533 		if (p->ainsn.loop.asize == 32)  in kprobe_emulate_loop()
536 		else if (p->ainsn.loop.asize == 64)  in kprobe_emulate_loop()
542 		if (p->ainsn.loop.asize == 32)  in kprobe_emulate_loop()
545 		else if (p->ainsn.loop.asize == 64)  in kprobe_emulate_loop()
552 	if (p->ainsn.loop.type == 0)	/* LOOPNE */  in kprobe_emulate_loop()
554 	else if (p->ainsn.loop.type == 1)	/* LOOPE */  in kprobe_emulate_loop()
558 		ip += p->ainsn.rel32;  in kprobe_emulate_loop()
584 static void kprobe_emulate_call_indirect(struct kprobe *p, struct pt_regs *regs)  in kprobe_emulate_call_indirect()  argument
586 	unsigned long offs = addrmode_regoffs[p->ainsn.indirect.reg];  in kprobe_emulate_call_indirect()
588 	int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + p->ainsn.size);  in kprobe_emulate_call_indirect()
593 static void kprobe_emulate_jmp_indirect(struct kprobe *p, struct pt_regs *regs)  in kprobe_emulate_jmp_indirect()  argument
595 	unsigned long offs = addrmode_regoffs[p->ainsn.indirect.reg];  in kprobe_emulate_jmp_indirect()
601 static int prepare_emulation(struct kprobe *p, struct insn *insn)  in prepare_emulation()  argument
614 		p->ainsn.emulate_op = kprobe_emulate_ifmodifiers;  in prepare_emulation()
615 		p->ainsn.opcode = opcode;  in prepare_emulation()
621 		p->ainsn.emulate_op = kprobe_emulate_ret;  in prepare_emulation()
630 		p->ainsn.emulate_op = kprobe_emulate_call;  in prepare_emulation()
632 			p->ainsn.rel32 = *(s16 *)&insn->immediate.value;  in prepare_emulation()
634 			p->ainsn.rel32 = *(s32 *)&insn->immediate.value;  in prepare_emulation()
638 		p->ainsn.emulate_op = kprobe_emulate_jmp;  in prepare_emulation()
640 			p->ainsn.rel32 = *(s8 *)&insn->immediate.value;  in prepare_emulation()
642 			p->ainsn.rel32 = *(s16 *)&insn->immediate.value;  in prepare_emulation()
644 			p->ainsn.rel32 = *(s32 *)&insn->immediate.value;  in prepare_emulation()
648 		p->ainsn.emulate_op = kprobe_emulate_jcc;  in prepare_emulation()
649 		p->ainsn.jcc.type = opcode & 0xf;  in prepare_emulation()
650 		p->ainsn.rel32 = insn->immediate.value;  in prepare_emulation()
656 			p->ainsn.emulate_op = kprobe_emulate_jcc;  in prepare_emulation()
657 			p->ainsn.jcc.type = opcode & 0xf;  in prepare_emulation()
659 				p->ainsn.rel32 = *(s16 *)&insn->immediate.value;  in prepare_emulation()
661 				p->ainsn.rel32 = *(s32 *)&insn->immediate.value;  in prepare_emulation()
673 		p->ainsn.emulate_op = kprobe_emulate_loop;  in prepare_emulation()
674 		p->ainsn.loop.type = opcode & 0x3;  in prepare_emulation()
675 		p->ainsn.loop.asize = insn->addr_bytes * 8;  in prepare_emulation()
676 		p->ainsn.rel32 = *(s8 *)&insn->immediate.value;  in prepare_emulation()
686 			p->ainsn.emulate_op = kprobe_emulate_call_indirect;  in prepare_emulation()
689 			p->ainsn.emulate_op = kprobe_emulate_jmp_indirect;  in prepare_emulation()
696 		if (!p->ainsn.emulate_op)  in prepare_emulation()
704 		p->ainsn.indirect.reg = X86_MODRM_RM(opcode);  in prepare_emulation()
707 			p->ainsn.indirect.reg += 8;  in prepare_emulation()
713 	p->ainsn.size = insn->length;  in prepare_emulation()
718 static int arch_copy_kprobe(struct kprobe *p)  in arch_copy_kprobe()  argument
725 	len = __copy_instruction(buf, p->addr, p->ainsn.insn, &insn);  in arch_copy_kprobe()
730 	ret = prepare_emulation(p, &insn);  in arch_copy_kprobe()
735 	len = prepare_singlestep(buf, p, &insn);  in arch_copy_kprobe()
740 	p->opcode = buf[0];  in arch_copy_kprobe()
742 	p->ainsn.tp_len = len;  in arch_copy_kprobe()
743 	perf_event_text_poke(p->ainsn.insn, NULL, 0, buf, len);  in arch_copy_kprobe()
746 	text_poke(p->ainsn.insn, buf, len);  in arch_copy_kprobe()
751 int arch_prepare_kprobe(struct kprobe *p)  in arch_prepare_kprobe()  argument
755 	if (alternatives_text_reserved(p->addr, p->addr))  in arch_prepare_kprobe()
758 	if (!can_probe((unsigned long)p->addr))  in arch_prepare_kprobe()
761 	memset(&p->ainsn, 0, sizeof(p->ainsn));  in arch_prepare_kprobe()
764 	p->ainsn.insn = get_insn_slot();  in arch_prepare_kprobe()
765 	if (!p->ainsn.insn)  in arch_prepare_kprobe()
768 	ret = arch_copy_kprobe(p);  in arch_prepare_kprobe()
770 		free_insn_slot(p->ainsn.insn, 0);  in arch_prepare_kprobe()
771 		p->ainsn.insn = NULL;  in arch_prepare_kprobe()
777 void arch_arm_kprobe(struct kprobe *p)  in arch_arm_kprobe()  argument
781 	text_poke(p->addr, &int3, 1);  in arch_arm_kprobe()
783 	perf_event_text_poke(p->addr, &p->opcode, 1, &int3, 1);  in arch_arm_kprobe()
786 void arch_disarm_kprobe(struct kprobe *p)  in arch_disarm_kprobe()  argument
790 	perf_event_text_poke(p->addr, &int3, 1, &p->opcode, 1);  in arch_disarm_kprobe()
791 	text_poke(p->addr, &p->opcode, 1);  in arch_disarm_kprobe()
795 void arch_remove_kprobe(struct kprobe *p)  in arch_remove_kprobe()  argument
797 	if (p->ainsn.insn) {  in arch_remove_kprobe()
799 		perf_event_text_poke(p->ainsn.insn, p->ainsn.insn,  in arch_remove_kprobe()
800 				     p->ainsn.tp_len, NULL, 0);  in arch_remove_kprobe()
801 		free_insn_slot(p->ainsn.insn, p->ainsn.boostable);  in arch_remove_kprobe()
802 		p->ainsn.insn = NULL;  in arch_remove_kprobe()
825 set_current_kprobe(struct kprobe *p, struct pt_regs *regs,  in set_current_kprobe()  argument
828 	__this_cpu_write(current_kprobe, p);  in set_current_kprobe()
853 static void setup_singlestep(struct kprobe *p, struct pt_regs *regs,  in setup_singlestep()  argument
856 	if (setup_detour_execution(p, regs, reenter))  in setup_singlestep()
860 	if (p->ainsn.boostable) {  in setup_singlestep()
869 		regs->ip = (unsigned long)p->ainsn.insn;  in setup_singlestep()
875 		set_current_kprobe(p, regs, kcb);  in setup_singlestep()
880 	if (p->ainsn.emulate_op) {  in setup_singlestep()
881 		p->ainsn.emulate_op(p, regs);  in setup_singlestep()
882 		kprobe_post_process(p, regs, kcb);  in setup_singlestep()
888 	regs->ip = (unsigned long)p->ainsn.insn;  in setup_singlestep()
910 static void resume_singlestep(struct kprobe *p, struct pt_regs *regs,  in resume_singlestep()  argument
913 	unsigned long copy_ip = (unsigned long)p->ainsn.insn;  in resume_singlestep()
914 	unsigned long orig_ip = (unsigned long)p->addr;  in resume_singlestep()
928 static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs,  in reenter_kprobe()  argument
935 		kprobes_inc_nmissed_count(p);  in reenter_kprobe()
936 		setup_singlestep(p, regs, kcb, 1);  in reenter_kprobe()
946 		dump_kprobe(p);  in reenter_kprobe()
971 	struct kprobe *p;  in kprobe_int3_handler()  local
985 	p = get_kprobe(addr);  in kprobe_int3_handler()
987 	if (p) {  in kprobe_int3_handler()
989 			if (reenter_kprobe(p, regs, kcb))  in kprobe_int3_handler()
992 			set_current_kprobe(p, regs, kcb);  in kprobe_int3_handler()
1002 			if (!p->pre_handler || !p->pre_handler(p, regs))  in kprobe_int3_handler()
1003 				setup_singlestep(p, regs, kcb, 0);  in kprobe_int3_handler()
1009 		p = kprobe_running();  in kprobe_int3_handler()
1010 		if ((unsigned long)p->ainsn.insn < regs->ip &&  in kprobe_int3_handler()
1011 		    (unsigned long)p->ainsn.insn + MAX_INSN_SIZE > regs->ip) {  in kprobe_int3_handler()
1013 			resume_singlestep(p, regs, kcb);  in kprobe_int3_handler()
1014 			kprobe_post_process(p, regs, kcb);  in kprobe_int3_handler()
1068 int arch_trampoline_kprobe(struct kprobe *p)  in arch_trampoline_kprobe()  argument