xref: /openbmc/linux/arch/x86/entry/common.c (revision abfbd895)
1 /*
2  * common.c - C code for kernel entry and exit
3  * Copyright (c) 2015 Andrew Lutomirski
4  * GPL v2
5  *
6  * Based on asm and ptrace code by many authors.  The code here originated
7  * in ptrace.c and signal.c.
8  */
9 
10 #include <linux/kernel.h>
11 #include <linux/sched.h>
12 #include <linux/mm.h>
13 #include <linux/smp.h>
14 #include <linux/errno.h>
15 #include <linux/ptrace.h>
16 #include <linux/tracehook.h>
17 #include <linux/audit.h>
18 #include <linux/seccomp.h>
19 #include <linux/signal.h>
20 #include <linux/export.h>
21 #include <linux/context_tracking.h>
22 #include <linux/user-return-notifier.h>
23 #include <linux/uprobes.h>
24 
25 #include <asm/desc.h>
26 #include <asm/traps.h>
27 #include <asm/vdso.h>
28 #include <asm/uaccess.h>
29 
30 #define CREATE_TRACE_POINTS
31 #include <trace/events/syscalls.h>
32 
33 static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
34 {
35 	unsigned long top_of_stack =
36 		(unsigned long)(regs + 1) + TOP_OF_KERNEL_STACK_PADDING;
37 	return (struct thread_info *)(top_of_stack - THREAD_SIZE);
38 }
39 
40 #ifdef CONFIG_CONTEXT_TRACKING
41 /* Called on entry from user mode with IRQs off. */
42 __visible void enter_from_user_mode(void)
43 {
44 	CT_WARN_ON(ct_state() != CONTEXT_USER);
45 	user_exit();
46 }
47 #endif
48 
49 static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)
50 {
51 #ifdef CONFIG_X86_64
52 	if (arch == AUDIT_ARCH_X86_64) {
53 		audit_syscall_entry(regs->orig_ax, regs->di,
54 				    regs->si, regs->dx, regs->r10);
55 	} else
56 #endif
57 	{
58 		audit_syscall_entry(regs->orig_ax, regs->bx,
59 				    regs->cx, regs->dx, regs->si);
60 	}
61 }
62 
63 /*
64  * We can return 0 to resume the syscall or anything else to go to phase
65  * 2.  If we resume the syscall, we need to put something appropriate in
66  * regs->orig_ax.
67  *
68  * NB: We don't have full pt_regs here, but regs->orig_ax and regs->ax
69  * are fully functional.
70  *
71  * For phase 2's benefit, our return value is:
72  * 0:			resume the syscall
73  * 1:			go to phase 2; no seccomp phase 2 needed
74  * anything else:	go to phase 2; pass return value to seccomp
75  */
76 unsigned long syscall_trace_enter_phase1(struct pt_regs *regs, u32 arch)
77 {
78 	struct thread_info *ti = pt_regs_to_thread_info(regs);
79 	unsigned long ret = 0;
80 	u32 work;
81 
82 	if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
83 		BUG_ON(regs != task_pt_regs(current));
84 
85 	work = ACCESS_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY;
86 
87 #ifdef CONFIG_CONTEXT_TRACKING
88 	/*
89 	 * If TIF_NOHZ is set, we are required to call user_exit() before
90 	 * doing anything that could touch RCU.
91 	 */
92 	if (work & _TIF_NOHZ) {
93 		enter_from_user_mode();
94 		work &= ~_TIF_NOHZ;
95 	}
96 #endif
97 
98 #ifdef CONFIG_SECCOMP
99 	/*
100 	 * Do seccomp first -- it should minimize exposure of other
101 	 * code, and keeping seccomp fast is probably more valuable
102 	 * than the rest of this.
103 	 */
104 	if (work & _TIF_SECCOMP) {
105 		struct seccomp_data sd;
106 
107 		sd.arch = arch;
108 		sd.nr = regs->orig_ax;
109 		sd.instruction_pointer = regs->ip;
110 #ifdef CONFIG_X86_64
111 		if (arch == AUDIT_ARCH_X86_64) {
112 			sd.args[0] = regs->di;
113 			sd.args[1] = regs->si;
114 			sd.args[2] = regs->dx;
115 			sd.args[3] = regs->r10;
116 			sd.args[4] = regs->r8;
117 			sd.args[5] = regs->r9;
118 		} else
119 #endif
120 		{
121 			sd.args[0] = regs->bx;
122 			sd.args[1] = regs->cx;
123 			sd.args[2] = regs->dx;
124 			sd.args[3] = regs->si;
125 			sd.args[4] = regs->di;
126 			sd.args[5] = regs->bp;
127 		}
128 
129 		BUILD_BUG_ON(SECCOMP_PHASE1_OK != 0);
130 		BUILD_BUG_ON(SECCOMP_PHASE1_SKIP != 1);
131 
132 		ret = seccomp_phase1(&sd);
133 		if (ret == SECCOMP_PHASE1_SKIP) {
134 			regs->orig_ax = -1;
135 			ret = 0;
136 		} else if (ret != SECCOMP_PHASE1_OK) {
137 			return ret;  /* Go directly to phase 2 */
138 		}
139 
140 		work &= ~_TIF_SECCOMP;
141 	}
142 #endif
143 
144 	/* Do our best to finish without phase 2. */
145 	if (work == 0)
146 		return ret;  /* seccomp and/or nohz only (ret == 0 here) */
147 
148 #ifdef CONFIG_AUDITSYSCALL
149 	if (work == _TIF_SYSCALL_AUDIT) {
150 		/*
151 		 * If there is no more work to be done except auditing,
152 		 * then audit in phase 1.  Phase 2 always audits, so, if
153 		 * we audit here, then we can't go on to phase 2.
154 		 */
155 		do_audit_syscall_entry(regs, arch);
156 		return 0;
157 	}
158 #endif
159 
160 	return 1;  /* Something is enabled that we can't handle in phase 1 */
161 }
162 
163 /* Returns the syscall nr to run (which should match regs->orig_ax). */
164 long syscall_trace_enter_phase2(struct pt_regs *regs, u32 arch,
165 				unsigned long phase1_result)
166 {
167 	struct thread_info *ti = pt_regs_to_thread_info(regs);
168 	long ret = 0;
169 	u32 work = ACCESS_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY;
170 
171 	if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
172 		BUG_ON(regs != task_pt_regs(current));
173 
174 	/*
175 	 * If we stepped into a sysenter/syscall insn, it trapped in
176 	 * kernel mode; do_debug() cleared TF and set TIF_SINGLESTEP.
177 	 * If user-mode had set TF itself, then it's still clear from
178 	 * do_debug() and we need to set it again to restore the user
179 	 * state.  If we entered on the slow path, TF was already set.
180 	 */
181 	if (work & _TIF_SINGLESTEP)
182 		regs->flags |= X86_EFLAGS_TF;
183 
184 #ifdef CONFIG_SECCOMP
185 	/*
186 	 * Call seccomp_phase2 before running the other hooks so that
187 	 * they can see any changes made by a seccomp tracer.
188 	 */
189 	if (phase1_result > 1 && seccomp_phase2(phase1_result)) {
190 		/* seccomp failures shouldn't expose any additional code. */
191 		return -1;
192 	}
193 #endif
194 
195 	if (unlikely(work & _TIF_SYSCALL_EMU))
196 		ret = -1L;
197 
198 	if ((ret || test_thread_flag(TIF_SYSCALL_TRACE)) &&
199 	    tracehook_report_syscall_entry(regs))
200 		ret = -1L;
201 
202 	if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
203 		trace_sys_enter(regs, regs->orig_ax);
204 
205 	do_audit_syscall_entry(regs, arch);
206 
207 	return ret ?: regs->orig_ax;
208 }
209 
210 long syscall_trace_enter(struct pt_regs *regs)
211 {
212 	u32 arch = is_ia32_task() ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64;
213 	unsigned long phase1_result = syscall_trace_enter_phase1(regs, arch);
214 
215 	if (phase1_result == 0)
216 		return regs->orig_ax;
217 	else
218 		return syscall_trace_enter_phase2(regs, arch, phase1_result);
219 }
220 
221 #define EXIT_TO_USERMODE_LOOP_FLAGS				\
222 	(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |	\
223 	 _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY)
224 
225 static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
226 {
227 	/*
228 	 * In order to return to user mode, we need to have IRQs off with
229 	 * none of _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_USER_RETURN_NOTIFY,
230 	 * _TIF_UPROBE, or _TIF_NEED_RESCHED set.  Several of these flags
231 	 * can be set at any time on preemptable kernels if we have IRQs on,
232 	 * so we need to loop.  Disabling preemption wouldn't help: doing the
233 	 * work to clear some of the flags can sleep.
234 	 */
235 	while (true) {
236 		/* We have work to do. */
237 		local_irq_enable();
238 
239 		if (cached_flags & _TIF_NEED_RESCHED)
240 			schedule();
241 
242 		if (cached_flags & _TIF_UPROBE)
243 			uprobe_notify_resume(regs);
244 
245 		/* deal with pending signal delivery */
246 		if (cached_flags & _TIF_SIGPENDING)
247 			do_signal(regs);
248 
249 		if (cached_flags & _TIF_NOTIFY_RESUME) {
250 			clear_thread_flag(TIF_NOTIFY_RESUME);
251 			tracehook_notify_resume(regs);
252 		}
253 
254 		if (cached_flags & _TIF_USER_RETURN_NOTIFY)
255 			fire_user_return_notifiers();
256 
257 		/* Disable IRQs and retry */
258 		local_irq_disable();
259 
260 		cached_flags = READ_ONCE(pt_regs_to_thread_info(regs)->flags);
261 
262 		if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
263 			break;
264 
265 	}
266 }
267 
268 /* Called with IRQs disabled. */
269 __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
270 {
271 	u32 cached_flags;
272 
273 	if (IS_ENABLED(CONFIG_PROVE_LOCKING) && WARN_ON(!irqs_disabled()))
274 		local_irq_disable();
275 
276 	lockdep_sys_exit();
277 
278 	cached_flags =
279 		READ_ONCE(pt_regs_to_thread_info(regs)->flags);
280 
281 	if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
282 		exit_to_usermode_loop(regs, cached_flags);
283 
284 	user_enter();
285 }
286 
287 #define SYSCALL_EXIT_WORK_FLAGS				\
288 	(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT |	\
289 	 _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)
290 
291 static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)
292 {
293 	bool step;
294 
295 	audit_syscall_exit(regs);
296 
297 	if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
298 		trace_sys_exit(regs, regs->ax);
299 
300 	/*
301 	 * If TIF_SYSCALL_EMU is set, we only get here because of
302 	 * TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
303 	 * We already reported this syscall instruction in
304 	 * syscall_trace_enter().
305 	 */
306 	step = unlikely(
307 		(cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
308 		== _TIF_SINGLESTEP);
309 	if (step || cached_flags & _TIF_SYSCALL_TRACE)
310 		tracehook_report_syscall_exit(regs, step);
311 }
312 
313 /*
314  * Called with IRQs on and fully valid regs.  Returns with IRQs off in a
315  * state such that we can immediately switch to user mode.
316  */
317 __visible inline void syscall_return_slowpath(struct pt_regs *regs)
318 {
319 	struct thread_info *ti = pt_regs_to_thread_info(regs);
320 	u32 cached_flags = READ_ONCE(ti->flags);
321 
322 	CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
323 
324 	if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
325 	    WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))
326 		local_irq_enable();
327 
328 	/*
329 	 * First do one-time work.  If these work items are enabled, we
330 	 * want to run them exactly once per syscall exit with IRQs on.
331 	 */
332 	if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))
333 		syscall_slow_exit_work(regs, cached_flags);
334 
335 #ifdef CONFIG_COMPAT
336 	/*
337 	 * Compat syscalls set TS_COMPAT.  Make sure we clear it before
338 	 * returning to user mode.
339 	 */
340 	ti->status &= ~TS_COMPAT;
341 #endif
342 
343 	local_irq_disable();
344 	prepare_exit_to_usermode(regs);
345 }
346 
347 #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
348 /*
349  * Does a 32-bit syscall.  Called with IRQs on and does all entry and
350  * exit work and returns with IRQs off.  This function is extremely hot
351  * in workloads that use it, and it's usually called from
352  * do_fast_syscall_32, so forcibly inline it to improve performance.
353  */
354 #ifdef CONFIG_X86_32
355 /* 32-bit kernels use a trap gate for INT80, and the asm code calls here. */
356 __visible
357 #else
358 /* 64-bit kernels use do_syscall_32_irqs_off() instead. */
359 static
360 #endif
361 __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
362 {
363 	struct thread_info *ti = pt_regs_to_thread_info(regs);
364 	unsigned int nr = (unsigned int)regs->orig_ax;
365 
366 #ifdef CONFIG_IA32_EMULATION
367 	ti->status |= TS_COMPAT;
368 #endif
369 
370 	if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
371 		/*
372 		 * Subtlety here: if ptrace pokes something larger than
373 		 * 2^32-1 into orig_ax, this truncates it.  This may or
374 		 * may not be necessary, but it matches the old asm
375 		 * behavior.
376 		 */
377 		nr = syscall_trace_enter(regs);
378 	}
379 
380 	if (likely(nr < IA32_NR_syscalls)) {
381 		/*
382 		 * It's possible that a 32-bit syscall implementation
383 		 * takes a 64-bit parameter but nonetheless assumes that
384 		 * the high bits are zero.  Make sure we zero-extend all
385 		 * of the args.
386 		 */
387 		regs->ax = ia32_sys_call_table[nr](
388 			(unsigned int)regs->bx, (unsigned int)regs->cx,
389 			(unsigned int)regs->dx, (unsigned int)regs->si,
390 			(unsigned int)regs->di, (unsigned int)regs->bp);
391 	}
392 
393 	syscall_return_slowpath(regs);
394 }
395 
396 #ifdef CONFIG_X86_64
397 /* Handles INT80 on 64-bit kernels */
398 __visible void do_syscall_32_irqs_off(struct pt_regs *regs)
399 {
400 	local_irq_enable();
401 	do_syscall_32_irqs_on(regs);
402 }
403 #endif
404 
405 /* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */
406 __visible long do_fast_syscall_32(struct pt_regs *regs)
407 {
408 	/*
409 	 * Called using the internal vDSO SYSENTER/SYSCALL32 calling
410 	 * convention.  Adjust regs so it looks like we entered using int80.
411 	 */
412 
413 	unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
414 		vdso_image_32.sym_int80_landing_pad;
415 
416 	/*
417 	 * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
418 	 * so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
419 	 * Fix it up.
420 	 */
421 	regs->ip = landing_pad;
422 
423 	/*
424 	 * Fetch EBP from where the vDSO stashed it.
425 	 *
426 	 * WARNING: We are in CONTEXT_USER and RCU isn't paying attention!
427 	 */
428 	local_irq_enable();
429 	if (
430 #ifdef CONFIG_X86_64
431 		/*
432 		 * Micro-optimization: the pointer we're following is explicitly
433 		 * 32 bits, so it can't be out of range.
434 		 */
435 		__get_user(*(u32 *)&regs->bp,
436 			    (u32 __user __force *)(unsigned long)(u32)regs->sp)
437 #else
438 		get_user(*(u32 *)&regs->bp,
439 			 (u32 __user __force *)(unsigned long)(u32)regs->sp)
440 #endif
441 		) {
442 
443 		/* User code screwed up. */
444 		local_irq_disable();
445 		regs->ax = -EFAULT;
446 #ifdef CONFIG_CONTEXT_TRACKING
447 		enter_from_user_mode();
448 #endif
449 		prepare_exit_to_usermode(regs);
450 		return 0;	/* Keep it simple: use IRET. */
451 	}
452 
453 	/* Now this is just like a normal syscall. */
454 	do_syscall_32_irqs_on(regs);
455 
456 #ifdef CONFIG_X86_64
457 	/*
458 	 * Opportunistic SYSRETL: if possible, try to return using SYSRETL.
459 	 * SYSRETL is available on all 64-bit CPUs, so we don't need to
460 	 * bother with SYSEXIT.
461 	 *
462 	 * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
463 	 * because the ECX fixup above will ensure that this is essentially
464 	 * never the case.
465 	 */
466 	return regs->cs == __USER32_CS && regs->ss == __USER_DS &&
467 		regs->ip == landing_pad &&
468 		(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)) == 0;
469 #else
470 	/*
471 	 * Opportunistic SYSEXIT: if possible, try to return using SYSEXIT.
472 	 *
473 	 * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
474 	 * because the ECX fixup above will ensure that this is essentially
475 	 * never the case.
476 	 *
477 	 * We don't allow syscalls at all from VM86 mode, but we still
478 	 * need to check VM, because we might be returning from sys_vm86.
479 	 */
480 	return static_cpu_has(X86_FEATURE_SEP) &&
481 		regs->cs == __USER_CS && regs->ss == __USER_DS &&
482 		regs->ip == landing_pad &&
483 		(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)) == 0;
484 #endif
485 }
486 #endif
487