1 /* 2 * common.c - C code for kernel entry and exit 3 * Copyright (c) 2015 Andrew Lutomirski 4 * GPL v2 5 * 6 * Based on asm and ptrace code by many authors. The code here originated 7 * in ptrace.c and signal.c. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/sched.h> 12 #include <linux/mm.h> 13 #include <linux/smp.h> 14 #include <linux/errno.h> 15 #include <linux/ptrace.h> 16 #include <linux/tracehook.h> 17 #include <linux/audit.h> 18 #include <linux/seccomp.h> 19 #include <linux/signal.h> 20 #include <linux/export.h> 21 #include <linux/context_tracking.h> 22 #include <linux/user-return-notifier.h> 23 #include <linux/uprobes.h> 24 25 #include <asm/desc.h> 26 #include <asm/traps.h> 27 #include <asm/vdso.h> 28 #include <asm/uaccess.h> 29 30 #define CREATE_TRACE_POINTS 31 #include <trace/events/syscalls.h> 32 33 #ifdef CONFIG_CONTEXT_TRACKING 34 /* Called on entry from user mode with IRQs off. */ 35 __visible void enter_from_user_mode(void) 36 { 37 CT_WARN_ON(ct_state() != CONTEXT_USER); 38 user_exit(); 39 } 40 #endif 41 42 static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch) 43 { 44 #ifdef CONFIG_X86_64 45 if (arch == AUDIT_ARCH_X86_64) { 46 audit_syscall_entry(regs->orig_ax, regs->di, 47 regs->si, regs->dx, regs->r10); 48 } else 49 #endif 50 { 51 audit_syscall_entry(regs->orig_ax, regs->bx, 52 regs->cx, regs->dx, regs->si); 53 } 54 } 55 56 /* 57 * We can return 0 to resume the syscall or anything else to go to phase 58 * 2. If we resume the syscall, we need to put something appropriate in 59 * regs->orig_ax. 60 * 61 * NB: We don't have full pt_regs here, but regs->orig_ax and regs->ax 62 * are fully functional. 63 * 64 * For phase 2's benefit, our return value is: 65 * 0: resume the syscall 66 * 1: go to phase 2; no seccomp phase 2 needed 67 * anything else: go to phase 2; pass return value to seccomp 68 */ 69 unsigned long syscall_trace_enter_phase1(struct pt_regs *regs, u32 arch) 70 { 71 unsigned long ret = 0; 72 u32 work; 73 74 BUG_ON(regs != task_pt_regs(current)); 75 76 work = ACCESS_ONCE(current_thread_info()->flags) & 77 _TIF_WORK_SYSCALL_ENTRY; 78 79 #ifdef CONFIG_CONTEXT_TRACKING 80 /* 81 * If TIF_NOHZ is set, we are required to call user_exit() before 82 * doing anything that could touch RCU. 83 */ 84 if (work & _TIF_NOHZ) { 85 enter_from_user_mode(); 86 work &= ~_TIF_NOHZ; 87 } 88 #endif 89 90 #ifdef CONFIG_SECCOMP 91 /* 92 * Do seccomp first -- it should minimize exposure of other 93 * code, and keeping seccomp fast is probably more valuable 94 * than the rest of this. 95 */ 96 if (work & _TIF_SECCOMP) { 97 struct seccomp_data sd; 98 99 sd.arch = arch; 100 sd.nr = regs->orig_ax; 101 sd.instruction_pointer = regs->ip; 102 #ifdef CONFIG_X86_64 103 if (arch == AUDIT_ARCH_X86_64) { 104 sd.args[0] = regs->di; 105 sd.args[1] = regs->si; 106 sd.args[2] = regs->dx; 107 sd.args[3] = regs->r10; 108 sd.args[4] = regs->r8; 109 sd.args[5] = regs->r9; 110 } else 111 #endif 112 { 113 sd.args[0] = regs->bx; 114 sd.args[1] = regs->cx; 115 sd.args[2] = regs->dx; 116 sd.args[3] = regs->si; 117 sd.args[4] = regs->di; 118 sd.args[5] = regs->bp; 119 } 120 121 BUILD_BUG_ON(SECCOMP_PHASE1_OK != 0); 122 BUILD_BUG_ON(SECCOMP_PHASE1_SKIP != 1); 123 124 ret = seccomp_phase1(&sd); 125 if (ret == SECCOMP_PHASE1_SKIP) { 126 regs->orig_ax = -1; 127 ret = 0; 128 } else if (ret != SECCOMP_PHASE1_OK) { 129 return ret; /* Go directly to phase 2 */ 130 } 131 132 work &= ~_TIF_SECCOMP; 133 } 134 #endif 135 136 /* Do our best to finish without phase 2. */ 137 if (work == 0) 138 return ret; /* seccomp and/or nohz only (ret == 0 here) */ 139 140 #ifdef CONFIG_AUDITSYSCALL 141 if (work == _TIF_SYSCALL_AUDIT) { 142 /* 143 * If there is no more work to be done except auditing, 144 * then audit in phase 1. Phase 2 always audits, so, if 145 * we audit here, then we can't go on to phase 2. 146 */ 147 do_audit_syscall_entry(regs, arch); 148 return 0; 149 } 150 #endif 151 152 return 1; /* Something is enabled that we can't handle in phase 1 */ 153 } 154 155 /* Returns the syscall nr to run (which should match regs->orig_ax). */ 156 long syscall_trace_enter_phase2(struct pt_regs *regs, u32 arch, 157 unsigned long phase1_result) 158 { 159 long ret = 0; 160 u32 work = ACCESS_ONCE(current_thread_info()->flags) & 161 _TIF_WORK_SYSCALL_ENTRY; 162 163 BUG_ON(regs != task_pt_regs(current)); 164 165 /* 166 * If we stepped into a sysenter/syscall insn, it trapped in 167 * kernel mode; do_debug() cleared TF and set TIF_SINGLESTEP. 168 * If user-mode had set TF itself, then it's still clear from 169 * do_debug() and we need to set it again to restore the user 170 * state. If we entered on the slow path, TF was already set. 171 */ 172 if (work & _TIF_SINGLESTEP) 173 regs->flags |= X86_EFLAGS_TF; 174 175 #ifdef CONFIG_SECCOMP 176 /* 177 * Call seccomp_phase2 before running the other hooks so that 178 * they can see any changes made by a seccomp tracer. 179 */ 180 if (phase1_result > 1 && seccomp_phase2(phase1_result)) { 181 /* seccomp failures shouldn't expose any additional code. */ 182 return -1; 183 } 184 #endif 185 186 if (unlikely(work & _TIF_SYSCALL_EMU)) 187 ret = -1L; 188 189 if ((ret || test_thread_flag(TIF_SYSCALL_TRACE)) && 190 tracehook_report_syscall_entry(regs)) 191 ret = -1L; 192 193 if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) 194 trace_sys_enter(regs, regs->orig_ax); 195 196 do_audit_syscall_entry(regs, arch); 197 198 return ret ?: regs->orig_ax; 199 } 200 201 long syscall_trace_enter(struct pt_regs *regs) 202 { 203 u32 arch = is_ia32_task() ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64; 204 unsigned long phase1_result = syscall_trace_enter_phase1(regs, arch); 205 206 if (phase1_result == 0) 207 return regs->orig_ax; 208 else 209 return syscall_trace_enter_phase2(regs, arch, phase1_result); 210 } 211 212 static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs) 213 { 214 unsigned long top_of_stack = 215 (unsigned long)(regs + 1) + TOP_OF_KERNEL_STACK_PADDING; 216 return (struct thread_info *)(top_of_stack - THREAD_SIZE); 217 } 218 219 /* Called with IRQs disabled. */ 220 __visible void prepare_exit_to_usermode(struct pt_regs *regs) 221 { 222 if (WARN_ON(!irqs_disabled())) 223 local_irq_disable(); 224 225 lockdep_sys_exit(); 226 227 /* 228 * In order to return to user mode, we need to have IRQs off with 229 * none of _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_USER_RETURN_NOTIFY, 230 * _TIF_UPROBE, or _TIF_NEED_RESCHED set. Several of these flags 231 * can be set at any time on preemptable kernels if we have IRQs on, 232 * so we need to loop. Disabling preemption wouldn't help: doing the 233 * work to clear some of the flags can sleep. 234 */ 235 while (true) { 236 u32 cached_flags = 237 READ_ONCE(pt_regs_to_thread_info(regs)->flags); 238 239 if (!(cached_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | 240 _TIF_UPROBE | _TIF_NEED_RESCHED | 241 _TIF_USER_RETURN_NOTIFY))) 242 break; 243 244 /* We have work to do. */ 245 local_irq_enable(); 246 247 if (cached_flags & _TIF_NEED_RESCHED) 248 schedule(); 249 250 if (cached_flags & _TIF_UPROBE) 251 uprobe_notify_resume(regs); 252 253 /* deal with pending signal delivery */ 254 if (cached_flags & _TIF_SIGPENDING) 255 do_signal(regs); 256 257 if (cached_flags & _TIF_NOTIFY_RESUME) { 258 clear_thread_flag(TIF_NOTIFY_RESUME); 259 tracehook_notify_resume(regs); 260 } 261 262 if (cached_flags & _TIF_USER_RETURN_NOTIFY) 263 fire_user_return_notifiers(); 264 265 /* Disable IRQs and retry */ 266 local_irq_disable(); 267 } 268 269 user_enter(); 270 } 271 272 /* 273 * Called with IRQs on and fully valid regs. Returns with IRQs off in a 274 * state such that we can immediately switch to user mode. 275 */ 276 __visible void syscall_return_slowpath(struct pt_regs *regs) 277 { 278 struct thread_info *ti = pt_regs_to_thread_info(regs); 279 u32 cached_flags = READ_ONCE(ti->flags); 280 bool step; 281 282 CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 283 284 if (WARN(irqs_disabled(), "syscall %ld left IRQs disabled", 285 regs->orig_ax)) 286 local_irq_enable(); 287 288 /* 289 * First do one-time work. If these work items are enabled, we 290 * want to run them exactly once per syscall exit with IRQs on. 291 */ 292 if (cached_flags & (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | 293 _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)) { 294 audit_syscall_exit(regs); 295 296 if (cached_flags & _TIF_SYSCALL_TRACEPOINT) 297 trace_sys_exit(regs, regs->ax); 298 299 /* 300 * If TIF_SYSCALL_EMU is set, we only get here because of 301 * TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP). 302 * We already reported this syscall instruction in 303 * syscall_trace_enter(). 304 */ 305 step = unlikely( 306 (cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU)) 307 == _TIF_SINGLESTEP); 308 if (step || cached_flags & _TIF_SYSCALL_TRACE) 309 tracehook_report_syscall_exit(regs, step); 310 } 311 312 #ifdef CONFIG_COMPAT 313 /* 314 * Compat syscalls set TS_COMPAT. Make sure we clear it before 315 * returning to user mode. 316 */ 317 ti->status &= ~TS_COMPAT; 318 #endif 319 320 local_irq_disable(); 321 prepare_exit_to_usermode(regs); 322 } 323 324 #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION) 325 /* 326 * Does a 32-bit syscall. Called with IRQs off and does all entry and 327 * exit work. 328 */ 329 __visible void do_int80_syscall_32(struct pt_regs *regs) 330 { 331 struct thread_info *ti = pt_regs_to_thread_info(regs); 332 unsigned int nr = (unsigned int)regs->orig_ax; 333 334 #ifdef CONFIG_IA32_EMULATION 335 ti->status |= TS_COMPAT; 336 #endif 337 338 local_irq_enable(); 339 340 if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) { 341 /* 342 * Subtlety here: if ptrace pokes something larger than 343 * 2^32-1 into orig_ax, this truncates it. This may or 344 * may not be necessary, but it matches the old asm 345 * behavior. 346 */ 347 nr = syscall_trace_enter(regs); 348 } 349 350 if (nr < IA32_NR_syscalls) { 351 /* 352 * It's possible that a 32-bit syscall implementation 353 * takes a 64-bit parameter but nonetheless assumes that 354 * the high bits are zero. Make sure we zero-extend all 355 * of the args. 356 */ 357 regs->ax = ia32_sys_call_table[nr]( 358 (unsigned int)regs->bx, (unsigned int)regs->cx, 359 (unsigned int)regs->dx, (unsigned int)regs->si, 360 (unsigned int)regs->di, (unsigned int)regs->bp); 361 } 362 363 syscall_return_slowpath(regs); 364 } 365 366 __visible void do_fast_syscall_32(struct pt_regs *regs) 367 { 368 /* 369 * Called using the internal vDSO SYSENTER/SYSCALL32 calling 370 * convention. Adjust regs so it looks like we entered using int80. 371 */ 372 373 unsigned long landing_pad = (unsigned long)current->mm->context.vdso + 374 vdso_image_32.sym_int80_landing_pad; 375 376 /* 377 * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward 378 * so that 'regs->ip -= 2' lands back on an int $0x80 instruction. 379 * Fix it up. 380 */ 381 regs->ip = landing_pad; 382 383 /* 384 * Fetch ECX from where the vDSO stashed it. 385 * 386 * WARNING: We are in CONTEXT_USER and RCU isn't paying attention! 387 */ 388 local_irq_enable(); 389 if (get_user(*(u32 *)®s->cx, 390 (u32 __user __force *)(unsigned long)(u32)regs->sp)) { 391 /* User code screwed up. */ 392 local_irq_disable(); 393 regs->ax = -EFAULT; 394 #ifdef CONFIG_CONTEXT_TRACKING 395 enter_from_user_mode(); 396 #endif 397 prepare_exit_to_usermode(regs); 398 return; 399 } 400 local_irq_disable(); 401 402 /* Now this is just like a normal syscall. */ 403 do_int80_syscall_32(regs); 404 return; 405 } 406 #endif 407