1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/context_tracking.h> 4 #include <linux/entry-common.h> 5 #include <linux/resume_user_mode.h> 6 #include <linux/highmem.h> 7 #include <linux/jump_label.h> 8 #include <linux/kmsan.h> 9 #include <linux/livepatch.h> 10 #include <linux/audit.h> 11 #include <linux/tick.h> 12 13 #include "common.h" 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/syscalls.h> 17 18 /* See comment for enter_from_user_mode() in entry-common.h */ 19 static __always_inline void __enter_from_user_mode(struct pt_regs *regs) 20 { 21 arch_enter_from_user_mode(regs); 22 lockdep_hardirqs_off(CALLER_ADDR0); 23 24 CT_WARN_ON(__ct_state() != CONTEXT_USER); 25 user_exit_irqoff(); 26 27 instrumentation_begin(); 28 kmsan_unpoison_entry_regs(regs); 29 trace_hardirqs_off_finish(); 30 instrumentation_end(); 31 } 32 33 void noinstr enter_from_user_mode(struct pt_regs *regs) 34 { 35 __enter_from_user_mode(regs); 36 } 37 38 static inline void syscall_enter_audit(struct pt_regs *regs, long syscall) 39 { 40 if (unlikely(audit_context())) { 41 unsigned long args[6]; 42 43 syscall_get_arguments(current, regs, args); 44 audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]); 45 } 46 } 47 48 static long syscall_trace_enter(struct pt_regs *regs, long syscall, 49 unsigned long work) 50 { 51 long ret = 0; 52 53 /* 54 * Handle Syscall User Dispatch. This must comes first, since 55 * the ABI here can be something that doesn't make sense for 56 * other syscall_work features. 57 */ 58 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { 59 if (syscall_user_dispatch(regs)) 60 return -1L; 61 } 62 63 /* Handle ptrace */ 64 if (work & (SYSCALL_WORK_SYSCALL_TRACE | SYSCALL_WORK_SYSCALL_EMU)) { 65 ret = ptrace_report_syscall_entry(regs); 66 if (ret || (work & SYSCALL_WORK_SYSCALL_EMU)) 67 return -1L; 68 } 69 70 /* Do seccomp after ptrace, to catch any tracer changes. */ 71 if (work & SYSCALL_WORK_SECCOMP) { 72 ret = __secure_computing(NULL); 73 if (ret == -1L) 74 return ret; 75 } 76 77 /* Either of the above might have changed the syscall number */ 78 syscall = syscall_get_nr(current, regs); 79 80 if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT)) 81 trace_sys_enter(regs, syscall); 82 83 syscall_enter_audit(regs, syscall); 84 85 return ret ? : syscall; 86 } 87 88 static __always_inline long 89 __syscall_enter_from_user_work(struct pt_regs *regs, long syscall) 90 { 91 unsigned long work = READ_ONCE(current_thread_info()->syscall_work); 92 93 if (work & SYSCALL_WORK_ENTER) 94 syscall = syscall_trace_enter(regs, syscall, work); 95 96 return syscall; 97 } 98 99 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall) 100 { 101 return __syscall_enter_from_user_work(regs, syscall); 102 } 103 104 noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall) 105 { 106 long ret; 107 108 __enter_from_user_mode(regs); 109 110 instrumentation_begin(); 111 local_irq_enable(); 112 ret = __syscall_enter_from_user_work(regs, syscall); 113 instrumentation_end(); 114 115 return ret; 116 } 117 118 noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs) 119 { 120 __enter_from_user_mode(regs); 121 instrumentation_begin(); 122 local_irq_enable(); 123 instrumentation_end(); 124 } 125 126 /* See comment for exit_to_user_mode() in entry-common.h */ 127 static __always_inline void __exit_to_user_mode(void) 128 { 129 instrumentation_begin(); 130 trace_hardirqs_on_prepare(); 131 lockdep_hardirqs_on_prepare(); 132 instrumentation_end(); 133 134 user_enter_irqoff(); 135 arch_exit_to_user_mode(); 136 lockdep_hardirqs_on(CALLER_ADDR0); 137 } 138 139 void noinstr exit_to_user_mode(void) 140 { 141 __exit_to_user_mode(); 142 } 143 144 /* Workaround to allow gradual conversion of architecture code */ 145 void __weak arch_do_signal_or_restart(struct pt_regs *regs) { } 146 147 static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, 148 unsigned long ti_work) 149 { 150 /* 151 * Before returning to user space ensure that all pending work 152 * items have been completed. 153 */ 154 while (ti_work & EXIT_TO_USER_MODE_WORK) { 155 156 local_irq_enable_exit_to_user(ti_work); 157 158 if (ti_work & _TIF_NEED_RESCHED) 159 schedule(); 160 161 if (ti_work & _TIF_UPROBE) 162 uprobe_notify_resume(regs); 163 164 if (ti_work & _TIF_PATCH_PENDING) 165 klp_update_patch_state(current); 166 167 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) 168 arch_do_signal_or_restart(regs); 169 170 if (ti_work & _TIF_NOTIFY_RESUME) 171 resume_user_mode_work(regs); 172 173 /* Architecture specific TIF work */ 174 arch_exit_to_user_mode_work(regs, ti_work); 175 176 /* 177 * Disable interrupts and reevaluate the work flags as they 178 * might have changed while interrupts and preemption was 179 * enabled above. 180 */ 181 local_irq_disable_exit_to_user(); 182 183 /* Check if any of the above work has queued a deferred wakeup */ 184 tick_nohz_user_enter_prepare(); 185 186 ti_work = read_thread_flags(); 187 } 188 189 /* Return the latest work state for arch_exit_to_user_mode() */ 190 return ti_work; 191 } 192 193 static void exit_to_user_mode_prepare(struct pt_regs *regs) 194 { 195 unsigned long ti_work; 196 197 lockdep_assert_irqs_disabled(); 198 199 /* Flush pending rcuog wakeup before the last need_resched() check */ 200 tick_nohz_user_enter_prepare(); 201 202 ti_work = read_thread_flags(); 203 if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) 204 ti_work = exit_to_user_mode_loop(regs, ti_work); 205 206 arch_exit_to_user_mode_prepare(regs, ti_work); 207 208 /* Ensure that the address limit is intact and no locks are held */ 209 addr_limit_user_check(); 210 kmap_assert_nomap(); 211 lockdep_assert_irqs_disabled(); 212 lockdep_sys_exit(); 213 } 214 215 /* 216 * If SYSCALL_EMU is set, then the only reason to report is when 217 * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall 218 * instruction has been already reported in syscall_enter_from_user_mode(). 219 */ 220 static inline bool report_single_step(unsigned long work) 221 { 222 if (work & SYSCALL_WORK_SYSCALL_EMU) 223 return false; 224 225 return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP; 226 } 227 228 static void syscall_exit_work(struct pt_regs *regs, unsigned long work) 229 { 230 bool step; 231 232 /* 233 * If the syscall was rolled back due to syscall user dispatching, 234 * then the tracers below are not invoked for the same reason as 235 * the entry side was not invoked in syscall_trace_enter(): The ABI 236 * of these syscalls is unknown. 237 */ 238 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { 239 if (unlikely(current->syscall_dispatch.on_dispatch)) { 240 current->syscall_dispatch.on_dispatch = false; 241 return; 242 } 243 } 244 245 audit_syscall_exit(regs); 246 247 if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT) 248 trace_sys_exit(regs, syscall_get_return_value(current, regs)); 249 250 step = report_single_step(work); 251 if (step || work & SYSCALL_WORK_SYSCALL_TRACE) 252 ptrace_report_syscall_exit(regs, step); 253 } 254 255 /* 256 * Syscall specific exit to user mode preparation. Runs with interrupts 257 * enabled. 258 */ 259 static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs) 260 { 261 unsigned long work = READ_ONCE(current_thread_info()->syscall_work); 262 unsigned long nr = syscall_get_nr(current, regs); 263 264 CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 265 266 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 267 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr)) 268 local_irq_enable(); 269 } 270 271 rseq_syscall(regs); 272 273 /* 274 * Do one-time syscall specific work. If these work items are 275 * enabled, we want to run them exactly once per syscall exit with 276 * interrupts enabled. 277 */ 278 if (unlikely(work & SYSCALL_WORK_EXIT)) 279 syscall_exit_work(regs, work); 280 } 281 282 static __always_inline void __syscall_exit_to_user_mode_work(struct pt_regs *regs) 283 { 284 syscall_exit_to_user_mode_prepare(regs); 285 local_irq_disable_exit_to_user(); 286 exit_to_user_mode_prepare(regs); 287 } 288 289 void syscall_exit_to_user_mode_work(struct pt_regs *regs) 290 { 291 __syscall_exit_to_user_mode_work(regs); 292 } 293 294 __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs) 295 { 296 instrumentation_begin(); 297 __syscall_exit_to_user_mode_work(regs); 298 instrumentation_end(); 299 __exit_to_user_mode(); 300 } 301 302 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs) 303 { 304 __enter_from_user_mode(regs); 305 } 306 307 noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs) 308 { 309 instrumentation_begin(); 310 exit_to_user_mode_prepare(regs); 311 instrumentation_end(); 312 __exit_to_user_mode(); 313 } 314 315 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) 316 { 317 irqentry_state_t ret = { 318 .exit_rcu = false, 319 }; 320 321 if (user_mode(regs)) { 322 irqentry_enter_from_user_mode(regs); 323 return ret; 324 } 325 326 /* 327 * If this entry hit the idle task invoke ct_irq_enter() whether 328 * RCU is watching or not. 329 * 330 * Interrupts can nest when the first interrupt invokes softirq 331 * processing on return which enables interrupts. 332 * 333 * Scheduler ticks in the idle task can mark quiescent state and 334 * terminate a grace period, if and only if the timer interrupt is 335 * not nested into another interrupt. 336 * 337 * Checking for rcu_is_watching() here would prevent the nesting 338 * interrupt to invoke ct_irq_enter(). If that nested interrupt is 339 * the tick then rcu_flavor_sched_clock_irq() would wrongfully 340 * assume that it is the first interrupt and eventually claim 341 * quiescent state and end grace periods prematurely. 342 * 343 * Unconditionally invoke ct_irq_enter() so RCU state stays 344 * consistent. 345 * 346 * TINY_RCU does not support EQS, so let the compiler eliminate 347 * this part when enabled. 348 */ 349 if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { 350 /* 351 * If RCU is not watching then the same careful 352 * sequence vs. lockdep and tracing is required 353 * as in irqentry_enter_from_user_mode(). 354 */ 355 lockdep_hardirqs_off(CALLER_ADDR0); 356 ct_irq_enter(); 357 instrumentation_begin(); 358 kmsan_unpoison_entry_regs(regs); 359 trace_hardirqs_off_finish(); 360 instrumentation_end(); 361 362 ret.exit_rcu = true; 363 return ret; 364 } 365 366 /* 367 * If RCU is watching then RCU only wants to check whether it needs 368 * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick() 369 * already contains a warning when RCU is not watching, so no point 370 * in having another one here. 371 */ 372 lockdep_hardirqs_off(CALLER_ADDR0); 373 instrumentation_begin(); 374 kmsan_unpoison_entry_regs(regs); 375 rcu_irq_enter_check_tick(); 376 trace_hardirqs_off_finish(); 377 instrumentation_end(); 378 379 return ret; 380 } 381 382 void raw_irqentry_exit_cond_resched(void) 383 { 384 if (!preempt_count()) { 385 /* Sanity check RCU and thread stack */ 386 rcu_irq_exit_check_preempt(); 387 if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) 388 WARN_ON_ONCE(!on_thread_stack()); 389 if (need_resched()) 390 preempt_schedule_irq(); 391 } 392 } 393 #ifdef CONFIG_PREEMPT_DYNAMIC 394 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 395 DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); 396 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 397 DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); 398 void dynamic_irqentry_exit_cond_resched(void) 399 { 400 if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) 401 return; 402 raw_irqentry_exit_cond_resched(); 403 } 404 #endif 405 #endif 406 407 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) 408 { 409 lockdep_assert_irqs_disabled(); 410 411 /* Check whether this returns to user mode */ 412 if (user_mode(regs)) { 413 irqentry_exit_to_user_mode(regs); 414 } else if (!regs_irqs_disabled(regs)) { 415 /* 416 * If RCU was not watching on entry this needs to be done 417 * carefully and needs the same ordering of lockdep/tracing 418 * and RCU as the return to user mode path. 419 */ 420 if (state.exit_rcu) { 421 instrumentation_begin(); 422 /* Tell the tracer that IRET will enable interrupts */ 423 trace_hardirqs_on_prepare(); 424 lockdep_hardirqs_on_prepare(); 425 instrumentation_end(); 426 ct_irq_exit(); 427 lockdep_hardirqs_on(CALLER_ADDR0); 428 return; 429 } 430 431 instrumentation_begin(); 432 if (IS_ENABLED(CONFIG_PREEMPTION)) 433 irqentry_exit_cond_resched(); 434 435 /* Covers both tracing and lockdep */ 436 trace_hardirqs_on(); 437 instrumentation_end(); 438 } else { 439 /* 440 * IRQ flags state is correct already. Just tell RCU if it 441 * was not watching on entry. 442 */ 443 if (state.exit_rcu) 444 ct_irq_exit(); 445 } 446 } 447 448 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) 449 { 450 irqentry_state_t irq_state; 451 452 irq_state.lockdep = lockdep_hardirqs_enabled(); 453 454 __nmi_enter(); 455 lockdep_hardirqs_off(CALLER_ADDR0); 456 lockdep_hardirq_enter(); 457 ct_nmi_enter(); 458 459 instrumentation_begin(); 460 kmsan_unpoison_entry_regs(regs); 461 trace_hardirqs_off_finish(); 462 ftrace_nmi_enter(); 463 instrumentation_end(); 464 465 return irq_state; 466 } 467 468 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state) 469 { 470 instrumentation_begin(); 471 ftrace_nmi_exit(); 472 if (irq_state.lockdep) { 473 trace_hardirqs_on_prepare(); 474 lockdep_hardirqs_on_prepare(); 475 } 476 instrumentation_end(); 477 478 ct_nmi_exit(); 479 lockdep_hardirq_exit(); 480 if (irq_state.lockdep) 481 lockdep_hardirqs_on(CALLER_ADDR0); 482 __nmi_exit(); 483 } 484