1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/context_tracking.h> 4 #include <linux/entry-common.h> 5 #include <linux/resume_user_mode.h> 6 #include <linux/highmem.h> 7 #include <linux/jump_label.h> 8 #include <linux/kmsan.h> 9 #include <linux/livepatch.h> 10 #include <linux/audit.h> 11 #include <linux/tick.h> 12 13 #include "common.h" 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/syscalls.h> 17 18 /* See comment for enter_from_user_mode() in entry-common.h */ 19 static __always_inline void __enter_from_user_mode(struct pt_regs *regs) 20 { 21 arch_enter_from_user_mode(regs); 22 lockdep_hardirqs_off(CALLER_ADDR0); 23 24 CT_WARN_ON(__ct_state() != CONTEXT_USER); 25 user_exit_irqoff(); 26 27 instrumentation_begin(); 28 kmsan_unpoison_entry_regs(regs); 29 trace_hardirqs_off_finish(); 30 instrumentation_end(); 31 } 32 33 void noinstr enter_from_user_mode(struct pt_regs *regs) 34 { 35 __enter_from_user_mode(regs); 36 } 37 38 static inline void syscall_enter_audit(struct pt_regs *regs, long syscall) 39 { 40 if (unlikely(audit_context())) { 41 unsigned long args[6]; 42 43 syscall_get_arguments(current, regs, args); 44 audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]); 45 } 46 } 47 48 static long syscall_trace_enter(struct pt_regs *regs, long syscall, 49 unsigned long work) 50 { 51 long ret = 0; 52 53 /* 54 * Handle Syscall User Dispatch. This must comes first, since 55 * the ABI here can be something that doesn't make sense for 56 * other syscall_work features. 57 */ 58 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { 59 if (syscall_user_dispatch(regs)) 60 return -1L; 61 } 62 63 /* Handle ptrace */ 64 if (work & (SYSCALL_WORK_SYSCALL_TRACE | SYSCALL_WORK_SYSCALL_EMU)) { 65 ret = ptrace_report_syscall_entry(regs); 66 if (ret || (work & SYSCALL_WORK_SYSCALL_EMU)) 67 return -1L; 68 } 69 70 /* Do seccomp after ptrace, to catch any tracer changes. */ 71 if (work & SYSCALL_WORK_SECCOMP) { 72 ret = __secure_computing(NULL); 73 if (ret == -1L) 74 return ret; 75 } 76 77 /* Either of the above might have changed the syscall number */ 78 syscall = syscall_get_nr(current, regs); 79 80 if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT)) 81 trace_sys_enter(regs, syscall); 82 83 syscall_enter_audit(regs, syscall); 84 85 return ret ? : syscall; 86 } 87 88 static __always_inline long 89 __syscall_enter_from_user_work(struct pt_regs *regs, long syscall) 90 { 91 unsigned long work = READ_ONCE(current_thread_info()->syscall_work); 92 93 if (work & SYSCALL_WORK_ENTER) 94 syscall = syscall_trace_enter(regs, syscall, work); 95 96 return syscall; 97 } 98 99 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall) 100 { 101 return __syscall_enter_from_user_work(regs, syscall); 102 } 103 104 noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall) 105 { 106 long ret; 107 108 __enter_from_user_mode(regs); 109 110 instrumentation_begin(); 111 local_irq_enable(); 112 ret = __syscall_enter_from_user_work(regs, syscall); 113 instrumentation_end(); 114 115 return ret; 116 } 117 118 noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs) 119 { 120 __enter_from_user_mode(regs); 121 instrumentation_begin(); 122 local_irq_enable(); 123 instrumentation_end(); 124 } 125 126 /* See comment for exit_to_user_mode() in entry-common.h */ 127 static __always_inline void __exit_to_user_mode(void) 128 { 129 instrumentation_begin(); 130 trace_hardirqs_on_prepare(); 131 lockdep_hardirqs_on_prepare(); 132 instrumentation_end(); 133 134 user_enter_irqoff(); 135 arch_exit_to_user_mode(); 136 lockdep_hardirqs_on(CALLER_ADDR0); 137 } 138 139 void noinstr exit_to_user_mode(void) 140 { 141 __exit_to_user_mode(); 142 } 143 144 /* Workaround to allow gradual conversion of architecture code */ 145 void __weak arch_do_signal_or_restart(struct pt_regs *regs) { } 146 147 static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, 148 unsigned long ti_work) 149 { 150 /* 151 * Before returning to user space ensure that all pending work 152 * items have been completed. 153 */ 154 while (ti_work & EXIT_TO_USER_MODE_WORK) { 155 156 local_irq_enable_exit_to_user(ti_work); 157 158 if (ti_work & _TIF_NEED_RESCHED) 159 schedule(); 160 161 if (ti_work & _TIF_UPROBE) 162 uprobe_notify_resume(regs); 163 164 if (ti_work & _TIF_PATCH_PENDING) 165 klp_update_patch_state(current); 166 167 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) 168 arch_do_signal_or_restart(regs); 169 170 if (ti_work & _TIF_NOTIFY_RESUME) 171 resume_user_mode_work(regs); 172 173 /* Architecture specific TIF work */ 174 arch_exit_to_user_mode_work(regs, ti_work); 175 176 /* 177 * Disable interrupts and reevaluate the work flags as they 178 * might have changed while interrupts and preemption was 179 * enabled above. 180 */ 181 local_irq_disable_exit_to_user(); 182 183 /* Check if any of the above work has queued a deferred wakeup */ 184 tick_nohz_user_enter_prepare(); 185 186 ti_work = read_thread_flags(); 187 } 188 189 /* Return the latest work state for arch_exit_to_user_mode() */ 190 return ti_work; 191 } 192 193 static void exit_to_user_mode_prepare(struct pt_regs *regs) 194 { 195 unsigned long ti_work; 196 197 lockdep_assert_irqs_disabled(); 198 199 /* Flush pending rcuog wakeup before the last need_resched() check */ 200 tick_nohz_user_enter_prepare(); 201 202 ti_work = read_thread_flags(); 203 if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) 204 ti_work = exit_to_user_mode_loop(regs, ti_work); 205 206 arch_exit_to_user_mode_prepare(regs, ti_work); 207 208 /* Ensure that kernel state is sane for a return to userspace */ 209 kmap_assert_nomap(); 210 lockdep_assert_irqs_disabled(); 211 lockdep_sys_exit(); 212 } 213 214 /* 215 * If SYSCALL_EMU is set, then the only reason to report is when 216 * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall 217 * instruction has been already reported in syscall_enter_from_user_mode(). 218 */ 219 static inline bool report_single_step(unsigned long work) 220 { 221 if (work & SYSCALL_WORK_SYSCALL_EMU) 222 return false; 223 224 return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP; 225 } 226 227 static void syscall_exit_work(struct pt_regs *regs, unsigned long work) 228 { 229 bool step; 230 231 /* 232 * If the syscall was rolled back due to syscall user dispatching, 233 * then the tracers below are not invoked for the same reason as 234 * the entry side was not invoked in syscall_trace_enter(): The ABI 235 * of these syscalls is unknown. 236 */ 237 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { 238 if (unlikely(current->syscall_dispatch.on_dispatch)) { 239 current->syscall_dispatch.on_dispatch = false; 240 return; 241 } 242 } 243 244 audit_syscall_exit(regs); 245 246 if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT) 247 trace_sys_exit(regs, syscall_get_return_value(current, regs)); 248 249 step = report_single_step(work); 250 if (step || work & SYSCALL_WORK_SYSCALL_TRACE) 251 ptrace_report_syscall_exit(regs, step); 252 } 253 254 /* 255 * Syscall specific exit to user mode preparation. Runs with interrupts 256 * enabled. 257 */ 258 static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs) 259 { 260 unsigned long work = READ_ONCE(current_thread_info()->syscall_work); 261 unsigned long nr = syscall_get_nr(current, regs); 262 263 CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 264 265 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 266 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr)) 267 local_irq_enable(); 268 } 269 270 rseq_syscall(regs); 271 272 /* 273 * Do one-time syscall specific work. If these work items are 274 * enabled, we want to run them exactly once per syscall exit with 275 * interrupts enabled. 276 */ 277 if (unlikely(work & SYSCALL_WORK_EXIT)) 278 syscall_exit_work(regs, work); 279 } 280 281 static __always_inline void __syscall_exit_to_user_mode_work(struct pt_regs *regs) 282 { 283 syscall_exit_to_user_mode_prepare(regs); 284 local_irq_disable_exit_to_user(); 285 exit_to_user_mode_prepare(regs); 286 } 287 288 void syscall_exit_to_user_mode_work(struct pt_regs *regs) 289 { 290 __syscall_exit_to_user_mode_work(regs); 291 } 292 293 __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs) 294 { 295 instrumentation_begin(); 296 __syscall_exit_to_user_mode_work(regs); 297 instrumentation_end(); 298 __exit_to_user_mode(); 299 } 300 301 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs) 302 { 303 __enter_from_user_mode(regs); 304 } 305 306 noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs) 307 { 308 instrumentation_begin(); 309 exit_to_user_mode_prepare(regs); 310 instrumentation_end(); 311 __exit_to_user_mode(); 312 } 313 314 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) 315 { 316 irqentry_state_t ret = { 317 .exit_rcu = false, 318 }; 319 320 if (user_mode(regs)) { 321 irqentry_enter_from_user_mode(regs); 322 return ret; 323 } 324 325 /* 326 * If this entry hit the idle task invoke ct_irq_enter() whether 327 * RCU is watching or not. 328 * 329 * Interrupts can nest when the first interrupt invokes softirq 330 * processing on return which enables interrupts. 331 * 332 * Scheduler ticks in the idle task can mark quiescent state and 333 * terminate a grace period, if and only if the timer interrupt is 334 * not nested into another interrupt. 335 * 336 * Checking for rcu_is_watching() here would prevent the nesting 337 * interrupt to invoke ct_irq_enter(). If that nested interrupt is 338 * the tick then rcu_flavor_sched_clock_irq() would wrongfully 339 * assume that it is the first interrupt and eventually claim 340 * quiescent state and end grace periods prematurely. 341 * 342 * Unconditionally invoke ct_irq_enter() so RCU state stays 343 * consistent. 344 * 345 * TINY_RCU does not support EQS, so let the compiler eliminate 346 * this part when enabled. 347 */ 348 if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { 349 /* 350 * If RCU is not watching then the same careful 351 * sequence vs. lockdep and tracing is required 352 * as in irqentry_enter_from_user_mode(). 353 */ 354 lockdep_hardirqs_off(CALLER_ADDR0); 355 ct_irq_enter(); 356 instrumentation_begin(); 357 kmsan_unpoison_entry_regs(regs); 358 trace_hardirqs_off_finish(); 359 instrumentation_end(); 360 361 ret.exit_rcu = true; 362 return ret; 363 } 364 365 /* 366 * If RCU is watching then RCU only wants to check whether it needs 367 * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick() 368 * already contains a warning when RCU is not watching, so no point 369 * in having another one here. 370 */ 371 lockdep_hardirqs_off(CALLER_ADDR0); 372 instrumentation_begin(); 373 kmsan_unpoison_entry_regs(regs); 374 rcu_irq_enter_check_tick(); 375 trace_hardirqs_off_finish(); 376 instrumentation_end(); 377 378 return ret; 379 } 380 381 void raw_irqentry_exit_cond_resched(void) 382 { 383 if (!preempt_count()) { 384 /* Sanity check RCU and thread stack */ 385 rcu_irq_exit_check_preempt(); 386 if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) 387 WARN_ON_ONCE(!on_thread_stack()); 388 if (need_resched()) 389 preempt_schedule_irq(); 390 } 391 } 392 #ifdef CONFIG_PREEMPT_DYNAMIC 393 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 394 DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); 395 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 396 DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); 397 void dynamic_irqentry_exit_cond_resched(void) 398 { 399 if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) 400 return; 401 raw_irqentry_exit_cond_resched(); 402 } 403 #endif 404 #endif 405 406 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) 407 { 408 lockdep_assert_irqs_disabled(); 409 410 /* Check whether this returns to user mode */ 411 if (user_mode(regs)) { 412 irqentry_exit_to_user_mode(regs); 413 } else if (!regs_irqs_disabled(regs)) { 414 /* 415 * If RCU was not watching on entry this needs to be done 416 * carefully and needs the same ordering of lockdep/tracing 417 * and RCU as the return to user mode path. 418 */ 419 if (state.exit_rcu) { 420 instrumentation_begin(); 421 /* Tell the tracer that IRET will enable interrupts */ 422 trace_hardirqs_on_prepare(); 423 lockdep_hardirqs_on_prepare(); 424 instrumentation_end(); 425 ct_irq_exit(); 426 lockdep_hardirqs_on(CALLER_ADDR0); 427 return; 428 } 429 430 instrumentation_begin(); 431 if (IS_ENABLED(CONFIG_PREEMPTION)) 432 irqentry_exit_cond_resched(); 433 434 /* Covers both tracing and lockdep */ 435 trace_hardirqs_on(); 436 instrumentation_end(); 437 } else { 438 /* 439 * IRQ flags state is correct already. Just tell RCU if it 440 * was not watching on entry. 441 */ 442 if (state.exit_rcu) 443 ct_irq_exit(); 444 } 445 } 446 447 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) 448 { 449 irqentry_state_t irq_state; 450 451 irq_state.lockdep = lockdep_hardirqs_enabled(); 452 453 __nmi_enter(); 454 lockdep_hardirqs_off(CALLER_ADDR0); 455 lockdep_hardirq_enter(); 456 ct_nmi_enter(); 457 458 instrumentation_begin(); 459 kmsan_unpoison_entry_regs(regs); 460 trace_hardirqs_off_finish(); 461 ftrace_nmi_enter(); 462 instrumentation_end(); 463 464 return irq_state; 465 } 466 467 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state) 468 { 469 instrumentation_begin(); 470 ftrace_nmi_exit(); 471 if (irq_state.lockdep) { 472 trace_hardirqs_on_prepare(); 473 lockdep_hardirqs_on_prepare(); 474 } 475 instrumentation_end(); 476 477 ct_nmi_exit(); 478 lockdep_hardirq_exit(); 479 if (irq_state.lockdep) 480 lockdep_hardirqs_on(CALLER_ADDR0); 481 __nmi_exit(); 482 } 483