1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/context_tracking.h> 4 #include <linux/entry-common.h> 5 #include <linux/resume_user_mode.h> 6 #include <linux/highmem.h> 7 #include <linux/jump_label.h> 8 #include <linux/kmsan.h> 9 #include <linux/livepatch.h> 10 #include <linux/audit.h> 11 #include <linux/tick.h> 12 13 #include "common.h" 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/syscalls.h> 17 18 /* See comment for enter_from_user_mode() in entry-common.h */ 19 static __always_inline void __enter_from_user_mode(struct pt_regs *regs) 20 { 21 arch_enter_from_user_mode(regs); 22 lockdep_hardirqs_off(CALLER_ADDR0); 23 24 CT_WARN_ON(__ct_state() != CONTEXT_USER); 25 user_exit_irqoff(); 26 27 instrumentation_begin(); 28 kmsan_unpoison_entry_regs(regs); 29 trace_hardirqs_off_finish(); 30 instrumentation_end(); 31 } 32 33 void noinstr enter_from_user_mode(struct pt_regs *regs) 34 { 35 __enter_from_user_mode(regs); 36 } 37 38 static inline void syscall_enter_audit(struct pt_regs *regs, long syscall) 39 { 40 if (unlikely(audit_context())) { 41 unsigned long args[6]; 42 43 syscall_get_arguments(current, regs, args); 44 audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]); 45 } 46 } 47 48 static long syscall_trace_enter(struct pt_regs *regs, long syscall, 49 unsigned long work) 50 { 51 long ret = 0; 52 53 /* 54 * Handle Syscall User Dispatch. This must comes first, since 55 * the ABI here can be something that doesn't make sense for 56 * other syscall_work features. 57 */ 58 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { 59 if (syscall_user_dispatch(regs)) 60 return -1L; 61 } 62 63 /* Handle ptrace */ 64 if (work & (SYSCALL_WORK_SYSCALL_TRACE | SYSCALL_WORK_SYSCALL_EMU)) { 65 ret = ptrace_report_syscall_entry(regs); 66 if (ret || (work & SYSCALL_WORK_SYSCALL_EMU)) 67 return -1L; 68 } 69 70 /* Do seccomp after ptrace, to catch any tracer changes. */ 71 if (work & SYSCALL_WORK_SECCOMP) { 72 ret = __secure_computing(NULL); 73 if (ret == -1L) 74 return ret; 75 } 76 77 /* Either of the above might have changed the syscall number */ 78 syscall = syscall_get_nr(current, regs); 79 80 if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT)) { 81 trace_sys_enter(regs, syscall); 82 /* 83 * Probes or BPF hooks in the tracepoint may have changed the 84 * system call number as well. 85 */ 86 syscall = syscall_get_nr(current, regs); 87 } 88 89 syscall_enter_audit(regs, syscall); 90 91 return ret ? : syscall; 92 } 93 94 static __always_inline long 95 __syscall_enter_from_user_work(struct pt_regs *regs, long syscall) 96 { 97 unsigned long work = READ_ONCE(current_thread_info()->syscall_work); 98 99 if (work & SYSCALL_WORK_ENTER) 100 syscall = syscall_trace_enter(regs, syscall, work); 101 102 return syscall; 103 } 104 105 long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall) 106 { 107 return __syscall_enter_from_user_work(regs, syscall); 108 } 109 110 noinstr long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall) 111 { 112 long ret; 113 114 __enter_from_user_mode(regs); 115 116 instrumentation_begin(); 117 local_irq_enable(); 118 ret = __syscall_enter_from_user_work(regs, syscall); 119 instrumentation_end(); 120 121 return ret; 122 } 123 124 noinstr void syscall_enter_from_user_mode_prepare(struct pt_regs *regs) 125 { 126 __enter_from_user_mode(regs); 127 instrumentation_begin(); 128 local_irq_enable(); 129 instrumentation_end(); 130 } 131 132 /* See comment for exit_to_user_mode() in entry-common.h */ 133 static __always_inline void __exit_to_user_mode(void) 134 { 135 instrumentation_begin(); 136 trace_hardirqs_on_prepare(); 137 lockdep_hardirqs_on_prepare(); 138 instrumentation_end(); 139 140 user_enter_irqoff(); 141 arch_exit_to_user_mode(); 142 lockdep_hardirqs_on(CALLER_ADDR0); 143 } 144 145 void noinstr exit_to_user_mode(void) 146 { 147 __exit_to_user_mode(); 148 } 149 150 /* Workaround to allow gradual conversion of architecture code */ 151 void __weak arch_do_signal_or_restart(struct pt_regs *regs) { } 152 153 static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, 154 unsigned long ti_work) 155 { 156 /* 157 * Before returning to user space ensure that all pending work 158 * items have been completed. 159 */ 160 while (ti_work & EXIT_TO_USER_MODE_WORK) { 161 162 local_irq_enable_exit_to_user(ti_work); 163 164 if (ti_work & _TIF_NEED_RESCHED) 165 schedule(); 166 167 if (ti_work & _TIF_UPROBE) 168 uprobe_notify_resume(regs); 169 170 if (ti_work & _TIF_PATCH_PENDING) 171 klp_update_patch_state(current); 172 173 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) 174 arch_do_signal_or_restart(regs); 175 176 if (ti_work & _TIF_NOTIFY_RESUME) 177 resume_user_mode_work(regs); 178 179 /* Architecture specific TIF work */ 180 arch_exit_to_user_mode_work(regs, ti_work); 181 182 /* 183 * Disable interrupts and reevaluate the work flags as they 184 * might have changed while interrupts and preemption was 185 * enabled above. 186 */ 187 local_irq_disable_exit_to_user(); 188 189 /* Check if any of the above work has queued a deferred wakeup */ 190 tick_nohz_user_enter_prepare(); 191 192 ti_work = read_thread_flags(); 193 } 194 195 /* Return the latest work state for arch_exit_to_user_mode() */ 196 return ti_work; 197 } 198 199 static void exit_to_user_mode_prepare(struct pt_regs *regs) 200 { 201 unsigned long ti_work; 202 203 lockdep_assert_irqs_disabled(); 204 205 /* Flush pending rcuog wakeup before the last need_resched() check */ 206 tick_nohz_user_enter_prepare(); 207 208 ti_work = read_thread_flags(); 209 if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) 210 ti_work = exit_to_user_mode_loop(regs, ti_work); 211 212 arch_exit_to_user_mode_prepare(regs, ti_work); 213 214 /* Ensure that kernel state is sane for a return to userspace */ 215 kmap_assert_nomap(); 216 lockdep_assert_irqs_disabled(); 217 lockdep_sys_exit(); 218 } 219 220 /* 221 * If SYSCALL_EMU is set, then the only reason to report is when 222 * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall 223 * instruction has been already reported in syscall_enter_from_user_mode(). 224 */ 225 static inline bool report_single_step(unsigned long work) 226 { 227 if (work & SYSCALL_WORK_SYSCALL_EMU) 228 return false; 229 230 return work & SYSCALL_WORK_SYSCALL_EXIT_TRAP; 231 } 232 233 static void syscall_exit_work(struct pt_regs *regs, unsigned long work) 234 { 235 bool step; 236 237 /* 238 * If the syscall was rolled back due to syscall user dispatching, 239 * then the tracers below are not invoked for the same reason as 240 * the entry side was not invoked in syscall_trace_enter(): The ABI 241 * of these syscalls is unknown. 242 */ 243 if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { 244 if (unlikely(current->syscall_dispatch.on_dispatch)) { 245 current->syscall_dispatch.on_dispatch = false; 246 return; 247 } 248 } 249 250 audit_syscall_exit(regs); 251 252 if (work & SYSCALL_WORK_SYSCALL_TRACEPOINT) 253 trace_sys_exit(regs, syscall_get_return_value(current, regs)); 254 255 step = report_single_step(work); 256 if (step || work & SYSCALL_WORK_SYSCALL_TRACE) 257 ptrace_report_syscall_exit(regs, step); 258 } 259 260 /* 261 * Syscall specific exit to user mode preparation. Runs with interrupts 262 * enabled. 263 */ 264 static void syscall_exit_to_user_mode_prepare(struct pt_regs *regs) 265 { 266 unsigned long work = READ_ONCE(current_thread_info()->syscall_work); 267 unsigned long nr = syscall_get_nr(current, regs); 268 269 CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 270 271 if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { 272 if (WARN(irqs_disabled(), "syscall %lu left IRQs disabled", nr)) 273 local_irq_enable(); 274 } 275 276 rseq_syscall(regs); 277 278 /* 279 * Do one-time syscall specific work. If these work items are 280 * enabled, we want to run them exactly once per syscall exit with 281 * interrupts enabled. 282 */ 283 if (unlikely(work & SYSCALL_WORK_EXIT)) 284 syscall_exit_work(regs, work); 285 } 286 287 static __always_inline void __syscall_exit_to_user_mode_work(struct pt_regs *regs) 288 { 289 syscall_exit_to_user_mode_prepare(regs); 290 local_irq_disable_exit_to_user(); 291 exit_to_user_mode_prepare(regs); 292 } 293 294 void syscall_exit_to_user_mode_work(struct pt_regs *regs) 295 { 296 __syscall_exit_to_user_mode_work(regs); 297 } 298 299 __visible noinstr void syscall_exit_to_user_mode(struct pt_regs *regs) 300 { 301 instrumentation_begin(); 302 __syscall_exit_to_user_mode_work(regs); 303 instrumentation_end(); 304 __exit_to_user_mode(); 305 } 306 307 noinstr void irqentry_enter_from_user_mode(struct pt_regs *regs) 308 { 309 __enter_from_user_mode(regs); 310 } 311 312 noinstr void irqentry_exit_to_user_mode(struct pt_regs *regs) 313 { 314 instrumentation_begin(); 315 exit_to_user_mode_prepare(regs); 316 instrumentation_end(); 317 __exit_to_user_mode(); 318 } 319 320 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) 321 { 322 irqentry_state_t ret = { 323 .exit_rcu = false, 324 }; 325 326 if (user_mode(regs)) { 327 irqentry_enter_from_user_mode(regs); 328 return ret; 329 } 330 331 /* 332 * If this entry hit the idle task invoke ct_irq_enter() whether 333 * RCU is watching or not. 334 * 335 * Interrupts can nest when the first interrupt invokes softirq 336 * processing on return which enables interrupts. 337 * 338 * Scheduler ticks in the idle task can mark quiescent state and 339 * terminate a grace period, if and only if the timer interrupt is 340 * not nested into another interrupt. 341 * 342 * Checking for rcu_is_watching() here would prevent the nesting 343 * interrupt to invoke ct_irq_enter(). If that nested interrupt is 344 * the tick then rcu_flavor_sched_clock_irq() would wrongfully 345 * assume that it is the first interrupt and eventually claim 346 * quiescent state and end grace periods prematurely. 347 * 348 * Unconditionally invoke ct_irq_enter() so RCU state stays 349 * consistent. 350 * 351 * TINY_RCU does not support EQS, so let the compiler eliminate 352 * this part when enabled. 353 */ 354 if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { 355 /* 356 * If RCU is not watching then the same careful 357 * sequence vs. lockdep and tracing is required 358 * as in irqentry_enter_from_user_mode(). 359 */ 360 lockdep_hardirqs_off(CALLER_ADDR0); 361 ct_irq_enter(); 362 instrumentation_begin(); 363 kmsan_unpoison_entry_regs(regs); 364 trace_hardirqs_off_finish(); 365 instrumentation_end(); 366 367 ret.exit_rcu = true; 368 return ret; 369 } 370 371 /* 372 * If RCU is watching then RCU only wants to check whether it needs 373 * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick() 374 * already contains a warning when RCU is not watching, so no point 375 * in having another one here. 376 */ 377 lockdep_hardirqs_off(CALLER_ADDR0); 378 instrumentation_begin(); 379 kmsan_unpoison_entry_regs(regs); 380 rcu_irq_enter_check_tick(); 381 trace_hardirqs_off_finish(); 382 instrumentation_end(); 383 384 return ret; 385 } 386 387 void raw_irqentry_exit_cond_resched(void) 388 { 389 if (!preempt_count()) { 390 /* Sanity check RCU and thread stack */ 391 rcu_irq_exit_check_preempt(); 392 if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) 393 WARN_ON_ONCE(!on_thread_stack()); 394 if (need_resched()) 395 preempt_schedule_irq(); 396 } 397 } 398 #ifdef CONFIG_PREEMPT_DYNAMIC 399 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 400 DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); 401 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 402 DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); 403 void dynamic_irqentry_exit_cond_resched(void) 404 { 405 if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) 406 return; 407 raw_irqentry_exit_cond_resched(); 408 } 409 #endif 410 #endif 411 412 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state) 413 { 414 lockdep_assert_irqs_disabled(); 415 416 /* Check whether this returns to user mode */ 417 if (user_mode(regs)) { 418 irqentry_exit_to_user_mode(regs); 419 } else if (!regs_irqs_disabled(regs)) { 420 /* 421 * If RCU was not watching on entry this needs to be done 422 * carefully and needs the same ordering of lockdep/tracing 423 * and RCU as the return to user mode path. 424 */ 425 if (state.exit_rcu) { 426 instrumentation_begin(); 427 /* Tell the tracer that IRET will enable interrupts */ 428 trace_hardirqs_on_prepare(); 429 lockdep_hardirqs_on_prepare(); 430 instrumentation_end(); 431 ct_irq_exit(); 432 lockdep_hardirqs_on(CALLER_ADDR0); 433 return; 434 } 435 436 instrumentation_begin(); 437 if (IS_ENABLED(CONFIG_PREEMPTION)) 438 irqentry_exit_cond_resched(); 439 440 /* Covers both tracing and lockdep */ 441 trace_hardirqs_on(); 442 instrumentation_end(); 443 } else { 444 /* 445 * IRQ flags state is correct already. Just tell RCU if it 446 * was not watching on entry. 447 */ 448 if (state.exit_rcu) 449 ct_irq_exit(); 450 } 451 } 452 453 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) 454 { 455 irqentry_state_t irq_state; 456 457 irq_state.lockdep = lockdep_hardirqs_enabled(); 458 459 __nmi_enter(); 460 lockdep_hardirqs_off(CALLER_ADDR0); 461 lockdep_hardirq_enter(); 462 ct_nmi_enter(); 463 464 instrumentation_begin(); 465 kmsan_unpoison_entry_regs(regs); 466 trace_hardirqs_off_finish(); 467 ftrace_nmi_enter(); 468 instrumentation_end(); 469 470 return irq_state; 471 } 472 473 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state) 474 { 475 instrumentation_begin(); 476 ftrace_nmi_exit(); 477 if (irq_state.lockdep) { 478 trace_hardirqs_on_prepare(); 479 lockdep_hardirqs_on_prepare(); 480 } 481 instrumentation_end(); 482 483 ct_nmi_exit(); 484 lockdep_hardirq_exit(); 485 if (irq_state.lockdep) 486 lockdep_hardirqs_on(CALLER_ADDR0); 487 __nmi_exit(); 488 } 489