1/* 2 * linux/arch/x86_64/entry.S 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs 6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz> 7 * 8 * entry.S contains the system-call and fault low-level handling routines. 9 * 10 * Some of this is documented in Documentation/x86/entry_64.txt 11 * 12 * A note on terminology: 13 * - iret frame: Architecture defined interrupt frame from SS to RIP 14 * at the top of the kernel process stack. 15 * 16 * Some macro usage: 17 * - ENTRY/END: Define functions in the symbol table. 18 * - TRACE_IRQ_*: Trace hardirq state for lock debugging. 19 * - idtentry: Define exception entry points. 20 */ 21#include <linux/linkage.h> 22#include <asm/segment.h> 23#include <asm/cache.h> 24#include <asm/errno.h> 25#include "calling.h" 26#include <asm/asm-offsets.h> 27#include <asm/msr.h> 28#include <asm/unistd.h> 29#include <asm/thread_info.h> 30#include <asm/hw_irq.h> 31#include <asm/page_types.h> 32#include <asm/irqflags.h> 33#include <asm/paravirt.h> 34#include <asm/percpu.h> 35#include <asm/asm.h> 36#include <asm/smap.h> 37#include <asm/pgtable_types.h> 38#include <asm/export.h> 39#include <asm/frame.h> 40#include <linux/err.h> 41 42.code64 43.section .entry.text, "ax" 44 45#ifdef CONFIG_PARAVIRT 46ENTRY(native_usergs_sysret64) 47 UNWIND_HINT_EMPTY 48 swapgs 49 sysretq 50END(native_usergs_sysret64) 51#endif /* CONFIG_PARAVIRT */ 52 53.macro TRACE_IRQS_IRETQ 54#ifdef CONFIG_TRACE_IRQFLAGS 55 bt $9, EFLAGS(%rsp) /* interrupts off? */ 56 jnc 1f 57 TRACE_IRQS_ON 581: 59#endif 60.endm 61 62/* 63 * When dynamic function tracer is enabled it will add a breakpoint 64 * to all locations that it is about to modify, sync CPUs, update 65 * all the code, sync CPUs, then remove the breakpoints. In this time 66 * if lockdep is enabled, it might jump back into the debug handler 67 * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF). 68 * 69 * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to 70 * make sure the stack pointer does not get reset back to the top 71 * of the debug stack, and instead just reuses the current stack. 72 */ 73#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS) 74 75.macro TRACE_IRQS_OFF_DEBUG 76 call debug_stack_set_zero 77 TRACE_IRQS_OFF 78 call debug_stack_reset 79.endm 80 81.macro TRACE_IRQS_ON_DEBUG 82 call debug_stack_set_zero 83 TRACE_IRQS_ON 84 call debug_stack_reset 85.endm 86 87.macro TRACE_IRQS_IRETQ_DEBUG 88 bt $9, EFLAGS(%rsp) /* interrupts off? */ 89 jnc 1f 90 TRACE_IRQS_ON_DEBUG 911: 92.endm 93 94#else 95# define TRACE_IRQS_OFF_DEBUG TRACE_IRQS_OFF 96# define TRACE_IRQS_ON_DEBUG TRACE_IRQS_ON 97# define TRACE_IRQS_IRETQ_DEBUG TRACE_IRQS_IRETQ 98#endif 99 100/* 101 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers. 102 * 103 * This is the only entry point used for 64-bit system calls. The 104 * hardware interface is reasonably well designed and the register to 105 * argument mapping Linux uses fits well with the registers that are 106 * available when SYSCALL is used. 107 * 108 * SYSCALL instructions can be found inlined in libc implementations as 109 * well as some other programs and libraries. There are also a handful 110 * of SYSCALL instructions in the vDSO used, for example, as a 111 * clock_gettimeofday fallback. 112 * 113 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11, 114 * then loads new ss, cs, and rip from previously programmed MSRs. 115 * rflags gets masked by a value from another MSR (so CLD and CLAC 116 * are not needed). SYSCALL does not save anything on the stack 117 * and does not change rsp. 118 * 119 * Registers on entry: 120 * rax system call number 121 * rcx return address 122 * r11 saved rflags (note: r11 is callee-clobbered register in C ABI) 123 * rdi arg0 124 * rsi arg1 125 * rdx arg2 126 * r10 arg3 (needs to be moved to rcx to conform to C ABI) 127 * r8 arg4 128 * r9 arg5 129 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI) 130 * 131 * Only called from user space. 132 * 133 * When user can change pt_regs->foo always force IRET. That is because 134 * it deals with uncanonical addresses better. SYSRET has trouble 135 * with them due to bugs in both AMD and Intel CPUs. 136 */ 137 138ENTRY(entry_SYSCALL_64) 139 UNWIND_HINT_EMPTY 140 /* 141 * Interrupts are off on entry. 142 * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON, 143 * it is too small to ever cause noticeable irq latency. 144 */ 145 146 swapgs 147 movq %rsp, PER_CPU_VAR(rsp_scratch) 148 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 149 150 TRACE_IRQS_OFF 151 152 /* Construct struct pt_regs on stack */ 153 pushq $__USER_DS /* pt_regs->ss */ 154 pushq PER_CPU_VAR(rsp_scratch) /* pt_regs->sp */ 155 pushq %r11 /* pt_regs->flags */ 156 pushq $__USER_CS /* pt_regs->cs */ 157 pushq %rcx /* pt_regs->ip */ 158GLOBAL(entry_SYSCALL_64_after_hwframe) 159 pushq %rax /* pt_regs->orig_ax */ 160 pushq %rdi /* pt_regs->di */ 161 pushq %rsi /* pt_regs->si */ 162 pushq %rdx /* pt_regs->dx */ 163 pushq %rcx /* pt_regs->cx */ 164 pushq $-ENOSYS /* pt_regs->ax */ 165 pushq %r8 /* pt_regs->r8 */ 166 pushq %r9 /* pt_regs->r9 */ 167 pushq %r10 /* pt_regs->r10 */ 168 pushq %r11 /* pt_regs->r11 */ 169 sub $(6*8), %rsp /* pt_regs->bp, bx, r12-15 not saved */ 170 UNWIND_HINT_REGS extra=0 171 172 /* 173 * If we need to do entry work or if we guess we'll need to do 174 * exit work, go straight to the slow path. 175 */ 176 movq PER_CPU_VAR(current_task), %r11 177 testl $_TIF_WORK_SYSCALL_ENTRY|_TIF_ALLWORK_MASK, TASK_TI_flags(%r11) 178 jnz entry_SYSCALL64_slow_path 179 180entry_SYSCALL_64_fastpath: 181 /* 182 * Easy case: enable interrupts and issue the syscall. If the syscall 183 * needs pt_regs, we'll call a stub that disables interrupts again 184 * and jumps to the slow path. 185 */ 186 TRACE_IRQS_ON 187 ENABLE_INTERRUPTS(CLBR_NONE) 188#if __SYSCALL_MASK == ~0 189 cmpq $__NR_syscall_max, %rax 190#else 191 andl $__SYSCALL_MASK, %eax 192 cmpl $__NR_syscall_max, %eax 193#endif 194 ja 1f /* return -ENOSYS (already in pt_regs->ax) */ 195 movq %r10, %rcx 196 197 /* 198 * This call instruction is handled specially in stub_ptregs_64. 199 * It might end up jumping to the slow path. If it jumps, RAX 200 * and all argument registers are clobbered. 201 */ 202 call *sys_call_table(, %rax, 8) 203.Lentry_SYSCALL_64_after_fastpath_call: 204 205 movq %rax, RAX(%rsp) 2061: 207 208 /* 209 * If we get here, then we know that pt_regs is clean for SYSRET64. 210 * If we see that no exit work is required (which we are required 211 * to check with IRQs off), then we can go straight to SYSRET64. 212 */ 213 DISABLE_INTERRUPTS(CLBR_ANY) 214 TRACE_IRQS_OFF 215 movq PER_CPU_VAR(current_task), %r11 216 testl $_TIF_ALLWORK_MASK, TASK_TI_flags(%r11) 217 jnz 1f 218 219 LOCKDEP_SYS_EXIT 220 TRACE_IRQS_ON /* user mode is traced as IRQs on */ 221 movq RIP(%rsp), %rcx 222 movq EFLAGS(%rsp), %r11 223 RESTORE_C_REGS_EXCEPT_RCX_R11 224 movq RSP(%rsp), %rsp 225 UNWIND_HINT_EMPTY 226 USERGS_SYSRET64 227 2281: 229 /* 230 * The fast path looked good when we started, but something changed 231 * along the way and we need to switch to the slow path. Calling 232 * raise(3) will trigger this, for example. IRQs are off. 233 */ 234 TRACE_IRQS_ON 235 ENABLE_INTERRUPTS(CLBR_ANY) 236 SAVE_EXTRA_REGS 237 movq %rsp, %rdi 238 call syscall_return_slowpath /* returns with IRQs disabled */ 239 jmp return_from_SYSCALL_64 240 241entry_SYSCALL64_slow_path: 242 /* IRQs are off. */ 243 SAVE_EXTRA_REGS 244 movq %rsp, %rdi 245 call do_syscall_64 /* returns with IRQs disabled */ 246 247return_from_SYSCALL_64: 248 RESTORE_EXTRA_REGS 249 TRACE_IRQS_IRETQ /* we're about to change IF */ 250 251 /* 252 * Try to use SYSRET instead of IRET if we're returning to 253 * a completely clean 64-bit userspace context. 254 */ 255 movq RCX(%rsp), %rcx 256 movq RIP(%rsp), %r11 257 cmpq %rcx, %r11 /* RCX == RIP */ 258 jne opportunistic_sysret_failed 259 260 /* 261 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP 262 * in kernel space. This essentially lets the user take over 263 * the kernel, since userspace controls RSP. 264 * 265 * If width of "canonical tail" ever becomes variable, this will need 266 * to be updated to remain correct on both old and new CPUs. 267 * 268 * Change top bits to match most significant bit (47th or 56th bit 269 * depending on paging mode) in the address. 270 */ 271 shl $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx 272 sar $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx 273 274 /* If this changed %rcx, it was not canonical */ 275 cmpq %rcx, %r11 276 jne opportunistic_sysret_failed 277 278 cmpq $__USER_CS, CS(%rsp) /* CS must match SYSRET */ 279 jne opportunistic_sysret_failed 280 281 movq R11(%rsp), %r11 282 cmpq %r11, EFLAGS(%rsp) /* R11 == RFLAGS */ 283 jne opportunistic_sysret_failed 284 285 /* 286 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot 287 * restore RF properly. If the slowpath sets it for whatever reason, we 288 * need to restore it correctly. 289 * 290 * SYSRET can restore TF, but unlike IRET, restoring TF results in a 291 * trap from userspace immediately after SYSRET. This would cause an 292 * infinite loop whenever #DB happens with register state that satisfies 293 * the opportunistic SYSRET conditions. For example, single-stepping 294 * this user code: 295 * 296 * movq $stuck_here, %rcx 297 * pushfq 298 * popq %r11 299 * stuck_here: 300 * 301 * would never get past 'stuck_here'. 302 */ 303 testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11 304 jnz opportunistic_sysret_failed 305 306 /* nothing to check for RSP */ 307 308 cmpq $__USER_DS, SS(%rsp) /* SS must match SYSRET */ 309 jne opportunistic_sysret_failed 310 311 /* 312 * We win! This label is here just for ease of understanding 313 * perf profiles. Nothing jumps here. 314 */ 315syscall_return_via_sysret: 316 /* rcx and r11 are already restored (see code above) */ 317 RESTORE_C_REGS_EXCEPT_RCX_R11 318 movq RSP(%rsp), %rsp 319 UNWIND_HINT_EMPTY 320 USERGS_SYSRET64 321 322opportunistic_sysret_failed: 323 SWAPGS 324 jmp restore_c_regs_and_iret 325END(entry_SYSCALL_64) 326 327ENTRY(stub_ptregs_64) 328 /* 329 * Syscalls marked as needing ptregs land here. 330 * If we are on the fast path, we need to save the extra regs, 331 * which we achieve by trying again on the slow path. If we are on 332 * the slow path, the extra regs are already saved. 333 * 334 * RAX stores a pointer to the C function implementing the syscall. 335 * IRQs are on. 336 */ 337 cmpq $.Lentry_SYSCALL_64_after_fastpath_call, (%rsp) 338 jne 1f 339 340 /* 341 * Called from fast path -- disable IRQs again, pop return address 342 * and jump to slow path 343 */ 344 DISABLE_INTERRUPTS(CLBR_ANY) 345 TRACE_IRQS_OFF 346 popq %rax 347 UNWIND_HINT_REGS extra=0 348 jmp entry_SYSCALL64_slow_path 349 3501: 351 jmp *%rax /* Called from C */ 352END(stub_ptregs_64) 353 354.macro ptregs_stub func 355ENTRY(ptregs_\func) 356 UNWIND_HINT_FUNC 357 leaq \func(%rip), %rax 358 jmp stub_ptregs_64 359END(ptregs_\func) 360.endm 361 362/* Instantiate ptregs_stub for each ptregs-using syscall */ 363#define __SYSCALL_64_QUAL_(sym) 364#define __SYSCALL_64_QUAL_ptregs(sym) ptregs_stub sym 365#define __SYSCALL_64(nr, sym, qual) __SYSCALL_64_QUAL_##qual(sym) 366#include <asm/syscalls_64.h> 367 368/* 369 * %rdi: prev task 370 * %rsi: next task 371 */ 372ENTRY(__switch_to_asm) 373 UNWIND_HINT_FUNC 374 /* 375 * Save callee-saved registers 376 * This must match the order in inactive_task_frame 377 */ 378 pushq %rbp 379 pushq %rbx 380 pushq %r12 381 pushq %r13 382 pushq %r14 383 pushq %r15 384 385 /* switch stack */ 386 movq %rsp, TASK_threadsp(%rdi) 387 movq TASK_threadsp(%rsi), %rsp 388 389#ifdef CONFIG_CC_STACKPROTECTOR 390 movq TASK_stack_canary(%rsi), %rbx 391 movq %rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset 392#endif 393 394 /* restore callee-saved registers */ 395 popq %r15 396 popq %r14 397 popq %r13 398 popq %r12 399 popq %rbx 400 popq %rbp 401 402 jmp __switch_to 403END(__switch_to_asm) 404 405/* 406 * A newly forked process directly context switches into this address. 407 * 408 * rax: prev task we switched from 409 * rbx: kernel thread func (NULL for user thread) 410 * r12: kernel thread arg 411 */ 412ENTRY(ret_from_fork) 413 UNWIND_HINT_EMPTY 414 movq %rax, %rdi 415 call schedule_tail /* rdi: 'prev' task parameter */ 416 417 testq %rbx, %rbx /* from kernel_thread? */ 418 jnz 1f /* kernel threads are uncommon */ 419 4202: 421 UNWIND_HINT_REGS 422 movq %rsp, %rdi 423 call syscall_return_slowpath /* returns with IRQs disabled */ 424 TRACE_IRQS_ON /* user mode is traced as IRQS on */ 425 SWAPGS 426 jmp restore_regs_and_iret 427 4281: 429 /* kernel thread */ 430 movq %r12, %rdi 431 call *%rbx 432 /* 433 * A kernel thread is allowed to return here after successfully 434 * calling do_execve(). Exit to userspace to complete the execve() 435 * syscall. 436 */ 437 movq $0, RAX(%rsp) 438 jmp 2b 439END(ret_from_fork) 440 441/* 442 * Build the entry stubs with some assembler magic. 443 * We pack 1 stub into every 8-byte block. 444 */ 445 .align 8 446ENTRY(irq_entries_start) 447 vector=FIRST_EXTERNAL_VECTOR 448 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 449 UNWIND_HINT_IRET_REGS 450 pushq $(~vector+0x80) /* Note: always in signed byte range */ 451 jmp common_interrupt 452 .align 8 453 vector=vector+1 454 .endr 455END(irq_entries_start) 456 457.macro DEBUG_ENTRY_ASSERT_IRQS_OFF 458#ifdef CONFIG_DEBUG_ENTRY 459 pushfq 460 testl $X86_EFLAGS_IF, (%rsp) 461 jz .Lokay_\@ 462 ud2 463.Lokay_\@: 464 addq $8, %rsp 465#endif 466.endm 467 468/* 469 * Enters the IRQ stack if we're not already using it. NMI-safe. Clobbers 470 * flags and puts old RSP into old_rsp, and leaves all other GPRs alone. 471 * Requires kernel GSBASE. 472 * 473 * The invariant is that, if irq_count != -1, then the IRQ stack is in use. 474 */ 475.macro ENTER_IRQ_STACK regs=1 old_rsp 476 DEBUG_ENTRY_ASSERT_IRQS_OFF 477 movq %rsp, \old_rsp 478 479 .if \regs 480 UNWIND_HINT_REGS base=\old_rsp 481 .endif 482 483 incl PER_CPU_VAR(irq_count) 484 jnz .Lirq_stack_push_old_rsp_\@ 485 486 /* 487 * Right now, if we just incremented irq_count to zero, we've 488 * claimed the IRQ stack but we haven't switched to it yet. 489 * 490 * If anything is added that can interrupt us here without using IST, 491 * it must be *extremely* careful to limit its stack usage. This 492 * could include kprobes and a hypothetical future IST-less #DB 493 * handler. 494 * 495 * The OOPS unwinder relies on the word at the top of the IRQ 496 * stack linking back to the previous RSP for the entire time we're 497 * on the IRQ stack. For this to work reliably, we need to write 498 * it before we actually move ourselves to the IRQ stack. 499 */ 500 501 movq \old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8) 502 movq PER_CPU_VAR(irq_stack_ptr), %rsp 503 504#ifdef CONFIG_DEBUG_ENTRY 505 /* 506 * If the first movq above becomes wrong due to IRQ stack layout 507 * changes, the only way we'll notice is if we try to unwind right 508 * here. Assert that we set up the stack right to catch this type 509 * of bug quickly. 510 */ 511 cmpq -8(%rsp), \old_rsp 512 je .Lirq_stack_okay\@ 513 ud2 514 .Lirq_stack_okay\@: 515#endif 516 517.Lirq_stack_push_old_rsp_\@: 518 pushq \old_rsp 519 520 .if \regs 521 UNWIND_HINT_REGS indirect=1 522 .endif 523.endm 524 525/* 526 * Undoes ENTER_IRQ_STACK. 527 */ 528.macro LEAVE_IRQ_STACK regs=1 529 DEBUG_ENTRY_ASSERT_IRQS_OFF 530 /* We need to be off the IRQ stack before decrementing irq_count. */ 531 popq %rsp 532 533 .if \regs 534 UNWIND_HINT_REGS 535 .endif 536 537 /* 538 * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming 539 * the irq stack but we're not on it. 540 */ 541 542 decl PER_CPU_VAR(irq_count) 543.endm 544 545/* 546 * Interrupt entry/exit. 547 * 548 * Interrupt entry points save only callee clobbered registers in fast path. 549 * 550 * Entry runs with interrupts off. 551 */ 552 553/* 0(%rsp): ~(interrupt number) */ 554 .macro interrupt func 555 cld 556 ALLOC_PT_GPREGS_ON_STACK 557 SAVE_C_REGS 558 SAVE_EXTRA_REGS 559 ENCODE_FRAME_POINTER 560 561 testb $3, CS(%rsp) 562 jz 1f 563 564 /* 565 * IRQ from user mode. Switch to kernel gsbase and inform context 566 * tracking that we're in kernel mode. 567 */ 568 SWAPGS 569 570 /* 571 * We need to tell lockdep that IRQs are off. We can't do this until 572 * we fix gsbase, and we should do it before enter_from_user_mode 573 * (which can take locks). Since TRACE_IRQS_OFF idempotent, 574 * the simplest way to handle it is to just call it twice if 575 * we enter from user mode. There's no reason to optimize this since 576 * TRACE_IRQS_OFF is a no-op if lockdep is off. 577 */ 578 TRACE_IRQS_OFF 579 580 CALL_enter_from_user_mode 581 5821: 583 ENTER_IRQ_STACK old_rsp=%rdi 584 /* We entered an interrupt context - irqs are off: */ 585 TRACE_IRQS_OFF 586 587 call \func /* rdi points to pt_regs */ 588 .endm 589 590 /* 591 * The interrupt stubs push (~vector+0x80) onto the stack and 592 * then jump to common_interrupt. 593 */ 594 .p2align CONFIG_X86_L1_CACHE_SHIFT 595common_interrupt: 596 ASM_CLAC 597 addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */ 598 interrupt do_IRQ 599 /* 0(%rsp): old RSP */ 600ret_from_intr: 601 DISABLE_INTERRUPTS(CLBR_ANY) 602 TRACE_IRQS_OFF 603 604 LEAVE_IRQ_STACK 605 606 testb $3, CS(%rsp) 607 jz retint_kernel 608 609 /* Interrupt came from user space */ 610GLOBAL(retint_user) 611 mov %rsp,%rdi 612 call prepare_exit_to_usermode 613 TRACE_IRQS_IRETQ 614 SWAPGS 615 jmp restore_regs_and_iret 616 617/* Returning to kernel space */ 618retint_kernel: 619#ifdef CONFIG_PREEMPT 620 /* Interrupts are off */ 621 /* Check if we need preemption */ 622 bt $9, EFLAGS(%rsp) /* were interrupts off? */ 623 jnc 1f 6240: cmpl $0, PER_CPU_VAR(__preempt_count) 625 jnz 1f 626 call preempt_schedule_irq 627 jmp 0b 6281: 629#endif 630 /* 631 * The iretq could re-enable interrupts: 632 */ 633 TRACE_IRQS_IRETQ 634 635/* 636 * At this label, code paths which return to kernel and to user, 637 * which come from interrupts/exception and from syscalls, merge. 638 */ 639GLOBAL(restore_regs_and_iret) 640 RESTORE_EXTRA_REGS 641restore_c_regs_and_iret: 642 RESTORE_C_REGS 643 REMOVE_PT_GPREGS_FROM_STACK 8 644 INTERRUPT_RETURN 645 646ENTRY(native_iret) 647 UNWIND_HINT_IRET_REGS 648 /* 649 * Are we returning to a stack segment from the LDT? Note: in 650 * 64-bit mode SS:RSP on the exception stack is always valid. 651 */ 652#ifdef CONFIG_X86_ESPFIX64 653 testb $4, (SS-RIP)(%rsp) 654 jnz native_irq_return_ldt 655#endif 656 657.global native_irq_return_iret 658native_irq_return_iret: 659 /* 660 * This may fault. Non-paranoid faults on return to userspace are 661 * handled by fixup_bad_iret. These include #SS, #GP, and #NP. 662 * Double-faults due to espfix64 are handled in do_double_fault. 663 * Other faults here are fatal. 664 */ 665 iretq 666 667#ifdef CONFIG_X86_ESPFIX64 668native_irq_return_ldt: 669 /* 670 * We are running with user GSBASE. All GPRs contain their user 671 * values. We have a percpu ESPFIX stack that is eight slots 672 * long (see ESPFIX_STACK_SIZE). espfix_waddr points to the bottom 673 * of the ESPFIX stack. 674 * 675 * We clobber RAX and RDI in this code. We stash RDI on the 676 * normal stack and RAX on the ESPFIX stack. 677 * 678 * The ESPFIX stack layout we set up looks like this: 679 * 680 * --- top of ESPFIX stack --- 681 * SS 682 * RSP 683 * RFLAGS 684 * CS 685 * RIP <-- RSP points here when we're done 686 * RAX <-- espfix_waddr points here 687 * --- bottom of ESPFIX stack --- 688 */ 689 690 pushq %rdi /* Stash user RDI */ 691 SWAPGS 692 movq PER_CPU_VAR(espfix_waddr), %rdi 693 movq %rax, (0*8)(%rdi) /* user RAX */ 694 movq (1*8)(%rsp), %rax /* user RIP */ 695 movq %rax, (1*8)(%rdi) 696 movq (2*8)(%rsp), %rax /* user CS */ 697 movq %rax, (2*8)(%rdi) 698 movq (3*8)(%rsp), %rax /* user RFLAGS */ 699 movq %rax, (3*8)(%rdi) 700 movq (5*8)(%rsp), %rax /* user SS */ 701 movq %rax, (5*8)(%rdi) 702 movq (4*8)(%rsp), %rax /* user RSP */ 703 movq %rax, (4*8)(%rdi) 704 /* Now RAX == RSP. */ 705 706 andl $0xffff0000, %eax /* RAX = (RSP & 0xffff0000) */ 707 popq %rdi /* Restore user RDI */ 708 709 /* 710 * espfix_stack[31:16] == 0. The page tables are set up such that 711 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of 712 * espfix_waddr for any X. That is, there are 65536 RO aliases of 713 * the same page. Set up RSP so that RSP[31:16] contains the 714 * respective 16 bits of the /userspace/ RSP and RSP nonetheless 715 * still points to an RO alias of the ESPFIX stack. 716 */ 717 orq PER_CPU_VAR(espfix_stack), %rax 718 SWAPGS 719 movq %rax, %rsp 720 UNWIND_HINT_IRET_REGS offset=8 721 722 /* 723 * At this point, we cannot write to the stack any more, but we can 724 * still read. 725 */ 726 popq %rax /* Restore user RAX */ 727 728 /* 729 * RSP now points to an ordinary IRET frame, except that the page 730 * is read-only and RSP[31:16] are preloaded with the userspace 731 * values. We can now IRET back to userspace. 732 */ 733 jmp native_irq_return_iret 734#endif 735END(common_interrupt) 736 737/* 738 * APIC interrupts. 739 */ 740.macro apicinterrupt3 num sym do_sym 741ENTRY(\sym) 742 UNWIND_HINT_IRET_REGS 743 ASM_CLAC 744 pushq $~(\num) 745.Lcommon_\sym: 746 interrupt \do_sym 747 jmp ret_from_intr 748END(\sym) 749.endm 750 751/* Make sure APIC interrupt handlers end up in the irqentry section: */ 752#define PUSH_SECTION_IRQENTRY .pushsection .irqentry.text, "ax" 753#define POP_SECTION_IRQENTRY .popsection 754 755.macro apicinterrupt num sym do_sym 756PUSH_SECTION_IRQENTRY 757apicinterrupt3 \num \sym \do_sym 758POP_SECTION_IRQENTRY 759.endm 760 761#ifdef CONFIG_SMP 762apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt 763apicinterrupt3 REBOOT_VECTOR reboot_interrupt smp_reboot_interrupt 764#endif 765 766#ifdef CONFIG_X86_UV 767apicinterrupt3 UV_BAU_MESSAGE uv_bau_message_intr1 uv_bau_message_interrupt 768#endif 769 770apicinterrupt LOCAL_TIMER_VECTOR apic_timer_interrupt smp_apic_timer_interrupt 771apicinterrupt X86_PLATFORM_IPI_VECTOR x86_platform_ipi smp_x86_platform_ipi 772 773#ifdef CONFIG_HAVE_KVM 774apicinterrupt3 POSTED_INTR_VECTOR kvm_posted_intr_ipi smp_kvm_posted_intr_ipi 775apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR kvm_posted_intr_wakeup_ipi smp_kvm_posted_intr_wakeup_ipi 776apicinterrupt3 POSTED_INTR_NESTED_VECTOR kvm_posted_intr_nested_ipi smp_kvm_posted_intr_nested_ipi 777#endif 778 779#ifdef CONFIG_X86_MCE_THRESHOLD 780apicinterrupt THRESHOLD_APIC_VECTOR threshold_interrupt smp_threshold_interrupt 781#endif 782 783#ifdef CONFIG_X86_MCE_AMD 784apicinterrupt DEFERRED_ERROR_VECTOR deferred_error_interrupt smp_deferred_error_interrupt 785#endif 786 787#ifdef CONFIG_X86_THERMAL_VECTOR 788apicinterrupt THERMAL_APIC_VECTOR thermal_interrupt smp_thermal_interrupt 789#endif 790 791#ifdef CONFIG_SMP 792apicinterrupt CALL_FUNCTION_SINGLE_VECTOR call_function_single_interrupt smp_call_function_single_interrupt 793apicinterrupt CALL_FUNCTION_VECTOR call_function_interrupt smp_call_function_interrupt 794apicinterrupt RESCHEDULE_VECTOR reschedule_interrupt smp_reschedule_interrupt 795#endif 796 797apicinterrupt ERROR_APIC_VECTOR error_interrupt smp_error_interrupt 798apicinterrupt SPURIOUS_APIC_VECTOR spurious_interrupt smp_spurious_interrupt 799 800#ifdef CONFIG_IRQ_WORK 801apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt 802#endif 803 804/* 805 * Exception entry points. 806 */ 807#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss) + (TSS_ist + ((x) - 1) * 8) 808 809.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 810ENTRY(\sym) 811 UNWIND_HINT_IRET_REGS offset=8 812 813 /* Sanity check */ 814 .if \shift_ist != -1 && \paranoid == 0 815 .error "using shift_ist requires paranoid=1" 816 .endif 817 818 ASM_CLAC 819 820 .ifeq \has_error_code 821 pushq $-1 /* ORIG_RAX: no syscall to restart */ 822 .endif 823 824 ALLOC_PT_GPREGS_ON_STACK 825 826 .if \paranoid 827 .if \paranoid == 1 828 testb $3, CS(%rsp) /* If coming from userspace, switch stacks */ 829 jnz 1f 830 .endif 831 call paranoid_entry 832 .else 833 call error_entry 834 .endif 835 UNWIND_HINT_REGS 836 /* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */ 837 838 .if \paranoid 839 .if \shift_ist != -1 840 TRACE_IRQS_OFF_DEBUG /* reload IDT in case of recursion */ 841 .else 842 TRACE_IRQS_OFF 843 .endif 844 .endif 845 846 movq %rsp, %rdi /* pt_regs pointer */ 847 848 .if \has_error_code 849 movq ORIG_RAX(%rsp), %rsi /* get error code */ 850 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ 851 .else 852 xorl %esi, %esi /* no error code */ 853 .endif 854 855 .if \shift_ist != -1 856 subq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist) 857 .endif 858 859 call \do_sym 860 861 .if \shift_ist != -1 862 addq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist) 863 .endif 864 865 /* these procedures expect "no swapgs" flag in ebx */ 866 .if \paranoid 867 jmp paranoid_exit 868 .else 869 jmp error_exit 870 .endif 871 872 .if \paranoid == 1 873 /* 874 * Paranoid entry from userspace. Switch stacks and treat it 875 * as a normal entry. This means that paranoid handlers 876 * run in real process context if user_mode(regs). 877 */ 8781: 879 call error_entry 880 881 882 movq %rsp, %rdi /* pt_regs pointer */ 883 call sync_regs 884 movq %rax, %rsp /* switch stack */ 885 886 movq %rsp, %rdi /* pt_regs pointer */ 887 888 .if \has_error_code 889 movq ORIG_RAX(%rsp), %rsi /* get error code */ 890 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ 891 .else 892 xorl %esi, %esi /* no error code */ 893 .endif 894 895 call \do_sym 896 897 jmp error_exit /* %ebx: no swapgs flag */ 898 .endif 899END(\sym) 900.endm 901 902idtentry divide_error do_divide_error has_error_code=0 903idtentry overflow do_overflow has_error_code=0 904idtentry bounds do_bounds has_error_code=0 905idtentry invalid_op do_invalid_op has_error_code=0 906idtentry device_not_available do_device_not_available has_error_code=0 907idtentry double_fault do_double_fault has_error_code=1 paranoid=2 908idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0 909idtentry invalid_TSS do_invalid_TSS has_error_code=1 910idtentry segment_not_present do_segment_not_present has_error_code=1 911idtentry spurious_interrupt_bug do_spurious_interrupt_bug has_error_code=0 912idtentry coprocessor_error do_coprocessor_error has_error_code=0 913idtentry alignment_check do_alignment_check has_error_code=1 914idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0 915 916 917 /* 918 * Reload gs selector with exception handling 919 * edi: new selector 920 */ 921ENTRY(native_load_gs_index) 922 FRAME_BEGIN 923 pushfq 924 DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI) 925 SWAPGS 926.Lgs_change: 927 movl %edi, %gs 9282: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE 929 SWAPGS 930 popfq 931 FRAME_END 932 ret 933ENDPROC(native_load_gs_index) 934EXPORT_SYMBOL(native_load_gs_index) 935 936 _ASM_EXTABLE(.Lgs_change, bad_gs) 937 .section .fixup, "ax" 938 /* running with kernelgs */ 939bad_gs: 940 SWAPGS /* switch back to user gs */ 941.macro ZAP_GS 942 /* This can't be a string because the preprocessor needs to see it. */ 943 movl $__USER_DS, %eax 944 movl %eax, %gs 945.endm 946 ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG 947 xorl %eax, %eax 948 movl %eax, %gs 949 jmp 2b 950 .previous 951 952/* Call softirq on interrupt stack. Interrupts are off. */ 953ENTRY(do_softirq_own_stack) 954 pushq %rbp 955 mov %rsp, %rbp 956 ENTER_IRQ_STACK regs=0 old_rsp=%r11 957 call __do_softirq 958 LEAVE_IRQ_STACK regs=0 959 leaveq 960 ret 961ENDPROC(do_softirq_own_stack) 962 963#ifdef CONFIG_XEN 964idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0 965 966/* 967 * A note on the "critical region" in our callback handler. 968 * We want to avoid stacking callback handlers due to events occurring 969 * during handling of the last event. To do this, we keep events disabled 970 * until we've done all processing. HOWEVER, we must enable events before 971 * popping the stack frame (can't be done atomically) and so it would still 972 * be possible to get enough handler activations to overflow the stack. 973 * Although unlikely, bugs of that kind are hard to track down, so we'd 974 * like to avoid the possibility. 975 * So, on entry to the handler we detect whether we interrupted an 976 * existing activation in its critical region -- if so, we pop the current 977 * activation and restart the handler using the previous one. 978 */ 979ENTRY(xen_do_hypervisor_callback) /* do_hypervisor_callback(struct *pt_regs) */ 980 981/* 982 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will 983 * see the correct pointer to the pt_regs 984 */ 985 UNWIND_HINT_FUNC 986 movq %rdi, %rsp /* we don't return, adjust the stack frame */ 987 UNWIND_HINT_REGS 988 989 ENTER_IRQ_STACK old_rsp=%r10 990 call xen_evtchn_do_upcall 991 LEAVE_IRQ_STACK 992 993#ifndef CONFIG_PREEMPT 994 call xen_maybe_preempt_hcall 995#endif 996 jmp error_exit 997END(xen_do_hypervisor_callback) 998 999/* 1000 * Hypervisor uses this for application faults while it executes. 1001 * We get here for two reasons: 1002 * 1. Fault while reloading DS, ES, FS or GS 1003 * 2. Fault while executing IRET 1004 * Category 1 we do not need to fix up as Xen has already reloaded all segment 1005 * registers that could be reloaded and zeroed the others. 1006 * Category 2 we fix up by killing the current process. We cannot use the 1007 * normal Linux return path in this case because if we use the IRET hypercall 1008 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1009 * We distinguish between categories by comparing each saved segment register 1010 * with its current contents: any discrepancy means we in category 1. 1011 */ 1012ENTRY(xen_failsafe_callback) 1013 UNWIND_HINT_EMPTY 1014 movl %ds, %ecx 1015 cmpw %cx, 0x10(%rsp) 1016 jne 1f 1017 movl %es, %ecx 1018 cmpw %cx, 0x18(%rsp) 1019 jne 1f 1020 movl %fs, %ecx 1021 cmpw %cx, 0x20(%rsp) 1022 jne 1f 1023 movl %gs, %ecx 1024 cmpw %cx, 0x28(%rsp) 1025 jne 1f 1026 /* All segments match their saved values => Category 2 (Bad IRET). */ 1027 movq (%rsp), %rcx 1028 movq 8(%rsp), %r11 1029 addq $0x30, %rsp 1030 pushq $0 /* RIP */ 1031 UNWIND_HINT_IRET_REGS offset=8 1032 jmp general_protection 10331: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */ 1034 movq (%rsp), %rcx 1035 movq 8(%rsp), %r11 1036 addq $0x30, %rsp 1037 UNWIND_HINT_IRET_REGS 1038 pushq $-1 /* orig_ax = -1 => not a system call */ 1039 ALLOC_PT_GPREGS_ON_STACK 1040 SAVE_C_REGS 1041 SAVE_EXTRA_REGS 1042 ENCODE_FRAME_POINTER 1043 jmp error_exit 1044END(xen_failsafe_callback) 1045 1046apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1047 xen_hvm_callback_vector xen_evtchn_do_upcall 1048 1049#endif /* CONFIG_XEN */ 1050 1051#if IS_ENABLED(CONFIG_HYPERV) 1052apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1053 hyperv_callback_vector hyperv_vector_handler 1054#endif /* CONFIG_HYPERV */ 1055 1056idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK 1057idtentry int3 do_int3 has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK 1058idtentry stack_segment do_stack_segment has_error_code=1 1059 1060#ifdef CONFIG_XEN 1061idtentry xendebug do_debug has_error_code=0 1062idtentry xenint3 do_int3 has_error_code=0 1063#endif 1064 1065idtentry general_protection do_general_protection has_error_code=1 1066idtentry page_fault do_page_fault has_error_code=1 1067 1068#ifdef CONFIG_KVM_GUEST 1069idtentry async_page_fault do_async_page_fault has_error_code=1 1070#endif 1071 1072#ifdef CONFIG_X86_MCE 1073idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vector(%rip) 1074#endif 1075 1076/* 1077 * Save all registers in pt_regs, and switch gs if needed. 1078 * Use slow, but surefire "are we in kernel?" check. 1079 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise 1080 */ 1081ENTRY(paranoid_entry) 1082 UNWIND_HINT_FUNC 1083 cld 1084 SAVE_C_REGS 8 1085 SAVE_EXTRA_REGS 8 1086 ENCODE_FRAME_POINTER 8 1087 movl $1, %ebx 1088 movl $MSR_GS_BASE, %ecx 1089 rdmsr 1090 testl %edx, %edx 1091 js 1f /* negative -> in kernel */ 1092 SWAPGS 1093 xorl %ebx, %ebx 10941: ret 1095END(paranoid_entry) 1096 1097/* 1098 * "Paranoid" exit path from exception stack. This is invoked 1099 * only on return from non-NMI IST interrupts that came 1100 * from kernel space. 1101 * 1102 * We may be returning to very strange contexts (e.g. very early 1103 * in syscall entry), so checking for preemption here would 1104 * be complicated. Fortunately, we there's no good reason 1105 * to try to handle preemption here. 1106 * 1107 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) 1108 */ 1109ENTRY(paranoid_exit) 1110 UNWIND_HINT_REGS 1111 DISABLE_INTERRUPTS(CLBR_ANY) 1112 TRACE_IRQS_OFF_DEBUG 1113 testl %ebx, %ebx /* swapgs needed? */ 1114 jnz paranoid_exit_no_swapgs 1115 TRACE_IRQS_IRETQ 1116 SWAPGS_UNSAFE_STACK 1117 jmp paranoid_exit_restore 1118paranoid_exit_no_swapgs: 1119 TRACE_IRQS_IRETQ_DEBUG 1120paranoid_exit_restore: 1121 RESTORE_EXTRA_REGS 1122 RESTORE_C_REGS 1123 REMOVE_PT_GPREGS_FROM_STACK 8 1124 INTERRUPT_RETURN 1125END(paranoid_exit) 1126 1127/* 1128 * Save all registers in pt_regs, and switch gs if needed. 1129 * Return: EBX=0: came from user mode; EBX=1: otherwise 1130 */ 1131ENTRY(error_entry) 1132 UNWIND_HINT_FUNC 1133 cld 1134 SAVE_C_REGS 8 1135 SAVE_EXTRA_REGS 8 1136 ENCODE_FRAME_POINTER 8 1137 xorl %ebx, %ebx 1138 testb $3, CS+8(%rsp) 1139 jz .Lerror_kernelspace 1140 1141 /* 1142 * We entered from user mode or we're pretending to have entered 1143 * from user mode due to an IRET fault. 1144 */ 1145 SWAPGS 1146 1147.Lerror_entry_from_usermode_after_swapgs: 1148 /* 1149 * We need to tell lockdep that IRQs are off. We can't do this until 1150 * we fix gsbase, and we should do it before enter_from_user_mode 1151 * (which can take locks). 1152 */ 1153 TRACE_IRQS_OFF 1154 CALL_enter_from_user_mode 1155 ret 1156 1157.Lerror_entry_done: 1158 TRACE_IRQS_OFF 1159 ret 1160 1161 /* 1162 * There are two places in the kernel that can potentially fault with 1163 * usergs. Handle them here. B stepping K8s sometimes report a 1164 * truncated RIP for IRET exceptions returning to compat mode. Check 1165 * for these here too. 1166 */ 1167.Lerror_kernelspace: 1168 incl %ebx 1169 leaq native_irq_return_iret(%rip), %rcx 1170 cmpq %rcx, RIP+8(%rsp) 1171 je .Lerror_bad_iret 1172 movl %ecx, %eax /* zero extend */ 1173 cmpq %rax, RIP+8(%rsp) 1174 je .Lbstep_iret 1175 cmpq $.Lgs_change, RIP+8(%rsp) 1176 jne .Lerror_entry_done 1177 1178 /* 1179 * hack: .Lgs_change can fail with user gsbase. If this happens, fix up 1180 * gsbase and proceed. We'll fix up the exception and land in 1181 * .Lgs_change's error handler with kernel gsbase. 1182 */ 1183 SWAPGS 1184 jmp .Lerror_entry_done 1185 1186.Lbstep_iret: 1187 /* Fix truncated RIP */ 1188 movq %rcx, RIP+8(%rsp) 1189 /* fall through */ 1190 1191.Lerror_bad_iret: 1192 /* 1193 * We came from an IRET to user mode, so we have user gsbase. 1194 * Switch to kernel gsbase: 1195 */ 1196 SWAPGS 1197 1198 /* 1199 * Pretend that the exception came from user mode: set up pt_regs 1200 * as if we faulted immediately after IRET and clear EBX so that 1201 * error_exit knows that we will be returning to user mode. 1202 */ 1203 mov %rsp, %rdi 1204 call fixup_bad_iret 1205 mov %rax, %rsp 1206 decl %ebx 1207 jmp .Lerror_entry_from_usermode_after_swapgs 1208END(error_entry) 1209 1210 1211/* 1212 * On entry, EBX is a "return to kernel mode" flag: 1213 * 1: already in kernel mode, don't need SWAPGS 1214 * 0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode 1215 */ 1216ENTRY(error_exit) 1217 UNWIND_HINT_REGS 1218 DISABLE_INTERRUPTS(CLBR_ANY) 1219 TRACE_IRQS_OFF 1220 testl %ebx, %ebx 1221 jnz retint_kernel 1222 jmp retint_user 1223END(error_exit) 1224 1225/* Runs on exception stack */ 1226/* XXX: broken on Xen PV */ 1227ENTRY(nmi) 1228 UNWIND_HINT_IRET_REGS 1229 /* 1230 * We allow breakpoints in NMIs. If a breakpoint occurs, then 1231 * the iretq it performs will take us out of NMI context. 1232 * This means that we can have nested NMIs where the next 1233 * NMI is using the top of the stack of the previous NMI. We 1234 * can't let it execute because the nested NMI will corrupt the 1235 * stack of the previous NMI. NMI handlers are not re-entrant 1236 * anyway. 1237 * 1238 * To handle this case we do the following: 1239 * Check the a special location on the stack that contains 1240 * a variable that is set when NMIs are executing. 1241 * The interrupted task's stack is also checked to see if it 1242 * is an NMI stack. 1243 * If the variable is not set and the stack is not the NMI 1244 * stack then: 1245 * o Set the special variable on the stack 1246 * o Copy the interrupt frame into an "outermost" location on the 1247 * stack 1248 * o Copy the interrupt frame into an "iret" location on the stack 1249 * o Continue processing the NMI 1250 * If the variable is set or the previous stack is the NMI stack: 1251 * o Modify the "iret" location to jump to the repeat_nmi 1252 * o return back to the first NMI 1253 * 1254 * Now on exit of the first NMI, we first clear the stack variable 1255 * The NMI stack will tell any nested NMIs at that point that it is 1256 * nested. Then we pop the stack normally with iret, and if there was 1257 * a nested NMI that updated the copy interrupt stack frame, a 1258 * jump will be made to the repeat_nmi code that will handle the second 1259 * NMI. 1260 * 1261 * However, espfix prevents us from directly returning to userspace 1262 * with a single IRET instruction. Similarly, IRET to user mode 1263 * can fault. We therefore handle NMIs from user space like 1264 * other IST entries. 1265 */ 1266 1267 ASM_CLAC 1268 1269 /* Use %rdx as our temp variable throughout */ 1270 pushq %rdx 1271 1272 testb $3, CS-RIP+8(%rsp) 1273 jz .Lnmi_from_kernel 1274 1275 /* 1276 * NMI from user mode. We need to run on the thread stack, but we 1277 * can't go through the normal entry paths: NMIs are masked, and 1278 * we don't want to enable interrupts, because then we'll end 1279 * up in an awkward situation in which IRQs are on but NMIs 1280 * are off. 1281 * 1282 * We also must not push anything to the stack before switching 1283 * stacks lest we corrupt the "NMI executing" variable. 1284 */ 1285 1286 SWAPGS_UNSAFE_STACK 1287 cld 1288 movq %rsp, %rdx 1289 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 1290 UNWIND_HINT_IRET_REGS base=%rdx offset=8 1291 pushq 5*8(%rdx) /* pt_regs->ss */ 1292 pushq 4*8(%rdx) /* pt_regs->rsp */ 1293 pushq 3*8(%rdx) /* pt_regs->flags */ 1294 pushq 2*8(%rdx) /* pt_regs->cs */ 1295 pushq 1*8(%rdx) /* pt_regs->rip */ 1296 UNWIND_HINT_IRET_REGS 1297 pushq $-1 /* pt_regs->orig_ax */ 1298 pushq %rdi /* pt_regs->di */ 1299 pushq %rsi /* pt_regs->si */ 1300 pushq (%rdx) /* pt_regs->dx */ 1301 pushq %rcx /* pt_regs->cx */ 1302 pushq %rax /* pt_regs->ax */ 1303 pushq %r8 /* pt_regs->r8 */ 1304 pushq %r9 /* pt_regs->r9 */ 1305 pushq %r10 /* pt_regs->r10 */ 1306 pushq %r11 /* pt_regs->r11 */ 1307 pushq %rbx /* pt_regs->rbx */ 1308 pushq %rbp /* pt_regs->rbp */ 1309 pushq %r12 /* pt_regs->r12 */ 1310 pushq %r13 /* pt_regs->r13 */ 1311 pushq %r14 /* pt_regs->r14 */ 1312 pushq %r15 /* pt_regs->r15 */ 1313 UNWIND_HINT_REGS 1314 ENCODE_FRAME_POINTER 1315 1316 /* 1317 * At this point we no longer need to worry about stack damage 1318 * due to nesting -- we're on the normal thread stack and we're 1319 * done with the NMI stack. 1320 */ 1321 1322 movq %rsp, %rdi 1323 movq $-1, %rsi 1324 call do_nmi 1325 1326 /* 1327 * Return back to user mode. We must *not* do the normal exit 1328 * work, because we don't want to enable interrupts. 1329 */ 1330 SWAPGS 1331 jmp restore_regs_and_iret 1332 1333.Lnmi_from_kernel: 1334 /* 1335 * Here's what our stack frame will look like: 1336 * +---------------------------------------------------------+ 1337 * | original SS | 1338 * | original Return RSP | 1339 * | original RFLAGS | 1340 * | original CS | 1341 * | original RIP | 1342 * +---------------------------------------------------------+ 1343 * | temp storage for rdx | 1344 * +---------------------------------------------------------+ 1345 * | "NMI executing" variable | 1346 * +---------------------------------------------------------+ 1347 * | iret SS } Copied from "outermost" frame | 1348 * | iret Return RSP } on each loop iteration; overwritten | 1349 * | iret RFLAGS } by a nested NMI to force another | 1350 * | iret CS } iteration if needed. | 1351 * | iret RIP } | 1352 * +---------------------------------------------------------+ 1353 * | outermost SS } initialized in first_nmi; | 1354 * | outermost Return RSP } will not be changed before | 1355 * | outermost RFLAGS } NMI processing is done. | 1356 * | outermost CS } Copied to "iret" frame on each | 1357 * | outermost RIP } iteration. | 1358 * +---------------------------------------------------------+ 1359 * | pt_regs | 1360 * +---------------------------------------------------------+ 1361 * 1362 * The "original" frame is used by hardware. Before re-enabling 1363 * NMIs, we need to be done with it, and we need to leave enough 1364 * space for the asm code here. 1365 * 1366 * We return by executing IRET while RSP points to the "iret" frame. 1367 * That will either return for real or it will loop back into NMI 1368 * processing. 1369 * 1370 * The "outermost" frame is copied to the "iret" frame on each 1371 * iteration of the loop, so each iteration starts with the "iret" 1372 * frame pointing to the final return target. 1373 */ 1374 1375 /* 1376 * Determine whether we're a nested NMI. 1377 * 1378 * If we interrupted kernel code between repeat_nmi and 1379 * end_repeat_nmi, then we are a nested NMI. We must not 1380 * modify the "iret" frame because it's being written by 1381 * the outer NMI. That's okay; the outer NMI handler is 1382 * about to about to call do_nmi anyway, so we can just 1383 * resume the outer NMI. 1384 */ 1385 1386 movq $repeat_nmi, %rdx 1387 cmpq 8(%rsp), %rdx 1388 ja 1f 1389 movq $end_repeat_nmi, %rdx 1390 cmpq 8(%rsp), %rdx 1391 ja nested_nmi_out 13921: 1393 1394 /* 1395 * Now check "NMI executing". If it's set, then we're nested. 1396 * This will not detect if we interrupted an outer NMI just 1397 * before IRET. 1398 */ 1399 cmpl $1, -8(%rsp) 1400 je nested_nmi 1401 1402 /* 1403 * Now test if the previous stack was an NMI stack. This covers 1404 * the case where we interrupt an outer NMI after it clears 1405 * "NMI executing" but before IRET. We need to be careful, though: 1406 * there is one case in which RSP could point to the NMI stack 1407 * despite there being no NMI active: naughty userspace controls 1408 * RSP at the very beginning of the SYSCALL targets. We can 1409 * pull a fast one on naughty userspace, though: we program 1410 * SYSCALL to mask DF, so userspace cannot cause DF to be set 1411 * if it controls the kernel's RSP. We set DF before we clear 1412 * "NMI executing". 1413 */ 1414 lea 6*8(%rsp), %rdx 1415 /* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */ 1416 cmpq %rdx, 4*8(%rsp) 1417 /* If the stack pointer is above the NMI stack, this is a normal NMI */ 1418 ja first_nmi 1419 1420 subq $EXCEPTION_STKSZ, %rdx 1421 cmpq %rdx, 4*8(%rsp) 1422 /* If it is below the NMI stack, it is a normal NMI */ 1423 jb first_nmi 1424 1425 /* Ah, it is within the NMI stack. */ 1426 1427 testb $(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp) 1428 jz first_nmi /* RSP was user controlled. */ 1429 1430 /* This is a nested NMI. */ 1431 1432nested_nmi: 1433 /* 1434 * Modify the "iret" frame to point to repeat_nmi, forcing another 1435 * iteration of NMI handling. 1436 */ 1437 subq $8, %rsp 1438 leaq -10*8(%rsp), %rdx 1439 pushq $__KERNEL_DS 1440 pushq %rdx 1441 pushfq 1442 pushq $__KERNEL_CS 1443 pushq $repeat_nmi 1444 1445 /* Put stack back */ 1446 addq $(6*8), %rsp 1447 1448nested_nmi_out: 1449 popq %rdx 1450 1451 /* We are returning to kernel mode, so this cannot result in a fault. */ 1452 INTERRUPT_RETURN 1453 1454first_nmi: 1455 /* Restore rdx. */ 1456 movq (%rsp), %rdx 1457 1458 /* Make room for "NMI executing". */ 1459 pushq $0 1460 1461 /* Leave room for the "iret" frame */ 1462 subq $(5*8), %rsp 1463 1464 /* Copy the "original" frame to the "outermost" frame */ 1465 .rept 5 1466 pushq 11*8(%rsp) 1467 .endr 1468 UNWIND_HINT_IRET_REGS 1469 1470 /* Everything up to here is safe from nested NMIs */ 1471 1472#ifdef CONFIG_DEBUG_ENTRY 1473 /* 1474 * For ease of testing, unmask NMIs right away. Disabled by 1475 * default because IRET is very expensive. 1476 */ 1477 pushq $0 /* SS */ 1478 pushq %rsp /* RSP (minus 8 because of the previous push) */ 1479 addq $8, (%rsp) /* Fix up RSP */ 1480 pushfq /* RFLAGS */ 1481 pushq $__KERNEL_CS /* CS */ 1482 pushq $1f /* RIP */ 1483 INTERRUPT_RETURN /* continues at repeat_nmi below */ 1484 UNWIND_HINT_IRET_REGS 14851: 1486#endif 1487 1488repeat_nmi: 1489 /* 1490 * If there was a nested NMI, the first NMI's iret will return 1491 * here. But NMIs are still enabled and we can take another 1492 * nested NMI. The nested NMI checks the interrupted RIP to see 1493 * if it is between repeat_nmi and end_repeat_nmi, and if so 1494 * it will just return, as we are about to repeat an NMI anyway. 1495 * This makes it safe to copy to the stack frame that a nested 1496 * NMI will update. 1497 * 1498 * RSP is pointing to "outermost RIP". gsbase is unknown, but, if 1499 * we're repeating an NMI, gsbase has the same value that it had on 1500 * the first iteration. paranoid_entry will load the kernel 1501 * gsbase if needed before we call do_nmi. "NMI executing" 1502 * is zero. 1503 */ 1504 movq $1, 10*8(%rsp) /* Set "NMI executing". */ 1505 1506 /* 1507 * Copy the "outermost" frame to the "iret" frame. NMIs that nest 1508 * here must not modify the "iret" frame while we're writing to 1509 * it or it will end up containing garbage. 1510 */ 1511 addq $(10*8), %rsp 1512 .rept 5 1513 pushq -6*8(%rsp) 1514 .endr 1515 subq $(5*8), %rsp 1516end_repeat_nmi: 1517 1518 /* 1519 * Everything below this point can be preempted by a nested NMI. 1520 * If this happens, then the inner NMI will change the "iret" 1521 * frame to point back to repeat_nmi. 1522 */ 1523 pushq $-1 /* ORIG_RAX: no syscall to restart */ 1524 ALLOC_PT_GPREGS_ON_STACK 1525 1526 /* 1527 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit 1528 * as we should not be calling schedule in NMI context. 1529 * Even with normal interrupts enabled. An NMI should not be 1530 * setting NEED_RESCHED or anything that normal interrupts and 1531 * exceptions might do. 1532 */ 1533 call paranoid_entry 1534 UNWIND_HINT_REGS 1535 1536 /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ 1537 movq %rsp, %rdi 1538 movq $-1, %rsi 1539 call do_nmi 1540 1541 testl %ebx, %ebx /* swapgs needed? */ 1542 jnz nmi_restore 1543nmi_swapgs: 1544 SWAPGS_UNSAFE_STACK 1545nmi_restore: 1546 RESTORE_EXTRA_REGS 1547 RESTORE_C_REGS 1548 1549 /* Point RSP at the "iret" frame. */ 1550 REMOVE_PT_GPREGS_FROM_STACK 6*8 1551 1552 /* 1553 * Clear "NMI executing". Set DF first so that we can easily 1554 * distinguish the remaining code between here and IRET from 1555 * the SYSCALL entry and exit paths. On a native kernel, we 1556 * could just inspect RIP, but, on paravirt kernels, 1557 * INTERRUPT_RETURN can translate into a jump into a 1558 * hypercall page. 1559 */ 1560 std 1561 movq $0, 5*8(%rsp) /* clear "NMI executing" */ 1562 1563 /* 1564 * INTERRUPT_RETURN reads the "iret" frame and exits the NMI 1565 * stack in a single instruction. We are returning to kernel 1566 * mode, so this cannot result in a fault. 1567 */ 1568 INTERRUPT_RETURN 1569END(nmi) 1570 1571ENTRY(ignore_sysret) 1572 UNWIND_HINT_EMPTY 1573 mov $-ENOSYS, %eax 1574 sysret 1575END(ignore_sysret) 1576 1577ENTRY(rewind_stack_do_exit) 1578 UNWIND_HINT_FUNC 1579 /* Prevent any naive code from trying to unwind to our caller. */ 1580 xorl %ebp, %ebp 1581 1582 movq PER_CPU_VAR(cpu_current_top_of_stack), %rax 1583 leaq -PTREGS_SIZE(%rax), %rsp 1584 UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE 1585 1586 call do_exit 1587END(rewind_stack_do_exit) 1588