1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 2905a36a2SIngo Molnar/* 3905a36a2SIngo Molnar * linux/arch/x86_64/entry.S 4905a36a2SIngo Molnar * 5905a36a2SIngo Molnar * Copyright (C) 1991, 1992 Linus Torvalds 6905a36a2SIngo Molnar * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs 7905a36a2SIngo Molnar * Copyright (C) 2000 Pavel Machek <pavel@suse.cz> 84d732138SIngo Molnar * 9905a36a2SIngo Molnar * entry.S contains the system-call and fault low-level handling routines. 10905a36a2SIngo Molnar * 11905a36a2SIngo Molnar * Some of this is documented in Documentation/x86/entry_64.txt 12905a36a2SIngo Molnar * 13905a36a2SIngo Molnar * A note on terminology: 14905a36a2SIngo Molnar * - iret frame: Architecture defined interrupt frame from SS to RIP 15905a36a2SIngo Molnar * at the top of the kernel process stack. 16905a36a2SIngo Molnar * 17905a36a2SIngo Molnar * Some macro usage: 184d732138SIngo Molnar * - ENTRY/END: Define functions in the symbol table. 194d732138SIngo Molnar * - TRACE_IRQ_*: Trace hardirq state for lock debugging. 204d732138SIngo Molnar * - idtentry: Define exception entry points. 21905a36a2SIngo Molnar */ 22905a36a2SIngo Molnar#include <linux/linkage.h> 23905a36a2SIngo Molnar#include <asm/segment.h> 24905a36a2SIngo Molnar#include <asm/cache.h> 25905a36a2SIngo Molnar#include <asm/errno.h> 26905a36a2SIngo Molnar#include <asm/asm-offsets.h> 27905a36a2SIngo Molnar#include <asm/msr.h> 28905a36a2SIngo Molnar#include <asm/unistd.h> 29905a36a2SIngo Molnar#include <asm/thread_info.h> 30905a36a2SIngo Molnar#include <asm/hw_irq.h> 31905a36a2SIngo Molnar#include <asm/page_types.h> 32905a36a2SIngo Molnar#include <asm/irqflags.h> 33905a36a2SIngo Molnar#include <asm/paravirt.h> 34905a36a2SIngo Molnar#include <asm/percpu.h> 35905a36a2SIngo Molnar#include <asm/asm.h> 36905a36a2SIngo Molnar#include <asm/smap.h> 37905a36a2SIngo Molnar#include <asm/pgtable_types.h> 38784d5699SAl Viro#include <asm/export.h> 398c1f7558SJosh Poimboeuf#include <asm/frame.h> 402641f08bSDavid Woodhouse#include <asm/nospec-branch.h> 41905a36a2SIngo Molnar#include <linux/err.h> 42905a36a2SIngo Molnar 436fd166aaSPeter Zijlstra#include "calling.h" 446fd166aaSPeter Zijlstra 45905a36a2SIngo Molnar.code64 46905a36a2SIngo Molnar.section .entry.text, "ax" 47905a36a2SIngo Molnar 48905a36a2SIngo Molnar#ifdef CONFIG_PARAVIRT 49905a36a2SIngo MolnarENTRY(native_usergs_sysret64) 508c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 51905a36a2SIngo Molnar swapgs 52905a36a2SIngo Molnar sysretq 538c1f7558SJosh PoimboeufEND(native_usergs_sysret64) 54905a36a2SIngo Molnar#endif /* CONFIG_PARAVIRT */ 55905a36a2SIngo Molnar 56ca37e57bSAndy Lutomirski.macro TRACE_IRQS_FLAGS flags:req 57905a36a2SIngo Molnar#ifdef CONFIG_TRACE_IRQFLAGS 58a368d7fdSJan Beulich btl $9, \flags /* interrupts off? */ 59905a36a2SIngo Molnar jnc 1f 60905a36a2SIngo Molnar TRACE_IRQS_ON 61905a36a2SIngo Molnar1: 62905a36a2SIngo Molnar#endif 63905a36a2SIngo Molnar.endm 64905a36a2SIngo Molnar 65ca37e57bSAndy Lutomirski.macro TRACE_IRQS_IRETQ 66ca37e57bSAndy Lutomirski TRACE_IRQS_FLAGS EFLAGS(%rsp) 67ca37e57bSAndy Lutomirski.endm 68ca37e57bSAndy Lutomirski 69905a36a2SIngo Molnar/* 70905a36a2SIngo Molnar * When dynamic function tracer is enabled it will add a breakpoint 71905a36a2SIngo Molnar * to all locations that it is about to modify, sync CPUs, update 72905a36a2SIngo Molnar * all the code, sync CPUs, then remove the breakpoints. In this time 73905a36a2SIngo Molnar * if lockdep is enabled, it might jump back into the debug handler 74905a36a2SIngo Molnar * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF). 75905a36a2SIngo Molnar * 76905a36a2SIngo Molnar * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to 77905a36a2SIngo Molnar * make sure the stack pointer does not get reset back to the top 78905a36a2SIngo Molnar * of the debug stack, and instead just reuses the current stack. 79905a36a2SIngo Molnar */ 80905a36a2SIngo Molnar#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS) 81905a36a2SIngo Molnar 82905a36a2SIngo Molnar.macro TRACE_IRQS_OFF_DEBUG 83905a36a2SIngo Molnar call debug_stack_set_zero 84905a36a2SIngo Molnar TRACE_IRQS_OFF 85905a36a2SIngo Molnar call debug_stack_reset 86905a36a2SIngo Molnar.endm 87905a36a2SIngo Molnar 88905a36a2SIngo Molnar.macro TRACE_IRQS_ON_DEBUG 89905a36a2SIngo Molnar call debug_stack_set_zero 90905a36a2SIngo Molnar TRACE_IRQS_ON 91905a36a2SIngo Molnar call debug_stack_reset 92905a36a2SIngo Molnar.endm 93905a36a2SIngo Molnar 94905a36a2SIngo Molnar.macro TRACE_IRQS_IRETQ_DEBUG 95905a36a2SIngo Molnar bt $9, EFLAGS(%rsp) /* interrupts off? */ 96905a36a2SIngo Molnar jnc 1f 97905a36a2SIngo Molnar TRACE_IRQS_ON_DEBUG 98905a36a2SIngo Molnar1: 99905a36a2SIngo Molnar.endm 100905a36a2SIngo Molnar 101905a36a2SIngo Molnar#else 102905a36a2SIngo Molnar# define TRACE_IRQS_OFF_DEBUG TRACE_IRQS_OFF 103905a36a2SIngo Molnar# define TRACE_IRQS_ON_DEBUG TRACE_IRQS_ON 104905a36a2SIngo Molnar# define TRACE_IRQS_IRETQ_DEBUG TRACE_IRQS_IRETQ 105905a36a2SIngo Molnar#endif 106905a36a2SIngo Molnar 107905a36a2SIngo Molnar/* 1084d732138SIngo Molnar * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers. 109905a36a2SIngo Molnar * 110fda57b22SAndy Lutomirski * This is the only entry point used for 64-bit system calls. The 111fda57b22SAndy Lutomirski * hardware interface is reasonably well designed and the register to 112fda57b22SAndy Lutomirski * argument mapping Linux uses fits well with the registers that are 113fda57b22SAndy Lutomirski * available when SYSCALL is used. 114fda57b22SAndy Lutomirski * 115fda57b22SAndy Lutomirski * SYSCALL instructions can be found inlined in libc implementations as 116fda57b22SAndy Lutomirski * well as some other programs and libraries. There are also a handful 117fda57b22SAndy Lutomirski * of SYSCALL instructions in the vDSO used, for example, as a 118fda57b22SAndy Lutomirski * clock_gettimeofday fallback. 119fda57b22SAndy Lutomirski * 1204d732138SIngo Molnar * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11, 121905a36a2SIngo Molnar * then loads new ss, cs, and rip from previously programmed MSRs. 122905a36a2SIngo Molnar * rflags gets masked by a value from another MSR (so CLD and CLAC 123905a36a2SIngo Molnar * are not needed). SYSCALL does not save anything on the stack 124905a36a2SIngo Molnar * and does not change rsp. 125905a36a2SIngo Molnar * 126905a36a2SIngo Molnar * Registers on entry: 127905a36a2SIngo Molnar * rax system call number 128905a36a2SIngo Molnar * rcx return address 129905a36a2SIngo Molnar * r11 saved rflags (note: r11 is callee-clobbered register in C ABI) 130905a36a2SIngo Molnar * rdi arg0 131905a36a2SIngo Molnar * rsi arg1 132905a36a2SIngo Molnar * rdx arg2 133905a36a2SIngo Molnar * r10 arg3 (needs to be moved to rcx to conform to C ABI) 134905a36a2SIngo Molnar * r8 arg4 135905a36a2SIngo Molnar * r9 arg5 136905a36a2SIngo Molnar * (note: r12-r15, rbp, rbx are callee-preserved in C ABI) 137905a36a2SIngo Molnar * 138905a36a2SIngo Molnar * Only called from user space. 139905a36a2SIngo Molnar * 140905a36a2SIngo Molnar * When user can change pt_regs->foo always force IRET. That is because 141905a36a2SIngo Molnar * it deals with uncanonical addresses better. SYSRET has trouble 142905a36a2SIngo Molnar * with them due to bugs in both AMD and Intel CPUs. 143905a36a2SIngo Molnar */ 144905a36a2SIngo Molnar 1453386bc8aSAndy Lutomirski .pushsection .entry_trampoline, "ax" 1463386bc8aSAndy Lutomirski 1473386bc8aSAndy Lutomirski/* 1483386bc8aSAndy Lutomirski * The code in here gets remapped into cpu_entry_area's trampoline. This means 1493386bc8aSAndy Lutomirski * that the assembler and linker have the wrong idea as to where this code 1503386bc8aSAndy Lutomirski * lives (and, in fact, it's mapped more than once, so it's not even at a 1513386bc8aSAndy Lutomirski * fixed address). So we can't reference any symbols outside the entry 1523386bc8aSAndy Lutomirski * trampoline and expect it to work. 1533386bc8aSAndy Lutomirski * 1543386bc8aSAndy Lutomirski * Instead, we carefully abuse %rip-relative addressing. 1553386bc8aSAndy Lutomirski * _entry_trampoline(%rip) refers to the start of the remapped) entry 1563386bc8aSAndy Lutomirski * trampoline. We can thus find cpu_entry_area with this macro: 1573386bc8aSAndy Lutomirski */ 1583386bc8aSAndy Lutomirski 1593386bc8aSAndy Lutomirski#define CPU_ENTRY_AREA \ 1603386bc8aSAndy Lutomirski _entry_trampoline - CPU_ENTRY_AREA_entry_trampoline(%rip) 1613386bc8aSAndy Lutomirski 1623386bc8aSAndy Lutomirski/* The top word of the SYSENTER stack is hot and is usable as scratch space. */ 1634fe2d8b1SDave Hansen#define RSP_SCRATCH CPU_ENTRY_AREA_entry_stack + \ 1644fe2d8b1SDave Hansen SIZEOF_entry_stack - 8 + CPU_ENTRY_AREA 1653386bc8aSAndy Lutomirski 1663386bc8aSAndy LutomirskiENTRY(entry_SYSCALL_64_trampoline) 1673386bc8aSAndy Lutomirski UNWIND_HINT_EMPTY 1683386bc8aSAndy Lutomirski swapgs 1693386bc8aSAndy Lutomirski 1703386bc8aSAndy Lutomirski /* Stash the user RSP. */ 1713386bc8aSAndy Lutomirski movq %rsp, RSP_SCRATCH 1723386bc8aSAndy Lutomirski 1738a09317bSDave Hansen /* Note: using %rsp as a scratch reg. */ 1748a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp 1758a09317bSDave Hansen 1763386bc8aSAndy Lutomirski /* Load the top of the task stack into RSP */ 1773386bc8aSAndy Lutomirski movq CPU_ENTRY_AREA_tss + TSS_sp1 + CPU_ENTRY_AREA, %rsp 1783386bc8aSAndy Lutomirski 1793386bc8aSAndy Lutomirski /* Start building the simulated IRET frame. */ 1803386bc8aSAndy Lutomirski pushq $__USER_DS /* pt_regs->ss */ 1813386bc8aSAndy Lutomirski pushq RSP_SCRATCH /* pt_regs->sp */ 1823386bc8aSAndy Lutomirski pushq %r11 /* pt_regs->flags */ 1833386bc8aSAndy Lutomirski pushq $__USER_CS /* pt_regs->cs */ 1843386bc8aSAndy Lutomirski pushq %rcx /* pt_regs->ip */ 1853386bc8aSAndy Lutomirski 1863386bc8aSAndy Lutomirski /* 1873386bc8aSAndy Lutomirski * x86 lacks a near absolute jump, and we can't jump to the real 1883386bc8aSAndy Lutomirski * entry text with a relative jump. We could push the target 1893386bc8aSAndy Lutomirski * address and then use retq, but this destroys the pipeline on 1903386bc8aSAndy Lutomirski * many CPUs (wasting over 20 cycles on Sandy Bridge). Instead, 1913386bc8aSAndy Lutomirski * spill RDI and restore it in a second-stage trampoline. 1923386bc8aSAndy Lutomirski */ 1933386bc8aSAndy Lutomirski pushq %rdi 1943386bc8aSAndy Lutomirski movq $entry_SYSCALL_64_stage2, %rdi 1952641f08bSDavid Woodhouse JMP_NOSPEC %rdi 1963386bc8aSAndy LutomirskiEND(entry_SYSCALL_64_trampoline) 1973386bc8aSAndy Lutomirski 1983386bc8aSAndy Lutomirski .popsection 1993386bc8aSAndy Lutomirski 2003386bc8aSAndy LutomirskiENTRY(entry_SYSCALL_64_stage2) 2013386bc8aSAndy Lutomirski UNWIND_HINT_EMPTY 2023386bc8aSAndy Lutomirski popq %rdi 2033386bc8aSAndy Lutomirski jmp entry_SYSCALL_64_after_hwframe 2043386bc8aSAndy LutomirskiEND(entry_SYSCALL_64_stage2) 2053386bc8aSAndy Lutomirski 206b2502b41SIngo MolnarENTRY(entry_SYSCALL_64) 2078c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 208905a36a2SIngo Molnar /* 209905a36a2SIngo Molnar * Interrupts are off on entry. 210905a36a2SIngo Molnar * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON, 211905a36a2SIngo Molnar * it is too small to ever cause noticeable irq latency. 212905a36a2SIngo Molnar */ 213905a36a2SIngo Molnar 2148a9949bcSAndy Lutomirski swapgs 2158a09317bSDave Hansen /* 21614b1fcc6SNadav Amit * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it 2178a09317bSDave Hansen * is not required to switch CR3. 2188a09317bSDave Hansen */ 219905a36a2SIngo Molnar movq %rsp, PER_CPU_VAR(rsp_scratch) 220905a36a2SIngo Molnar movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 221905a36a2SIngo Molnar 222905a36a2SIngo Molnar /* Construct struct pt_regs on stack */ 223905a36a2SIngo Molnar pushq $__USER_DS /* pt_regs->ss */ 224905a36a2SIngo Molnar pushq PER_CPU_VAR(rsp_scratch) /* pt_regs->sp */ 225905a36a2SIngo Molnar pushq %r11 /* pt_regs->flags */ 226905a36a2SIngo Molnar pushq $__USER_CS /* pt_regs->cs */ 227905a36a2SIngo Molnar pushq %rcx /* pt_regs->ip */ 2288a9949bcSAndy LutomirskiGLOBAL(entry_SYSCALL_64_after_hwframe) 229905a36a2SIngo Molnar pushq %rax /* pt_regs->orig_ax */ 23030907fd1SDominik Brodowski 23130907fd1SDominik Brodowski PUSH_AND_CLEAR_REGS rax=$-ENOSYS 232905a36a2SIngo Molnar 233548c3050SAndy Lutomirski TRACE_IRQS_OFF 234548c3050SAndy Lutomirski 2351e423bffSAndy Lutomirski /* IRQs are off. */ 236dfe64506SLinus Torvalds movq %rax, %rdi 237dfe64506SLinus Torvalds movq %rsp, %rsi 2381e423bffSAndy Lutomirski call do_syscall_64 /* returns with IRQs disabled */ 2391e423bffSAndy Lutomirski 24029ea1b25SAndy Lutomirski TRACE_IRQS_IRETQ /* we're about to change IF */ 241905a36a2SIngo Molnar 242905a36a2SIngo Molnar /* 243905a36a2SIngo Molnar * Try to use SYSRET instead of IRET if we're returning to 2448a055d7fSAndy Lutomirski * a completely clean 64-bit userspace context. If we're not, 2458a055d7fSAndy Lutomirski * go to the slow exit path. 246905a36a2SIngo Molnar */ 247905a36a2SIngo Molnar movq RCX(%rsp), %rcx 248905a36a2SIngo Molnar movq RIP(%rsp), %r11 2498a055d7fSAndy Lutomirski 2508a055d7fSAndy Lutomirski cmpq %rcx, %r11 /* SYSRET requires RCX == RIP */ 2518a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 252905a36a2SIngo Molnar 253905a36a2SIngo Molnar /* 254905a36a2SIngo Molnar * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP 255905a36a2SIngo Molnar * in kernel space. This essentially lets the user take over 256905a36a2SIngo Molnar * the kernel, since userspace controls RSP. 257905a36a2SIngo Molnar * 258905a36a2SIngo Molnar * If width of "canonical tail" ever becomes variable, this will need 259905a36a2SIngo Molnar * to be updated to remain correct on both old and new CPUs. 260361b4b58SKirill A. Shutemov * 261cbe0317bSKirill A. Shutemov * Change top bits to match most significant bit (47th or 56th bit 262cbe0317bSKirill A. Shutemov * depending on paging mode) in the address. 263905a36a2SIngo Molnar */ 26409e61a77SKirill A. Shutemov#ifdef CONFIG_X86_5LEVEL 26539b95522SKirill A. Shutemov ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \ 26639b95522SKirill A. Shutemov "shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57 26709e61a77SKirill A. Shutemov#else 268905a36a2SIngo Molnar shl $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx 269905a36a2SIngo Molnar sar $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx 27009e61a77SKirill A. Shutemov#endif 2714d732138SIngo Molnar 272905a36a2SIngo Molnar /* If this changed %rcx, it was not canonical */ 273905a36a2SIngo Molnar cmpq %rcx, %r11 2748a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 275905a36a2SIngo Molnar 276905a36a2SIngo Molnar cmpq $__USER_CS, CS(%rsp) /* CS must match SYSRET */ 2778a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 278905a36a2SIngo Molnar 279905a36a2SIngo Molnar movq R11(%rsp), %r11 280905a36a2SIngo Molnar cmpq %r11, EFLAGS(%rsp) /* R11 == RFLAGS */ 2818a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 282905a36a2SIngo Molnar 283905a36a2SIngo Molnar /* 2843e035305SBorislav Petkov * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot 2853e035305SBorislav Petkov * restore RF properly. If the slowpath sets it for whatever reason, we 2863e035305SBorislav Petkov * need to restore it correctly. 2873e035305SBorislav Petkov * 2883e035305SBorislav Petkov * SYSRET can restore TF, but unlike IRET, restoring TF results in a 2893e035305SBorislav Petkov * trap from userspace immediately after SYSRET. This would cause an 2903e035305SBorislav Petkov * infinite loop whenever #DB happens with register state that satisfies 2913e035305SBorislav Petkov * the opportunistic SYSRET conditions. For example, single-stepping 2923e035305SBorislav Petkov * this user code: 293905a36a2SIngo Molnar * 294905a36a2SIngo Molnar * movq $stuck_here, %rcx 295905a36a2SIngo Molnar * pushfq 296905a36a2SIngo Molnar * popq %r11 297905a36a2SIngo Molnar * stuck_here: 298905a36a2SIngo Molnar * 299905a36a2SIngo Molnar * would never get past 'stuck_here'. 300905a36a2SIngo Molnar */ 301905a36a2SIngo Molnar testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11 3028a055d7fSAndy Lutomirski jnz swapgs_restore_regs_and_return_to_usermode 303905a36a2SIngo Molnar 304905a36a2SIngo Molnar /* nothing to check for RSP */ 305905a36a2SIngo Molnar 306905a36a2SIngo Molnar cmpq $__USER_DS, SS(%rsp) /* SS must match SYSRET */ 3078a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 308905a36a2SIngo Molnar 309905a36a2SIngo Molnar /* 310905a36a2SIngo Molnar * We win! This label is here just for ease of understanding 311905a36a2SIngo Molnar * perf profiles. Nothing jumps here. 312905a36a2SIngo Molnar */ 313905a36a2SIngo Molnarsyscall_return_via_sysret: 314905a36a2SIngo Molnar /* rcx and r11 are already restored (see code above) */ 3158c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 316502af0d7SDominik Brodowski POP_REGS pop_rdi=0 skip_r11rcx=1 3173e3b9293SAndy Lutomirski 3183e3b9293SAndy Lutomirski /* 3193e3b9293SAndy Lutomirski * Now all regs are restored except RSP and RDI. 3203e3b9293SAndy Lutomirski * Save old stack pointer and switch to trampoline stack. 3213e3b9293SAndy Lutomirski */ 3223e3b9293SAndy Lutomirski movq %rsp, %rdi 323c482feefSAndy Lutomirski movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp 3243e3b9293SAndy Lutomirski 3253e3b9293SAndy Lutomirski pushq RSP-RDI(%rdi) /* RSP */ 3263e3b9293SAndy Lutomirski pushq (%rdi) /* RDI */ 3273e3b9293SAndy Lutomirski 3283e3b9293SAndy Lutomirski /* 3293e3b9293SAndy Lutomirski * We are on the trampoline stack. All regs except RDI are live. 3303e3b9293SAndy Lutomirski * We can do future final exit work right here. 3313e3b9293SAndy Lutomirski */ 3326fd166aaSPeter Zijlstra SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 3333e3b9293SAndy Lutomirski 3344fbb3910SAndy Lutomirski popq %rdi 3353e3b9293SAndy Lutomirski popq %rsp 336905a36a2SIngo Molnar USERGS_SYSRET64 337b2502b41SIngo MolnarEND(entry_SYSCALL_64) 338905a36a2SIngo Molnar 339905a36a2SIngo Molnar/* 3400100301bSBrian Gerst * %rdi: prev task 3410100301bSBrian Gerst * %rsi: next task 3420100301bSBrian Gerst */ 3430100301bSBrian GerstENTRY(__switch_to_asm) 3448c1f7558SJosh Poimboeuf UNWIND_HINT_FUNC 3450100301bSBrian Gerst /* 3460100301bSBrian Gerst * Save callee-saved registers 3470100301bSBrian Gerst * This must match the order in inactive_task_frame 3480100301bSBrian Gerst */ 3490100301bSBrian Gerst pushq %rbp 3500100301bSBrian Gerst pushq %rbx 3510100301bSBrian Gerst pushq %r12 3520100301bSBrian Gerst pushq %r13 3530100301bSBrian Gerst pushq %r14 3540100301bSBrian Gerst pushq %r15 3550100301bSBrian Gerst 3560100301bSBrian Gerst /* switch stack */ 3570100301bSBrian Gerst movq %rsp, TASK_threadsp(%rdi) 3580100301bSBrian Gerst movq TASK_threadsp(%rsi), %rsp 3590100301bSBrian Gerst 360050e9baaSLinus Torvalds#ifdef CONFIG_STACKPROTECTOR 3610100301bSBrian Gerst movq TASK_stack_canary(%rsi), %rbx 3620100301bSBrian Gerst movq %rbx, PER_CPU_VAR(irq_stack_union)+stack_canary_offset 3630100301bSBrian Gerst#endif 3640100301bSBrian Gerst 365c995efd5SDavid Woodhouse#ifdef CONFIG_RETPOLINE 366c995efd5SDavid Woodhouse /* 367c995efd5SDavid Woodhouse * When switching from a shallower to a deeper call stack 368c995efd5SDavid Woodhouse * the RSB may either underflow or use entries populated 369c995efd5SDavid Woodhouse * with userspace addresses. On CPUs where those concerns 370c995efd5SDavid Woodhouse * exist, overwrite the RSB with entries which capture 371c995efd5SDavid Woodhouse * speculative execution to prevent attack. 372c995efd5SDavid Woodhouse */ 373d1c99108SDavid Woodhouse FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW 374c995efd5SDavid Woodhouse#endif 375c995efd5SDavid Woodhouse 3760100301bSBrian Gerst /* restore callee-saved registers */ 3770100301bSBrian Gerst popq %r15 3780100301bSBrian Gerst popq %r14 3790100301bSBrian Gerst popq %r13 3800100301bSBrian Gerst popq %r12 3810100301bSBrian Gerst popq %rbx 3820100301bSBrian Gerst popq %rbp 3830100301bSBrian Gerst 3840100301bSBrian Gerst jmp __switch_to 3850100301bSBrian GerstEND(__switch_to_asm) 3860100301bSBrian Gerst 3870100301bSBrian Gerst/* 388905a36a2SIngo Molnar * A newly forked process directly context switches into this address. 389905a36a2SIngo Molnar * 3900100301bSBrian Gerst * rax: prev task we switched from 391616d2483SBrian Gerst * rbx: kernel thread func (NULL for user thread) 392616d2483SBrian Gerst * r12: kernel thread arg 393905a36a2SIngo Molnar */ 394905a36a2SIngo MolnarENTRY(ret_from_fork) 3958c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 3960100301bSBrian Gerst movq %rax, %rdi 3974d732138SIngo Molnar call schedule_tail /* rdi: 'prev' task parameter */ 398905a36a2SIngo Molnar 399616d2483SBrian Gerst testq %rbx, %rbx /* from kernel_thread? */ 400616d2483SBrian Gerst jnz 1f /* kernel threads are uncommon */ 401905a36a2SIngo Molnar 402616d2483SBrian Gerst2: 4038c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 404ebd57499SJosh Poimboeuf movq %rsp, %rdi 40524d978b7SAndy Lutomirski call syscall_return_slowpath /* returns with IRQs disabled */ 40624d978b7SAndy Lutomirski TRACE_IRQS_ON /* user mode is traced as IRQS on */ 4078a055d7fSAndy Lutomirski jmp swapgs_restore_regs_and_return_to_usermode 408616d2483SBrian Gerst 409616d2483SBrian Gerst1: 410616d2483SBrian Gerst /* kernel thread */ 411*d31a5802SJosh Poimboeuf UNWIND_HINT_EMPTY 412616d2483SBrian Gerst movq %r12, %rdi 4132641f08bSDavid Woodhouse CALL_NOSPEC %rbx 414616d2483SBrian Gerst /* 415616d2483SBrian Gerst * A kernel thread is allowed to return here after successfully 416616d2483SBrian Gerst * calling do_execve(). Exit to userspace to complete the execve() 417616d2483SBrian Gerst * syscall. 418616d2483SBrian Gerst */ 419616d2483SBrian Gerst movq $0, RAX(%rsp) 420616d2483SBrian Gerst jmp 2b 421905a36a2SIngo MolnarEND(ret_from_fork) 422905a36a2SIngo Molnar 423905a36a2SIngo Molnar/* 424905a36a2SIngo Molnar * Build the entry stubs with some assembler magic. 425905a36a2SIngo Molnar * We pack 1 stub into every 8-byte block. 426905a36a2SIngo Molnar */ 427905a36a2SIngo Molnar .align 8 428905a36a2SIngo MolnarENTRY(irq_entries_start) 429905a36a2SIngo Molnar vector=FIRST_EXTERNAL_VECTOR 430905a36a2SIngo Molnar .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 4318c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 432905a36a2SIngo Molnar pushq $(~vector+0x80) /* Note: always in signed byte range */ 433905a36a2SIngo Molnar jmp common_interrupt 434905a36a2SIngo Molnar .align 8 4358c1f7558SJosh Poimboeuf vector=vector+1 436905a36a2SIngo Molnar .endr 437905a36a2SIngo MolnarEND(irq_entries_start) 438905a36a2SIngo Molnar 4391d3e53e8SAndy Lutomirski.macro DEBUG_ENTRY_ASSERT_IRQS_OFF 4401d3e53e8SAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 441e17f8234SBoris Ostrovsky pushq %rax 442e17f8234SBoris Ostrovsky SAVE_FLAGS(CLBR_RAX) 443e17f8234SBoris Ostrovsky testl $X86_EFLAGS_IF, %eax 4441d3e53e8SAndy Lutomirski jz .Lokay_\@ 4451d3e53e8SAndy Lutomirski ud2 4461d3e53e8SAndy Lutomirski.Lokay_\@: 447e17f8234SBoris Ostrovsky popq %rax 4481d3e53e8SAndy Lutomirski#endif 4491d3e53e8SAndy Lutomirski.endm 4501d3e53e8SAndy Lutomirski 4511d3e53e8SAndy Lutomirski/* 4521d3e53e8SAndy Lutomirski * Enters the IRQ stack if we're not already using it. NMI-safe. Clobbers 4531d3e53e8SAndy Lutomirski * flags and puts old RSP into old_rsp, and leaves all other GPRs alone. 4541d3e53e8SAndy Lutomirski * Requires kernel GSBASE. 4551d3e53e8SAndy Lutomirski * 4561d3e53e8SAndy Lutomirski * The invariant is that, if irq_count != -1, then the IRQ stack is in use. 4571d3e53e8SAndy Lutomirski */ 4582ba64741SDominik Brodowski.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0 4591d3e53e8SAndy Lutomirski DEBUG_ENTRY_ASSERT_IRQS_OFF 4602ba64741SDominik Brodowski 4612ba64741SDominik Brodowski .if \save_ret 4622ba64741SDominik Brodowski /* 4632ba64741SDominik Brodowski * If save_ret is set, the original stack contains one additional 4642ba64741SDominik Brodowski * entry -- the return address. Therefore, move the address one 4652ba64741SDominik Brodowski * entry below %rsp to \old_rsp. 4662ba64741SDominik Brodowski */ 4672ba64741SDominik Brodowski leaq 8(%rsp), \old_rsp 4682ba64741SDominik Brodowski .else 4691d3e53e8SAndy Lutomirski movq %rsp, \old_rsp 4702ba64741SDominik Brodowski .endif 4718c1f7558SJosh Poimboeuf 4728c1f7558SJosh Poimboeuf .if \regs 4738c1f7558SJosh Poimboeuf UNWIND_HINT_REGS base=\old_rsp 4748c1f7558SJosh Poimboeuf .endif 4758c1f7558SJosh Poimboeuf 4761d3e53e8SAndy Lutomirski incl PER_CPU_VAR(irq_count) 47729955909SAndy Lutomirski jnz .Lirq_stack_push_old_rsp_\@ 4781d3e53e8SAndy Lutomirski 4791d3e53e8SAndy Lutomirski /* 4801d3e53e8SAndy Lutomirski * Right now, if we just incremented irq_count to zero, we've 4811d3e53e8SAndy Lutomirski * claimed the IRQ stack but we haven't switched to it yet. 4821d3e53e8SAndy Lutomirski * 4831d3e53e8SAndy Lutomirski * If anything is added that can interrupt us here without using IST, 4841d3e53e8SAndy Lutomirski * it must be *extremely* careful to limit its stack usage. This 4851d3e53e8SAndy Lutomirski * could include kprobes and a hypothetical future IST-less #DB 4861d3e53e8SAndy Lutomirski * handler. 48729955909SAndy Lutomirski * 48829955909SAndy Lutomirski * The OOPS unwinder relies on the word at the top of the IRQ 48929955909SAndy Lutomirski * stack linking back to the previous RSP for the entire time we're 49029955909SAndy Lutomirski * on the IRQ stack. For this to work reliably, we need to write 49129955909SAndy Lutomirski * it before we actually move ourselves to the IRQ stack. 4921d3e53e8SAndy Lutomirski */ 4931d3e53e8SAndy Lutomirski 49429955909SAndy Lutomirski movq \old_rsp, PER_CPU_VAR(irq_stack_union + IRQ_STACK_SIZE - 8) 49529955909SAndy Lutomirski movq PER_CPU_VAR(irq_stack_ptr), %rsp 49629955909SAndy Lutomirski 49729955909SAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 49829955909SAndy Lutomirski /* 49929955909SAndy Lutomirski * If the first movq above becomes wrong due to IRQ stack layout 50029955909SAndy Lutomirski * changes, the only way we'll notice is if we try to unwind right 50129955909SAndy Lutomirski * here. Assert that we set up the stack right to catch this type 50229955909SAndy Lutomirski * of bug quickly. 50329955909SAndy Lutomirski */ 50429955909SAndy Lutomirski cmpq -8(%rsp), \old_rsp 50529955909SAndy Lutomirski je .Lirq_stack_okay\@ 50629955909SAndy Lutomirski ud2 50729955909SAndy Lutomirski .Lirq_stack_okay\@: 50829955909SAndy Lutomirski#endif 50929955909SAndy Lutomirski 51029955909SAndy Lutomirski.Lirq_stack_push_old_rsp_\@: 5111d3e53e8SAndy Lutomirski pushq \old_rsp 5128c1f7558SJosh Poimboeuf 5138c1f7558SJosh Poimboeuf .if \regs 5148c1f7558SJosh Poimboeuf UNWIND_HINT_REGS indirect=1 5158c1f7558SJosh Poimboeuf .endif 5162ba64741SDominik Brodowski 5172ba64741SDominik Brodowski .if \save_ret 5182ba64741SDominik Brodowski /* 5192ba64741SDominik Brodowski * Push the return address to the stack. This return address can 5202ba64741SDominik Brodowski * be found at the "real" original RSP, which was offset by 8 at 5212ba64741SDominik Brodowski * the beginning of this macro. 5222ba64741SDominik Brodowski */ 5232ba64741SDominik Brodowski pushq -8(\old_rsp) 5242ba64741SDominik Brodowski .endif 5251d3e53e8SAndy Lutomirski.endm 5261d3e53e8SAndy Lutomirski 5271d3e53e8SAndy Lutomirski/* 5281d3e53e8SAndy Lutomirski * Undoes ENTER_IRQ_STACK. 5291d3e53e8SAndy Lutomirski */ 5308c1f7558SJosh Poimboeuf.macro LEAVE_IRQ_STACK regs=1 5311d3e53e8SAndy Lutomirski DEBUG_ENTRY_ASSERT_IRQS_OFF 5321d3e53e8SAndy Lutomirski /* We need to be off the IRQ stack before decrementing irq_count. */ 5331d3e53e8SAndy Lutomirski popq %rsp 5341d3e53e8SAndy Lutomirski 5358c1f7558SJosh Poimboeuf .if \regs 5368c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 5378c1f7558SJosh Poimboeuf .endif 5388c1f7558SJosh Poimboeuf 5391d3e53e8SAndy Lutomirski /* 5401d3e53e8SAndy Lutomirski * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming 5411d3e53e8SAndy Lutomirski * the irq stack but we're not on it. 5421d3e53e8SAndy Lutomirski */ 5431d3e53e8SAndy Lutomirski 5441d3e53e8SAndy Lutomirski decl PER_CPU_VAR(irq_count) 5451d3e53e8SAndy Lutomirski.endm 5461d3e53e8SAndy Lutomirski 547905a36a2SIngo Molnar/* 548f3d415eaSDominik Brodowski * Interrupt entry helper function. 549905a36a2SIngo Molnar * 550f3d415eaSDominik Brodowski * Entry runs with interrupts off. Stack layout at entry: 551f3d415eaSDominik Brodowski * +----------------------------------------------------+ 552f3d415eaSDominik Brodowski * | regs->ss | 553f3d415eaSDominik Brodowski * | regs->rsp | 554f3d415eaSDominik Brodowski * | regs->eflags | 555f3d415eaSDominik Brodowski * | regs->cs | 556f3d415eaSDominik Brodowski * | regs->ip | 557f3d415eaSDominik Brodowski * +----------------------------------------------------+ 558f3d415eaSDominik Brodowski * | regs->orig_ax = ~(interrupt number) | 559f3d415eaSDominik Brodowski * +----------------------------------------------------+ 560f3d415eaSDominik Brodowski * | return address | 561f3d415eaSDominik Brodowski * +----------------------------------------------------+ 562905a36a2SIngo Molnar */ 563f3d415eaSDominik BrodowskiENTRY(interrupt_entry) 564f3d415eaSDominik Brodowski UNWIND_HINT_FUNC 565f3d415eaSDominik Brodowski ASM_CLAC 566905a36a2SIngo Molnar cld 5677f2590a1SAndy Lutomirski 568f3d415eaSDominik Brodowski testb $3, CS-ORIG_RAX+8(%rsp) 5697f2590a1SAndy Lutomirski jz 1f 5707f2590a1SAndy Lutomirski SWAPGS 571f3d415eaSDominik Brodowski 572f3d415eaSDominik Brodowski /* 573f3d415eaSDominik Brodowski * Switch to the thread stack. The IRET frame and orig_ax are 574f3d415eaSDominik Brodowski * on the stack, as well as the return address. RDI..R12 are 575f3d415eaSDominik Brodowski * not (yet) on the stack and space has not (yet) been 576f3d415eaSDominik Brodowski * allocated for them. 577f3d415eaSDominik Brodowski */ 57890a6acc4SDominik Brodowski pushq %rdi 579f3d415eaSDominik Brodowski 58090a6acc4SDominik Brodowski /* Need to switch before accessing the thread stack. */ 58190a6acc4SDominik Brodowski SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi 58290a6acc4SDominik Brodowski movq %rsp, %rdi 58390a6acc4SDominik Brodowski movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 584f3d415eaSDominik Brodowski 585f3d415eaSDominik Brodowski /* 586f3d415eaSDominik Brodowski * We have RDI, return address, and orig_ax on the stack on 587f3d415eaSDominik Brodowski * top of the IRET frame. That means offset=24 588f3d415eaSDominik Brodowski */ 589f3d415eaSDominik Brodowski UNWIND_HINT_IRET_REGS base=%rdi offset=24 59090a6acc4SDominik Brodowski 59190a6acc4SDominik Brodowski pushq 7*8(%rdi) /* regs->ss */ 59290a6acc4SDominik Brodowski pushq 6*8(%rdi) /* regs->rsp */ 59390a6acc4SDominik Brodowski pushq 5*8(%rdi) /* regs->eflags */ 59490a6acc4SDominik Brodowski pushq 4*8(%rdi) /* regs->cs */ 59590a6acc4SDominik Brodowski pushq 3*8(%rdi) /* regs->ip */ 59690a6acc4SDominik Brodowski pushq 2*8(%rdi) /* regs->orig_ax */ 59790a6acc4SDominik Brodowski pushq 8(%rdi) /* return address */ 59890a6acc4SDominik Brodowski UNWIND_HINT_FUNC 59990a6acc4SDominik Brodowski 60090a6acc4SDominik Brodowski movq (%rdi), %rdi 6017f2590a1SAndy Lutomirski1: 6027f2590a1SAndy Lutomirski 6030e34d226SDominik Brodowski PUSH_AND_CLEAR_REGS save_ret=1 6040e34d226SDominik Brodowski ENCODE_FRAME_POINTER 8 605905a36a2SIngo Molnar 6062ba64741SDominik Brodowski testb $3, CS+8(%rsp) 607905a36a2SIngo Molnar jz 1f 60802bc7768SAndy Lutomirski 60902bc7768SAndy Lutomirski /* 6107f2590a1SAndy Lutomirski * IRQ from user mode. 6117f2590a1SAndy Lutomirski * 612f1075053SAndy Lutomirski * We need to tell lockdep that IRQs are off. We can't do this until 613f1075053SAndy Lutomirski * we fix gsbase, and we should do it before enter_from_user_mode 614f3d415eaSDominik Brodowski * (which can take locks). Since TRACE_IRQS_OFF is idempotent, 615f1075053SAndy Lutomirski * the simplest way to handle it is to just call it twice if 616f1075053SAndy Lutomirski * we enter from user mode. There's no reason to optimize this since 617f1075053SAndy Lutomirski * TRACE_IRQS_OFF is a no-op if lockdep is off. 618f1075053SAndy Lutomirski */ 619f1075053SAndy Lutomirski TRACE_IRQS_OFF 620f1075053SAndy Lutomirski 621478dc89cSAndy Lutomirski CALL_enter_from_user_mode 62202bc7768SAndy Lutomirski 623905a36a2SIngo Molnar1: 6242ba64741SDominik Brodowski ENTER_IRQ_STACK old_rsp=%rdi save_ret=1 625905a36a2SIngo Molnar /* We entered an interrupt context - irqs are off: */ 626905a36a2SIngo Molnar TRACE_IRQS_OFF 627905a36a2SIngo Molnar 6282ba64741SDominik Brodowski ret 6292ba64741SDominik BrodowskiEND(interrupt_entry) 6302ba64741SDominik Brodowski 631f3d415eaSDominik Brodowski 632f3d415eaSDominik Brodowski/* Interrupt entry/exit. */ 633905a36a2SIngo Molnar 634905a36a2SIngo Molnar /* 635905a36a2SIngo Molnar * The interrupt stubs push (~vector+0x80) onto the stack and 636905a36a2SIngo Molnar * then jump to common_interrupt. 637905a36a2SIngo Molnar */ 638905a36a2SIngo Molnar .p2align CONFIG_X86_L1_CACHE_SHIFT 639905a36a2SIngo Molnarcommon_interrupt: 640905a36a2SIngo Molnar addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */ 6413aa99fc3SDominik Brodowski call interrupt_entry 6423aa99fc3SDominik Brodowski UNWIND_HINT_REGS indirect=1 6433aa99fc3SDominik Brodowski call do_IRQ /* rdi points to pt_regs */ 644905a36a2SIngo Molnar /* 0(%rsp): old RSP */ 645905a36a2SIngo Molnarret_from_intr: 6462140a994SJan Beulich DISABLE_INTERRUPTS(CLBR_ANY) 647905a36a2SIngo Molnar TRACE_IRQS_OFF 648905a36a2SIngo Molnar 6491d3e53e8SAndy Lutomirski LEAVE_IRQ_STACK 650905a36a2SIngo Molnar 651905a36a2SIngo Molnar testb $3, CS(%rsp) 652905a36a2SIngo Molnar jz retint_kernel 65302bc7768SAndy Lutomirski 654905a36a2SIngo Molnar /* Interrupt came from user space */ 65502bc7768SAndy LutomirskiGLOBAL(retint_user) 65602bc7768SAndy Lutomirski mov %rsp,%rdi 65702bc7768SAndy Lutomirski call prepare_exit_to_usermode 658905a36a2SIngo Molnar TRACE_IRQS_IRETQ 65926c4ef9cSAndy Lutomirski 6608a055d7fSAndy LutomirskiGLOBAL(swapgs_restore_regs_and_return_to_usermode) 66126c4ef9cSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 66226c4ef9cSAndy Lutomirski /* Assert that pt_regs indicates user mode. */ 6631e4c4f61SBorislav Petkov testb $3, CS(%rsp) 66426c4ef9cSAndy Lutomirski jnz 1f 66526c4ef9cSAndy Lutomirski ud2 66626c4ef9cSAndy Lutomirski1: 66726c4ef9cSAndy Lutomirski#endif 668502af0d7SDominik Brodowski POP_REGS pop_rdi=0 6693e3b9293SAndy Lutomirski 6703e3b9293SAndy Lutomirski /* 6713e3b9293SAndy Lutomirski * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS. 6723e3b9293SAndy Lutomirski * Save old stack pointer and switch to trampoline stack. 6733e3b9293SAndy Lutomirski */ 6743e3b9293SAndy Lutomirski movq %rsp, %rdi 675c482feefSAndy Lutomirski movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp 6763e3b9293SAndy Lutomirski 6773e3b9293SAndy Lutomirski /* Copy the IRET frame to the trampoline stack. */ 6783e3b9293SAndy Lutomirski pushq 6*8(%rdi) /* SS */ 6793e3b9293SAndy Lutomirski pushq 5*8(%rdi) /* RSP */ 6803e3b9293SAndy Lutomirski pushq 4*8(%rdi) /* EFLAGS */ 6813e3b9293SAndy Lutomirski pushq 3*8(%rdi) /* CS */ 6823e3b9293SAndy Lutomirski pushq 2*8(%rdi) /* RIP */ 6833e3b9293SAndy Lutomirski 6843e3b9293SAndy Lutomirski /* Push user RDI on the trampoline stack. */ 6853e3b9293SAndy Lutomirski pushq (%rdi) 6863e3b9293SAndy Lutomirski 6873e3b9293SAndy Lutomirski /* 6883e3b9293SAndy Lutomirski * We are on the trampoline stack. All regs except RDI are live. 6893e3b9293SAndy Lutomirski * We can do future final exit work right here. 6903e3b9293SAndy Lutomirski */ 6913e3b9293SAndy Lutomirski 6926fd166aaSPeter Zijlstra SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 6938a09317bSDave Hansen 6943e3b9293SAndy Lutomirski /* Restore RDI. */ 6953e3b9293SAndy Lutomirski popq %rdi 6963e3b9293SAndy Lutomirski SWAPGS 69726c4ef9cSAndy Lutomirski INTERRUPT_RETURN 69826c4ef9cSAndy Lutomirski 699905a36a2SIngo Molnar 700905a36a2SIngo Molnar/* Returning to kernel space */ 701905a36a2SIngo Molnarretint_kernel: 702905a36a2SIngo Molnar#ifdef CONFIG_PREEMPT 703905a36a2SIngo Molnar /* Interrupts are off */ 704905a36a2SIngo Molnar /* Check if we need preemption */ 7054d732138SIngo Molnar bt $9, EFLAGS(%rsp) /* were interrupts off? */ 706905a36a2SIngo Molnar jnc 1f 707905a36a2SIngo Molnar0: cmpl $0, PER_CPU_VAR(__preempt_count) 708905a36a2SIngo Molnar jnz 1f 709905a36a2SIngo Molnar call preempt_schedule_irq 710905a36a2SIngo Molnar jmp 0b 711905a36a2SIngo Molnar1: 712905a36a2SIngo Molnar#endif 713905a36a2SIngo Molnar /* 714905a36a2SIngo Molnar * The iretq could re-enable interrupts: 715905a36a2SIngo Molnar */ 716905a36a2SIngo Molnar TRACE_IRQS_IRETQ 717905a36a2SIngo Molnar 71826c4ef9cSAndy LutomirskiGLOBAL(restore_regs_and_return_to_kernel) 71926c4ef9cSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 72026c4ef9cSAndy Lutomirski /* Assert that pt_regs indicates kernel mode. */ 7211e4c4f61SBorislav Petkov testb $3, CS(%rsp) 72226c4ef9cSAndy Lutomirski jz 1f 72326c4ef9cSAndy Lutomirski ud2 72426c4ef9cSAndy Lutomirski1: 72526c4ef9cSAndy Lutomirski#endif 726502af0d7SDominik Brodowski POP_REGS 727e872045bSAndy Lutomirski addq $8, %rsp /* skip regs->orig_ax */ 72810bcc80eSMathieu Desnoyers /* 72910bcc80eSMathieu Desnoyers * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization 73010bcc80eSMathieu Desnoyers * when returning from IPI handler. 73110bcc80eSMathieu Desnoyers */ 732905a36a2SIngo Molnar INTERRUPT_RETURN 733905a36a2SIngo Molnar 734905a36a2SIngo MolnarENTRY(native_iret) 7358c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 736905a36a2SIngo Molnar /* 737905a36a2SIngo Molnar * Are we returning to a stack segment from the LDT? Note: in 738905a36a2SIngo Molnar * 64-bit mode SS:RSP on the exception stack is always valid. 739905a36a2SIngo Molnar */ 740905a36a2SIngo Molnar#ifdef CONFIG_X86_ESPFIX64 741905a36a2SIngo Molnar testb $4, (SS-RIP)(%rsp) 742905a36a2SIngo Molnar jnz native_irq_return_ldt 743905a36a2SIngo Molnar#endif 744905a36a2SIngo Molnar 745905a36a2SIngo Molnar.global native_irq_return_iret 746905a36a2SIngo Molnarnative_irq_return_iret: 747905a36a2SIngo Molnar /* 748905a36a2SIngo Molnar * This may fault. Non-paranoid faults on return to userspace are 749905a36a2SIngo Molnar * handled by fixup_bad_iret. These include #SS, #GP, and #NP. 750905a36a2SIngo Molnar * Double-faults due to espfix64 are handled in do_double_fault. 751905a36a2SIngo Molnar * Other faults here are fatal. 752905a36a2SIngo Molnar */ 753905a36a2SIngo Molnar iretq 754905a36a2SIngo Molnar 755905a36a2SIngo Molnar#ifdef CONFIG_X86_ESPFIX64 756905a36a2SIngo Molnarnative_irq_return_ldt: 75785063facSAndy Lutomirski /* 75885063facSAndy Lutomirski * We are running with user GSBASE. All GPRs contain their user 75985063facSAndy Lutomirski * values. We have a percpu ESPFIX stack that is eight slots 76085063facSAndy Lutomirski * long (see ESPFIX_STACK_SIZE). espfix_waddr points to the bottom 76185063facSAndy Lutomirski * of the ESPFIX stack. 76285063facSAndy Lutomirski * 76385063facSAndy Lutomirski * We clobber RAX and RDI in this code. We stash RDI on the 76485063facSAndy Lutomirski * normal stack and RAX on the ESPFIX stack. 76585063facSAndy Lutomirski * 76685063facSAndy Lutomirski * The ESPFIX stack layout we set up looks like this: 76785063facSAndy Lutomirski * 76885063facSAndy Lutomirski * --- top of ESPFIX stack --- 76985063facSAndy Lutomirski * SS 77085063facSAndy Lutomirski * RSP 77185063facSAndy Lutomirski * RFLAGS 77285063facSAndy Lutomirski * CS 77385063facSAndy Lutomirski * RIP <-- RSP points here when we're done 77485063facSAndy Lutomirski * RAX <-- espfix_waddr points here 77585063facSAndy Lutomirski * --- bottom of ESPFIX stack --- 77685063facSAndy Lutomirski */ 77785063facSAndy Lutomirski 77885063facSAndy Lutomirski pushq %rdi /* Stash user RDI */ 7798a09317bSDave Hansen SWAPGS /* to kernel GS */ 7808a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */ 7818a09317bSDave Hansen 782905a36a2SIngo Molnar movq PER_CPU_VAR(espfix_waddr), %rdi 78385063facSAndy Lutomirski movq %rax, (0*8)(%rdi) /* user RAX */ 78485063facSAndy Lutomirski movq (1*8)(%rsp), %rax /* user RIP */ 785905a36a2SIngo Molnar movq %rax, (1*8)(%rdi) 78685063facSAndy Lutomirski movq (2*8)(%rsp), %rax /* user CS */ 787905a36a2SIngo Molnar movq %rax, (2*8)(%rdi) 78885063facSAndy Lutomirski movq (3*8)(%rsp), %rax /* user RFLAGS */ 789905a36a2SIngo Molnar movq %rax, (3*8)(%rdi) 79085063facSAndy Lutomirski movq (5*8)(%rsp), %rax /* user SS */ 791905a36a2SIngo Molnar movq %rax, (5*8)(%rdi) 79285063facSAndy Lutomirski movq (4*8)(%rsp), %rax /* user RSP */ 793905a36a2SIngo Molnar movq %rax, (4*8)(%rdi) 79485063facSAndy Lutomirski /* Now RAX == RSP. */ 79585063facSAndy Lutomirski 79685063facSAndy Lutomirski andl $0xffff0000, %eax /* RAX = (RSP & 0xffff0000) */ 79785063facSAndy Lutomirski 79885063facSAndy Lutomirski /* 79985063facSAndy Lutomirski * espfix_stack[31:16] == 0. The page tables are set up such that 80085063facSAndy Lutomirski * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of 80185063facSAndy Lutomirski * espfix_waddr for any X. That is, there are 65536 RO aliases of 80285063facSAndy Lutomirski * the same page. Set up RSP so that RSP[31:16] contains the 80385063facSAndy Lutomirski * respective 16 bits of the /userspace/ RSP and RSP nonetheless 80485063facSAndy Lutomirski * still points to an RO alias of the ESPFIX stack. 80585063facSAndy Lutomirski */ 806905a36a2SIngo Molnar orq PER_CPU_VAR(espfix_stack), %rax 8078a09317bSDave Hansen 8086fd166aaSPeter Zijlstra SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 8098a09317bSDave Hansen SWAPGS /* to user GS */ 8108a09317bSDave Hansen popq %rdi /* Restore user RDI */ 8118a09317bSDave Hansen 812905a36a2SIngo Molnar movq %rax, %rsp 8138c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS offset=8 81485063facSAndy Lutomirski 81585063facSAndy Lutomirski /* 81685063facSAndy Lutomirski * At this point, we cannot write to the stack any more, but we can 81785063facSAndy Lutomirski * still read. 81885063facSAndy Lutomirski */ 81985063facSAndy Lutomirski popq %rax /* Restore user RAX */ 82085063facSAndy Lutomirski 82185063facSAndy Lutomirski /* 82285063facSAndy Lutomirski * RSP now points to an ordinary IRET frame, except that the page 82385063facSAndy Lutomirski * is read-only and RSP[31:16] are preloaded with the userspace 82485063facSAndy Lutomirski * values. We can now IRET back to userspace. 82585063facSAndy Lutomirski */ 826905a36a2SIngo Molnar jmp native_irq_return_iret 827905a36a2SIngo Molnar#endif 828905a36a2SIngo MolnarEND(common_interrupt) 829905a36a2SIngo Molnar 830905a36a2SIngo Molnar/* 831905a36a2SIngo Molnar * APIC interrupts. 832905a36a2SIngo Molnar */ 833905a36a2SIngo Molnar.macro apicinterrupt3 num sym do_sym 834905a36a2SIngo MolnarENTRY(\sym) 8358c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 836905a36a2SIngo Molnar pushq $~(\num) 837905a36a2SIngo Molnar.Lcommon_\sym: 8383aa99fc3SDominik Brodowski call interrupt_entry 8393aa99fc3SDominik Brodowski UNWIND_HINT_REGS indirect=1 8403aa99fc3SDominik Brodowski call \do_sym /* rdi points to pt_regs */ 841905a36a2SIngo Molnar jmp ret_from_intr 842905a36a2SIngo MolnarEND(\sym) 843905a36a2SIngo Molnar.endm 844905a36a2SIngo Molnar 845469f0023SAlexander Potapenko/* Make sure APIC interrupt handlers end up in the irqentry section: */ 846469f0023SAlexander Potapenko#define PUSH_SECTION_IRQENTRY .pushsection .irqentry.text, "ax" 847469f0023SAlexander Potapenko#define POP_SECTION_IRQENTRY .popsection 848469f0023SAlexander Potapenko 849905a36a2SIngo Molnar.macro apicinterrupt num sym do_sym 850469f0023SAlexander PotapenkoPUSH_SECTION_IRQENTRY 851905a36a2SIngo Molnarapicinterrupt3 \num \sym \do_sym 852469f0023SAlexander PotapenkoPOP_SECTION_IRQENTRY 853905a36a2SIngo Molnar.endm 854905a36a2SIngo Molnar 855905a36a2SIngo Molnar#ifdef CONFIG_SMP 8564d732138SIngo Molnarapicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt 8574d732138SIngo Molnarapicinterrupt3 REBOOT_VECTOR reboot_interrupt smp_reboot_interrupt 858905a36a2SIngo Molnar#endif 859905a36a2SIngo Molnar 860905a36a2SIngo Molnar#ifdef CONFIG_X86_UV 8614d732138SIngo Molnarapicinterrupt3 UV_BAU_MESSAGE uv_bau_message_intr1 uv_bau_message_interrupt 862905a36a2SIngo Molnar#endif 8634d732138SIngo Molnar 8644d732138SIngo Molnarapicinterrupt LOCAL_TIMER_VECTOR apic_timer_interrupt smp_apic_timer_interrupt 8654d732138SIngo Molnarapicinterrupt X86_PLATFORM_IPI_VECTOR x86_platform_ipi smp_x86_platform_ipi 866905a36a2SIngo Molnar 867905a36a2SIngo Molnar#ifdef CONFIG_HAVE_KVM 8684d732138SIngo Molnarapicinterrupt3 POSTED_INTR_VECTOR kvm_posted_intr_ipi smp_kvm_posted_intr_ipi 8694d732138SIngo Molnarapicinterrupt3 POSTED_INTR_WAKEUP_VECTOR kvm_posted_intr_wakeup_ipi smp_kvm_posted_intr_wakeup_ipi 870210f84b0SWincy Vanapicinterrupt3 POSTED_INTR_NESTED_VECTOR kvm_posted_intr_nested_ipi smp_kvm_posted_intr_nested_ipi 871905a36a2SIngo Molnar#endif 872905a36a2SIngo Molnar 873905a36a2SIngo Molnar#ifdef CONFIG_X86_MCE_THRESHOLD 8744d732138SIngo Molnarapicinterrupt THRESHOLD_APIC_VECTOR threshold_interrupt smp_threshold_interrupt 875905a36a2SIngo Molnar#endif 876905a36a2SIngo Molnar 8779dda1658SIngo Molnar#ifdef CONFIG_X86_MCE_AMD 8784d732138SIngo Molnarapicinterrupt DEFERRED_ERROR_VECTOR deferred_error_interrupt smp_deferred_error_interrupt 8799dda1658SIngo Molnar#endif 8809dda1658SIngo Molnar 881905a36a2SIngo Molnar#ifdef CONFIG_X86_THERMAL_VECTOR 8824d732138SIngo Molnarapicinterrupt THERMAL_APIC_VECTOR thermal_interrupt smp_thermal_interrupt 883905a36a2SIngo Molnar#endif 884905a36a2SIngo Molnar 885905a36a2SIngo Molnar#ifdef CONFIG_SMP 8864d732138SIngo Molnarapicinterrupt CALL_FUNCTION_SINGLE_VECTOR call_function_single_interrupt smp_call_function_single_interrupt 8874d732138SIngo Molnarapicinterrupt CALL_FUNCTION_VECTOR call_function_interrupt smp_call_function_interrupt 8884d732138SIngo Molnarapicinterrupt RESCHEDULE_VECTOR reschedule_interrupt smp_reschedule_interrupt 889905a36a2SIngo Molnar#endif 890905a36a2SIngo Molnar 8914d732138SIngo Molnarapicinterrupt ERROR_APIC_VECTOR error_interrupt smp_error_interrupt 8924d732138SIngo Molnarapicinterrupt SPURIOUS_APIC_VECTOR spurious_interrupt smp_spurious_interrupt 893905a36a2SIngo Molnar 894905a36a2SIngo Molnar#ifdef CONFIG_IRQ_WORK 8954d732138SIngo Molnarapicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt 896905a36a2SIngo Molnar#endif 897905a36a2SIngo Molnar 898905a36a2SIngo Molnar/* 899905a36a2SIngo Molnar * Exception entry points. 900905a36a2SIngo Molnar */ 901c482feefSAndy Lutomirski#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8) 902905a36a2SIngo Molnar 903905a36a2SIngo Molnar.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 904905a36a2SIngo MolnarENTRY(\sym) 90598990a33SJosh Poimboeuf UNWIND_HINT_IRET_REGS offset=\has_error_code*8 9068c1f7558SJosh Poimboeuf 907905a36a2SIngo Molnar /* Sanity check */ 908905a36a2SIngo Molnar .if \shift_ist != -1 && \paranoid == 0 909905a36a2SIngo Molnar .error "using shift_ist requires paranoid=1" 910905a36a2SIngo Molnar .endif 911905a36a2SIngo Molnar 912905a36a2SIngo Molnar ASM_CLAC 913905a36a2SIngo Molnar 91482c62fa0SJosh Poimboeuf .if \has_error_code == 0 915905a36a2SIngo Molnar pushq $-1 /* ORIG_RAX: no syscall to restart */ 916905a36a2SIngo Molnar .endif 917905a36a2SIngo Molnar 918071ccc96SAndy Lutomirski .if \paranoid == 1 9199e809d15SDominik Brodowski testb $3, CS-ORIG_RAX(%rsp) /* If coming from userspace, switch stacks */ 9207f2590a1SAndy Lutomirski jnz .Lfrom_usermode_switch_stack_\@ 921905a36a2SIngo Molnar .endif 9227f2590a1SAndy Lutomirski 9237f2590a1SAndy Lutomirski .if \paranoid 924905a36a2SIngo Molnar call paranoid_entry 925905a36a2SIngo Molnar .else 926905a36a2SIngo Molnar call error_entry 927905a36a2SIngo Molnar .endif 9288c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 929905a36a2SIngo Molnar /* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */ 930905a36a2SIngo Molnar 931905a36a2SIngo Molnar .if \paranoid 932905a36a2SIngo Molnar .if \shift_ist != -1 933905a36a2SIngo Molnar TRACE_IRQS_OFF_DEBUG /* reload IDT in case of recursion */ 934905a36a2SIngo Molnar .else 935905a36a2SIngo Molnar TRACE_IRQS_OFF 936905a36a2SIngo Molnar .endif 937905a36a2SIngo Molnar .endif 938905a36a2SIngo Molnar 939905a36a2SIngo Molnar movq %rsp, %rdi /* pt_regs pointer */ 940905a36a2SIngo Molnar 941905a36a2SIngo Molnar .if \has_error_code 942905a36a2SIngo Molnar movq ORIG_RAX(%rsp), %rsi /* get error code */ 943905a36a2SIngo Molnar movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ 944905a36a2SIngo Molnar .else 945905a36a2SIngo Molnar xorl %esi, %esi /* no error code */ 946905a36a2SIngo Molnar .endif 947905a36a2SIngo Molnar 948905a36a2SIngo Molnar .if \shift_ist != -1 949905a36a2SIngo Molnar subq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist) 950905a36a2SIngo Molnar .endif 951905a36a2SIngo Molnar 952905a36a2SIngo Molnar call \do_sym 953905a36a2SIngo Molnar 954905a36a2SIngo Molnar .if \shift_ist != -1 955905a36a2SIngo Molnar addq $EXCEPTION_STKSZ, CPU_TSS_IST(\shift_ist) 956905a36a2SIngo Molnar .endif 957905a36a2SIngo Molnar 958905a36a2SIngo Molnar /* these procedures expect "no swapgs" flag in ebx */ 959905a36a2SIngo Molnar .if \paranoid 960905a36a2SIngo Molnar jmp paranoid_exit 961905a36a2SIngo Molnar .else 962905a36a2SIngo Molnar jmp error_exit 963905a36a2SIngo Molnar .endif 964905a36a2SIngo Molnar 965071ccc96SAndy Lutomirski .if \paranoid == 1 966905a36a2SIngo Molnar /* 9677f2590a1SAndy Lutomirski * Entry from userspace. Switch stacks and treat it 968905a36a2SIngo Molnar * as a normal entry. This means that paranoid handlers 969905a36a2SIngo Molnar * run in real process context if user_mode(regs). 970905a36a2SIngo Molnar */ 9717f2590a1SAndy Lutomirski.Lfrom_usermode_switch_stack_\@: 972905a36a2SIngo Molnar call error_entry 973905a36a2SIngo Molnar 974905a36a2SIngo Molnar movq %rsp, %rdi /* pt_regs pointer */ 975905a36a2SIngo Molnar 976905a36a2SIngo Molnar .if \has_error_code 977905a36a2SIngo Molnar movq ORIG_RAX(%rsp), %rsi /* get error code */ 978905a36a2SIngo Molnar movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ 979905a36a2SIngo Molnar .else 980905a36a2SIngo Molnar xorl %esi, %esi /* no error code */ 981905a36a2SIngo Molnar .endif 982905a36a2SIngo Molnar 983905a36a2SIngo Molnar call \do_sym 984905a36a2SIngo Molnar 985905a36a2SIngo Molnar jmp error_exit /* %ebx: no swapgs flag */ 986905a36a2SIngo Molnar .endif 987905a36a2SIngo MolnarEND(\sym) 988905a36a2SIngo Molnar.endm 989905a36a2SIngo Molnar 990905a36a2SIngo Molnaridtentry divide_error do_divide_error has_error_code=0 991905a36a2SIngo Molnaridtentry overflow do_overflow has_error_code=0 992905a36a2SIngo Molnaridtentry bounds do_bounds has_error_code=0 993905a36a2SIngo Molnaridtentry invalid_op do_invalid_op has_error_code=0 994905a36a2SIngo Molnaridtentry device_not_available do_device_not_available has_error_code=0 995905a36a2SIngo Molnaridtentry double_fault do_double_fault has_error_code=1 paranoid=2 996905a36a2SIngo Molnaridtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0 997905a36a2SIngo Molnaridtentry invalid_TSS do_invalid_TSS has_error_code=1 998905a36a2SIngo Molnaridtentry segment_not_present do_segment_not_present has_error_code=1 999905a36a2SIngo Molnaridtentry spurious_interrupt_bug do_spurious_interrupt_bug has_error_code=0 1000905a36a2SIngo Molnaridtentry coprocessor_error do_coprocessor_error has_error_code=0 1001905a36a2SIngo Molnaridtentry alignment_check do_alignment_check has_error_code=1 1002905a36a2SIngo Molnaridtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0 1003905a36a2SIngo Molnar 1004905a36a2SIngo Molnar 10054d732138SIngo Molnar /* 10064d732138SIngo Molnar * Reload gs selector with exception handling 10074d732138SIngo Molnar * edi: new selector 10084d732138SIngo Molnar */ 1009905a36a2SIngo MolnarENTRY(native_load_gs_index) 10108c1f7558SJosh Poimboeuf FRAME_BEGIN 1011905a36a2SIngo Molnar pushfq 1012905a36a2SIngo Molnar DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI) 1013ca37e57bSAndy Lutomirski TRACE_IRQS_OFF 1014905a36a2SIngo Molnar SWAPGS 101542c748bbSBorislav Petkov.Lgs_change: 1016905a36a2SIngo Molnar movl %edi, %gs 101796e5d28aSBorislav Petkov2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE 1018905a36a2SIngo Molnar SWAPGS 1019ca37e57bSAndy Lutomirski TRACE_IRQS_FLAGS (%rsp) 1020905a36a2SIngo Molnar popfq 10218c1f7558SJosh Poimboeuf FRAME_END 1022905a36a2SIngo Molnar ret 10238c1f7558SJosh PoimboeufENDPROC(native_load_gs_index) 1024784d5699SAl ViroEXPORT_SYMBOL(native_load_gs_index) 1025905a36a2SIngo Molnar 102642c748bbSBorislav Petkov _ASM_EXTABLE(.Lgs_change, bad_gs) 1027905a36a2SIngo Molnar .section .fixup, "ax" 1028905a36a2SIngo Molnar /* running with kernelgs */ 1029905a36a2SIngo Molnarbad_gs: 1030905a36a2SIngo Molnar SWAPGS /* switch back to user gs */ 1031b038c842SAndy Lutomirski.macro ZAP_GS 1032b038c842SAndy Lutomirski /* This can't be a string because the preprocessor needs to see it. */ 1033b038c842SAndy Lutomirski movl $__USER_DS, %eax 1034b038c842SAndy Lutomirski movl %eax, %gs 1035b038c842SAndy Lutomirski.endm 1036b038c842SAndy Lutomirski ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG 1037905a36a2SIngo Molnar xorl %eax, %eax 1038905a36a2SIngo Molnar movl %eax, %gs 1039905a36a2SIngo Molnar jmp 2b 1040905a36a2SIngo Molnar .previous 1041905a36a2SIngo Molnar 1042905a36a2SIngo Molnar/* Call softirq on interrupt stack. Interrupts are off. */ 1043905a36a2SIngo MolnarENTRY(do_softirq_own_stack) 1044905a36a2SIngo Molnar pushq %rbp 1045905a36a2SIngo Molnar mov %rsp, %rbp 10468c1f7558SJosh Poimboeuf ENTER_IRQ_STACK regs=0 old_rsp=%r11 1047905a36a2SIngo Molnar call __do_softirq 10488c1f7558SJosh Poimboeuf LEAVE_IRQ_STACK regs=0 1049905a36a2SIngo Molnar leaveq 1050905a36a2SIngo Molnar ret 10518c1f7558SJosh PoimboeufENDPROC(do_softirq_own_stack) 1052905a36a2SIngo Molnar 1053905a36a2SIngo Molnar#ifdef CONFIG_XEN 10545878d5d6SJuergen Grossidtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0 1055905a36a2SIngo Molnar 1056905a36a2SIngo Molnar/* 1057905a36a2SIngo Molnar * A note on the "critical region" in our callback handler. 1058905a36a2SIngo Molnar * We want to avoid stacking callback handlers due to events occurring 1059905a36a2SIngo Molnar * during handling of the last event. To do this, we keep events disabled 1060905a36a2SIngo Molnar * until we've done all processing. HOWEVER, we must enable events before 1061905a36a2SIngo Molnar * popping the stack frame (can't be done atomically) and so it would still 1062905a36a2SIngo Molnar * be possible to get enough handler activations to overflow the stack. 1063905a36a2SIngo Molnar * Although unlikely, bugs of that kind are hard to track down, so we'd 1064905a36a2SIngo Molnar * like to avoid the possibility. 1065905a36a2SIngo Molnar * So, on entry to the handler we detect whether we interrupted an 1066905a36a2SIngo Molnar * existing activation in its critical region -- if so, we pop the current 1067905a36a2SIngo Molnar * activation and restart the handler using the previous one. 1068905a36a2SIngo Molnar */ 10694d732138SIngo MolnarENTRY(xen_do_hypervisor_callback) /* do_hypervisor_callback(struct *pt_regs) */ 10704d732138SIngo Molnar 1071905a36a2SIngo Molnar/* 1072905a36a2SIngo Molnar * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will 1073905a36a2SIngo Molnar * see the correct pointer to the pt_regs 1074905a36a2SIngo Molnar */ 10758c1f7558SJosh Poimboeuf UNWIND_HINT_FUNC 10764d732138SIngo Molnar movq %rdi, %rsp /* we don't return, adjust the stack frame */ 10778c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 10781d3e53e8SAndy Lutomirski 10791d3e53e8SAndy Lutomirski ENTER_IRQ_STACK old_rsp=%r10 1080905a36a2SIngo Molnar call xen_evtchn_do_upcall 10811d3e53e8SAndy Lutomirski LEAVE_IRQ_STACK 10821d3e53e8SAndy Lutomirski 1083905a36a2SIngo Molnar#ifndef CONFIG_PREEMPT 1084905a36a2SIngo Molnar call xen_maybe_preempt_hcall 1085905a36a2SIngo Molnar#endif 1086905a36a2SIngo Molnar jmp error_exit 1087905a36a2SIngo MolnarEND(xen_do_hypervisor_callback) 1088905a36a2SIngo Molnar 1089905a36a2SIngo Molnar/* 1090905a36a2SIngo Molnar * Hypervisor uses this for application faults while it executes. 1091905a36a2SIngo Molnar * We get here for two reasons: 1092905a36a2SIngo Molnar * 1. Fault while reloading DS, ES, FS or GS 1093905a36a2SIngo Molnar * 2. Fault while executing IRET 1094905a36a2SIngo Molnar * Category 1 we do not need to fix up as Xen has already reloaded all segment 1095905a36a2SIngo Molnar * registers that could be reloaded and zeroed the others. 1096905a36a2SIngo Molnar * Category 2 we fix up by killing the current process. We cannot use the 1097905a36a2SIngo Molnar * normal Linux return path in this case because if we use the IRET hypercall 1098905a36a2SIngo Molnar * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1099905a36a2SIngo Molnar * We distinguish between categories by comparing each saved segment register 1100905a36a2SIngo Molnar * with its current contents: any discrepancy means we in category 1. 1101905a36a2SIngo Molnar */ 1102905a36a2SIngo MolnarENTRY(xen_failsafe_callback) 11038c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 1104905a36a2SIngo Molnar movl %ds, %ecx 1105905a36a2SIngo Molnar cmpw %cx, 0x10(%rsp) 1106905a36a2SIngo Molnar jne 1f 1107905a36a2SIngo Molnar movl %es, %ecx 1108905a36a2SIngo Molnar cmpw %cx, 0x18(%rsp) 1109905a36a2SIngo Molnar jne 1f 1110905a36a2SIngo Molnar movl %fs, %ecx 1111905a36a2SIngo Molnar cmpw %cx, 0x20(%rsp) 1112905a36a2SIngo Molnar jne 1f 1113905a36a2SIngo Molnar movl %gs, %ecx 1114905a36a2SIngo Molnar cmpw %cx, 0x28(%rsp) 1115905a36a2SIngo Molnar jne 1f 1116905a36a2SIngo Molnar /* All segments match their saved values => Category 2 (Bad IRET). */ 1117905a36a2SIngo Molnar movq (%rsp), %rcx 1118905a36a2SIngo Molnar movq 8(%rsp), %r11 1119905a36a2SIngo Molnar addq $0x30, %rsp 1120905a36a2SIngo Molnar pushq $0 /* RIP */ 11218c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS offset=8 1122905a36a2SIngo Molnar jmp general_protection 1123905a36a2SIngo Molnar1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */ 1124905a36a2SIngo Molnar movq (%rsp), %rcx 1125905a36a2SIngo Molnar movq 8(%rsp), %r11 1126905a36a2SIngo Molnar addq $0x30, %rsp 11278c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 1128905a36a2SIngo Molnar pushq $-1 /* orig_ax = -1 => not a system call */ 11293f01daecSDominik Brodowski PUSH_AND_CLEAR_REGS 1130946c1911SJosh Poimboeuf ENCODE_FRAME_POINTER 1131905a36a2SIngo Molnar jmp error_exit 1132905a36a2SIngo MolnarEND(xen_failsafe_callback) 1133905a36a2SIngo Molnar 1134905a36a2SIngo Molnarapicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1135905a36a2SIngo Molnar xen_hvm_callback_vector xen_evtchn_do_upcall 1136905a36a2SIngo Molnar 1137905a36a2SIngo Molnar#endif /* CONFIG_XEN */ 1138905a36a2SIngo Molnar 1139905a36a2SIngo Molnar#if IS_ENABLED(CONFIG_HYPERV) 1140905a36a2SIngo Molnarapicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1141905a36a2SIngo Molnar hyperv_callback_vector hyperv_vector_handler 114293286261SVitaly Kuznetsov 114393286261SVitaly Kuznetsovapicinterrupt3 HYPERV_REENLIGHTENMENT_VECTOR \ 114493286261SVitaly Kuznetsov hyperv_reenlightenment_vector hyperv_reenlightenment_intr 1145248e742aSMichael Kelley 1146248e742aSMichael Kelleyapicinterrupt3 HYPERV_STIMER0_VECTOR \ 1147248e742aSMichael Kelley hv_stimer0_callback_vector hv_stimer0_vector_handler 1148905a36a2SIngo Molnar#endif /* CONFIG_HYPERV */ 1149905a36a2SIngo Molnar 1150905a36a2SIngo Molnaridtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK 1151d8ba61baSAndy Lutomirskiidtentry int3 do_int3 has_error_code=0 1152905a36a2SIngo Molnaridtentry stack_segment do_stack_segment has_error_code=1 11534d732138SIngo Molnar 1154905a36a2SIngo Molnar#ifdef CONFIG_XEN 115543e41110SJuergen Grossidtentry xennmi do_nmi has_error_code=0 11565878d5d6SJuergen Grossidtentry xendebug do_debug has_error_code=0 11575878d5d6SJuergen Grossidtentry xenint3 do_int3 has_error_code=0 1158905a36a2SIngo Molnar#endif 11594d732138SIngo Molnar 1160905a36a2SIngo Molnaridtentry general_protection do_general_protection has_error_code=1 116111a7ffb0SThomas Gleixneridtentry page_fault do_page_fault has_error_code=1 11624d732138SIngo Molnar 1163905a36a2SIngo Molnar#ifdef CONFIG_KVM_GUEST 1164905a36a2SIngo Molnaridtentry async_page_fault do_async_page_fault has_error_code=1 1165905a36a2SIngo Molnar#endif 11664d732138SIngo Molnar 1167905a36a2SIngo Molnar#ifdef CONFIG_X86_MCE 11686f41c34dSThomas Gleixneridtentry machine_check do_mce has_error_code=0 paranoid=1 1169905a36a2SIngo Molnar#endif 1170905a36a2SIngo Molnar 1171905a36a2SIngo Molnar/* 11729e809d15SDominik Brodowski * Save all registers in pt_regs, and switch gs if needed. 1173905a36a2SIngo Molnar * Use slow, but surefire "are we in kernel?" check. 1174905a36a2SIngo Molnar * Return: ebx=0: need swapgs on exit, ebx=1: otherwise 1175905a36a2SIngo Molnar */ 1176905a36a2SIngo MolnarENTRY(paranoid_entry) 11778c1f7558SJosh Poimboeuf UNWIND_HINT_FUNC 1178905a36a2SIngo Molnar cld 11799e809d15SDominik Brodowski PUSH_AND_CLEAR_REGS save_ret=1 11809e809d15SDominik Brodowski ENCODE_FRAME_POINTER 8 1181905a36a2SIngo Molnar movl $1, %ebx 1182905a36a2SIngo Molnar movl $MSR_GS_BASE, %ecx 1183905a36a2SIngo Molnar rdmsr 1184905a36a2SIngo Molnar testl %edx, %edx 1185905a36a2SIngo Molnar js 1f /* negative -> in kernel */ 1186905a36a2SIngo Molnar SWAPGS 1187905a36a2SIngo Molnar xorl %ebx, %ebx 11888a09317bSDave Hansen 11898a09317bSDave Hansen1: 11908a09317bSDave Hansen SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14 11918a09317bSDave Hansen 11928a09317bSDave Hansen ret 1193905a36a2SIngo MolnarEND(paranoid_entry) 1194905a36a2SIngo Molnar 1195905a36a2SIngo Molnar/* 1196905a36a2SIngo Molnar * "Paranoid" exit path from exception stack. This is invoked 1197905a36a2SIngo Molnar * only on return from non-NMI IST interrupts that came 1198905a36a2SIngo Molnar * from kernel space. 1199905a36a2SIngo Molnar * 1200905a36a2SIngo Molnar * We may be returning to very strange contexts (e.g. very early 1201905a36a2SIngo Molnar * in syscall entry), so checking for preemption here would 1202905a36a2SIngo Molnar * be complicated. Fortunately, we there's no good reason 1203905a36a2SIngo Molnar * to try to handle preemption here. 12044d732138SIngo Molnar * 12054d732138SIngo Molnar * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) 1206905a36a2SIngo Molnar */ 1207905a36a2SIngo MolnarENTRY(paranoid_exit) 12088c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 12092140a994SJan Beulich DISABLE_INTERRUPTS(CLBR_ANY) 1210905a36a2SIngo Molnar TRACE_IRQS_OFF_DEBUG 1211905a36a2SIngo Molnar testl %ebx, %ebx /* swapgs needed? */ 1212e5317832SAndy Lutomirski jnz .Lparanoid_exit_no_swapgs 1213905a36a2SIngo Molnar TRACE_IRQS_IRETQ 121421e94459SPeter Zijlstra RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 1215905a36a2SIngo Molnar SWAPGS_UNSAFE_STACK 1216e5317832SAndy Lutomirski jmp .Lparanoid_exit_restore 1217e5317832SAndy Lutomirski.Lparanoid_exit_no_swapgs: 1218905a36a2SIngo Molnar TRACE_IRQS_IRETQ_DEBUG 1219e4865757SIngo Molnar RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 1220e5317832SAndy Lutomirski.Lparanoid_exit_restore: 1221e5317832SAndy Lutomirski jmp restore_regs_and_return_to_kernel 1222905a36a2SIngo MolnarEND(paranoid_exit) 1223905a36a2SIngo Molnar 1224905a36a2SIngo Molnar/* 12259e809d15SDominik Brodowski * Save all registers in pt_regs, and switch GS if needed. 1226539f5113SAndy Lutomirski * Return: EBX=0: came from user mode; EBX=1: otherwise 1227905a36a2SIngo Molnar */ 1228905a36a2SIngo MolnarENTRY(error_entry) 12299e809d15SDominik Brodowski UNWIND_HINT_FUNC 1230905a36a2SIngo Molnar cld 12319e809d15SDominik Brodowski PUSH_AND_CLEAR_REGS save_ret=1 12329e809d15SDominik Brodowski ENCODE_FRAME_POINTER 8 1233905a36a2SIngo Molnar testb $3, CS+8(%rsp) 1234cb6f64edSAndy Lutomirski jz .Lerror_kernelspace 1235539f5113SAndy Lutomirski 1236cb6f64edSAndy Lutomirski /* 1237cb6f64edSAndy Lutomirski * We entered from user mode or we're pretending to have entered 1238cb6f64edSAndy Lutomirski * from user mode due to an IRET fault. 1239cb6f64edSAndy Lutomirski */ 1240905a36a2SIngo Molnar SWAPGS 12418a09317bSDave Hansen /* We have user CR3. Change to kernel CR3. */ 12428a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rax 1243539f5113SAndy Lutomirski 1244cb6f64edSAndy Lutomirski.Lerror_entry_from_usermode_after_swapgs: 12457f2590a1SAndy Lutomirski /* Put us onto the real thread stack. */ 12467f2590a1SAndy Lutomirski popq %r12 /* save return addr in %12 */ 12477f2590a1SAndy Lutomirski movq %rsp, %rdi /* arg0 = pt_regs pointer */ 12487f2590a1SAndy Lutomirski call sync_regs 12497f2590a1SAndy Lutomirski movq %rax, %rsp /* switch stack */ 12507f2590a1SAndy Lutomirski ENCODE_FRAME_POINTER 12517f2590a1SAndy Lutomirski pushq %r12 12527f2590a1SAndy Lutomirski 1253f1075053SAndy Lutomirski /* 1254f1075053SAndy Lutomirski * We need to tell lockdep that IRQs are off. We can't do this until 1255f1075053SAndy Lutomirski * we fix gsbase, and we should do it before enter_from_user_mode 1256f1075053SAndy Lutomirski * (which can take locks). 1257f1075053SAndy Lutomirski */ 1258f1075053SAndy Lutomirski TRACE_IRQS_OFF 1259478dc89cSAndy Lutomirski CALL_enter_from_user_mode 1260f1075053SAndy Lutomirski ret 126102bc7768SAndy Lutomirski 1262cb6f64edSAndy Lutomirski.Lerror_entry_done: 1263905a36a2SIngo Molnar TRACE_IRQS_OFF 1264905a36a2SIngo Molnar ret 1265905a36a2SIngo Molnar 1266905a36a2SIngo Molnar /* 1267905a36a2SIngo Molnar * There are two places in the kernel that can potentially fault with 1268905a36a2SIngo Molnar * usergs. Handle them here. B stepping K8s sometimes report a 1269905a36a2SIngo Molnar * truncated RIP for IRET exceptions returning to compat mode. Check 1270905a36a2SIngo Molnar * for these here too. 1271905a36a2SIngo Molnar */ 1272cb6f64edSAndy Lutomirski.Lerror_kernelspace: 1273905a36a2SIngo Molnar incl %ebx 1274905a36a2SIngo Molnar leaq native_irq_return_iret(%rip), %rcx 1275905a36a2SIngo Molnar cmpq %rcx, RIP+8(%rsp) 1276cb6f64edSAndy Lutomirski je .Lerror_bad_iret 1277905a36a2SIngo Molnar movl %ecx, %eax /* zero extend */ 1278905a36a2SIngo Molnar cmpq %rax, RIP+8(%rsp) 1279cb6f64edSAndy Lutomirski je .Lbstep_iret 128042c748bbSBorislav Petkov cmpq $.Lgs_change, RIP+8(%rsp) 1281cb6f64edSAndy Lutomirski jne .Lerror_entry_done 1282539f5113SAndy Lutomirski 1283539f5113SAndy Lutomirski /* 128442c748bbSBorislav Petkov * hack: .Lgs_change can fail with user gsbase. If this happens, fix up 1285539f5113SAndy Lutomirski * gsbase and proceed. We'll fix up the exception and land in 128642c748bbSBorislav Petkov * .Lgs_change's error handler with kernel gsbase. 1287539f5113SAndy Lutomirski */ 12882fa5f04fSWanpeng Li SWAPGS 12898a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rax 12902fa5f04fSWanpeng Li jmp .Lerror_entry_done 1291905a36a2SIngo Molnar 1292cb6f64edSAndy Lutomirski.Lbstep_iret: 1293905a36a2SIngo Molnar /* Fix truncated RIP */ 1294905a36a2SIngo Molnar movq %rcx, RIP+8(%rsp) 1295905a36a2SIngo Molnar /* fall through */ 1296905a36a2SIngo Molnar 1297cb6f64edSAndy Lutomirski.Lerror_bad_iret: 1298539f5113SAndy Lutomirski /* 12998a09317bSDave Hansen * We came from an IRET to user mode, so we have user 13008a09317bSDave Hansen * gsbase and CR3. Switch to kernel gsbase and CR3: 1301539f5113SAndy Lutomirski */ 1302905a36a2SIngo Molnar SWAPGS 13038a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rax 1304539f5113SAndy Lutomirski 1305539f5113SAndy Lutomirski /* 1306539f5113SAndy Lutomirski * Pretend that the exception came from user mode: set up pt_regs 1307539f5113SAndy Lutomirski * as if we faulted immediately after IRET and clear EBX so that 1308539f5113SAndy Lutomirski * error_exit knows that we will be returning to user mode. 1309539f5113SAndy Lutomirski */ 1310905a36a2SIngo Molnar mov %rsp, %rdi 1311905a36a2SIngo Molnar call fixup_bad_iret 1312905a36a2SIngo Molnar mov %rax, %rsp 1313539f5113SAndy Lutomirski decl %ebx 1314cb6f64edSAndy Lutomirski jmp .Lerror_entry_from_usermode_after_swapgs 1315905a36a2SIngo MolnarEND(error_entry) 1316905a36a2SIngo Molnar 1317905a36a2SIngo Molnar 1318539f5113SAndy Lutomirski/* 131975ca5b22SNicolas Iooss * On entry, EBX is a "return to kernel mode" flag: 1320539f5113SAndy Lutomirski * 1: already in kernel mode, don't need SWAPGS 1321539f5113SAndy Lutomirski * 0: user gsbase is loaded, we need SWAPGS and standard preparation for return to usermode 1322539f5113SAndy Lutomirski */ 1323905a36a2SIngo MolnarENTRY(error_exit) 13248c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 13252140a994SJan Beulich DISABLE_INTERRUPTS(CLBR_ANY) 1326905a36a2SIngo Molnar TRACE_IRQS_OFF 13272140a994SJan Beulich testl %ebx, %ebx 1328905a36a2SIngo Molnar jnz retint_kernel 1329905a36a2SIngo Molnar jmp retint_user 1330905a36a2SIngo MolnarEND(error_exit) 1331905a36a2SIngo Molnar 1332929bacecSAndy Lutomirski/* 1333929bacecSAndy Lutomirski * Runs on exception stack. Xen PV does not go through this path at all, 1334929bacecSAndy Lutomirski * so we can use real assembly here. 13358a09317bSDave Hansen * 13368a09317bSDave Hansen * Registers: 13378a09317bSDave Hansen * %r14: Used to save/restore the CR3 of the interrupted context 13388a09317bSDave Hansen * when PAGE_TABLE_ISOLATION is in use. Do not clobber. 1339929bacecSAndy Lutomirski */ 1340905a36a2SIngo MolnarENTRY(nmi) 13418c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 1342929bacecSAndy Lutomirski 1343fc57a7c6SAndy Lutomirski /* 1344905a36a2SIngo Molnar * We allow breakpoints in NMIs. If a breakpoint occurs, then 1345905a36a2SIngo Molnar * the iretq it performs will take us out of NMI context. 1346905a36a2SIngo Molnar * This means that we can have nested NMIs where the next 1347905a36a2SIngo Molnar * NMI is using the top of the stack of the previous NMI. We 1348905a36a2SIngo Molnar * can't let it execute because the nested NMI will corrupt the 1349905a36a2SIngo Molnar * stack of the previous NMI. NMI handlers are not re-entrant 1350905a36a2SIngo Molnar * anyway. 1351905a36a2SIngo Molnar * 1352905a36a2SIngo Molnar * To handle this case we do the following: 1353905a36a2SIngo Molnar * Check the a special location on the stack that contains 1354905a36a2SIngo Molnar * a variable that is set when NMIs are executing. 1355905a36a2SIngo Molnar * The interrupted task's stack is also checked to see if it 1356905a36a2SIngo Molnar * is an NMI stack. 1357905a36a2SIngo Molnar * If the variable is not set and the stack is not the NMI 1358905a36a2SIngo Molnar * stack then: 1359905a36a2SIngo Molnar * o Set the special variable on the stack 13600b22930eSAndy Lutomirski * o Copy the interrupt frame into an "outermost" location on the 13610b22930eSAndy Lutomirski * stack 13620b22930eSAndy Lutomirski * o Copy the interrupt frame into an "iret" location on the stack 1363905a36a2SIngo Molnar * o Continue processing the NMI 1364905a36a2SIngo Molnar * If the variable is set or the previous stack is the NMI stack: 13650b22930eSAndy Lutomirski * o Modify the "iret" location to jump to the repeat_nmi 1366905a36a2SIngo Molnar * o return back to the first NMI 1367905a36a2SIngo Molnar * 1368905a36a2SIngo Molnar * Now on exit of the first NMI, we first clear the stack variable 1369905a36a2SIngo Molnar * The NMI stack will tell any nested NMIs at that point that it is 1370905a36a2SIngo Molnar * nested. Then we pop the stack normally with iret, and if there was 1371905a36a2SIngo Molnar * a nested NMI that updated the copy interrupt stack frame, a 1372905a36a2SIngo Molnar * jump will be made to the repeat_nmi code that will handle the second 1373905a36a2SIngo Molnar * NMI. 13749b6e6a83SAndy Lutomirski * 13759b6e6a83SAndy Lutomirski * However, espfix prevents us from directly returning to userspace 13769b6e6a83SAndy Lutomirski * with a single IRET instruction. Similarly, IRET to user mode 13779b6e6a83SAndy Lutomirski * can fault. We therefore handle NMIs from user space like 13789b6e6a83SAndy Lutomirski * other IST entries. 1379905a36a2SIngo Molnar */ 1380905a36a2SIngo Molnar 1381e93c1730SAndy Lutomirski ASM_CLAC 1382e93c1730SAndy Lutomirski 1383905a36a2SIngo Molnar /* Use %rdx as our temp variable throughout */ 1384905a36a2SIngo Molnar pushq %rdx 1385905a36a2SIngo Molnar 13869b6e6a83SAndy Lutomirski testb $3, CS-RIP+8(%rsp) 13879b6e6a83SAndy Lutomirski jz .Lnmi_from_kernel 1388905a36a2SIngo Molnar 1389905a36a2SIngo Molnar /* 13909b6e6a83SAndy Lutomirski * NMI from user mode. We need to run on the thread stack, but we 13919b6e6a83SAndy Lutomirski * can't go through the normal entry paths: NMIs are masked, and 13929b6e6a83SAndy Lutomirski * we don't want to enable interrupts, because then we'll end 13939b6e6a83SAndy Lutomirski * up in an awkward situation in which IRQs are on but NMIs 13949b6e6a83SAndy Lutomirski * are off. 139583c133cfSAndy Lutomirski * 139683c133cfSAndy Lutomirski * We also must not push anything to the stack before switching 139783c133cfSAndy Lutomirski * stacks lest we corrupt the "NMI executing" variable. 13989b6e6a83SAndy Lutomirski */ 13999b6e6a83SAndy Lutomirski 1400929bacecSAndy Lutomirski swapgs 14019b6e6a83SAndy Lutomirski cld 14028a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx 14039b6e6a83SAndy Lutomirski movq %rsp, %rdx 14049b6e6a83SAndy Lutomirski movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 14058c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS base=%rdx offset=8 14069b6e6a83SAndy Lutomirski pushq 5*8(%rdx) /* pt_regs->ss */ 14079b6e6a83SAndy Lutomirski pushq 4*8(%rdx) /* pt_regs->rsp */ 14089b6e6a83SAndy Lutomirski pushq 3*8(%rdx) /* pt_regs->flags */ 14099b6e6a83SAndy Lutomirski pushq 2*8(%rdx) /* pt_regs->cs */ 14109b6e6a83SAndy Lutomirski pushq 1*8(%rdx) /* pt_regs->rip */ 14118c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 14129b6e6a83SAndy Lutomirski pushq $-1 /* pt_regs->orig_ax */ 141330907fd1SDominik Brodowski PUSH_AND_CLEAR_REGS rdx=(%rdx) 1414946c1911SJosh Poimboeuf ENCODE_FRAME_POINTER 14159b6e6a83SAndy Lutomirski 14169b6e6a83SAndy Lutomirski /* 14179b6e6a83SAndy Lutomirski * At this point we no longer need to worry about stack damage 14189b6e6a83SAndy Lutomirski * due to nesting -- we're on the normal thread stack and we're 14199b6e6a83SAndy Lutomirski * done with the NMI stack. 14209b6e6a83SAndy Lutomirski */ 14219b6e6a83SAndy Lutomirski 14229b6e6a83SAndy Lutomirski movq %rsp, %rdi 14239b6e6a83SAndy Lutomirski movq $-1, %rsi 14249b6e6a83SAndy Lutomirski call do_nmi 14259b6e6a83SAndy Lutomirski 14269b6e6a83SAndy Lutomirski /* 14279b6e6a83SAndy Lutomirski * Return back to user mode. We must *not* do the normal exit 1428946c1911SJosh Poimboeuf * work, because we don't want to enable interrupts. 14299b6e6a83SAndy Lutomirski */ 14308a055d7fSAndy Lutomirski jmp swapgs_restore_regs_and_return_to_usermode 14319b6e6a83SAndy Lutomirski 14329b6e6a83SAndy Lutomirski.Lnmi_from_kernel: 14339b6e6a83SAndy Lutomirski /* 14340b22930eSAndy Lutomirski * Here's what our stack frame will look like: 14350b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14360b22930eSAndy Lutomirski * | original SS | 14370b22930eSAndy Lutomirski * | original Return RSP | 14380b22930eSAndy Lutomirski * | original RFLAGS | 14390b22930eSAndy Lutomirski * | original CS | 14400b22930eSAndy Lutomirski * | original RIP | 14410b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14420b22930eSAndy Lutomirski * | temp storage for rdx | 14430b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14440b22930eSAndy Lutomirski * | "NMI executing" variable | 14450b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14460b22930eSAndy Lutomirski * | iret SS } Copied from "outermost" frame | 14470b22930eSAndy Lutomirski * | iret Return RSP } on each loop iteration; overwritten | 14480b22930eSAndy Lutomirski * | iret RFLAGS } by a nested NMI to force another | 14490b22930eSAndy Lutomirski * | iret CS } iteration if needed. | 14500b22930eSAndy Lutomirski * | iret RIP } | 14510b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14520b22930eSAndy Lutomirski * | outermost SS } initialized in first_nmi; | 14530b22930eSAndy Lutomirski * | outermost Return RSP } will not be changed before | 14540b22930eSAndy Lutomirski * | outermost RFLAGS } NMI processing is done. | 14550b22930eSAndy Lutomirski * | outermost CS } Copied to "iret" frame on each | 14560b22930eSAndy Lutomirski * | outermost RIP } iteration. | 14570b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14580b22930eSAndy Lutomirski * | pt_regs | 14590b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14600b22930eSAndy Lutomirski * 14610b22930eSAndy Lutomirski * The "original" frame is used by hardware. Before re-enabling 14620b22930eSAndy Lutomirski * NMIs, we need to be done with it, and we need to leave enough 14630b22930eSAndy Lutomirski * space for the asm code here. 14640b22930eSAndy Lutomirski * 14650b22930eSAndy Lutomirski * We return by executing IRET while RSP points to the "iret" frame. 14660b22930eSAndy Lutomirski * That will either return for real or it will loop back into NMI 14670b22930eSAndy Lutomirski * processing. 14680b22930eSAndy Lutomirski * 14690b22930eSAndy Lutomirski * The "outermost" frame is copied to the "iret" frame on each 14700b22930eSAndy Lutomirski * iteration of the loop, so each iteration starts with the "iret" 14710b22930eSAndy Lutomirski * frame pointing to the final return target. 14720b22930eSAndy Lutomirski */ 14730b22930eSAndy Lutomirski 14740b22930eSAndy Lutomirski /* 14750b22930eSAndy Lutomirski * Determine whether we're a nested NMI. 14760b22930eSAndy Lutomirski * 1477a27507caSAndy Lutomirski * If we interrupted kernel code between repeat_nmi and 1478a27507caSAndy Lutomirski * end_repeat_nmi, then we are a nested NMI. We must not 1479a27507caSAndy Lutomirski * modify the "iret" frame because it's being written by 1480a27507caSAndy Lutomirski * the outer NMI. That's okay; the outer NMI handler is 1481a27507caSAndy Lutomirski * about to about to call do_nmi anyway, so we can just 1482a27507caSAndy Lutomirski * resume the outer NMI. 1483a27507caSAndy Lutomirski */ 1484a27507caSAndy Lutomirski 1485a27507caSAndy Lutomirski movq $repeat_nmi, %rdx 1486a27507caSAndy Lutomirski cmpq 8(%rsp), %rdx 1487a27507caSAndy Lutomirski ja 1f 1488a27507caSAndy Lutomirski movq $end_repeat_nmi, %rdx 1489a27507caSAndy Lutomirski cmpq 8(%rsp), %rdx 1490a27507caSAndy Lutomirski ja nested_nmi_out 1491a27507caSAndy Lutomirski1: 1492a27507caSAndy Lutomirski 1493a27507caSAndy Lutomirski /* 1494a27507caSAndy Lutomirski * Now check "NMI executing". If it's set, then we're nested. 14950b22930eSAndy Lutomirski * This will not detect if we interrupted an outer NMI just 14960b22930eSAndy Lutomirski * before IRET. 1497905a36a2SIngo Molnar */ 1498905a36a2SIngo Molnar cmpl $1, -8(%rsp) 1499905a36a2SIngo Molnar je nested_nmi 1500905a36a2SIngo Molnar 1501905a36a2SIngo Molnar /* 15020b22930eSAndy Lutomirski * Now test if the previous stack was an NMI stack. This covers 15030b22930eSAndy Lutomirski * the case where we interrupt an outer NMI after it clears 1504810bc075SAndy Lutomirski * "NMI executing" but before IRET. We need to be careful, though: 1505810bc075SAndy Lutomirski * there is one case in which RSP could point to the NMI stack 1506810bc075SAndy Lutomirski * despite there being no NMI active: naughty userspace controls 1507810bc075SAndy Lutomirski * RSP at the very beginning of the SYSCALL targets. We can 1508810bc075SAndy Lutomirski * pull a fast one on naughty userspace, though: we program 1509810bc075SAndy Lutomirski * SYSCALL to mask DF, so userspace cannot cause DF to be set 1510810bc075SAndy Lutomirski * if it controls the kernel's RSP. We set DF before we clear 1511810bc075SAndy Lutomirski * "NMI executing". 1512905a36a2SIngo Molnar */ 1513905a36a2SIngo Molnar lea 6*8(%rsp), %rdx 1514905a36a2SIngo Molnar /* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */ 1515905a36a2SIngo Molnar cmpq %rdx, 4*8(%rsp) 1516905a36a2SIngo Molnar /* If the stack pointer is above the NMI stack, this is a normal NMI */ 1517905a36a2SIngo Molnar ja first_nmi 15184d732138SIngo Molnar 1519905a36a2SIngo Molnar subq $EXCEPTION_STKSZ, %rdx 1520905a36a2SIngo Molnar cmpq %rdx, 4*8(%rsp) 1521905a36a2SIngo Molnar /* If it is below the NMI stack, it is a normal NMI */ 1522905a36a2SIngo Molnar jb first_nmi 1523810bc075SAndy Lutomirski 1524810bc075SAndy Lutomirski /* Ah, it is within the NMI stack. */ 1525810bc075SAndy Lutomirski 1526810bc075SAndy Lutomirski testb $(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp) 1527810bc075SAndy Lutomirski jz first_nmi /* RSP was user controlled. */ 1528810bc075SAndy Lutomirski 1529810bc075SAndy Lutomirski /* This is a nested NMI. */ 1530905a36a2SIngo Molnar 1531905a36a2SIngo Molnarnested_nmi: 1532905a36a2SIngo Molnar /* 15330b22930eSAndy Lutomirski * Modify the "iret" frame to point to repeat_nmi, forcing another 15340b22930eSAndy Lutomirski * iteration of NMI handling. 1535905a36a2SIngo Molnar */ 153623a781e9SAndy Lutomirski subq $8, %rsp 1537905a36a2SIngo Molnar leaq -10*8(%rsp), %rdx 1538905a36a2SIngo Molnar pushq $__KERNEL_DS 1539905a36a2SIngo Molnar pushq %rdx 1540905a36a2SIngo Molnar pushfq 1541905a36a2SIngo Molnar pushq $__KERNEL_CS 1542905a36a2SIngo Molnar pushq $repeat_nmi 1543905a36a2SIngo Molnar 1544905a36a2SIngo Molnar /* Put stack back */ 1545905a36a2SIngo Molnar addq $(6*8), %rsp 1546905a36a2SIngo Molnar 1547905a36a2SIngo Molnarnested_nmi_out: 1548905a36a2SIngo Molnar popq %rdx 1549905a36a2SIngo Molnar 15500b22930eSAndy Lutomirski /* We are returning to kernel mode, so this cannot result in a fault. */ 1551929bacecSAndy Lutomirski iretq 1552905a36a2SIngo Molnar 1553905a36a2SIngo Molnarfirst_nmi: 15540b22930eSAndy Lutomirski /* Restore rdx. */ 1555905a36a2SIngo Molnar movq (%rsp), %rdx 1556905a36a2SIngo Molnar 155736f1a77bSAndy Lutomirski /* Make room for "NMI executing". */ 155836f1a77bSAndy Lutomirski pushq $0 1559905a36a2SIngo Molnar 15600b22930eSAndy Lutomirski /* Leave room for the "iret" frame */ 1561905a36a2SIngo Molnar subq $(5*8), %rsp 1562905a36a2SIngo Molnar 15630b22930eSAndy Lutomirski /* Copy the "original" frame to the "outermost" frame */ 1564905a36a2SIngo Molnar .rept 5 1565905a36a2SIngo Molnar pushq 11*8(%rsp) 1566905a36a2SIngo Molnar .endr 15678c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 1568905a36a2SIngo Molnar 1569905a36a2SIngo Molnar /* Everything up to here is safe from nested NMIs */ 1570905a36a2SIngo Molnar 1571a97439aaSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 1572a97439aaSAndy Lutomirski /* 1573a97439aaSAndy Lutomirski * For ease of testing, unmask NMIs right away. Disabled by 1574a97439aaSAndy Lutomirski * default because IRET is very expensive. 1575a97439aaSAndy Lutomirski */ 1576a97439aaSAndy Lutomirski pushq $0 /* SS */ 1577a97439aaSAndy Lutomirski pushq %rsp /* RSP (minus 8 because of the previous push) */ 1578a97439aaSAndy Lutomirski addq $8, (%rsp) /* Fix up RSP */ 1579a97439aaSAndy Lutomirski pushfq /* RFLAGS */ 1580a97439aaSAndy Lutomirski pushq $__KERNEL_CS /* CS */ 1581a97439aaSAndy Lutomirski pushq $1f /* RIP */ 1582929bacecSAndy Lutomirski iretq /* continues at repeat_nmi below */ 15838c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 1584a97439aaSAndy Lutomirski1: 1585a97439aaSAndy Lutomirski#endif 1586a97439aaSAndy Lutomirski 15870b22930eSAndy Lutomirskirepeat_nmi: 1588905a36a2SIngo Molnar /* 1589905a36a2SIngo Molnar * If there was a nested NMI, the first NMI's iret will return 1590905a36a2SIngo Molnar * here. But NMIs are still enabled and we can take another 1591905a36a2SIngo Molnar * nested NMI. The nested NMI checks the interrupted RIP to see 1592905a36a2SIngo Molnar * if it is between repeat_nmi and end_repeat_nmi, and if so 1593905a36a2SIngo Molnar * it will just return, as we are about to repeat an NMI anyway. 1594905a36a2SIngo Molnar * This makes it safe to copy to the stack frame that a nested 1595905a36a2SIngo Molnar * NMI will update. 15960b22930eSAndy Lutomirski * 15970b22930eSAndy Lutomirski * RSP is pointing to "outermost RIP". gsbase is unknown, but, if 15980b22930eSAndy Lutomirski * we're repeating an NMI, gsbase has the same value that it had on 15990b22930eSAndy Lutomirski * the first iteration. paranoid_entry will load the kernel 160036f1a77bSAndy Lutomirski * gsbase if needed before we call do_nmi. "NMI executing" 160136f1a77bSAndy Lutomirski * is zero. 1602905a36a2SIngo Molnar */ 160336f1a77bSAndy Lutomirski movq $1, 10*8(%rsp) /* Set "NMI executing". */ 1604905a36a2SIngo Molnar 16050b22930eSAndy Lutomirski /* 16060b22930eSAndy Lutomirski * Copy the "outermost" frame to the "iret" frame. NMIs that nest 16070b22930eSAndy Lutomirski * here must not modify the "iret" frame while we're writing to 16080b22930eSAndy Lutomirski * it or it will end up containing garbage. 16090b22930eSAndy Lutomirski */ 1610905a36a2SIngo Molnar addq $(10*8), %rsp 1611905a36a2SIngo Molnar .rept 5 1612905a36a2SIngo Molnar pushq -6*8(%rsp) 1613905a36a2SIngo Molnar .endr 1614905a36a2SIngo Molnar subq $(5*8), %rsp 1615905a36a2SIngo Molnarend_repeat_nmi: 1616905a36a2SIngo Molnar 1617905a36a2SIngo Molnar /* 16180b22930eSAndy Lutomirski * Everything below this point can be preempted by a nested NMI. 16190b22930eSAndy Lutomirski * If this happens, then the inner NMI will change the "iret" 16200b22930eSAndy Lutomirski * frame to point back to repeat_nmi. 1621905a36a2SIngo Molnar */ 1622905a36a2SIngo Molnar pushq $-1 /* ORIG_RAX: no syscall to restart */ 1623905a36a2SIngo Molnar 1624905a36a2SIngo Molnar /* 1625905a36a2SIngo Molnar * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit 1626905a36a2SIngo Molnar * as we should not be calling schedule in NMI context. 1627905a36a2SIngo Molnar * Even with normal interrupts enabled. An NMI should not be 1628905a36a2SIngo Molnar * setting NEED_RESCHED or anything that normal interrupts and 1629905a36a2SIngo Molnar * exceptions might do. 1630905a36a2SIngo Molnar */ 1631905a36a2SIngo Molnar call paranoid_entry 16328c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 1633905a36a2SIngo Molnar 1634905a36a2SIngo Molnar /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ 1635905a36a2SIngo Molnar movq %rsp, %rdi 1636905a36a2SIngo Molnar movq $-1, %rsi 1637905a36a2SIngo Molnar call do_nmi 1638905a36a2SIngo Molnar 163921e94459SPeter Zijlstra RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 16408a09317bSDave Hansen 1641905a36a2SIngo Molnar testl %ebx, %ebx /* swapgs needed? */ 1642905a36a2SIngo Molnar jnz nmi_restore 1643905a36a2SIngo Molnarnmi_swapgs: 1644905a36a2SIngo Molnar SWAPGS_UNSAFE_STACK 1645905a36a2SIngo Molnarnmi_restore: 1646502af0d7SDominik Brodowski POP_REGS 16470b22930eSAndy Lutomirski 1648471ee483SAndy Lutomirski /* 1649471ee483SAndy Lutomirski * Skip orig_ax and the "outermost" frame to point RSP at the "iret" 1650471ee483SAndy Lutomirski * at the "iret" frame. 1651471ee483SAndy Lutomirski */ 1652471ee483SAndy Lutomirski addq $6*8, %rsp 1653905a36a2SIngo Molnar 1654810bc075SAndy Lutomirski /* 1655810bc075SAndy Lutomirski * Clear "NMI executing". Set DF first so that we can easily 1656810bc075SAndy Lutomirski * distinguish the remaining code between here and IRET from 1657929bacecSAndy Lutomirski * the SYSCALL entry and exit paths. 1658929bacecSAndy Lutomirski * 1659929bacecSAndy Lutomirski * We arguably should just inspect RIP instead, but I (Andy) wrote 1660929bacecSAndy Lutomirski * this code when I had the misapprehension that Xen PV supported 1661929bacecSAndy Lutomirski * NMIs, and Xen PV would break that approach. 1662810bc075SAndy Lutomirski */ 1663810bc075SAndy Lutomirski std 1664810bc075SAndy Lutomirski movq $0, 5*8(%rsp) /* clear "NMI executing" */ 16650b22930eSAndy Lutomirski 16660b22930eSAndy Lutomirski /* 1667929bacecSAndy Lutomirski * iretq reads the "iret" frame and exits the NMI stack in a 1668929bacecSAndy Lutomirski * single instruction. We are returning to kernel mode, so this 1669929bacecSAndy Lutomirski * cannot result in a fault. Similarly, we don't need to worry 1670929bacecSAndy Lutomirski * about espfix64 on the way back to kernel mode. 16710b22930eSAndy Lutomirski */ 1672929bacecSAndy Lutomirski iretq 1673905a36a2SIngo MolnarEND(nmi) 1674905a36a2SIngo Molnar 1675905a36a2SIngo MolnarENTRY(ignore_sysret) 16768c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 1677905a36a2SIngo Molnar mov $-ENOSYS, %eax 1678905a36a2SIngo Molnar sysret 1679905a36a2SIngo MolnarEND(ignore_sysret) 16802deb4be2SAndy Lutomirski 16812deb4be2SAndy LutomirskiENTRY(rewind_stack_do_exit) 16828c1f7558SJosh Poimboeuf UNWIND_HINT_FUNC 16832deb4be2SAndy Lutomirski /* Prevent any naive code from trying to unwind to our caller. */ 16842deb4be2SAndy Lutomirski xorl %ebp, %ebp 16852deb4be2SAndy Lutomirski 16862deb4be2SAndy Lutomirski movq PER_CPU_VAR(cpu_current_top_of_stack), %rax 16878c1f7558SJosh Poimboeuf leaq -PTREGS_SIZE(%rax), %rsp 16888c1f7558SJosh Poimboeuf UNWIND_HINT_FUNC sp_offset=PTREGS_SIZE 16892deb4be2SAndy Lutomirski 16902deb4be2SAndy Lutomirski call do_exit 16912deb4be2SAndy LutomirskiEND(rewind_stack_do_exit) 1692