1b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 2905a36a2SIngo Molnar/* 3905a36a2SIngo Molnar * linux/arch/x86_64/entry.S 4905a36a2SIngo Molnar * 5905a36a2SIngo Molnar * Copyright (C) 1991, 1992 Linus Torvalds 6905a36a2SIngo Molnar * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs 7905a36a2SIngo Molnar * Copyright (C) 2000 Pavel Machek <pavel@suse.cz> 84d732138SIngo Molnar * 9905a36a2SIngo Molnar * entry.S contains the system-call and fault low-level handling routines. 10905a36a2SIngo Molnar * 11cb1aaebeSMauro Carvalho Chehab * Some of this is documented in Documentation/x86/entry_64.rst 12905a36a2SIngo Molnar * 13905a36a2SIngo Molnar * A note on terminology: 14905a36a2SIngo Molnar * - iret frame: Architecture defined interrupt frame from SS to RIP 15905a36a2SIngo Molnar * at the top of the kernel process stack. 16905a36a2SIngo Molnar * 17905a36a2SIngo Molnar * Some macro usage: 186dcc5627SJiri Slaby * - SYM_FUNC_START/END:Define functions in the symbol table. 194d732138SIngo Molnar * - TRACE_IRQ_*: Trace hardirq state for lock debugging. 204d732138SIngo Molnar * - idtentry: Define exception entry points. 21905a36a2SIngo Molnar */ 22905a36a2SIngo Molnar#include <linux/linkage.h> 23905a36a2SIngo Molnar#include <asm/segment.h> 24905a36a2SIngo Molnar#include <asm/cache.h> 25905a36a2SIngo Molnar#include <asm/errno.h> 26905a36a2SIngo Molnar#include <asm/asm-offsets.h> 27905a36a2SIngo Molnar#include <asm/msr.h> 28905a36a2SIngo Molnar#include <asm/unistd.h> 29905a36a2SIngo Molnar#include <asm/thread_info.h> 30905a36a2SIngo Molnar#include <asm/hw_irq.h> 31905a36a2SIngo Molnar#include <asm/page_types.h> 32905a36a2SIngo Molnar#include <asm/irqflags.h> 33905a36a2SIngo Molnar#include <asm/paravirt.h> 34905a36a2SIngo Molnar#include <asm/percpu.h> 35905a36a2SIngo Molnar#include <asm/asm.h> 36905a36a2SIngo Molnar#include <asm/smap.h> 37905a36a2SIngo Molnar#include <asm/pgtable_types.h> 38784d5699SAl Viro#include <asm/export.h> 398c1f7558SJosh Poimboeuf#include <asm/frame.h> 40cfa82a00SThomas Gleixner#include <asm/trapnr.h> 412641f08bSDavid Woodhouse#include <asm/nospec-branch.h> 42905a36a2SIngo Molnar#include <linux/err.h> 43905a36a2SIngo Molnar 446fd166aaSPeter Zijlstra#include "calling.h" 456fd166aaSPeter Zijlstra 46905a36a2SIngo Molnar.code64 47905a36a2SIngo Molnar.section .entry.text, "ax" 48905a36a2SIngo Molnar 49905a36a2SIngo Molnar#ifdef CONFIG_PARAVIRT 50bc7b11c0SJiri SlabySYM_CODE_START(native_usergs_sysret64) 518c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 52905a36a2SIngo Molnar swapgs 53905a36a2SIngo Molnar sysretq 54bc7b11c0SJiri SlabySYM_CODE_END(native_usergs_sysret64) 55905a36a2SIngo Molnar#endif /* CONFIG_PARAVIRT */ 56905a36a2SIngo Molnar 57ca37e57bSAndy Lutomirski.macro TRACE_IRQS_FLAGS flags:req 58905a36a2SIngo Molnar#ifdef CONFIG_TRACE_IRQFLAGS 59a368d7fdSJan Beulich btl $9, \flags /* interrupts off? */ 60905a36a2SIngo Molnar jnc 1f 61905a36a2SIngo Molnar TRACE_IRQS_ON 62905a36a2SIngo Molnar1: 63905a36a2SIngo Molnar#endif 64905a36a2SIngo Molnar.endm 65905a36a2SIngo Molnar 66ca37e57bSAndy Lutomirski.macro TRACE_IRQS_IRETQ 67ca37e57bSAndy Lutomirski TRACE_IRQS_FLAGS EFLAGS(%rsp) 68ca37e57bSAndy Lutomirski.endm 69ca37e57bSAndy Lutomirski 70905a36a2SIngo Molnar/* 71905a36a2SIngo Molnar * When dynamic function tracer is enabled it will add a breakpoint 72905a36a2SIngo Molnar * to all locations that it is about to modify, sync CPUs, update 73905a36a2SIngo Molnar * all the code, sync CPUs, then remove the breakpoints. In this time 74905a36a2SIngo Molnar * if lockdep is enabled, it might jump back into the debug handler 75905a36a2SIngo Molnar * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF). 76905a36a2SIngo Molnar * 77905a36a2SIngo Molnar * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to 78905a36a2SIngo Molnar * make sure the stack pointer does not get reset back to the top 79905a36a2SIngo Molnar * of the debug stack, and instead just reuses the current stack. 80905a36a2SIngo Molnar */ 81905a36a2SIngo Molnar#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS) 82905a36a2SIngo Molnar 83905a36a2SIngo Molnar.macro TRACE_IRQS_OFF_DEBUG 84905a36a2SIngo Molnar call debug_stack_set_zero 85905a36a2SIngo Molnar TRACE_IRQS_OFF 86905a36a2SIngo Molnar call debug_stack_reset 87905a36a2SIngo Molnar.endm 88905a36a2SIngo Molnar 89905a36a2SIngo Molnar.macro TRACE_IRQS_ON_DEBUG 90905a36a2SIngo Molnar call debug_stack_set_zero 91905a36a2SIngo Molnar TRACE_IRQS_ON 92905a36a2SIngo Molnar call debug_stack_reset 93905a36a2SIngo Molnar.endm 94905a36a2SIngo Molnar 95905a36a2SIngo Molnar.macro TRACE_IRQS_IRETQ_DEBUG 966709812fSJan Beulich btl $9, EFLAGS(%rsp) /* interrupts off? */ 97905a36a2SIngo Molnar jnc 1f 98905a36a2SIngo Molnar TRACE_IRQS_ON_DEBUG 99905a36a2SIngo Molnar1: 100905a36a2SIngo Molnar.endm 101905a36a2SIngo Molnar 102905a36a2SIngo Molnar#else 103905a36a2SIngo Molnar# define TRACE_IRQS_OFF_DEBUG TRACE_IRQS_OFF 104905a36a2SIngo Molnar# define TRACE_IRQS_ON_DEBUG TRACE_IRQS_ON 105905a36a2SIngo Molnar# define TRACE_IRQS_IRETQ_DEBUG TRACE_IRQS_IRETQ 106905a36a2SIngo Molnar#endif 107905a36a2SIngo Molnar 108905a36a2SIngo Molnar/* 1094d732138SIngo Molnar * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers. 110905a36a2SIngo Molnar * 111fda57b22SAndy Lutomirski * This is the only entry point used for 64-bit system calls. The 112fda57b22SAndy Lutomirski * hardware interface is reasonably well designed and the register to 113fda57b22SAndy Lutomirski * argument mapping Linux uses fits well with the registers that are 114fda57b22SAndy Lutomirski * available when SYSCALL is used. 115fda57b22SAndy Lutomirski * 116fda57b22SAndy Lutomirski * SYSCALL instructions can be found inlined in libc implementations as 117fda57b22SAndy Lutomirski * well as some other programs and libraries. There are also a handful 118fda57b22SAndy Lutomirski * of SYSCALL instructions in the vDSO used, for example, as a 119fda57b22SAndy Lutomirski * clock_gettimeofday fallback. 120fda57b22SAndy Lutomirski * 1214d732138SIngo Molnar * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11, 122905a36a2SIngo Molnar * then loads new ss, cs, and rip from previously programmed MSRs. 123905a36a2SIngo Molnar * rflags gets masked by a value from another MSR (so CLD and CLAC 124905a36a2SIngo Molnar * are not needed). SYSCALL does not save anything on the stack 125905a36a2SIngo Molnar * and does not change rsp. 126905a36a2SIngo Molnar * 127905a36a2SIngo Molnar * Registers on entry: 128905a36a2SIngo Molnar * rax system call number 129905a36a2SIngo Molnar * rcx return address 130905a36a2SIngo Molnar * r11 saved rflags (note: r11 is callee-clobbered register in C ABI) 131905a36a2SIngo Molnar * rdi arg0 132905a36a2SIngo Molnar * rsi arg1 133905a36a2SIngo Molnar * rdx arg2 134905a36a2SIngo Molnar * r10 arg3 (needs to be moved to rcx to conform to C ABI) 135905a36a2SIngo Molnar * r8 arg4 136905a36a2SIngo Molnar * r9 arg5 137905a36a2SIngo Molnar * (note: r12-r15, rbp, rbx are callee-preserved in C ABI) 138905a36a2SIngo Molnar * 139905a36a2SIngo Molnar * Only called from user space. 140905a36a2SIngo Molnar * 141905a36a2SIngo Molnar * When user can change pt_regs->foo always force IRET. That is because 142905a36a2SIngo Molnar * it deals with uncanonical addresses better. SYSRET has trouble 143905a36a2SIngo Molnar * with them due to bugs in both AMD and Intel CPUs. 144905a36a2SIngo Molnar */ 145905a36a2SIngo Molnar 146bc7b11c0SJiri SlabySYM_CODE_START(entry_SYSCALL_64) 1478c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 148905a36a2SIngo Molnar /* 149905a36a2SIngo Molnar * Interrupts are off on entry. 150905a36a2SIngo Molnar * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON, 151905a36a2SIngo Molnar * it is too small to ever cause noticeable irq latency. 152905a36a2SIngo Molnar */ 153905a36a2SIngo Molnar 1548a9949bcSAndy Lutomirski swapgs 155bf904d27SAndy Lutomirski /* tss.sp2 is scratch space. */ 15698f05b51SAndy Lutomirski movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) 157bf904d27SAndy Lutomirski SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp 158905a36a2SIngo Molnar movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 159905a36a2SIngo Molnar 160905a36a2SIngo Molnar /* Construct struct pt_regs on stack */ 161905a36a2SIngo Molnar pushq $__USER_DS /* pt_regs->ss */ 16298f05b51SAndy Lutomirski pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */ 163905a36a2SIngo Molnar pushq %r11 /* pt_regs->flags */ 164905a36a2SIngo Molnar pushq $__USER_CS /* pt_regs->cs */ 165905a36a2SIngo Molnar pushq %rcx /* pt_regs->ip */ 16626ba4e57SJiri SlabySYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL) 167905a36a2SIngo Molnar pushq %rax /* pt_regs->orig_ax */ 16830907fd1SDominik Brodowski 16930907fd1SDominik Brodowski PUSH_AND_CLEAR_REGS rax=$-ENOSYS 170905a36a2SIngo Molnar 1711e423bffSAndy Lutomirski /* IRQs are off. */ 172dfe64506SLinus Torvalds movq %rax, %rdi 173dfe64506SLinus Torvalds movq %rsp, %rsi 1741e423bffSAndy Lutomirski call do_syscall_64 /* returns with IRQs disabled */ 1751e423bffSAndy Lutomirski 176905a36a2SIngo Molnar /* 177905a36a2SIngo Molnar * Try to use SYSRET instead of IRET if we're returning to 1788a055d7fSAndy Lutomirski * a completely clean 64-bit userspace context. If we're not, 1798a055d7fSAndy Lutomirski * go to the slow exit path. 180905a36a2SIngo Molnar */ 181905a36a2SIngo Molnar movq RCX(%rsp), %rcx 182905a36a2SIngo Molnar movq RIP(%rsp), %r11 1838a055d7fSAndy Lutomirski 1848a055d7fSAndy Lutomirski cmpq %rcx, %r11 /* SYSRET requires RCX == RIP */ 1858a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 186905a36a2SIngo Molnar 187905a36a2SIngo Molnar /* 188905a36a2SIngo Molnar * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP 189905a36a2SIngo Molnar * in kernel space. This essentially lets the user take over 190905a36a2SIngo Molnar * the kernel, since userspace controls RSP. 191905a36a2SIngo Molnar * 192905a36a2SIngo Molnar * If width of "canonical tail" ever becomes variable, this will need 193905a36a2SIngo Molnar * to be updated to remain correct on both old and new CPUs. 194361b4b58SKirill A. Shutemov * 195cbe0317bSKirill A. Shutemov * Change top bits to match most significant bit (47th or 56th bit 196cbe0317bSKirill A. Shutemov * depending on paging mode) in the address. 197905a36a2SIngo Molnar */ 19809e61a77SKirill A. Shutemov#ifdef CONFIG_X86_5LEVEL 19939b95522SKirill A. Shutemov ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \ 20039b95522SKirill A. Shutemov "shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57 20109e61a77SKirill A. Shutemov#else 202905a36a2SIngo Molnar shl $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx 203905a36a2SIngo Molnar sar $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx 20409e61a77SKirill A. Shutemov#endif 2054d732138SIngo Molnar 206905a36a2SIngo Molnar /* If this changed %rcx, it was not canonical */ 207905a36a2SIngo Molnar cmpq %rcx, %r11 2088a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 209905a36a2SIngo Molnar 210905a36a2SIngo Molnar cmpq $__USER_CS, CS(%rsp) /* CS must match SYSRET */ 2118a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 212905a36a2SIngo Molnar 213905a36a2SIngo Molnar movq R11(%rsp), %r11 214905a36a2SIngo Molnar cmpq %r11, EFLAGS(%rsp) /* R11 == RFLAGS */ 2158a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 216905a36a2SIngo Molnar 217905a36a2SIngo Molnar /* 2183e035305SBorislav Petkov * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot 2193e035305SBorislav Petkov * restore RF properly. If the slowpath sets it for whatever reason, we 2203e035305SBorislav Petkov * need to restore it correctly. 2213e035305SBorislav Petkov * 2223e035305SBorislav Petkov * SYSRET can restore TF, but unlike IRET, restoring TF results in a 2233e035305SBorislav Petkov * trap from userspace immediately after SYSRET. This would cause an 2243e035305SBorislav Petkov * infinite loop whenever #DB happens with register state that satisfies 2253e035305SBorislav Petkov * the opportunistic SYSRET conditions. For example, single-stepping 2263e035305SBorislav Petkov * this user code: 227905a36a2SIngo Molnar * 228905a36a2SIngo Molnar * movq $stuck_here, %rcx 229905a36a2SIngo Molnar * pushfq 230905a36a2SIngo Molnar * popq %r11 231905a36a2SIngo Molnar * stuck_here: 232905a36a2SIngo Molnar * 233905a36a2SIngo Molnar * would never get past 'stuck_here'. 234905a36a2SIngo Molnar */ 235905a36a2SIngo Molnar testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11 2368a055d7fSAndy Lutomirski jnz swapgs_restore_regs_and_return_to_usermode 237905a36a2SIngo Molnar 238905a36a2SIngo Molnar /* nothing to check for RSP */ 239905a36a2SIngo Molnar 240905a36a2SIngo Molnar cmpq $__USER_DS, SS(%rsp) /* SS must match SYSRET */ 2418a055d7fSAndy Lutomirski jne swapgs_restore_regs_and_return_to_usermode 242905a36a2SIngo Molnar 243905a36a2SIngo Molnar /* 244905a36a2SIngo Molnar * We win! This label is here just for ease of understanding 245905a36a2SIngo Molnar * perf profiles. Nothing jumps here. 246905a36a2SIngo Molnar */ 247905a36a2SIngo Molnarsyscall_return_via_sysret: 248905a36a2SIngo Molnar /* rcx and r11 are already restored (see code above) */ 249502af0d7SDominik Brodowski POP_REGS pop_rdi=0 skip_r11rcx=1 2503e3b9293SAndy Lutomirski 2513e3b9293SAndy Lutomirski /* 2523e3b9293SAndy Lutomirski * Now all regs are restored except RSP and RDI. 2533e3b9293SAndy Lutomirski * Save old stack pointer and switch to trampoline stack. 2543e3b9293SAndy Lutomirski */ 2553e3b9293SAndy Lutomirski movq %rsp, %rdi 256c482feefSAndy Lutomirski movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp 2571fb14363SJosh Poimboeuf UNWIND_HINT_EMPTY 2583e3b9293SAndy Lutomirski 2593e3b9293SAndy Lutomirski pushq RSP-RDI(%rdi) /* RSP */ 2603e3b9293SAndy Lutomirski pushq (%rdi) /* RDI */ 2613e3b9293SAndy Lutomirski 2623e3b9293SAndy Lutomirski /* 2633e3b9293SAndy Lutomirski * We are on the trampoline stack. All regs except RDI are live. 2643e3b9293SAndy Lutomirski * We can do future final exit work right here. 2653e3b9293SAndy Lutomirski */ 266afaef01cSAlexander Popov STACKLEAK_ERASE_NOCLOBBER 267afaef01cSAlexander Popov 2686fd166aaSPeter Zijlstra SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 2693e3b9293SAndy Lutomirski 2704fbb3910SAndy Lutomirski popq %rdi 2713e3b9293SAndy Lutomirski popq %rsp 272905a36a2SIngo Molnar USERGS_SYSRET64 273bc7b11c0SJiri SlabySYM_CODE_END(entry_SYSCALL_64) 274905a36a2SIngo Molnar 275905a36a2SIngo Molnar/* 2760100301bSBrian Gerst * %rdi: prev task 2770100301bSBrian Gerst * %rsi: next task 2780100301bSBrian Gerst */ 279b9f6976bSThomas Gleixner.pushsection .text, "ax" 28096c64806SJosh PoimboeufSYM_FUNC_START(__switch_to_asm) 2810100301bSBrian Gerst /* 2820100301bSBrian Gerst * Save callee-saved registers 2830100301bSBrian Gerst * This must match the order in inactive_task_frame 2840100301bSBrian Gerst */ 2850100301bSBrian Gerst pushq %rbp 2860100301bSBrian Gerst pushq %rbx 2870100301bSBrian Gerst pushq %r12 2880100301bSBrian Gerst pushq %r13 2890100301bSBrian Gerst pushq %r14 2900100301bSBrian Gerst pushq %r15 2910100301bSBrian Gerst 2920100301bSBrian Gerst /* switch stack */ 2930100301bSBrian Gerst movq %rsp, TASK_threadsp(%rdi) 2940100301bSBrian Gerst movq TASK_threadsp(%rsi), %rsp 2950100301bSBrian Gerst 296050e9baaSLinus Torvalds#ifdef CONFIG_STACKPROTECTOR 2970100301bSBrian Gerst movq TASK_stack_canary(%rsi), %rbx 298e6401c13SAndy Lutomirski movq %rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset 2990100301bSBrian Gerst#endif 3000100301bSBrian Gerst 301c995efd5SDavid Woodhouse#ifdef CONFIG_RETPOLINE 302c995efd5SDavid Woodhouse /* 303c995efd5SDavid Woodhouse * When switching from a shallower to a deeper call stack 304c995efd5SDavid Woodhouse * the RSB may either underflow or use entries populated 305c995efd5SDavid Woodhouse * with userspace addresses. On CPUs where those concerns 306c995efd5SDavid Woodhouse * exist, overwrite the RSB with entries which capture 307c995efd5SDavid Woodhouse * speculative execution to prevent attack. 308c995efd5SDavid Woodhouse */ 309d1c99108SDavid Woodhouse FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW 310c995efd5SDavid Woodhouse#endif 311c995efd5SDavid Woodhouse 3120100301bSBrian Gerst /* restore callee-saved registers */ 3130100301bSBrian Gerst popq %r15 3140100301bSBrian Gerst popq %r14 3150100301bSBrian Gerst popq %r13 3160100301bSBrian Gerst popq %r12 3170100301bSBrian Gerst popq %rbx 3180100301bSBrian Gerst popq %rbp 3190100301bSBrian Gerst 3200100301bSBrian Gerst jmp __switch_to 32196c64806SJosh PoimboeufSYM_FUNC_END(__switch_to_asm) 322b9f6976bSThomas Gleixner.popsection 3230100301bSBrian Gerst 3240100301bSBrian Gerst/* 325905a36a2SIngo Molnar * A newly forked process directly context switches into this address. 326905a36a2SIngo Molnar * 3270100301bSBrian Gerst * rax: prev task we switched from 328616d2483SBrian Gerst * rbx: kernel thread func (NULL for user thread) 329616d2483SBrian Gerst * r12: kernel thread arg 330905a36a2SIngo Molnar */ 331b9f6976bSThomas Gleixner.pushsection .text, "ax" 332bc7b11c0SJiri SlabySYM_CODE_START(ret_from_fork) 3338c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 3340100301bSBrian Gerst movq %rax, %rdi 3354d732138SIngo Molnar call schedule_tail /* rdi: 'prev' task parameter */ 336905a36a2SIngo Molnar 337616d2483SBrian Gerst testq %rbx, %rbx /* from kernel_thread? */ 338616d2483SBrian Gerst jnz 1f /* kernel threads are uncommon */ 339905a36a2SIngo Molnar 340616d2483SBrian Gerst2: 3418c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 342ebd57499SJosh Poimboeuf movq %rsp, %rdi 34324d978b7SAndy Lutomirski call syscall_return_slowpath /* returns with IRQs disabled */ 3448a055d7fSAndy Lutomirski jmp swapgs_restore_regs_and_return_to_usermode 345616d2483SBrian Gerst 346616d2483SBrian Gerst1: 347616d2483SBrian Gerst /* kernel thread */ 348d31a5802SJosh Poimboeuf UNWIND_HINT_EMPTY 349616d2483SBrian Gerst movq %r12, %rdi 35034fdce69SPeter Zijlstra CALL_NOSPEC rbx 351616d2483SBrian Gerst /* 352616d2483SBrian Gerst * A kernel thread is allowed to return here after successfully 353616d2483SBrian Gerst * calling do_execve(). Exit to userspace to complete the execve() 354616d2483SBrian Gerst * syscall. 355616d2483SBrian Gerst */ 356616d2483SBrian Gerst movq $0, RAX(%rsp) 357616d2483SBrian Gerst jmp 2b 358bc7b11c0SJiri SlabySYM_CODE_END(ret_from_fork) 359b9f6976bSThomas Gleixner.popsection 360905a36a2SIngo Molnar 3611d3e53e8SAndy Lutomirski.macro DEBUG_ENTRY_ASSERT_IRQS_OFF 3621d3e53e8SAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 363e17f8234SBoris Ostrovsky pushq %rax 364e17f8234SBoris Ostrovsky SAVE_FLAGS(CLBR_RAX) 365e17f8234SBoris Ostrovsky testl $X86_EFLAGS_IF, %eax 3661d3e53e8SAndy Lutomirski jz .Lokay_\@ 3671d3e53e8SAndy Lutomirski ud2 3681d3e53e8SAndy Lutomirski.Lokay_\@: 369e17f8234SBoris Ostrovsky popq %rax 3701d3e53e8SAndy Lutomirski#endif 3711d3e53e8SAndy Lutomirski.endm 3721d3e53e8SAndy Lutomirski 3731d3e53e8SAndy Lutomirski/* 3741d3e53e8SAndy Lutomirski * Enters the IRQ stack if we're not already using it. NMI-safe. Clobbers 3751d3e53e8SAndy Lutomirski * flags and puts old RSP into old_rsp, and leaves all other GPRs alone. 3761d3e53e8SAndy Lutomirski * Requires kernel GSBASE. 3771d3e53e8SAndy Lutomirski * 3781d3e53e8SAndy Lutomirski * The invariant is that, if irq_count != -1, then the IRQ stack is in use. 3791d3e53e8SAndy Lutomirski */ 3802ba64741SDominik Brodowski.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0 3811d3e53e8SAndy Lutomirski DEBUG_ENTRY_ASSERT_IRQS_OFF 3822ba64741SDominik Brodowski 3832ba64741SDominik Brodowski .if \save_ret 3842ba64741SDominik Brodowski /* 3852ba64741SDominik Brodowski * If save_ret is set, the original stack contains one additional 3862ba64741SDominik Brodowski * entry -- the return address. Therefore, move the address one 3872ba64741SDominik Brodowski * entry below %rsp to \old_rsp. 3882ba64741SDominik Brodowski */ 3892ba64741SDominik Brodowski leaq 8(%rsp), \old_rsp 3902ba64741SDominik Brodowski .else 3911d3e53e8SAndy Lutomirski movq %rsp, \old_rsp 3922ba64741SDominik Brodowski .endif 3938c1f7558SJosh Poimboeuf 3948c1f7558SJosh Poimboeuf .if \regs 3958c1f7558SJosh Poimboeuf UNWIND_HINT_REGS base=\old_rsp 3968c1f7558SJosh Poimboeuf .endif 3978c1f7558SJosh Poimboeuf 3981d3e53e8SAndy Lutomirski incl PER_CPU_VAR(irq_count) 39929955909SAndy Lutomirski jnz .Lirq_stack_push_old_rsp_\@ 4001d3e53e8SAndy Lutomirski 4011d3e53e8SAndy Lutomirski /* 4021d3e53e8SAndy Lutomirski * Right now, if we just incremented irq_count to zero, we've 4031d3e53e8SAndy Lutomirski * claimed the IRQ stack but we haven't switched to it yet. 4041d3e53e8SAndy Lutomirski * 4051d3e53e8SAndy Lutomirski * If anything is added that can interrupt us here without using IST, 4061d3e53e8SAndy Lutomirski * it must be *extremely* careful to limit its stack usage. This 4071d3e53e8SAndy Lutomirski * could include kprobes and a hypothetical future IST-less #DB 4081d3e53e8SAndy Lutomirski * handler. 40929955909SAndy Lutomirski * 41029955909SAndy Lutomirski * The OOPS unwinder relies on the word at the top of the IRQ 41129955909SAndy Lutomirski * stack linking back to the previous RSP for the entire time we're 41229955909SAndy Lutomirski * on the IRQ stack. For this to work reliably, we need to write 41329955909SAndy Lutomirski * it before we actually move ourselves to the IRQ stack. 4141d3e53e8SAndy Lutomirski */ 4151d3e53e8SAndy Lutomirski 416e6401c13SAndy Lutomirski movq \old_rsp, PER_CPU_VAR(irq_stack_backing_store + IRQ_STACK_SIZE - 8) 417758a2e31SThomas Gleixner movq PER_CPU_VAR(hardirq_stack_ptr), %rsp 41829955909SAndy Lutomirski 41929955909SAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 42029955909SAndy Lutomirski /* 42129955909SAndy Lutomirski * If the first movq above becomes wrong due to IRQ stack layout 42229955909SAndy Lutomirski * changes, the only way we'll notice is if we try to unwind right 42329955909SAndy Lutomirski * here. Assert that we set up the stack right to catch this type 42429955909SAndy Lutomirski * of bug quickly. 42529955909SAndy Lutomirski */ 42629955909SAndy Lutomirski cmpq -8(%rsp), \old_rsp 42729955909SAndy Lutomirski je .Lirq_stack_okay\@ 42829955909SAndy Lutomirski ud2 42929955909SAndy Lutomirski .Lirq_stack_okay\@: 43029955909SAndy Lutomirski#endif 43129955909SAndy Lutomirski 43229955909SAndy Lutomirski.Lirq_stack_push_old_rsp_\@: 4331d3e53e8SAndy Lutomirski pushq \old_rsp 4348c1f7558SJosh Poimboeuf 4358c1f7558SJosh Poimboeuf .if \regs 4368c1f7558SJosh Poimboeuf UNWIND_HINT_REGS indirect=1 4378c1f7558SJosh Poimboeuf .endif 4382ba64741SDominik Brodowski 4392ba64741SDominik Brodowski .if \save_ret 4402ba64741SDominik Brodowski /* 4412ba64741SDominik Brodowski * Push the return address to the stack. This return address can 4422ba64741SDominik Brodowski * be found at the "real" original RSP, which was offset by 8 at 4432ba64741SDominik Brodowski * the beginning of this macro. 4442ba64741SDominik Brodowski */ 4452ba64741SDominik Brodowski pushq -8(\old_rsp) 4462ba64741SDominik Brodowski .endif 4471d3e53e8SAndy Lutomirski.endm 4481d3e53e8SAndy Lutomirski 4491d3e53e8SAndy Lutomirski/* 4501d3e53e8SAndy Lutomirski * Undoes ENTER_IRQ_STACK. 4511d3e53e8SAndy Lutomirski */ 4528c1f7558SJosh Poimboeuf.macro LEAVE_IRQ_STACK regs=1 4531d3e53e8SAndy Lutomirski DEBUG_ENTRY_ASSERT_IRQS_OFF 4541d3e53e8SAndy Lutomirski /* We need to be off the IRQ stack before decrementing irq_count. */ 4551d3e53e8SAndy Lutomirski popq %rsp 4561d3e53e8SAndy Lutomirski 4578c1f7558SJosh Poimboeuf .if \regs 4588c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 4598c1f7558SJosh Poimboeuf .endif 4608c1f7558SJosh Poimboeuf 4611d3e53e8SAndy Lutomirski /* 4621d3e53e8SAndy Lutomirski * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming 4631d3e53e8SAndy Lutomirski * the irq stack but we're not on it. 4641d3e53e8SAndy Lutomirski */ 4651d3e53e8SAndy Lutomirski 4661d3e53e8SAndy Lutomirski decl PER_CPU_VAR(irq_count) 4671d3e53e8SAndy Lutomirski.endm 4681d3e53e8SAndy Lutomirski 469cfa82a00SThomas Gleixner/** 470cfa82a00SThomas Gleixner * idtentry_body - Macro to emit code calling the C function 471cfa82a00SThomas Gleixner * @cfunc: C function to be called 472cfa82a00SThomas Gleixner * @has_error_code: Hardware pushed error code on stack 473cfa82a00SThomas Gleixner */ 474e2dcb5f1SThomas Gleixner.macro idtentry_body cfunc has_error_code:req 475cfa82a00SThomas Gleixner 476cfa82a00SThomas Gleixner call error_entry 477cfa82a00SThomas Gleixner UNWIND_HINT_REGS 478cfa82a00SThomas Gleixner 479cfa82a00SThomas Gleixner movq %rsp, %rdi /* pt_regs pointer into 1st argument*/ 480cfa82a00SThomas Gleixner 481cfa82a00SThomas Gleixner .if \has_error_code == 1 482cfa82a00SThomas Gleixner movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ 483cfa82a00SThomas Gleixner movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ 484cfa82a00SThomas Gleixner .endif 485cfa82a00SThomas Gleixner 486cfa82a00SThomas Gleixner call \cfunc 487cfa82a00SThomas Gleixner 488424c7d0aSThomas Gleixner jmp error_return 489cfa82a00SThomas Gleixner.endm 490cfa82a00SThomas Gleixner 491cfa82a00SThomas Gleixner/** 492cfa82a00SThomas Gleixner * idtentry - Macro to generate entry stubs for simple IDT entries 493cfa82a00SThomas Gleixner * @vector: Vector number 494cfa82a00SThomas Gleixner * @asmsym: ASM symbol for the entry point 495cfa82a00SThomas Gleixner * @cfunc: C function to be called 496cfa82a00SThomas Gleixner * @has_error_code: Hardware pushed error code on stack 497cfa82a00SThomas Gleixner * 498cfa82a00SThomas Gleixner * The macro emits code to set up the kernel context for straight forward 499cfa82a00SThomas Gleixner * and simple IDT entries. No IST stack, no paranoid entry checks. 500cfa82a00SThomas Gleixner */ 501e2dcb5f1SThomas Gleixner.macro idtentry vector asmsym cfunc has_error_code:req 502cfa82a00SThomas GleixnerSYM_CODE_START(\asmsym) 503cfa82a00SThomas Gleixner UNWIND_HINT_IRET_REGS offset=\has_error_code*8 504cfa82a00SThomas Gleixner ASM_CLAC 505cfa82a00SThomas Gleixner 506cfa82a00SThomas Gleixner .if \has_error_code == 0 507cfa82a00SThomas Gleixner pushq $-1 /* ORIG_RAX: no syscall to restart */ 508cfa82a00SThomas Gleixner .endif 509cfa82a00SThomas Gleixner 510cfa82a00SThomas Gleixner .if \vector == X86_TRAP_BP 511cfa82a00SThomas Gleixner /* 512cfa82a00SThomas Gleixner * If coming from kernel space, create a 6-word gap to allow the 513cfa82a00SThomas Gleixner * int3 handler to emulate a call instruction. 514cfa82a00SThomas Gleixner */ 515cfa82a00SThomas Gleixner testb $3, CS-ORIG_RAX(%rsp) 516cfa82a00SThomas Gleixner jnz .Lfrom_usermode_no_gap_\@ 517cfa82a00SThomas Gleixner .rept 6 518cfa82a00SThomas Gleixner pushq 5*8(%rsp) 519cfa82a00SThomas Gleixner .endr 520cfa82a00SThomas Gleixner UNWIND_HINT_IRET_REGS offset=8 521cfa82a00SThomas Gleixner.Lfrom_usermode_no_gap_\@: 522cfa82a00SThomas Gleixner .endif 523cfa82a00SThomas Gleixner 524e2dcb5f1SThomas Gleixner idtentry_body \cfunc \has_error_code 525cfa82a00SThomas Gleixner 526cfa82a00SThomas Gleixner_ASM_NOKPROBE(\asmsym) 527cfa82a00SThomas GleixnerSYM_CODE_END(\asmsym) 528cfa82a00SThomas Gleixner.endm 529cfa82a00SThomas Gleixner 530cfa82a00SThomas Gleixner/* 5310bf7c314SThomas Gleixner * Interrupt entry/exit. 5320bf7c314SThomas Gleixner * 5330bf7c314SThomas Gleixner + The interrupt stubs push (vector) onto the stack, which is the error_code 5340bf7c314SThomas Gleixner * position of idtentry exceptions, and jump to one of the two idtentry points 5350bf7c314SThomas Gleixner * (common/spurious). 5360bf7c314SThomas Gleixner * 5370bf7c314SThomas Gleixner * common_interrupt is a hotpath, align it to a cache line 5380bf7c314SThomas Gleixner */ 5390bf7c314SThomas Gleixner.macro idtentry_irq vector cfunc 5400bf7c314SThomas Gleixner .p2align CONFIG_X86_L1_CACHE_SHIFT 5410bf7c314SThomas Gleixner idtentry \vector asm_\cfunc \cfunc has_error_code=1 5420bf7c314SThomas Gleixner.endm 5430bf7c314SThomas Gleixner 5440bf7c314SThomas Gleixner/* 545cfa82a00SThomas Gleixner * MCE and DB exceptions 546cfa82a00SThomas Gleixner */ 547cfa82a00SThomas Gleixner#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + (x) * 8) 548cfa82a00SThomas Gleixner 549cfa82a00SThomas Gleixner/** 550cfa82a00SThomas Gleixner * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB 551cfa82a00SThomas Gleixner * @vector: Vector number 552cfa82a00SThomas Gleixner * @asmsym: ASM symbol for the entry point 553cfa82a00SThomas Gleixner * @cfunc: C function to be called 554cfa82a00SThomas Gleixner * 555cfa82a00SThomas Gleixner * The macro emits code to set up the kernel context for #MC and #DB 556cfa82a00SThomas Gleixner * 557cfa82a00SThomas Gleixner * If the entry comes from user space it uses the normal entry path 558cfa82a00SThomas Gleixner * including the return to user space work and preemption checks on 559cfa82a00SThomas Gleixner * exit. 560cfa82a00SThomas Gleixner * 561cfa82a00SThomas Gleixner * If hits in kernel mode then it needs to go through the paranoid 562cfa82a00SThomas Gleixner * entry as the exception can hit any random state. No preemption 563cfa82a00SThomas Gleixner * check on exit to keep the paranoid path simple. 564cfa82a00SThomas Gleixner * 565cfa82a00SThomas Gleixner * If the trap is #DB then the interrupt stack entry in the IST is 566cfa82a00SThomas Gleixner * moved to the second stack, so a potential recursion will have a 567cfa82a00SThomas Gleixner * fresh IST. 568cfa82a00SThomas Gleixner */ 569cfa82a00SThomas Gleixner.macro idtentry_mce_db vector asmsym cfunc 570cfa82a00SThomas GleixnerSYM_CODE_START(\asmsym) 571cfa82a00SThomas Gleixner UNWIND_HINT_IRET_REGS 572cfa82a00SThomas Gleixner ASM_CLAC 573cfa82a00SThomas Gleixner 574cfa82a00SThomas Gleixner pushq $-1 /* ORIG_RAX: no syscall to restart */ 575cfa82a00SThomas Gleixner 576cfa82a00SThomas Gleixner /* 577cfa82a00SThomas Gleixner * If the entry is from userspace, switch stacks and treat it as 578cfa82a00SThomas Gleixner * a normal entry. 579cfa82a00SThomas Gleixner */ 580cfa82a00SThomas Gleixner testb $3, CS-ORIG_RAX(%rsp) 581cfa82a00SThomas Gleixner jnz .Lfrom_usermode_switch_stack_\@ 582cfa82a00SThomas Gleixner 583cfa82a00SThomas Gleixner /* 584cfa82a00SThomas Gleixner * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX. 585cfa82a00SThomas Gleixner * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS 586cfa82a00SThomas Gleixner */ 587cfa82a00SThomas Gleixner call paranoid_entry 588cfa82a00SThomas Gleixner 589cfa82a00SThomas Gleixner UNWIND_HINT_REGS 590cfa82a00SThomas Gleixner 591cfa82a00SThomas Gleixner .if \vector == X86_TRAP_DB 592cfa82a00SThomas Gleixner TRACE_IRQS_OFF_DEBUG 593cfa82a00SThomas Gleixner .else 594cfa82a00SThomas Gleixner TRACE_IRQS_OFF 595cfa82a00SThomas Gleixner .endif 596cfa82a00SThomas Gleixner 597cfa82a00SThomas Gleixner movq %rsp, %rdi /* pt_regs pointer */ 598cfa82a00SThomas Gleixner 599cfa82a00SThomas Gleixner .if \vector == X86_TRAP_DB 600cfa82a00SThomas Gleixner subq $DB_STACK_OFFSET, CPU_TSS_IST(IST_INDEX_DB) 601cfa82a00SThomas Gleixner .endif 602cfa82a00SThomas Gleixner 603cfa82a00SThomas Gleixner call \cfunc 604cfa82a00SThomas Gleixner 605cfa82a00SThomas Gleixner .if \vector == X86_TRAP_DB 606cfa82a00SThomas Gleixner addq $DB_STACK_OFFSET, CPU_TSS_IST(IST_INDEX_DB) 607cfa82a00SThomas Gleixner .endif 608cfa82a00SThomas Gleixner 609cfa82a00SThomas Gleixner jmp paranoid_exit 610cfa82a00SThomas Gleixner 611cfa82a00SThomas Gleixner /* Switch to the regular task stack and use the noist entry point */ 612cfa82a00SThomas Gleixner.Lfrom_usermode_switch_stack_\@: 613e2dcb5f1SThomas Gleixner idtentry_body noist_\cfunc, has_error_code=0 614cfa82a00SThomas Gleixner 615cfa82a00SThomas Gleixner_ASM_NOKPROBE(\asmsym) 616cfa82a00SThomas GleixnerSYM_CODE_END(\asmsym) 617cfa82a00SThomas Gleixner.endm 618cfa82a00SThomas Gleixner 619cfa82a00SThomas Gleixner/* 620cfa82a00SThomas Gleixner * Double fault entry. Straight paranoid. No checks from which context 621cfa82a00SThomas Gleixner * this comes because for the espfix induced #DF this would do the wrong 622cfa82a00SThomas Gleixner * thing. 623cfa82a00SThomas Gleixner */ 624cfa82a00SThomas Gleixner.macro idtentry_df vector asmsym cfunc 625cfa82a00SThomas GleixnerSYM_CODE_START(\asmsym) 626cfa82a00SThomas Gleixner UNWIND_HINT_IRET_REGS offset=8 627cfa82a00SThomas Gleixner ASM_CLAC 628cfa82a00SThomas Gleixner 629cfa82a00SThomas Gleixner /* 630cfa82a00SThomas Gleixner * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX. 631cfa82a00SThomas Gleixner * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS 632cfa82a00SThomas Gleixner */ 633cfa82a00SThomas Gleixner call paranoid_entry 634cfa82a00SThomas Gleixner UNWIND_HINT_REGS 635cfa82a00SThomas Gleixner 636cfa82a00SThomas Gleixner movq %rsp, %rdi /* pt_regs pointer into first argument */ 637cfa82a00SThomas Gleixner movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ 638cfa82a00SThomas Gleixner movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ 639cfa82a00SThomas Gleixner call \cfunc 640cfa82a00SThomas Gleixner 641cfa82a00SThomas Gleixner jmp paranoid_exit 642cfa82a00SThomas Gleixner 643cfa82a00SThomas Gleixner_ASM_NOKPROBE(\asmsym) 644cfa82a00SThomas GleixnerSYM_CODE_END(\asmsym) 645cfa82a00SThomas Gleixner.endm 646cfa82a00SThomas Gleixner 647905a36a2SIngo Molnar/* 64853aaf262SThomas Gleixner * Include the defines which emit the idt entries which are shared 64953aaf262SThomas Gleixner * shared between 32 and 64 bit. 65053aaf262SThomas Gleixner */ 65153aaf262SThomas Gleixner#include <asm/idtentry.h> 65253aaf262SThomas Gleixner 65353aaf262SThomas Gleixner/* 654f3d415eaSDominik Brodowski * Interrupt entry helper function. 655905a36a2SIngo Molnar * 656f3d415eaSDominik Brodowski * Entry runs with interrupts off. Stack layout at entry: 657f3d415eaSDominik Brodowski * +----------------------------------------------------+ 658f3d415eaSDominik Brodowski * | regs->ss | 659f3d415eaSDominik Brodowski * | regs->rsp | 660f3d415eaSDominik Brodowski * | regs->eflags | 661f3d415eaSDominik Brodowski * | regs->cs | 662f3d415eaSDominik Brodowski * | regs->ip | 663f3d415eaSDominik Brodowski * +----------------------------------------------------+ 664f3d415eaSDominik Brodowski * | regs->orig_ax = ~(interrupt number) | 665f3d415eaSDominik Brodowski * +----------------------------------------------------+ 666f3d415eaSDominik Brodowski * | return address | 667f3d415eaSDominik Brodowski * +----------------------------------------------------+ 668905a36a2SIngo Molnar */ 669bc7b11c0SJiri SlabySYM_CODE_START(interrupt_entry) 67081b67439SJosh Poimboeuf UNWIND_HINT_IRET_REGS offset=16 671f3d415eaSDominik Brodowski ASM_CLAC 672905a36a2SIngo Molnar cld 6737f2590a1SAndy Lutomirski 674f3d415eaSDominik Brodowski testb $3, CS-ORIG_RAX+8(%rsp) 6757f2590a1SAndy Lutomirski jz 1f 6767f2590a1SAndy Lutomirski SWAPGS 67718ec54fdSJosh Poimboeuf FENCE_SWAPGS_USER_ENTRY 678f3d415eaSDominik Brodowski /* 679f3d415eaSDominik Brodowski * Switch to the thread stack. The IRET frame and orig_ax are 680f3d415eaSDominik Brodowski * on the stack, as well as the return address. RDI..R12 are 681f3d415eaSDominik Brodowski * not (yet) on the stack and space has not (yet) been 682f3d415eaSDominik Brodowski * allocated for them. 683f3d415eaSDominik Brodowski */ 68490a6acc4SDominik Brodowski pushq %rdi 685f3d415eaSDominik Brodowski 68690a6acc4SDominik Brodowski /* Need to switch before accessing the thread stack. */ 68790a6acc4SDominik Brodowski SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi 68890a6acc4SDominik Brodowski movq %rsp, %rdi 68990a6acc4SDominik Brodowski movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 690f3d415eaSDominik Brodowski 691f3d415eaSDominik Brodowski /* 692f3d415eaSDominik Brodowski * We have RDI, return address, and orig_ax on the stack on 693f3d415eaSDominik Brodowski * top of the IRET frame. That means offset=24 694f3d415eaSDominik Brodowski */ 695f3d415eaSDominik Brodowski UNWIND_HINT_IRET_REGS base=%rdi offset=24 69690a6acc4SDominik Brodowski 69790a6acc4SDominik Brodowski pushq 7*8(%rdi) /* regs->ss */ 69890a6acc4SDominik Brodowski pushq 6*8(%rdi) /* regs->rsp */ 69990a6acc4SDominik Brodowski pushq 5*8(%rdi) /* regs->eflags */ 70090a6acc4SDominik Brodowski pushq 4*8(%rdi) /* regs->cs */ 70190a6acc4SDominik Brodowski pushq 3*8(%rdi) /* regs->ip */ 70281b67439SJosh Poimboeuf UNWIND_HINT_IRET_REGS 70390a6acc4SDominik Brodowski pushq 2*8(%rdi) /* regs->orig_ax */ 70490a6acc4SDominik Brodowski pushq 8(%rdi) /* return address */ 70590a6acc4SDominik Brodowski 70690a6acc4SDominik Brodowski movq (%rdi), %rdi 70764dbc122SJosh Poimboeuf jmp 2f 7087f2590a1SAndy Lutomirski1: 70918ec54fdSJosh Poimboeuf FENCE_SWAPGS_KERNEL_ENTRY 71018ec54fdSJosh Poimboeuf2: 7110e34d226SDominik Brodowski PUSH_AND_CLEAR_REGS save_ret=1 7120e34d226SDominik Brodowski ENCODE_FRAME_POINTER 8 713905a36a2SIngo Molnar 7142ba64741SDominik Brodowski testb $3, CS+8(%rsp) 715905a36a2SIngo Molnar jz 1f 71602bc7768SAndy Lutomirski 71702bc7768SAndy Lutomirski /* 7187f2590a1SAndy Lutomirski * IRQ from user mode. 7197f2590a1SAndy Lutomirski * 720f1075053SAndy Lutomirski * We need to tell lockdep that IRQs are off. We can't do this until 721f1075053SAndy Lutomirski * we fix gsbase, and we should do it before enter_from_user_mode 722f3d415eaSDominik Brodowski * (which can take locks). Since TRACE_IRQS_OFF is idempotent, 723f1075053SAndy Lutomirski * the simplest way to handle it is to just call it twice if 724f1075053SAndy Lutomirski * we enter from user mode. There's no reason to optimize this since 725f1075053SAndy Lutomirski * TRACE_IRQS_OFF is a no-op if lockdep is off. 726f1075053SAndy Lutomirski */ 727f1075053SAndy Lutomirski TRACE_IRQS_OFF 728f1075053SAndy Lutomirski 729478dc89cSAndy Lutomirski CALL_enter_from_user_mode 73002bc7768SAndy Lutomirski 731905a36a2SIngo Molnar1: 7322ba64741SDominik Brodowski ENTER_IRQ_STACK old_rsp=%rdi save_ret=1 733905a36a2SIngo Molnar /* We entered an interrupt context - irqs are off: */ 734905a36a2SIngo Molnar TRACE_IRQS_OFF 735905a36a2SIngo Molnar 7362ba64741SDominik Brodowski ret 737bc7b11c0SJiri SlabySYM_CODE_END(interrupt_entry) 738a50480cbSAndrea Righi_ASM_NOKPROBE(interrupt_entry) 7392ba64741SDominik Brodowski 740*fa5e5c40SThomas GleixnerSYM_CODE_START_LOCAL(common_interrupt_return) 741905a36a2SIngo Molnarret_from_intr: 7422140a994SJan Beulich DISABLE_INTERRUPTS(CLBR_ANY) 743905a36a2SIngo Molnar TRACE_IRQS_OFF 744905a36a2SIngo Molnar 7451d3e53e8SAndy Lutomirski LEAVE_IRQ_STACK 746905a36a2SIngo Molnar 747905a36a2SIngo Molnar testb $3, CS(%rsp) 748905a36a2SIngo Molnar jz retint_kernel 74902bc7768SAndy Lutomirski 750905a36a2SIngo Molnar /* Interrupt came from user space */ 75130a2441cSJiri Slaby.Lretint_user: 75202bc7768SAndy Lutomirski mov %rsp,%rdi 75302bc7768SAndy Lutomirski call prepare_exit_to_usermode 75426c4ef9cSAndy Lutomirski 75526ba4e57SJiri SlabySYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) 75626c4ef9cSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 75726c4ef9cSAndy Lutomirski /* Assert that pt_regs indicates user mode. */ 7581e4c4f61SBorislav Petkov testb $3, CS(%rsp) 75926c4ef9cSAndy Lutomirski jnz 1f 76026c4ef9cSAndy Lutomirski ud2 76126c4ef9cSAndy Lutomirski1: 76226c4ef9cSAndy Lutomirski#endif 763502af0d7SDominik Brodowski POP_REGS pop_rdi=0 7643e3b9293SAndy Lutomirski 7653e3b9293SAndy Lutomirski /* 7663e3b9293SAndy Lutomirski * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS. 7673e3b9293SAndy Lutomirski * Save old stack pointer and switch to trampoline stack. 7683e3b9293SAndy Lutomirski */ 7693e3b9293SAndy Lutomirski movq %rsp, %rdi 770c482feefSAndy Lutomirski movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp 7711fb14363SJosh Poimboeuf UNWIND_HINT_EMPTY 7723e3b9293SAndy Lutomirski 7733e3b9293SAndy Lutomirski /* Copy the IRET frame to the trampoline stack. */ 7743e3b9293SAndy Lutomirski pushq 6*8(%rdi) /* SS */ 7753e3b9293SAndy Lutomirski pushq 5*8(%rdi) /* RSP */ 7763e3b9293SAndy Lutomirski pushq 4*8(%rdi) /* EFLAGS */ 7773e3b9293SAndy Lutomirski pushq 3*8(%rdi) /* CS */ 7783e3b9293SAndy Lutomirski pushq 2*8(%rdi) /* RIP */ 7793e3b9293SAndy Lutomirski 7803e3b9293SAndy Lutomirski /* Push user RDI on the trampoline stack. */ 7813e3b9293SAndy Lutomirski pushq (%rdi) 7823e3b9293SAndy Lutomirski 7833e3b9293SAndy Lutomirski /* 7843e3b9293SAndy Lutomirski * We are on the trampoline stack. All regs except RDI are live. 7853e3b9293SAndy Lutomirski * We can do future final exit work right here. 7863e3b9293SAndy Lutomirski */ 787afaef01cSAlexander Popov STACKLEAK_ERASE_NOCLOBBER 7883e3b9293SAndy Lutomirski 7896fd166aaSPeter Zijlstra SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 7908a09317bSDave Hansen 7913e3b9293SAndy Lutomirski /* Restore RDI. */ 7923e3b9293SAndy Lutomirski popq %rdi 7933e3b9293SAndy Lutomirski SWAPGS 79426c4ef9cSAndy Lutomirski INTERRUPT_RETURN 79526c4ef9cSAndy Lutomirski 796905a36a2SIngo Molnar 797905a36a2SIngo Molnar/* Returning to kernel space */ 798905a36a2SIngo Molnarretint_kernel: 79948593975SThomas Gleixner#ifdef CONFIG_PREEMPTION 800905a36a2SIngo Molnar /* Interrupts are off */ 801905a36a2SIngo Molnar /* Check if we need preemption */ 8026709812fSJan Beulich btl $9, EFLAGS(%rsp) /* were interrupts off? */ 803905a36a2SIngo Molnar jnc 1f 804b5b447b6SValentin Schneider cmpl $0, PER_CPU_VAR(__preempt_count) 805905a36a2SIngo Molnar jnz 1f 806905a36a2SIngo Molnar call preempt_schedule_irq 807905a36a2SIngo Molnar1: 808905a36a2SIngo Molnar#endif 809905a36a2SIngo Molnar /* 810905a36a2SIngo Molnar * The iretq could re-enable interrupts: 811905a36a2SIngo Molnar */ 812905a36a2SIngo Molnar TRACE_IRQS_IRETQ 813905a36a2SIngo Molnar 81426ba4e57SJiri SlabySYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) 81526c4ef9cSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 81626c4ef9cSAndy Lutomirski /* Assert that pt_regs indicates kernel mode. */ 8171e4c4f61SBorislav Petkov testb $3, CS(%rsp) 81826c4ef9cSAndy Lutomirski jz 1f 81926c4ef9cSAndy Lutomirski ud2 82026c4ef9cSAndy Lutomirski1: 82126c4ef9cSAndy Lutomirski#endif 822502af0d7SDominik Brodowski POP_REGS 823e872045bSAndy Lutomirski addq $8, %rsp /* skip regs->orig_ax */ 82410bcc80eSMathieu Desnoyers /* 82510bcc80eSMathieu Desnoyers * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization 82610bcc80eSMathieu Desnoyers * when returning from IPI handler. 82710bcc80eSMathieu Desnoyers */ 828905a36a2SIngo Molnar INTERRUPT_RETURN 829905a36a2SIngo Molnar 830cc66936eSJiri SlabySYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL) 8318c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 832905a36a2SIngo Molnar /* 833905a36a2SIngo Molnar * Are we returning to a stack segment from the LDT? Note: in 834905a36a2SIngo Molnar * 64-bit mode SS:RSP on the exception stack is always valid. 835905a36a2SIngo Molnar */ 836905a36a2SIngo Molnar#ifdef CONFIG_X86_ESPFIX64 837905a36a2SIngo Molnar testb $4, (SS-RIP)(%rsp) 838905a36a2SIngo Molnar jnz native_irq_return_ldt 839905a36a2SIngo Molnar#endif 840905a36a2SIngo Molnar 841cc66936eSJiri SlabySYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL) 842905a36a2SIngo Molnar /* 843905a36a2SIngo Molnar * This may fault. Non-paranoid faults on return to userspace are 844905a36a2SIngo Molnar * handled by fixup_bad_iret. These include #SS, #GP, and #NP. 845c29c775aSThomas Gleixner * Double-faults due to espfix64 are handled in exc_double_fault. 846905a36a2SIngo Molnar * Other faults here are fatal. 847905a36a2SIngo Molnar */ 848905a36a2SIngo Molnar iretq 849905a36a2SIngo Molnar 850905a36a2SIngo Molnar#ifdef CONFIG_X86_ESPFIX64 851905a36a2SIngo Molnarnative_irq_return_ldt: 85285063facSAndy Lutomirski /* 85385063facSAndy Lutomirski * We are running with user GSBASE. All GPRs contain their user 85485063facSAndy Lutomirski * values. We have a percpu ESPFIX stack that is eight slots 85585063facSAndy Lutomirski * long (see ESPFIX_STACK_SIZE). espfix_waddr points to the bottom 85685063facSAndy Lutomirski * of the ESPFIX stack. 85785063facSAndy Lutomirski * 85885063facSAndy Lutomirski * We clobber RAX and RDI in this code. We stash RDI on the 85985063facSAndy Lutomirski * normal stack and RAX on the ESPFIX stack. 86085063facSAndy Lutomirski * 86185063facSAndy Lutomirski * The ESPFIX stack layout we set up looks like this: 86285063facSAndy Lutomirski * 86385063facSAndy Lutomirski * --- top of ESPFIX stack --- 86485063facSAndy Lutomirski * SS 86585063facSAndy Lutomirski * RSP 86685063facSAndy Lutomirski * RFLAGS 86785063facSAndy Lutomirski * CS 86885063facSAndy Lutomirski * RIP <-- RSP points here when we're done 86985063facSAndy Lutomirski * RAX <-- espfix_waddr points here 87085063facSAndy Lutomirski * --- bottom of ESPFIX stack --- 87185063facSAndy Lutomirski */ 87285063facSAndy Lutomirski 87385063facSAndy Lutomirski pushq %rdi /* Stash user RDI */ 8748a09317bSDave Hansen SWAPGS /* to kernel GS */ 8758a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */ 8768a09317bSDave Hansen 877905a36a2SIngo Molnar movq PER_CPU_VAR(espfix_waddr), %rdi 87885063facSAndy Lutomirski movq %rax, (0*8)(%rdi) /* user RAX */ 87985063facSAndy Lutomirski movq (1*8)(%rsp), %rax /* user RIP */ 880905a36a2SIngo Molnar movq %rax, (1*8)(%rdi) 88185063facSAndy Lutomirski movq (2*8)(%rsp), %rax /* user CS */ 882905a36a2SIngo Molnar movq %rax, (2*8)(%rdi) 88385063facSAndy Lutomirski movq (3*8)(%rsp), %rax /* user RFLAGS */ 884905a36a2SIngo Molnar movq %rax, (3*8)(%rdi) 88585063facSAndy Lutomirski movq (5*8)(%rsp), %rax /* user SS */ 886905a36a2SIngo Molnar movq %rax, (5*8)(%rdi) 88785063facSAndy Lutomirski movq (4*8)(%rsp), %rax /* user RSP */ 888905a36a2SIngo Molnar movq %rax, (4*8)(%rdi) 88985063facSAndy Lutomirski /* Now RAX == RSP. */ 89085063facSAndy Lutomirski 89185063facSAndy Lutomirski andl $0xffff0000, %eax /* RAX = (RSP & 0xffff0000) */ 89285063facSAndy Lutomirski 89385063facSAndy Lutomirski /* 89485063facSAndy Lutomirski * espfix_stack[31:16] == 0. The page tables are set up such that 89585063facSAndy Lutomirski * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of 89685063facSAndy Lutomirski * espfix_waddr for any X. That is, there are 65536 RO aliases of 89785063facSAndy Lutomirski * the same page. Set up RSP so that RSP[31:16] contains the 89885063facSAndy Lutomirski * respective 16 bits of the /userspace/ RSP and RSP nonetheless 89985063facSAndy Lutomirski * still points to an RO alias of the ESPFIX stack. 90085063facSAndy Lutomirski */ 901905a36a2SIngo Molnar orq PER_CPU_VAR(espfix_stack), %rax 9028a09317bSDave Hansen 9036fd166aaSPeter Zijlstra SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 9048a09317bSDave Hansen SWAPGS /* to user GS */ 9058a09317bSDave Hansen popq %rdi /* Restore user RDI */ 9068a09317bSDave Hansen 907905a36a2SIngo Molnar movq %rax, %rsp 9088c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS offset=8 90985063facSAndy Lutomirski 91085063facSAndy Lutomirski /* 91185063facSAndy Lutomirski * At this point, we cannot write to the stack any more, but we can 91285063facSAndy Lutomirski * still read. 91385063facSAndy Lutomirski */ 91485063facSAndy Lutomirski popq %rax /* Restore user RAX */ 91585063facSAndy Lutomirski 91685063facSAndy Lutomirski /* 91785063facSAndy Lutomirski * RSP now points to an ordinary IRET frame, except that the page 91885063facSAndy Lutomirski * is read-only and RSP[31:16] are preloaded with the userspace 91985063facSAndy Lutomirski * values. We can now IRET back to userspace. 92085063facSAndy Lutomirski */ 921905a36a2SIngo Molnar jmp native_irq_return_iret 922905a36a2SIngo Molnar#endif 923*fa5e5c40SThomas GleixnerSYM_CODE_END(common_interrupt_return) 924*fa5e5c40SThomas Gleixner_ASM_NOKPROBE(common_interrupt_return) 925905a36a2SIngo Molnar 926905a36a2SIngo Molnar/* 927905a36a2SIngo Molnar * APIC interrupts. 928905a36a2SIngo Molnar */ 929905a36a2SIngo Molnar.macro apicinterrupt3 num sym do_sym 930bc7b11c0SJiri SlabySYM_CODE_START(\sym) 9318c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 932905a36a2SIngo Molnar pushq $~(\num) 9333aa99fc3SDominik Brodowski call interrupt_entry 9343aa99fc3SDominik Brodowski UNWIND_HINT_REGS indirect=1 9353aa99fc3SDominik Brodowski call \do_sym /* rdi points to pt_regs */ 936905a36a2SIngo Molnar jmp ret_from_intr 937bc7b11c0SJiri SlabySYM_CODE_END(\sym) 938a50480cbSAndrea Righi_ASM_NOKPROBE(\sym) 939905a36a2SIngo Molnar.endm 940905a36a2SIngo Molnar 941469f0023SAlexander Potapenko/* Make sure APIC interrupt handlers end up in the irqentry section: */ 942469f0023SAlexander Potapenko#define PUSH_SECTION_IRQENTRY .pushsection .irqentry.text, "ax" 943469f0023SAlexander Potapenko#define POP_SECTION_IRQENTRY .popsection 944469f0023SAlexander Potapenko 945905a36a2SIngo Molnar.macro apicinterrupt num sym do_sym 946469f0023SAlexander PotapenkoPUSH_SECTION_IRQENTRY 947905a36a2SIngo Molnarapicinterrupt3 \num \sym \do_sym 948469f0023SAlexander PotapenkoPOP_SECTION_IRQENTRY 949905a36a2SIngo Molnar.endm 950905a36a2SIngo Molnar 951905a36a2SIngo Molnar#ifdef CONFIG_SMP 9524d732138SIngo Molnarapicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt 9534d732138SIngo Molnarapicinterrupt3 REBOOT_VECTOR reboot_interrupt smp_reboot_interrupt 954905a36a2SIngo Molnar#endif 955905a36a2SIngo Molnar 956905a36a2SIngo Molnar#ifdef CONFIG_X86_UV 9574d732138SIngo Molnarapicinterrupt3 UV_BAU_MESSAGE uv_bau_message_intr1 uv_bau_message_interrupt 958905a36a2SIngo Molnar#endif 9594d732138SIngo Molnar 9604d732138SIngo Molnarapicinterrupt LOCAL_TIMER_VECTOR apic_timer_interrupt smp_apic_timer_interrupt 9614d732138SIngo Molnarapicinterrupt X86_PLATFORM_IPI_VECTOR x86_platform_ipi smp_x86_platform_ipi 962905a36a2SIngo Molnar 963905a36a2SIngo Molnar#ifdef CONFIG_HAVE_KVM 9644d732138SIngo Molnarapicinterrupt3 POSTED_INTR_VECTOR kvm_posted_intr_ipi smp_kvm_posted_intr_ipi 9654d732138SIngo Molnarapicinterrupt3 POSTED_INTR_WAKEUP_VECTOR kvm_posted_intr_wakeup_ipi smp_kvm_posted_intr_wakeup_ipi 966210f84b0SWincy Vanapicinterrupt3 POSTED_INTR_NESTED_VECTOR kvm_posted_intr_nested_ipi smp_kvm_posted_intr_nested_ipi 967905a36a2SIngo Molnar#endif 968905a36a2SIngo Molnar 969905a36a2SIngo Molnar#ifdef CONFIG_X86_MCE_THRESHOLD 9704d732138SIngo Molnarapicinterrupt THRESHOLD_APIC_VECTOR threshold_interrupt smp_threshold_interrupt 971905a36a2SIngo Molnar#endif 972905a36a2SIngo Molnar 9739dda1658SIngo Molnar#ifdef CONFIG_X86_MCE_AMD 9744d732138SIngo Molnarapicinterrupt DEFERRED_ERROR_VECTOR deferred_error_interrupt smp_deferred_error_interrupt 9759dda1658SIngo Molnar#endif 9769dda1658SIngo Molnar 977905a36a2SIngo Molnar#ifdef CONFIG_X86_THERMAL_VECTOR 9784d732138SIngo Molnarapicinterrupt THERMAL_APIC_VECTOR thermal_interrupt smp_thermal_interrupt 979905a36a2SIngo Molnar#endif 980905a36a2SIngo Molnar 981905a36a2SIngo Molnar#ifdef CONFIG_SMP 9824d732138SIngo Molnarapicinterrupt CALL_FUNCTION_SINGLE_VECTOR call_function_single_interrupt smp_call_function_single_interrupt 9834d732138SIngo Molnarapicinterrupt CALL_FUNCTION_VECTOR call_function_interrupt smp_call_function_interrupt 9844d732138SIngo Molnarapicinterrupt RESCHEDULE_VECTOR reschedule_interrupt smp_reschedule_interrupt 985905a36a2SIngo Molnar#endif 986905a36a2SIngo Molnar 9874d732138SIngo Molnarapicinterrupt ERROR_APIC_VECTOR error_interrupt smp_error_interrupt 988633260faSThomas Gleixnerapicinterrupt SPURIOUS_APIC_VECTOR spurious_apic_interrupt smp_spurious_apic_interrupt 989905a36a2SIngo Molnar 990905a36a2SIngo Molnar#ifdef CONFIG_IRQ_WORK 9914d732138SIngo Molnarapicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt 992905a36a2SIngo Molnar#endif 993905a36a2SIngo Molnar 994905a36a2SIngo Molnar/* 9954d732138SIngo Molnar * Reload gs selector with exception handling 9964d732138SIngo Molnar * edi: new selector 997b9f6976bSThomas Gleixner * 998b9f6976bSThomas Gleixner * Is in entry.text as it shouldn't be instrumented. 9994d732138SIngo Molnar */ 1000410367e3SThomas GleixnerSYM_FUNC_START(asm_load_gs_index) 10018c1f7558SJosh Poimboeuf FRAME_BEGIN 1002c9317202SThomas Gleixner swapgs 100342c748bbSBorislav Petkov.Lgs_change: 1004905a36a2SIngo Molnar movl %edi, %gs 100596e5d28aSBorislav Petkov2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE 1006c9317202SThomas Gleixner swapgs 10078c1f7558SJosh Poimboeuf FRAME_END 1008905a36a2SIngo Molnar ret 1009410367e3SThomas GleixnerSYM_FUNC_END(asm_load_gs_index) 1010410367e3SThomas GleixnerEXPORT_SYMBOL(asm_load_gs_index) 1011905a36a2SIngo Molnar 101298ededb6SJiri Slaby _ASM_EXTABLE(.Lgs_change, .Lbad_gs) 1013905a36a2SIngo Molnar .section .fixup, "ax" 1014905a36a2SIngo Molnar /* running with kernelgs */ 1015ef77e688SJiri SlabySYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) 1016c9317202SThomas Gleixner swapgs /* switch back to user gs */ 1017b038c842SAndy Lutomirski.macro ZAP_GS 1018b038c842SAndy Lutomirski /* This can't be a string because the preprocessor needs to see it. */ 1019b038c842SAndy Lutomirski movl $__USER_DS, %eax 1020b038c842SAndy Lutomirski movl %eax, %gs 1021b038c842SAndy Lutomirski.endm 1022b038c842SAndy Lutomirski ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG 1023905a36a2SIngo Molnar xorl %eax, %eax 1024905a36a2SIngo Molnar movl %eax, %gs 1025905a36a2SIngo Molnar jmp 2b 1026ef77e688SJiri SlabySYM_CODE_END(.Lbad_gs) 1027905a36a2SIngo Molnar .previous 1028905a36a2SIngo Molnar 1029931b9414SThomas Gleixner/* 1030931b9414SThomas Gleixner * rdi: New stack pointer points to the top word of the stack 1031931b9414SThomas Gleixner * rsi: Function pointer 1032931b9414SThomas Gleixner * rdx: Function argument (can be NULL if none) 1033931b9414SThomas Gleixner */ 1034931b9414SThomas GleixnerSYM_FUNC_START(asm_call_on_stack) 1035931b9414SThomas Gleixner /* 1036931b9414SThomas Gleixner * Save the frame pointer unconditionally. This allows the ORC 1037931b9414SThomas Gleixner * unwinder to handle the stack switch. 1038931b9414SThomas Gleixner */ 1039931b9414SThomas Gleixner pushq %rbp 1040931b9414SThomas Gleixner mov %rsp, %rbp 1041931b9414SThomas Gleixner 1042931b9414SThomas Gleixner /* 1043931b9414SThomas Gleixner * The unwinder relies on the word at the top of the new stack 1044931b9414SThomas Gleixner * page linking back to the previous RSP. 1045931b9414SThomas Gleixner */ 1046931b9414SThomas Gleixner mov %rsp, (%rdi) 1047931b9414SThomas Gleixner mov %rdi, %rsp 1048931b9414SThomas Gleixner /* Move the argument to the right place */ 1049931b9414SThomas Gleixner mov %rdx, %rdi 1050931b9414SThomas Gleixner 1051931b9414SThomas Gleixner1: 1052931b9414SThomas Gleixner .pushsection .discard.instr_begin 1053931b9414SThomas Gleixner .long 1b - . 1054931b9414SThomas Gleixner .popsection 1055931b9414SThomas Gleixner 1056931b9414SThomas Gleixner CALL_NOSPEC rsi 1057931b9414SThomas Gleixner 1058931b9414SThomas Gleixner2: 1059931b9414SThomas Gleixner .pushsection .discard.instr_end 1060931b9414SThomas Gleixner .long 2b - . 1061931b9414SThomas Gleixner .popsection 1062931b9414SThomas Gleixner 1063931b9414SThomas Gleixner /* Restore the previous stack pointer from RBP. */ 1064931b9414SThomas Gleixner leaveq 1065931b9414SThomas Gleixner ret 1066931b9414SThomas GleixnerSYM_FUNC_END(asm_call_on_stack) 1067931b9414SThomas Gleixner 106828c11b0fSJuergen Gross#ifdef CONFIG_XEN_PV 1069905a36a2SIngo Molnar/* 1070905a36a2SIngo Molnar * A note on the "critical region" in our callback handler. 1071905a36a2SIngo Molnar * We want to avoid stacking callback handlers due to events occurring 1072905a36a2SIngo Molnar * during handling of the last event. To do this, we keep events disabled 1073905a36a2SIngo Molnar * until we've done all processing. HOWEVER, we must enable events before 1074905a36a2SIngo Molnar * popping the stack frame (can't be done atomically) and so it would still 1075905a36a2SIngo Molnar * be possible to get enough handler activations to overflow the stack. 1076905a36a2SIngo Molnar * Although unlikely, bugs of that kind are hard to track down, so we'd 1077905a36a2SIngo Molnar * like to avoid the possibility. 1078905a36a2SIngo Molnar * So, on entry to the handler we detect whether we interrupted an 1079905a36a2SIngo Molnar * existing activation in its critical region -- if so, we pop the current 1080905a36a2SIngo Molnar * activation and restart the handler using the previous one. 10812f6474e4SThomas Gleixner * 10822f6474e4SThomas Gleixner * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs) 1083905a36a2SIngo Molnar */ 10842f6474e4SThomas GleixnerSYM_CODE_START_LOCAL(exc_xen_hypervisor_callback) 10854d732138SIngo Molnar 1086905a36a2SIngo Molnar/* 1087905a36a2SIngo Molnar * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will 1088905a36a2SIngo Molnar * see the correct pointer to the pt_regs 1089905a36a2SIngo Molnar */ 10908c1f7558SJosh Poimboeuf UNWIND_HINT_FUNC 10914d732138SIngo Molnar movq %rdi, %rsp /* we don't return, adjust the stack frame */ 10928c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 10931d3e53e8SAndy Lutomirski 10942f6474e4SThomas Gleixner call xen_pv_evtchn_do_upcall 10951d3e53e8SAndy Lutomirski 10962f6474e4SThomas Gleixner jmp error_return 10972f6474e4SThomas GleixnerSYM_CODE_END(exc_xen_hypervisor_callback) 1098905a36a2SIngo Molnar 1099905a36a2SIngo Molnar/* 1100905a36a2SIngo Molnar * Hypervisor uses this for application faults while it executes. 1101905a36a2SIngo Molnar * We get here for two reasons: 1102905a36a2SIngo Molnar * 1. Fault while reloading DS, ES, FS or GS 1103905a36a2SIngo Molnar * 2. Fault while executing IRET 1104905a36a2SIngo Molnar * Category 1 we do not need to fix up as Xen has already reloaded all segment 1105905a36a2SIngo Molnar * registers that could be reloaded and zeroed the others. 1106905a36a2SIngo Molnar * Category 2 we fix up by killing the current process. We cannot use the 1107905a36a2SIngo Molnar * normal Linux return path in this case because if we use the IRET hypercall 1108905a36a2SIngo Molnar * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 1109905a36a2SIngo Molnar * We distinguish between categories by comparing each saved segment register 1110905a36a2SIngo Molnar * with its current contents: any discrepancy means we in category 1. 1111905a36a2SIngo Molnar */ 1112bc7b11c0SJiri SlabySYM_CODE_START(xen_failsafe_callback) 11138c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 1114905a36a2SIngo Molnar movl %ds, %ecx 1115905a36a2SIngo Molnar cmpw %cx, 0x10(%rsp) 1116905a36a2SIngo Molnar jne 1f 1117905a36a2SIngo Molnar movl %es, %ecx 1118905a36a2SIngo Molnar cmpw %cx, 0x18(%rsp) 1119905a36a2SIngo Molnar jne 1f 1120905a36a2SIngo Molnar movl %fs, %ecx 1121905a36a2SIngo Molnar cmpw %cx, 0x20(%rsp) 1122905a36a2SIngo Molnar jne 1f 1123905a36a2SIngo Molnar movl %gs, %ecx 1124905a36a2SIngo Molnar cmpw %cx, 0x28(%rsp) 1125905a36a2SIngo Molnar jne 1f 1126905a36a2SIngo Molnar /* All segments match their saved values => Category 2 (Bad IRET). */ 1127905a36a2SIngo Molnar movq (%rsp), %rcx 1128905a36a2SIngo Molnar movq 8(%rsp), %r11 1129905a36a2SIngo Molnar addq $0x30, %rsp 1130905a36a2SIngo Molnar pushq $0 /* RIP */ 11318c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS offset=8 1132be4c11afSThomas Gleixner jmp asm_exc_general_protection 1133905a36a2SIngo Molnar1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */ 1134905a36a2SIngo Molnar movq (%rsp), %rcx 1135905a36a2SIngo Molnar movq 8(%rsp), %r11 1136905a36a2SIngo Molnar addq $0x30, %rsp 11378c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 1138905a36a2SIngo Molnar pushq $-1 /* orig_ax = -1 => not a system call */ 11393f01daecSDominik Brodowski PUSH_AND_CLEAR_REGS 1140946c1911SJosh Poimboeuf ENCODE_FRAME_POINTER 1141e88d9741SThomas Gleixner jmp error_return 1142bc7b11c0SJiri SlabySYM_CODE_END(xen_failsafe_callback) 114328c11b0fSJuergen Gross#endif /* CONFIG_XEN_PV */ 1144905a36a2SIngo Molnar 114528c11b0fSJuergen Gross#ifdef CONFIG_XEN_PVHVM 1146905a36a2SIngo Molnarapicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1147905a36a2SIngo Molnar xen_hvm_callback_vector xen_evtchn_do_upcall 114828c11b0fSJuergen Gross#endif 1149905a36a2SIngo Molnar 1150905a36a2SIngo Molnar 1151905a36a2SIngo Molnar#if IS_ENABLED(CONFIG_HYPERV) 1152905a36a2SIngo Molnarapicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1153905a36a2SIngo Molnar hyperv_callback_vector hyperv_vector_handler 115493286261SVitaly Kuznetsov 115593286261SVitaly Kuznetsovapicinterrupt3 HYPERV_REENLIGHTENMENT_VECTOR \ 115693286261SVitaly Kuznetsov hyperv_reenlightenment_vector hyperv_reenlightenment_intr 1157248e742aSMichael Kelley 1158248e742aSMichael Kelleyapicinterrupt3 HYPERV_STIMER0_VECTOR \ 1159248e742aSMichael Kelley hv_stimer0_callback_vector hv_stimer0_vector_handler 1160905a36a2SIngo Molnar#endif /* CONFIG_HYPERV */ 1161905a36a2SIngo Molnar 1162498ad393SZhao Yakui#if IS_ENABLED(CONFIG_ACRN_GUEST) 1163498ad393SZhao Yakuiapicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ 1164498ad393SZhao Yakui acrn_hv_callback_vector acrn_hv_vector_handler 1165498ad393SZhao Yakui#endif 1166498ad393SZhao Yakui 1167905a36a2SIngo Molnar/* 11689e809d15SDominik Brodowski * Save all registers in pt_regs, and switch gs if needed. 1169905a36a2SIngo Molnar * Use slow, but surefire "are we in kernel?" check. 1170905a36a2SIngo Molnar * Return: ebx=0: need swapgs on exit, ebx=1: otherwise 1171905a36a2SIngo Molnar */ 1172ef1e0315SJiri SlabySYM_CODE_START_LOCAL(paranoid_entry) 11738c1f7558SJosh Poimboeuf UNWIND_HINT_FUNC 1174905a36a2SIngo Molnar cld 11759e809d15SDominik Brodowski PUSH_AND_CLEAR_REGS save_ret=1 11769e809d15SDominik Brodowski ENCODE_FRAME_POINTER 8 1177905a36a2SIngo Molnar movl $1, %ebx 1178905a36a2SIngo Molnar movl $MSR_GS_BASE, %ecx 1179905a36a2SIngo Molnar rdmsr 1180905a36a2SIngo Molnar testl %edx, %edx 1181905a36a2SIngo Molnar js 1f /* negative -> in kernel */ 1182905a36a2SIngo Molnar SWAPGS 1183905a36a2SIngo Molnar xorl %ebx, %ebx 11848a09317bSDave Hansen 11858a09317bSDave Hansen1: 118616561f27SDave Hansen /* 118716561f27SDave Hansen * Always stash CR3 in %r14. This value will be restored, 1188ae852495SAndy Lutomirski * verbatim, at exit. Needed if paranoid_entry interrupted 1189ae852495SAndy Lutomirski * another entry that already switched to the user CR3 value 1190ae852495SAndy Lutomirski * but has not yet returned to userspace. 119116561f27SDave Hansen * 119216561f27SDave Hansen * This is also why CS (stashed in the "iret frame" by the 119316561f27SDave Hansen * hardware at entry) can not be used: this may be a return 1194ae852495SAndy Lutomirski * to kernel code, but with a user CR3 value. 119516561f27SDave Hansen */ 11968a09317bSDave Hansen SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14 11978a09317bSDave Hansen 119818ec54fdSJosh Poimboeuf /* 119918ec54fdSJosh Poimboeuf * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an 120018ec54fdSJosh Poimboeuf * unconditional CR3 write, even in the PTI case. So do an lfence 120118ec54fdSJosh Poimboeuf * to prevent GS speculation, regardless of whether PTI is enabled. 120218ec54fdSJosh Poimboeuf */ 120318ec54fdSJosh Poimboeuf FENCE_SWAPGS_KERNEL_ENTRY 120418ec54fdSJosh Poimboeuf 12058a09317bSDave Hansen ret 1206ef1e0315SJiri SlabySYM_CODE_END(paranoid_entry) 1207905a36a2SIngo Molnar 1208905a36a2SIngo Molnar/* 1209905a36a2SIngo Molnar * "Paranoid" exit path from exception stack. This is invoked 1210905a36a2SIngo Molnar * only on return from non-NMI IST interrupts that came 1211905a36a2SIngo Molnar * from kernel space. 1212905a36a2SIngo Molnar * 1213905a36a2SIngo Molnar * We may be returning to very strange contexts (e.g. very early 1214905a36a2SIngo Molnar * in syscall entry), so checking for preemption here would 1215905a36a2SIngo Molnar * be complicated. Fortunately, we there's no good reason 1216905a36a2SIngo Molnar * to try to handle preemption here. 12174d732138SIngo Molnar * 12184d732138SIngo Molnar * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) 1219905a36a2SIngo Molnar */ 1220ef1e0315SJiri SlabySYM_CODE_START_LOCAL(paranoid_exit) 12218c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 12222140a994SJan Beulich DISABLE_INTERRUPTS(CLBR_ANY) 1223905a36a2SIngo Molnar TRACE_IRQS_OFF_DEBUG 1224905a36a2SIngo Molnar testl %ebx, %ebx /* swapgs needed? */ 1225e5317832SAndy Lutomirski jnz .Lparanoid_exit_no_swapgs 1226905a36a2SIngo Molnar TRACE_IRQS_IRETQ 122716561f27SDave Hansen /* Always restore stashed CR3 value (see paranoid_entry) */ 122821e94459SPeter Zijlstra RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 1229905a36a2SIngo Molnar SWAPGS_UNSAFE_STACK 123045c08383SThomas Gleixner jmp restore_regs_and_return_to_kernel 1231e5317832SAndy Lutomirski.Lparanoid_exit_no_swapgs: 1232905a36a2SIngo Molnar TRACE_IRQS_IRETQ_DEBUG 123316561f27SDave Hansen /* Always restore stashed CR3 value (see paranoid_entry) */ 1234e4865757SIngo Molnar RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 1235e5317832SAndy Lutomirski jmp restore_regs_and_return_to_kernel 1236ef1e0315SJiri SlabySYM_CODE_END(paranoid_exit) 1237905a36a2SIngo Molnar 1238905a36a2SIngo Molnar/* 12399e809d15SDominik Brodowski * Save all registers in pt_regs, and switch GS if needed. 1240905a36a2SIngo Molnar */ 1241ef1e0315SJiri SlabySYM_CODE_START_LOCAL(error_entry) 12429e809d15SDominik Brodowski UNWIND_HINT_FUNC 1243905a36a2SIngo Molnar cld 12449e809d15SDominik Brodowski PUSH_AND_CLEAR_REGS save_ret=1 12459e809d15SDominik Brodowski ENCODE_FRAME_POINTER 8 1246905a36a2SIngo Molnar testb $3, CS+8(%rsp) 1247cb6f64edSAndy Lutomirski jz .Lerror_kernelspace 1248539f5113SAndy Lutomirski 1249cb6f64edSAndy Lutomirski /* 1250cb6f64edSAndy Lutomirski * We entered from user mode or we're pretending to have entered 1251cb6f64edSAndy Lutomirski * from user mode due to an IRET fault. 1252cb6f64edSAndy Lutomirski */ 1253905a36a2SIngo Molnar SWAPGS 125418ec54fdSJosh Poimboeuf FENCE_SWAPGS_USER_ENTRY 12558a09317bSDave Hansen /* We have user CR3. Change to kernel CR3. */ 12568a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rax 1257539f5113SAndy Lutomirski 1258cb6f64edSAndy Lutomirski.Lerror_entry_from_usermode_after_swapgs: 12597f2590a1SAndy Lutomirski /* Put us onto the real thread stack. */ 12607f2590a1SAndy Lutomirski popq %r12 /* save return addr in %12 */ 12617f2590a1SAndy Lutomirski movq %rsp, %rdi /* arg0 = pt_regs pointer */ 12627f2590a1SAndy Lutomirski call sync_regs 12637f2590a1SAndy Lutomirski movq %rax, %rsp /* switch stack */ 12647f2590a1SAndy Lutomirski ENCODE_FRAME_POINTER 12657f2590a1SAndy Lutomirski pushq %r12 1266f1075053SAndy Lutomirski ret 126702bc7768SAndy Lutomirski 126818ec54fdSJosh Poimboeuf.Lerror_entry_done_lfence: 126918ec54fdSJosh Poimboeuf FENCE_SWAPGS_KERNEL_ENTRY 1270cb6f64edSAndy Lutomirski.Lerror_entry_done: 1271905a36a2SIngo Molnar ret 1272905a36a2SIngo Molnar 1273905a36a2SIngo Molnar /* 1274905a36a2SIngo Molnar * There are two places in the kernel that can potentially fault with 1275905a36a2SIngo Molnar * usergs. Handle them here. B stepping K8s sometimes report a 1276905a36a2SIngo Molnar * truncated RIP for IRET exceptions returning to compat mode. Check 1277905a36a2SIngo Molnar * for these here too. 1278905a36a2SIngo Molnar */ 1279cb6f64edSAndy Lutomirski.Lerror_kernelspace: 1280905a36a2SIngo Molnar leaq native_irq_return_iret(%rip), %rcx 1281905a36a2SIngo Molnar cmpq %rcx, RIP+8(%rsp) 1282cb6f64edSAndy Lutomirski je .Lerror_bad_iret 1283905a36a2SIngo Molnar movl %ecx, %eax /* zero extend */ 1284905a36a2SIngo Molnar cmpq %rax, RIP+8(%rsp) 1285cb6f64edSAndy Lutomirski je .Lbstep_iret 128642c748bbSBorislav Petkov cmpq $.Lgs_change, RIP+8(%rsp) 128718ec54fdSJosh Poimboeuf jne .Lerror_entry_done_lfence 1288539f5113SAndy Lutomirski 1289539f5113SAndy Lutomirski /* 129042c748bbSBorislav Petkov * hack: .Lgs_change can fail with user gsbase. If this happens, fix up 1291539f5113SAndy Lutomirski * gsbase and proceed. We'll fix up the exception and land in 129242c748bbSBorislav Petkov * .Lgs_change's error handler with kernel gsbase. 1293539f5113SAndy Lutomirski */ 12942fa5f04fSWanpeng Li SWAPGS 129518ec54fdSJosh Poimboeuf FENCE_SWAPGS_USER_ENTRY 12962fa5f04fSWanpeng Li jmp .Lerror_entry_done 1297905a36a2SIngo Molnar 1298cb6f64edSAndy Lutomirski.Lbstep_iret: 1299905a36a2SIngo Molnar /* Fix truncated RIP */ 1300905a36a2SIngo Molnar movq %rcx, RIP+8(%rsp) 1301905a36a2SIngo Molnar /* fall through */ 1302905a36a2SIngo Molnar 1303cb6f64edSAndy Lutomirski.Lerror_bad_iret: 1304539f5113SAndy Lutomirski /* 13058a09317bSDave Hansen * We came from an IRET to user mode, so we have user 13068a09317bSDave Hansen * gsbase and CR3. Switch to kernel gsbase and CR3: 1307539f5113SAndy Lutomirski */ 1308905a36a2SIngo Molnar SWAPGS 130918ec54fdSJosh Poimboeuf FENCE_SWAPGS_USER_ENTRY 13108a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rax 1311539f5113SAndy Lutomirski 1312539f5113SAndy Lutomirski /* 1313539f5113SAndy Lutomirski * Pretend that the exception came from user mode: set up pt_regs 1314b3681dd5SAndy Lutomirski * as if we faulted immediately after IRET. 1315539f5113SAndy Lutomirski */ 1316905a36a2SIngo Molnar mov %rsp, %rdi 1317905a36a2SIngo Molnar call fixup_bad_iret 1318905a36a2SIngo Molnar mov %rax, %rsp 1319cb6f64edSAndy Lutomirski jmp .Lerror_entry_from_usermode_after_swapgs 1320ef1e0315SJiri SlabySYM_CODE_END(error_entry) 1321905a36a2SIngo Molnar 1322424c7d0aSThomas GleixnerSYM_CODE_START_LOCAL(error_return) 1323424c7d0aSThomas Gleixner UNWIND_HINT_REGS 1324424c7d0aSThomas Gleixner DEBUG_ENTRY_ASSERT_IRQS_OFF 1325424c7d0aSThomas Gleixner testb $3, CS(%rsp) 1326424c7d0aSThomas Gleixner jz restore_regs_and_return_to_kernel 1327424c7d0aSThomas Gleixner jmp swapgs_restore_regs_and_return_to_usermode 1328424c7d0aSThomas GleixnerSYM_CODE_END(error_return) 1329424c7d0aSThomas Gleixner 1330929bacecSAndy Lutomirski/* 1331929bacecSAndy Lutomirski * Runs on exception stack. Xen PV does not go through this path at all, 1332929bacecSAndy Lutomirski * so we can use real assembly here. 13338a09317bSDave Hansen * 13348a09317bSDave Hansen * Registers: 13358a09317bSDave Hansen * %r14: Used to save/restore the CR3 of the interrupted context 13368a09317bSDave Hansen * when PAGE_TABLE_ISOLATION is in use. Do not clobber. 1337929bacecSAndy Lutomirski */ 13386271fef0SThomas GleixnerSYM_CODE_START(asm_exc_nmi) 13398c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 1340929bacecSAndy Lutomirski 1341fc57a7c6SAndy Lutomirski /* 1342905a36a2SIngo Molnar * We allow breakpoints in NMIs. If a breakpoint occurs, then 1343905a36a2SIngo Molnar * the iretq it performs will take us out of NMI context. 1344905a36a2SIngo Molnar * This means that we can have nested NMIs where the next 1345905a36a2SIngo Molnar * NMI is using the top of the stack of the previous NMI. We 1346905a36a2SIngo Molnar * can't let it execute because the nested NMI will corrupt the 1347905a36a2SIngo Molnar * stack of the previous NMI. NMI handlers are not re-entrant 1348905a36a2SIngo Molnar * anyway. 1349905a36a2SIngo Molnar * 1350905a36a2SIngo Molnar * To handle this case we do the following: 1351905a36a2SIngo Molnar * Check the a special location on the stack that contains 1352905a36a2SIngo Molnar * a variable that is set when NMIs are executing. 1353905a36a2SIngo Molnar * The interrupted task's stack is also checked to see if it 1354905a36a2SIngo Molnar * is an NMI stack. 1355905a36a2SIngo Molnar * If the variable is not set and the stack is not the NMI 1356905a36a2SIngo Molnar * stack then: 1357905a36a2SIngo Molnar * o Set the special variable on the stack 13580b22930eSAndy Lutomirski * o Copy the interrupt frame into an "outermost" location on the 13590b22930eSAndy Lutomirski * stack 13600b22930eSAndy Lutomirski * o Copy the interrupt frame into an "iret" location on the stack 1361905a36a2SIngo Molnar * o Continue processing the NMI 1362905a36a2SIngo Molnar * If the variable is set or the previous stack is the NMI stack: 13630b22930eSAndy Lutomirski * o Modify the "iret" location to jump to the repeat_nmi 1364905a36a2SIngo Molnar * o return back to the first NMI 1365905a36a2SIngo Molnar * 1366905a36a2SIngo Molnar * Now on exit of the first NMI, we first clear the stack variable 1367905a36a2SIngo Molnar * The NMI stack will tell any nested NMIs at that point that it is 1368905a36a2SIngo Molnar * nested. Then we pop the stack normally with iret, and if there was 1369905a36a2SIngo Molnar * a nested NMI that updated the copy interrupt stack frame, a 1370905a36a2SIngo Molnar * jump will be made to the repeat_nmi code that will handle the second 1371905a36a2SIngo Molnar * NMI. 13729b6e6a83SAndy Lutomirski * 13739b6e6a83SAndy Lutomirski * However, espfix prevents us from directly returning to userspace 13749b6e6a83SAndy Lutomirski * with a single IRET instruction. Similarly, IRET to user mode 13759b6e6a83SAndy Lutomirski * can fault. We therefore handle NMIs from user space like 13769b6e6a83SAndy Lutomirski * other IST entries. 1377905a36a2SIngo Molnar */ 1378905a36a2SIngo Molnar 1379e93c1730SAndy Lutomirski ASM_CLAC 1380e93c1730SAndy Lutomirski 1381905a36a2SIngo Molnar /* Use %rdx as our temp variable throughout */ 1382905a36a2SIngo Molnar pushq %rdx 1383905a36a2SIngo Molnar 13849b6e6a83SAndy Lutomirski testb $3, CS-RIP+8(%rsp) 13859b6e6a83SAndy Lutomirski jz .Lnmi_from_kernel 1386905a36a2SIngo Molnar 1387905a36a2SIngo Molnar /* 13889b6e6a83SAndy Lutomirski * NMI from user mode. We need to run on the thread stack, but we 13899b6e6a83SAndy Lutomirski * can't go through the normal entry paths: NMIs are masked, and 13909b6e6a83SAndy Lutomirski * we don't want to enable interrupts, because then we'll end 13919b6e6a83SAndy Lutomirski * up in an awkward situation in which IRQs are on but NMIs 13929b6e6a83SAndy Lutomirski * are off. 139383c133cfSAndy Lutomirski * 139483c133cfSAndy Lutomirski * We also must not push anything to the stack before switching 139583c133cfSAndy Lutomirski * stacks lest we corrupt the "NMI executing" variable. 13969b6e6a83SAndy Lutomirski */ 13979b6e6a83SAndy Lutomirski 1398929bacecSAndy Lutomirski swapgs 13999b6e6a83SAndy Lutomirski cld 140018ec54fdSJosh Poimboeuf FENCE_SWAPGS_USER_ENTRY 14018a09317bSDave Hansen SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx 14029b6e6a83SAndy Lutomirski movq %rsp, %rdx 14039b6e6a83SAndy Lutomirski movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 14048c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS base=%rdx offset=8 14059b6e6a83SAndy Lutomirski pushq 5*8(%rdx) /* pt_regs->ss */ 14069b6e6a83SAndy Lutomirski pushq 4*8(%rdx) /* pt_regs->rsp */ 14079b6e6a83SAndy Lutomirski pushq 3*8(%rdx) /* pt_regs->flags */ 14089b6e6a83SAndy Lutomirski pushq 2*8(%rdx) /* pt_regs->cs */ 14099b6e6a83SAndy Lutomirski pushq 1*8(%rdx) /* pt_regs->rip */ 14108c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 14119b6e6a83SAndy Lutomirski pushq $-1 /* pt_regs->orig_ax */ 141230907fd1SDominik Brodowski PUSH_AND_CLEAR_REGS rdx=(%rdx) 1413946c1911SJosh Poimboeuf ENCODE_FRAME_POINTER 14149b6e6a83SAndy Lutomirski 14159b6e6a83SAndy Lutomirski /* 14169b6e6a83SAndy Lutomirski * At this point we no longer need to worry about stack damage 14179b6e6a83SAndy Lutomirski * due to nesting -- we're on the normal thread stack and we're 14189b6e6a83SAndy Lutomirski * done with the NMI stack. 14199b6e6a83SAndy Lutomirski */ 14209b6e6a83SAndy Lutomirski 14219b6e6a83SAndy Lutomirski movq %rsp, %rdi 14229b6e6a83SAndy Lutomirski movq $-1, %rsi 14236271fef0SThomas Gleixner call exc_nmi 14249b6e6a83SAndy Lutomirski 14259b6e6a83SAndy Lutomirski /* 14269b6e6a83SAndy Lutomirski * Return back to user mode. We must *not* do the normal exit 1427946c1911SJosh Poimboeuf * work, because we don't want to enable interrupts. 14289b6e6a83SAndy Lutomirski */ 14298a055d7fSAndy Lutomirski jmp swapgs_restore_regs_and_return_to_usermode 14309b6e6a83SAndy Lutomirski 14319b6e6a83SAndy Lutomirski.Lnmi_from_kernel: 14329b6e6a83SAndy Lutomirski /* 14330b22930eSAndy Lutomirski * Here's what our stack frame will look like: 14340b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14350b22930eSAndy Lutomirski * | original SS | 14360b22930eSAndy Lutomirski * | original Return RSP | 14370b22930eSAndy Lutomirski * | original RFLAGS | 14380b22930eSAndy Lutomirski * | original CS | 14390b22930eSAndy Lutomirski * | original RIP | 14400b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14410b22930eSAndy Lutomirski * | temp storage for rdx | 14420b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14430b22930eSAndy Lutomirski * | "NMI executing" variable | 14440b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14450b22930eSAndy Lutomirski * | iret SS } Copied from "outermost" frame | 14460b22930eSAndy Lutomirski * | iret Return RSP } on each loop iteration; overwritten | 14470b22930eSAndy Lutomirski * | iret RFLAGS } by a nested NMI to force another | 14480b22930eSAndy Lutomirski * | iret CS } iteration if needed. | 14490b22930eSAndy Lutomirski * | iret RIP } | 14500b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14510b22930eSAndy Lutomirski * | outermost SS } initialized in first_nmi; | 14520b22930eSAndy Lutomirski * | outermost Return RSP } will not be changed before | 14530b22930eSAndy Lutomirski * | outermost RFLAGS } NMI processing is done. | 14540b22930eSAndy Lutomirski * | outermost CS } Copied to "iret" frame on each | 14550b22930eSAndy Lutomirski * | outermost RIP } iteration. | 14560b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14570b22930eSAndy Lutomirski * | pt_regs | 14580b22930eSAndy Lutomirski * +---------------------------------------------------------+ 14590b22930eSAndy Lutomirski * 14600b22930eSAndy Lutomirski * The "original" frame is used by hardware. Before re-enabling 14610b22930eSAndy Lutomirski * NMIs, we need to be done with it, and we need to leave enough 14620b22930eSAndy Lutomirski * space for the asm code here. 14630b22930eSAndy Lutomirski * 14640b22930eSAndy Lutomirski * We return by executing IRET while RSP points to the "iret" frame. 14650b22930eSAndy Lutomirski * That will either return for real or it will loop back into NMI 14660b22930eSAndy Lutomirski * processing. 14670b22930eSAndy Lutomirski * 14680b22930eSAndy Lutomirski * The "outermost" frame is copied to the "iret" frame on each 14690b22930eSAndy Lutomirski * iteration of the loop, so each iteration starts with the "iret" 14700b22930eSAndy Lutomirski * frame pointing to the final return target. 14710b22930eSAndy Lutomirski */ 14720b22930eSAndy Lutomirski 14730b22930eSAndy Lutomirski /* 14740b22930eSAndy Lutomirski * Determine whether we're a nested NMI. 14750b22930eSAndy Lutomirski * 1476a27507caSAndy Lutomirski * If we interrupted kernel code between repeat_nmi and 1477a27507caSAndy Lutomirski * end_repeat_nmi, then we are a nested NMI. We must not 1478a27507caSAndy Lutomirski * modify the "iret" frame because it's being written by 1479a27507caSAndy Lutomirski * the outer NMI. That's okay; the outer NMI handler is 14806271fef0SThomas Gleixner * about to about to call exc_nmi() anyway, so we can just 1481a27507caSAndy Lutomirski * resume the outer NMI. 1482a27507caSAndy Lutomirski */ 1483a27507caSAndy Lutomirski 1484a27507caSAndy Lutomirski movq $repeat_nmi, %rdx 1485a27507caSAndy Lutomirski cmpq 8(%rsp), %rdx 1486a27507caSAndy Lutomirski ja 1f 1487a27507caSAndy Lutomirski movq $end_repeat_nmi, %rdx 1488a27507caSAndy Lutomirski cmpq 8(%rsp), %rdx 1489a27507caSAndy Lutomirski ja nested_nmi_out 1490a27507caSAndy Lutomirski1: 1491a27507caSAndy Lutomirski 1492a27507caSAndy Lutomirski /* 1493a27507caSAndy Lutomirski * Now check "NMI executing". If it's set, then we're nested. 14940b22930eSAndy Lutomirski * This will not detect if we interrupted an outer NMI just 14950b22930eSAndy Lutomirski * before IRET. 1496905a36a2SIngo Molnar */ 1497905a36a2SIngo Molnar cmpl $1, -8(%rsp) 1498905a36a2SIngo Molnar je nested_nmi 1499905a36a2SIngo Molnar 1500905a36a2SIngo Molnar /* 15010b22930eSAndy Lutomirski * Now test if the previous stack was an NMI stack. This covers 15020b22930eSAndy Lutomirski * the case where we interrupt an outer NMI after it clears 1503810bc075SAndy Lutomirski * "NMI executing" but before IRET. We need to be careful, though: 1504810bc075SAndy Lutomirski * there is one case in which RSP could point to the NMI stack 1505810bc075SAndy Lutomirski * despite there being no NMI active: naughty userspace controls 1506810bc075SAndy Lutomirski * RSP at the very beginning of the SYSCALL targets. We can 1507810bc075SAndy Lutomirski * pull a fast one on naughty userspace, though: we program 1508810bc075SAndy Lutomirski * SYSCALL to mask DF, so userspace cannot cause DF to be set 1509810bc075SAndy Lutomirski * if it controls the kernel's RSP. We set DF before we clear 1510810bc075SAndy Lutomirski * "NMI executing". 1511905a36a2SIngo Molnar */ 1512905a36a2SIngo Molnar lea 6*8(%rsp), %rdx 1513905a36a2SIngo Molnar /* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */ 1514905a36a2SIngo Molnar cmpq %rdx, 4*8(%rsp) 1515905a36a2SIngo Molnar /* If the stack pointer is above the NMI stack, this is a normal NMI */ 1516905a36a2SIngo Molnar ja first_nmi 15174d732138SIngo Molnar 1518905a36a2SIngo Molnar subq $EXCEPTION_STKSZ, %rdx 1519905a36a2SIngo Molnar cmpq %rdx, 4*8(%rsp) 1520905a36a2SIngo Molnar /* If it is below the NMI stack, it is a normal NMI */ 1521905a36a2SIngo Molnar jb first_nmi 1522810bc075SAndy Lutomirski 1523810bc075SAndy Lutomirski /* Ah, it is within the NMI stack. */ 1524810bc075SAndy Lutomirski 1525810bc075SAndy Lutomirski testb $(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp) 1526810bc075SAndy Lutomirski jz first_nmi /* RSP was user controlled. */ 1527810bc075SAndy Lutomirski 1528810bc075SAndy Lutomirski /* This is a nested NMI. */ 1529905a36a2SIngo Molnar 1530905a36a2SIngo Molnarnested_nmi: 1531905a36a2SIngo Molnar /* 15320b22930eSAndy Lutomirski * Modify the "iret" frame to point to repeat_nmi, forcing another 15330b22930eSAndy Lutomirski * iteration of NMI handling. 1534905a36a2SIngo Molnar */ 153523a781e9SAndy Lutomirski subq $8, %rsp 1536905a36a2SIngo Molnar leaq -10*8(%rsp), %rdx 1537905a36a2SIngo Molnar pushq $__KERNEL_DS 1538905a36a2SIngo Molnar pushq %rdx 1539905a36a2SIngo Molnar pushfq 1540905a36a2SIngo Molnar pushq $__KERNEL_CS 1541905a36a2SIngo Molnar pushq $repeat_nmi 1542905a36a2SIngo Molnar 1543905a36a2SIngo Molnar /* Put stack back */ 1544905a36a2SIngo Molnar addq $(6*8), %rsp 1545905a36a2SIngo Molnar 1546905a36a2SIngo Molnarnested_nmi_out: 1547905a36a2SIngo Molnar popq %rdx 1548905a36a2SIngo Molnar 15490b22930eSAndy Lutomirski /* We are returning to kernel mode, so this cannot result in a fault. */ 1550929bacecSAndy Lutomirski iretq 1551905a36a2SIngo Molnar 1552905a36a2SIngo Molnarfirst_nmi: 15530b22930eSAndy Lutomirski /* Restore rdx. */ 1554905a36a2SIngo Molnar movq (%rsp), %rdx 1555905a36a2SIngo Molnar 155636f1a77bSAndy Lutomirski /* Make room for "NMI executing". */ 155736f1a77bSAndy Lutomirski pushq $0 1558905a36a2SIngo Molnar 15590b22930eSAndy Lutomirski /* Leave room for the "iret" frame */ 1560905a36a2SIngo Molnar subq $(5*8), %rsp 1561905a36a2SIngo Molnar 15620b22930eSAndy Lutomirski /* Copy the "original" frame to the "outermost" frame */ 1563905a36a2SIngo Molnar .rept 5 1564905a36a2SIngo Molnar pushq 11*8(%rsp) 1565905a36a2SIngo Molnar .endr 15668c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 1567905a36a2SIngo Molnar 1568905a36a2SIngo Molnar /* Everything up to here is safe from nested NMIs */ 1569905a36a2SIngo Molnar 1570a97439aaSAndy Lutomirski#ifdef CONFIG_DEBUG_ENTRY 1571a97439aaSAndy Lutomirski /* 1572a97439aaSAndy Lutomirski * For ease of testing, unmask NMIs right away. Disabled by 1573a97439aaSAndy Lutomirski * default because IRET is very expensive. 1574a97439aaSAndy Lutomirski */ 1575a97439aaSAndy Lutomirski pushq $0 /* SS */ 1576a97439aaSAndy Lutomirski pushq %rsp /* RSP (minus 8 because of the previous push) */ 1577a97439aaSAndy Lutomirski addq $8, (%rsp) /* Fix up RSP */ 1578a97439aaSAndy Lutomirski pushfq /* RFLAGS */ 1579a97439aaSAndy Lutomirski pushq $__KERNEL_CS /* CS */ 1580a97439aaSAndy Lutomirski pushq $1f /* RIP */ 1581929bacecSAndy Lutomirski iretq /* continues at repeat_nmi below */ 15828c1f7558SJosh Poimboeuf UNWIND_HINT_IRET_REGS 1583a97439aaSAndy Lutomirski1: 1584a97439aaSAndy Lutomirski#endif 1585a97439aaSAndy Lutomirski 15860b22930eSAndy Lutomirskirepeat_nmi: 1587905a36a2SIngo Molnar /* 1588905a36a2SIngo Molnar * If there was a nested NMI, the first NMI's iret will return 1589905a36a2SIngo Molnar * here. But NMIs are still enabled and we can take another 1590905a36a2SIngo Molnar * nested NMI. The nested NMI checks the interrupted RIP to see 1591905a36a2SIngo Molnar * if it is between repeat_nmi and end_repeat_nmi, and if so 1592905a36a2SIngo Molnar * it will just return, as we are about to repeat an NMI anyway. 1593905a36a2SIngo Molnar * This makes it safe to copy to the stack frame that a nested 1594905a36a2SIngo Molnar * NMI will update. 15950b22930eSAndy Lutomirski * 15960b22930eSAndy Lutomirski * RSP is pointing to "outermost RIP". gsbase is unknown, but, if 15970b22930eSAndy Lutomirski * we're repeating an NMI, gsbase has the same value that it had on 15980b22930eSAndy Lutomirski * the first iteration. paranoid_entry will load the kernel 15996271fef0SThomas Gleixner * gsbase if needed before we call exc_nmi(). "NMI executing" 160036f1a77bSAndy Lutomirski * is zero. 1601905a36a2SIngo Molnar */ 160236f1a77bSAndy Lutomirski movq $1, 10*8(%rsp) /* Set "NMI executing". */ 1603905a36a2SIngo Molnar 16040b22930eSAndy Lutomirski /* 16050b22930eSAndy Lutomirski * Copy the "outermost" frame to the "iret" frame. NMIs that nest 16060b22930eSAndy Lutomirski * here must not modify the "iret" frame while we're writing to 16070b22930eSAndy Lutomirski * it or it will end up containing garbage. 16080b22930eSAndy Lutomirski */ 1609905a36a2SIngo Molnar addq $(10*8), %rsp 1610905a36a2SIngo Molnar .rept 5 1611905a36a2SIngo Molnar pushq -6*8(%rsp) 1612905a36a2SIngo Molnar .endr 1613905a36a2SIngo Molnar subq $(5*8), %rsp 1614905a36a2SIngo Molnarend_repeat_nmi: 1615905a36a2SIngo Molnar 1616905a36a2SIngo Molnar /* 16170b22930eSAndy Lutomirski * Everything below this point can be preempted by a nested NMI. 16180b22930eSAndy Lutomirski * If this happens, then the inner NMI will change the "iret" 16190b22930eSAndy Lutomirski * frame to point back to repeat_nmi. 1620905a36a2SIngo Molnar */ 1621905a36a2SIngo Molnar pushq $-1 /* ORIG_RAX: no syscall to restart */ 1622905a36a2SIngo Molnar 1623905a36a2SIngo Molnar /* 1624905a36a2SIngo Molnar * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit 1625905a36a2SIngo Molnar * as we should not be calling schedule in NMI context. 1626905a36a2SIngo Molnar * Even with normal interrupts enabled. An NMI should not be 1627905a36a2SIngo Molnar * setting NEED_RESCHED or anything that normal interrupts and 1628905a36a2SIngo Molnar * exceptions might do. 1629905a36a2SIngo Molnar */ 1630905a36a2SIngo Molnar call paranoid_entry 16318c1f7558SJosh Poimboeuf UNWIND_HINT_REGS 1632905a36a2SIngo Molnar 16336271fef0SThomas Gleixner /* paranoidentry exc_nmi(), 0; without TRACE_IRQS_OFF */ 1634905a36a2SIngo Molnar movq %rsp, %rdi 1635905a36a2SIngo Molnar movq $-1, %rsi 16366271fef0SThomas Gleixner call exc_nmi 1637905a36a2SIngo Molnar 163816561f27SDave Hansen /* Always restore stashed CR3 value (see paranoid_entry) */ 163921e94459SPeter Zijlstra RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 16408a09317bSDave Hansen 1641905a36a2SIngo Molnar testl %ebx, %ebx /* swapgs needed? */ 1642905a36a2SIngo Molnar jnz nmi_restore 1643905a36a2SIngo Molnarnmi_swapgs: 1644905a36a2SIngo Molnar SWAPGS_UNSAFE_STACK 1645905a36a2SIngo Molnarnmi_restore: 1646502af0d7SDominik Brodowski POP_REGS 16470b22930eSAndy Lutomirski 1648471ee483SAndy Lutomirski /* 1649471ee483SAndy Lutomirski * Skip orig_ax and the "outermost" frame to point RSP at the "iret" 1650471ee483SAndy Lutomirski * at the "iret" frame. 1651471ee483SAndy Lutomirski */ 1652471ee483SAndy Lutomirski addq $6*8, %rsp 1653905a36a2SIngo Molnar 1654810bc075SAndy Lutomirski /* 1655810bc075SAndy Lutomirski * Clear "NMI executing". Set DF first so that we can easily 1656810bc075SAndy Lutomirski * distinguish the remaining code between here and IRET from 1657929bacecSAndy Lutomirski * the SYSCALL entry and exit paths. 1658929bacecSAndy Lutomirski * 1659929bacecSAndy Lutomirski * We arguably should just inspect RIP instead, but I (Andy) wrote 1660929bacecSAndy Lutomirski * this code when I had the misapprehension that Xen PV supported 1661929bacecSAndy Lutomirski * NMIs, and Xen PV would break that approach. 1662810bc075SAndy Lutomirski */ 1663810bc075SAndy Lutomirski std 1664810bc075SAndy Lutomirski movq $0, 5*8(%rsp) /* clear "NMI executing" */ 16650b22930eSAndy Lutomirski 16660b22930eSAndy Lutomirski /* 1667929bacecSAndy Lutomirski * iretq reads the "iret" frame and exits the NMI stack in a 1668929bacecSAndy Lutomirski * single instruction. We are returning to kernel mode, so this 1669929bacecSAndy Lutomirski * cannot result in a fault. Similarly, we don't need to worry 1670929bacecSAndy Lutomirski * about espfix64 on the way back to kernel mode. 16710b22930eSAndy Lutomirski */ 1672929bacecSAndy Lutomirski iretq 16736271fef0SThomas GleixnerSYM_CODE_END(asm_exc_nmi) 1674905a36a2SIngo Molnar 1675dffb3f9dSAndy Lutomirski#ifndef CONFIG_IA32_EMULATION 1676dffb3f9dSAndy Lutomirski/* 1677dffb3f9dSAndy Lutomirski * This handles SYSCALL from 32-bit code. There is no way to program 1678dffb3f9dSAndy Lutomirski * MSRs to fully disable 32-bit SYSCALL. 1679dffb3f9dSAndy Lutomirski */ 1680bc7b11c0SJiri SlabySYM_CODE_START(ignore_sysret) 16818c1f7558SJosh Poimboeuf UNWIND_HINT_EMPTY 1682905a36a2SIngo Molnar mov $-ENOSYS, %eax 1683b2b1d94cSJan Beulich sysretl 1684bc7b11c0SJiri SlabySYM_CODE_END(ignore_sysret) 1685dffb3f9dSAndy Lutomirski#endif 16862deb4be2SAndy Lutomirski 1687b9f6976bSThomas Gleixner.pushsection .text, "ax" 1688bc7b11c0SJiri SlabySYM_CODE_START(rewind_stack_do_exit) 16898c1f7558SJosh Poimboeuf UNWIND_HINT_FUNC 16902deb4be2SAndy Lutomirski /* Prevent any naive code from trying to unwind to our caller. */ 16912deb4be2SAndy Lutomirski xorl %ebp, %ebp 16922deb4be2SAndy Lutomirski 16932deb4be2SAndy Lutomirski movq PER_CPU_VAR(cpu_current_top_of_stack), %rax 16948c1f7558SJosh Poimboeuf leaq -PTREGS_SIZE(%rax), %rsp 1695f977df7bSJann Horn UNWIND_HINT_REGS 16962deb4be2SAndy Lutomirski 16972deb4be2SAndy Lutomirski call do_exit 1698bc7b11c0SJiri SlabySYM_CODE_END(rewind_stack_do_exit) 1699b9f6976bSThomas Gleixner.popsection 1700