1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * linux/arch/x86_64/entry.S 4 * 5 * Copyright (C) 1991, 1992 Linus Torvalds 6 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs 7 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz> 8 * 9 * entry.S contains the system-call and fault low-level handling routines. 10 * 11 * Some of this is documented in Documentation/x86/entry_64.rst 12 * 13 * A note on terminology: 14 * - iret frame: Architecture defined interrupt frame from SS to RIP 15 * at the top of the kernel process stack. 16 * 17 * Some macro usage: 18 * - SYM_FUNC_START/END:Define functions in the symbol table. 19 * - idtentry: Define exception entry points. 20 */ 21#include <linux/linkage.h> 22#include <asm/segment.h> 23#include <asm/cache.h> 24#include <asm/errno.h> 25#include <asm/asm-offsets.h> 26#include <asm/msr.h> 27#include <asm/unistd.h> 28#include <asm/thread_info.h> 29#include <asm/hw_irq.h> 30#include <asm/page_types.h> 31#include <asm/irqflags.h> 32#include <asm/paravirt.h> 33#include <asm/percpu.h> 34#include <asm/asm.h> 35#include <asm/smap.h> 36#include <asm/pgtable_types.h> 37#include <asm/export.h> 38#include <asm/frame.h> 39#include <asm/trapnr.h> 40#include <asm/nospec-branch.h> 41#include <linux/err.h> 42 43#include "calling.h" 44 45.code64 46.section .entry.text, "ax" 47 48#ifdef CONFIG_PARAVIRT 49SYM_CODE_START(native_usergs_sysret64) 50 UNWIND_HINT_EMPTY 51 swapgs 52 sysretq 53SYM_CODE_END(native_usergs_sysret64) 54#endif /* CONFIG_PARAVIRT */ 55 56/* 57 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers. 58 * 59 * This is the only entry point used for 64-bit system calls. The 60 * hardware interface is reasonably well designed and the register to 61 * argument mapping Linux uses fits well with the registers that are 62 * available when SYSCALL is used. 63 * 64 * SYSCALL instructions can be found inlined in libc implementations as 65 * well as some other programs and libraries. There are also a handful 66 * of SYSCALL instructions in the vDSO used, for example, as a 67 * clock_gettimeofday fallback. 68 * 69 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11, 70 * then loads new ss, cs, and rip from previously programmed MSRs. 71 * rflags gets masked by a value from another MSR (so CLD and CLAC 72 * are not needed). SYSCALL does not save anything on the stack 73 * and does not change rsp. 74 * 75 * Registers on entry: 76 * rax system call number 77 * rcx return address 78 * r11 saved rflags (note: r11 is callee-clobbered register in C ABI) 79 * rdi arg0 80 * rsi arg1 81 * rdx arg2 82 * r10 arg3 (needs to be moved to rcx to conform to C ABI) 83 * r8 arg4 84 * r9 arg5 85 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI) 86 * 87 * Only called from user space. 88 * 89 * When user can change pt_regs->foo always force IRET. That is because 90 * it deals with uncanonical addresses better. SYSRET has trouble 91 * with them due to bugs in both AMD and Intel CPUs. 92 */ 93 94SYM_CODE_START(entry_SYSCALL_64) 95 UNWIND_HINT_EMPTY 96 97 swapgs 98 /* tss.sp2 is scratch space. */ 99 movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2) 100 SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp 101 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 102 103 /* Construct struct pt_regs on stack */ 104 pushq $__USER_DS /* pt_regs->ss */ 105 pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */ 106 pushq %r11 /* pt_regs->flags */ 107 pushq $__USER_CS /* pt_regs->cs */ 108 pushq %rcx /* pt_regs->ip */ 109SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL) 110 pushq %rax /* pt_regs->orig_ax */ 111 112 PUSH_AND_CLEAR_REGS rax=$-ENOSYS 113 114 /* IRQs are off. */ 115 movq %rax, %rdi 116 movq %rsp, %rsi 117 call do_syscall_64 /* returns with IRQs disabled */ 118 119 /* 120 * Try to use SYSRET instead of IRET if we're returning to 121 * a completely clean 64-bit userspace context. If we're not, 122 * go to the slow exit path. 123 */ 124 movq RCX(%rsp), %rcx 125 movq RIP(%rsp), %r11 126 127 cmpq %rcx, %r11 /* SYSRET requires RCX == RIP */ 128 jne swapgs_restore_regs_and_return_to_usermode 129 130 /* 131 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP 132 * in kernel space. This essentially lets the user take over 133 * the kernel, since userspace controls RSP. 134 * 135 * If width of "canonical tail" ever becomes variable, this will need 136 * to be updated to remain correct on both old and new CPUs. 137 * 138 * Change top bits to match most significant bit (47th or 56th bit 139 * depending on paging mode) in the address. 140 */ 141#ifdef CONFIG_X86_5LEVEL 142 ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \ 143 "shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57 144#else 145 shl $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx 146 sar $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx 147#endif 148 149 /* If this changed %rcx, it was not canonical */ 150 cmpq %rcx, %r11 151 jne swapgs_restore_regs_and_return_to_usermode 152 153 cmpq $__USER_CS, CS(%rsp) /* CS must match SYSRET */ 154 jne swapgs_restore_regs_and_return_to_usermode 155 156 movq R11(%rsp), %r11 157 cmpq %r11, EFLAGS(%rsp) /* R11 == RFLAGS */ 158 jne swapgs_restore_regs_and_return_to_usermode 159 160 /* 161 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot 162 * restore RF properly. If the slowpath sets it for whatever reason, we 163 * need to restore it correctly. 164 * 165 * SYSRET can restore TF, but unlike IRET, restoring TF results in a 166 * trap from userspace immediately after SYSRET. This would cause an 167 * infinite loop whenever #DB happens with register state that satisfies 168 * the opportunistic SYSRET conditions. For example, single-stepping 169 * this user code: 170 * 171 * movq $stuck_here, %rcx 172 * pushfq 173 * popq %r11 174 * stuck_here: 175 * 176 * would never get past 'stuck_here'. 177 */ 178 testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11 179 jnz swapgs_restore_regs_and_return_to_usermode 180 181 /* nothing to check for RSP */ 182 183 cmpq $__USER_DS, SS(%rsp) /* SS must match SYSRET */ 184 jne swapgs_restore_regs_and_return_to_usermode 185 186 /* 187 * We win! This label is here just for ease of understanding 188 * perf profiles. Nothing jumps here. 189 */ 190syscall_return_via_sysret: 191 /* rcx and r11 are already restored (see code above) */ 192 POP_REGS pop_rdi=0 skip_r11rcx=1 193 194 /* 195 * Now all regs are restored except RSP and RDI. 196 * Save old stack pointer and switch to trampoline stack. 197 */ 198 movq %rsp, %rdi 199 movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp 200 UNWIND_HINT_EMPTY 201 202 pushq RSP-RDI(%rdi) /* RSP */ 203 pushq (%rdi) /* RDI */ 204 205 /* 206 * We are on the trampoline stack. All regs except RDI are live. 207 * We can do future final exit work right here. 208 */ 209 STACKLEAK_ERASE_NOCLOBBER 210 211 SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 212 213 popq %rdi 214 popq %rsp 215 USERGS_SYSRET64 216SYM_CODE_END(entry_SYSCALL_64) 217 218/* 219 * %rdi: prev task 220 * %rsi: next task 221 */ 222.pushsection .text, "ax" 223SYM_FUNC_START(__switch_to_asm) 224 /* 225 * Save callee-saved registers 226 * This must match the order in inactive_task_frame 227 */ 228 pushq %rbp 229 pushq %rbx 230 pushq %r12 231 pushq %r13 232 pushq %r14 233 pushq %r15 234 235 /* switch stack */ 236 movq %rsp, TASK_threadsp(%rdi) 237 movq TASK_threadsp(%rsi), %rsp 238 239#ifdef CONFIG_STACKPROTECTOR 240 movq TASK_stack_canary(%rsi), %rbx 241 movq %rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset 242#endif 243 244#ifdef CONFIG_RETPOLINE 245 /* 246 * When switching from a shallower to a deeper call stack 247 * the RSB may either underflow or use entries populated 248 * with userspace addresses. On CPUs where those concerns 249 * exist, overwrite the RSB with entries which capture 250 * speculative execution to prevent attack. 251 */ 252 FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW 253#endif 254 255 /* restore callee-saved registers */ 256 popq %r15 257 popq %r14 258 popq %r13 259 popq %r12 260 popq %rbx 261 popq %rbp 262 263 jmp __switch_to 264SYM_FUNC_END(__switch_to_asm) 265.popsection 266 267/* 268 * A newly forked process directly context switches into this address. 269 * 270 * rax: prev task we switched from 271 * rbx: kernel thread func (NULL for user thread) 272 * r12: kernel thread arg 273 */ 274.pushsection .text, "ax" 275SYM_CODE_START(ret_from_fork) 276 UNWIND_HINT_EMPTY 277 movq %rax, %rdi 278 call schedule_tail /* rdi: 'prev' task parameter */ 279 280 testq %rbx, %rbx /* from kernel_thread? */ 281 jnz 1f /* kernel threads are uncommon */ 282 2832: 284 UNWIND_HINT_REGS 285 movq %rsp, %rdi 286 call syscall_return_slowpath /* returns with IRQs disabled */ 287 jmp swapgs_restore_regs_and_return_to_usermode 288 2891: 290 /* kernel thread */ 291 UNWIND_HINT_EMPTY 292 movq %r12, %rdi 293 CALL_NOSPEC rbx 294 /* 295 * A kernel thread is allowed to return here after successfully 296 * calling do_execve(). Exit to userspace to complete the execve() 297 * syscall. 298 */ 299 movq $0, RAX(%rsp) 300 jmp 2b 301SYM_CODE_END(ret_from_fork) 302.popsection 303 304.macro DEBUG_ENTRY_ASSERT_IRQS_OFF 305#ifdef CONFIG_DEBUG_ENTRY 306 pushq %rax 307 SAVE_FLAGS(CLBR_RAX) 308 testl $X86_EFLAGS_IF, %eax 309 jz .Lokay_\@ 310 ud2 311.Lokay_\@: 312 popq %rax 313#endif 314.endm 315 316/** 317 * idtentry_body - Macro to emit code calling the C function 318 * @cfunc: C function to be called 319 * @has_error_code: Hardware pushed error code on stack 320 */ 321.macro idtentry_body cfunc has_error_code:req 322 323 call error_entry 324 UNWIND_HINT_REGS 325 326 movq %rsp, %rdi /* pt_regs pointer into 1st argument*/ 327 328 .if \has_error_code == 1 329 movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ 330 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ 331 .endif 332 333 call \cfunc 334 335 jmp error_return 336.endm 337 338/** 339 * idtentry - Macro to generate entry stubs for simple IDT entries 340 * @vector: Vector number 341 * @asmsym: ASM symbol for the entry point 342 * @cfunc: C function to be called 343 * @has_error_code: Hardware pushed error code on stack 344 * 345 * The macro emits code to set up the kernel context for straight forward 346 * and simple IDT entries. No IST stack, no paranoid entry checks. 347 */ 348.macro idtentry vector asmsym cfunc has_error_code:req 349SYM_CODE_START(\asmsym) 350 UNWIND_HINT_IRET_REGS offset=\has_error_code*8 351 ASM_CLAC 352 353 .if \has_error_code == 0 354 pushq $-1 /* ORIG_RAX: no syscall to restart */ 355 .endif 356 357 .if \vector == X86_TRAP_BP 358 /* 359 * If coming from kernel space, create a 6-word gap to allow the 360 * int3 handler to emulate a call instruction. 361 */ 362 testb $3, CS-ORIG_RAX(%rsp) 363 jnz .Lfrom_usermode_no_gap_\@ 364 .rept 6 365 pushq 5*8(%rsp) 366 .endr 367 UNWIND_HINT_IRET_REGS offset=8 368.Lfrom_usermode_no_gap_\@: 369 .endif 370 371 idtentry_body \cfunc \has_error_code 372 373_ASM_NOKPROBE(\asmsym) 374SYM_CODE_END(\asmsym) 375.endm 376 377/* 378 * Interrupt entry/exit. 379 * 380 + The interrupt stubs push (vector) onto the stack, which is the error_code 381 * position of idtentry exceptions, and jump to one of the two idtentry points 382 * (common/spurious). 383 * 384 * common_interrupt is a hotpath, align it to a cache line 385 */ 386.macro idtentry_irq vector cfunc 387 .p2align CONFIG_X86_L1_CACHE_SHIFT 388 idtentry \vector asm_\cfunc \cfunc has_error_code=1 389.endm 390 391/* 392 * System vectors which invoke their handlers directly and are not 393 * going through the regular common device interrupt handling code. 394 */ 395.macro idtentry_sysvec vector cfunc 396 idtentry \vector asm_\cfunc \cfunc has_error_code=0 397.endm 398 399/** 400 * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB 401 * @vector: Vector number 402 * @asmsym: ASM symbol for the entry point 403 * @cfunc: C function to be called 404 * 405 * The macro emits code to set up the kernel context for #MC and #DB 406 * 407 * If the entry comes from user space it uses the normal entry path 408 * including the return to user space work and preemption checks on 409 * exit. 410 * 411 * If hits in kernel mode then it needs to go through the paranoid 412 * entry as the exception can hit any random state. No preemption 413 * check on exit to keep the paranoid path simple. 414 */ 415.macro idtentry_mce_db vector asmsym cfunc 416SYM_CODE_START(\asmsym) 417 UNWIND_HINT_IRET_REGS 418 ASM_CLAC 419 420 pushq $-1 /* ORIG_RAX: no syscall to restart */ 421 422 /* 423 * If the entry is from userspace, switch stacks and treat it as 424 * a normal entry. 425 */ 426 testb $3, CS-ORIG_RAX(%rsp) 427 jnz .Lfrom_usermode_switch_stack_\@ 428 429 /* 430 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX. 431 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS 432 */ 433 call paranoid_entry 434 435 UNWIND_HINT_REGS 436 437 movq %rsp, %rdi /* pt_regs pointer */ 438 439 call \cfunc 440 441 jmp paranoid_exit 442 443 /* Switch to the regular task stack and use the noist entry point */ 444.Lfrom_usermode_switch_stack_\@: 445 idtentry_body noist_\cfunc, has_error_code=0 446 447_ASM_NOKPROBE(\asmsym) 448SYM_CODE_END(\asmsym) 449.endm 450 451/* 452 * Double fault entry. Straight paranoid. No checks from which context 453 * this comes because for the espfix induced #DF this would do the wrong 454 * thing. 455 */ 456.macro idtentry_df vector asmsym cfunc 457SYM_CODE_START(\asmsym) 458 UNWIND_HINT_IRET_REGS offset=8 459 ASM_CLAC 460 461 /* 462 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX. 463 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS 464 */ 465 call paranoid_entry 466 UNWIND_HINT_REGS 467 468 movq %rsp, %rdi /* pt_regs pointer into first argument */ 469 movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ 470 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ 471 call \cfunc 472 473 jmp paranoid_exit 474 475_ASM_NOKPROBE(\asmsym) 476SYM_CODE_END(\asmsym) 477.endm 478 479/* 480 * Include the defines which emit the idt entries which are shared 481 * shared between 32 and 64 bit and emit the __irqentry_text_* markers 482 * so the stacktrace boundary checks work. 483 */ 484 .align 16 485 .globl __irqentry_text_start 486__irqentry_text_start: 487 488#include <asm/idtentry.h> 489 490 .align 16 491 .globl __irqentry_text_end 492__irqentry_text_end: 493 494SYM_CODE_START_LOCAL(common_interrupt_return) 495SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) 496#ifdef CONFIG_DEBUG_ENTRY 497 /* Assert that pt_regs indicates user mode. */ 498 testb $3, CS(%rsp) 499 jnz 1f 500 ud2 5011: 502#endif 503 POP_REGS pop_rdi=0 504 505 /* 506 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS. 507 * Save old stack pointer and switch to trampoline stack. 508 */ 509 movq %rsp, %rdi 510 movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp 511 UNWIND_HINT_EMPTY 512 513 /* Copy the IRET frame to the trampoline stack. */ 514 pushq 6*8(%rdi) /* SS */ 515 pushq 5*8(%rdi) /* RSP */ 516 pushq 4*8(%rdi) /* EFLAGS */ 517 pushq 3*8(%rdi) /* CS */ 518 pushq 2*8(%rdi) /* RIP */ 519 520 /* Push user RDI on the trampoline stack. */ 521 pushq (%rdi) 522 523 /* 524 * We are on the trampoline stack. All regs except RDI are live. 525 * We can do future final exit work right here. 526 */ 527 STACKLEAK_ERASE_NOCLOBBER 528 529 SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 530 531 /* Restore RDI. */ 532 popq %rdi 533 SWAPGS 534 INTERRUPT_RETURN 535 536 537SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) 538#ifdef CONFIG_DEBUG_ENTRY 539 /* Assert that pt_regs indicates kernel mode. */ 540 testb $3, CS(%rsp) 541 jz 1f 542 ud2 5431: 544#endif 545 POP_REGS 546 addq $8, %rsp /* skip regs->orig_ax */ 547 /* 548 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization 549 * when returning from IPI handler. 550 */ 551 INTERRUPT_RETURN 552 553SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL) 554 UNWIND_HINT_IRET_REGS 555 /* 556 * Are we returning to a stack segment from the LDT? Note: in 557 * 64-bit mode SS:RSP on the exception stack is always valid. 558 */ 559#ifdef CONFIG_X86_ESPFIX64 560 testb $4, (SS-RIP)(%rsp) 561 jnz native_irq_return_ldt 562#endif 563 564SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL) 565 /* 566 * This may fault. Non-paranoid faults on return to userspace are 567 * handled by fixup_bad_iret. These include #SS, #GP, and #NP. 568 * Double-faults due to espfix64 are handled in exc_double_fault. 569 * Other faults here are fatal. 570 */ 571 iretq 572 573#ifdef CONFIG_X86_ESPFIX64 574native_irq_return_ldt: 575 /* 576 * We are running with user GSBASE. All GPRs contain their user 577 * values. We have a percpu ESPFIX stack that is eight slots 578 * long (see ESPFIX_STACK_SIZE). espfix_waddr points to the bottom 579 * of the ESPFIX stack. 580 * 581 * We clobber RAX and RDI in this code. We stash RDI on the 582 * normal stack and RAX on the ESPFIX stack. 583 * 584 * The ESPFIX stack layout we set up looks like this: 585 * 586 * --- top of ESPFIX stack --- 587 * SS 588 * RSP 589 * RFLAGS 590 * CS 591 * RIP <-- RSP points here when we're done 592 * RAX <-- espfix_waddr points here 593 * --- bottom of ESPFIX stack --- 594 */ 595 596 pushq %rdi /* Stash user RDI */ 597 SWAPGS /* to kernel GS */ 598 SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */ 599 600 movq PER_CPU_VAR(espfix_waddr), %rdi 601 movq %rax, (0*8)(%rdi) /* user RAX */ 602 movq (1*8)(%rsp), %rax /* user RIP */ 603 movq %rax, (1*8)(%rdi) 604 movq (2*8)(%rsp), %rax /* user CS */ 605 movq %rax, (2*8)(%rdi) 606 movq (3*8)(%rsp), %rax /* user RFLAGS */ 607 movq %rax, (3*8)(%rdi) 608 movq (5*8)(%rsp), %rax /* user SS */ 609 movq %rax, (5*8)(%rdi) 610 movq (4*8)(%rsp), %rax /* user RSP */ 611 movq %rax, (4*8)(%rdi) 612 /* Now RAX == RSP. */ 613 614 andl $0xffff0000, %eax /* RAX = (RSP & 0xffff0000) */ 615 616 /* 617 * espfix_stack[31:16] == 0. The page tables are set up such that 618 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of 619 * espfix_waddr for any X. That is, there are 65536 RO aliases of 620 * the same page. Set up RSP so that RSP[31:16] contains the 621 * respective 16 bits of the /userspace/ RSP and RSP nonetheless 622 * still points to an RO alias of the ESPFIX stack. 623 */ 624 orq PER_CPU_VAR(espfix_stack), %rax 625 626 SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi 627 SWAPGS /* to user GS */ 628 popq %rdi /* Restore user RDI */ 629 630 movq %rax, %rsp 631 UNWIND_HINT_IRET_REGS offset=8 632 633 /* 634 * At this point, we cannot write to the stack any more, but we can 635 * still read. 636 */ 637 popq %rax /* Restore user RAX */ 638 639 /* 640 * RSP now points to an ordinary IRET frame, except that the page 641 * is read-only and RSP[31:16] are preloaded with the userspace 642 * values. We can now IRET back to userspace. 643 */ 644 jmp native_irq_return_iret 645#endif 646SYM_CODE_END(common_interrupt_return) 647_ASM_NOKPROBE(common_interrupt_return) 648 649/* 650 * Reload gs selector with exception handling 651 * edi: new selector 652 * 653 * Is in entry.text as it shouldn't be instrumented. 654 */ 655SYM_FUNC_START(asm_load_gs_index) 656 FRAME_BEGIN 657 swapgs 658.Lgs_change: 659 movl %edi, %gs 6602: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE 661 swapgs 662 FRAME_END 663 ret 664SYM_FUNC_END(asm_load_gs_index) 665EXPORT_SYMBOL(asm_load_gs_index) 666 667 _ASM_EXTABLE(.Lgs_change, .Lbad_gs) 668 .section .fixup, "ax" 669 /* running with kernelgs */ 670SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) 671 swapgs /* switch back to user gs */ 672.macro ZAP_GS 673 /* This can't be a string because the preprocessor needs to see it. */ 674 movl $__USER_DS, %eax 675 movl %eax, %gs 676.endm 677 ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG 678 xorl %eax, %eax 679 movl %eax, %gs 680 jmp 2b 681SYM_CODE_END(.Lbad_gs) 682 .previous 683 684/* 685 * rdi: New stack pointer points to the top word of the stack 686 * rsi: Function pointer 687 * rdx: Function argument (can be NULL if none) 688 */ 689SYM_FUNC_START(asm_call_on_stack) 690 /* 691 * Save the frame pointer unconditionally. This allows the ORC 692 * unwinder to handle the stack switch. 693 */ 694 pushq %rbp 695 mov %rsp, %rbp 696 697 /* 698 * The unwinder relies on the word at the top of the new stack 699 * page linking back to the previous RSP. 700 */ 701 mov %rsp, (%rdi) 702 mov %rdi, %rsp 703 /* Move the argument to the right place */ 704 mov %rdx, %rdi 705 7061: 707 .pushsection .discard.instr_begin 708 .long 1b - . 709 .popsection 710 711 CALL_NOSPEC rsi 712 7132: 714 .pushsection .discard.instr_end 715 .long 2b - . 716 .popsection 717 718 /* Restore the previous stack pointer from RBP. */ 719 leaveq 720 ret 721SYM_FUNC_END(asm_call_on_stack) 722 723#ifdef CONFIG_XEN_PV 724/* 725 * A note on the "critical region" in our callback handler. 726 * We want to avoid stacking callback handlers due to events occurring 727 * during handling of the last event. To do this, we keep events disabled 728 * until we've done all processing. HOWEVER, we must enable events before 729 * popping the stack frame (can't be done atomically) and so it would still 730 * be possible to get enough handler activations to overflow the stack. 731 * Although unlikely, bugs of that kind are hard to track down, so we'd 732 * like to avoid the possibility. 733 * So, on entry to the handler we detect whether we interrupted an 734 * existing activation in its critical region -- if so, we pop the current 735 * activation and restart the handler using the previous one. 736 * 737 * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs) 738 */ 739SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback) 740 741/* 742 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will 743 * see the correct pointer to the pt_regs 744 */ 745 UNWIND_HINT_FUNC 746 movq %rdi, %rsp /* we don't return, adjust the stack frame */ 747 UNWIND_HINT_REGS 748 749 call xen_pv_evtchn_do_upcall 750 751 jmp error_return 752SYM_CODE_END(exc_xen_hypervisor_callback) 753 754/* 755 * Hypervisor uses this for application faults while it executes. 756 * We get here for two reasons: 757 * 1. Fault while reloading DS, ES, FS or GS 758 * 2. Fault while executing IRET 759 * Category 1 we do not need to fix up as Xen has already reloaded all segment 760 * registers that could be reloaded and zeroed the others. 761 * Category 2 we fix up by killing the current process. We cannot use the 762 * normal Linux return path in this case because if we use the IRET hypercall 763 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 764 * We distinguish between categories by comparing each saved segment register 765 * with its current contents: any discrepancy means we in category 1. 766 */ 767SYM_CODE_START(xen_failsafe_callback) 768 UNWIND_HINT_EMPTY 769 movl %ds, %ecx 770 cmpw %cx, 0x10(%rsp) 771 jne 1f 772 movl %es, %ecx 773 cmpw %cx, 0x18(%rsp) 774 jne 1f 775 movl %fs, %ecx 776 cmpw %cx, 0x20(%rsp) 777 jne 1f 778 movl %gs, %ecx 779 cmpw %cx, 0x28(%rsp) 780 jne 1f 781 /* All segments match their saved values => Category 2 (Bad IRET). */ 782 movq (%rsp), %rcx 783 movq 8(%rsp), %r11 784 addq $0x30, %rsp 785 pushq $0 /* RIP */ 786 UNWIND_HINT_IRET_REGS offset=8 787 jmp asm_exc_general_protection 7881: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */ 789 movq (%rsp), %rcx 790 movq 8(%rsp), %r11 791 addq $0x30, %rsp 792 UNWIND_HINT_IRET_REGS 793 pushq $-1 /* orig_ax = -1 => not a system call */ 794 PUSH_AND_CLEAR_REGS 795 ENCODE_FRAME_POINTER 796 jmp error_return 797SYM_CODE_END(xen_failsafe_callback) 798#endif /* CONFIG_XEN_PV */ 799 800/* 801 * Save all registers in pt_regs, and switch gs if needed. 802 * Use slow, but surefire "are we in kernel?" check. 803 * Return: ebx=0: need swapgs on exit, ebx=1: otherwise 804 */ 805SYM_CODE_START_LOCAL(paranoid_entry) 806 UNWIND_HINT_FUNC 807 cld 808 PUSH_AND_CLEAR_REGS save_ret=1 809 ENCODE_FRAME_POINTER 8 810 movl $1, %ebx 811 movl $MSR_GS_BASE, %ecx 812 rdmsr 813 testl %edx, %edx 814 js 1f /* negative -> in kernel */ 815 SWAPGS 816 xorl %ebx, %ebx 817 8181: 819 /* 820 * Always stash CR3 in %r14. This value will be restored, 821 * verbatim, at exit. Needed if paranoid_entry interrupted 822 * another entry that already switched to the user CR3 value 823 * but has not yet returned to userspace. 824 * 825 * This is also why CS (stashed in the "iret frame" by the 826 * hardware at entry) can not be used: this may be a return 827 * to kernel code, but with a user CR3 value. 828 */ 829 SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14 830 831 /* 832 * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an 833 * unconditional CR3 write, even in the PTI case. So do an lfence 834 * to prevent GS speculation, regardless of whether PTI is enabled. 835 */ 836 FENCE_SWAPGS_KERNEL_ENTRY 837 838 ret 839SYM_CODE_END(paranoid_entry) 840 841/* 842 * "Paranoid" exit path from exception stack. This is invoked 843 * only on return from non-NMI IST interrupts that came 844 * from kernel space. 845 * 846 * We may be returning to very strange contexts (e.g. very early 847 * in syscall entry), so checking for preemption here would 848 * be complicated. Fortunately, we there's no good reason 849 * to try to handle preemption here. 850 * 851 * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) 852 */ 853SYM_CODE_START_LOCAL(paranoid_exit) 854 UNWIND_HINT_REGS 855 testl %ebx, %ebx /* swapgs needed? */ 856 jnz .Lparanoid_exit_no_swapgs 857 /* Always restore stashed CR3 value (see paranoid_entry) */ 858 RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 859 SWAPGS_UNSAFE_STACK 860 jmp restore_regs_and_return_to_kernel 861.Lparanoid_exit_no_swapgs: 862 /* Always restore stashed CR3 value (see paranoid_entry) */ 863 RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 864 jmp restore_regs_and_return_to_kernel 865SYM_CODE_END(paranoid_exit) 866 867/* 868 * Save all registers in pt_regs, and switch GS if needed. 869 */ 870SYM_CODE_START_LOCAL(error_entry) 871 UNWIND_HINT_FUNC 872 cld 873 PUSH_AND_CLEAR_REGS save_ret=1 874 ENCODE_FRAME_POINTER 8 875 testb $3, CS+8(%rsp) 876 jz .Lerror_kernelspace 877 878 /* 879 * We entered from user mode or we're pretending to have entered 880 * from user mode due to an IRET fault. 881 */ 882 SWAPGS 883 FENCE_SWAPGS_USER_ENTRY 884 /* We have user CR3. Change to kernel CR3. */ 885 SWITCH_TO_KERNEL_CR3 scratch_reg=%rax 886 887.Lerror_entry_from_usermode_after_swapgs: 888 /* Put us onto the real thread stack. */ 889 popq %r12 /* save return addr in %12 */ 890 movq %rsp, %rdi /* arg0 = pt_regs pointer */ 891 call sync_regs 892 movq %rax, %rsp /* switch stack */ 893 ENCODE_FRAME_POINTER 894 pushq %r12 895 ret 896 897.Lerror_entry_done_lfence: 898 FENCE_SWAPGS_KERNEL_ENTRY 899.Lerror_entry_done: 900 ret 901 902 /* 903 * There are two places in the kernel that can potentially fault with 904 * usergs. Handle them here. B stepping K8s sometimes report a 905 * truncated RIP for IRET exceptions returning to compat mode. Check 906 * for these here too. 907 */ 908.Lerror_kernelspace: 909 leaq native_irq_return_iret(%rip), %rcx 910 cmpq %rcx, RIP+8(%rsp) 911 je .Lerror_bad_iret 912 movl %ecx, %eax /* zero extend */ 913 cmpq %rax, RIP+8(%rsp) 914 je .Lbstep_iret 915 cmpq $.Lgs_change, RIP+8(%rsp) 916 jne .Lerror_entry_done_lfence 917 918 /* 919 * hack: .Lgs_change can fail with user gsbase. If this happens, fix up 920 * gsbase and proceed. We'll fix up the exception and land in 921 * .Lgs_change's error handler with kernel gsbase. 922 */ 923 SWAPGS 924 FENCE_SWAPGS_USER_ENTRY 925 jmp .Lerror_entry_done 926 927.Lbstep_iret: 928 /* Fix truncated RIP */ 929 movq %rcx, RIP+8(%rsp) 930 /* fall through */ 931 932.Lerror_bad_iret: 933 /* 934 * We came from an IRET to user mode, so we have user 935 * gsbase and CR3. Switch to kernel gsbase and CR3: 936 */ 937 SWAPGS 938 FENCE_SWAPGS_USER_ENTRY 939 SWITCH_TO_KERNEL_CR3 scratch_reg=%rax 940 941 /* 942 * Pretend that the exception came from user mode: set up pt_regs 943 * as if we faulted immediately after IRET. 944 */ 945 mov %rsp, %rdi 946 call fixup_bad_iret 947 mov %rax, %rsp 948 jmp .Lerror_entry_from_usermode_after_swapgs 949SYM_CODE_END(error_entry) 950 951SYM_CODE_START_LOCAL(error_return) 952 UNWIND_HINT_REGS 953 DEBUG_ENTRY_ASSERT_IRQS_OFF 954 testb $3, CS(%rsp) 955 jz restore_regs_and_return_to_kernel 956 jmp swapgs_restore_regs_and_return_to_usermode 957SYM_CODE_END(error_return) 958 959/* 960 * Runs on exception stack. Xen PV does not go through this path at all, 961 * so we can use real assembly here. 962 * 963 * Registers: 964 * %r14: Used to save/restore the CR3 of the interrupted context 965 * when PAGE_TABLE_ISOLATION is in use. Do not clobber. 966 */ 967SYM_CODE_START(asm_exc_nmi) 968 UNWIND_HINT_IRET_REGS 969 970 /* 971 * We allow breakpoints in NMIs. If a breakpoint occurs, then 972 * the iretq it performs will take us out of NMI context. 973 * This means that we can have nested NMIs where the next 974 * NMI is using the top of the stack of the previous NMI. We 975 * can't let it execute because the nested NMI will corrupt the 976 * stack of the previous NMI. NMI handlers are not re-entrant 977 * anyway. 978 * 979 * To handle this case we do the following: 980 * Check the a special location on the stack that contains 981 * a variable that is set when NMIs are executing. 982 * The interrupted task's stack is also checked to see if it 983 * is an NMI stack. 984 * If the variable is not set and the stack is not the NMI 985 * stack then: 986 * o Set the special variable on the stack 987 * o Copy the interrupt frame into an "outermost" location on the 988 * stack 989 * o Copy the interrupt frame into an "iret" location on the stack 990 * o Continue processing the NMI 991 * If the variable is set or the previous stack is the NMI stack: 992 * o Modify the "iret" location to jump to the repeat_nmi 993 * o return back to the first NMI 994 * 995 * Now on exit of the first NMI, we first clear the stack variable 996 * The NMI stack will tell any nested NMIs at that point that it is 997 * nested. Then we pop the stack normally with iret, and if there was 998 * a nested NMI that updated the copy interrupt stack frame, a 999 * jump will be made to the repeat_nmi code that will handle the second 1000 * NMI. 1001 * 1002 * However, espfix prevents us from directly returning to userspace 1003 * with a single IRET instruction. Similarly, IRET to user mode 1004 * can fault. We therefore handle NMIs from user space like 1005 * other IST entries. 1006 */ 1007 1008 ASM_CLAC 1009 1010 /* Use %rdx as our temp variable throughout */ 1011 pushq %rdx 1012 1013 testb $3, CS-RIP+8(%rsp) 1014 jz .Lnmi_from_kernel 1015 1016 /* 1017 * NMI from user mode. We need to run on the thread stack, but we 1018 * can't go through the normal entry paths: NMIs are masked, and 1019 * we don't want to enable interrupts, because then we'll end 1020 * up in an awkward situation in which IRQs are on but NMIs 1021 * are off. 1022 * 1023 * We also must not push anything to the stack before switching 1024 * stacks lest we corrupt the "NMI executing" variable. 1025 */ 1026 1027 swapgs 1028 cld 1029 FENCE_SWAPGS_USER_ENTRY 1030 SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx 1031 movq %rsp, %rdx 1032 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp 1033 UNWIND_HINT_IRET_REGS base=%rdx offset=8 1034 pushq 5*8(%rdx) /* pt_regs->ss */ 1035 pushq 4*8(%rdx) /* pt_regs->rsp */ 1036 pushq 3*8(%rdx) /* pt_regs->flags */ 1037 pushq 2*8(%rdx) /* pt_regs->cs */ 1038 pushq 1*8(%rdx) /* pt_regs->rip */ 1039 UNWIND_HINT_IRET_REGS 1040 pushq $-1 /* pt_regs->orig_ax */ 1041 PUSH_AND_CLEAR_REGS rdx=(%rdx) 1042 ENCODE_FRAME_POINTER 1043 1044 /* 1045 * At this point we no longer need to worry about stack damage 1046 * due to nesting -- we're on the normal thread stack and we're 1047 * done with the NMI stack. 1048 */ 1049 1050 movq %rsp, %rdi 1051 movq $-1, %rsi 1052 call exc_nmi 1053 1054 /* 1055 * Return back to user mode. We must *not* do the normal exit 1056 * work, because we don't want to enable interrupts. 1057 */ 1058 jmp swapgs_restore_regs_and_return_to_usermode 1059 1060.Lnmi_from_kernel: 1061 /* 1062 * Here's what our stack frame will look like: 1063 * +---------------------------------------------------------+ 1064 * | original SS | 1065 * | original Return RSP | 1066 * | original RFLAGS | 1067 * | original CS | 1068 * | original RIP | 1069 * +---------------------------------------------------------+ 1070 * | temp storage for rdx | 1071 * +---------------------------------------------------------+ 1072 * | "NMI executing" variable | 1073 * +---------------------------------------------------------+ 1074 * | iret SS } Copied from "outermost" frame | 1075 * | iret Return RSP } on each loop iteration; overwritten | 1076 * | iret RFLAGS } by a nested NMI to force another | 1077 * | iret CS } iteration if needed. | 1078 * | iret RIP } | 1079 * +---------------------------------------------------------+ 1080 * | outermost SS } initialized in first_nmi; | 1081 * | outermost Return RSP } will not be changed before | 1082 * | outermost RFLAGS } NMI processing is done. | 1083 * | outermost CS } Copied to "iret" frame on each | 1084 * | outermost RIP } iteration. | 1085 * +---------------------------------------------------------+ 1086 * | pt_regs | 1087 * +---------------------------------------------------------+ 1088 * 1089 * The "original" frame is used by hardware. Before re-enabling 1090 * NMIs, we need to be done with it, and we need to leave enough 1091 * space for the asm code here. 1092 * 1093 * We return by executing IRET while RSP points to the "iret" frame. 1094 * That will either return for real or it will loop back into NMI 1095 * processing. 1096 * 1097 * The "outermost" frame is copied to the "iret" frame on each 1098 * iteration of the loop, so each iteration starts with the "iret" 1099 * frame pointing to the final return target. 1100 */ 1101 1102 /* 1103 * Determine whether we're a nested NMI. 1104 * 1105 * If we interrupted kernel code between repeat_nmi and 1106 * end_repeat_nmi, then we are a nested NMI. We must not 1107 * modify the "iret" frame because it's being written by 1108 * the outer NMI. That's okay; the outer NMI handler is 1109 * about to about to call exc_nmi() anyway, so we can just 1110 * resume the outer NMI. 1111 */ 1112 1113 movq $repeat_nmi, %rdx 1114 cmpq 8(%rsp), %rdx 1115 ja 1f 1116 movq $end_repeat_nmi, %rdx 1117 cmpq 8(%rsp), %rdx 1118 ja nested_nmi_out 11191: 1120 1121 /* 1122 * Now check "NMI executing". If it's set, then we're nested. 1123 * This will not detect if we interrupted an outer NMI just 1124 * before IRET. 1125 */ 1126 cmpl $1, -8(%rsp) 1127 je nested_nmi 1128 1129 /* 1130 * Now test if the previous stack was an NMI stack. This covers 1131 * the case where we interrupt an outer NMI after it clears 1132 * "NMI executing" but before IRET. We need to be careful, though: 1133 * there is one case in which RSP could point to the NMI stack 1134 * despite there being no NMI active: naughty userspace controls 1135 * RSP at the very beginning of the SYSCALL targets. We can 1136 * pull a fast one on naughty userspace, though: we program 1137 * SYSCALL to mask DF, so userspace cannot cause DF to be set 1138 * if it controls the kernel's RSP. We set DF before we clear 1139 * "NMI executing". 1140 */ 1141 lea 6*8(%rsp), %rdx 1142 /* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */ 1143 cmpq %rdx, 4*8(%rsp) 1144 /* If the stack pointer is above the NMI stack, this is a normal NMI */ 1145 ja first_nmi 1146 1147 subq $EXCEPTION_STKSZ, %rdx 1148 cmpq %rdx, 4*8(%rsp) 1149 /* If it is below the NMI stack, it is a normal NMI */ 1150 jb first_nmi 1151 1152 /* Ah, it is within the NMI stack. */ 1153 1154 testb $(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp) 1155 jz first_nmi /* RSP was user controlled. */ 1156 1157 /* This is a nested NMI. */ 1158 1159nested_nmi: 1160 /* 1161 * Modify the "iret" frame to point to repeat_nmi, forcing another 1162 * iteration of NMI handling. 1163 */ 1164 subq $8, %rsp 1165 leaq -10*8(%rsp), %rdx 1166 pushq $__KERNEL_DS 1167 pushq %rdx 1168 pushfq 1169 pushq $__KERNEL_CS 1170 pushq $repeat_nmi 1171 1172 /* Put stack back */ 1173 addq $(6*8), %rsp 1174 1175nested_nmi_out: 1176 popq %rdx 1177 1178 /* We are returning to kernel mode, so this cannot result in a fault. */ 1179 iretq 1180 1181first_nmi: 1182 /* Restore rdx. */ 1183 movq (%rsp), %rdx 1184 1185 /* Make room for "NMI executing". */ 1186 pushq $0 1187 1188 /* Leave room for the "iret" frame */ 1189 subq $(5*8), %rsp 1190 1191 /* Copy the "original" frame to the "outermost" frame */ 1192 .rept 5 1193 pushq 11*8(%rsp) 1194 .endr 1195 UNWIND_HINT_IRET_REGS 1196 1197 /* Everything up to here is safe from nested NMIs */ 1198 1199#ifdef CONFIG_DEBUG_ENTRY 1200 /* 1201 * For ease of testing, unmask NMIs right away. Disabled by 1202 * default because IRET is very expensive. 1203 */ 1204 pushq $0 /* SS */ 1205 pushq %rsp /* RSP (minus 8 because of the previous push) */ 1206 addq $8, (%rsp) /* Fix up RSP */ 1207 pushfq /* RFLAGS */ 1208 pushq $__KERNEL_CS /* CS */ 1209 pushq $1f /* RIP */ 1210 iretq /* continues at repeat_nmi below */ 1211 UNWIND_HINT_IRET_REGS 12121: 1213#endif 1214 1215repeat_nmi: 1216 /* 1217 * If there was a nested NMI, the first NMI's iret will return 1218 * here. But NMIs are still enabled and we can take another 1219 * nested NMI. The nested NMI checks the interrupted RIP to see 1220 * if it is between repeat_nmi and end_repeat_nmi, and if so 1221 * it will just return, as we are about to repeat an NMI anyway. 1222 * This makes it safe to copy to the stack frame that a nested 1223 * NMI will update. 1224 * 1225 * RSP is pointing to "outermost RIP". gsbase is unknown, but, if 1226 * we're repeating an NMI, gsbase has the same value that it had on 1227 * the first iteration. paranoid_entry will load the kernel 1228 * gsbase if needed before we call exc_nmi(). "NMI executing" 1229 * is zero. 1230 */ 1231 movq $1, 10*8(%rsp) /* Set "NMI executing". */ 1232 1233 /* 1234 * Copy the "outermost" frame to the "iret" frame. NMIs that nest 1235 * here must not modify the "iret" frame while we're writing to 1236 * it or it will end up containing garbage. 1237 */ 1238 addq $(10*8), %rsp 1239 .rept 5 1240 pushq -6*8(%rsp) 1241 .endr 1242 subq $(5*8), %rsp 1243end_repeat_nmi: 1244 1245 /* 1246 * Everything below this point can be preempted by a nested NMI. 1247 * If this happens, then the inner NMI will change the "iret" 1248 * frame to point back to repeat_nmi. 1249 */ 1250 pushq $-1 /* ORIG_RAX: no syscall to restart */ 1251 1252 /* 1253 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit 1254 * as we should not be calling schedule in NMI context. 1255 * Even with normal interrupts enabled. An NMI should not be 1256 * setting NEED_RESCHED or anything that normal interrupts and 1257 * exceptions might do. 1258 */ 1259 call paranoid_entry 1260 UNWIND_HINT_REGS 1261 1262 movq %rsp, %rdi 1263 movq $-1, %rsi 1264 call exc_nmi 1265 1266 /* Always restore stashed CR3 value (see paranoid_entry) */ 1267 RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 1268 1269 testl %ebx, %ebx /* swapgs needed? */ 1270 jnz nmi_restore 1271nmi_swapgs: 1272 SWAPGS_UNSAFE_STACK 1273nmi_restore: 1274 POP_REGS 1275 1276 /* 1277 * Skip orig_ax and the "outermost" frame to point RSP at the "iret" 1278 * at the "iret" frame. 1279 */ 1280 addq $6*8, %rsp 1281 1282 /* 1283 * Clear "NMI executing". Set DF first so that we can easily 1284 * distinguish the remaining code between here and IRET from 1285 * the SYSCALL entry and exit paths. 1286 * 1287 * We arguably should just inspect RIP instead, but I (Andy) wrote 1288 * this code when I had the misapprehension that Xen PV supported 1289 * NMIs, and Xen PV would break that approach. 1290 */ 1291 std 1292 movq $0, 5*8(%rsp) /* clear "NMI executing" */ 1293 1294 /* 1295 * iretq reads the "iret" frame and exits the NMI stack in a 1296 * single instruction. We are returning to kernel mode, so this 1297 * cannot result in a fault. Similarly, we don't need to worry 1298 * about espfix64 on the way back to kernel mode. 1299 */ 1300 iretq 1301SYM_CODE_END(asm_exc_nmi) 1302 1303#ifndef CONFIG_IA32_EMULATION 1304/* 1305 * This handles SYSCALL from 32-bit code. There is no way to program 1306 * MSRs to fully disable 32-bit SYSCALL. 1307 */ 1308SYM_CODE_START(ignore_sysret) 1309 UNWIND_HINT_EMPTY 1310 mov $-ENOSYS, %eax 1311 sysretl 1312SYM_CODE_END(ignore_sysret) 1313#endif 1314 1315.pushsection .text, "ax" 1316SYM_CODE_START(rewind_stack_do_exit) 1317 UNWIND_HINT_FUNC 1318 /* Prevent any naive code from trying to unwind to our caller. */ 1319 xorl %ebp, %ebp 1320 1321 movq PER_CPU_VAR(cpu_current_top_of_stack), %rax 1322 leaq -PTREGS_SIZE(%rax), %rsp 1323 UNWIND_HINT_REGS 1324 1325 call do_exit 1326SYM_CODE_END(rewind_stack_do_exit) 1327.popsection 1328