1/* 2 * Copyright (C) 1991,1992 Linus Torvalds 3 * 4 * entry_32.S contains the system-call and low-level fault and trap handling routines. 5 * 6 * Stack layout while running C code: 7 * ptrace needs to have all registers on the stack. 8 * If the order here is changed, it needs to be 9 * updated in fork.c:copy_process(), signal.c:do_signal(), 10 * ptrace.c and ptrace.h 11 * 12 * 0(%esp) - %ebx 13 * 4(%esp) - %ecx 14 * 8(%esp) - %edx 15 * C(%esp) - %esi 16 * 10(%esp) - %edi 17 * 14(%esp) - %ebp 18 * 18(%esp) - %eax 19 * 1C(%esp) - %ds 20 * 20(%esp) - %es 21 * 24(%esp) - %fs 22 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 23 * 2C(%esp) - orig_eax 24 * 30(%esp) - %eip 25 * 34(%esp) - %cs 26 * 38(%esp) - %eflags 27 * 3C(%esp) - %oldesp 28 * 40(%esp) - %oldss 29 */ 30 31#include <linux/linkage.h> 32#include <linux/err.h> 33#include <asm/thread_info.h> 34#include <asm/irqflags.h> 35#include <asm/errno.h> 36#include <asm/segment.h> 37#include <asm/smp.h> 38#include <asm/page_types.h> 39#include <asm/percpu.h> 40#include <asm/processor-flags.h> 41#include <asm/ftrace.h> 42#include <asm/irq_vectors.h> 43#include <asm/cpufeature.h> 44#include <asm/alternative-asm.h> 45#include <asm/asm.h> 46#include <asm/smap.h> 47 48 .section .entry.text, "ax" 49 50/* 51 * We use macros for low-level operations which need to be overridden 52 * for paravirtualization. The following will never clobber any registers: 53 * INTERRUPT_RETURN (aka. "iret") 54 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 55 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 56 * 57 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 58 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 59 * Allowing a register to be clobbered can shrink the paravirt replacement 60 * enough to patch inline, increasing performance. 61 */ 62 63#ifdef CONFIG_PREEMPT 64# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 65#else 66# define preempt_stop(clobbers) 67# define resume_kernel restore_all 68#endif 69 70.macro TRACE_IRQS_IRET 71#ifdef CONFIG_TRACE_IRQFLAGS 72 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? 73 jz 1f 74 TRACE_IRQS_ON 751: 76#endif 77.endm 78 79/* 80 * User gs save/restore 81 * 82 * %gs is used for userland TLS and kernel only uses it for stack 83 * canary which is required to be at %gs:20 by gcc. Read the comment 84 * at the top of stackprotector.h for more info. 85 * 86 * Local labels 98 and 99 are used. 87 */ 88#ifdef CONFIG_X86_32_LAZY_GS 89 90 /* unfortunately push/pop can't be no-op */ 91.macro PUSH_GS 92 pushl $0 93.endm 94.macro POP_GS pop=0 95 addl $(4 + \pop), %esp 96.endm 97.macro POP_GS_EX 98.endm 99 100 /* all the rest are no-op */ 101.macro PTGS_TO_GS 102.endm 103.macro PTGS_TO_GS_EX 104.endm 105.macro GS_TO_REG reg 106.endm 107.macro REG_TO_PTGS reg 108.endm 109.macro SET_KERNEL_GS reg 110.endm 111 112#else /* CONFIG_X86_32_LAZY_GS */ 113 114.macro PUSH_GS 115 pushl %gs 116.endm 117 118.macro POP_GS pop=0 11998: popl %gs 120 .if \pop <> 0 121 add $\pop, %esp 122 .endif 123.endm 124.macro POP_GS_EX 125.pushsection .fixup, "ax" 12699: movl $0, (%esp) 127 jmp 98b 128.popsection 129 _ASM_EXTABLE(98b, 99b) 130.endm 131 132.macro PTGS_TO_GS 13398: mov PT_GS(%esp), %gs 134.endm 135.macro PTGS_TO_GS_EX 136.pushsection .fixup, "ax" 13799: movl $0, PT_GS(%esp) 138 jmp 98b 139.popsection 140 _ASM_EXTABLE(98b, 99b) 141.endm 142 143.macro GS_TO_REG reg 144 movl %gs, \reg 145.endm 146.macro REG_TO_PTGS reg 147 movl \reg, PT_GS(%esp) 148.endm 149.macro SET_KERNEL_GS reg 150 movl $(__KERNEL_STACK_CANARY), \reg 151 movl \reg, %gs 152.endm 153 154#endif /* CONFIG_X86_32_LAZY_GS */ 155 156.macro SAVE_ALL pt_regs_ax=%eax 157 cld 158 PUSH_GS 159 pushl %fs 160 pushl %es 161 pushl %ds 162 pushl \pt_regs_ax 163 pushl %ebp 164 pushl %edi 165 pushl %esi 166 pushl %edx 167 pushl %ecx 168 pushl %ebx 169 movl $(__USER_DS), %edx 170 movl %edx, %ds 171 movl %edx, %es 172 movl $(__KERNEL_PERCPU), %edx 173 movl %edx, %fs 174 SET_KERNEL_GS %edx 175.endm 176 177.macro RESTORE_INT_REGS 178 popl %ebx 179 popl %ecx 180 popl %edx 181 popl %esi 182 popl %edi 183 popl %ebp 184 popl %eax 185.endm 186 187.macro RESTORE_REGS pop=0 188 RESTORE_INT_REGS 1891: popl %ds 1902: popl %es 1913: popl %fs 192 POP_GS \pop 193.pushsection .fixup, "ax" 1944: movl $0, (%esp) 195 jmp 1b 1965: movl $0, (%esp) 197 jmp 2b 1986: movl $0, (%esp) 199 jmp 3b 200.popsection 201 _ASM_EXTABLE(1b, 4b) 202 _ASM_EXTABLE(2b, 5b) 203 _ASM_EXTABLE(3b, 6b) 204 POP_GS_EX 205.endm 206 207ENTRY(ret_from_fork) 208 pushl %eax 209 call schedule_tail 210 GET_THREAD_INFO(%ebp) 211 popl %eax 212 pushl $0x0202 # Reset kernel eflags 213 popfl 214 215 /* When we fork, we trace the syscall return in the child, too. */ 216 movl %esp, %eax 217 call syscall_return_slowpath 218 jmp restore_all 219END(ret_from_fork) 220 221ENTRY(ret_from_kernel_thread) 222 pushl %eax 223 call schedule_tail 224 GET_THREAD_INFO(%ebp) 225 popl %eax 226 pushl $0x0202 # Reset kernel eflags 227 popfl 228 movl PT_EBP(%esp), %eax 229 call *PT_EBX(%esp) 230 movl $0, PT_EAX(%esp) 231 232 /* 233 * Kernel threads return to userspace as if returning from a syscall. 234 * We should check whether anything actually uses this path and, if so, 235 * consider switching it over to ret_from_fork. 236 */ 237 movl %esp, %eax 238 call syscall_return_slowpath 239 jmp restore_all 240ENDPROC(ret_from_kernel_thread) 241 242/* 243 * Return to user mode is not as complex as all this looks, 244 * but we want the default path for a system call return to 245 * go as quickly as possible which is why some of this is 246 * less clear than it otherwise should be. 247 */ 248 249 # userspace resumption stub bypassing syscall exit tracing 250 ALIGN 251ret_from_exception: 252 preempt_stop(CLBR_ANY) 253ret_from_intr: 254 GET_THREAD_INFO(%ebp) 255#ifdef CONFIG_VM86 256 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 257 movb PT_CS(%esp), %al 258 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 259#else 260 /* 261 * We can be coming here from child spawned by kernel_thread(). 262 */ 263 movl PT_CS(%esp), %eax 264 andl $SEGMENT_RPL_MASK, %eax 265#endif 266 cmpl $USER_RPL, %eax 267 jb resume_kernel # not returning to v8086 or userspace 268 269ENTRY(resume_userspace) 270 DISABLE_INTERRUPTS(CLBR_ANY) 271 TRACE_IRQS_OFF 272 movl %esp, %eax 273 call prepare_exit_to_usermode 274 jmp restore_all 275END(ret_from_exception) 276 277#ifdef CONFIG_PREEMPT 278ENTRY(resume_kernel) 279 DISABLE_INTERRUPTS(CLBR_ANY) 280need_resched: 281 cmpl $0, PER_CPU_VAR(__preempt_count) 282 jnz restore_all 283 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? 284 jz restore_all 285 call preempt_schedule_irq 286 jmp need_resched 287END(resume_kernel) 288#endif 289 290 # SYSENTER call handler stub 291ENTRY(entry_SYSENTER_32) 292 movl TSS_sysenter_sp0(%esp), %esp 293sysenter_past_esp: 294 pushl $__USER_DS /* pt_regs->ss */ 295 pushl %ecx /* pt_regs->cx */ 296 pushfl /* pt_regs->flags (except IF = 0) */ 297 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */ 298 pushl $__USER_CS /* pt_regs->cs */ 299 pushl $0 /* pt_regs->ip = 0 (placeholder) */ 300 pushl %eax /* pt_regs->orig_ax */ 301 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 302 303 /* 304 * User mode is traced as though IRQs are on, and SYSENTER 305 * turned them off. 306 */ 307 TRACE_IRQS_OFF 308 309 movl %esp, %eax 310 call do_fast_syscall_32 311 testl %eax, %eax 312 jz .Lsyscall_32_done 313 314/* Opportunistic SYSEXIT */ 315 TRACE_IRQS_ON /* User mode traces as IRQs on. */ 316 movl PT_EIP(%esp), %edx /* pt_regs->ip */ 317 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */ 3181: mov PT_FS(%esp), %fs 319 PTGS_TO_GS 320 popl %ebx /* pt_regs->bx */ 321 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */ 322 popl %esi /* pt_regs->si */ 323 popl %edi /* pt_regs->di */ 324 popl %ebp /* pt_regs->bp */ 325 popl %eax /* pt_regs->ax */ 326 327 /* 328 * Return back to the vDSO, which will pop ecx and edx. 329 * Don't bother with DS and ES (they already contain __USER_DS). 330 */ 331 ENABLE_INTERRUPTS_SYSEXIT 332 333.pushsection .fixup, "ax" 3342: movl $0, PT_FS(%esp) 335 jmp 1b 336.popsection 337 _ASM_EXTABLE(1b, 2b) 338 PTGS_TO_GS_EX 339ENDPROC(entry_SYSENTER_32) 340 341 # system call handler stub 342ENTRY(entry_INT80_32) 343 ASM_CLAC 344 pushl %eax /* pt_regs->orig_ax */ 345 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 346 347 /* 348 * User mode is traced as though IRQs are on. Unlike the 64-bit 349 * case, INT80 is a trap gate on 32-bit kernels, so interrupts 350 * are already on (unless user code is messing around with iopl). 351 */ 352 353 movl %esp, %eax 354 call do_syscall_32_irqs_on 355.Lsyscall_32_done: 356 357restore_all: 358 TRACE_IRQS_IRET 359restore_all_notrace: 360#ifdef CONFIG_X86_ESPFIX32 361 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 362 /* 363 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we 364 * are returning to the kernel. 365 * See comments in process.c:copy_thread() for details. 366 */ 367 movb PT_OLDSS(%esp), %ah 368 movb PT_CS(%esp), %al 369 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 370 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 371 je ldt_ss # returning to user-space with LDT SS 372#endif 373restore_nocheck: 374 RESTORE_REGS 4 # skip orig_eax/error_code 375irq_return: 376 INTERRUPT_RETURN 377.section .fixup, "ax" 378ENTRY(iret_exc ) 379 pushl $0 # no error code 380 pushl $do_iret_error 381 jmp error_code 382.previous 383 _ASM_EXTABLE(irq_return, iret_exc) 384 385#ifdef CONFIG_X86_ESPFIX32 386ldt_ss: 387#ifdef CONFIG_PARAVIRT 388 /* 389 * The kernel can't run on a non-flat stack if paravirt mode 390 * is active. Rather than try to fixup the high bits of 391 * ESP, bypass this code entirely. This may break DOSemu 392 * and/or Wine support in a paravirt VM, although the option 393 * is still available to implement the setting of the high 394 * 16-bits in the INTERRUPT_RETURN paravirt-op. 395 */ 396 cmpl $0, pv_info+PARAVIRT_enabled 397 jne restore_nocheck 398#endif 399 400/* 401 * Setup and switch to ESPFIX stack 402 * 403 * We're returning to userspace with a 16 bit stack. The CPU will not 404 * restore the high word of ESP for us on executing iret... This is an 405 * "official" bug of all the x86-compatible CPUs, which we can work 406 * around to make dosemu and wine happy. We do this by preloading the 407 * high word of ESP with the high word of the userspace ESP while 408 * compensating for the offset by changing to the ESPFIX segment with 409 * a base address that matches for the difference. 410 */ 411#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 412 mov %esp, %edx /* load kernel esp */ 413 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 414 mov %dx, %ax /* eax: new kernel esp */ 415 sub %eax, %edx /* offset (low word is 0) */ 416 shr $16, %edx 417 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 418 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 419 pushl $__ESPFIX_SS 420 pushl %eax /* new kernel esp */ 421 /* 422 * Disable interrupts, but do not irqtrace this section: we 423 * will soon execute iret and the tracer was already set to 424 * the irqstate after the IRET: 425 */ 426 DISABLE_INTERRUPTS(CLBR_EAX) 427 lss (%esp), %esp /* switch to espfix segment */ 428 jmp restore_nocheck 429#endif 430ENDPROC(entry_INT80_32) 431 432.macro FIXUP_ESPFIX_STACK 433/* 434 * Switch back for ESPFIX stack to the normal zerobased stack 435 * 436 * We can't call C functions using the ESPFIX stack. This code reads 437 * the high word of the segment base from the GDT and swiches to the 438 * normal stack and adjusts ESP with the matching offset. 439 */ 440#ifdef CONFIG_X86_ESPFIX32 441 /* fixup the stack */ 442 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 443 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 444 shl $16, %eax 445 addl %esp, %eax /* the adjusted stack pointer */ 446 pushl $__KERNEL_DS 447 pushl %eax 448 lss (%esp), %esp /* switch to the normal stack segment */ 449#endif 450.endm 451.macro UNWIND_ESPFIX_STACK 452#ifdef CONFIG_X86_ESPFIX32 453 movl %ss, %eax 454 /* see if on espfix stack */ 455 cmpw $__ESPFIX_SS, %ax 456 jne 27f 457 movl $__KERNEL_DS, %eax 458 movl %eax, %ds 459 movl %eax, %es 460 /* switch to normal stack */ 461 FIXUP_ESPFIX_STACK 46227: 463#endif 464.endm 465 466/* 467 * Build the entry stubs with some assembler magic. 468 * We pack 1 stub into every 8-byte block. 469 */ 470 .align 8 471ENTRY(irq_entries_start) 472 vector=FIRST_EXTERNAL_VECTOR 473 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 474 pushl $(~vector+0x80) /* Note: always in signed byte range */ 475 vector=vector+1 476 jmp common_interrupt 477 .align 8 478 .endr 479END(irq_entries_start) 480 481/* 482 * the CPU automatically disables interrupts when executing an IRQ vector, 483 * so IRQ-flags tracing has to follow that: 484 */ 485 .p2align CONFIG_X86_L1_CACHE_SHIFT 486common_interrupt: 487 ASM_CLAC 488 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ 489 SAVE_ALL 490 TRACE_IRQS_OFF 491 movl %esp, %eax 492 call do_IRQ 493 jmp ret_from_intr 494ENDPROC(common_interrupt) 495 496#define BUILD_INTERRUPT3(name, nr, fn) \ 497ENTRY(name) \ 498 ASM_CLAC; \ 499 pushl $~(nr); \ 500 SAVE_ALL; \ 501 TRACE_IRQS_OFF \ 502 movl %esp, %eax; \ 503 call fn; \ 504 jmp ret_from_intr; \ 505ENDPROC(name) 506 507 508#ifdef CONFIG_TRACING 509# define TRACE_BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name) 510#else 511# define TRACE_BUILD_INTERRUPT(name, nr) 512#endif 513 514#define BUILD_INTERRUPT(name, nr) \ 515 BUILD_INTERRUPT3(name, nr, smp_##name); \ 516 TRACE_BUILD_INTERRUPT(name, nr) 517 518/* The include is where all of the SMP etc. interrupts come from */ 519#include <asm/entry_arch.h> 520 521ENTRY(coprocessor_error) 522 ASM_CLAC 523 pushl $0 524 pushl $do_coprocessor_error 525 jmp error_code 526END(coprocessor_error) 527 528ENTRY(simd_coprocessor_error) 529 ASM_CLAC 530 pushl $0 531#ifdef CONFIG_X86_INVD_BUG 532 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 533 ALTERNATIVE "pushl $do_general_protection", \ 534 "pushl $do_simd_coprocessor_error", \ 535 X86_FEATURE_XMM 536#else 537 pushl $do_simd_coprocessor_error 538#endif 539 jmp error_code 540END(simd_coprocessor_error) 541 542ENTRY(device_not_available) 543 ASM_CLAC 544 pushl $-1 # mark this as an int 545 pushl $do_device_not_available 546 jmp error_code 547END(device_not_available) 548 549#ifdef CONFIG_PARAVIRT 550ENTRY(native_iret) 551 iret 552 _ASM_EXTABLE(native_iret, iret_exc) 553END(native_iret) 554 555ENTRY(native_irq_enable_sysexit) 556 sti 557 sysexit 558END(native_irq_enable_sysexit) 559#endif 560 561ENTRY(overflow) 562 ASM_CLAC 563 pushl $0 564 pushl $do_overflow 565 jmp error_code 566END(overflow) 567 568ENTRY(bounds) 569 ASM_CLAC 570 pushl $0 571 pushl $do_bounds 572 jmp error_code 573END(bounds) 574 575ENTRY(invalid_op) 576 ASM_CLAC 577 pushl $0 578 pushl $do_invalid_op 579 jmp error_code 580END(invalid_op) 581 582ENTRY(coprocessor_segment_overrun) 583 ASM_CLAC 584 pushl $0 585 pushl $do_coprocessor_segment_overrun 586 jmp error_code 587END(coprocessor_segment_overrun) 588 589ENTRY(invalid_TSS) 590 ASM_CLAC 591 pushl $do_invalid_TSS 592 jmp error_code 593END(invalid_TSS) 594 595ENTRY(segment_not_present) 596 ASM_CLAC 597 pushl $do_segment_not_present 598 jmp error_code 599END(segment_not_present) 600 601ENTRY(stack_segment) 602 ASM_CLAC 603 pushl $do_stack_segment 604 jmp error_code 605END(stack_segment) 606 607ENTRY(alignment_check) 608 ASM_CLAC 609 pushl $do_alignment_check 610 jmp error_code 611END(alignment_check) 612 613ENTRY(divide_error) 614 ASM_CLAC 615 pushl $0 # no error code 616 pushl $do_divide_error 617 jmp error_code 618END(divide_error) 619 620#ifdef CONFIG_X86_MCE 621ENTRY(machine_check) 622 ASM_CLAC 623 pushl $0 624 pushl machine_check_vector 625 jmp error_code 626END(machine_check) 627#endif 628 629ENTRY(spurious_interrupt_bug) 630 ASM_CLAC 631 pushl $0 632 pushl $do_spurious_interrupt_bug 633 jmp error_code 634END(spurious_interrupt_bug) 635 636#ifdef CONFIG_XEN 637/* 638 * Xen doesn't set %esp to be precisely what the normal SYSENTER 639 * entry point expects, so fix it up before using the normal path. 640 */ 641ENTRY(xen_sysenter_target) 642 addl $5*4, %esp /* remove xen-provided frame */ 643 jmp sysenter_past_esp 644 645ENTRY(xen_hypervisor_callback) 646 pushl $-1 /* orig_ax = -1 => not a system call */ 647 SAVE_ALL 648 TRACE_IRQS_OFF 649 650 /* 651 * Check to see if we got the event in the critical 652 * region in xen_iret_direct, after we've reenabled 653 * events and checked for pending events. This simulates 654 * iret instruction's behaviour where it delivers a 655 * pending interrupt when enabling interrupts: 656 */ 657 movl PT_EIP(%esp), %eax 658 cmpl $xen_iret_start_crit, %eax 659 jb 1f 660 cmpl $xen_iret_end_crit, %eax 661 jae 1f 662 663 jmp xen_iret_crit_fixup 664 665ENTRY(xen_do_upcall) 6661: mov %esp, %eax 667 call xen_evtchn_do_upcall 668#ifndef CONFIG_PREEMPT 669 call xen_maybe_preempt_hcall 670#endif 671 jmp ret_from_intr 672ENDPROC(xen_hypervisor_callback) 673 674/* 675 * Hypervisor uses this for application faults while it executes. 676 * We get here for two reasons: 677 * 1. Fault while reloading DS, ES, FS or GS 678 * 2. Fault while executing IRET 679 * Category 1 we fix up by reattempting the load, and zeroing the segment 680 * register if the load fails. 681 * Category 2 we fix up by jumping to do_iret_error. We cannot use the 682 * normal Linux return path in this case because if we use the IRET hypercall 683 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 684 * We distinguish between categories by maintaining a status value in EAX. 685 */ 686ENTRY(xen_failsafe_callback) 687 pushl %eax 688 movl $1, %eax 6891: mov 4(%esp), %ds 6902: mov 8(%esp), %es 6913: mov 12(%esp), %fs 6924: mov 16(%esp), %gs 693 /* EAX == 0 => Category 1 (Bad segment) 694 EAX != 0 => Category 2 (Bad IRET) */ 695 testl %eax, %eax 696 popl %eax 697 lea 16(%esp), %esp 698 jz 5f 699 jmp iret_exc 7005: pushl $-1 /* orig_ax = -1 => not a system call */ 701 SAVE_ALL 702 jmp ret_from_exception 703 704.section .fixup, "ax" 7056: xorl %eax, %eax 706 movl %eax, 4(%esp) 707 jmp 1b 7087: xorl %eax, %eax 709 movl %eax, 8(%esp) 710 jmp 2b 7118: xorl %eax, %eax 712 movl %eax, 12(%esp) 713 jmp 3b 7149: xorl %eax, %eax 715 movl %eax, 16(%esp) 716 jmp 4b 717.previous 718 _ASM_EXTABLE(1b, 6b) 719 _ASM_EXTABLE(2b, 7b) 720 _ASM_EXTABLE(3b, 8b) 721 _ASM_EXTABLE(4b, 9b) 722ENDPROC(xen_failsafe_callback) 723 724BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 725 xen_evtchn_do_upcall) 726 727#endif /* CONFIG_XEN */ 728 729#if IS_ENABLED(CONFIG_HYPERV) 730 731BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 732 hyperv_vector_handler) 733 734#endif /* CONFIG_HYPERV */ 735 736#ifdef CONFIG_FUNCTION_TRACER 737#ifdef CONFIG_DYNAMIC_FTRACE 738 739ENTRY(mcount) 740 ret 741END(mcount) 742 743ENTRY(ftrace_caller) 744 pushl %eax 745 pushl %ecx 746 pushl %edx 747 pushl $0 /* Pass NULL as regs pointer */ 748 movl 4*4(%esp), %eax 749 movl 0x4(%ebp), %edx 750 movl function_trace_op, %ecx 751 subl $MCOUNT_INSN_SIZE, %eax 752 753.globl ftrace_call 754ftrace_call: 755 call ftrace_stub 756 757 addl $4, %esp /* skip NULL pointer */ 758 popl %edx 759 popl %ecx 760 popl %eax 761ftrace_ret: 762#ifdef CONFIG_FUNCTION_GRAPH_TRACER 763.globl ftrace_graph_call 764ftrace_graph_call: 765 jmp ftrace_stub 766#endif 767 768.globl ftrace_stub 769ftrace_stub: 770 ret 771END(ftrace_caller) 772 773ENTRY(ftrace_regs_caller) 774 pushf /* push flags before compare (in cs location) */ 775 776 /* 777 * i386 does not save SS and ESP when coming from kernel. 778 * Instead, to get sp, ®s->sp is used (see ptrace.h). 779 * Unfortunately, that means eflags must be at the same location 780 * as the current return ip is. We move the return ip into the 781 * ip location, and move flags into the return ip location. 782 */ 783 pushl 4(%esp) /* save return ip into ip slot */ 784 785 pushl $0 /* Load 0 into orig_ax */ 786 pushl %gs 787 pushl %fs 788 pushl %es 789 pushl %ds 790 pushl %eax 791 pushl %ebp 792 pushl %edi 793 pushl %esi 794 pushl %edx 795 pushl %ecx 796 pushl %ebx 797 798 movl 13*4(%esp), %eax /* Get the saved flags */ 799 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ 800 /* clobbering return ip */ 801 movl $__KERNEL_CS, 13*4(%esp) 802 803 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 804 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 805 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 806 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 807 pushl %esp /* Save pt_regs as 4th parameter */ 808 809GLOBAL(ftrace_regs_call) 810 call ftrace_stub 811 812 addl $4, %esp /* Skip pt_regs */ 813 movl 14*4(%esp), %eax /* Move flags back into cs */ 814 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ 815 movl 12*4(%esp), %eax /* Get return ip from regs->ip */ 816 movl %eax, 14*4(%esp) /* Put return ip back for ret */ 817 818 popl %ebx 819 popl %ecx 820 popl %edx 821 popl %esi 822 popl %edi 823 popl %ebp 824 popl %eax 825 popl %ds 826 popl %es 827 popl %fs 828 popl %gs 829 addl $8, %esp /* Skip orig_ax and ip */ 830 popf /* Pop flags at end (no addl to corrupt flags) */ 831 jmp ftrace_ret 832 833 popf 834 jmp ftrace_stub 835#else /* ! CONFIG_DYNAMIC_FTRACE */ 836 837ENTRY(mcount) 838 cmpl $__PAGE_OFFSET, %esp 839 jb ftrace_stub /* Paging not enabled yet? */ 840 841 cmpl $ftrace_stub, ftrace_trace_function 842 jnz trace 843#ifdef CONFIG_FUNCTION_GRAPH_TRACER 844 cmpl $ftrace_stub, ftrace_graph_return 845 jnz ftrace_graph_caller 846 847 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 848 jnz ftrace_graph_caller 849#endif 850.globl ftrace_stub 851ftrace_stub: 852 ret 853 854 /* taken from glibc */ 855trace: 856 pushl %eax 857 pushl %ecx 858 pushl %edx 859 movl 0xc(%esp), %eax 860 movl 0x4(%ebp), %edx 861 subl $MCOUNT_INSN_SIZE, %eax 862 863 call *ftrace_trace_function 864 865 popl %edx 866 popl %ecx 867 popl %eax 868 jmp ftrace_stub 869END(mcount) 870#endif /* CONFIG_DYNAMIC_FTRACE */ 871#endif /* CONFIG_FUNCTION_TRACER */ 872 873#ifdef CONFIG_FUNCTION_GRAPH_TRACER 874ENTRY(ftrace_graph_caller) 875 pushl %eax 876 pushl %ecx 877 pushl %edx 878 movl 0xc(%esp), %eax 879 lea 0x4(%ebp), %edx 880 movl (%ebp), %ecx 881 subl $MCOUNT_INSN_SIZE, %eax 882 call prepare_ftrace_return 883 popl %edx 884 popl %ecx 885 popl %eax 886 ret 887END(ftrace_graph_caller) 888 889.globl return_to_handler 890return_to_handler: 891 pushl %eax 892 pushl %edx 893 movl %ebp, %eax 894 call ftrace_return_to_handler 895 movl %eax, %ecx 896 popl %edx 897 popl %eax 898 jmp *%ecx 899#endif 900 901#ifdef CONFIG_TRACING 902ENTRY(trace_page_fault) 903 ASM_CLAC 904 pushl $trace_do_page_fault 905 jmp error_code 906END(trace_page_fault) 907#endif 908 909ENTRY(page_fault) 910 ASM_CLAC 911 pushl $do_page_fault 912 ALIGN 913error_code: 914 /* the function address is in %gs's slot on the stack */ 915 pushl %fs 916 pushl %es 917 pushl %ds 918 pushl %eax 919 pushl %ebp 920 pushl %edi 921 pushl %esi 922 pushl %edx 923 pushl %ecx 924 pushl %ebx 925 cld 926 movl $(__KERNEL_PERCPU), %ecx 927 movl %ecx, %fs 928 UNWIND_ESPFIX_STACK 929 GS_TO_REG %ecx 930 movl PT_GS(%esp), %edi # get the function address 931 movl PT_ORIG_EAX(%esp), %edx # get the error code 932 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 933 REG_TO_PTGS %ecx 934 SET_KERNEL_GS %ecx 935 movl $(__USER_DS), %ecx 936 movl %ecx, %ds 937 movl %ecx, %es 938 TRACE_IRQS_OFF 939 movl %esp, %eax # pt_regs pointer 940 call *%edi 941 jmp ret_from_exception 942END(page_fault) 943 944/* 945 * Debug traps and NMI can happen at the one SYSENTER instruction 946 * that sets up the real kernel stack. Check here, since we can't 947 * allow the wrong stack to be used. 948 * 949 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 950 * already pushed 3 words if it hits on the sysenter instruction: 951 * eflags, cs and eip. 952 * 953 * We just load the right stack, and push the three (known) values 954 * by hand onto the new stack - while updating the return eip past 955 * the instruction that would have done it for sysenter. 956 */ 957.macro FIX_STACK offset ok label 958 cmpw $__KERNEL_CS, 4(%esp) 959 jne \ok 960\label: 961 movl TSS_sysenter_sp0 + \offset(%esp), %esp 962 pushfl 963 pushl $__KERNEL_CS 964 pushl $sysenter_past_esp 965.endm 966 967ENTRY(debug) 968 ASM_CLAC 969 cmpl $entry_SYSENTER_32, (%esp) 970 jne debug_stack_correct 971 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 972debug_stack_correct: 973 pushl $-1 # mark this as an int 974 SAVE_ALL 975 TRACE_IRQS_OFF 976 xorl %edx, %edx # error code 0 977 movl %esp, %eax # pt_regs pointer 978 call do_debug 979 jmp ret_from_exception 980END(debug) 981 982/* 983 * NMI is doubly nasty. It can happen _while_ we're handling 984 * a debug fault, and the debug fault hasn't yet been able to 985 * clear up the stack. So we first check whether we got an 986 * NMI on the sysenter entry path, but after that we need to 987 * check whether we got an NMI on the debug path where the debug 988 * fault happened on the sysenter path. 989 */ 990ENTRY(nmi) 991 ASM_CLAC 992#ifdef CONFIG_X86_ESPFIX32 993 pushl %eax 994 movl %ss, %eax 995 cmpw $__ESPFIX_SS, %ax 996 popl %eax 997 je nmi_espfix_stack 998#endif 999 cmpl $entry_SYSENTER_32, (%esp) 1000 je nmi_stack_fixup 1001 pushl %eax 1002 movl %esp, %eax 1003 /* 1004 * Do not access memory above the end of our stack page, 1005 * it might not exist. 1006 */ 1007 andl $(THREAD_SIZE-1), %eax 1008 cmpl $(THREAD_SIZE-20), %eax 1009 popl %eax 1010 jae nmi_stack_correct 1011 cmpl $entry_SYSENTER_32, 12(%esp) 1012 je nmi_debug_stack_check 1013nmi_stack_correct: 1014 pushl %eax 1015 SAVE_ALL 1016 xorl %edx, %edx # zero error code 1017 movl %esp, %eax # pt_regs pointer 1018 call do_nmi 1019 jmp restore_all_notrace 1020 1021nmi_stack_fixup: 1022 FIX_STACK 12, nmi_stack_correct, 1 1023 jmp nmi_stack_correct 1024 1025nmi_debug_stack_check: 1026 cmpw $__KERNEL_CS, 16(%esp) 1027 jne nmi_stack_correct 1028 cmpl $debug, (%esp) 1029 jb nmi_stack_correct 1030 cmpl $debug_esp_fix_insn, (%esp) 1031 ja nmi_stack_correct 1032 FIX_STACK 24, nmi_stack_correct, 1 1033 jmp nmi_stack_correct 1034 1035#ifdef CONFIG_X86_ESPFIX32 1036nmi_espfix_stack: 1037 /* 1038 * create the pointer to lss back 1039 */ 1040 pushl %ss 1041 pushl %esp 1042 addl $4, (%esp) 1043 /* copy the iret frame of 12 bytes */ 1044 .rept 3 1045 pushl 16(%esp) 1046 .endr 1047 pushl %eax 1048 SAVE_ALL 1049 FIXUP_ESPFIX_STACK # %eax == %esp 1050 xorl %edx, %edx # zero error code 1051 call do_nmi 1052 RESTORE_REGS 1053 lss 12+4(%esp), %esp # back to espfix stack 1054 jmp irq_return 1055#endif 1056END(nmi) 1057 1058ENTRY(int3) 1059 ASM_CLAC 1060 pushl $-1 # mark this as an int 1061 SAVE_ALL 1062 TRACE_IRQS_OFF 1063 xorl %edx, %edx # zero error code 1064 movl %esp, %eax # pt_regs pointer 1065 call do_int3 1066 jmp ret_from_exception 1067END(int3) 1068 1069ENTRY(general_protection) 1070 pushl $do_general_protection 1071 jmp error_code 1072END(general_protection) 1073 1074#ifdef CONFIG_KVM_GUEST 1075ENTRY(async_page_fault) 1076 ASM_CLAC 1077 pushl $do_async_page_fault 1078 jmp error_code 1079END(async_page_fault) 1080#endif 1081