1/* 2 * Copyright (C) 1991,1992 Linus Torvalds 3 * 4 * entry_32.S contains the system-call and low-level fault and trap handling routines. 5 * 6 * Stack layout while running C code: 7 * ptrace needs to have all registers on the stack. 8 * If the order here is changed, it needs to be 9 * updated in fork.c:copy_process(), signal.c:do_signal(), 10 * ptrace.c and ptrace.h 11 * 12 * 0(%esp) - %ebx 13 * 4(%esp) - %ecx 14 * 8(%esp) - %edx 15 * C(%esp) - %esi 16 * 10(%esp) - %edi 17 * 14(%esp) - %ebp 18 * 18(%esp) - %eax 19 * 1C(%esp) - %ds 20 * 20(%esp) - %es 21 * 24(%esp) - %fs 22 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS 23 * 2C(%esp) - orig_eax 24 * 30(%esp) - %eip 25 * 34(%esp) - %cs 26 * 38(%esp) - %eflags 27 * 3C(%esp) - %oldesp 28 * 40(%esp) - %oldss 29 */ 30 31#include <linux/linkage.h> 32#include <linux/err.h> 33#include <asm/thread_info.h> 34#include <asm/irqflags.h> 35#include <asm/errno.h> 36#include <asm/segment.h> 37#include <asm/smp.h> 38#include <asm/page_types.h> 39#include <asm/percpu.h> 40#include <asm/processor-flags.h> 41#include <asm/ftrace.h> 42#include <asm/irq_vectors.h> 43#include <asm/cpufeature.h> 44#include <asm/alternative-asm.h> 45#include <asm/asm.h> 46#include <asm/smap.h> 47 48 .section .entry.text, "ax" 49 50/* 51 * We use macros for low-level operations which need to be overridden 52 * for paravirtualization. The following will never clobber any registers: 53 * INTERRUPT_RETURN (aka. "iret") 54 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") 55 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). 56 * 57 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must 58 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). 59 * Allowing a register to be clobbered can shrink the paravirt replacement 60 * enough to patch inline, increasing performance. 61 */ 62 63#ifdef CONFIG_PREEMPT 64# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF 65#else 66# define preempt_stop(clobbers) 67# define resume_kernel restore_all 68#endif 69 70.macro TRACE_IRQS_IRET 71#ifdef CONFIG_TRACE_IRQFLAGS 72 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? 73 jz 1f 74 TRACE_IRQS_ON 751: 76#endif 77.endm 78 79/* 80 * User gs save/restore 81 * 82 * %gs is used for userland TLS and kernel only uses it for stack 83 * canary which is required to be at %gs:20 by gcc. Read the comment 84 * at the top of stackprotector.h for more info. 85 * 86 * Local labels 98 and 99 are used. 87 */ 88#ifdef CONFIG_X86_32_LAZY_GS 89 90 /* unfortunately push/pop can't be no-op */ 91.macro PUSH_GS 92 pushl $0 93.endm 94.macro POP_GS pop=0 95 addl $(4 + \pop), %esp 96.endm 97.macro POP_GS_EX 98.endm 99 100 /* all the rest are no-op */ 101.macro PTGS_TO_GS 102.endm 103.macro PTGS_TO_GS_EX 104.endm 105.macro GS_TO_REG reg 106.endm 107.macro REG_TO_PTGS reg 108.endm 109.macro SET_KERNEL_GS reg 110.endm 111 112#else /* CONFIG_X86_32_LAZY_GS */ 113 114.macro PUSH_GS 115 pushl %gs 116.endm 117 118.macro POP_GS pop=0 11998: popl %gs 120 .if \pop <> 0 121 add $\pop, %esp 122 .endif 123.endm 124.macro POP_GS_EX 125.pushsection .fixup, "ax" 12699: movl $0, (%esp) 127 jmp 98b 128.popsection 129 _ASM_EXTABLE(98b, 99b) 130.endm 131 132.macro PTGS_TO_GS 13398: mov PT_GS(%esp), %gs 134.endm 135.macro PTGS_TO_GS_EX 136.pushsection .fixup, "ax" 13799: movl $0, PT_GS(%esp) 138 jmp 98b 139.popsection 140 _ASM_EXTABLE(98b, 99b) 141.endm 142 143.macro GS_TO_REG reg 144 movl %gs, \reg 145.endm 146.macro REG_TO_PTGS reg 147 movl \reg, PT_GS(%esp) 148.endm 149.macro SET_KERNEL_GS reg 150 movl $(__KERNEL_STACK_CANARY), \reg 151 movl \reg, %gs 152.endm 153 154#endif /* CONFIG_X86_32_LAZY_GS */ 155 156.macro SAVE_ALL pt_regs_ax=%eax 157 cld 158 PUSH_GS 159 pushl %fs 160 pushl %es 161 pushl %ds 162 pushl \pt_regs_ax 163 pushl %ebp 164 pushl %edi 165 pushl %esi 166 pushl %edx 167 pushl %ecx 168 pushl %ebx 169 movl $(__USER_DS), %edx 170 movl %edx, %ds 171 movl %edx, %es 172 movl $(__KERNEL_PERCPU), %edx 173 movl %edx, %fs 174 SET_KERNEL_GS %edx 175.endm 176 177.macro RESTORE_INT_REGS 178 popl %ebx 179 popl %ecx 180 popl %edx 181 popl %esi 182 popl %edi 183 popl %ebp 184 popl %eax 185.endm 186 187.macro RESTORE_REGS pop=0 188 RESTORE_INT_REGS 1891: popl %ds 1902: popl %es 1913: popl %fs 192 POP_GS \pop 193.pushsection .fixup, "ax" 1944: movl $0, (%esp) 195 jmp 1b 1965: movl $0, (%esp) 197 jmp 2b 1986: movl $0, (%esp) 199 jmp 3b 200.popsection 201 _ASM_EXTABLE(1b, 4b) 202 _ASM_EXTABLE(2b, 5b) 203 _ASM_EXTABLE(3b, 6b) 204 POP_GS_EX 205.endm 206 207ENTRY(ret_from_fork) 208 pushl %eax 209 call schedule_tail 210 GET_THREAD_INFO(%ebp) 211 popl %eax 212 pushl $0x0202 # Reset kernel eflags 213 popfl 214 215 /* When we fork, we trace the syscall return in the child, too. */ 216 movl %esp, %eax 217 call syscall_return_slowpath 218 jmp restore_all 219END(ret_from_fork) 220 221ENTRY(ret_from_kernel_thread) 222 pushl %eax 223 call schedule_tail 224 GET_THREAD_INFO(%ebp) 225 popl %eax 226 pushl $0x0202 # Reset kernel eflags 227 popfl 228 movl PT_EBP(%esp), %eax 229 call *PT_EBX(%esp) 230 movl $0, PT_EAX(%esp) 231 232 /* 233 * Kernel threads return to userspace as if returning from a syscall. 234 * We should check whether anything actually uses this path and, if so, 235 * consider switching it over to ret_from_fork. 236 */ 237 movl %esp, %eax 238 call syscall_return_slowpath 239 jmp restore_all 240ENDPROC(ret_from_kernel_thread) 241 242/* 243 * Return to user mode is not as complex as all this looks, 244 * but we want the default path for a system call return to 245 * go as quickly as possible which is why some of this is 246 * less clear than it otherwise should be. 247 */ 248 249 # userspace resumption stub bypassing syscall exit tracing 250 ALIGN 251ret_from_exception: 252 preempt_stop(CLBR_ANY) 253ret_from_intr: 254 GET_THREAD_INFO(%ebp) 255#ifdef CONFIG_VM86 256 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS 257 movb PT_CS(%esp), %al 258 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 259#else 260 /* 261 * We can be coming here from child spawned by kernel_thread(). 262 */ 263 movl PT_CS(%esp), %eax 264 andl $SEGMENT_RPL_MASK, %eax 265#endif 266 cmpl $USER_RPL, %eax 267 jb resume_kernel # not returning to v8086 or userspace 268 269ENTRY(resume_userspace) 270 DISABLE_INTERRUPTS(CLBR_ANY) 271 TRACE_IRQS_OFF 272 movl %esp, %eax 273 call prepare_exit_to_usermode 274 jmp restore_all 275END(ret_from_exception) 276 277#ifdef CONFIG_PREEMPT 278ENTRY(resume_kernel) 279 DISABLE_INTERRUPTS(CLBR_ANY) 280need_resched: 281 cmpl $0, PER_CPU_VAR(__preempt_count) 282 jnz restore_all 283 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? 284 jz restore_all 285 call preempt_schedule_irq 286 jmp need_resched 287END(resume_kernel) 288#endif 289 290 # SYSENTER call handler stub 291ENTRY(entry_SYSENTER_32) 292 movl TSS_sysenter_sp0(%esp), %esp 293sysenter_past_esp: 294 pushl $__USER_DS /* pt_regs->ss */ 295 pushl %ebp /* pt_regs->sp (stashed in bp) */ 296 pushfl /* pt_regs->flags (except IF = 0) */ 297 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */ 298 pushl $__USER_CS /* pt_regs->cs */ 299 pushl $0 /* pt_regs->ip = 0 (placeholder) */ 300 pushl %eax /* pt_regs->orig_ax */ 301 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 302 303 /* 304 * User mode is traced as though IRQs are on, and SYSENTER 305 * turned them off. 306 */ 307 TRACE_IRQS_OFF 308 309 movl %esp, %eax 310 call do_fast_syscall_32 311 /* XEN PV guests always use IRET path */ 312 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ 313 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV 314 315/* Opportunistic SYSEXIT */ 316 TRACE_IRQS_ON /* User mode traces as IRQs on. */ 317 movl PT_EIP(%esp), %edx /* pt_regs->ip */ 318 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */ 3191: mov PT_FS(%esp), %fs 320 PTGS_TO_GS 321 popl %ebx /* pt_regs->bx */ 322 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */ 323 popl %esi /* pt_regs->si */ 324 popl %edi /* pt_regs->di */ 325 popl %ebp /* pt_regs->bp */ 326 popl %eax /* pt_regs->ax */ 327 328 /* 329 * Return back to the vDSO, which will pop ecx and edx. 330 * Don't bother with DS and ES (they already contain __USER_DS). 331 */ 332 sti 333 sysexit 334 335.pushsection .fixup, "ax" 3362: movl $0, PT_FS(%esp) 337 jmp 1b 338.popsection 339 _ASM_EXTABLE(1b, 2b) 340 PTGS_TO_GS_EX 341ENDPROC(entry_SYSENTER_32) 342 343 # system call handler stub 344ENTRY(entry_INT80_32) 345 ASM_CLAC 346 pushl %eax /* pt_regs->orig_ax */ 347 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */ 348 349 /* 350 * User mode is traced as though IRQs are on. Unlike the 64-bit 351 * case, INT80 is a trap gate on 32-bit kernels, so interrupts 352 * are already on (unless user code is messing around with iopl). 353 */ 354 355 movl %esp, %eax 356 call do_syscall_32_irqs_on 357.Lsyscall_32_done: 358 359restore_all: 360 TRACE_IRQS_IRET 361restore_all_notrace: 362#ifdef CONFIG_X86_ESPFIX32 363 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS 364 /* 365 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we 366 * are returning to the kernel. 367 * See comments in process.c:copy_thread() for details. 368 */ 369 movb PT_OLDSS(%esp), %ah 370 movb PT_CS(%esp), %al 371 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax 372 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax 373 je ldt_ss # returning to user-space with LDT SS 374#endif 375restore_nocheck: 376 RESTORE_REGS 4 # skip orig_eax/error_code 377irq_return: 378 INTERRUPT_RETURN 379.section .fixup, "ax" 380ENTRY(iret_exc ) 381 pushl $0 # no error code 382 pushl $do_iret_error 383 jmp error_code 384.previous 385 _ASM_EXTABLE(irq_return, iret_exc) 386 387#ifdef CONFIG_X86_ESPFIX32 388ldt_ss: 389#ifdef CONFIG_PARAVIRT 390 /* 391 * The kernel can't run on a non-flat stack if paravirt mode 392 * is active. Rather than try to fixup the high bits of 393 * ESP, bypass this code entirely. This may break DOSemu 394 * and/or Wine support in a paravirt VM, although the option 395 * is still available to implement the setting of the high 396 * 16-bits in the INTERRUPT_RETURN paravirt-op. 397 */ 398 cmpl $0, pv_info+PARAVIRT_enabled 399 jne restore_nocheck 400#endif 401 402/* 403 * Setup and switch to ESPFIX stack 404 * 405 * We're returning to userspace with a 16 bit stack. The CPU will not 406 * restore the high word of ESP for us on executing iret... This is an 407 * "official" bug of all the x86-compatible CPUs, which we can work 408 * around to make dosemu and wine happy. We do this by preloading the 409 * high word of ESP with the high word of the userspace ESP while 410 * compensating for the offset by changing to the ESPFIX segment with 411 * a base address that matches for the difference. 412 */ 413#define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8) 414 mov %esp, %edx /* load kernel esp */ 415 mov PT_OLDESP(%esp), %eax /* load userspace esp */ 416 mov %dx, %ax /* eax: new kernel esp */ 417 sub %eax, %edx /* offset (low word is 0) */ 418 shr $16, %edx 419 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */ 420 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */ 421 pushl $__ESPFIX_SS 422 pushl %eax /* new kernel esp */ 423 /* 424 * Disable interrupts, but do not irqtrace this section: we 425 * will soon execute iret and the tracer was already set to 426 * the irqstate after the IRET: 427 */ 428 DISABLE_INTERRUPTS(CLBR_EAX) 429 lss (%esp), %esp /* switch to espfix segment */ 430 jmp restore_nocheck 431#endif 432ENDPROC(entry_INT80_32) 433 434.macro FIXUP_ESPFIX_STACK 435/* 436 * Switch back for ESPFIX stack to the normal zerobased stack 437 * 438 * We can't call C functions using the ESPFIX stack. This code reads 439 * the high word of the segment base from the GDT and swiches to the 440 * normal stack and adjusts ESP with the matching offset. 441 */ 442#ifdef CONFIG_X86_ESPFIX32 443 /* fixup the stack */ 444 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */ 445 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */ 446 shl $16, %eax 447 addl %esp, %eax /* the adjusted stack pointer */ 448 pushl $__KERNEL_DS 449 pushl %eax 450 lss (%esp), %esp /* switch to the normal stack segment */ 451#endif 452.endm 453.macro UNWIND_ESPFIX_STACK 454#ifdef CONFIG_X86_ESPFIX32 455 movl %ss, %eax 456 /* see if on espfix stack */ 457 cmpw $__ESPFIX_SS, %ax 458 jne 27f 459 movl $__KERNEL_DS, %eax 460 movl %eax, %ds 461 movl %eax, %es 462 /* switch to normal stack */ 463 FIXUP_ESPFIX_STACK 46427: 465#endif 466.endm 467 468/* 469 * Build the entry stubs with some assembler magic. 470 * We pack 1 stub into every 8-byte block. 471 */ 472 .align 8 473ENTRY(irq_entries_start) 474 vector=FIRST_EXTERNAL_VECTOR 475 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) 476 pushl $(~vector+0x80) /* Note: always in signed byte range */ 477 vector=vector+1 478 jmp common_interrupt 479 .align 8 480 .endr 481END(irq_entries_start) 482 483/* 484 * the CPU automatically disables interrupts when executing an IRQ vector, 485 * so IRQ-flags tracing has to follow that: 486 */ 487 .p2align CONFIG_X86_L1_CACHE_SHIFT 488common_interrupt: 489 ASM_CLAC 490 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ 491 SAVE_ALL 492 TRACE_IRQS_OFF 493 movl %esp, %eax 494 call do_IRQ 495 jmp ret_from_intr 496ENDPROC(common_interrupt) 497 498#define BUILD_INTERRUPT3(name, nr, fn) \ 499ENTRY(name) \ 500 ASM_CLAC; \ 501 pushl $~(nr); \ 502 SAVE_ALL; \ 503 TRACE_IRQS_OFF \ 504 movl %esp, %eax; \ 505 call fn; \ 506 jmp ret_from_intr; \ 507ENDPROC(name) 508 509 510#ifdef CONFIG_TRACING 511# define TRACE_BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name) 512#else 513# define TRACE_BUILD_INTERRUPT(name, nr) 514#endif 515 516#define BUILD_INTERRUPT(name, nr) \ 517 BUILD_INTERRUPT3(name, nr, smp_##name); \ 518 TRACE_BUILD_INTERRUPT(name, nr) 519 520/* The include is where all of the SMP etc. interrupts come from */ 521#include <asm/entry_arch.h> 522 523ENTRY(coprocessor_error) 524 ASM_CLAC 525 pushl $0 526 pushl $do_coprocessor_error 527 jmp error_code 528END(coprocessor_error) 529 530ENTRY(simd_coprocessor_error) 531 ASM_CLAC 532 pushl $0 533#ifdef CONFIG_X86_INVD_BUG 534 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 535 ALTERNATIVE "pushl $do_general_protection", \ 536 "pushl $do_simd_coprocessor_error", \ 537 X86_FEATURE_XMM 538#else 539 pushl $do_simd_coprocessor_error 540#endif 541 jmp error_code 542END(simd_coprocessor_error) 543 544ENTRY(device_not_available) 545 ASM_CLAC 546 pushl $-1 # mark this as an int 547 pushl $do_device_not_available 548 jmp error_code 549END(device_not_available) 550 551#ifdef CONFIG_PARAVIRT 552ENTRY(native_iret) 553 iret 554 _ASM_EXTABLE(native_iret, iret_exc) 555END(native_iret) 556#endif 557 558ENTRY(overflow) 559 ASM_CLAC 560 pushl $0 561 pushl $do_overflow 562 jmp error_code 563END(overflow) 564 565ENTRY(bounds) 566 ASM_CLAC 567 pushl $0 568 pushl $do_bounds 569 jmp error_code 570END(bounds) 571 572ENTRY(invalid_op) 573 ASM_CLAC 574 pushl $0 575 pushl $do_invalid_op 576 jmp error_code 577END(invalid_op) 578 579ENTRY(coprocessor_segment_overrun) 580 ASM_CLAC 581 pushl $0 582 pushl $do_coprocessor_segment_overrun 583 jmp error_code 584END(coprocessor_segment_overrun) 585 586ENTRY(invalid_TSS) 587 ASM_CLAC 588 pushl $do_invalid_TSS 589 jmp error_code 590END(invalid_TSS) 591 592ENTRY(segment_not_present) 593 ASM_CLAC 594 pushl $do_segment_not_present 595 jmp error_code 596END(segment_not_present) 597 598ENTRY(stack_segment) 599 ASM_CLAC 600 pushl $do_stack_segment 601 jmp error_code 602END(stack_segment) 603 604ENTRY(alignment_check) 605 ASM_CLAC 606 pushl $do_alignment_check 607 jmp error_code 608END(alignment_check) 609 610ENTRY(divide_error) 611 ASM_CLAC 612 pushl $0 # no error code 613 pushl $do_divide_error 614 jmp error_code 615END(divide_error) 616 617#ifdef CONFIG_X86_MCE 618ENTRY(machine_check) 619 ASM_CLAC 620 pushl $0 621 pushl machine_check_vector 622 jmp error_code 623END(machine_check) 624#endif 625 626ENTRY(spurious_interrupt_bug) 627 ASM_CLAC 628 pushl $0 629 pushl $do_spurious_interrupt_bug 630 jmp error_code 631END(spurious_interrupt_bug) 632 633#ifdef CONFIG_XEN 634/* 635 * Xen doesn't set %esp to be precisely what the normal SYSENTER 636 * entry point expects, so fix it up before using the normal path. 637 */ 638ENTRY(xen_sysenter_target) 639 addl $5*4, %esp /* remove xen-provided frame */ 640 jmp sysenter_past_esp 641 642ENTRY(xen_hypervisor_callback) 643 pushl $-1 /* orig_ax = -1 => not a system call */ 644 SAVE_ALL 645 TRACE_IRQS_OFF 646 647 /* 648 * Check to see if we got the event in the critical 649 * region in xen_iret_direct, after we've reenabled 650 * events and checked for pending events. This simulates 651 * iret instruction's behaviour where it delivers a 652 * pending interrupt when enabling interrupts: 653 */ 654 movl PT_EIP(%esp), %eax 655 cmpl $xen_iret_start_crit, %eax 656 jb 1f 657 cmpl $xen_iret_end_crit, %eax 658 jae 1f 659 660 jmp xen_iret_crit_fixup 661 662ENTRY(xen_do_upcall) 6631: mov %esp, %eax 664 call xen_evtchn_do_upcall 665#ifndef CONFIG_PREEMPT 666 call xen_maybe_preempt_hcall 667#endif 668 jmp ret_from_intr 669ENDPROC(xen_hypervisor_callback) 670 671/* 672 * Hypervisor uses this for application faults while it executes. 673 * We get here for two reasons: 674 * 1. Fault while reloading DS, ES, FS or GS 675 * 2. Fault while executing IRET 676 * Category 1 we fix up by reattempting the load, and zeroing the segment 677 * register if the load fails. 678 * Category 2 we fix up by jumping to do_iret_error. We cannot use the 679 * normal Linux return path in this case because if we use the IRET hypercall 680 * to pop the stack frame we end up in an infinite loop of failsafe callbacks. 681 * We distinguish between categories by maintaining a status value in EAX. 682 */ 683ENTRY(xen_failsafe_callback) 684 pushl %eax 685 movl $1, %eax 6861: mov 4(%esp), %ds 6872: mov 8(%esp), %es 6883: mov 12(%esp), %fs 6894: mov 16(%esp), %gs 690 /* EAX == 0 => Category 1 (Bad segment) 691 EAX != 0 => Category 2 (Bad IRET) */ 692 testl %eax, %eax 693 popl %eax 694 lea 16(%esp), %esp 695 jz 5f 696 jmp iret_exc 6975: pushl $-1 /* orig_ax = -1 => not a system call */ 698 SAVE_ALL 699 jmp ret_from_exception 700 701.section .fixup, "ax" 7026: xorl %eax, %eax 703 movl %eax, 4(%esp) 704 jmp 1b 7057: xorl %eax, %eax 706 movl %eax, 8(%esp) 707 jmp 2b 7088: xorl %eax, %eax 709 movl %eax, 12(%esp) 710 jmp 3b 7119: xorl %eax, %eax 712 movl %eax, 16(%esp) 713 jmp 4b 714.previous 715 _ASM_EXTABLE(1b, 6b) 716 _ASM_EXTABLE(2b, 7b) 717 _ASM_EXTABLE(3b, 8b) 718 _ASM_EXTABLE(4b, 9b) 719ENDPROC(xen_failsafe_callback) 720 721BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 722 xen_evtchn_do_upcall) 723 724#endif /* CONFIG_XEN */ 725 726#if IS_ENABLED(CONFIG_HYPERV) 727 728BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, 729 hyperv_vector_handler) 730 731#endif /* CONFIG_HYPERV */ 732 733#ifdef CONFIG_FUNCTION_TRACER 734#ifdef CONFIG_DYNAMIC_FTRACE 735 736ENTRY(mcount) 737 ret 738END(mcount) 739 740ENTRY(ftrace_caller) 741 pushl %eax 742 pushl %ecx 743 pushl %edx 744 pushl $0 /* Pass NULL as regs pointer */ 745 movl 4*4(%esp), %eax 746 movl 0x4(%ebp), %edx 747 movl function_trace_op, %ecx 748 subl $MCOUNT_INSN_SIZE, %eax 749 750.globl ftrace_call 751ftrace_call: 752 call ftrace_stub 753 754 addl $4, %esp /* skip NULL pointer */ 755 popl %edx 756 popl %ecx 757 popl %eax 758ftrace_ret: 759#ifdef CONFIG_FUNCTION_GRAPH_TRACER 760.globl ftrace_graph_call 761ftrace_graph_call: 762 jmp ftrace_stub 763#endif 764 765.globl ftrace_stub 766ftrace_stub: 767 ret 768END(ftrace_caller) 769 770ENTRY(ftrace_regs_caller) 771 pushf /* push flags before compare (in cs location) */ 772 773 /* 774 * i386 does not save SS and ESP when coming from kernel. 775 * Instead, to get sp, ®s->sp is used (see ptrace.h). 776 * Unfortunately, that means eflags must be at the same location 777 * as the current return ip is. We move the return ip into the 778 * ip location, and move flags into the return ip location. 779 */ 780 pushl 4(%esp) /* save return ip into ip slot */ 781 782 pushl $0 /* Load 0 into orig_ax */ 783 pushl %gs 784 pushl %fs 785 pushl %es 786 pushl %ds 787 pushl %eax 788 pushl %ebp 789 pushl %edi 790 pushl %esi 791 pushl %edx 792 pushl %ecx 793 pushl %ebx 794 795 movl 13*4(%esp), %eax /* Get the saved flags */ 796 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */ 797 /* clobbering return ip */ 798 movl $__KERNEL_CS, 13*4(%esp) 799 800 movl 12*4(%esp), %eax /* Load ip (1st parameter) */ 801 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */ 802 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */ 803 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */ 804 pushl %esp /* Save pt_regs as 4th parameter */ 805 806GLOBAL(ftrace_regs_call) 807 call ftrace_stub 808 809 addl $4, %esp /* Skip pt_regs */ 810 movl 14*4(%esp), %eax /* Move flags back into cs */ 811 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */ 812 movl 12*4(%esp), %eax /* Get return ip from regs->ip */ 813 movl %eax, 14*4(%esp) /* Put return ip back for ret */ 814 815 popl %ebx 816 popl %ecx 817 popl %edx 818 popl %esi 819 popl %edi 820 popl %ebp 821 popl %eax 822 popl %ds 823 popl %es 824 popl %fs 825 popl %gs 826 addl $8, %esp /* Skip orig_ax and ip */ 827 popf /* Pop flags at end (no addl to corrupt flags) */ 828 jmp ftrace_ret 829 830 popf 831 jmp ftrace_stub 832#else /* ! CONFIG_DYNAMIC_FTRACE */ 833 834ENTRY(mcount) 835 cmpl $__PAGE_OFFSET, %esp 836 jb ftrace_stub /* Paging not enabled yet? */ 837 838 cmpl $ftrace_stub, ftrace_trace_function 839 jnz trace 840#ifdef CONFIG_FUNCTION_GRAPH_TRACER 841 cmpl $ftrace_stub, ftrace_graph_return 842 jnz ftrace_graph_caller 843 844 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry 845 jnz ftrace_graph_caller 846#endif 847.globl ftrace_stub 848ftrace_stub: 849 ret 850 851 /* taken from glibc */ 852trace: 853 pushl %eax 854 pushl %ecx 855 pushl %edx 856 movl 0xc(%esp), %eax 857 movl 0x4(%ebp), %edx 858 subl $MCOUNT_INSN_SIZE, %eax 859 860 call *ftrace_trace_function 861 862 popl %edx 863 popl %ecx 864 popl %eax 865 jmp ftrace_stub 866END(mcount) 867#endif /* CONFIG_DYNAMIC_FTRACE */ 868#endif /* CONFIG_FUNCTION_TRACER */ 869 870#ifdef CONFIG_FUNCTION_GRAPH_TRACER 871ENTRY(ftrace_graph_caller) 872 pushl %eax 873 pushl %ecx 874 pushl %edx 875 movl 0xc(%esp), %eax 876 lea 0x4(%ebp), %edx 877 movl (%ebp), %ecx 878 subl $MCOUNT_INSN_SIZE, %eax 879 call prepare_ftrace_return 880 popl %edx 881 popl %ecx 882 popl %eax 883 ret 884END(ftrace_graph_caller) 885 886.globl return_to_handler 887return_to_handler: 888 pushl %eax 889 pushl %edx 890 movl %ebp, %eax 891 call ftrace_return_to_handler 892 movl %eax, %ecx 893 popl %edx 894 popl %eax 895 jmp *%ecx 896#endif 897 898#ifdef CONFIG_TRACING 899ENTRY(trace_page_fault) 900 ASM_CLAC 901 pushl $trace_do_page_fault 902 jmp error_code 903END(trace_page_fault) 904#endif 905 906ENTRY(page_fault) 907 ASM_CLAC 908 pushl $do_page_fault 909 ALIGN 910error_code: 911 /* the function address is in %gs's slot on the stack */ 912 pushl %fs 913 pushl %es 914 pushl %ds 915 pushl %eax 916 pushl %ebp 917 pushl %edi 918 pushl %esi 919 pushl %edx 920 pushl %ecx 921 pushl %ebx 922 cld 923 movl $(__KERNEL_PERCPU), %ecx 924 movl %ecx, %fs 925 UNWIND_ESPFIX_STACK 926 GS_TO_REG %ecx 927 movl PT_GS(%esp), %edi # get the function address 928 movl PT_ORIG_EAX(%esp), %edx # get the error code 929 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart 930 REG_TO_PTGS %ecx 931 SET_KERNEL_GS %ecx 932 movl $(__USER_DS), %ecx 933 movl %ecx, %ds 934 movl %ecx, %es 935 TRACE_IRQS_OFF 936 movl %esp, %eax # pt_regs pointer 937 call *%edi 938 jmp ret_from_exception 939END(page_fault) 940 941/* 942 * Debug traps and NMI can happen at the one SYSENTER instruction 943 * that sets up the real kernel stack. Check here, since we can't 944 * allow the wrong stack to be used. 945 * 946 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have 947 * already pushed 3 words if it hits on the sysenter instruction: 948 * eflags, cs and eip. 949 * 950 * We just load the right stack, and push the three (known) values 951 * by hand onto the new stack - while updating the return eip past 952 * the instruction that would have done it for sysenter. 953 */ 954.macro FIX_STACK offset ok label 955 cmpw $__KERNEL_CS, 4(%esp) 956 jne \ok 957\label: 958 movl TSS_sysenter_sp0 + \offset(%esp), %esp 959 pushfl 960 pushl $__KERNEL_CS 961 pushl $sysenter_past_esp 962.endm 963 964ENTRY(debug) 965 ASM_CLAC 966 cmpl $entry_SYSENTER_32, (%esp) 967 jne debug_stack_correct 968 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 969debug_stack_correct: 970 pushl $-1 # mark this as an int 971 SAVE_ALL 972 TRACE_IRQS_OFF 973 xorl %edx, %edx # error code 0 974 movl %esp, %eax # pt_regs pointer 975 call do_debug 976 jmp ret_from_exception 977END(debug) 978 979/* 980 * NMI is doubly nasty. It can happen _while_ we're handling 981 * a debug fault, and the debug fault hasn't yet been able to 982 * clear up the stack. So we first check whether we got an 983 * NMI on the sysenter entry path, but after that we need to 984 * check whether we got an NMI on the debug path where the debug 985 * fault happened on the sysenter path. 986 */ 987ENTRY(nmi) 988 ASM_CLAC 989#ifdef CONFIG_X86_ESPFIX32 990 pushl %eax 991 movl %ss, %eax 992 cmpw $__ESPFIX_SS, %ax 993 popl %eax 994 je nmi_espfix_stack 995#endif 996 cmpl $entry_SYSENTER_32, (%esp) 997 je nmi_stack_fixup 998 pushl %eax 999 movl %esp, %eax 1000 /* 1001 * Do not access memory above the end of our stack page, 1002 * it might not exist. 1003 */ 1004 andl $(THREAD_SIZE-1), %eax 1005 cmpl $(THREAD_SIZE-20), %eax 1006 popl %eax 1007 jae nmi_stack_correct 1008 cmpl $entry_SYSENTER_32, 12(%esp) 1009 je nmi_debug_stack_check 1010nmi_stack_correct: 1011 pushl %eax 1012 SAVE_ALL 1013 xorl %edx, %edx # zero error code 1014 movl %esp, %eax # pt_regs pointer 1015 call do_nmi 1016 jmp restore_all_notrace 1017 1018nmi_stack_fixup: 1019 FIX_STACK 12, nmi_stack_correct, 1 1020 jmp nmi_stack_correct 1021 1022nmi_debug_stack_check: 1023 cmpw $__KERNEL_CS, 16(%esp) 1024 jne nmi_stack_correct 1025 cmpl $debug, (%esp) 1026 jb nmi_stack_correct 1027 cmpl $debug_esp_fix_insn, (%esp) 1028 ja nmi_stack_correct 1029 FIX_STACK 24, nmi_stack_correct, 1 1030 jmp nmi_stack_correct 1031 1032#ifdef CONFIG_X86_ESPFIX32 1033nmi_espfix_stack: 1034 /* 1035 * create the pointer to lss back 1036 */ 1037 pushl %ss 1038 pushl %esp 1039 addl $4, (%esp) 1040 /* copy the iret frame of 12 bytes */ 1041 .rept 3 1042 pushl 16(%esp) 1043 .endr 1044 pushl %eax 1045 SAVE_ALL 1046 FIXUP_ESPFIX_STACK # %eax == %esp 1047 xorl %edx, %edx # zero error code 1048 call do_nmi 1049 RESTORE_REGS 1050 lss 12+4(%esp), %esp # back to espfix stack 1051 jmp irq_return 1052#endif 1053END(nmi) 1054 1055ENTRY(int3) 1056 ASM_CLAC 1057 pushl $-1 # mark this as an int 1058 SAVE_ALL 1059 TRACE_IRQS_OFF 1060 xorl %edx, %edx # zero error code 1061 movl %esp, %eax # pt_regs pointer 1062 call do_int3 1063 jmp ret_from_exception 1064END(int3) 1065 1066ENTRY(general_protection) 1067 pushl $do_general_protection 1068 jmp error_code 1069END(general_protection) 1070 1071#ifdef CONFIG_KVM_GUEST 1072ENTRY(async_page_fault) 1073 ASM_CLAC 1074 pushl $do_async_page_fault 1075 jmp error_code 1076END(async_page_fault) 1077#endif 1078