1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Asm versions of Xen pv-ops, suitable for direct use. 4 * 5 * We only bother with direct forms (ie, vcpu in percpu data) of the 6 * operations here; the indirect forms are better handled in C. 7 */ 8 9#include <asm/errno.h> 10#include <asm/asm-offsets.h> 11#include <asm/percpu.h> 12#include <asm/processor-flags.h> 13#include <asm/segment.h> 14#include <asm/thread_info.h> 15#include <asm/asm.h> 16#include <asm/frame.h> 17#include <asm/unwind_hints.h> 18 19#include <xen/interface/xen.h> 20 21#include <linux/init.h> 22#include <linux/linkage.h> 23#include <../entry/calling.h> 24 25/* 26 * Enable events. This clears the event mask and tests the pending 27 * event status with one and operation. If there are pending events, 28 * then enter the hypervisor to get them handled. 29 */ 30SYM_FUNC_START(xen_irq_enable_direct) 31 FRAME_BEGIN 32 /* Unmask events */ 33 movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 34 35 /* 36 * Preempt here doesn't matter because that will deal with any 37 * pending interrupts. The pending check may end up being run 38 * on the wrong CPU, but that doesn't hurt. 39 */ 40 41 /* Test for pending */ 42 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending 43 jz 1f 44 45 call check_events 461: 47 FRAME_END 48 RET 49SYM_FUNC_END(xen_irq_enable_direct) 50 51 52/* 53 * Disabling events is simply a matter of making the event mask 54 * non-zero. 55 */ 56SYM_FUNC_START(xen_irq_disable_direct) 57 movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 58 RET 59SYM_FUNC_END(xen_irq_disable_direct) 60 61/* 62 * (xen_)save_fl is used to get the current interrupt enable status. 63 * Callers expect the status to be in X86_EFLAGS_IF, and other bits 64 * may be set in the return value. We take advantage of this by 65 * making sure that X86_EFLAGS_IF has the right value (and other bits 66 * in that byte are 0), but other bits in the return value are 67 * undefined. We need to toggle the state of the bit, because Xen and 68 * x86 use opposite senses (mask vs enable). 69 */ 70SYM_FUNC_START(xen_save_fl_direct) 71 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 72 setz %ah 73 addb %ah, %ah 74 RET 75SYM_FUNC_END(xen_save_fl_direct) 76 77/* 78 * Force an event check by making a hypercall, but preserve regs 79 * before making the call. 80 */ 81SYM_FUNC_START(check_events) 82 FRAME_BEGIN 83 push %rax 84 push %rcx 85 push %rdx 86 push %rsi 87 push %rdi 88 push %r8 89 push %r9 90 push %r10 91 push %r11 92 call xen_force_evtchn_callback 93 pop %r11 94 pop %r10 95 pop %r9 96 pop %r8 97 pop %rdi 98 pop %rsi 99 pop %rdx 100 pop %rcx 101 pop %rax 102 FRAME_END 103 RET 104SYM_FUNC_END(check_events) 105 106SYM_FUNC_START(xen_read_cr2) 107 FRAME_BEGIN 108 _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX 109 _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX 110 FRAME_END 111 RET 112SYM_FUNC_END(xen_read_cr2); 113 114SYM_FUNC_START(xen_read_cr2_direct) 115 FRAME_BEGIN 116 _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX 117 FRAME_END 118 RET 119SYM_FUNC_END(xen_read_cr2_direct); 120 121.macro xen_pv_trap name 122SYM_CODE_START(xen_\name) 123 UNWIND_HINT_EMPTY 124 pop %rcx 125 pop %r11 126 jmp \name 127SYM_CODE_END(xen_\name) 128_ASM_NOKPROBE(xen_\name) 129.endm 130 131xen_pv_trap asm_exc_divide_error 132xen_pv_trap asm_xenpv_exc_debug 133xen_pv_trap asm_exc_int3 134xen_pv_trap asm_xenpv_exc_nmi 135xen_pv_trap asm_exc_overflow 136xen_pv_trap asm_exc_bounds 137xen_pv_trap asm_exc_invalid_op 138xen_pv_trap asm_exc_device_not_available 139xen_pv_trap asm_xenpv_exc_double_fault 140xen_pv_trap asm_exc_coproc_segment_overrun 141xen_pv_trap asm_exc_invalid_tss 142xen_pv_trap asm_exc_segment_not_present 143xen_pv_trap asm_exc_stack_segment 144xen_pv_trap asm_exc_general_protection 145xen_pv_trap asm_exc_page_fault 146xen_pv_trap asm_exc_spurious_interrupt_bug 147xen_pv_trap asm_exc_coprocessor_error 148xen_pv_trap asm_exc_alignment_check 149#ifdef CONFIG_X86_MCE 150xen_pv_trap asm_xenpv_exc_machine_check 151#endif /* CONFIG_X86_MCE */ 152xen_pv_trap asm_exc_simd_coprocessor_error 153#ifdef CONFIG_IA32_EMULATION 154xen_pv_trap entry_INT80_compat 155#endif 156xen_pv_trap asm_exc_xen_unknown_trap 157xen_pv_trap asm_exc_xen_hypervisor_callback 158 159 __INIT 160SYM_CODE_START(xen_early_idt_handler_array) 161 i = 0 162 .rept NUM_EXCEPTION_VECTORS 163 UNWIND_HINT_EMPTY 164 pop %rcx 165 pop %r11 166 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE 167 i = i + 1 168 .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc 169 .endr 170SYM_CODE_END(xen_early_idt_handler_array) 171 __FINIT 172 173hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 174/* 175 * Xen64 iret frame: 176 * 177 * ss 178 * rsp 179 * rflags 180 * cs 181 * rip <-- standard iret frame 182 * 183 * flags 184 * 185 * rcx } 186 * r11 }<-- pushed by hypercall page 187 * rsp->rax } 188 */ 189SYM_CODE_START(xen_iret) 190 UNWIND_HINT_EMPTY 191 pushq $0 192 jmp hypercall_iret 193SYM_CODE_END(xen_iret) 194 195/* 196 * XEN pv doesn't use trampoline stack, PER_CPU_VAR(cpu_tss_rw + TSS_sp0) is 197 * also the kernel stack. Reusing swapgs_restore_regs_and_return_to_usermode() 198 * in XEN pv would cause %rsp to move up to the top of the kernel stack and 199 * leave the IRET frame below %rsp, which is dangerous to be corrupted if #NMI 200 * interrupts. And swapgs_restore_regs_and_return_to_usermode() pushing the IRET 201 * frame at the same address is useless. 202 */ 203SYM_CODE_START(xenpv_restore_regs_and_return_to_usermode) 204 UNWIND_HINT_REGS 205 POP_REGS 206 207 /* stackleak_erase() can work safely on the kernel stack. */ 208 STACKLEAK_ERASE_NOCLOBBER 209 210 addq $8, %rsp /* skip regs->orig_ax */ 211 jmp xen_iret 212SYM_CODE_END(xenpv_restore_regs_and_return_to_usermode) 213 214/* 215 * Xen handles syscall callbacks much like ordinary exceptions, which 216 * means we have: 217 * - kernel gs 218 * - kernel rsp 219 * - an iret-like stack frame on the stack (including rcx and r11): 220 * ss 221 * rsp 222 * rflags 223 * cs 224 * rip 225 * r11 226 * rsp->rcx 227 */ 228 229/* Normal 64-bit system call target */ 230SYM_CODE_START(xen_entry_SYSCALL_64) 231 UNWIND_HINT_EMPTY 232 popq %rcx 233 popq %r11 234 235 /* 236 * Neither Xen nor the kernel really knows what the old SS and 237 * CS were. The kernel expects __USER_DS and __USER_CS, so 238 * report those values even though Xen will guess its own values. 239 */ 240 movq $__USER_DS, 4*8(%rsp) 241 movq $__USER_CS, 1*8(%rsp) 242 243 jmp entry_SYSCALL_64_after_hwframe 244SYM_CODE_END(xen_entry_SYSCALL_64) 245 246#ifdef CONFIG_IA32_EMULATION 247 248/* 32-bit compat syscall target */ 249SYM_CODE_START(xen_entry_SYSCALL_compat) 250 UNWIND_HINT_EMPTY 251 popq %rcx 252 popq %r11 253 254 /* 255 * Neither Xen nor the kernel really knows what the old SS and 256 * CS were. The kernel expects __USER32_DS and __USER32_CS, so 257 * report those values even though Xen will guess its own values. 258 */ 259 movq $__USER32_DS, 4*8(%rsp) 260 movq $__USER32_CS, 1*8(%rsp) 261 262 jmp entry_SYSCALL_compat_after_hwframe 263SYM_CODE_END(xen_entry_SYSCALL_compat) 264 265/* 32-bit compat sysenter target */ 266SYM_CODE_START(xen_entry_SYSENTER_compat) 267 UNWIND_HINT_EMPTY 268 /* 269 * NB: Xen is polite and clears TF from EFLAGS for us. This means 270 * that we don't need to guard against single step exceptions here. 271 */ 272 popq %rcx 273 popq %r11 274 275 /* 276 * Neither Xen nor the kernel really knows what the old SS and 277 * CS were. The kernel expects __USER32_DS and __USER32_CS, so 278 * report those values even though Xen will guess its own values. 279 */ 280 movq $__USER32_DS, 4*8(%rsp) 281 movq $__USER32_CS, 1*8(%rsp) 282 283 jmp entry_SYSENTER_compat_after_hwframe 284SYM_CODE_END(xen_entry_SYSENTER_compat) 285 286#else /* !CONFIG_IA32_EMULATION */ 287 288SYM_CODE_START(xen_entry_SYSCALL_compat) 289SYM_CODE_START(xen_entry_SYSENTER_compat) 290 UNWIND_HINT_EMPTY 291 lea 16(%rsp), %rsp /* strip %rcx, %r11 */ 292 mov $-ENOSYS, %rax 293 pushq $0 294 jmp hypercall_iret 295SYM_CODE_END(xen_entry_SYSENTER_compat) 296SYM_CODE_END(xen_entry_SYSCALL_compat) 297 298#endif /* CONFIG_IA32_EMULATION */ 299