1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Asm versions of Xen pv-ops, suitable for direct use. 4 * 5 * We only bother with direct forms (ie, vcpu in percpu data) of the 6 * operations here; the indirect forms are better handled in C. 7 */ 8 9#include <asm/errno.h> 10#include <asm/asm-offsets.h> 11#include <asm/percpu.h> 12#include <asm/processor-flags.h> 13#include <asm/segment.h> 14#include <asm/thread_info.h> 15#include <asm/asm.h> 16#include <asm/frame.h> 17 18#include <xen/interface/xen.h> 19 20#include <linux/init.h> 21#include <linux/linkage.h> 22 23/* 24 * Enable events. This clears the event mask and tests the pending 25 * event status with one and operation. If there are pending events, 26 * then enter the hypervisor to get them handled. 27 */ 28SYM_FUNC_START(xen_irq_enable_direct) 29 FRAME_BEGIN 30 /* Unmask events */ 31 movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 32 33 /* 34 * Preempt here doesn't matter because that will deal with any 35 * pending interrupts. The pending check may end up being run 36 * on the wrong CPU, but that doesn't hurt. 37 */ 38 39 /* Test for pending */ 40 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending 41 jz 1f 42 43 call check_events 441: 45 FRAME_END 46 ret 47SYM_FUNC_END(xen_irq_enable_direct) 48 49 50/* 51 * Disabling events is simply a matter of making the event mask 52 * non-zero. 53 */ 54SYM_FUNC_START(xen_irq_disable_direct) 55 movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 56 ret 57SYM_FUNC_END(xen_irq_disable_direct) 58 59/* 60 * (xen_)save_fl is used to get the current interrupt enable status. 61 * Callers expect the status to be in X86_EFLAGS_IF, and other bits 62 * may be set in the return value. We take advantage of this by 63 * making sure that X86_EFLAGS_IF has the right value (and other bits 64 * in that byte are 0), but other bits in the return value are 65 * undefined. We need to toggle the state of the bit, because Xen and 66 * x86 use opposite senses (mask vs enable). 67 */ 68SYM_FUNC_START(xen_save_fl_direct) 69 testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 70 setz %ah 71 addb %ah, %ah 72 ret 73SYM_FUNC_END(xen_save_fl_direct) 74 75 76/* 77 * In principle the caller should be passing us a value return from 78 * xen_save_fl_direct, but for robustness sake we test only the 79 * X86_EFLAGS_IF flag rather than the whole byte. After setting the 80 * interrupt mask state, it checks for unmasked pending events and 81 * enters the hypervisor to get them delivered if so. 82 */ 83SYM_FUNC_START(xen_restore_fl_direct) 84 FRAME_BEGIN 85 testw $X86_EFLAGS_IF, %di 86 setz PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask 87 /* 88 * Preempt here doesn't matter because that will deal with any 89 * pending interrupts. The pending check may end up being run 90 * on the wrong CPU, but that doesn't hurt. 91 */ 92 93 /* check for unmasked and pending */ 94 cmpw $0x0001, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending 95 jnz 1f 96 call check_events 971: 98 FRAME_END 99 ret 100SYM_FUNC_END(xen_restore_fl_direct) 101 102 103/* 104 * Force an event check by making a hypercall, but preserve regs 105 * before making the call. 106 */ 107SYM_FUNC_START(check_events) 108 FRAME_BEGIN 109 push %rax 110 push %rcx 111 push %rdx 112 push %rsi 113 push %rdi 114 push %r8 115 push %r9 116 push %r10 117 push %r11 118 call xen_force_evtchn_callback 119 pop %r11 120 pop %r10 121 pop %r9 122 pop %r8 123 pop %rdi 124 pop %rsi 125 pop %rdx 126 pop %rcx 127 pop %rax 128 FRAME_END 129 ret 130SYM_FUNC_END(check_events) 131 132SYM_FUNC_START(xen_read_cr2) 133 FRAME_BEGIN 134 _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX 135 _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX 136 FRAME_END 137 ret 138SYM_FUNC_END(xen_read_cr2); 139 140SYM_FUNC_START(xen_read_cr2_direct) 141 FRAME_BEGIN 142 _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX 143 FRAME_END 144 ret 145SYM_FUNC_END(xen_read_cr2_direct); 146 147.macro xen_pv_trap name 148SYM_CODE_START(xen_\name) 149 pop %rcx 150 pop %r11 151 jmp \name 152SYM_CODE_END(xen_\name) 153_ASM_NOKPROBE(xen_\name) 154.endm 155 156xen_pv_trap asm_exc_divide_error 157xen_pv_trap asm_xenpv_exc_debug 158xen_pv_trap asm_exc_int3 159xen_pv_trap asm_xenpv_exc_nmi 160xen_pv_trap asm_exc_overflow 161xen_pv_trap asm_exc_bounds 162xen_pv_trap asm_exc_invalid_op 163xen_pv_trap asm_exc_device_not_available 164xen_pv_trap asm_xenpv_exc_double_fault 165xen_pv_trap asm_exc_coproc_segment_overrun 166xen_pv_trap asm_exc_invalid_tss 167xen_pv_trap asm_exc_segment_not_present 168xen_pv_trap asm_exc_stack_segment 169xen_pv_trap asm_exc_general_protection 170xen_pv_trap asm_exc_page_fault 171xen_pv_trap asm_exc_spurious_interrupt_bug 172xen_pv_trap asm_exc_coprocessor_error 173xen_pv_trap asm_exc_alignment_check 174#ifdef CONFIG_X86_MCE 175xen_pv_trap asm_xenpv_exc_machine_check 176#endif /* CONFIG_X86_MCE */ 177xen_pv_trap asm_exc_simd_coprocessor_error 178#ifdef CONFIG_IA32_EMULATION 179xen_pv_trap entry_INT80_compat 180#endif 181xen_pv_trap asm_exc_xen_unknown_trap 182xen_pv_trap asm_exc_xen_hypervisor_callback 183 184 __INIT 185SYM_CODE_START(xen_early_idt_handler_array) 186 i = 0 187 .rept NUM_EXCEPTION_VECTORS 188 pop %rcx 189 pop %r11 190 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE 191 i = i + 1 192 .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc 193 .endr 194SYM_CODE_END(xen_early_idt_handler_array) 195 __FINIT 196 197hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 198/* 199 * Xen64 iret frame: 200 * 201 * ss 202 * rsp 203 * rflags 204 * cs 205 * rip <-- standard iret frame 206 * 207 * flags 208 * 209 * rcx } 210 * r11 }<-- pushed by hypercall page 211 * rsp->rax } 212 */ 213SYM_CODE_START(xen_iret) 214 pushq $0 215 jmp hypercall_iret 216SYM_CODE_END(xen_iret) 217 218/* 219 * Xen handles syscall callbacks much like ordinary exceptions, which 220 * means we have: 221 * - kernel gs 222 * - kernel rsp 223 * - an iret-like stack frame on the stack (including rcx and r11): 224 * ss 225 * rsp 226 * rflags 227 * cs 228 * rip 229 * r11 230 * rsp->rcx 231 */ 232 233/* Normal 64-bit system call target */ 234SYM_FUNC_START(xen_syscall_target) 235 popq %rcx 236 popq %r11 237 238 /* 239 * Neither Xen nor the kernel really knows what the old SS and 240 * CS were. The kernel expects __USER_DS and __USER_CS, so 241 * report those values even though Xen will guess its own values. 242 */ 243 movq $__USER_DS, 4*8(%rsp) 244 movq $__USER_CS, 1*8(%rsp) 245 246 jmp entry_SYSCALL_64_after_hwframe 247SYM_FUNC_END(xen_syscall_target) 248 249#ifdef CONFIG_IA32_EMULATION 250 251/* 32-bit compat syscall target */ 252SYM_FUNC_START(xen_syscall32_target) 253 popq %rcx 254 popq %r11 255 256 /* 257 * Neither Xen nor the kernel really knows what the old SS and 258 * CS were. The kernel expects __USER32_DS and __USER32_CS, so 259 * report those values even though Xen will guess its own values. 260 */ 261 movq $__USER32_DS, 4*8(%rsp) 262 movq $__USER32_CS, 1*8(%rsp) 263 264 jmp entry_SYSCALL_compat_after_hwframe 265SYM_FUNC_END(xen_syscall32_target) 266 267/* 32-bit compat sysenter target */ 268SYM_FUNC_START(xen_sysenter_target) 269 /* 270 * NB: Xen is polite and clears TF from EFLAGS for us. This means 271 * that we don't need to guard against single step exceptions here. 272 */ 273 popq %rcx 274 popq %r11 275 276 /* 277 * Neither Xen nor the kernel really knows what the old SS and 278 * CS were. The kernel expects __USER32_DS and __USER32_CS, so 279 * report those values even though Xen will guess its own values. 280 */ 281 movq $__USER32_DS, 4*8(%rsp) 282 movq $__USER32_CS, 1*8(%rsp) 283 284 jmp entry_SYSENTER_compat_after_hwframe 285SYM_FUNC_END(xen_sysenter_target) 286 287#else /* !CONFIG_IA32_EMULATION */ 288 289SYM_FUNC_START_ALIAS(xen_syscall32_target) 290SYM_FUNC_START(xen_sysenter_target) 291 lea 16(%rsp), %rsp /* strip %rcx, %r11 */ 292 mov $-ENOSYS, %rax 293 pushq $0 294 jmp hypercall_iret 295SYM_FUNC_END(xen_sysenter_target) 296SYM_FUNC_END_ALIAS(xen_syscall32_target) 297 298#endif /* CONFIG_IA32_EMULATION */ 299