1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_SYNC_CORE_H 3 #define _ASM_X86_SYNC_CORE_H 4 5 #include <linux/preempt.h> 6 #include <asm/processor.h> 7 #include <asm/cpufeature.h> 8 #include <asm/special_insns.h> 9 10 #ifdef CONFIG_X86_32 11 static inline void iret_to_self(void) 12 { 13 asm volatile ( 14 "pushfl\n\t" 15 "pushl %%cs\n\t" 16 "pushl $1f\n\t" 17 "iret\n\t" 18 "1:" 19 : ASM_CALL_CONSTRAINT : : "memory"); 20 } 21 #else 22 static inline void iret_to_self(void) 23 { 24 unsigned int tmp; 25 26 asm volatile ( 27 "mov %%ss, %0\n\t" 28 "pushq %q0\n\t" 29 "pushq %%rsp\n\t" 30 "addq $8, (%%rsp)\n\t" 31 "pushfq\n\t" 32 "mov %%cs, %0\n\t" 33 "pushq %q0\n\t" 34 "pushq $1f\n\t" 35 "iretq\n\t" 36 "1:" 37 : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory"); 38 } 39 #endif /* CONFIG_X86_32 */ 40 41 /* 42 * This function forces the icache and prefetched instruction stream to 43 * catch up with reality in two very specific cases: 44 * 45 * a) Text was modified using one virtual address and is about to be executed 46 * from the same physical page at a different virtual address. 47 * 48 * b) Text was modified on a different CPU, may subsequently be 49 * executed on this CPU, and you want to make sure the new version 50 * gets executed. This generally means you're calling this in an IPI. 51 * 52 * If you're calling this for a different reason, you're probably doing 53 * it wrong. 54 * 55 * Like all of Linux's memory ordering operations, this is a 56 * compiler barrier as well. 57 */ 58 static inline void sync_core(void) 59 { 60 /* 61 * The SERIALIZE instruction is the most straightforward way to 62 * do this, but it is not universally available. 63 */ 64 if (static_cpu_has(X86_FEATURE_SERIALIZE)) { 65 serialize(); 66 return; 67 } 68 69 /* 70 * For all other processors, there are quite a few ways to do this. 71 * IRET-to-self is nice because it works on every CPU, at any CPL 72 * (so it's compatible with paravirtualization), and it never exits 73 * to a hypervisor. The only downsides are that it's a bit slow 74 * (it seems to be a bit more than 2x slower than the fastest 75 * options) and that it unmasks NMIs. The "push %cs" is needed, 76 * because in paravirtual environments __KERNEL_CS may not be a 77 * valid CS value when we do IRET directly. 78 * 79 * In case NMI unmasking or performance ever becomes a problem, 80 * the next best option appears to be MOV-to-CR2 and an 81 * unconditional jump. That sequence also works on all CPUs, 82 * but it will fault at CPL3 (i.e. Xen PV). 83 * 84 * CPUID is the conventional way, but it's nasty: it doesn't 85 * exist on some 486-like CPUs, and it usually exits to a 86 * hypervisor. 87 */ 88 iret_to_self(); 89 } 90 91 /* 92 * Ensure that a core serializing instruction is issued before returning 93 * to user-mode. x86 implements return to user-space through sysexit, 94 * sysrel, and sysretq, which are not core serializing. 95 */ 96 static inline void sync_core_before_usermode(void) 97 { 98 /* With PTI, we unconditionally serialize before running user code. */ 99 if (static_cpu_has(X86_FEATURE_PTI)) 100 return; 101 102 /* 103 * Even if we're in an interrupt, we might reschedule before returning, 104 * in which case we could switch to a different thread in the same mm 105 * and return using SYSRET or SYSEXIT. Instead of trying to keep 106 * track of our need to sync the core, just sync right away. 107 */ 108 sync_core(); 109 } 110 111 #endif /* _ASM_X86_SYNC_CORE_H */ 112