11965aae3SH. Peter Anvin #ifndef _ASM_X86_MMU_CONTEXT_H 21965aae3SH. Peter Anvin #define _ASM_X86_MMU_CONTEXT_H 3bb898558SAl Viro 4bb898558SAl Viro #include <asm/desc.h> 560063497SArun Sharma #include <linux/atomic.h> 6d17d8f9dSDave Hansen #include <linux/mm_types.h> 7d17d8f9dSDave Hansen 8d17d8f9dSDave Hansen #include <trace/events/tlb.h> 9d17d8f9dSDave Hansen 10bb898558SAl Viro #include <asm/pgalloc.h> 11bb898558SAl Viro #include <asm/tlbflush.h> 12bb898558SAl Viro #include <asm/paravirt.h> 13fe3d197fSDave Hansen #include <asm/mpx.h> 14bb898558SAl Viro #ifndef CONFIG_PARAVIRT 15bb898558SAl Viro static inline void paravirt_activate_mm(struct mm_struct *prev, 16bb898558SAl Viro struct mm_struct *next) 17bb898558SAl Viro { 18bb898558SAl Viro } 19bb898558SAl Viro #endif /* !CONFIG_PARAVIRT */ 20bb898558SAl Viro 217911d3f7SAndy Lutomirski #ifdef CONFIG_PERF_EVENTS 22a6673429SAndy Lutomirski extern struct static_key rdpmc_always_available; 23a6673429SAndy Lutomirski 247911d3f7SAndy Lutomirski static inline void load_mm_cr4(struct mm_struct *mm) 257911d3f7SAndy Lutomirski { 26a833581eSPeter Zijlstra if (static_key_false(&rdpmc_always_available) || 27a6673429SAndy Lutomirski atomic_read(&mm->context.perf_rdpmc_allowed)) 287911d3f7SAndy Lutomirski cr4_set_bits(X86_CR4_PCE); 297911d3f7SAndy Lutomirski else 307911d3f7SAndy Lutomirski cr4_clear_bits(X86_CR4_PCE); 317911d3f7SAndy Lutomirski } 327911d3f7SAndy Lutomirski #else 337911d3f7SAndy Lutomirski static inline void load_mm_cr4(struct mm_struct *mm) {} 347911d3f7SAndy Lutomirski #endif 357911d3f7SAndy Lutomirski 36a5b9e5a2SAndy Lutomirski #ifdef CONFIG_MODIFY_LDT_SYSCALL 37bb898558SAl Viro /* 3837868fe1SAndy Lutomirski * ldt_structs can be allocated, used, and freed, but they are never 3937868fe1SAndy Lutomirski * modified while live. 4037868fe1SAndy Lutomirski */ 4137868fe1SAndy Lutomirski struct ldt_struct { 4237868fe1SAndy Lutomirski /* 4337868fe1SAndy Lutomirski * Xen requires page-aligned LDTs with special permissions. This is 4437868fe1SAndy Lutomirski * needed to prevent us from installing evil descriptors such as 4537868fe1SAndy Lutomirski * call gates. On native, we could merge the ldt_struct and LDT 4637868fe1SAndy Lutomirski * allocations, but it's not worth trying to optimize. 4737868fe1SAndy Lutomirski */ 4837868fe1SAndy Lutomirski struct desc_struct *entries; 4937868fe1SAndy Lutomirski int size; 5037868fe1SAndy Lutomirski }; 5137868fe1SAndy Lutomirski 52a5b9e5a2SAndy Lutomirski /* 53a5b9e5a2SAndy Lutomirski * Used for LDT copy/destruction. 54a5b9e5a2SAndy Lutomirski */ 5539a0526fSDave Hansen int init_new_context_ldt(struct task_struct *tsk, struct mm_struct *mm); 5639a0526fSDave Hansen void destroy_context_ldt(struct mm_struct *mm); 57a5b9e5a2SAndy Lutomirski #else /* CONFIG_MODIFY_LDT_SYSCALL */ 5839a0526fSDave Hansen static inline int init_new_context_ldt(struct task_struct *tsk, 59a5b9e5a2SAndy Lutomirski struct mm_struct *mm) 60a5b9e5a2SAndy Lutomirski { 61a5b9e5a2SAndy Lutomirski return 0; 62a5b9e5a2SAndy Lutomirski } 6339a0526fSDave Hansen static inline void destroy_context_ldt(struct mm_struct *mm) {} 64a5b9e5a2SAndy Lutomirski #endif 65a5b9e5a2SAndy Lutomirski 6637868fe1SAndy Lutomirski static inline void load_mm_ldt(struct mm_struct *mm) 6737868fe1SAndy Lutomirski { 68a5b9e5a2SAndy Lutomirski #ifdef CONFIG_MODIFY_LDT_SYSCALL 6937868fe1SAndy Lutomirski struct ldt_struct *ldt; 7037868fe1SAndy Lutomirski 7137868fe1SAndy Lutomirski /* lockless_dereference synchronizes with smp_store_release */ 7237868fe1SAndy Lutomirski ldt = lockless_dereference(mm->context.ldt); 7337868fe1SAndy Lutomirski 7437868fe1SAndy Lutomirski /* 7537868fe1SAndy Lutomirski * Any change to mm->context.ldt is followed by an IPI to all 7637868fe1SAndy Lutomirski * CPUs with the mm active. The LDT will not be freed until 7737868fe1SAndy Lutomirski * after the IPI is handled by all such CPUs. This means that, 7837868fe1SAndy Lutomirski * if the ldt_struct changes before we return, the values we see 7937868fe1SAndy Lutomirski * will be safe, and the new values will be loaded before we run 8037868fe1SAndy Lutomirski * any user code. 8137868fe1SAndy Lutomirski * 8237868fe1SAndy Lutomirski * NB: don't try to convert this to use RCU without extreme care. 8337868fe1SAndy Lutomirski * We would still need IRQs off, because we don't want to change 8437868fe1SAndy Lutomirski * the local LDT after an IPI loaded a newer value than the one 8537868fe1SAndy Lutomirski * that we can see. 8637868fe1SAndy Lutomirski */ 8737868fe1SAndy Lutomirski 8837868fe1SAndy Lutomirski if (unlikely(ldt)) 8937868fe1SAndy Lutomirski set_ldt(ldt->entries, ldt->size); 9037868fe1SAndy Lutomirski else 9137868fe1SAndy Lutomirski clear_LDT(); 92a5b9e5a2SAndy Lutomirski #else 93a5b9e5a2SAndy Lutomirski clear_LDT(); 94a5b9e5a2SAndy Lutomirski #endif 9537868fe1SAndy Lutomirski 9637868fe1SAndy Lutomirski DEBUG_LOCKS_WARN_ON(preemptible()); 9737868fe1SAndy Lutomirski } 9837868fe1SAndy Lutomirski 996826c8ffSBrian Gerst static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 1006826c8ffSBrian Gerst { 1016826c8ffSBrian Gerst #ifdef CONFIG_SMP 102c6ae41e7SAlex Shi if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) 103c6ae41e7SAlex Shi this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY); 104bb898558SAl Viro #endif 1056826c8ffSBrian Gerst } 1066826c8ffSBrian Gerst 10739a0526fSDave Hansen static inline int init_new_context(struct task_struct *tsk, 10839a0526fSDave Hansen struct mm_struct *mm) 10939a0526fSDave Hansen { 11039a0526fSDave Hansen init_new_context_ldt(tsk, mm); 11139a0526fSDave Hansen return 0; 11239a0526fSDave Hansen } 11339a0526fSDave Hansen static inline void destroy_context(struct mm_struct *mm) 11439a0526fSDave Hansen { 11539a0526fSDave Hansen destroy_context_ldt(mm); 11639a0526fSDave Hansen } 11739a0526fSDave Hansen 11869c0319aSAndy Lutomirski extern void switch_mm(struct mm_struct *prev, struct mm_struct *next, 11969c0319aSAndy Lutomirski struct task_struct *tsk); 1206826c8ffSBrian Gerst 121078194f8SAndy Lutomirski extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, 122078194f8SAndy Lutomirski struct task_struct *tsk); 123078194f8SAndy Lutomirski #define switch_mm_irqs_off switch_mm_irqs_off 124bb898558SAl Viro 125bb898558SAl Viro #define activate_mm(prev, next) \ 126bb898558SAl Viro do { \ 127bb898558SAl Viro paravirt_activate_mm((prev), (next)); \ 128bb898558SAl Viro switch_mm((prev), (next), NULL); \ 129bb898558SAl Viro } while (0); 130bb898558SAl Viro 1316826c8ffSBrian Gerst #ifdef CONFIG_X86_32 1326826c8ffSBrian Gerst #define deactivate_mm(tsk, mm) \ 1336826c8ffSBrian Gerst do { \ 134ccbeed3aSTejun Heo lazy_load_gs(0); \ 1356826c8ffSBrian Gerst } while (0) 1366826c8ffSBrian Gerst #else 1376826c8ffSBrian Gerst #define deactivate_mm(tsk, mm) \ 1386826c8ffSBrian Gerst do { \ 1396826c8ffSBrian Gerst load_gs_index(0); \ 1406826c8ffSBrian Gerst loadsegment(fs, 0); \ 1416826c8ffSBrian Gerst } while (0) 1426826c8ffSBrian Gerst #endif 143bb898558SAl Viro 144a1ea1c03SDave Hansen static inline void arch_dup_mmap(struct mm_struct *oldmm, 145a1ea1c03SDave Hansen struct mm_struct *mm) 146a1ea1c03SDave Hansen { 147a1ea1c03SDave Hansen paravirt_arch_dup_mmap(oldmm, mm); 148a1ea1c03SDave Hansen } 149a1ea1c03SDave Hansen 150a1ea1c03SDave Hansen static inline void arch_exit_mmap(struct mm_struct *mm) 151a1ea1c03SDave Hansen { 152a1ea1c03SDave Hansen paravirt_arch_exit_mmap(mm); 153a1ea1c03SDave Hansen } 154a1ea1c03SDave Hansen 155b0e9b09bSDave Hansen #ifdef CONFIG_X86_64 156b0e9b09bSDave Hansen static inline bool is_64bit_mm(struct mm_struct *mm) 157b0e9b09bSDave Hansen { 15897f2645fSMasahiro Yamada return !IS_ENABLED(CONFIG_IA32_EMULATION) || 159b0e9b09bSDave Hansen !(mm->context.ia32_compat == TIF_IA32); 160b0e9b09bSDave Hansen } 161b0e9b09bSDave Hansen #else 162b0e9b09bSDave Hansen static inline bool is_64bit_mm(struct mm_struct *mm) 163b0e9b09bSDave Hansen { 164b0e9b09bSDave Hansen return false; 165b0e9b09bSDave Hansen } 166b0e9b09bSDave Hansen #endif 167b0e9b09bSDave Hansen 168fe3d197fSDave Hansen static inline void arch_bprm_mm_init(struct mm_struct *mm, 169fe3d197fSDave Hansen struct vm_area_struct *vma) 170fe3d197fSDave Hansen { 171fe3d197fSDave Hansen mpx_mm_init(mm); 172fe3d197fSDave Hansen } 173fe3d197fSDave Hansen 1741de4fa14SDave Hansen static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma, 1751de4fa14SDave Hansen unsigned long start, unsigned long end) 1761de4fa14SDave Hansen { 177c922228eSDave Hansen /* 178c922228eSDave Hansen * mpx_notify_unmap() goes and reads a rarely-hot 179c922228eSDave Hansen * cacheline in the mm_struct. That can be expensive 180c922228eSDave Hansen * enough to be seen in profiles. 181c922228eSDave Hansen * 182c922228eSDave Hansen * The mpx_notify_unmap() call and its contents have been 183c922228eSDave Hansen * observed to affect munmap() performance on hardware 184c922228eSDave Hansen * where MPX is not present. 185c922228eSDave Hansen * 186c922228eSDave Hansen * The unlikely() optimizes for the fast case: no MPX 187c922228eSDave Hansen * in the CPU, or no MPX use in the process. Even if 188c922228eSDave Hansen * we get this wrong (in the unlikely event that MPX 189c922228eSDave Hansen * is widely enabled on some system) the overhead of 190c922228eSDave Hansen * MPX itself (reading bounds tables) is expected to 191c922228eSDave Hansen * overwhelm the overhead of getting this unlikely() 192c922228eSDave Hansen * consistently wrong. 193c922228eSDave Hansen */ 194c922228eSDave Hansen if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX))) 1951de4fa14SDave Hansen mpx_notify_unmap(mm, vma, start, end); 1961de4fa14SDave Hansen } 1971de4fa14SDave Hansen 1988f62c883SDave Hansen static inline int vma_pkey(struct vm_area_struct *vma) 1998f62c883SDave Hansen { 2008f62c883SDave Hansen u16 pkey = 0; 2018f62c883SDave Hansen #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS 2028f62c883SDave Hansen unsigned long vma_pkey_mask = VM_PKEY_BIT0 | VM_PKEY_BIT1 | 2038f62c883SDave Hansen VM_PKEY_BIT2 | VM_PKEY_BIT3; 2048f62c883SDave Hansen pkey = (vma->vm_flags & vma_pkey_mask) >> VM_PKEY_SHIFT; 2058f62c883SDave Hansen #endif 2068f62c883SDave Hansen return pkey; 2078f62c883SDave Hansen } 2088f62c883SDave Hansen 20933a709b2SDave Hansen static inline bool __pkru_allows_pkey(u16 pkey, bool write) 21033a709b2SDave Hansen { 21133a709b2SDave Hansen u32 pkru = read_pkru(); 21233a709b2SDave Hansen 21333a709b2SDave Hansen if (!__pkru_allows_read(pkru, pkey)) 21433a709b2SDave Hansen return false; 21533a709b2SDave Hansen if (write && !__pkru_allows_write(pkru, pkey)) 21633a709b2SDave Hansen return false; 21733a709b2SDave Hansen 21833a709b2SDave Hansen return true; 21933a709b2SDave Hansen } 22033a709b2SDave Hansen 22133a709b2SDave Hansen /* 22233a709b2SDave Hansen * We only want to enforce protection keys on the current process 22333a709b2SDave Hansen * because we effectively have no access to PKRU for other 22433a709b2SDave Hansen * processes or any way to tell *which * PKRU in a threaded 22533a709b2SDave Hansen * process we could use. 22633a709b2SDave Hansen * 22733a709b2SDave Hansen * So do not enforce things if the VMA is not from the current 22833a709b2SDave Hansen * mm, or if we are in a kernel thread. 22933a709b2SDave Hansen */ 23033a709b2SDave Hansen static inline bool vma_is_foreign(struct vm_area_struct *vma) 23133a709b2SDave Hansen { 23233a709b2SDave Hansen if (!current->mm) 23333a709b2SDave Hansen return true; 23433a709b2SDave Hansen /* 23533a709b2SDave Hansen * Should PKRU be enforced on the access to this VMA? If 23633a709b2SDave Hansen * the VMA is from another process, then PKRU has no 23733a709b2SDave Hansen * relevance and should not be enforced. 23833a709b2SDave Hansen */ 23933a709b2SDave Hansen if (current->mm != vma->vm_mm) 24033a709b2SDave Hansen return true; 24133a709b2SDave Hansen 24233a709b2SDave Hansen return false; 24333a709b2SDave Hansen } 24433a709b2SDave Hansen 2451b2ee126SDave Hansen static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, 246d61172b4SDave Hansen bool write, bool execute, bool foreign) 24733a709b2SDave Hansen { 248d61172b4SDave Hansen /* pkeys never affect instruction fetches */ 249d61172b4SDave Hansen if (execute) 250d61172b4SDave Hansen return true; 25133a709b2SDave Hansen /* allow access if the VMA is not one from this process */ 2521b2ee126SDave Hansen if (foreign || vma_is_foreign(vma)) 25333a709b2SDave Hansen return true; 25433a709b2SDave Hansen return __pkru_allows_pkey(vma_pkey(vma), write); 25533a709b2SDave Hansen } 25633a709b2SDave Hansen 25733a709b2SDave Hansen static inline bool arch_pte_access_permitted(pte_t pte, bool write) 25833a709b2SDave Hansen { 25933a709b2SDave Hansen return __pkru_allows_pkey(pte_flags_pkey(pte_flags(pte)), write); 26033a709b2SDave Hansen } 26133a709b2SDave Hansen 2621965aae3SH. Peter Anvin #endif /* _ASM_X86_MMU_CONTEXT_H */ 263