1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 21965aae3SH. Peter Anvin #ifndef _ASM_X86_MMU_CONTEXT_H 31965aae3SH. Peter Anvin #define _ASM_X86_MMU_CONTEXT_H 4bb898558SAl Viro 5bb898558SAl Viro #include <asm/desc.h> 660063497SArun Sharma #include <linux/atomic.h> 7d17d8f9dSDave Hansen #include <linux/mm_types.h> 87d06d9c9SDave Hansen #include <linux/pkeys.h> 9d17d8f9dSDave Hansen 10d17d8f9dSDave Hansen #include <trace/events/tlb.h> 11d17d8f9dSDave Hansen 12bb898558SAl Viro #include <asm/pgalloc.h> 13bb898558SAl Viro #include <asm/tlbflush.h> 14bb898558SAl Viro #include <asm/paravirt.h> 15d97080ebSNadav Amit #include <asm/debugreg.h> 16f39681edSAndy Lutomirski 17f39681edSAndy Lutomirski extern atomic64_t last_mm_ctx_id; 18f39681edSAndy Lutomirski 19fdc0269eSJuergen Gross #ifndef CONFIG_PARAVIRT_XXL 20bb898558SAl Viro static inline void paravirt_activate_mm(struct mm_struct *prev, 21bb898558SAl Viro struct mm_struct *next) 22bb898558SAl Viro { 23bb898558SAl Viro } 24fdc0269eSJuergen Gross #endif /* !CONFIG_PARAVIRT_XXL */ 25bb898558SAl Viro 267911d3f7SAndy Lutomirski #ifdef CONFIG_PERF_EVENTS 27405b4537SAnthony Steinhauser DECLARE_STATIC_KEY_FALSE(rdpmc_never_available_key); 28631fe154SDavidlohr Bueso DECLARE_STATIC_KEY_FALSE(rdpmc_always_available_key); 29cb2a0235SThomas Gleixner void cr4_update_pce(void *ignored); 307911d3f7SAndy Lutomirski #endif 317911d3f7SAndy Lutomirski 32a5b9e5a2SAndy Lutomirski #ifdef CONFIG_MODIFY_LDT_SYSCALL 33bb898558SAl Viro /* 3437868fe1SAndy Lutomirski * ldt_structs can be allocated, used, and freed, but they are never 3537868fe1SAndy Lutomirski * modified while live. 3637868fe1SAndy Lutomirski */ 3737868fe1SAndy Lutomirski struct ldt_struct { 3837868fe1SAndy Lutomirski /* 3937868fe1SAndy Lutomirski * Xen requires page-aligned LDTs with special permissions. This is 4037868fe1SAndy Lutomirski * needed to prevent us from installing evil descriptors such as 4137868fe1SAndy Lutomirski * call gates. On native, we could merge the ldt_struct and LDT 4237868fe1SAndy Lutomirski * allocations, but it's not worth trying to optimize. 4337868fe1SAndy Lutomirski */ 4437868fe1SAndy Lutomirski struct desc_struct *entries; 45bbf79d21SBorislav Petkov unsigned int nr_entries; 46f55f0501SAndy Lutomirski 47f55f0501SAndy Lutomirski /* 48f55f0501SAndy Lutomirski * If PTI is in use, then the entries array is not mapped while we're 49f55f0501SAndy Lutomirski * in user mode. The whole array will be aliased at the addressed 50f55f0501SAndy Lutomirski * given by ldt_slot_va(slot). We use two slots so that we can allocate 51f55f0501SAndy Lutomirski * and map, and enable a new LDT without invalidating the mapping 52f55f0501SAndy Lutomirski * of an older, still-in-use LDT. 53f55f0501SAndy Lutomirski * 54f55f0501SAndy Lutomirski * slot will be -1 if this LDT doesn't have an alias mapping. 55f55f0501SAndy Lutomirski */ 56f55f0501SAndy Lutomirski int slot; 5737868fe1SAndy Lutomirski }; 5837868fe1SAndy Lutomirski 59a5b9e5a2SAndy Lutomirski /* 60a5b9e5a2SAndy Lutomirski * Used for LDT copy/destruction. 61a5b9e5a2SAndy Lutomirski */ 62a4828f81SThomas Gleixner static inline void init_new_context_ldt(struct mm_struct *mm) 63a4828f81SThomas Gleixner { 64a4828f81SThomas Gleixner mm->context.ldt = NULL; 65a4828f81SThomas Gleixner init_rwsem(&mm->context.ldt_usr_sem); 66a4828f81SThomas Gleixner } 67a4828f81SThomas Gleixner int ldt_dup_context(struct mm_struct *oldmm, struct mm_struct *mm); 6839a0526fSDave Hansen void destroy_context_ldt(struct mm_struct *mm); 69f55f0501SAndy Lutomirski void ldt_arch_exit_mmap(struct mm_struct *mm); 70a5b9e5a2SAndy Lutomirski #else /* CONFIG_MODIFY_LDT_SYSCALL */ 71a4828f81SThomas Gleixner static inline void init_new_context_ldt(struct mm_struct *mm) { } 72a4828f81SThomas Gleixner static inline int ldt_dup_context(struct mm_struct *oldmm, 73a5b9e5a2SAndy Lutomirski struct mm_struct *mm) 74a5b9e5a2SAndy Lutomirski { 75a5b9e5a2SAndy Lutomirski return 0; 76a5b9e5a2SAndy Lutomirski } 7739a0526fSDave Hansen static inline void destroy_context_ldt(struct mm_struct *mm) { } 78f55f0501SAndy Lutomirski static inline void ldt_arch_exit_mmap(struct mm_struct *mm) { } 79a5b9e5a2SAndy Lutomirski #endif 80a5b9e5a2SAndy Lutomirski 81186525bdSIngo Molnar #ifdef CONFIG_MODIFY_LDT_SYSCALL 82186525bdSIngo Molnar extern void load_mm_ldt(struct mm_struct *mm); 83186525bdSIngo Molnar extern void switch_ldt(struct mm_struct *prev, struct mm_struct *next); 84186525bdSIngo Molnar #else 8537868fe1SAndy Lutomirski static inline void load_mm_ldt(struct mm_struct *mm) 8637868fe1SAndy Lutomirski { 87f55f0501SAndy Lutomirski clear_LDT(); 88f55f0501SAndy Lutomirski } 8973534258SAndy Lutomirski static inline void switch_ldt(struct mm_struct *prev, struct mm_struct *next) 9073534258SAndy Lutomirski { 9137868fe1SAndy Lutomirski DEBUG_LOCKS_WARN_ON(preemptible()); 9237868fe1SAndy Lutomirski } 93186525bdSIngo Molnar #endif 9437868fe1SAndy Lutomirski 95186525bdSIngo Molnar extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk); 966826c8ffSBrian Gerst 97a31e184eSDave Hansen /* 98a31e184eSDave Hansen * Init a new mm. Used on mm copies, like at fork() 99a31e184eSDave Hansen * and on mm's that are brand-new, like at execve(). 100a31e184eSDave Hansen */ 10139a0526fSDave Hansen static inline int init_new_context(struct task_struct *tsk, 10239a0526fSDave Hansen struct mm_struct *mm) 10339a0526fSDave Hansen { 104c2b3496bSPeter Zijlstra mutex_init(&mm->context.lock); 105c2b3496bSPeter Zijlstra 106f39681edSAndy Lutomirski mm->context.ctx_id = atomic64_inc_return(&last_mm_ctx_id); 107f39681edSAndy Lutomirski atomic64_set(&mm->context.tlb_gen, 0); 108f39681edSAndy Lutomirski 109e8c24d3aSDave Hansen #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS 110e8c24d3aSDave Hansen if (cpu_feature_enabled(X86_FEATURE_OSPKE)) { 1112fa9d1cfSDave Hansen /* pkey 0 is the default and allocated implicitly */ 112e8c24d3aSDave Hansen mm->context.pkey_allocation_map = 0x1; 113e8c24d3aSDave Hansen /* -1 means unallocated or invalid */ 114e8c24d3aSDave Hansen mm->context.execute_only_pkey = -1; 115e8c24d3aSDave Hansen } 116e8c24d3aSDave Hansen #endif 117a4828f81SThomas Gleixner init_new_context_ldt(mm); 118a4828f81SThomas Gleixner return 0; 11939a0526fSDave Hansen } 12039a0526fSDave Hansen static inline void destroy_context(struct mm_struct *mm) 12139a0526fSDave Hansen { 12239a0526fSDave Hansen destroy_context_ldt(mm); 12339a0526fSDave Hansen } 12439a0526fSDave Hansen 12569c0319aSAndy Lutomirski extern void switch_mm(struct mm_struct *prev, struct mm_struct *next, 12669c0319aSAndy Lutomirski struct task_struct *tsk); 1276826c8ffSBrian Gerst 128078194f8SAndy Lutomirski extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, 129078194f8SAndy Lutomirski struct task_struct *tsk); 130078194f8SAndy Lutomirski #define switch_mm_irqs_off switch_mm_irqs_off 131bb898558SAl Viro 132bb898558SAl Viro #define activate_mm(prev, next) \ 133bb898558SAl Viro do { \ 134bb898558SAl Viro paravirt_activate_mm((prev), (next)); \ 135bb898558SAl Viro switch_mm((prev), (next), NULL); \ 136bb898558SAl Viro } while (0); 137bb898558SAl Viro 1386826c8ffSBrian Gerst #ifdef CONFIG_X86_32 1396826c8ffSBrian Gerst #define deactivate_mm(tsk, mm) \ 1406826c8ffSBrian Gerst do { \ 141ccbeed3aSTejun Heo lazy_load_gs(0); \ 1426826c8ffSBrian Gerst } while (0) 1436826c8ffSBrian Gerst #else 1446826c8ffSBrian Gerst #define deactivate_mm(tsk, mm) \ 1456826c8ffSBrian Gerst do { \ 1466826c8ffSBrian Gerst load_gs_index(0); \ 1476826c8ffSBrian Gerst loadsegment(fs, 0); \ 1486826c8ffSBrian Gerst } while (0) 1496826c8ffSBrian Gerst #endif 150bb898558SAl Viro 151a31e184eSDave Hansen static inline void arch_dup_pkeys(struct mm_struct *oldmm, 152a31e184eSDave Hansen struct mm_struct *mm) 153a31e184eSDave Hansen { 154a31e184eSDave Hansen #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS 155a31e184eSDave Hansen if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) 156a31e184eSDave Hansen return; 157a31e184eSDave Hansen 158a31e184eSDave Hansen /* Duplicate the oldmm pkey state in mm: */ 159a31e184eSDave Hansen mm->context.pkey_allocation_map = oldmm->context.pkey_allocation_map; 160a31e184eSDave Hansen mm->context.execute_only_pkey = oldmm->context.execute_only_pkey; 161a31e184eSDave Hansen #endif 162a31e184eSDave Hansen } 163a31e184eSDave Hansen 164c10e83f5SThomas Gleixner static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) 165a1ea1c03SDave Hansen { 166a31e184eSDave Hansen arch_dup_pkeys(oldmm, mm); 167a1ea1c03SDave Hansen paravirt_arch_dup_mmap(oldmm, mm); 168a4828f81SThomas Gleixner return ldt_dup_context(oldmm, mm); 169a1ea1c03SDave Hansen } 170a1ea1c03SDave Hansen 171a1ea1c03SDave Hansen static inline void arch_exit_mmap(struct mm_struct *mm) 172a1ea1c03SDave Hansen { 173a1ea1c03SDave Hansen paravirt_arch_exit_mmap(mm); 174f55f0501SAndy Lutomirski ldt_arch_exit_mmap(mm); 175a1ea1c03SDave Hansen } 176a1ea1c03SDave Hansen 177b0e9b09bSDave Hansen #ifdef CONFIG_X86_64 178b0e9b09bSDave Hansen static inline bool is_64bit_mm(struct mm_struct *mm) 179b0e9b09bSDave Hansen { 18097f2645fSMasahiro Yamada return !IS_ENABLED(CONFIG_IA32_EMULATION) || 181b0e9b09bSDave Hansen !(mm->context.ia32_compat == TIF_IA32); 182b0e9b09bSDave Hansen } 183b0e9b09bSDave Hansen #else 184b0e9b09bSDave Hansen static inline bool is_64bit_mm(struct mm_struct *mm) 185b0e9b09bSDave Hansen { 186b0e9b09bSDave Hansen return false; 187b0e9b09bSDave Hansen } 188b0e9b09bSDave Hansen #endif 189b0e9b09bSDave Hansen 1905a28fc94SDave Hansen static inline void arch_unmap(struct mm_struct *mm, unsigned long start, 1915a28fc94SDave Hansen unsigned long end) 1921de4fa14SDave Hansen { 1931de4fa14SDave Hansen } 1941de4fa14SDave Hansen 19533a709b2SDave Hansen /* 19633a709b2SDave Hansen * We only want to enforce protection keys on the current process 19733a709b2SDave Hansen * because we effectively have no access to PKRU for other 19833a709b2SDave Hansen * processes or any way to tell *which * PKRU in a threaded 19933a709b2SDave Hansen * process we could use. 20033a709b2SDave Hansen * 20133a709b2SDave Hansen * So do not enforce things if the VMA is not from the current 20233a709b2SDave Hansen * mm, or if we are in a kernel thread. 20333a709b2SDave Hansen */ 2041b2ee126SDave Hansen static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, 205d61172b4SDave Hansen bool write, bool execute, bool foreign) 20633a709b2SDave Hansen { 207d61172b4SDave Hansen /* pkeys never affect instruction fetches */ 208d61172b4SDave Hansen if (execute) 209d61172b4SDave Hansen return true; 21033a709b2SDave Hansen /* allow access if the VMA is not one from this process */ 2111b2ee126SDave Hansen if (foreign || vma_is_foreign(vma)) 21233a709b2SDave Hansen return true; 21333a709b2SDave Hansen return __pkru_allows_pkey(vma_pkey(vma), write); 21433a709b2SDave Hansen } 21533a709b2SDave Hansen 2168c5cc19eSThomas Gleixner unsigned long __get_current_cr3_fast(void); 217d6e41f11SAndy Lutomirski 218cefa929cSAndy Lutomirski typedef struct { 219cefa929cSAndy Lutomirski struct mm_struct *mm; 220cefa929cSAndy Lutomirski } temp_mm_state_t; 221cefa929cSAndy Lutomirski 222cefa929cSAndy Lutomirski /* 223cefa929cSAndy Lutomirski * Using a temporary mm allows to set temporary mappings that are not accessible 224cefa929cSAndy Lutomirski * by other CPUs. Such mappings are needed to perform sensitive memory writes 225cefa929cSAndy Lutomirski * that override the kernel memory protections (e.g., W^X), without exposing the 226cefa929cSAndy Lutomirski * temporary page-table mappings that are required for these write operations to 227cefa929cSAndy Lutomirski * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the 228cefa929cSAndy Lutomirski * mapping is torn down. 229cefa929cSAndy Lutomirski * 230cefa929cSAndy Lutomirski * Context: The temporary mm needs to be used exclusively by a single core. To 231cefa929cSAndy Lutomirski * harden security IRQs must be disabled while the temporary mm is 232cefa929cSAndy Lutomirski * loaded, thereby preventing interrupt handler bugs from overriding 233cefa929cSAndy Lutomirski * the kernel memory protection. 234cefa929cSAndy Lutomirski */ 235cefa929cSAndy Lutomirski static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm) 236cefa929cSAndy Lutomirski { 237cefa929cSAndy Lutomirski temp_mm_state_t temp_state; 238cefa929cSAndy Lutomirski 239cefa929cSAndy Lutomirski lockdep_assert_irqs_disabled(); 240cefa929cSAndy Lutomirski temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm); 241cefa929cSAndy Lutomirski switch_mm_irqs_off(NULL, mm, current); 242d97080ebSNadav Amit 243d97080ebSNadav Amit /* 244d97080ebSNadav Amit * If breakpoints are enabled, disable them while the temporary mm is 245d97080ebSNadav Amit * used. Userspace might set up watchpoints on addresses that are used 246d97080ebSNadav Amit * in the temporary mm, which would lead to wrong signals being sent or 247d97080ebSNadav Amit * crashes. 248d97080ebSNadav Amit * 249d97080ebSNadav Amit * Note that breakpoints are not disabled selectively, which also causes 250d97080ebSNadav Amit * kernel breakpoints (e.g., perf's) to be disabled. This might be 251d97080ebSNadav Amit * undesirable, but still seems reasonable as the code that runs in the 252d97080ebSNadav Amit * temporary mm should be short. 253d97080ebSNadav Amit */ 254d97080ebSNadav Amit if (hw_breakpoint_active()) 255d97080ebSNadav Amit hw_breakpoint_disable(); 256d97080ebSNadav Amit 257cefa929cSAndy Lutomirski return temp_state; 258cefa929cSAndy Lutomirski } 259cefa929cSAndy Lutomirski 260cefa929cSAndy Lutomirski static inline void unuse_temporary_mm(temp_mm_state_t prev_state) 261cefa929cSAndy Lutomirski { 262cefa929cSAndy Lutomirski lockdep_assert_irqs_disabled(); 263cefa929cSAndy Lutomirski switch_mm_irqs_off(NULL, prev_state.mm, current); 264d97080ebSNadav Amit 265d97080ebSNadav Amit /* 266d97080ebSNadav Amit * Restore the breakpoints if they were disabled before the temporary mm 267d97080ebSNadav Amit * was loaded. 268d97080ebSNadav Amit */ 269d97080ebSNadav Amit if (hw_breakpoint_active()) 270d97080ebSNadav Amit hw_breakpoint_restore(); 271cefa929cSAndy Lutomirski } 272cefa929cSAndy Lutomirski 2731965aae3SH. Peter Anvin #endif /* _ASM_X86_MMU_CONTEXT_H */ 274