11965aae3SH. Peter Anvin #ifndef _ASM_X86_MMU_CONTEXT_H 21965aae3SH. Peter Anvin #define _ASM_X86_MMU_CONTEXT_H 3bb898558SAl Viro 4bb898558SAl Viro #include <asm/desc.h> 560063497SArun Sharma #include <linux/atomic.h> 6d17d8f9dSDave Hansen #include <linux/mm_types.h> 7d17d8f9dSDave Hansen 8d17d8f9dSDave Hansen #include <trace/events/tlb.h> 9d17d8f9dSDave Hansen 10bb898558SAl Viro #include <asm/pgalloc.h> 11bb898558SAl Viro #include <asm/tlbflush.h> 12bb898558SAl Viro #include <asm/paravirt.h> 13fe3d197fSDave Hansen #include <asm/mpx.h> 14bb898558SAl Viro #ifndef CONFIG_PARAVIRT 15bb898558SAl Viro static inline void paravirt_activate_mm(struct mm_struct *prev, 16bb898558SAl Viro struct mm_struct *next) 17bb898558SAl Viro { 18bb898558SAl Viro } 19bb898558SAl Viro #endif /* !CONFIG_PARAVIRT */ 20bb898558SAl Viro 217911d3f7SAndy Lutomirski #ifdef CONFIG_PERF_EVENTS 22a6673429SAndy Lutomirski extern struct static_key rdpmc_always_available; 23a6673429SAndy Lutomirski 247911d3f7SAndy Lutomirski static inline void load_mm_cr4(struct mm_struct *mm) 257911d3f7SAndy Lutomirski { 26a833581eSPeter Zijlstra if (static_key_false(&rdpmc_always_available) || 27a6673429SAndy Lutomirski atomic_read(&mm->context.perf_rdpmc_allowed)) 287911d3f7SAndy Lutomirski cr4_set_bits(X86_CR4_PCE); 297911d3f7SAndy Lutomirski else 307911d3f7SAndy Lutomirski cr4_clear_bits(X86_CR4_PCE); 317911d3f7SAndy Lutomirski } 327911d3f7SAndy Lutomirski #else 337911d3f7SAndy Lutomirski static inline void load_mm_cr4(struct mm_struct *mm) {} 347911d3f7SAndy Lutomirski #endif 357911d3f7SAndy Lutomirski 36bb898558SAl Viro /* 37bb898558SAl Viro * Used for LDT copy/destruction. 38bb898558SAl Viro */ 39bb898558SAl Viro int init_new_context(struct task_struct *tsk, struct mm_struct *mm); 40bb898558SAl Viro void destroy_context(struct mm_struct *mm); 41bb898558SAl Viro 426826c8ffSBrian Gerst 436826c8ffSBrian Gerst static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 446826c8ffSBrian Gerst { 456826c8ffSBrian Gerst #ifdef CONFIG_SMP 46c6ae41e7SAlex Shi if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) 47c6ae41e7SAlex Shi this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY); 48bb898558SAl Viro #endif 496826c8ffSBrian Gerst } 506826c8ffSBrian Gerst 516826c8ffSBrian Gerst static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, 526826c8ffSBrian Gerst struct task_struct *tsk) 536826c8ffSBrian Gerst { 546826c8ffSBrian Gerst unsigned cpu = smp_processor_id(); 556826c8ffSBrian Gerst 566826c8ffSBrian Gerst if (likely(prev != next)) { 576826c8ffSBrian Gerst #ifdef CONFIG_SMP 58c6ae41e7SAlex Shi this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK); 59c6ae41e7SAlex Shi this_cpu_write(cpu_tlbstate.active_mm, next); 606826c8ffSBrian Gerst #endif 6178f1c4d6SRusty Russell cpumask_set_cpu(cpu, mm_cpumask(next)); 626826c8ffSBrian Gerst 636826c8ffSBrian Gerst /* Re-load page tables */ 646826c8ffSBrian Gerst load_cr3(next->pgd); 65d17d8f9dSDave Hansen trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); 666826c8ffSBrian Gerst 678f898fbbSRik van Riel /* Stop flush ipis for the previous mm */ 68831d52bcSSuresh Siddha cpumask_clear_cpu(cpu, mm_cpumask(prev)); 69831d52bcSSuresh Siddha 707911d3f7SAndy Lutomirski /* Load per-mm CR4 state */ 717911d3f7SAndy Lutomirski load_mm_cr4(next); 727911d3f7SAndy Lutomirski 73c4a7bba2SAndy Lutomirski /* 74c4a7bba2SAndy Lutomirski * Load the LDT, if the LDT is different. 75c4a7bba2SAndy Lutomirski * 7622c4bd9fSAndy Lutomirski * It's possible that prev->context.ldt doesn't match 7722c4bd9fSAndy Lutomirski * the LDT register. This can happen if leave_mm(prev) 7822c4bd9fSAndy Lutomirski * was called and then modify_ldt changed 7922c4bd9fSAndy Lutomirski * prev->context.ldt but suppressed an IPI to this CPU. 8022c4bd9fSAndy Lutomirski * In this case, prev->context.ldt != NULL, because we 8122c4bd9fSAndy Lutomirski * never free an LDT while the mm still exists. That 8222c4bd9fSAndy Lutomirski * means that next->context.ldt != prev->context.ldt, 8322c4bd9fSAndy Lutomirski * because mms never share an LDT. 84c4a7bba2SAndy Lutomirski */ 856826c8ffSBrian Gerst if (unlikely(prev->context.ldt != next->context.ldt)) 866826c8ffSBrian Gerst load_LDT_nolock(&next->context); 876826c8ffSBrian Gerst } 886826c8ffSBrian Gerst #ifdef CONFIG_SMP 896826c8ffSBrian Gerst else { 90c6ae41e7SAlex Shi this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK); 91c6ae41e7SAlex Shi BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next); 926826c8ffSBrian Gerst 938f898fbbSRik van Riel if (!cpumask_test_cpu(cpu, mm_cpumask(next))) { 948f898fbbSRik van Riel /* 958f898fbbSRik van Riel * On established mms, the mm_cpumask is only changed 968f898fbbSRik van Riel * from irq context, from ptep_clear_flush() while in 978f898fbbSRik van Riel * lazy tlb mode, and here. Irqs are blocked during 988f898fbbSRik van Riel * schedule, protecting us from simultaneous changes. 998f898fbbSRik van Riel */ 1008f898fbbSRik van Riel cpumask_set_cpu(cpu, mm_cpumask(next)); 1018f898fbbSRik van Riel /* 1028f898fbbSRik van Riel * We were in lazy tlb mode and leave_mm disabled 1036826c8ffSBrian Gerst * tlb flush IPI delivery. We must reload CR3 1046826c8ffSBrian Gerst * to make sure to use no freed page tables. 1056826c8ffSBrian Gerst */ 1066826c8ffSBrian Gerst load_cr3(next->pgd); 107d17d8f9dSDave Hansen trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); 1087911d3f7SAndy Lutomirski load_mm_cr4(next); 1096826c8ffSBrian Gerst load_LDT_nolock(&next->context); 1106826c8ffSBrian Gerst } 1116826c8ffSBrian Gerst } 1126826c8ffSBrian Gerst #endif 1136826c8ffSBrian Gerst } 114bb898558SAl Viro 115bb898558SAl Viro #define activate_mm(prev, next) \ 116bb898558SAl Viro do { \ 117bb898558SAl Viro paravirt_activate_mm((prev), (next)); \ 118bb898558SAl Viro switch_mm((prev), (next), NULL); \ 119bb898558SAl Viro } while (0); 120bb898558SAl Viro 1216826c8ffSBrian Gerst #ifdef CONFIG_X86_32 1226826c8ffSBrian Gerst #define deactivate_mm(tsk, mm) \ 1236826c8ffSBrian Gerst do { \ 124ccbeed3aSTejun Heo lazy_load_gs(0); \ 1256826c8ffSBrian Gerst } while (0) 1266826c8ffSBrian Gerst #else 1276826c8ffSBrian Gerst #define deactivate_mm(tsk, mm) \ 1286826c8ffSBrian Gerst do { \ 1296826c8ffSBrian Gerst load_gs_index(0); \ 1306826c8ffSBrian Gerst loadsegment(fs, 0); \ 1316826c8ffSBrian Gerst } while (0) 1326826c8ffSBrian Gerst #endif 133bb898558SAl Viro 134a1ea1c03SDave Hansen static inline void arch_dup_mmap(struct mm_struct *oldmm, 135a1ea1c03SDave Hansen struct mm_struct *mm) 136a1ea1c03SDave Hansen { 137a1ea1c03SDave Hansen paravirt_arch_dup_mmap(oldmm, mm); 138a1ea1c03SDave Hansen } 139a1ea1c03SDave Hansen 140a1ea1c03SDave Hansen static inline void arch_exit_mmap(struct mm_struct *mm) 141a1ea1c03SDave Hansen { 142a1ea1c03SDave Hansen paravirt_arch_exit_mmap(mm); 143a1ea1c03SDave Hansen } 144a1ea1c03SDave Hansen 145b0e9b09bSDave Hansen #ifdef CONFIG_X86_64 146b0e9b09bSDave Hansen static inline bool is_64bit_mm(struct mm_struct *mm) 147b0e9b09bSDave Hansen { 148b0e9b09bSDave Hansen return !config_enabled(CONFIG_IA32_EMULATION) || 149b0e9b09bSDave Hansen !(mm->context.ia32_compat == TIF_IA32); 150b0e9b09bSDave Hansen } 151b0e9b09bSDave Hansen #else 152b0e9b09bSDave Hansen static inline bool is_64bit_mm(struct mm_struct *mm) 153b0e9b09bSDave Hansen { 154b0e9b09bSDave Hansen return false; 155b0e9b09bSDave Hansen } 156b0e9b09bSDave Hansen #endif 157b0e9b09bSDave Hansen 158fe3d197fSDave Hansen static inline void arch_bprm_mm_init(struct mm_struct *mm, 159fe3d197fSDave Hansen struct vm_area_struct *vma) 160fe3d197fSDave Hansen { 161fe3d197fSDave Hansen mpx_mm_init(mm); 162fe3d197fSDave Hansen } 163fe3d197fSDave Hansen 1641de4fa14SDave Hansen static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma, 1651de4fa14SDave Hansen unsigned long start, unsigned long end) 1661de4fa14SDave Hansen { 167c922228eSDave Hansen /* 168c922228eSDave Hansen * mpx_notify_unmap() goes and reads a rarely-hot 169c922228eSDave Hansen * cacheline in the mm_struct. That can be expensive 170c922228eSDave Hansen * enough to be seen in profiles. 171c922228eSDave Hansen * 172c922228eSDave Hansen * The mpx_notify_unmap() call and its contents have been 173c922228eSDave Hansen * observed to affect munmap() performance on hardware 174c922228eSDave Hansen * where MPX is not present. 175c922228eSDave Hansen * 176c922228eSDave Hansen * The unlikely() optimizes for the fast case: no MPX 177c922228eSDave Hansen * in the CPU, or no MPX use in the process. Even if 178c922228eSDave Hansen * we get this wrong (in the unlikely event that MPX 179c922228eSDave Hansen * is widely enabled on some system) the overhead of 180c922228eSDave Hansen * MPX itself (reading bounds tables) is expected to 181c922228eSDave Hansen * overwhelm the overhead of getting this unlikely() 182c922228eSDave Hansen * consistently wrong. 183c922228eSDave Hansen */ 184c922228eSDave Hansen if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX))) 1851de4fa14SDave Hansen mpx_notify_unmap(mm, vma, start, end); 1861de4fa14SDave Hansen } 1871de4fa14SDave Hansen 1881965aae3SH. Peter Anvin #endif /* _ASM_X86_MMU_CONTEXT_H */ 189