1 #ifndef _ASM_X86_MMU_CONTEXT_H 2 #define _ASM_X86_MMU_CONTEXT_H 3 4 #include <asm/desc.h> 5 #include <linux/atomic.h> 6 #include <linux/mm_types.h> 7 8 #include <trace/events/tlb.h> 9 10 #include <asm/pgalloc.h> 11 #include <asm/tlbflush.h> 12 #include <asm/paravirt.h> 13 #include <asm/mpx.h> 14 #ifndef CONFIG_PARAVIRT 15 static inline void paravirt_activate_mm(struct mm_struct *prev, 16 struct mm_struct *next) 17 { 18 } 19 #endif /* !CONFIG_PARAVIRT */ 20 21 #ifdef CONFIG_PERF_EVENTS 22 extern struct static_key rdpmc_always_available; 23 24 static inline void load_mm_cr4(struct mm_struct *mm) 25 { 26 if (static_key_false(&rdpmc_always_available) || 27 atomic_read(&mm->context.perf_rdpmc_allowed)) 28 cr4_set_bits(X86_CR4_PCE); 29 else 30 cr4_clear_bits(X86_CR4_PCE); 31 } 32 #else 33 static inline void load_mm_cr4(struct mm_struct *mm) {} 34 #endif 35 36 /* 37 * ldt_structs can be allocated, used, and freed, but they are never 38 * modified while live. 39 */ 40 struct ldt_struct { 41 /* 42 * Xen requires page-aligned LDTs with special permissions. This is 43 * needed to prevent us from installing evil descriptors such as 44 * call gates. On native, we could merge the ldt_struct and LDT 45 * allocations, but it's not worth trying to optimize. 46 */ 47 struct desc_struct *entries; 48 int size; 49 }; 50 51 static inline void load_mm_ldt(struct mm_struct *mm) 52 { 53 struct ldt_struct *ldt; 54 55 /* lockless_dereference synchronizes with smp_store_release */ 56 ldt = lockless_dereference(mm->context.ldt); 57 58 /* 59 * Any change to mm->context.ldt is followed by an IPI to all 60 * CPUs with the mm active. The LDT will not be freed until 61 * after the IPI is handled by all such CPUs. This means that, 62 * if the ldt_struct changes before we return, the values we see 63 * will be safe, and the new values will be loaded before we run 64 * any user code. 65 * 66 * NB: don't try to convert this to use RCU without extreme care. 67 * We would still need IRQs off, because we don't want to change 68 * the local LDT after an IPI loaded a newer value than the one 69 * that we can see. 70 */ 71 72 if (unlikely(ldt)) 73 set_ldt(ldt->entries, ldt->size); 74 else 75 clear_LDT(); 76 77 DEBUG_LOCKS_WARN_ON(preemptible()); 78 } 79 80 /* 81 * Used for LDT copy/destruction. 82 */ 83 int init_new_context(struct task_struct *tsk, struct mm_struct *mm); 84 void destroy_context(struct mm_struct *mm); 85 86 87 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) 88 { 89 #ifdef CONFIG_SMP 90 if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) 91 this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY); 92 #endif 93 } 94 95 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, 96 struct task_struct *tsk) 97 { 98 unsigned cpu = smp_processor_id(); 99 100 if (likely(prev != next)) { 101 #ifdef CONFIG_SMP 102 this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK); 103 this_cpu_write(cpu_tlbstate.active_mm, next); 104 #endif 105 cpumask_set_cpu(cpu, mm_cpumask(next)); 106 107 /* Re-load page tables */ 108 load_cr3(next->pgd); 109 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); 110 111 /* Stop flush ipis for the previous mm */ 112 cpumask_clear_cpu(cpu, mm_cpumask(prev)); 113 114 /* Load per-mm CR4 state */ 115 load_mm_cr4(next); 116 117 /* 118 * Load the LDT, if the LDT is different. 119 * 120 * It's possible that prev->context.ldt doesn't match 121 * the LDT register. This can happen if leave_mm(prev) 122 * was called and then modify_ldt changed 123 * prev->context.ldt but suppressed an IPI to this CPU. 124 * In this case, prev->context.ldt != NULL, because we 125 * never set context.ldt to NULL while the mm still 126 * exists. That means that next->context.ldt != 127 * prev->context.ldt, because mms never share an LDT. 128 */ 129 if (unlikely(prev->context.ldt != next->context.ldt)) 130 load_mm_ldt(next); 131 } 132 #ifdef CONFIG_SMP 133 else { 134 this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK); 135 BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next); 136 137 if (!cpumask_test_cpu(cpu, mm_cpumask(next))) { 138 /* 139 * On established mms, the mm_cpumask is only changed 140 * from irq context, from ptep_clear_flush() while in 141 * lazy tlb mode, and here. Irqs are blocked during 142 * schedule, protecting us from simultaneous changes. 143 */ 144 cpumask_set_cpu(cpu, mm_cpumask(next)); 145 /* 146 * We were in lazy tlb mode and leave_mm disabled 147 * tlb flush IPI delivery. We must reload CR3 148 * to make sure to use no freed page tables. 149 */ 150 load_cr3(next->pgd); 151 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); 152 load_mm_cr4(next); 153 load_mm_ldt(next); 154 } 155 } 156 #endif 157 } 158 159 #define activate_mm(prev, next) \ 160 do { \ 161 paravirt_activate_mm((prev), (next)); \ 162 switch_mm((prev), (next), NULL); \ 163 } while (0); 164 165 #ifdef CONFIG_X86_32 166 #define deactivate_mm(tsk, mm) \ 167 do { \ 168 lazy_load_gs(0); \ 169 } while (0) 170 #else 171 #define deactivate_mm(tsk, mm) \ 172 do { \ 173 load_gs_index(0); \ 174 loadsegment(fs, 0); \ 175 } while (0) 176 #endif 177 178 static inline void arch_dup_mmap(struct mm_struct *oldmm, 179 struct mm_struct *mm) 180 { 181 paravirt_arch_dup_mmap(oldmm, mm); 182 } 183 184 static inline void arch_exit_mmap(struct mm_struct *mm) 185 { 186 paravirt_arch_exit_mmap(mm); 187 } 188 189 #ifdef CONFIG_X86_64 190 static inline bool is_64bit_mm(struct mm_struct *mm) 191 { 192 return !config_enabled(CONFIG_IA32_EMULATION) || 193 !(mm->context.ia32_compat == TIF_IA32); 194 } 195 #else 196 static inline bool is_64bit_mm(struct mm_struct *mm) 197 { 198 return false; 199 } 200 #endif 201 202 static inline void arch_bprm_mm_init(struct mm_struct *mm, 203 struct vm_area_struct *vma) 204 { 205 mpx_mm_init(mm); 206 } 207 208 static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma, 209 unsigned long start, unsigned long end) 210 { 211 /* 212 * mpx_notify_unmap() goes and reads a rarely-hot 213 * cacheline in the mm_struct. That can be expensive 214 * enough to be seen in profiles. 215 * 216 * The mpx_notify_unmap() call and its contents have been 217 * observed to affect munmap() performance on hardware 218 * where MPX is not present. 219 * 220 * The unlikely() optimizes for the fast case: no MPX 221 * in the CPU, or no MPX use in the process. Even if 222 * we get this wrong (in the unlikely event that MPX 223 * is widely enabled on some system) the overhead of 224 * MPX itself (reading bounds tables) is expected to 225 * overwhelm the overhead of getting this unlikely() 226 * consistently wrong. 227 */ 228 if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX))) 229 mpx_notify_unmap(mm, vma, start, end); 230 } 231 232 #endif /* _ASM_X86_MMU_CONTEXT_H */ 233