1 #ifndef _ASM_X86_TLBFLUSH_H 2 #define _ASM_X86_TLBFLUSH_H 3 4 #include <linux/mm.h> 5 #include <linux/sched.h> 6 7 #include <asm/processor.h> 8 #include <asm/cpufeature.h> 9 #include <asm/special_insns.h> 10 #include <asm/smp.h> 11 12 static inline void __invpcid(unsigned long pcid, unsigned long addr, 13 unsigned long type) 14 { 15 struct { u64 d[2]; } desc = { { pcid, addr } }; 16 17 /* 18 * The memory clobber is because the whole point is to invalidate 19 * stale TLB entries and, especially if we're flushing global 20 * mappings, we don't want the compiler to reorder any subsequent 21 * memory accesses before the TLB flush. 22 * 23 * The hex opcode is invpcid (%ecx), %eax in 32-bit mode and 24 * invpcid (%rcx), %rax in long mode. 25 */ 26 asm volatile (".byte 0x66, 0x0f, 0x38, 0x82, 0x01" 27 : : "m" (desc), "a" (type), "c" (&desc) : "memory"); 28 } 29 30 #define INVPCID_TYPE_INDIV_ADDR 0 31 #define INVPCID_TYPE_SINGLE_CTXT 1 32 #define INVPCID_TYPE_ALL_INCL_GLOBAL 2 33 #define INVPCID_TYPE_ALL_NON_GLOBAL 3 34 35 /* Flush all mappings for a given pcid and addr, not including globals. */ 36 static inline void invpcid_flush_one(unsigned long pcid, 37 unsigned long addr) 38 { 39 __invpcid(pcid, addr, INVPCID_TYPE_INDIV_ADDR); 40 } 41 42 /* Flush all mappings for a given PCID, not including globals. */ 43 static inline void invpcid_flush_single_context(unsigned long pcid) 44 { 45 __invpcid(pcid, 0, INVPCID_TYPE_SINGLE_CTXT); 46 } 47 48 /* Flush all mappings, including globals, for all PCIDs. */ 49 static inline void invpcid_flush_all(void) 50 { 51 __invpcid(0, 0, INVPCID_TYPE_ALL_INCL_GLOBAL); 52 } 53 54 /* Flush all mappings for all PCIDs except globals. */ 55 static inline void invpcid_flush_all_nonglobals(void) 56 { 57 __invpcid(0, 0, INVPCID_TYPE_ALL_NON_GLOBAL); 58 } 59 60 #ifdef CONFIG_PARAVIRT 61 #include <asm/paravirt.h> 62 #else 63 #define __flush_tlb() __native_flush_tlb() 64 #define __flush_tlb_global() __native_flush_tlb_global() 65 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr) 66 #endif 67 68 struct tlb_state { 69 /* 70 * cpu_tlbstate.loaded_mm should match CR3 whenever interrupts 71 * are on. This means that it may not match current->active_mm, 72 * which will contain the previous user mm when we're in lazy TLB 73 * mode even if we've already switched back to swapper_pg_dir. 74 */ 75 struct mm_struct *loaded_mm; 76 int state; 77 78 /* 79 * Access to this CR4 shadow and to H/W CR4 is protected by 80 * disabling interrupts when modifying either one. 81 */ 82 unsigned long cr4; 83 }; 84 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate); 85 86 /* Initialize cr4 shadow for this CPU. */ 87 static inline void cr4_init_shadow(void) 88 { 89 this_cpu_write(cpu_tlbstate.cr4, __read_cr4()); 90 } 91 92 /* Set in this cpu's CR4. */ 93 static inline void cr4_set_bits(unsigned long mask) 94 { 95 unsigned long cr4; 96 97 cr4 = this_cpu_read(cpu_tlbstate.cr4); 98 if ((cr4 | mask) != cr4) { 99 cr4 |= mask; 100 this_cpu_write(cpu_tlbstate.cr4, cr4); 101 __write_cr4(cr4); 102 } 103 } 104 105 /* Clear in this cpu's CR4. */ 106 static inline void cr4_clear_bits(unsigned long mask) 107 { 108 unsigned long cr4; 109 110 cr4 = this_cpu_read(cpu_tlbstate.cr4); 111 if ((cr4 & ~mask) != cr4) { 112 cr4 &= ~mask; 113 this_cpu_write(cpu_tlbstate.cr4, cr4); 114 __write_cr4(cr4); 115 } 116 } 117 118 static inline void cr4_toggle_bits(unsigned long mask) 119 { 120 unsigned long cr4; 121 122 cr4 = this_cpu_read(cpu_tlbstate.cr4); 123 cr4 ^= mask; 124 this_cpu_write(cpu_tlbstate.cr4, cr4); 125 __write_cr4(cr4); 126 } 127 128 /* Read the CR4 shadow. */ 129 static inline unsigned long cr4_read_shadow(void) 130 { 131 return this_cpu_read(cpu_tlbstate.cr4); 132 } 133 134 /* 135 * Save some of cr4 feature set we're using (e.g. Pentium 4MB 136 * enable and PPro Global page enable), so that any CPU's that boot 137 * up after us can get the correct flags. This should only be used 138 * during boot on the boot cpu. 139 */ 140 extern unsigned long mmu_cr4_features; 141 extern u32 *trampoline_cr4_features; 142 143 static inline void cr4_set_bits_and_update_boot(unsigned long mask) 144 { 145 mmu_cr4_features |= mask; 146 if (trampoline_cr4_features) 147 *trampoline_cr4_features = mmu_cr4_features; 148 cr4_set_bits(mask); 149 } 150 151 static inline void __native_flush_tlb(void) 152 { 153 /* 154 * If current->mm == NULL then we borrow a mm which may change during a 155 * task switch and therefore we must not be preempted while we write CR3 156 * back: 157 */ 158 preempt_disable(); 159 native_write_cr3(__native_read_cr3()); 160 preempt_enable(); 161 } 162 163 static inline void __native_flush_tlb_global_irq_disabled(void) 164 { 165 unsigned long cr4; 166 167 cr4 = this_cpu_read(cpu_tlbstate.cr4); 168 /* clear PGE */ 169 native_write_cr4(cr4 & ~X86_CR4_PGE); 170 /* write old PGE again and flush TLBs */ 171 native_write_cr4(cr4); 172 } 173 174 static inline void __native_flush_tlb_global(void) 175 { 176 unsigned long flags; 177 178 if (static_cpu_has(X86_FEATURE_INVPCID)) { 179 /* 180 * Using INVPCID is considerably faster than a pair of writes 181 * to CR4 sandwiched inside an IRQ flag save/restore. 182 */ 183 invpcid_flush_all(); 184 return; 185 } 186 187 /* 188 * Read-modify-write to CR4 - protect it from preemption and 189 * from interrupts. (Use the raw variant because this code can 190 * be called from deep inside debugging code.) 191 */ 192 raw_local_irq_save(flags); 193 194 __native_flush_tlb_global_irq_disabled(); 195 196 raw_local_irq_restore(flags); 197 } 198 199 static inline void __native_flush_tlb_single(unsigned long addr) 200 { 201 asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); 202 } 203 204 static inline void __flush_tlb_all(void) 205 { 206 if (boot_cpu_has(X86_FEATURE_PGE)) 207 __flush_tlb_global(); 208 else 209 __flush_tlb(); 210 } 211 212 static inline void __flush_tlb_one(unsigned long addr) 213 { 214 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); 215 __flush_tlb_single(addr); 216 } 217 218 #define TLB_FLUSH_ALL -1UL 219 220 /* 221 * TLB flushing: 222 * 223 * - flush_tlb_all() flushes all processes TLBs 224 * - flush_tlb_mm(mm) flushes the specified mm context TLB's 225 * - flush_tlb_page(vma, vmaddr) flushes one page 226 * - flush_tlb_range(vma, start, end) flushes a range of pages 227 * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages 228 * - flush_tlb_others(cpumask, info) flushes TLBs on other cpus 229 * 230 * ..but the i386 has somewhat limited tlb flushing capabilities, 231 * and page-granular flushes are available only on i486 and up. 232 */ 233 struct flush_tlb_info { 234 struct mm_struct *mm; 235 unsigned long start; 236 unsigned long end; 237 }; 238 239 #define local_flush_tlb() __flush_tlb() 240 241 #define flush_tlb_mm(mm) flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL) 242 243 #define flush_tlb_range(vma, start, end) \ 244 flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags) 245 246 extern void flush_tlb_all(void); 247 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, 248 unsigned long end, unsigned long vmflag); 249 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); 250 251 static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a) 252 { 253 flush_tlb_mm_range(vma->vm_mm, a, a + PAGE_SIZE, VM_NONE); 254 } 255 256 void native_flush_tlb_others(const struct cpumask *cpumask, 257 const struct flush_tlb_info *info); 258 259 #define TLBSTATE_OK 1 260 #define TLBSTATE_LAZY 2 261 262 static inline void arch_tlbbatch_add_mm(struct arch_tlbflush_unmap_batch *batch, 263 struct mm_struct *mm) 264 { 265 cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm)); 266 } 267 268 extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch); 269 270 #ifndef CONFIG_PARAVIRT 271 #define flush_tlb_others(mask, info) \ 272 native_flush_tlb_others(mask, info) 273 #endif 274 275 #endif /* _ASM_X86_TLBFLUSH_H */ 276