1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * This file contains the routines for TLB flushing. 4 * On machines where the MMU does not use a hash table to store virtual to 5 * physical translations (ie, SW loaded TLBs or Book3E compilant processors, 6 * this does -not- include 603 however which shares the implementation with 7 * hash based processors) 8 * 9 * -- BenH 10 * 11 * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org> 12 * IBM Corp. 13 * 14 * Derived from arch/ppc/mm/init.c: 15 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 16 * 17 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 18 * and Cort Dougan (PReP) (cort@cs.nmt.edu) 19 * Copyright (C) 1996 Paul Mackerras 20 * 21 * Derived from "arch/i386/mm/init.c" 22 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 23 */ 24 25 #include <linux/kernel.h> 26 #include <linux/export.h> 27 #include <linux/mm.h> 28 #include <linux/init.h> 29 #include <linux/highmem.h> 30 #include <linux/pagemap.h> 31 #include <linux/preempt.h> 32 #include <linux/spinlock.h> 33 #include <linux/memblock.h> 34 #include <linux/of_fdt.h> 35 #include <linux/hugetlb.h> 36 37 #include <asm/pgalloc.h> 38 #include <asm/tlbflush.h> 39 #include <asm/tlb.h> 40 #include <asm/code-patching.h> 41 #include <asm/cputhreads.h> 42 #include <asm/hugetlb.h> 43 #include <asm/paca.h> 44 45 #include <mm/mmu_decl.h> 46 47 /* 48 * This struct lists the sw-supported page sizes. The hardawre MMU may support 49 * other sizes not listed here. The .ind field is only used on MMUs that have 50 * indirect page table entries. 51 */ 52 #ifdef CONFIG_PPC_E500 53 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { 54 [MMU_PAGE_4K] = { 55 .shift = 12, 56 .enc = BOOK3E_PAGESZ_4K, 57 }, 58 [MMU_PAGE_2M] = { 59 .shift = 21, 60 .enc = BOOK3E_PAGESZ_2M, 61 }, 62 [MMU_PAGE_4M] = { 63 .shift = 22, 64 .enc = BOOK3E_PAGESZ_4M, 65 }, 66 [MMU_PAGE_16M] = { 67 .shift = 24, 68 .enc = BOOK3E_PAGESZ_16M, 69 }, 70 [MMU_PAGE_64M] = { 71 .shift = 26, 72 .enc = BOOK3E_PAGESZ_64M, 73 }, 74 [MMU_PAGE_256M] = { 75 .shift = 28, 76 .enc = BOOK3E_PAGESZ_256M, 77 }, 78 [MMU_PAGE_1G] = { 79 .shift = 30, 80 .enc = BOOK3E_PAGESZ_1GB, 81 }, 82 }; 83 84 static inline int mmu_get_tsize(int psize) 85 { 86 return mmu_psize_defs[psize].enc; 87 } 88 #else 89 static inline int mmu_get_tsize(int psize) 90 { 91 /* This isn't used on !Book3E for now */ 92 return 0; 93 } 94 #endif 95 96 #ifdef CONFIG_PPC_8xx 97 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { 98 [MMU_PAGE_4K] = { 99 .shift = 12, 100 }, 101 [MMU_PAGE_16K] = { 102 .shift = 14, 103 }, 104 [MMU_PAGE_512K] = { 105 .shift = 19, 106 }, 107 [MMU_PAGE_8M] = { 108 .shift = 23, 109 }, 110 }; 111 #endif 112 113 #ifdef CONFIG_PPC_E500 114 /* next_tlbcam_idx is used to round-robin tlbcam entry assignment */ 115 DEFINE_PER_CPU(int, next_tlbcam_idx); 116 EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx); 117 #endif 118 119 /* 120 * Base TLB flushing operations: 121 * 122 * - flush_tlb_mm(mm) flushes the specified mm context TLB's 123 * - flush_tlb_page(vma, vmaddr) flushes one page 124 * - flush_tlb_range(vma, start, end) flushes a range of pages 125 * - flush_tlb_kernel_range(start, end) flushes kernel pages 126 * 127 * - local_* variants of page and mm only apply to the current 128 * processor 129 */ 130 131 #ifndef CONFIG_PPC_8xx 132 /* 133 * These are the base non-SMP variants of page and mm flushing 134 */ 135 void local_flush_tlb_mm(struct mm_struct *mm) 136 { 137 unsigned int pid; 138 139 preempt_disable(); 140 pid = mm->context.id; 141 if (pid != MMU_NO_CONTEXT) 142 _tlbil_pid(pid); 143 preempt_enable(); 144 } 145 EXPORT_SYMBOL(local_flush_tlb_mm); 146 147 void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, 148 int tsize, int ind) 149 { 150 unsigned int pid; 151 152 preempt_disable(); 153 pid = mm ? mm->context.id : 0; 154 if (pid != MMU_NO_CONTEXT) 155 _tlbil_va(vmaddr, pid, tsize, ind); 156 preempt_enable(); 157 } 158 159 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 160 { 161 __local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr, 162 mmu_get_tsize(mmu_virtual_psize), 0); 163 } 164 EXPORT_SYMBOL(local_flush_tlb_page); 165 166 void local_flush_tlb_page_psize(struct mm_struct *mm, 167 unsigned long vmaddr, int psize) 168 { 169 __local_flush_tlb_page(mm, vmaddr, mmu_get_tsize(psize), 0); 170 } 171 EXPORT_SYMBOL(local_flush_tlb_page_psize); 172 173 #endif 174 175 /* 176 * And here are the SMP non-local implementations 177 */ 178 #ifdef CONFIG_SMP 179 180 static DEFINE_RAW_SPINLOCK(tlbivax_lock); 181 182 struct tlb_flush_param { 183 unsigned long addr; 184 unsigned int pid; 185 unsigned int tsize; 186 unsigned int ind; 187 }; 188 189 static void do_flush_tlb_mm_ipi(void *param) 190 { 191 struct tlb_flush_param *p = param; 192 193 _tlbil_pid(p ? p->pid : 0); 194 } 195 196 static void do_flush_tlb_page_ipi(void *param) 197 { 198 struct tlb_flush_param *p = param; 199 200 _tlbil_va(p->addr, p->pid, p->tsize, p->ind); 201 } 202 203 204 /* Note on invalidations and PID: 205 * 206 * We snapshot the PID with preempt disabled. At this point, it can still 207 * change either because: 208 * - our context is being stolen (PID -> NO_CONTEXT) on another CPU 209 * - we are invaliating some target that isn't currently running here 210 * and is concurrently acquiring a new PID on another CPU 211 * - some other CPU is re-acquiring a lost PID for this mm 212 * etc... 213 * 214 * However, this shouldn't be a problem as we only guarantee 215 * invalidation of TLB entries present prior to this call, so we 216 * don't care about the PID changing, and invalidating a stale PID 217 * is generally harmless. 218 */ 219 220 void flush_tlb_mm(struct mm_struct *mm) 221 { 222 unsigned int pid; 223 224 preempt_disable(); 225 pid = mm->context.id; 226 if (unlikely(pid == MMU_NO_CONTEXT)) 227 goto no_context; 228 if (!mm_is_core_local(mm)) { 229 struct tlb_flush_param p = { .pid = pid }; 230 /* Ignores smp_processor_id() even if set. */ 231 smp_call_function_many(mm_cpumask(mm), 232 do_flush_tlb_mm_ipi, &p, 1); 233 } 234 _tlbil_pid(pid); 235 no_context: 236 preempt_enable(); 237 } 238 EXPORT_SYMBOL(flush_tlb_mm); 239 240 void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, 241 int tsize, int ind) 242 { 243 struct cpumask *cpu_mask; 244 unsigned int pid; 245 246 /* 247 * This function as well as __local_flush_tlb_page() must only be called 248 * for user contexts. 249 */ 250 if (WARN_ON(!mm)) 251 return; 252 253 preempt_disable(); 254 pid = mm->context.id; 255 if (unlikely(pid == MMU_NO_CONTEXT)) 256 goto bail; 257 cpu_mask = mm_cpumask(mm); 258 if (!mm_is_core_local(mm)) { 259 /* If broadcast tlbivax is supported, use it */ 260 if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) { 261 int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL); 262 if (lock) 263 raw_spin_lock(&tlbivax_lock); 264 _tlbivax_bcast(vmaddr, pid, tsize, ind); 265 if (lock) 266 raw_spin_unlock(&tlbivax_lock); 267 goto bail; 268 } else { 269 struct tlb_flush_param p = { 270 .pid = pid, 271 .addr = vmaddr, 272 .tsize = tsize, 273 .ind = ind, 274 }; 275 /* Ignores smp_processor_id() even if set in cpu_mask */ 276 smp_call_function_many(cpu_mask, 277 do_flush_tlb_page_ipi, &p, 1); 278 } 279 } 280 _tlbil_va(vmaddr, pid, tsize, ind); 281 bail: 282 preempt_enable(); 283 } 284 285 void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 286 { 287 #ifdef CONFIG_HUGETLB_PAGE 288 if (vma && is_vm_hugetlb_page(vma)) 289 flush_hugetlb_page(vma, vmaddr); 290 #endif 291 292 __flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr, 293 mmu_get_tsize(mmu_virtual_psize), 0); 294 } 295 EXPORT_SYMBOL(flush_tlb_page); 296 297 #endif /* CONFIG_SMP */ 298 299 /* 300 * Flush kernel TLB entries in the given range 301 */ 302 #ifndef CONFIG_PPC_8xx 303 void flush_tlb_kernel_range(unsigned long start, unsigned long end) 304 { 305 #ifdef CONFIG_SMP 306 preempt_disable(); 307 smp_call_function(do_flush_tlb_mm_ipi, NULL, 1); 308 _tlbil_pid(0); 309 preempt_enable(); 310 #else 311 _tlbil_pid(0); 312 #endif 313 } 314 EXPORT_SYMBOL(flush_tlb_kernel_range); 315 #endif 316 317 /* 318 * Currently, for range flushing, we just do a full mm flush. This should 319 * be optimized based on a threshold on the size of the range, since 320 * some implementation can stack multiple tlbivax before a tlbsync but 321 * for now, we keep it that way 322 */ 323 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 324 unsigned long end) 325 326 { 327 if (end - start == PAGE_SIZE && !(start & ~PAGE_MASK)) 328 flush_tlb_page(vma, start); 329 else 330 flush_tlb_mm(vma->vm_mm); 331 } 332 EXPORT_SYMBOL(flush_tlb_range); 333 334 void tlb_flush(struct mmu_gather *tlb) 335 { 336 flush_tlb_mm(tlb->mm); 337 } 338 339 #ifndef CONFIG_PPC64 340 void __init early_init_mmu(void) 341 { 342 unsigned long root = of_get_flat_dt_root(); 343 344 if (IS_ENABLED(CONFIG_PPC_47x) && IS_ENABLED(CONFIG_SMP) && 345 of_get_flat_dt_prop(root, "cooperative-partition", NULL)) 346 mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST); 347 } 348 #endif /* CONFIG_PPC64 */ 349