1 /* 2 * This file contains the routines for flushing entries from the 3 * TLB and MMU hash table. 4 * 5 * Derived from arch/ppc64/mm/init.c: 6 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 7 * 8 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 9 * and Cort Dougan (PReP) (cort@cs.nmt.edu) 10 * Copyright (C) 1996 Paul Mackerras 11 * 12 * Derived from "arch/i386/mm/init.c" 13 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 14 * 15 * Dave Engebretsen <engebret@us.ibm.com> 16 * Rework for PPC64 port. 17 * 18 * This program is free software; you can redistribute it and/or 19 * modify it under the terms of the GNU General Public License 20 * as published by the Free Software Foundation; either version 21 * 2 of the License, or (at your option) any later version. 22 */ 23 24 #include <linux/kernel.h> 25 #include <linux/mm.h> 26 #include <linux/percpu.h> 27 #include <linux/hardirq.h> 28 #include <asm/pgalloc.h> 29 #include <asm/tlbflush.h> 30 #include <asm/tlb.h> 31 #include <asm/bug.h> 32 #include <asm/pte-walk.h> 33 34 35 #include <trace/events/thp.h> 36 37 DEFINE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); 38 39 /* 40 * A linux PTE was changed and the corresponding hash table entry 41 * neesd to be flushed. This function will either perform the flush 42 * immediately or will batch it up if the current CPU has an active 43 * batch on it. 44 */ 45 void hpte_need_flush(struct mm_struct *mm, unsigned long addr, 46 pte_t *ptep, unsigned long pte, int huge) 47 { 48 unsigned long vpn; 49 struct ppc64_tlb_batch *batch = &get_cpu_var(ppc64_tlb_batch); 50 unsigned long vsid; 51 unsigned int psize; 52 int ssize; 53 real_pte_t rpte; 54 int i, offset; 55 56 i = batch->index; 57 58 /* 59 * Get page size (maybe move back to caller). 60 * 61 * NOTE: when using special 64K mappings in 4K environment like 62 * for SPEs, we obtain the page size from the slice, which thus 63 * must still exist (and thus the VMA not reused) at the time 64 * of this call 65 */ 66 if (huge) { 67 #ifdef CONFIG_HUGETLB_PAGE 68 psize = get_slice_psize(mm, addr); 69 /* Mask the address for the correct page size */ 70 addr &= ~((1UL << mmu_psize_defs[psize].shift) - 1); 71 if (unlikely(psize == MMU_PAGE_16G)) 72 offset = PTRS_PER_PUD; 73 else 74 offset = PTRS_PER_PMD; 75 #else 76 BUG(); 77 psize = pte_pagesize_index(mm, addr, pte); /* shutup gcc */ 78 #endif 79 } else { 80 psize = pte_pagesize_index(mm, addr, pte); 81 /* 82 * Mask the address for the standard page size. If we 83 * have a 64k page kernel, but the hardware does not 84 * support 64k pages, this might be different from the 85 * hardware page size encoded in the slice table. 86 */ 87 addr &= PAGE_MASK; 88 offset = PTRS_PER_PTE; 89 } 90 91 92 /* Build full vaddr */ 93 if (!is_kernel_addr(addr)) { 94 ssize = user_segment_size(addr); 95 vsid = get_user_vsid(&mm->context, addr, ssize); 96 } else { 97 vsid = get_kernel_vsid(addr, mmu_kernel_ssize); 98 ssize = mmu_kernel_ssize; 99 } 100 WARN_ON(vsid == 0); 101 vpn = hpt_vpn(addr, vsid, ssize); 102 rpte = __real_pte(__pte(pte), ptep, offset); 103 104 /* 105 * Check if we have an active batch on this CPU. If not, just 106 * flush now and return. 107 */ 108 if (!batch->active) { 109 flush_hash_page(vpn, rpte, psize, ssize, mm_is_thread_local(mm)); 110 put_cpu_var(ppc64_tlb_batch); 111 return; 112 } 113 114 /* 115 * This can happen when we are in the middle of a TLB batch and 116 * we encounter memory pressure (eg copy_page_range when it tries 117 * to allocate a new pte). If we have to reclaim memory and end 118 * up scanning and resetting referenced bits then our batch context 119 * will change mid stream. 120 * 121 * We also need to ensure only one page size is present in a given 122 * batch 123 */ 124 if (i != 0 && (mm != batch->mm || batch->psize != psize || 125 batch->ssize != ssize)) { 126 __flush_tlb_pending(batch); 127 i = 0; 128 } 129 if (i == 0) { 130 batch->mm = mm; 131 batch->psize = psize; 132 batch->ssize = ssize; 133 } 134 batch->pte[i] = rpte; 135 batch->vpn[i] = vpn; 136 batch->index = ++i; 137 if (i >= PPC64_TLB_BATCH_NR) 138 __flush_tlb_pending(batch); 139 put_cpu_var(ppc64_tlb_batch); 140 } 141 142 /* 143 * This function is called when terminating an mmu batch or when a batch 144 * is full. It will perform the flush of all the entries currently stored 145 * in a batch. 146 * 147 * Must be called from within some kind of spinlock/non-preempt region... 148 */ 149 void __flush_tlb_pending(struct ppc64_tlb_batch *batch) 150 { 151 int i, local; 152 153 i = batch->index; 154 local = mm_is_thread_local(batch->mm); 155 if (i == 1) 156 flush_hash_page(batch->vpn[0], batch->pte[0], 157 batch->psize, batch->ssize, local); 158 else 159 flush_hash_range(i, local); 160 batch->index = 0; 161 } 162 163 void hash__tlb_flush(struct mmu_gather *tlb) 164 { 165 struct ppc64_tlb_batch *tlbbatch = &get_cpu_var(ppc64_tlb_batch); 166 167 /* 168 * If there's a TLB batch pending, then we must flush it because the 169 * pages are going to be freed and we really don't want to have a CPU 170 * access a freed page because it has a stale TLB 171 */ 172 if (tlbbatch->index) 173 __flush_tlb_pending(tlbbatch); 174 175 put_cpu_var(ppc64_tlb_batch); 176 } 177 178 /** 179 * __flush_hash_table_range - Flush all HPTEs for a given address range 180 * from the hash table (and the TLB). But keeps 181 * the linux PTEs intact. 182 * 183 * @mm : mm_struct of the target address space (generally init_mm) 184 * @start : starting address 185 * @end : ending address (not included in the flush) 186 * 187 * This function is mostly to be used by some IO hotplug code in order 188 * to remove all hash entries from a given address range used to map IO 189 * space on a removed PCI-PCI bidge without tearing down the full mapping 190 * since 64K pages may overlap with other bridges when using 64K pages 191 * with 4K HW pages on IO space. 192 * 193 * Because of that usage pattern, it is implemented for small size rather 194 * than speed. 195 */ 196 void __flush_hash_table_range(struct mm_struct *mm, unsigned long start, 197 unsigned long end) 198 { 199 bool is_thp; 200 int hugepage_shift; 201 unsigned long flags; 202 203 start = _ALIGN_DOWN(start, PAGE_SIZE); 204 end = _ALIGN_UP(end, PAGE_SIZE); 205 206 BUG_ON(!mm->pgd); 207 208 /* 209 * Note: Normally, we should only ever use a batch within a 210 * PTE locked section. This violates the rule, but will work 211 * since we don't actually modify the PTEs, we just flush the 212 * hash while leaving the PTEs intact (including their reference 213 * to being hashed). This is not the most performance oriented 214 * way to do things but is fine for our needs here. 215 */ 216 local_irq_save(flags); 217 arch_enter_lazy_mmu_mode(); 218 for (; start < end; start += PAGE_SIZE) { 219 pte_t *ptep = find_current_mm_pte(mm->pgd, start, &is_thp, 220 &hugepage_shift); 221 unsigned long pte; 222 223 if (ptep == NULL) 224 continue; 225 pte = pte_val(*ptep); 226 if (is_thp) 227 trace_hugepage_invalidate(start, pte); 228 if (!(pte & H_PAGE_HASHPTE)) 229 continue; 230 if (unlikely(is_thp)) 231 hpte_do_hugepage_flush(mm, start, (pmd_t *)ptep, pte); 232 else 233 hpte_need_flush(mm, start, ptep, pte, hugepage_shift); 234 } 235 arch_leave_lazy_mmu_mode(); 236 local_irq_restore(flags); 237 } 238 239 void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr) 240 { 241 pte_t *pte; 242 pte_t *start_pte; 243 unsigned long flags; 244 245 addr = _ALIGN_DOWN(addr, PMD_SIZE); 246 /* 247 * Note: Normally, we should only ever use a batch within a 248 * PTE locked section. This violates the rule, but will work 249 * since we don't actually modify the PTEs, we just flush the 250 * hash while leaving the PTEs intact (including their reference 251 * to being hashed). This is not the most performance oriented 252 * way to do things but is fine for our needs here. 253 */ 254 local_irq_save(flags); 255 arch_enter_lazy_mmu_mode(); 256 start_pte = pte_offset_map(pmd, addr); 257 for (pte = start_pte; pte < start_pte + PTRS_PER_PTE; pte++) { 258 unsigned long pteval = pte_val(*pte); 259 if (pteval & H_PAGE_HASHPTE) 260 hpte_need_flush(mm, addr, pte, pteval, 0); 261 addr += PAGE_SIZE; 262 } 263 arch_leave_lazy_mmu_mode(); 264 local_irq_restore(flags); 265 } 266