1 /* 2 * This file contains common routines for dealing with free of page tables 3 * Along with common page table handling code 4 * 5 * Derived from arch/powerpc/mm/tlb_64.c: 6 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 7 * 8 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 9 * and Cort Dougan (PReP) (cort@cs.nmt.edu) 10 * Copyright (C) 1996 Paul Mackerras 11 * 12 * Derived from "arch/i386/mm/init.c" 13 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 14 * 15 * Dave Engebretsen <engebret@us.ibm.com> 16 * Rework for PPC64 port. 17 * 18 * This program is free software; you can redistribute it and/or 19 * modify it under the terms of the GNU General Public License 20 * as published by the Free Software Foundation; either version 21 * 2 of the License, or (at your option) any later version. 22 */ 23 24 #include <linux/kernel.h> 25 #include <linux/mm.h> 26 #include <linux/init.h> 27 #include <linux/percpu.h> 28 #include <linux/hardirq.h> 29 #include <asm/pgalloc.h> 30 #include <asm/tlbflush.h> 31 #include <asm/tlb.h> 32 33 static DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur); 34 static unsigned long pte_freelist_forced_free; 35 36 struct pte_freelist_batch 37 { 38 struct rcu_head rcu; 39 unsigned int index; 40 pgtable_free_t tables[0]; 41 }; 42 43 #define PTE_FREELIST_SIZE \ 44 ((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \ 45 / sizeof(pgtable_free_t)) 46 47 static void pte_free_smp_sync(void *arg) 48 { 49 /* Do nothing, just ensure we sync with all CPUs */ 50 } 51 52 /* This is only called when we are critically out of memory 53 * (and fail to get a page in pte_free_tlb). 54 */ 55 static void pgtable_free_now(pgtable_free_t pgf) 56 { 57 pte_freelist_forced_free++; 58 59 smp_call_function(pte_free_smp_sync, NULL, 1); 60 61 pgtable_free(pgf); 62 } 63 64 static void pte_free_rcu_callback(struct rcu_head *head) 65 { 66 struct pte_freelist_batch *batch = 67 container_of(head, struct pte_freelist_batch, rcu); 68 unsigned int i; 69 70 for (i = 0; i < batch->index; i++) 71 pgtable_free(batch->tables[i]); 72 73 free_page((unsigned long)batch); 74 } 75 76 static void pte_free_submit(struct pte_freelist_batch *batch) 77 { 78 INIT_RCU_HEAD(&batch->rcu); 79 call_rcu(&batch->rcu, pte_free_rcu_callback); 80 } 81 82 void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf) 83 { 84 /* This is safe since tlb_gather_mmu has disabled preemption */ 85 struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur); 86 87 if (atomic_read(&tlb->mm->mm_users) < 2 || 88 cpumask_equal(mm_cpumask(tlb->mm), cpumask_of(smp_processor_id()))){ 89 pgtable_free(pgf); 90 return; 91 } 92 93 if (*batchp == NULL) { 94 *batchp = (struct pte_freelist_batch *)__get_free_page(GFP_ATOMIC); 95 if (*batchp == NULL) { 96 pgtable_free_now(pgf); 97 return; 98 } 99 (*batchp)->index = 0; 100 } 101 (*batchp)->tables[(*batchp)->index++] = pgf; 102 if ((*batchp)->index == PTE_FREELIST_SIZE) { 103 pte_free_submit(*batchp); 104 *batchp = NULL; 105 } 106 } 107 108 void pte_free_finish(void) 109 { 110 /* This is safe since tlb_gather_mmu has disabled preemption */ 111 struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur); 112 113 if (*batchp == NULL) 114 return; 115 pte_free_submit(*batchp); 116 *batchp = NULL; 117 } 118 119 /* 120 * Handle i/d cache flushing, called from set_pte_at() or ptep_set_access_flags() 121 */ 122 static pte_t do_dcache_icache_coherency(pte_t pte) 123 { 124 unsigned long pfn = pte_pfn(pte); 125 struct page *page; 126 127 if (unlikely(!pfn_valid(pfn))) 128 return pte; 129 page = pfn_to_page(pfn); 130 131 if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags)) { 132 pr_debug("do_dcache_icache_coherency... flushing\n"); 133 flush_dcache_icache_page(page); 134 set_bit(PG_arch_1, &page->flags); 135 } 136 else 137 pr_debug("do_dcache_icache_coherency... already clean\n"); 138 return __pte(pte_val(pte) | _PAGE_HWEXEC); 139 } 140 141 static inline int is_exec_fault(void) 142 { 143 return current->thread.regs && TRAP(current->thread.regs) == 0x400; 144 } 145 146 /* We only try to do i/d cache coherency on stuff that looks like 147 * reasonably "normal" PTEs. We currently require a PTE to be present 148 * and we avoid _PAGE_SPECIAL and _PAGE_NO_CACHE 149 */ 150 static inline int pte_looks_normal(pte_t pte) 151 { 152 return (pte_val(pte) & 153 (_PAGE_PRESENT | _PAGE_SPECIAL | _PAGE_NO_CACHE)) == 154 (_PAGE_PRESENT); 155 } 156 157 #if defined(CONFIG_PPC_STD_MMU) 158 /* Server-style MMU handles coherency when hashing if HW exec permission 159 * is supposed per page (currently 64-bit only). Else, we always flush 160 * valid PTEs in set_pte. 161 */ 162 static inline int pte_need_exec_flush(pte_t pte, int set_pte) 163 { 164 return set_pte && pte_looks_normal(pte) && 165 !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) || 166 cpu_has_feature(CPU_FTR_NOEXECUTE)); 167 } 168 #elif _PAGE_HWEXEC == 0 169 /* Embedded type MMU without HW exec support (8xx only so far), we flush 170 * the cache for any present PTE 171 */ 172 static inline int pte_need_exec_flush(pte_t pte, int set_pte) 173 { 174 return set_pte && pte_looks_normal(pte); 175 } 176 #else 177 /* Other embedded CPUs with HW exec support per-page, we flush on exec 178 * fault if HWEXEC is not set 179 */ 180 static inline int pte_need_exec_flush(pte_t pte, int set_pte) 181 { 182 return pte_looks_normal(pte) && is_exec_fault() && 183 !(pte_val(pte) & _PAGE_HWEXEC); 184 } 185 #endif 186 187 /* 188 * set_pte stores a linux PTE into the linux page table. 189 */ 190 void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) 191 { 192 #ifdef CONFIG_DEBUG_VM 193 WARN_ON(pte_present(*ptep)); 194 #endif 195 /* Note: mm->context.id might not yet have been assigned as 196 * this context might not have been activated yet when this 197 * is called. 198 */ 199 pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); 200 if (pte_need_exec_flush(pte, 1)) 201 pte = do_dcache_icache_coherency(pte); 202 203 /* Perform the setting of the PTE */ 204 __set_pte_at(mm, addr, ptep, pte, 0); 205 } 206 207 /* 208 * This is called when relaxing access to a PTE. It's also called in the page 209 * fault path when we don't hit any of the major fault cases, ie, a minor 210 * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have 211 * handled those two for us, we additionally deal with missing execute 212 * permission here on some processors 213 */ 214 int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, 215 pte_t *ptep, pte_t entry, int dirty) 216 { 217 int changed; 218 if (!dirty && pte_need_exec_flush(entry, 0)) 219 entry = do_dcache_icache_coherency(entry); 220 changed = !pte_same(*(ptep), entry); 221 if (changed) { 222 assert_pte_locked(vma->vm_mm, address); 223 __ptep_set_access_flags(ptep, entry); 224 flush_tlb_page_nohash(vma, address); 225 } 226 return changed; 227 } 228 229 #ifdef CONFIG_DEBUG_VM 230 void assert_pte_locked(struct mm_struct *mm, unsigned long addr) 231 { 232 pgd_t *pgd; 233 pud_t *pud; 234 pmd_t *pmd; 235 236 if (mm == &init_mm) 237 return; 238 pgd = mm->pgd + pgd_index(addr); 239 BUG_ON(pgd_none(*pgd)); 240 pud = pud_offset(pgd, addr); 241 BUG_ON(pud_none(*pud)); 242 pmd = pmd_offset(pud, addr); 243 BUG_ON(!pmd_present(*pmd)); 244 BUG_ON(!spin_is_locked(pte_lockptr(mm, pmd))); 245 } 246 #endif /* CONFIG_DEBUG_VM */ 247 248