1 /* 2 * This file contains common routines for dealing with free of page tables 3 * Along with common page table handling code 4 * 5 * Derived from arch/powerpc/mm/tlb_64.c: 6 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 7 * 8 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 9 * and Cort Dougan (PReP) (cort@cs.nmt.edu) 10 * Copyright (C) 1996 Paul Mackerras 11 * 12 * Derived from "arch/i386/mm/init.c" 13 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 14 * 15 * Dave Engebretsen <engebret@us.ibm.com> 16 * Rework for PPC64 port. 17 * 18 * This program is free software; you can redistribute it and/or 19 * modify it under the terms of the GNU General Public License 20 * as published by the Free Software Foundation; either version 21 * 2 of the License, or (at your option) any later version. 22 */ 23 24 #include <linux/kernel.h> 25 #include <linux/gfp.h> 26 #include <linux/mm.h> 27 #include <linux/percpu.h> 28 #include <linux/hardirq.h> 29 #include <linux/hugetlb.h> 30 #include <asm/pgalloc.h> 31 #include <asm/tlbflush.h> 32 #include <asm/tlb.h> 33 34 static inline int is_exec_fault(void) 35 { 36 return current->thread.regs && TRAP(current->thread.regs) == 0x400; 37 } 38 39 /* We only try to do i/d cache coherency on stuff that looks like 40 * reasonably "normal" PTEs. We currently require a PTE to be present 41 * and we avoid _PAGE_SPECIAL and cache inhibited pte. We also only do that 42 * on userspace PTEs 43 */ 44 static inline int pte_looks_normal(pte_t pte) 45 { 46 47 #if defined(CONFIG_PPC_BOOK3S_64) 48 if ((pte_val(pte) & (_PAGE_PRESENT | _PAGE_SPECIAL)) == _PAGE_PRESENT) { 49 if (pte_ci(pte)) 50 return 0; 51 if (pte_user(pte)) 52 return 1; 53 } 54 return 0; 55 #else 56 return (pte_val(pte) & 57 (_PAGE_PRESENT | _PAGE_SPECIAL | _PAGE_NO_CACHE | _PAGE_USER)) == 58 (_PAGE_PRESENT | _PAGE_USER); 59 #endif 60 } 61 62 static struct page *maybe_pte_to_page(pte_t pte) 63 { 64 unsigned long pfn = pte_pfn(pte); 65 struct page *page; 66 67 if (unlikely(!pfn_valid(pfn))) 68 return NULL; 69 page = pfn_to_page(pfn); 70 if (PageReserved(page)) 71 return NULL; 72 return page; 73 } 74 75 #if defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0 76 77 /* Server-style MMU handles coherency when hashing if HW exec permission 78 * is supposed per page (currently 64-bit only). If not, then, we always 79 * flush the cache for valid PTEs in set_pte. Embedded CPU without HW exec 80 * support falls into the same category. 81 */ 82 83 static pte_t set_pte_filter(pte_t pte) 84 { 85 if (radix_enabled()) 86 return pte; 87 88 pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); 89 if (pte_looks_normal(pte) && !(cpu_has_feature(CPU_FTR_COHERENT_ICACHE) || 90 cpu_has_feature(CPU_FTR_NOEXECUTE))) { 91 struct page *pg = maybe_pte_to_page(pte); 92 if (!pg) 93 return pte; 94 if (!test_bit(PG_arch_1, &pg->flags)) { 95 flush_dcache_icache_page(pg); 96 set_bit(PG_arch_1, &pg->flags); 97 } 98 } 99 return pte; 100 } 101 102 static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma, 103 int dirty) 104 { 105 return pte; 106 } 107 108 #else /* defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0 */ 109 110 /* Embedded type MMU with HW exec support. This is a bit more complicated 111 * as we don't have two bits to spare for _PAGE_EXEC and _PAGE_HWEXEC so 112 * instead we "filter out" the exec permission for non clean pages. 113 */ 114 static pte_t set_pte_filter(pte_t pte) 115 { 116 struct page *pg; 117 118 /* No exec permission in the first place, move on */ 119 if (!(pte_val(pte) & _PAGE_EXEC) || !pte_looks_normal(pte)) 120 return pte; 121 122 /* If you set _PAGE_EXEC on weird pages you're on your own */ 123 pg = maybe_pte_to_page(pte); 124 if (unlikely(!pg)) 125 return pte; 126 127 /* If the page clean, we move on */ 128 if (test_bit(PG_arch_1, &pg->flags)) 129 return pte; 130 131 /* If it's an exec fault, we flush the cache and make it clean */ 132 if (is_exec_fault()) { 133 flush_dcache_icache_page(pg); 134 set_bit(PG_arch_1, &pg->flags); 135 return pte; 136 } 137 138 /* Else, we filter out _PAGE_EXEC */ 139 return __pte(pte_val(pte) & ~_PAGE_EXEC); 140 } 141 142 static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma, 143 int dirty) 144 { 145 struct page *pg; 146 147 /* So here, we only care about exec faults, as we use them 148 * to recover lost _PAGE_EXEC and perform I$/D$ coherency 149 * if necessary. Also if _PAGE_EXEC is already set, same deal, 150 * we just bail out 151 */ 152 if (dirty || (pte_val(pte) & _PAGE_EXEC) || !is_exec_fault()) 153 return pte; 154 155 #ifdef CONFIG_DEBUG_VM 156 /* So this is an exec fault, _PAGE_EXEC is not set. If it was 157 * an error we would have bailed out earlier in do_page_fault() 158 * but let's make sure of it 159 */ 160 if (WARN_ON(!(vma->vm_flags & VM_EXEC))) 161 return pte; 162 #endif /* CONFIG_DEBUG_VM */ 163 164 /* If you set _PAGE_EXEC on weird pages you're on your own */ 165 pg = maybe_pte_to_page(pte); 166 if (unlikely(!pg)) 167 goto bail; 168 169 /* If the page is already clean, we move on */ 170 if (test_bit(PG_arch_1, &pg->flags)) 171 goto bail; 172 173 /* Clean the page and set PG_arch_1 */ 174 flush_dcache_icache_page(pg); 175 set_bit(PG_arch_1, &pg->flags); 176 177 bail: 178 return __pte(pte_val(pte) | _PAGE_EXEC); 179 } 180 181 #endif /* !(defined(CONFIG_PPC_STD_MMU) || _PAGE_EXEC == 0) */ 182 183 /* 184 * set_pte stores a linux PTE into the linux page table. 185 */ 186 void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, 187 pte_t pte) 188 { 189 /* 190 * When handling numa faults, we already have the pte marked 191 * _PAGE_PRESENT, but we can be sure that it is not in hpte. 192 * Hence we can use set_pte_at for them. 193 */ 194 VM_WARN_ON(pte_present(*ptep) && !pte_protnone(*ptep)); 195 196 /* Add the pte bit when trying to set a pte */ 197 pte = __pte(pte_val(pte) | _PAGE_PTE); 198 199 /* Note: mm->context.id might not yet have been assigned as 200 * this context might not have been activated yet when this 201 * is called. 202 */ 203 pte = set_pte_filter(pte); 204 205 /* Perform the setting of the PTE */ 206 __set_pte_at(mm, addr, ptep, pte, 0); 207 } 208 209 /* 210 * This is called when relaxing access to a PTE. It's also called in the page 211 * fault path when we don't hit any of the major fault cases, ie, a minor 212 * update of _PAGE_ACCESSED, _PAGE_DIRTY, etc... The generic code will have 213 * handled those two for us, we additionally deal with missing execute 214 * permission here on some processors 215 */ 216 int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, 217 pte_t *ptep, pte_t entry, int dirty) 218 { 219 int changed; 220 entry = set_access_flags_filter(entry, vma, dirty); 221 changed = !pte_same(*(ptep), entry); 222 if (changed) { 223 if (!is_vm_hugetlb_page(vma)) 224 assert_pte_locked(vma->vm_mm, address); 225 __ptep_set_access_flags(vma->vm_mm, ptep, entry, address); 226 flush_tlb_page(vma, address); 227 } 228 return changed; 229 } 230 231 #ifdef CONFIG_DEBUG_VM 232 void assert_pte_locked(struct mm_struct *mm, unsigned long addr) 233 { 234 pgd_t *pgd; 235 pud_t *pud; 236 pmd_t *pmd; 237 238 if (mm == &init_mm) 239 return; 240 pgd = mm->pgd + pgd_index(addr); 241 BUG_ON(pgd_none(*pgd)); 242 pud = pud_offset(pgd, addr); 243 BUG_ON(pud_none(*pud)); 244 pmd = pmd_offset(pud, addr); 245 /* 246 * khugepaged to collapse normal pages to hugepage, first set 247 * pmd to none to force page fault/gup to take mmap_sem. After 248 * pmd is set to none, we do a pte_clear which does this assertion 249 * so if we find pmd none, return. 250 */ 251 if (pmd_none(*pmd)) 252 return; 253 BUG_ON(!pmd_present(*pmd)); 254 assert_spin_locked(pte_lockptr(mm, pmd)); 255 } 256 #endif /* CONFIG_DEBUG_VM */ 257 258 unsigned long vmalloc_to_phys(void *va) 259 { 260 unsigned long pfn = vmalloc_to_pfn(va); 261 262 BUG_ON(!pfn); 263 return __pa(pfn_to_kaddr(pfn)) + offset_in_page(va); 264 } 265 EXPORT_SYMBOL_GPL(vmalloc_to_phys); 266