1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/mm/flush.c 4 * 5 * Copyright (C) 1995-2002 Russell King 6 */ 7 #include <linux/module.h> 8 #include <linux/mm.h> 9 #include <linux/pagemap.h> 10 #include <linux/highmem.h> 11 12 #include <asm/cacheflush.h> 13 #include <asm/cachetype.h> 14 #include <asm/highmem.h> 15 #include <asm/smp_plat.h> 16 #include <asm/tlbflush.h> 17 #include <linux/hugetlb.h> 18 19 #include "mm.h" 20 21 #ifdef CONFIG_ARM_HEAVY_MB 22 void (*soc_mb)(void); 23 24 void arm_heavy_mb(void) 25 { 26 #ifdef CONFIG_OUTER_CACHE_SYNC 27 if (outer_cache.sync) 28 outer_cache.sync(); 29 #endif 30 if (soc_mb) 31 soc_mb(); 32 } 33 EXPORT_SYMBOL(arm_heavy_mb); 34 #endif 35 36 #ifdef CONFIG_CPU_CACHE_VIPT 37 38 static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr) 39 { 40 unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT); 41 const int zero = 0; 42 43 set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL)); 44 45 asm( "mcrr p15, 0, %1, %0, c14\n" 46 " mcr p15, 0, %2, c7, c10, 4" 47 : 48 : "r" (to), "r" (to + PAGE_SIZE - 1), "r" (zero) 49 : "cc"); 50 } 51 52 static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len) 53 { 54 unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT); 55 unsigned long offset = vaddr & (PAGE_SIZE - 1); 56 unsigned long to; 57 58 set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL)); 59 to = va + offset; 60 flush_icache_range(to, to + len); 61 } 62 63 void flush_cache_mm(struct mm_struct *mm) 64 { 65 if (cache_is_vivt()) { 66 vivt_flush_cache_mm(mm); 67 return; 68 } 69 70 if (cache_is_vipt_aliasing()) { 71 asm( "mcr p15, 0, %0, c7, c14, 0\n" 72 " mcr p15, 0, %0, c7, c10, 4" 73 : 74 : "r" (0) 75 : "cc"); 76 } 77 } 78 79 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 80 { 81 if (cache_is_vivt()) { 82 vivt_flush_cache_range(vma, start, end); 83 return; 84 } 85 86 if (cache_is_vipt_aliasing()) { 87 asm( "mcr p15, 0, %0, c7, c14, 0\n" 88 " mcr p15, 0, %0, c7, c10, 4" 89 : 90 : "r" (0) 91 : "cc"); 92 } 93 94 if (vma->vm_flags & VM_EXEC) 95 __flush_icache_all(); 96 } 97 98 void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn) 99 { 100 if (cache_is_vivt()) { 101 vivt_flush_cache_page(vma, user_addr, pfn); 102 return; 103 } 104 105 if (cache_is_vipt_aliasing()) { 106 flush_pfn_alias(pfn, user_addr); 107 __flush_icache_all(); 108 } 109 110 if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged()) 111 __flush_icache_all(); 112 } 113 114 #else 115 #define flush_pfn_alias(pfn,vaddr) do { } while (0) 116 #define flush_icache_alias(pfn,vaddr,len) do { } while (0) 117 #endif 118 119 #define FLAG_PA_IS_EXEC 1 120 #define FLAG_PA_CORE_IN_MM 2 121 122 static void flush_ptrace_access_other(void *args) 123 { 124 __flush_icache_all(); 125 } 126 127 static inline 128 void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr, 129 unsigned long len, unsigned int flags) 130 { 131 if (cache_is_vivt()) { 132 if (flags & FLAG_PA_CORE_IN_MM) { 133 unsigned long addr = (unsigned long)kaddr; 134 __cpuc_coherent_kern_range(addr, addr + len); 135 } 136 return; 137 } 138 139 if (cache_is_vipt_aliasing()) { 140 flush_pfn_alias(page_to_pfn(page), uaddr); 141 __flush_icache_all(); 142 return; 143 } 144 145 /* VIPT non-aliasing D-cache */ 146 if (flags & FLAG_PA_IS_EXEC) { 147 unsigned long addr = (unsigned long)kaddr; 148 if (icache_is_vipt_aliasing()) 149 flush_icache_alias(page_to_pfn(page), uaddr, len); 150 else 151 __cpuc_coherent_kern_range(addr, addr + len); 152 if (cache_ops_need_broadcast()) 153 smp_call_function(flush_ptrace_access_other, 154 NULL, 1); 155 } 156 } 157 158 static 159 void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, 160 unsigned long uaddr, void *kaddr, unsigned long len) 161 { 162 unsigned int flags = 0; 163 if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) 164 flags |= FLAG_PA_CORE_IN_MM; 165 if (vma->vm_flags & VM_EXEC) 166 flags |= FLAG_PA_IS_EXEC; 167 __flush_ptrace_access(page, uaddr, kaddr, len, flags); 168 } 169 170 void flush_uprobe_xol_access(struct page *page, unsigned long uaddr, 171 void *kaddr, unsigned long len) 172 { 173 unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC; 174 175 __flush_ptrace_access(page, uaddr, kaddr, len, flags); 176 } 177 178 /* 179 * Copy user data from/to a page which is mapped into a different 180 * processes address space. Really, we want to allow our "user 181 * space" model to handle this. 182 * 183 * Note that this code needs to run on the current CPU. 184 */ 185 void copy_to_user_page(struct vm_area_struct *vma, struct page *page, 186 unsigned long uaddr, void *dst, const void *src, 187 unsigned long len) 188 { 189 #ifdef CONFIG_SMP 190 preempt_disable(); 191 #endif 192 memcpy(dst, src, len); 193 flush_ptrace_access(vma, page, uaddr, dst, len); 194 #ifdef CONFIG_SMP 195 preempt_enable(); 196 #endif 197 } 198 199 void __flush_dcache_page(struct address_space *mapping, struct page *page) 200 { 201 /* 202 * Writeback any data associated with the kernel mapping of this 203 * page. This ensures that data in the physical page is mutually 204 * coherent with the kernels mapping. 205 */ 206 if (!PageHighMem(page)) { 207 size_t page_size = PAGE_SIZE << compound_order(page); 208 __cpuc_flush_dcache_area(page_address(page), page_size); 209 } else { 210 unsigned long i; 211 if (cache_is_vipt_nonaliasing()) { 212 for (i = 0; i < (1 << compound_order(page)); i++) { 213 void *addr = kmap_atomic(page + i); 214 __cpuc_flush_dcache_area(addr, PAGE_SIZE); 215 kunmap_atomic(addr); 216 } 217 } else { 218 for (i = 0; i < (1 << compound_order(page)); i++) { 219 void *addr = kmap_high_get(page + i); 220 if (addr) { 221 __cpuc_flush_dcache_area(addr, PAGE_SIZE); 222 kunmap_high(page + i); 223 } 224 } 225 } 226 } 227 228 /* 229 * If this is a page cache page, and we have an aliasing VIPT cache, 230 * we only need to do one flush - which would be at the relevant 231 * userspace colour, which is congruent with page->index. 232 */ 233 if (mapping && cache_is_vipt_aliasing()) 234 flush_pfn_alias(page_to_pfn(page), 235 page->index << PAGE_SHIFT); 236 } 237 238 static void __flush_dcache_aliases(struct address_space *mapping, struct page *page) 239 { 240 struct mm_struct *mm = current->active_mm; 241 struct vm_area_struct *mpnt; 242 pgoff_t pgoff; 243 244 /* 245 * There are possible user space mappings of this page: 246 * - VIVT cache: we need to also write back and invalidate all user 247 * data in the current VM view associated with this page. 248 * - aliasing VIPT: we only need to find one mapping of this page. 249 */ 250 pgoff = page->index; 251 252 flush_dcache_mmap_lock(mapping); 253 vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) { 254 unsigned long offset; 255 256 /* 257 * If this VMA is not in our MM, we can ignore it. 258 */ 259 if (mpnt->vm_mm != mm) 260 continue; 261 if (!(mpnt->vm_flags & VM_MAYSHARE)) 262 continue; 263 offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT; 264 flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page)); 265 } 266 flush_dcache_mmap_unlock(mapping); 267 } 268 269 #if __LINUX_ARM_ARCH__ >= 6 270 void __sync_icache_dcache(pte_t pteval) 271 { 272 unsigned long pfn; 273 struct page *page; 274 struct address_space *mapping; 275 276 if (cache_is_vipt_nonaliasing() && !pte_exec(pteval)) 277 /* only flush non-aliasing VIPT caches for exec mappings */ 278 return; 279 pfn = pte_pfn(pteval); 280 if (!pfn_valid(pfn)) 281 return; 282 283 page = pfn_to_page(pfn); 284 if (cache_is_vipt_aliasing()) 285 mapping = page_mapping_file(page); 286 else 287 mapping = NULL; 288 289 if (!test_and_set_bit(PG_dcache_clean, &page->flags)) 290 __flush_dcache_page(mapping, page); 291 292 if (pte_exec(pteval)) 293 __flush_icache_all(); 294 } 295 #endif 296 297 /* 298 * Ensure cache coherency between kernel mapping and userspace mapping 299 * of this page. 300 * 301 * We have three cases to consider: 302 * - VIPT non-aliasing cache: fully coherent so nothing required. 303 * - VIVT: fully aliasing, so we need to handle every alias in our 304 * current VM view. 305 * - VIPT aliasing: need to handle one alias in our current VM view. 306 * 307 * If we need to handle aliasing: 308 * If the page only exists in the page cache and there are no user 309 * space mappings, we can be lazy and remember that we may have dirty 310 * kernel cache lines for later. Otherwise, we assume we have 311 * aliasing mappings. 312 * 313 * Note that we disable the lazy flush for SMP configurations where 314 * the cache maintenance operations are not automatically broadcasted. 315 */ 316 void flush_dcache_page(struct page *page) 317 { 318 struct address_space *mapping; 319 320 /* 321 * The zero page is never written to, so never has any dirty 322 * cache lines, and therefore never needs to be flushed. 323 */ 324 if (page == ZERO_PAGE(0)) 325 return; 326 327 if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) { 328 if (test_bit(PG_dcache_clean, &page->flags)) 329 clear_bit(PG_dcache_clean, &page->flags); 330 return; 331 } 332 333 mapping = page_mapping_file(page); 334 335 if (!cache_ops_need_broadcast() && 336 mapping && !page_mapcount(page)) 337 clear_bit(PG_dcache_clean, &page->flags); 338 else { 339 __flush_dcache_page(mapping, page); 340 if (mapping && cache_is_vivt()) 341 __flush_dcache_aliases(mapping, page); 342 else if (mapping) 343 __flush_icache_all(); 344 set_bit(PG_dcache_clean, &page->flags); 345 } 346 } 347 EXPORT_SYMBOL(flush_dcache_page); 348 349 /* 350 * Ensure cache coherency for the kernel mapping of this page. We can 351 * assume that the page is pinned via kmap. 352 * 353 * If the page only exists in the page cache and there are no user 354 * space mappings, this is a no-op since the page was already marked 355 * dirty at creation. Otherwise, we need to flush the dirty kernel 356 * cache lines directly. 357 */ 358 void flush_kernel_dcache_page(struct page *page) 359 { 360 if (cache_is_vivt() || cache_is_vipt_aliasing()) { 361 struct address_space *mapping; 362 363 mapping = page_mapping_file(page); 364 365 if (!mapping || mapping_mapped(mapping)) { 366 void *addr; 367 368 addr = page_address(page); 369 /* 370 * kmap_atomic() doesn't set the page virtual 371 * address for highmem pages, and 372 * kunmap_atomic() takes care of cache 373 * flushing already. 374 */ 375 if (!IS_ENABLED(CONFIG_HIGHMEM) || addr) 376 __cpuc_flush_dcache_area(addr, PAGE_SIZE); 377 } 378 } 379 } 380 EXPORT_SYMBOL(flush_kernel_dcache_page); 381 382 /* 383 * Flush an anonymous page so that users of get_user_pages() 384 * can safely access the data. The expected sequence is: 385 * 386 * get_user_pages() 387 * -> flush_anon_page 388 * memcpy() to/from page 389 * if written to page, flush_dcache_page() 390 */ 391 void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr) 392 { 393 unsigned long pfn; 394 395 /* VIPT non-aliasing caches need do nothing */ 396 if (cache_is_vipt_nonaliasing()) 397 return; 398 399 /* 400 * Write back and invalidate userspace mapping. 401 */ 402 pfn = page_to_pfn(page); 403 if (cache_is_vivt()) { 404 flush_cache_page(vma, vmaddr, pfn); 405 } else { 406 /* 407 * For aliasing VIPT, we can flush an alias of the 408 * userspace address only. 409 */ 410 flush_pfn_alias(pfn, vmaddr); 411 __flush_icache_all(); 412 } 413 414 /* 415 * Invalidate kernel mapping. No data should be contained 416 * in this mapping of the page. FIXME: this is overkill 417 * since we actually ask for a write-back and invalidate. 418 */ 419 __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); 420 } 421