1 /* 2 * arch/arm/include/asm/tlbflush.h 3 * 4 * Copyright (C) 1999-2003 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #ifndef _ASMARM_TLBFLUSH_H 11 #define _ASMARM_TLBFLUSH_H 12 13 14 #ifndef CONFIG_MMU 15 16 #define tlb_flush(tlb) ((void) tlb) 17 18 #else /* CONFIG_MMU */ 19 20 #include <asm/glue.h> 21 22 #define TLB_V3_PAGE (1 << 0) 23 #define TLB_V4_U_PAGE (1 << 1) 24 #define TLB_V4_D_PAGE (1 << 2) 25 #define TLB_V4_I_PAGE (1 << 3) 26 #define TLB_V6_U_PAGE (1 << 4) 27 #define TLB_V6_D_PAGE (1 << 5) 28 #define TLB_V6_I_PAGE (1 << 6) 29 30 #define TLB_V3_FULL (1 << 8) 31 #define TLB_V4_U_FULL (1 << 9) 32 #define TLB_V4_D_FULL (1 << 10) 33 #define TLB_V4_I_FULL (1 << 11) 34 #define TLB_V6_U_FULL (1 << 12) 35 #define TLB_V6_D_FULL (1 << 13) 36 #define TLB_V6_I_FULL (1 << 14) 37 38 #define TLB_V6_U_ASID (1 << 16) 39 #define TLB_V6_D_ASID (1 << 17) 40 #define TLB_V6_I_ASID (1 << 18) 41 42 #define TLB_L2CLEAN_FR (1 << 29) /* Feroceon */ 43 #define TLB_DCLEAN (1 << 30) 44 #define TLB_WB (1 << 31) 45 46 /* 47 * MMU TLB Model 48 * ============= 49 * 50 * We have the following to choose from: 51 * v3 - ARMv3 52 * v4 - ARMv4 without write buffer 53 * v4wb - ARMv4 with write buffer without I TLB flush entry instruction 54 * v4wbi - ARMv4 with write buffer with I TLB flush entry instruction 55 * fr - Feroceon (v4wbi with non-outer-cacheable page table walks) 56 * v6wbi - ARMv6 with write buffer with I TLB flush entry instruction 57 * v7wbi - identical to v6wbi 58 */ 59 #undef _TLB 60 #undef MULTI_TLB 61 62 #define v3_tlb_flags (TLB_V3_FULL | TLB_V3_PAGE) 63 64 #ifdef CONFIG_CPU_TLB_V3 65 # define v3_possible_flags v3_tlb_flags 66 # define v3_always_flags v3_tlb_flags 67 # ifdef _TLB 68 # define MULTI_TLB 1 69 # else 70 # define _TLB v3 71 # endif 72 #else 73 # define v3_possible_flags 0 74 # define v3_always_flags (-1UL) 75 #endif 76 77 #define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE) 78 79 #ifdef CONFIG_CPU_TLB_V4WT 80 # define v4_possible_flags v4_tlb_flags 81 # define v4_always_flags v4_tlb_flags 82 # ifdef _TLB 83 # define MULTI_TLB 1 84 # else 85 # define _TLB v4 86 # endif 87 #else 88 # define v4_possible_flags 0 89 # define v4_always_flags (-1UL) 90 #endif 91 92 #define v4wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \ 93 TLB_V4_I_FULL | TLB_V4_D_FULL | \ 94 TLB_V4_I_PAGE | TLB_V4_D_PAGE) 95 96 #ifdef CONFIG_CPU_TLB_V4WBI 97 # define v4wbi_possible_flags v4wbi_tlb_flags 98 # define v4wbi_always_flags v4wbi_tlb_flags 99 # ifdef _TLB 100 # define MULTI_TLB 1 101 # else 102 # define _TLB v4wbi 103 # endif 104 #else 105 # define v4wbi_possible_flags 0 106 # define v4wbi_always_flags (-1UL) 107 #endif 108 109 #define fr_tlb_flags (TLB_WB | TLB_DCLEAN | TLB_L2CLEAN_FR | \ 110 TLB_V4_I_FULL | TLB_V4_D_FULL | \ 111 TLB_V4_I_PAGE | TLB_V4_D_PAGE) 112 113 #ifdef CONFIG_CPU_TLB_FEROCEON 114 # define fr_possible_flags fr_tlb_flags 115 # define fr_always_flags fr_tlb_flags 116 # ifdef _TLB 117 # define MULTI_TLB 1 118 # else 119 # define _TLB v4wbi 120 # endif 121 #else 122 # define fr_possible_flags 0 123 # define fr_always_flags (-1UL) 124 #endif 125 126 #define v4wb_tlb_flags (TLB_WB | TLB_DCLEAN | \ 127 TLB_V4_I_FULL | TLB_V4_D_FULL | \ 128 TLB_V4_D_PAGE) 129 130 #ifdef CONFIG_CPU_TLB_V4WB 131 # define v4wb_possible_flags v4wb_tlb_flags 132 # define v4wb_always_flags v4wb_tlb_flags 133 # ifdef _TLB 134 # define MULTI_TLB 1 135 # else 136 # define _TLB v4wb 137 # endif 138 #else 139 # define v4wb_possible_flags 0 140 # define v4wb_always_flags (-1UL) 141 #endif 142 143 #define v6wbi_tlb_flags (TLB_WB | TLB_DCLEAN | \ 144 TLB_V6_I_FULL | TLB_V6_D_FULL | \ 145 TLB_V6_I_PAGE | TLB_V6_D_PAGE | \ 146 TLB_V6_I_ASID | TLB_V6_D_ASID) 147 148 #ifdef CONFIG_CPU_TLB_V6 149 # define v6wbi_possible_flags v6wbi_tlb_flags 150 # define v6wbi_always_flags v6wbi_tlb_flags 151 # ifdef _TLB 152 # define MULTI_TLB 1 153 # else 154 # define _TLB v6wbi 155 # endif 156 #else 157 # define v6wbi_possible_flags 0 158 # define v6wbi_always_flags (-1UL) 159 #endif 160 161 #ifdef CONFIG_CPU_TLB_V7 162 # define v7wbi_possible_flags v6wbi_tlb_flags 163 # define v7wbi_always_flags v6wbi_tlb_flags 164 # ifdef _TLB 165 # define MULTI_TLB 1 166 # else 167 # define _TLB v7wbi 168 # endif 169 #else 170 # define v7wbi_possible_flags 0 171 # define v7wbi_always_flags (-1UL) 172 #endif 173 174 #ifndef _TLB 175 #error Unknown TLB model 176 #endif 177 178 #ifndef __ASSEMBLY__ 179 180 #include <linux/sched.h> 181 182 struct cpu_tlb_fns { 183 void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *); 184 void (*flush_kern_range)(unsigned long, unsigned long); 185 unsigned long tlb_flags; 186 }; 187 188 /* 189 * Select the calling method 190 */ 191 #ifdef MULTI_TLB 192 193 #define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range 194 #define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range 195 196 #else 197 198 #define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range) 199 #define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range) 200 201 extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); 202 extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long); 203 204 #endif 205 206 extern struct cpu_tlb_fns cpu_tlb; 207 208 #define __cpu_tlb_flags cpu_tlb.tlb_flags 209 210 /* 211 * TLB Management 212 * ============== 213 * 214 * The arch/arm/mm/tlb-*.S files implement these methods. 215 * 216 * The TLB specific code is expected to perform whatever tests it 217 * needs to determine if it should invalidate the TLB for each 218 * call. Start addresses are inclusive and end addresses are 219 * exclusive; it is safe to round these addresses down. 220 * 221 * flush_tlb_all() 222 * 223 * Invalidate the entire TLB. 224 * 225 * flush_tlb_mm(mm) 226 * 227 * Invalidate all TLB entries in a particular address 228 * space. 229 * - mm - mm_struct describing address space 230 * 231 * flush_tlb_range(mm,start,end) 232 * 233 * Invalidate a range of TLB entries in the specified 234 * address space. 235 * - mm - mm_struct describing address space 236 * - start - start address (may not be aligned) 237 * - end - end address (exclusive, may not be aligned) 238 * 239 * flush_tlb_page(vaddr,vma) 240 * 241 * Invalidate the specified page in the specified address range. 242 * - vaddr - virtual address (may not be aligned) 243 * - vma - vma_struct describing address range 244 * 245 * flush_kern_tlb_page(kaddr) 246 * 247 * Invalidate the TLB entry for the specified page. The address 248 * will be in the kernels virtual memory space. Current uses 249 * only require the D-TLB to be invalidated. 250 * - kaddr - Kernel virtual memory address 251 */ 252 253 /* 254 * We optimise the code below by: 255 * - building a set of TLB flags that might be set in __cpu_tlb_flags 256 * - building a set of TLB flags that will always be set in __cpu_tlb_flags 257 * - if we're going to need __cpu_tlb_flags, access it once and only once 258 * 259 * This allows us to build optimal assembly for the single-CPU type case, 260 * and as close to optimal given the compiler constrants for multi-CPU 261 * case. We could do better for the multi-CPU case if the compiler 262 * implemented the "%?" method, but this has been discontinued due to too 263 * many people getting it wrong. 264 */ 265 #define possible_tlb_flags (v3_possible_flags | \ 266 v4_possible_flags | \ 267 v4wbi_possible_flags | \ 268 fr_possible_flags | \ 269 v4wb_possible_flags | \ 270 v6wbi_possible_flags | \ 271 v7wbi_possible_flags) 272 273 #define always_tlb_flags (v3_always_flags & \ 274 v4_always_flags & \ 275 v4wbi_always_flags & \ 276 fr_always_flags & \ 277 v4wb_always_flags & \ 278 v6wbi_always_flags & \ 279 v7wbi_always_flags) 280 281 #define tlb_flag(f) ((always_tlb_flags & (f)) || (__tlb_flag & possible_tlb_flags & (f))) 282 283 static inline void local_flush_tlb_all(void) 284 { 285 const int zero = 0; 286 const unsigned int __tlb_flag = __cpu_tlb_flags; 287 288 if (tlb_flag(TLB_WB)) 289 dsb(); 290 291 if (tlb_flag(TLB_V3_FULL)) 292 asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); 293 if (tlb_flag(TLB_V4_U_FULL | TLB_V6_U_FULL)) 294 asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); 295 if (tlb_flag(TLB_V4_D_FULL | TLB_V6_D_FULL)) 296 asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); 297 if (tlb_flag(TLB_V4_I_FULL | TLB_V6_I_FULL)) 298 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); 299 300 if (tlb_flag(TLB_V6_I_FULL | TLB_V6_D_FULL | 301 TLB_V6_I_PAGE | TLB_V6_D_PAGE | 302 TLB_V6_I_ASID | TLB_V6_D_ASID)) { 303 /* flush the branch target cache */ 304 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); 305 dsb(); 306 isb(); 307 } 308 } 309 310 static inline void local_flush_tlb_mm(struct mm_struct *mm) 311 { 312 const int zero = 0; 313 const int asid = ASID(mm); 314 const unsigned int __tlb_flag = __cpu_tlb_flags; 315 316 if (tlb_flag(TLB_WB)) 317 dsb(); 318 319 if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) { 320 if (tlb_flag(TLB_V3_FULL)) 321 asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); 322 if (tlb_flag(TLB_V4_U_FULL)) 323 asm("mcr p15, 0, %0, c8, c7, 0" : : "r" (zero) : "cc"); 324 if (tlb_flag(TLB_V4_D_FULL)) 325 asm("mcr p15, 0, %0, c8, c6, 0" : : "r" (zero) : "cc"); 326 if (tlb_flag(TLB_V4_I_FULL)) 327 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); 328 } 329 330 if (tlb_flag(TLB_V6_U_ASID)) 331 asm("mcr p15, 0, %0, c8, c7, 2" : : "r" (asid) : "cc"); 332 if (tlb_flag(TLB_V6_D_ASID)) 333 asm("mcr p15, 0, %0, c8, c6, 2" : : "r" (asid) : "cc"); 334 if (tlb_flag(TLB_V6_I_ASID)) 335 asm("mcr p15, 0, %0, c8, c5, 2" : : "r" (asid) : "cc"); 336 337 if (tlb_flag(TLB_V6_I_FULL | TLB_V6_D_FULL | 338 TLB_V6_I_PAGE | TLB_V6_D_PAGE | 339 TLB_V6_I_ASID | TLB_V6_D_ASID)) { 340 /* flush the branch target cache */ 341 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); 342 dsb(); 343 } 344 } 345 346 static inline void 347 local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) 348 { 349 const int zero = 0; 350 const unsigned int __tlb_flag = __cpu_tlb_flags; 351 352 uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); 353 354 if (tlb_flag(TLB_WB)) 355 dsb(); 356 357 if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) { 358 if (tlb_flag(TLB_V3_PAGE)) 359 asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr) : "cc"); 360 if (tlb_flag(TLB_V4_U_PAGE)) 361 asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); 362 if (tlb_flag(TLB_V4_D_PAGE)) 363 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); 364 if (tlb_flag(TLB_V4_I_PAGE)) 365 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); 366 if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) 367 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); 368 } 369 370 if (tlb_flag(TLB_V6_U_PAGE)) 371 asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (uaddr) : "cc"); 372 if (tlb_flag(TLB_V6_D_PAGE)) 373 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (uaddr) : "cc"); 374 if (tlb_flag(TLB_V6_I_PAGE)) 375 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (uaddr) : "cc"); 376 377 if (tlb_flag(TLB_V6_I_FULL | TLB_V6_D_FULL | 378 TLB_V6_I_PAGE | TLB_V6_D_PAGE | 379 TLB_V6_I_ASID | TLB_V6_D_ASID)) { 380 /* flush the branch target cache */ 381 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); 382 dsb(); 383 } 384 } 385 386 static inline void local_flush_tlb_kernel_page(unsigned long kaddr) 387 { 388 const int zero = 0; 389 const unsigned int __tlb_flag = __cpu_tlb_flags; 390 391 kaddr &= PAGE_MASK; 392 393 if (tlb_flag(TLB_WB)) 394 dsb(); 395 396 if (tlb_flag(TLB_V3_PAGE)) 397 asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (kaddr) : "cc"); 398 if (tlb_flag(TLB_V4_U_PAGE)) 399 asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); 400 if (tlb_flag(TLB_V4_D_PAGE)) 401 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); 402 if (tlb_flag(TLB_V4_I_PAGE)) 403 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); 404 if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL)) 405 asm("mcr p15, 0, %0, c8, c5, 0" : : "r" (zero) : "cc"); 406 407 if (tlb_flag(TLB_V6_U_PAGE)) 408 asm("mcr p15, 0, %0, c8, c7, 1" : : "r" (kaddr) : "cc"); 409 if (tlb_flag(TLB_V6_D_PAGE)) 410 asm("mcr p15, 0, %0, c8, c6, 1" : : "r" (kaddr) : "cc"); 411 if (tlb_flag(TLB_V6_I_PAGE)) 412 asm("mcr p15, 0, %0, c8, c5, 1" : : "r" (kaddr) : "cc"); 413 414 if (tlb_flag(TLB_V6_I_FULL | TLB_V6_D_FULL | 415 TLB_V6_I_PAGE | TLB_V6_D_PAGE | 416 TLB_V6_I_ASID | TLB_V6_D_ASID)) { 417 /* flush the branch target cache */ 418 asm("mcr p15, 0, %0, c7, c5, 6" : : "r" (zero) : "cc"); 419 dsb(); 420 isb(); 421 } 422 } 423 424 /* 425 * flush_pmd_entry 426 * 427 * Flush a PMD entry (word aligned, or double-word aligned) to 428 * RAM if the TLB for the CPU we are running on requires this. 429 * This is typically used when we are creating PMD entries. 430 * 431 * clean_pmd_entry 432 * 433 * Clean (but don't drain the write buffer) if the CPU requires 434 * these operations. This is typically used when we are removing 435 * PMD entries. 436 */ 437 static inline void flush_pmd_entry(pmd_t *pmd) 438 { 439 const unsigned int __tlb_flag = __cpu_tlb_flags; 440 441 if (tlb_flag(TLB_DCLEAN)) 442 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" 443 : : "r" (pmd) : "cc"); 444 445 if (tlb_flag(TLB_L2CLEAN_FR)) 446 asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd" 447 : : "r" (pmd) : "cc"); 448 449 if (tlb_flag(TLB_WB)) 450 dsb(); 451 } 452 453 static inline void clean_pmd_entry(pmd_t *pmd) 454 { 455 const unsigned int __tlb_flag = __cpu_tlb_flags; 456 457 if (tlb_flag(TLB_DCLEAN)) 458 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pmd" 459 : : "r" (pmd) : "cc"); 460 461 if (tlb_flag(TLB_L2CLEAN_FR)) 462 asm("mcr p15, 1, %0, c15, c9, 1 @ L2 flush_pmd" 463 : : "r" (pmd) : "cc"); 464 } 465 466 #undef tlb_flag 467 #undef always_tlb_flags 468 #undef possible_tlb_flags 469 470 /* 471 * Convert calls to our calling convention. 472 */ 473 #define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma) 474 #define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e) 475 476 #ifndef CONFIG_SMP 477 #define flush_tlb_all local_flush_tlb_all 478 #define flush_tlb_mm local_flush_tlb_mm 479 #define flush_tlb_page local_flush_tlb_page 480 #define flush_tlb_kernel_page local_flush_tlb_kernel_page 481 #define flush_tlb_range local_flush_tlb_range 482 #define flush_tlb_kernel_range local_flush_tlb_kernel_range 483 #else 484 extern void flush_tlb_all(void); 485 extern void flush_tlb_mm(struct mm_struct *mm); 486 extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr); 487 extern void flush_tlb_kernel_page(unsigned long kaddr); 488 extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); 489 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); 490 #endif 491 492 /* 493 * if PG_dcache_dirty is set for the page, we need to ensure that any 494 * cache entries for the kernels virtual memory range are written 495 * back to the page. 496 */ 497 extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte); 498 499 #endif 500 501 #endif /* CONFIG_MMU */ 502 503 #endif 504