1/* 2 * linux/arch/arm/mm/cache-v7.S 3 * 4 * Copyright (C) 2001 Deep Blue Solutions Ltd. 5 * Copyright (C) 2005 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This is the "shell" of the ARMv7 processor support. 12 */ 13#include <linux/linkage.h> 14#include <linux/init.h> 15#include <asm/assembler.h> 16#include <asm/errno.h> 17#include <asm/unwind.h> 18 19#include "proc-macros.S" 20 21/* 22 * v7_flush_icache_all() 23 * 24 * Flush the whole I-cache. 25 * 26 * Registers: 27 * r0 - set to 0 28 */ 29ENTRY(v7_flush_icache_all) 30 mov r0, #0 31 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable 32 ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate 33 mov pc, lr 34ENDPROC(v7_flush_icache_all) 35 36 /* 37 * v7_flush_dcache_louis() 38 * 39 * Flush the D-cache up to the Level of Unification Inner Shareable 40 * 41 * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) 42 */ 43 44ENTRY(v7_flush_dcache_louis) 45 dmb @ ensure ordering with previous memory accesses 46 mrc p15, 1, r0, c0, c0, 1 @ read clidr, r0 = clidr 47 ALT_SMP(ands r3, r0, #(7 << 21)) @ extract LoUIS from clidr 48 ALT_UP(ands r3, r0, #(7 << 27)) @ extract LoUU from clidr 49 ALT_SMP(mov r3, r3, lsr #20) @ r3 = LoUIS * 2 50 ALT_UP(mov r3, r3, lsr #26) @ r3 = LoUU * 2 51 moveq pc, lr @ return if level == 0 52 mov r10, #0 @ r10 (starting level) = 0 53 b flush_levels @ start flushing cache levels 54ENDPROC(v7_flush_dcache_louis) 55 56/* 57 * v7_flush_dcache_all() 58 * 59 * Flush the whole D-cache. 60 * 61 * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode) 62 * 63 * - mm - mm_struct describing address space 64 */ 65ENTRY(v7_flush_dcache_all) 66 dmb @ ensure ordering with previous memory accesses 67 mrc p15, 1, r0, c0, c0, 1 @ read clidr 68 ands r3, r0, #0x7000000 @ extract loc from clidr 69 mov r3, r3, lsr #23 @ left align loc bit field 70 beq finished @ if loc is 0, then no need to clean 71 mov r10, #0 @ start clean at cache level 0 72flush_levels: 73 add r2, r10, r10, lsr #1 @ work out 3x current cache level 74 mov r1, r0, lsr r2 @ extract cache type bits from clidr 75 and r1, r1, #7 @ mask of the bits for current cache only 76 cmp r1, #2 @ see what cache we have at this level 77 blt skip @ skip if no cache, or just i-cache 78#ifdef CONFIG_PREEMPT 79 save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic 80#endif 81 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 82 isb @ isb to sych the new cssr&csidr 83 mrc p15, 1, r1, c0, c0, 0 @ read the new csidr 84#ifdef CONFIG_PREEMPT 85 restore_irqs_notrace r9 86#endif 87 and r2, r1, #7 @ extract the length of the cache lines 88 add r2, r2, #4 @ add 4 (line length offset) 89 ldr r4, =0x3ff 90 ands r4, r4, r1, lsr #3 @ find maximum number on the way size 91 clz r5, r4 @ find bit position of way size increment 92 ldr r7, =0x7fff 93 ands r7, r7, r1, lsr #13 @ extract max number of the index size 94loop1: 95 mov r9, r4 @ create working copy of max way size 96loop2: 97 ARM( orr r11, r10, r9, lsl r5 ) @ factor way and cache number into r11 98 THUMB( lsl r6, r9, r5 ) 99 THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11 100 ARM( orr r11, r11, r7, lsl r2 ) @ factor index number into r11 101 THUMB( lsl r6, r7, r2 ) 102 THUMB( orr r11, r11, r6 ) @ factor index number into r11 103 mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way 104 subs r9, r9, #1 @ decrement the way 105 bge loop2 106 subs r7, r7, #1 @ decrement the index 107 bge loop1 108skip: 109 add r10, r10, #2 @ increment cache number 110 cmp r3, r10 111 bgt flush_levels 112finished: 113 mov r10, #0 @ swith back to cache level 0 114 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 115 dsb 116 isb 117 mov pc, lr 118ENDPROC(v7_flush_dcache_all) 119 120/* 121 * v7_flush_cache_all() 122 * 123 * Flush the entire cache system. 124 * The data cache flush is now achieved using atomic clean / invalidates 125 * working outwards from L1 cache. This is done using Set/Way based cache 126 * maintenance instructions. 127 * The instruction cache can still be invalidated back to the point of 128 * unification in a single instruction. 129 * 130 */ 131ENTRY(v7_flush_kern_cache_all) 132 ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) 133 THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) 134 bl v7_flush_dcache_all 135 mov r0, #0 136 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable 137 ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate 138 ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) 139 THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) 140 mov pc, lr 141ENDPROC(v7_flush_kern_cache_all) 142 143 /* 144 * v7_flush_kern_cache_louis(void) 145 * 146 * Flush the data cache up to Level of Unification Inner Shareable. 147 * Invalidate the I-cache to the point of unification. 148 */ 149ENTRY(v7_flush_kern_cache_louis) 150 ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} ) 151 THUMB( stmfd sp!, {r4-r7, r9-r11, lr} ) 152 bl v7_flush_dcache_louis 153 mov r0, #0 154 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable 155 ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate 156 ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} ) 157 THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} ) 158 mov pc, lr 159ENDPROC(v7_flush_kern_cache_louis) 160 161/* 162 * v7_flush_cache_all() 163 * 164 * Flush all TLB entries in a particular address space 165 * 166 * - mm - mm_struct describing address space 167 */ 168ENTRY(v7_flush_user_cache_all) 169 /*FALLTHROUGH*/ 170 171/* 172 * v7_flush_cache_range(start, end, flags) 173 * 174 * Flush a range of TLB entries in the specified address space. 175 * 176 * - start - start address (may not be aligned) 177 * - end - end address (exclusive, may not be aligned) 178 * - flags - vm_area_struct flags describing address space 179 * 180 * It is assumed that: 181 * - we have a VIPT cache. 182 */ 183ENTRY(v7_flush_user_cache_range) 184 mov pc, lr 185ENDPROC(v7_flush_user_cache_all) 186ENDPROC(v7_flush_user_cache_range) 187 188/* 189 * v7_coherent_kern_range(start,end) 190 * 191 * Ensure that the I and D caches are coherent within specified 192 * region. This is typically used when code has been written to 193 * a memory region, and will be executed. 194 * 195 * - start - virtual start address of region 196 * - end - virtual end address of region 197 * 198 * It is assumed that: 199 * - the Icache does not read data from the write buffer 200 */ 201ENTRY(v7_coherent_kern_range) 202 /* FALLTHROUGH */ 203 204/* 205 * v7_coherent_user_range(start,end) 206 * 207 * Ensure that the I and D caches are coherent within specified 208 * region. This is typically used when code has been written to 209 * a memory region, and will be executed. 210 * 211 * - start - virtual start address of region 212 * - end - virtual end address of region 213 * 214 * It is assumed that: 215 * - the Icache does not read data from the write buffer 216 */ 217ENTRY(v7_coherent_user_range) 218 UNWIND(.fnstart ) 219 dcache_line_size r2, r3 220 sub r3, r2, #1 221 bic r12, r0, r3 222#ifdef CONFIG_ARM_ERRATA_764369 223 ALT_SMP(W(dsb)) 224 ALT_UP(W(nop)) 225#endif 2261: 227 USER( mcr p15, 0, r12, c7, c11, 1 ) @ clean D line to the point of unification 228 add r12, r12, r2 229 cmp r12, r1 230 blo 1b 231 dsb 232 icache_line_size r2, r3 233 sub r3, r2, #1 234 bic r12, r0, r3 2352: 236 USER( mcr p15, 0, r12, c7, c5, 1 ) @ invalidate I line 237 add r12, r12, r2 238 cmp r12, r1 239 blo 2b 240 mov r0, #0 241 ALT_SMP(mcr p15, 0, r0, c7, c1, 6) @ invalidate BTB Inner Shareable 242 ALT_UP(mcr p15, 0, r0, c7, c5, 6) @ invalidate BTB 243 dsb 244 isb 245 mov pc, lr 246 247/* 248 * Fault handling for the cache operation above. If the virtual address in r0 249 * isn't mapped, fail with -EFAULT. 250 */ 2519001: 252#ifdef CONFIG_ARM_ERRATA_775420 253 dsb 254#endif 255 mov r0, #-EFAULT 256 mov pc, lr 257 UNWIND(.fnend ) 258ENDPROC(v7_coherent_kern_range) 259ENDPROC(v7_coherent_user_range) 260 261/* 262 * v7_flush_kern_dcache_area(void *addr, size_t size) 263 * 264 * Ensure that the data held in the page kaddr is written back 265 * to the page in question. 266 * 267 * - addr - kernel address 268 * - size - region size 269 */ 270ENTRY(v7_flush_kern_dcache_area) 271 dcache_line_size r2, r3 272 add r1, r0, r1 273 sub r3, r2, #1 274 bic r0, r0, r3 275#ifdef CONFIG_ARM_ERRATA_764369 276 ALT_SMP(W(dsb)) 277 ALT_UP(W(nop)) 278#endif 2791: 280 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line 281 add r0, r0, r2 282 cmp r0, r1 283 blo 1b 284 dsb 285 mov pc, lr 286ENDPROC(v7_flush_kern_dcache_area) 287 288/* 289 * v7_dma_inv_range(start,end) 290 * 291 * Invalidate the data cache within the specified region; we will 292 * be performing a DMA operation in this region and we want to 293 * purge old data in the cache. 294 * 295 * - start - virtual start address of region 296 * - end - virtual end address of region 297 */ 298v7_dma_inv_range: 299 dcache_line_size r2, r3 300 sub r3, r2, #1 301 tst r0, r3 302 bic r0, r0, r3 303#ifdef CONFIG_ARM_ERRATA_764369 304 ALT_SMP(W(dsb)) 305 ALT_UP(W(nop)) 306#endif 307 mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line 308 309 tst r1, r3 310 bic r1, r1, r3 311 mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D / U line 3121: 313 mcr p15, 0, r0, c7, c6, 1 @ invalidate D / U line 314 add r0, r0, r2 315 cmp r0, r1 316 blo 1b 317 dsb 318 mov pc, lr 319ENDPROC(v7_dma_inv_range) 320 321/* 322 * v7_dma_clean_range(start,end) 323 * - start - virtual start address of region 324 * - end - virtual end address of region 325 */ 326v7_dma_clean_range: 327 dcache_line_size r2, r3 328 sub r3, r2, #1 329 bic r0, r0, r3 330#ifdef CONFIG_ARM_ERRATA_764369 331 ALT_SMP(W(dsb)) 332 ALT_UP(W(nop)) 333#endif 3341: 335 mcr p15, 0, r0, c7, c10, 1 @ clean D / U line 336 add r0, r0, r2 337 cmp r0, r1 338 blo 1b 339 dsb 340 mov pc, lr 341ENDPROC(v7_dma_clean_range) 342 343/* 344 * v7_dma_flush_range(start,end) 345 * - start - virtual start address of region 346 * - end - virtual end address of region 347 */ 348ENTRY(v7_dma_flush_range) 349 dcache_line_size r2, r3 350 sub r3, r2, #1 351 bic r0, r0, r3 352#ifdef CONFIG_ARM_ERRATA_764369 353 ALT_SMP(W(dsb)) 354 ALT_UP(W(nop)) 355#endif 3561: 357 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line 358 add r0, r0, r2 359 cmp r0, r1 360 blo 1b 361 dsb 362 mov pc, lr 363ENDPROC(v7_dma_flush_range) 364 365/* 366 * dma_map_area(start, size, dir) 367 * - start - kernel virtual start address 368 * - size - size of region 369 * - dir - DMA direction 370 */ 371ENTRY(v7_dma_map_area) 372 add r1, r1, r0 373 teq r2, #DMA_FROM_DEVICE 374 beq v7_dma_inv_range 375 b v7_dma_clean_range 376ENDPROC(v7_dma_map_area) 377 378/* 379 * dma_unmap_area(start, size, dir) 380 * - start - kernel virtual start address 381 * - size - size of region 382 * - dir - DMA direction 383 */ 384ENTRY(v7_dma_unmap_area) 385 add r1, r1, r0 386 teq r2, #DMA_TO_DEVICE 387 bne v7_dma_inv_range 388 mov pc, lr 389ENDPROC(v7_dma_unmap_area) 390 391 __INITDATA 392 393 @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) 394 define_cache_functions v7 395