1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * linux/arch/arm/mm/cache-v7m.S 4 * 5 * Based on linux/arch/arm/mm/cache-v7.S 6 * 7 * Copyright (C) 2001 Deep Blue Solutions Ltd. 8 * Copyright (C) 2005 ARM Ltd. 9 * 10 * This is the "shell" of the ARMv7M processor support. 11 */ 12 #include <linux/linkage.h> 13 #include <linux/init.h> 14 #include <asm/assembler.h> 15 #include <asm/errno.h> 16 #include <asm/unwind.h> 17 #include <asm/v7m.h> 18 19 #include "proc-macros.S" 20 21 .arch armv7-m 22 23 /* Generic V7M read/write macros for memory mapped cache operations */ 24 .macro v7m_cache_read, rt, reg 25 movw \rt, #:lower16:BASEADDR_V7M_SCB + \reg 26 movt \rt, #:upper16:BASEADDR_V7M_SCB + \reg 27 ldr \rt, [\rt] 28 .endm 29 30 .macro v7m_cacheop, rt, tmp, op, c = al 31 movw\c \tmp, #:lower16:BASEADDR_V7M_SCB + \op 32 movt\c \tmp, #:upper16:BASEADDR_V7M_SCB + \op 33 str\c \rt, [\tmp] 34 .endm 35 36 37 .macro read_ccsidr, rt 38 v7m_cache_read \rt, V7M_SCB_CCSIDR 39 .endm 40 41 .macro read_clidr, rt 42 v7m_cache_read \rt, V7M_SCB_CLIDR 43 .endm 44 45 .macro write_csselr, rt, tmp 46 v7m_cacheop \rt, \tmp, V7M_SCB_CSSELR 47 .endm 48 49 /* 50 * dcisw: Invalidate data cache by set/way 51 */ 52 .macro dcisw, rt, tmp 53 v7m_cacheop \rt, \tmp, V7M_SCB_DCISW 54 .endm 55 56 /* 57 * dccisw: Clean and invalidate data cache by set/way 58 */ 59 .macro dccisw, rt, tmp 60 v7m_cacheop \rt, \tmp, V7M_SCB_DCCISW 61 .endm 62 63 /* 64 * dccimvac: Clean and invalidate data cache line by MVA to PoC. 65 */ 66 .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo 67 .macro dccimvac\c, rt, tmp 68 v7m_cacheop \rt, \tmp, V7M_SCB_DCCIMVAC, \c 69 .endm 70 .endr 71 72 /* 73 * dcimvac: Invalidate data cache line by MVA to PoC 74 */ 75 .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo 76 .macro dcimvac\c, rt, tmp 77 v7m_cacheop \rt, \tmp, V7M_SCB_DCIMVAC, \c 78 .endm 79 .endr 80 81 /* 82 * dccmvau: Clean data cache line by MVA to PoU 83 */ 84 .macro dccmvau, rt, tmp 85 v7m_cacheop \rt, \tmp, V7M_SCB_DCCMVAU 86 .endm 87 88 /* 89 * dccmvac: Clean data cache line by MVA to PoC 90 */ 91 .macro dccmvac, rt, tmp 92 v7m_cacheop \rt, \tmp, V7M_SCB_DCCMVAC 93 .endm 94 95 /* 96 * icimvau: Invalidate instruction caches by MVA to PoU 97 */ 98 .macro icimvau, rt, tmp 99 v7m_cacheop \rt, \tmp, V7M_SCB_ICIMVAU 100 .endm 101 102 /* 103 * Invalidate the icache, inner shareable if SMP, invalidate BTB for UP. 104 * rt data ignored by ICIALLU(IS), so can be used for the address 105 */ 106 .macro invalidate_icache, rt 107 v7m_cacheop \rt, \rt, V7M_SCB_ICIALLU 108 mov \rt, #0 109 .endm 110 111 /* 112 * Invalidate the BTB, inner shareable if SMP. 113 * rt data ignored by BPIALL, so it can be used for the address 114 */ 115 .macro invalidate_bp, rt 116 v7m_cacheop \rt, \rt, V7M_SCB_BPIALL 117 mov \rt, #0 118 .endm 119 120 ENTRY(v7m_invalidate_l1) 121 mov r0, #0 122 123 write_csselr r0, r1 124 read_ccsidr r0 125 126 movw r1, #0x7fff 127 and r2, r1, r0, lsr #13 128 129 movw r1, #0x3ff 130 131 and r3, r1, r0, lsr #3 @ NumWays - 1 132 add r2, r2, #1 @ NumSets 133 134 and r0, r0, #0x7 135 add r0, r0, #4 @ SetShift 136 137 clz r1, r3 @ WayShift 138 add r4, r3, #1 @ NumWays 139 1: sub r2, r2, #1 @ NumSets-- 140 mov r3, r4 @ Temp = NumWays 141 2: subs r3, r3, #1 @ Temp-- 142 mov r5, r3, lsl r1 143 mov r6, r2, lsl r0 144 orr r5, r5, r6 @ Reg = (Temp<<WayShift)|(NumSets<<SetShift) 145 dcisw r5, r6 146 bgt 2b 147 cmp r2, #0 148 bgt 1b 149 dsb st 150 isb 151 ret lr 152 ENDPROC(v7m_invalidate_l1) 153 154 /* 155 * v7m_flush_icache_all() 156 * 157 * Flush the whole I-cache. 158 * 159 * Registers: 160 * r0 - set to 0 161 */ 162 ENTRY(v7m_flush_icache_all) 163 invalidate_icache r0 164 ret lr 165 ENDPROC(v7m_flush_icache_all) 166 167 /* 168 * v7m_flush_dcache_all() 169 * 170 * Flush the whole D-cache. 171 * 172 * Corrupted registers: r0-r7, r9-r11 173 */ 174 ENTRY(v7m_flush_dcache_all) 175 dmb @ ensure ordering with previous memory accesses 176 read_clidr r0 177 mov r3, r0, lsr #23 @ move LoC into position 178 ands r3, r3, #7 << 1 @ extract LoC*2 from clidr 179 beq finished @ if loc is 0, then no need to clean 180 start_flush_levels: 181 mov r10, #0 @ start clean at cache level 0 182 flush_levels: 183 add r2, r10, r10, lsr #1 @ work out 3x current cache level 184 mov r1, r0, lsr r2 @ extract cache type bits from clidr 185 and r1, r1, #7 @ mask of the bits for current cache only 186 cmp r1, #2 @ see what cache we have at this level 187 blt skip @ skip if no cache, or just i-cache 188 #ifdef CONFIG_PREEMPTION 189 save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic 190 #endif 191 write_csselr r10, r1 @ set current cache level 192 isb @ isb to sych the new cssr&csidr 193 read_ccsidr r1 @ read the new csidr 194 #ifdef CONFIG_PREEMPTION 195 restore_irqs_notrace r9 196 #endif 197 and r2, r1, #7 @ extract the length of the cache lines 198 add r2, r2, #4 @ add 4 (line length offset) 199 movw r4, #0x3ff 200 ands r4, r4, r1, lsr #3 @ find maximum number on the way size 201 clz r5, r4 @ find bit position of way size increment 202 movw r7, #0x7fff 203 ands r7, r7, r1, lsr #13 @ extract max number of the index size 204 loop1: 205 mov r9, r7 @ create working copy of max index 206 loop2: 207 lsl r6, r4, r5 208 orr r11, r10, r6 @ factor way and cache number into r11 209 lsl r6, r9, r2 210 orr r11, r11, r6 @ factor index number into r11 211 dccisw r11, r6 @ clean/invalidate by set/way 212 subs r9, r9, #1 @ decrement the index 213 bge loop2 214 subs r4, r4, #1 @ decrement the way 215 bge loop1 216 skip: 217 add r10, r10, #2 @ increment cache number 218 cmp r3, r10 219 bgt flush_levels 220 finished: 221 mov r10, #0 @ switch back to cache level 0 222 write_csselr r10, r3 @ select current cache level in cssr 223 dsb st 224 isb 225 ret lr 226 ENDPROC(v7m_flush_dcache_all) 227 228 /* 229 * v7m_flush_cache_all() 230 * 231 * Flush the entire cache system. 232 * The data cache flush is now achieved using atomic clean / invalidates 233 * working outwards from L1 cache. This is done using Set/Way based cache 234 * maintenance instructions. 235 * The instruction cache can still be invalidated back to the point of 236 * unification in a single instruction. 237 * 238 */ 239 ENTRY(v7m_flush_kern_cache_all) 240 stmfd sp!, {r4-r7, r9-r11, lr} 241 bl v7m_flush_dcache_all 242 invalidate_icache r0 243 ldmfd sp!, {r4-r7, r9-r11, lr} 244 ret lr 245 ENDPROC(v7m_flush_kern_cache_all) 246 247 /* 248 * v7m_flush_cache_all() 249 * 250 * Flush all TLB entries in a particular address space 251 * 252 * - mm - mm_struct describing address space 253 */ 254 ENTRY(v7m_flush_user_cache_all) 255 /*FALLTHROUGH*/ 256 257 /* 258 * v7m_flush_cache_range(start, end, flags) 259 * 260 * Flush a range of TLB entries in the specified address space. 261 * 262 * - start - start address (may not be aligned) 263 * - end - end address (exclusive, may not be aligned) 264 * - flags - vm_area_struct flags describing address space 265 * 266 * It is assumed that: 267 * - we have a VIPT cache. 268 */ 269 ENTRY(v7m_flush_user_cache_range) 270 ret lr 271 ENDPROC(v7m_flush_user_cache_all) 272 ENDPROC(v7m_flush_user_cache_range) 273 274 /* 275 * v7m_coherent_kern_range(start,end) 276 * 277 * Ensure that the I and D caches are coherent within specified 278 * region. This is typically used when code has been written to 279 * a memory region, and will be executed. 280 * 281 * - start - virtual start address of region 282 * - end - virtual end address of region 283 * 284 * It is assumed that: 285 * - the Icache does not read data from the write buffer 286 */ 287 ENTRY(v7m_coherent_kern_range) 288 /* FALLTHROUGH */ 289 290 /* 291 * v7m_coherent_user_range(start,end) 292 * 293 * Ensure that the I and D caches are coherent within specified 294 * region. This is typically used when code has been written to 295 * a memory region, and will be executed. 296 * 297 * - start - virtual start address of region 298 * - end - virtual end address of region 299 * 300 * It is assumed that: 301 * - the Icache does not read data from the write buffer 302 */ 303 ENTRY(v7m_coherent_user_range) 304 UNWIND(.fnstart ) 305 dcache_line_size r2, r3 306 sub r3, r2, #1 307 bic r12, r0, r3 308 1: 309 /* 310 * We use open coded version of dccmvau otherwise USER() would 311 * point at movw instruction. 312 */ 313 dccmvau r12, r3 314 add r12, r12, r2 315 cmp r12, r1 316 blo 1b 317 dsb ishst 318 icache_line_size r2, r3 319 sub r3, r2, #1 320 bic r12, r0, r3 321 2: 322 icimvau r12, r3 323 add r12, r12, r2 324 cmp r12, r1 325 blo 2b 326 invalidate_bp r0 327 dsb ishst 328 isb 329 ret lr 330 UNWIND(.fnend ) 331 ENDPROC(v7m_coherent_kern_range) 332 ENDPROC(v7m_coherent_user_range) 333 334 /* 335 * v7m_flush_kern_dcache_area(void *addr, size_t size) 336 * 337 * Ensure that the data held in the page kaddr is written back 338 * to the page in question. 339 * 340 * - addr - kernel address 341 * - size - region size 342 */ 343 ENTRY(v7m_flush_kern_dcache_area) 344 dcache_line_size r2, r3 345 add r1, r0, r1 346 sub r3, r2, #1 347 bic r0, r0, r3 348 1: 349 dccimvac r0, r3 @ clean & invalidate D line / unified line 350 add r0, r0, r2 351 cmp r0, r1 352 blo 1b 353 dsb st 354 ret lr 355 ENDPROC(v7m_flush_kern_dcache_area) 356 357 /* 358 * v7m_dma_inv_range(start,end) 359 * 360 * Invalidate the data cache within the specified region; we will 361 * be performing a DMA operation in this region and we want to 362 * purge old data in the cache. 363 * 364 * - start - virtual start address of region 365 * - end - virtual end address of region 366 */ 367 v7m_dma_inv_range: 368 dcache_line_size r2, r3 369 sub r3, r2, #1 370 tst r0, r3 371 bic r0, r0, r3 372 dccimvacne r0, r3 373 addne r0, r0, r2 374 subne r3, r2, #1 @ restore r3, corrupted by v7m's dccimvac 375 tst r1, r3 376 bic r1, r1, r3 377 dccimvacne r1, r3 378 cmp r0, r1 379 1: 380 dcimvaclo r0, r3 381 addlo r0, r0, r2 382 cmplo r0, r1 383 blo 1b 384 dsb st 385 ret lr 386 ENDPROC(v7m_dma_inv_range) 387 388 /* 389 * v7m_dma_clean_range(start,end) 390 * - start - virtual start address of region 391 * - end - virtual end address of region 392 */ 393 v7m_dma_clean_range: 394 dcache_line_size r2, r3 395 sub r3, r2, #1 396 bic r0, r0, r3 397 1: 398 dccmvac r0, r3 @ clean D / U line 399 add r0, r0, r2 400 cmp r0, r1 401 blo 1b 402 dsb st 403 ret lr 404 ENDPROC(v7m_dma_clean_range) 405 406 /* 407 * v7m_dma_flush_range(start,end) 408 * - start - virtual start address of region 409 * - end - virtual end address of region 410 */ 411 ENTRY(v7m_dma_flush_range) 412 dcache_line_size r2, r3 413 sub r3, r2, #1 414 bic r0, r0, r3 415 1: 416 dccimvac r0, r3 @ clean & invalidate D / U line 417 add r0, r0, r2 418 cmp r0, r1 419 blo 1b 420 dsb st 421 ret lr 422 ENDPROC(v7m_dma_flush_range) 423 424 /* 425 * dma_map_area(start, size, dir) 426 * - start - kernel virtual start address 427 * - size - size of region 428 * - dir - DMA direction 429 */ 430 ENTRY(v7m_dma_map_area) 431 add r1, r1, r0 432 teq r2, #DMA_FROM_DEVICE 433 beq v7m_dma_inv_range 434 b v7m_dma_clean_range 435 ENDPROC(v7m_dma_map_area) 436 437 /* 438 * dma_unmap_area(start, size, dir) 439 * - start - kernel virtual start address 440 * - size - size of region 441 * - dir - DMA direction 442 */ 443 ENTRY(v7m_dma_unmap_area) 444 add r1, r1, r0 445 teq r2, #DMA_TO_DEVICE 446 bne v7m_dma_inv_range 447 ret lr 448 ENDPROC(v7m_dma_unmap_area) 449 450 .globl v7m_flush_kern_cache_louis 451 .equ v7m_flush_kern_cache_louis, v7m_flush_kern_cache_all 452 453 __INITDATA 454 455 @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) 456 define_cache_functions v7m 457