1/* 2 * Cache maintenance 3 * 4 * Copyright (C) 2001 Deep Blue Solutions Ltd. 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 20#include <linux/errno.h> 21#include <linux/linkage.h> 22#include <linux/init.h> 23#include <asm/assembler.h> 24#include <asm/cpufeature.h> 25#include <asm/alternative-asm.h> 26 27#include "proc-macros.S" 28 29/* 30 * __flush_dcache_all() 31 * 32 * Flush the whole D-cache. 33 * 34 * Corrupted registers: x0-x7, x9-x11 35 */ 36__flush_dcache_all: 37 dmb sy // ensure ordering with previous memory accesses 38 mrs x0, clidr_el1 // read clidr 39 and x3, x0, #0x7000000 // extract loc from clidr 40 lsr x3, x3, #23 // left align loc bit field 41 cbz x3, finished // if loc is 0, then no need to clean 42 mov x10, #0 // start clean at cache level 0 43loop1: 44 add x2, x10, x10, lsr #1 // work out 3x current cache level 45 lsr x1, x0, x2 // extract cache type bits from clidr 46 and x1, x1, #7 // mask of the bits for current cache only 47 cmp x1, #2 // see what cache we have at this level 48 b.lt skip // skip if no cache, or just i-cache 49 save_and_disable_irqs x9 // make CSSELR and CCSIDR access atomic 50 msr csselr_el1, x10 // select current cache level in csselr 51 isb // isb to sych the new cssr&csidr 52 mrs x1, ccsidr_el1 // read the new ccsidr 53 restore_irqs x9 54 and x2, x1, #7 // extract the length of the cache lines 55 add x2, x2, #4 // add 4 (line length offset) 56 mov x4, #0x3ff 57 and x4, x4, x1, lsr #3 // find maximum number on the way size 58 clz w5, w4 // find bit position of way size increment 59 mov x7, #0x7fff 60 and x7, x7, x1, lsr #13 // extract max number of the index size 61loop2: 62 mov x9, x4 // create working copy of max way size 63loop3: 64 lsl x6, x9, x5 65 orr x11, x10, x6 // factor way and cache number into x11 66 lsl x6, x7, x2 67 orr x11, x11, x6 // factor index number into x11 68 dc cisw, x11 // clean & invalidate by set/way 69 subs x9, x9, #1 // decrement the way 70 b.ge loop3 71 subs x7, x7, #1 // decrement the index 72 b.ge loop2 73skip: 74 add x10, x10, #2 // increment cache number 75 cmp x3, x10 76 b.gt loop1 77finished: 78 mov x10, #0 // swith back to cache level 0 79 msr csselr_el1, x10 // select current cache level in csselr 80 dsb sy 81 isb 82 ret 83ENDPROC(__flush_dcache_all) 84 85/* 86 * flush_cache_all() 87 * 88 * Flush the entire cache system. The data cache flush is now achieved 89 * using atomic clean / invalidates working outwards from L1 cache. This 90 * is done using Set/Way based cache maintainance instructions. The 91 * instruction cache can still be invalidated back to the point of 92 * unification in a single instruction. 93 */ 94ENTRY(flush_cache_all) 95 mov x12, lr 96 bl __flush_dcache_all 97 mov x0, #0 98 ic ialluis // I+BTB cache invalidate 99 ret x12 100ENDPROC(flush_cache_all) 101 102/* 103 * flush_icache_range(start,end) 104 * 105 * Ensure that the I and D caches are coherent within specified region. 106 * This is typically used when code has been written to a memory region, 107 * and will be executed. 108 * 109 * - start - virtual start address of region 110 * - end - virtual end address of region 111 */ 112ENTRY(flush_icache_range) 113 /* FALLTHROUGH */ 114 115/* 116 * __flush_cache_user_range(start,end) 117 * 118 * Ensure that the I and D caches are coherent within specified region. 119 * This is typically used when code has been written to a memory region, 120 * and will be executed. 121 * 122 * - start - virtual start address of region 123 * - end - virtual end address of region 124 */ 125ENTRY(__flush_cache_user_range) 126 dcache_line_size x2, x3 127 sub x3, x2, #1 128 bic x4, x0, x3 1291: 130USER(9f, dc cvau, x4 ) // clean D line to PoU 131 add x4, x4, x2 132 cmp x4, x1 133 b.lo 1b 134 dsb ish 135 136 icache_line_size x2, x3 137 sub x3, x2, #1 138 bic x4, x0, x3 1391: 140USER(9f, ic ivau, x4 ) // invalidate I line PoU 141 add x4, x4, x2 142 cmp x4, x1 143 b.lo 1b 144 dsb ish 145 isb 146 mov x0, #0 147 ret 1489: 149 mov x0, #-EFAULT 150 ret 151ENDPROC(flush_icache_range) 152ENDPROC(__flush_cache_user_range) 153 154/* 155 * __flush_dcache_area(kaddr, size) 156 * 157 * Ensure that the data held in the page kaddr is written back to the 158 * page in question. 159 * 160 * - kaddr - kernel address 161 * - size - size in question 162 */ 163ENTRY(__flush_dcache_area) 164 dcache_line_size x2, x3 165 add x1, x0, x1 166 sub x3, x2, #1 167 bic x0, x0, x3 1681: dc civac, x0 // clean & invalidate D line / unified line 169 add x0, x0, x2 170 cmp x0, x1 171 b.lo 1b 172 dsb sy 173 ret 174ENDPROC(__flush_dcache_area) 175 176/* 177 * __inval_cache_range(start, end) 178 * - start - start address of region 179 * - end - end address of region 180 */ 181ENTRY(__inval_cache_range) 182 /* FALLTHROUGH */ 183 184/* 185 * __dma_inv_range(start, end) 186 * - start - virtual start address of region 187 * - end - virtual end address of region 188 */ 189__dma_inv_range: 190 dcache_line_size x2, x3 191 sub x3, x2, #1 192 tst x1, x3 // end cache line aligned? 193 bic x1, x1, x3 194 b.eq 1f 195 dc civac, x1 // clean & invalidate D / U line 1961: tst x0, x3 // start cache line aligned? 197 bic x0, x0, x3 198 b.eq 2f 199 dc civac, x0 // clean & invalidate D / U line 200 b 3f 2012: dc ivac, x0 // invalidate D / U line 2023: add x0, x0, x2 203 cmp x0, x1 204 b.lo 2b 205 dsb sy 206 ret 207ENDPROC(__inval_cache_range) 208ENDPROC(__dma_inv_range) 209 210/* 211 * __dma_clean_range(start, end) 212 * - start - virtual start address of region 213 * - end - virtual end address of region 214 */ 215__dma_clean_range: 216 dcache_line_size x2, x3 217 sub x3, x2, #1 218 bic x0, x0, x3 2191: alternative_insn "dc cvac, x0", "dc civac, x0", ARM64_WORKAROUND_CLEAN_CACHE 220 add x0, x0, x2 221 cmp x0, x1 222 b.lo 1b 223 dsb sy 224 ret 225ENDPROC(__dma_clean_range) 226 227/* 228 * __dma_flush_range(start, end) 229 * - start - virtual start address of region 230 * - end - virtual end address of region 231 */ 232ENTRY(__dma_flush_range) 233 dcache_line_size x2, x3 234 sub x3, x2, #1 235 bic x0, x0, x3 2361: dc civac, x0 // clean & invalidate D / U line 237 add x0, x0, x2 238 cmp x0, x1 239 b.lo 1b 240 dsb sy 241 ret 242ENDPROC(__dma_flush_range) 243 244/* 245 * __dma_map_area(start, size, dir) 246 * - start - kernel virtual start address 247 * - size - size of region 248 * - dir - DMA direction 249 */ 250ENTRY(__dma_map_area) 251 add x1, x1, x0 252 cmp w2, #DMA_FROM_DEVICE 253 b.eq __dma_inv_range 254 b __dma_clean_range 255ENDPROC(__dma_map_area) 256 257/* 258 * __dma_unmap_area(start, size, dir) 259 * - start - kernel virtual start address 260 * - size - size of region 261 * - dir - DMA direction 262 */ 263ENTRY(__dma_unmap_area) 264 add x1, x1, x0 265 cmp w2, #DMA_TO_DEVICE 266 b.ne __dma_inv_range 267 ret 268ENDPROC(__dma_unmap_area) 269