1/* 2 * Cache maintenance 3 * 4 * Copyright (C) 2001 Deep Blue Solutions Ltd. 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 20#include <linux/errno.h> 21#include <linux/linkage.h> 22#include <linux/init.h> 23#include <asm/assembler.h> 24#include <asm/cpufeature.h> 25#include <asm/alternative.h> 26#include <asm/asm-uaccess.h> 27 28/* 29 * flush_icache_range(start,end) 30 * 31 * Ensure that the I and D caches are coherent within specified region. 32 * This is typically used when code has been written to a memory region, 33 * and will be executed. 34 * 35 * - start - virtual start address of region 36 * - end - virtual end address of region 37 */ 38ENTRY(flush_icache_range) 39 /* FALLTHROUGH */ 40 41/* 42 * __flush_cache_user_range(start,end) 43 * 44 * Ensure that the I and D caches are coherent within specified region. 45 * This is typically used when code has been written to a memory region, 46 * and will be executed. 47 * 48 * - start - virtual start address of region 49 * - end - virtual end address of region 50 */ 51ENTRY(__flush_cache_user_range) 52 uaccess_ttbr0_enable x2, x3, x4 53 dcache_line_size x2, x3 54 sub x3, x2, #1 55 bic x4, x0, x3 561: 57user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE 58 add x4, x4, x2 59 cmp x4, x1 60 b.lo 1b 61 dsb ish 62 63 invalidate_icache_by_line x0, x1, x2, x3, 9f 64 mov x0, #0 651: 66 uaccess_ttbr0_disable x1, x2 67 ret 689: 69 mov x0, #-EFAULT 70 b 1b 71ENDPROC(flush_icache_range) 72ENDPROC(__flush_cache_user_range) 73 74/* 75 * invalidate_icache_range(start,end) 76 * 77 * Ensure that the I cache is invalid within specified region. 78 * 79 * - start - virtual start address of region 80 * - end - virtual end address of region 81 */ 82ENTRY(invalidate_icache_range) 83 uaccess_ttbr0_enable x2, x3, x4 84 85 invalidate_icache_by_line x0, x1, x2, x3, 2f 86 mov x0, xzr 871: 88 uaccess_ttbr0_disable x1, x2 89 ret 902: 91 mov x0, #-EFAULT 92 b 1b 93ENDPROC(invalidate_icache_range) 94 95/* 96 * __flush_dcache_area(kaddr, size) 97 * 98 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) 99 * are cleaned and invalidated to the PoC. 100 * 101 * - kaddr - kernel address 102 * - size - size in question 103 */ 104ENTRY(__flush_dcache_area) 105 dcache_by_line_op civac, sy, x0, x1, x2, x3 106 ret 107ENDPIPROC(__flush_dcache_area) 108 109/* 110 * __clean_dcache_area_pou(kaddr, size) 111 * 112 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) 113 * are cleaned to the PoU. 114 * 115 * - kaddr - kernel address 116 * - size - size in question 117 */ 118ENTRY(__clean_dcache_area_pou) 119 dcache_by_line_op cvau, ish, x0, x1, x2, x3 120 ret 121ENDPROC(__clean_dcache_area_pou) 122 123/* 124 * __inval_dcache_area(kaddr, size) 125 * 126 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) 127 * are invalidated. Any partial lines at the ends of the interval are 128 * also cleaned to PoC to prevent data loss. 129 * 130 * - kaddr - kernel address 131 * - size - size in question 132 */ 133ENTRY(__inval_dcache_area) 134 /* FALLTHROUGH */ 135 136/* 137 * __dma_inv_area(start, size) 138 * - start - virtual start address of region 139 * - size - size in question 140 */ 141__dma_inv_area: 142 add x1, x1, x0 143 dcache_line_size x2, x3 144 sub x3, x2, #1 145 tst x1, x3 // end cache line aligned? 146 bic x1, x1, x3 147 b.eq 1f 148 dc civac, x1 // clean & invalidate D / U line 1491: tst x0, x3 // start cache line aligned? 150 bic x0, x0, x3 151 b.eq 2f 152 dc civac, x0 // clean & invalidate D / U line 153 b 3f 1542: dc ivac, x0 // invalidate D / U line 1553: add x0, x0, x2 156 cmp x0, x1 157 b.lo 2b 158 dsb sy 159 ret 160ENDPIPROC(__inval_dcache_area) 161ENDPROC(__dma_inv_area) 162 163/* 164 * __clean_dcache_area_poc(kaddr, size) 165 * 166 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) 167 * are cleaned to the PoC. 168 * 169 * - kaddr - kernel address 170 * - size - size in question 171 */ 172ENTRY(__clean_dcache_area_poc) 173 /* FALLTHROUGH */ 174 175/* 176 * __dma_clean_area(start, size) 177 * - start - virtual start address of region 178 * - size - size in question 179 */ 180__dma_clean_area: 181 dcache_by_line_op cvac, sy, x0, x1, x2, x3 182 ret 183ENDPIPROC(__clean_dcache_area_poc) 184ENDPROC(__dma_clean_area) 185 186/* 187 * __clean_dcache_area_pop(kaddr, size) 188 * 189 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) 190 * are cleaned to the PoP. 191 * 192 * - kaddr - kernel address 193 * - size - size in question 194 */ 195ENTRY(__clean_dcache_area_pop) 196 dcache_by_line_op cvap, sy, x0, x1, x2, x3 197 ret 198ENDPIPROC(__clean_dcache_area_pop) 199 200/* 201 * __dma_flush_area(start, size) 202 * 203 * clean & invalidate D / U line 204 * 205 * - start - virtual start address of region 206 * - size - size in question 207 */ 208ENTRY(__dma_flush_area) 209 dcache_by_line_op civac, sy, x0, x1, x2, x3 210 ret 211ENDPIPROC(__dma_flush_area) 212 213/* 214 * __dma_map_area(start, size, dir) 215 * - start - kernel virtual start address 216 * - size - size of region 217 * - dir - DMA direction 218 */ 219ENTRY(__dma_map_area) 220 cmp w2, #DMA_FROM_DEVICE 221 b.eq __dma_inv_area 222 b __dma_clean_area 223ENDPIPROC(__dma_map_area) 224 225/* 226 * __dma_unmap_area(start, size, dir) 227 * - start - kernel virtual start address 228 * - size - size of region 229 * - dir - DMA direction 230 */ 231ENTRY(__dma_unmap_area) 232 cmp w2, #DMA_TO_DEVICE 233 b.ne __dma_inv_area 234 ret 235ENDPIPROC(__dma_unmap_area) 236