1/* 2 * Cache maintenance 3 * 4 * Copyright (C) 2001 Deep Blue Solutions Ltd. 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 20#include <linux/errno.h> 21#include <linux/linkage.h> 22#include <linux/init.h> 23#include <asm/assembler.h> 24#include <asm/cpufeature.h> 25#include <asm/alternative.h> 26#include <asm/asm-uaccess.h> 27 28/* 29 * flush_icache_range(start,end) 30 * 31 * Ensure that the I and D caches are coherent within specified region. 32 * This is typically used when code has been written to a memory region, 33 * and will be executed. 34 * 35 * - start - virtual start address of region 36 * - end - virtual end address of region 37 */ 38ENTRY(flush_icache_range) 39 /* FALLTHROUGH */ 40 41/* 42 * __flush_cache_user_range(start,end) 43 * 44 * Ensure that the I and D caches are coherent within specified region. 45 * This is typically used when code has been written to a memory region, 46 * and will be executed. 47 * 48 * - start - virtual start address of region 49 * - end - virtual end address of region 50 */ 51ENTRY(__flush_cache_user_range) 52 uaccess_ttbr0_enable x2, x3 53 dcache_line_size x2, x3 54 sub x3, x2, #1 55 bic x4, x0, x3 561: 57user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE 58 add x4, x4, x2 59 cmp x4, x1 60 b.lo 1b 61 dsb ish 62 63 icache_line_size x2, x3 64 sub x3, x2, #1 65 bic x4, x0, x3 661: 67USER(9f, ic ivau, x4 ) // invalidate I line PoU 68 add x4, x4, x2 69 cmp x4, x1 70 b.lo 1b 71 dsb ish 72 isb 73 mov x0, #0 741: 75 uaccess_ttbr0_disable x1 76 ret 779: 78 mov x0, #-EFAULT 79 b 1b 80ENDPROC(flush_icache_range) 81ENDPROC(__flush_cache_user_range) 82 83/* 84 * __flush_dcache_area(kaddr, size) 85 * 86 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) 87 * are cleaned and invalidated to the PoC. 88 * 89 * - kaddr - kernel address 90 * - size - size in question 91 */ 92ENTRY(__flush_dcache_area) 93 dcache_by_line_op civac, sy, x0, x1, x2, x3 94 ret 95ENDPIPROC(__flush_dcache_area) 96 97/* 98 * __clean_dcache_area_pou(kaddr, size) 99 * 100 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) 101 * are cleaned to the PoU. 102 * 103 * - kaddr - kernel address 104 * - size - size in question 105 */ 106ENTRY(__clean_dcache_area_pou) 107 dcache_by_line_op cvau, ish, x0, x1, x2, x3 108 ret 109ENDPROC(__clean_dcache_area_pou) 110 111/* 112 * __dma_inv_area(start, size) 113 * - start - virtual start address of region 114 * - size - size in question 115 */ 116__dma_inv_area: 117 add x1, x1, x0 118 /* FALLTHROUGH */ 119 120/* 121 * __inval_cache_range(start, end) 122 * - start - start address of region 123 * - end - end address of region 124 */ 125ENTRY(__inval_cache_range) 126 dcache_line_size x2, x3 127 sub x3, x2, #1 128 tst x1, x3 // end cache line aligned? 129 bic x1, x1, x3 130 b.eq 1f 131 dc civac, x1 // clean & invalidate D / U line 1321: tst x0, x3 // start cache line aligned? 133 bic x0, x0, x3 134 b.eq 2f 135 dc civac, x0 // clean & invalidate D / U line 136 b 3f 1372: dc ivac, x0 // invalidate D / U line 1383: add x0, x0, x2 139 cmp x0, x1 140 b.lo 2b 141 dsb sy 142 ret 143ENDPIPROC(__inval_cache_range) 144ENDPROC(__dma_inv_area) 145 146/* 147 * __clean_dcache_area_poc(kaddr, size) 148 * 149 * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) 150 * are cleaned to the PoC. 151 * 152 * - kaddr - kernel address 153 * - size - size in question 154 */ 155ENTRY(__clean_dcache_area_poc) 156 /* FALLTHROUGH */ 157 158/* 159 * __dma_clean_area(start, size) 160 * - start - virtual start address of region 161 * - size - size in question 162 */ 163__dma_clean_area: 164 dcache_by_line_op cvac, sy, x0, x1, x2, x3 165 ret 166ENDPIPROC(__clean_dcache_area_poc) 167ENDPROC(__dma_clean_area) 168 169/* 170 * __dma_flush_area(start, size) 171 * 172 * clean & invalidate D / U line 173 * 174 * - start - virtual start address of region 175 * - size - size in question 176 */ 177ENTRY(__dma_flush_area) 178 dcache_by_line_op civac, sy, x0, x1, x2, x3 179 ret 180ENDPIPROC(__dma_flush_area) 181 182/* 183 * __dma_map_area(start, size, dir) 184 * - start - kernel virtual start address 185 * - size - size of region 186 * - dir - DMA direction 187 */ 188ENTRY(__dma_map_area) 189 cmp w2, #DMA_FROM_DEVICE 190 b.eq __dma_inv_area 191 b __dma_clean_area 192ENDPIPROC(__dma_map_area) 193 194/* 195 * __dma_unmap_area(start, size, dir) 196 * - start - kernel virtual start address 197 * - size - size of region 198 * - dir - DMA direction 199 */ 200ENTRY(__dma_unmap_area) 201 cmp w2, #DMA_TO_DEVICE 202 b.ne __dma_inv_area 203 ret 204ENDPIPROC(__dma_unmap_area) 205