1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Cache maintenance 4 * 5 * Copyright (C) 2001 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 9#include <linux/errno.h> 10#include <linux/linkage.h> 11#include <linux/init.h> 12#include <asm/assembler.h> 13#include <asm/cpufeature.h> 14#include <asm/alternative.h> 15#include <asm/asm-uaccess.h> 16 17/* 18 * caches_clean_inval_pou_macro(start,end) [fixup] 19 * 20 * Ensure that the I and D caches are coherent within specified region. 21 * This is typically used when code has been written to a memory region, 22 * and will be executed. 23 * 24 * - start - virtual start address of region 25 * - end - virtual end address of region 26 * - fixup - optional label to branch to on user fault 27 */ 28.macro caches_clean_inval_pou_macro, fixup 29alternative_if ARM64_HAS_CACHE_IDC 30 dsb ishst 31 b .Ldc_skip_\@ 32alternative_else_nop_endif 33 mov x2, x0 34 mov x3, x1 35 dcache_by_line_op cvau, ish, x2, x3, x4, x5, \fixup 36.Ldc_skip_\@: 37alternative_if ARM64_HAS_CACHE_DIC 38 isb 39 b .Lic_skip_\@ 40alternative_else_nop_endif 41 invalidate_icache_by_line x0, x1, x2, x3, \fixup 42.Lic_skip_\@: 43.endm 44 45/* 46 * caches_clean_inval_pou(start,end) 47 * 48 * Ensure that the I and D caches are coherent within specified region. 49 * This is typically used when code has been written to a memory region, 50 * and will be executed. 51 * 52 * - start - virtual start address of region 53 * - end - virtual end address of region 54 */ 55SYM_FUNC_START(caches_clean_inval_pou) 56 caches_clean_inval_pou_macro 57 ret 58SYM_FUNC_END(caches_clean_inval_pou) 59 60/* 61 * caches_clean_inval_user_pou(start,end) 62 * 63 * Ensure that the I and D caches are coherent within specified region. 64 * This is typically used when code has been written to a memory region, 65 * and will be executed. 66 * 67 * - start - virtual start address of region 68 * - end - virtual end address of region 69 */ 70SYM_FUNC_START(caches_clean_inval_user_pou) 71 uaccess_ttbr0_enable x2, x3, x4 72 73 caches_clean_inval_pou_macro 2f 74 mov x0, xzr 751: 76 uaccess_ttbr0_disable x1, x2 77 ret 782: 79 mov x0, #-EFAULT 80 b 1b 81SYM_FUNC_END(caches_clean_inval_user_pou) 82 83/* 84 * icache_inval_pou(start,end) 85 * 86 * Ensure that the I cache is invalid within specified region. 87 * 88 * - start - virtual start address of region 89 * - end - virtual end address of region 90 */ 91SYM_FUNC_START(icache_inval_pou) 92alternative_if ARM64_HAS_CACHE_DIC 93 isb 94 ret 95alternative_else_nop_endif 96 97 invalidate_icache_by_line x0, x1, x2, x3 98 ret 99SYM_FUNC_END(icache_inval_pou) 100 101/* 102 * dcache_clean_inval_poc(start, end) 103 * 104 * Ensure that any D-cache lines for the interval [start, end) 105 * are cleaned and invalidated to the PoC. 106 * 107 * - start - virtual start address of region 108 * - end - virtual end address of region 109 */ 110SYM_FUNC_START(__pi_dcache_clean_inval_poc) 111 dcache_by_line_op civac, sy, x0, x1, x2, x3 112 ret 113SYM_FUNC_END(__pi_dcache_clean_inval_poc) 114SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc) 115 116/* 117 * dcache_clean_pou(start, end) 118 * 119 * Ensure that any D-cache lines for the interval [start, end) 120 * are cleaned to the PoU. 121 * 122 * - start - virtual start address of region 123 * - end - virtual end address of region 124 */ 125SYM_FUNC_START(dcache_clean_pou) 126alternative_if ARM64_HAS_CACHE_IDC 127 dsb ishst 128 ret 129alternative_else_nop_endif 130 dcache_by_line_op cvau, ish, x0, x1, x2, x3 131 ret 132SYM_FUNC_END(dcache_clean_pou) 133 134/* 135 * dcache_inval_poc(start, end) 136 * 137 * Ensure that any D-cache lines for the interval [start, end) 138 * are invalidated. Any partial lines at the ends of the interval are 139 * also cleaned to PoC to prevent data loss. 140 * 141 * - start - kernel start address of region 142 * - end - kernel end address of region 143 */ 144SYM_FUNC_START(__pi_dcache_inval_poc) 145 dcache_line_size x2, x3 146 sub x3, x2, #1 147 tst x1, x3 // end cache line aligned? 148 bic x1, x1, x3 149 b.eq 1f 150 dc civac, x1 // clean & invalidate D / U line 1511: tst x0, x3 // start cache line aligned? 152 bic x0, x0, x3 153 b.eq 2f 154 dc civac, x0 // clean & invalidate D / U line 155 b 3f 1562: dc ivac, x0 // invalidate D / U line 1573: add x0, x0, x2 158 cmp x0, x1 159 b.lo 2b 160 dsb sy 161 ret 162SYM_FUNC_END(__pi_dcache_inval_poc) 163SYM_FUNC_ALIAS(dcache_inval_poc, __pi_dcache_inval_poc) 164 165/* 166 * dcache_clean_poc(start, end) 167 * 168 * Ensure that any D-cache lines for the interval [start, end) 169 * are cleaned to the PoC. 170 * 171 * - start - virtual start address of region 172 * - end - virtual end address of region 173 */ 174SYM_FUNC_START(__pi_dcache_clean_poc) 175 dcache_by_line_op cvac, sy, x0, x1, x2, x3 176 ret 177SYM_FUNC_END(__pi_dcache_clean_poc) 178SYM_FUNC_ALIAS(dcache_clean_poc, __pi_dcache_clean_poc) 179 180/* 181 * dcache_clean_pop(start, end) 182 * 183 * Ensure that any D-cache lines for the interval [start, end) 184 * are cleaned to the PoP. 185 * 186 * - start - virtual start address of region 187 * - end - virtual end address of region 188 */ 189SYM_FUNC_START(__pi_dcache_clean_pop) 190 alternative_if_not ARM64_HAS_DCPOP 191 b dcache_clean_poc 192 alternative_else_nop_endif 193 dcache_by_line_op cvap, sy, x0, x1, x2, x3 194 ret 195SYM_FUNC_END(__pi_dcache_clean_pop) 196SYM_FUNC_ALIAS(dcache_clean_pop, __pi_dcache_clean_pop) 197 198/* 199 * __dma_flush_area(start, size) 200 * 201 * clean & invalidate D / U line 202 * 203 * - start - virtual start address of region 204 * - size - size in question 205 */ 206SYM_FUNC_START(__pi___dma_flush_area) 207 add x1, x0, x1 208 dcache_by_line_op civac, sy, x0, x1, x2, x3 209 ret 210SYM_FUNC_END(__pi___dma_flush_area) 211SYM_FUNC_ALIAS(__dma_flush_area, __pi___dma_flush_area) 212 213/* 214 * __dma_map_area(start, size, dir) 215 * - start - kernel virtual start address 216 * - size - size of region 217 * - dir - DMA direction 218 */ 219SYM_FUNC_START(__pi___dma_map_area) 220 add x1, x0, x1 221 cmp w2, #DMA_FROM_DEVICE 222 b.eq __pi_dcache_inval_poc 223 b __pi_dcache_clean_poc 224SYM_FUNC_END(__pi___dma_map_area) 225SYM_FUNC_ALIAS(__dma_map_area, __pi___dma_map_area) 226 227/* 228 * __dma_unmap_area(start, size, dir) 229 * - start - kernel virtual start address 230 * - size - size of region 231 * - dir - DMA direction 232 */ 233SYM_FUNC_START(__pi___dma_unmap_area) 234 add x1, x0, x1 235 cmp w2, #DMA_TO_DEVICE 236 b.ne __pi_dcache_inval_poc 237 ret 238SYM_FUNC_END(__pi___dma_unmap_area) 239SYM_FUNC_ALIAS(__dma_unmap_area, __pi___dma_unmap_area) 240