xref: /openbmc/linux/arch/arm64/mm/cache.S (revision abfbd895)
1/*
2 * Cache maintenance
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include <linux/errno.h>
21#include <linux/linkage.h>
22#include <linux/init.h>
23#include <asm/assembler.h>
24#include <asm/cpufeature.h>
25#include <asm/alternative.h>
26
27#include "proc-macros.S"
28
29/*
30 *	flush_icache_range(start,end)
31 *
32 *	Ensure that the I and D caches are coherent within specified region.
33 *	This is typically used when code has been written to a memory region,
34 *	and will be executed.
35 *
36 *	- start   - virtual start address of region
37 *	- end     - virtual end address of region
38 */
39ENTRY(flush_icache_range)
40	/* FALLTHROUGH */
41
42/*
43 *	__flush_cache_user_range(start,end)
44 *
45 *	Ensure that the I and D caches are coherent within specified region.
46 *	This is typically used when code has been written to a memory region,
47 *	and will be executed.
48 *
49 *	- start   - virtual start address of region
50 *	- end     - virtual end address of region
51 */
52ENTRY(__flush_cache_user_range)
53	dcache_line_size x2, x3
54	sub	x3, x2, #1
55	bic	x4, x0, x3
561:
57USER(9f, dc	cvau, x4	)		// clean D line to PoU
58	add	x4, x4, x2
59	cmp	x4, x1
60	b.lo	1b
61	dsb	ish
62
63	icache_line_size x2, x3
64	sub	x3, x2, #1
65	bic	x4, x0, x3
661:
67USER(9f, ic	ivau, x4	)		// invalidate I line PoU
68	add	x4, x4, x2
69	cmp	x4, x1
70	b.lo	1b
71	dsb	ish
72	isb
73	mov	x0, #0
74	ret
759:
76	mov	x0, #-EFAULT
77	ret
78ENDPROC(flush_icache_range)
79ENDPROC(__flush_cache_user_range)
80
81/*
82 *	__flush_dcache_area(kaddr, size)
83 *
84 *	Ensure that the data held in the page kaddr is written back to the
85 *	page in question.
86 *
87 *	- kaddr   - kernel address
88 *	- size    - size in question
89 */
90ENTRY(__flush_dcache_area)
91	dcache_line_size x2, x3
92	add	x1, x0, x1
93	sub	x3, x2, #1
94	bic	x0, x0, x3
951:	dc	civac, x0			// clean & invalidate D line / unified line
96	add	x0, x0, x2
97	cmp	x0, x1
98	b.lo	1b
99	dsb	sy
100	ret
101ENDPIPROC(__flush_dcache_area)
102
103/*
104 *	__inval_cache_range(start, end)
105 *	- start   - start address of region
106 *	- end     - end address of region
107 */
108ENTRY(__inval_cache_range)
109	/* FALLTHROUGH */
110
111/*
112 *	__dma_inv_range(start, end)
113 *	- start   - virtual start address of region
114 *	- end     - virtual end address of region
115 */
116__dma_inv_range:
117	dcache_line_size x2, x3
118	sub	x3, x2, #1
119	tst	x1, x3				// end cache line aligned?
120	bic	x1, x1, x3
121	b.eq	1f
122	dc	civac, x1			// clean & invalidate D / U line
1231:	tst	x0, x3				// start cache line aligned?
124	bic	x0, x0, x3
125	b.eq	2f
126	dc	civac, x0			// clean & invalidate D / U line
127	b	3f
1282:	dc	ivac, x0			// invalidate D / U line
1293:	add	x0, x0, x2
130	cmp	x0, x1
131	b.lo	2b
132	dsb	sy
133	ret
134ENDPIPROC(__inval_cache_range)
135ENDPROC(__dma_inv_range)
136
137/*
138 *	__dma_clean_range(start, end)
139 *	- start   - virtual start address of region
140 *	- end     - virtual end address of region
141 */
142__dma_clean_range:
143	dcache_line_size x2, x3
144	sub	x3, x2, #1
145	bic	x0, x0, x3
1461:
147alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
148	dc	cvac, x0
149alternative_else
150	dc	civac, x0
151alternative_endif
152	add	x0, x0, x2
153	cmp	x0, x1
154	b.lo	1b
155	dsb	sy
156	ret
157ENDPROC(__dma_clean_range)
158
159/*
160 *	__dma_flush_range(start, end)
161 *	- start   - virtual start address of region
162 *	- end     - virtual end address of region
163 */
164ENTRY(__dma_flush_range)
165	dcache_line_size x2, x3
166	sub	x3, x2, #1
167	bic	x0, x0, x3
1681:	dc	civac, x0			// clean & invalidate D / U line
169	add	x0, x0, x2
170	cmp	x0, x1
171	b.lo	1b
172	dsb	sy
173	ret
174ENDPIPROC(__dma_flush_range)
175
176/*
177 *	__dma_map_area(start, size, dir)
178 *	- start	- kernel virtual start address
179 *	- size	- size of region
180 *	- dir	- DMA direction
181 */
182ENTRY(__dma_map_area)
183	add	x1, x1, x0
184	cmp	w2, #DMA_FROM_DEVICE
185	b.eq	__dma_inv_range
186	b	__dma_clean_range
187ENDPIPROC(__dma_map_area)
188
189/*
190 *	__dma_unmap_area(start, size, dir)
191 *	- start	- kernel virtual start address
192 *	- size	- size of region
193 *	- dir	- DMA direction
194 */
195ENTRY(__dma_unmap_area)
196	add	x1, x1, x0
197	cmp	w2, #DMA_TO_DEVICE
198	b.ne	__dma_inv_range
199	ret
200ENDPIPROC(__dma_unmap_area)
201