1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * IOMMU API for ARM architected SMMUv3 implementations.
4  *
5  * Copyright (C) 2015 ARM Limited
6  */
7 
8 #ifndef _ARM_SMMU_V3_H
9 #define _ARM_SMMU_V3_H
10 
11 #include <linux/bitfield.h>
12 #include <linux/iommu.h>
13 #include <linux/kernel.h>
14 #include <linux/mmzone.h>
15 #include <linux/sizes.h>
16 
17 /* MMIO registers */
18 #define ARM_SMMU_IDR0			0x0
19 #define IDR0_ST_LVL			GENMASK(28, 27)
20 #define IDR0_ST_LVL_2LVL		1
21 #define IDR0_STALL_MODEL		GENMASK(25, 24)
22 #define IDR0_STALL_MODEL_STALL		0
23 #define IDR0_STALL_MODEL_FORCE		2
24 #define IDR0_TTENDIAN			GENMASK(22, 21)
25 #define IDR0_TTENDIAN_MIXED		0
26 #define IDR0_TTENDIAN_LE		2
27 #define IDR0_TTENDIAN_BE		3
28 #define IDR0_CD2L			(1 << 19)
29 #define IDR0_VMID16			(1 << 18)
30 #define IDR0_PRI			(1 << 16)
31 #define IDR0_SEV			(1 << 14)
32 #define IDR0_MSI			(1 << 13)
33 #define IDR0_ASID16			(1 << 12)
34 #define IDR0_ATS			(1 << 10)
35 #define IDR0_HYP			(1 << 9)
36 #define IDR0_COHACC			(1 << 4)
37 #define IDR0_TTF			GENMASK(3, 2)
38 #define IDR0_TTF_AARCH64		2
39 #define IDR0_TTF_AARCH32_64		3
40 #define IDR0_S1P			(1 << 1)
41 #define IDR0_S2P			(1 << 0)
42 
43 #define ARM_SMMU_IDR1			0x4
44 #define IDR1_TABLES_PRESET		(1 << 30)
45 #define IDR1_QUEUES_PRESET		(1 << 29)
46 #define IDR1_REL			(1 << 28)
47 #define IDR1_CMDQS			GENMASK(25, 21)
48 #define IDR1_EVTQS			GENMASK(20, 16)
49 #define IDR1_PRIQS			GENMASK(15, 11)
50 #define IDR1_SSIDSIZE			GENMASK(10, 6)
51 #define IDR1_SIDSIZE			GENMASK(5, 0)
52 
53 #define ARM_SMMU_IDR3			0xc
54 #define IDR3_RIL			(1 << 10)
55 
56 #define ARM_SMMU_IDR5			0x14
57 #define IDR5_STALL_MAX			GENMASK(31, 16)
58 #define IDR5_GRAN64K			(1 << 6)
59 #define IDR5_GRAN16K			(1 << 5)
60 #define IDR5_GRAN4K			(1 << 4)
61 #define IDR5_OAS			GENMASK(2, 0)
62 #define IDR5_OAS_32_BIT			0
63 #define IDR5_OAS_36_BIT			1
64 #define IDR5_OAS_40_BIT			2
65 #define IDR5_OAS_42_BIT			3
66 #define IDR5_OAS_44_BIT			4
67 #define IDR5_OAS_48_BIT			5
68 #define IDR5_OAS_52_BIT			6
69 #define IDR5_VAX			GENMASK(11, 10)
70 #define IDR5_VAX_52_BIT			1
71 
72 #define ARM_SMMU_CR0			0x20
73 #define CR0_ATSCHK			(1 << 4)
74 #define CR0_CMDQEN			(1 << 3)
75 #define CR0_EVTQEN			(1 << 2)
76 #define CR0_PRIQEN			(1 << 1)
77 #define CR0_SMMUEN			(1 << 0)
78 
79 #define ARM_SMMU_CR0ACK			0x24
80 
81 #define ARM_SMMU_CR1			0x28
82 #define CR1_TABLE_SH			GENMASK(11, 10)
83 #define CR1_TABLE_OC			GENMASK(9, 8)
84 #define CR1_TABLE_IC			GENMASK(7, 6)
85 #define CR1_QUEUE_SH			GENMASK(5, 4)
86 #define CR1_QUEUE_OC			GENMASK(3, 2)
87 #define CR1_QUEUE_IC			GENMASK(1, 0)
88 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
89 #define CR1_CACHE_NC			0
90 #define CR1_CACHE_WB			1
91 #define CR1_CACHE_WT			2
92 
93 #define ARM_SMMU_CR2			0x2c
94 #define CR2_PTM				(1 << 2)
95 #define CR2_RECINVSID			(1 << 1)
96 #define CR2_E2H				(1 << 0)
97 
98 #define ARM_SMMU_GBPA			0x44
99 #define GBPA_UPDATE			(1 << 31)
100 #define GBPA_ABORT			(1 << 20)
101 
102 #define ARM_SMMU_IRQ_CTRL		0x50
103 #define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
104 #define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
105 #define IRQ_CTRL_GERROR_IRQEN		(1 << 0)
106 
107 #define ARM_SMMU_IRQ_CTRLACK		0x54
108 
109 #define ARM_SMMU_GERROR			0x60
110 #define GERROR_SFM_ERR			(1 << 8)
111 #define GERROR_MSI_GERROR_ABT_ERR	(1 << 7)
112 #define GERROR_MSI_PRIQ_ABT_ERR		(1 << 6)
113 #define GERROR_MSI_EVTQ_ABT_ERR		(1 << 5)
114 #define GERROR_MSI_CMDQ_ABT_ERR		(1 << 4)
115 #define GERROR_PRIQ_ABT_ERR		(1 << 3)
116 #define GERROR_EVTQ_ABT_ERR		(1 << 2)
117 #define GERROR_CMDQ_ERR			(1 << 0)
118 #define GERROR_ERR_MASK			0x1fd
119 
120 #define ARM_SMMU_GERRORN		0x64
121 
122 #define ARM_SMMU_GERROR_IRQ_CFG0	0x68
123 #define ARM_SMMU_GERROR_IRQ_CFG1	0x70
124 #define ARM_SMMU_GERROR_IRQ_CFG2	0x74
125 
126 #define ARM_SMMU_STRTAB_BASE		0x80
127 #define STRTAB_BASE_RA			(1UL << 62)
128 #define STRTAB_BASE_ADDR_MASK		GENMASK_ULL(51, 6)
129 
130 #define ARM_SMMU_STRTAB_BASE_CFG	0x88
131 #define STRTAB_BASE_CFG_FMT		GENMASK(17, 16)
132 #define STRTAB_BASE_CFG_FMT_LINEAR	0
133 #define STRTAB_BASE_CFG_FMT_2LVL	1
134 #define STRTAB_BASE_CFG_SPLIT		GENMASK(10, 6)
135 #define STRTAB_BASE_CFG_LOG2SIZE	GENMASK(5, 0)
136 
137 #define ARM_SMMU_CMDQ_BASE		0x90
138 #define ARM_SMMU_CMDQ_PROD		0x98
139 #define ARM_SMMU_CMDQ_CONS		0x9c
140 
141 #define ARM_SMMU_EVTQ_BASE		0xa0
142 #define ARM_SMMU_EVTQ_PROD		0xa8
143 #define ARM_SMMU_EVTQ_CONS		0xac
144 #define ARM_SMMU_EVTQ_IRQ_CFG0		0xb0
145 #define ARM_SMMU_EVTQ_IRQ_CFG1		0xb8
146 #define ARM_SMMU_EVTQ_IRQ_CFG2		0xbc
147 
148 #define ARM_SMMU_PRIQ_BASE		0xc0
149 #define ARM_SMMU_PRIQ_PROD		0xc8
150 #define ARM_SMMU_PRIQ_CONS		0xcc
151 #define ARM_SMMU_PRIQ_IRQ_CFG0		0xd0
152 #define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
153 #define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
154 
155 #define ARM_SMMU_REG_SZ			0xe00
156 
157 /* Common MSI config fields */
158 #define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
159 #define MSI_CFG2_SH			GENMASK(5, 4)
160 #define MSI_CFG2_MEMATTR		GENMASK(3, 0)
161 
162 /* Common memory attribute values */
163 #define ARM_SMMU_SH_NSH			0
164 #define ARM_SMMU_SH_OSH			2
165 #define ARM_SMMU_SH_ISH			3
166 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE	0x1
167 #define ARM_SMMU_MEMATTR_OIWB		0xf
168 
169 #define Q_IDX(llq, p)			((p) & ((1 << (llq)->max_n_shift) - 1))
170 #define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
171 #define Q_OVERFLOW_FLAG			(1U << 31)
172 #define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
173 #define Q_ENT(q, p)			((q)->base +			\
174 					 Q_IDX(&((q)->llq), p) *	\
175 					 (q)->ent_dwords)
176 
177 #define Q_BASE_RWA			(1UL << 62)
178 #define Q_BASE_ADDR_MASK		GENMASK_ULL(51, 5)
179 #define Q_BASE_LOG2SIZE			GENMASK(4, 0)
180 
181 /* Ensure DMA allocations are naturally aligned */
182 #ifdef CONFIG_CMA_ALIGNMENT
183 #define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
184 #else
185 #define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + MAX_ORDER)
186 #endif
187 
188 /*
189  * Stream table.
190  *
191  * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
192  * 2lvl: 128k L1 entries,
193  *       256 lazy entries per table (each table covers a PCI bus)
194  */
195 #define STRTAB_L1_SZ_SHIFT		20
196 #define STRTAB_SPLIT			8
197 
198 #define STRTAB_L1_DESC_DWORDS		1
199 #define STRTAB_L1_DESC_SPAN		GENMASK_ULL(4, 0)
200 #define STRTAB_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 6)
201 
202 #define STRTAB_STE_DWORDS		8
203 #define STRTAB_STE_0_V			(1UL << 0)
204 #define STRTAB_STE_0_CFG		GENMASK_ULL(3, 1)
205 #define STRTAB_STE_0_CFG_ABORT		0
206 #define STRTAB_STE_0_CFG_BYPASS		4
207 #define STRTAB_STE_0_CFG_S1_TRANS	5
208 #define STRTAB_STE_0_CFG_S2_TRANS	6
209 
210 #define STRTAB_STE_0_S1FMT		GENMASK_ULL(5, 4)
211 #define STRTAB_STE_0_S1FMT_LINEAR	0
212 #define STRTAB_STE_0_S1FMT_64K_L2	2
213 #define STRTAB_STE_0_S1CTXPTR_MASK	GENMASK_ULL(51, 6)
214 #define STRTAB_STE_0_S1CDMAX		GENMASK_ULL(63, 59)
215 
216 #define STRTAB_STE_1_S1DSS		GENMASK_ULL(1, 0)
217 #define STRTAB_STE_1_S1DSS_TERMINATE	0x0
218 #define STRTAB_STE_1_S1DSS_BYPASS	0x1
219 #define STRTAB_STE_1_S1DSS_SSID0	0x2
220 
221 #define STRTAB_STE_1_S1C_CACHE_NC	0UL
222 #define STRTAB_STE_1_S1C_CACHE_WBRA	1UL
223 #define STRTAB_STE_1_S1C_CACHE_WT	2UL
224 #define STRTAB_STE_1_S1C_CACHE_WB	3UL
225 #define STRTAB_STE_1_S1CIR		GENMASK_ULL(3, 2)
226 #define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)
227 #define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)
228 
229 #define STRTAB_STE_1_S1STALLD		(1UL << 27)
230 
231 #define STRTAB_STE_1_EATS		GENMASK_ULL(29, 28)
232 #define STRTAB_STE_1_EATS_ABT		0UL
233 #define STRTAB_STE_1_EATS_TRANS		1UL
234 #define STRTAB_STE_1_EATS_S1CHK		2UL
235 
236 #define STRTAB_STE_1_STRW		GENMASK_ULL(31, 30)
237 #define STRTAB_STE_1_STRW_NSEL1		0UL
238 #define STRTAB_STE_1_STRW_EL2		2UL
239 
240 #define STRTAB_STE_1_SHCFG		GENMASK_ULL(45, 44)
241 #define STRTAB_STE_1_SHCFG_INCOMING	1UL
242 
243 #define STRTAB_STE_2_S2VMID		GENMASK_ULL(15, 0)
244 #define STRTAB_STE_2_VTCR		GENMASK_ULL(50, 32)
245 #define STRTAB_STE_2_VTCR_S2T0SZ	GENMASK_ULL(5, 0)
246 #define STRTAB_STE_2_VTCR_S2SL0		GENMASK_ULL(7, 6)
247 #define STRTAB_STE_2_VTCR_S2IR0		GENMASK_ULL(9, 8)
248 #define STRTAB_STE_2_VTCR_S2OR0		GENMASK_ULL(11, 10)
249 #define STRTAB_STE_2_VTCR_S2SH0		GENMASK_ULL(13, 12)
250 #define STRTAB_STE_2_VTCR_S2TG		GENMASK_ULL(15, 14)
251 #define STRTAB_STE_2_VTCR_S2PS		GENMASK_ULL(18, 16)
252 #define STRTAB_STE_2_S2AA64		(1UL << 51)
253 #define STRTAB_STE_2_S2ENDI		(1UL << 52)
254 #define STRTAB_STE_2_S2PTW		(1UL << 54)
255 #define STRTAB_STE_2_S2R		(1UL << 58)
256 
257 #define STRTAB_STE_3_S2TTB_MASK		GENMASK_ULL(51, 4)
258 
259 /*
260  * Context descriptors.
261  *
262  * Linear: when less than 1024 SSIDs are supported
263  * 2lvl: at most 1024 L1 entries,
264  *       1024 lazy entries per table.
265  */
266 #define CTXDESC_SPLIT			10
267 #define CTXDESC_L2_ENTRIES		(1 << CTXDESC_SPLIT)
268 
269 #define CTXDESC_L1_DESC_DWORDS		1
270 #define CTXDESC_L1_DESC_V		(1UL << 0)
271 #define CTXDESC_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 12)
272 
273 #define CTXDESC_CD_DWORDS		8
274 #define CTXDESC_CD_0_TCR_T0SZ		GENMASK_ULL(5, 0)
275 #define CTXDESC_CD_0_TCR_TG0		GENMASK_ULL(7, 6)
276 #define CTXDESC_CD_0_TCR_IRGN0		GENMASK_ULL(9, 8)
277 #define CTXDESC_CD_0_TCR_ORGN0		GENMASK_ULL(11, 10)
278 #define CTXDESC_CD_0_TCR_SH0		GENMASK_ULL(13, 12)
279 #define CTXDESC_CD_0_TCR_EPD0		(1ULL << 14)
280 #define CTXDESC_CD_0_TCR_EPD1		(1ULL << 30)
281 
282 #define CTXDESC_CD_0_ENDI		(1UL << 15)
283 #define CTXDESC_CD_0_V			(1UL << 31)
284 
285 #define CTXDESC_CD_0_TCR_IPS		GENMASK_ULL(34, 32)
286 #define CTXDESC_CD_0_TCR_TBI0		(1ULL << 38)
287 
288 #define CTXDESC_CD_0_AA64		(1UL << 41)
289 #define CTXDESC_CD_0_S			(1UL << 44)
290 #define CTXDESC_CD_0_R			(1UL << 45)
291 #define CTXDESC_CD_0_A			(1UL << 46)
292 #define CTXDESC_CD_0_ASET		(1UL << 47)
293 #define CTXDESC_CD_0_ASID		GENMASK_ULL(63, 48)
294 
295 #define CTXDESC_CD_1_TTB0_MASK		GENMASK_ULL(51, 4)
296 
297 /*
298  * When the SMMU only supports linear context descriptor tables, pick a
299  * reasonable size limit (64kB).
300  */
301 #define CTXDESC_LINEAR_CDMAX		ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
302 
303 /* Command queue */
304 #define CMDQ_ENT_SZ_SHIFT		4
305 #define CMDQ_ENT_DWORDS			((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
306 #define CMDQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
307 
308 #define CMDQ_CONS_ERR			GENMASK(30, 24)
309 #define CMDQ_ERR_CERROR_NONE_IDX	0
310 #define CMDQ_ERR_CERROR_ILL_IDX		1
311 #define CMDQ_ERR_CERROR_ABT_IDX		2
312 #define CMDQ_ERR_CERROR_ATC_INV_IDX	3
313 
314 #define CMDQ_PROD_OWNED_FLAG		Q_OVERFLOW_FLAG
315 
316 /*
317  * This is used to size the command queue and therefore must be at least
318  * BITS_PER_LONG so that the valid_map works correctly (it relies on the
319  * total number of queue entries being a multiple of BITS_PER_LONG).
320  */
321 #define CMDQ_BATCH_ENTRIES		BITS_PER_LONG
322 
323 #define CMDQ_0_OP			GENMASK_ULL(7, 0)
324 #define CMDQ_0_SSV			(1UL << 11)
325 
326 #define CMDQ_PREFETCH_0_SID		GENMASK_ULL(63, 32)
327 #define CMDQ_PREFETCH_1_SIZE		GENMASK_ULL(4, 0)
328 #define CMDQ_PREFETCH_1_ADDR_MASK	GENMASK_ULL(63, 12)
329 
330 #define CMDQ_CFGI_0_SSID		GENMASK_ULL(31, 12)
331 #define CMDQ_CFGI_0_SID			GENMASK_ULL(63, 32)
332 #define CMDQ_CFGI_1_LEAF		(1UL << 0)
333 #define CMDQ_CFGI_1_RANGE		GENMASK_ULL(4, 0)
334 
335 #define CMDQ_TLBI_0_NUM			GENMASK_ULL(16, 12)
336 #define CMDQ_TLBI_RANGE_NUM_MAX		31
337 #define CMDQ_TLBI_0_SCALE		GENMASK_ULL(24, 20)
338 #define CMDQ_TLBI_0_VMID		GENMASK_ULL(47, 32)
339 #define CMDQ_TLBI_0_ASID		GENMASK_ULL(63, 48)
340 #define CMDQ_TLBI_1_LEAF		(1UL << 0)
341 #define CMDQ_TLBI_1_TTL			GENMASK_ULL(9, 8)
342 #define CMDQ_TLBI_1_TG			GENMASK_ULL(11, 10)
343 #define CMDQ_TLBI_1_VA_MASK		GENMASK_ULL(63, 12)
344 #define CMDQ_TLBI_1_IPA_MASK		GENMASK_ULL(51, 12)
345 
346 #define CMDQ_ATC_0_SSID			GENMASK_ULL(31, 12)
347 #define CMDQ_ATC_0_SID			GENMASK_ULL(63, 32)
348 #define CMDQ_ATC_0_GLOBAL		(1UL << 9)
349 #define CMDQ_ATC_1_SIZE			GENMASK_ULL(5, 0)
350 #define CMDQ_ATC_1_ADDR_MASK		GENMASK_ULL(63, 12)
351 
352 #define CMDQ_PRI_0_SSID			GENMASK_ULL(31, 12)
353 #define CMDQ_PRI_0_SID			GENMASK_ULL(63, 32)
354 #define CMDQ_PRI_1_GRPID		GENMASK_ULL(8, 0)
355 #define CMDQ_PRI_1_RESP			GENMASK_ULL(13, 12)
356 
357 #define CMDQ_RESUME_0_RESP_TERM		0UL
358 #define CMDQ_RESUME_0_RESP_RETRY	1UL
359 #define CMDQ_RESUME_0_RESP_ABORT	2UL
360 #define CMDQ_RESUME_0_RESP		GENMASK_ULL(13, 12)
361 #define CMDQ_RESUME_0_SID		GENMASK_ULL(63, 32)
362 #define CMDQ_RESUME_1_STAG		GENMASK_ULL(15, 0)
363 
364 #define CMDQ_SYNC_0_CS			GENMASK_ULL(13, 12)
365 #define CMDQ_SYNC_0_CS_NONE		0
366 #define CMDQ_SYNC_0_CS_IRQ		1
367 #define CMDQ_SYNC_0_CS_SEV		2
368 #define CMDQ_SYNC_0_MSH			GENMASK_ULL(23, 22)
369 #define CMDQ_SYNC_0_MSIATTR		GENMASK_ULL(27, 24)
370 #define CMDQ_SYNC_0_MSIDATA		GENMASK_ULL(63, 32)
371 #define CMDQ_SYNC_1_MSIADDR_MASK	GENMASK_ULL(51, 2)
372 
373 /* Event queue */
374 #define EVTQ_ENT_SZ_SHIFT		5
375 #define EVTQ_ENT_DWORDS			((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
376 #define EVTQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
377 
378 #define EVTQ_0_ID			GENMASK_ULL(7, 0)
379 
380 #define EVT_ID_TRANSLATION_FAULT	0x10
381 #define EVT_ID_ADDR_SIZE_FAULT		0x11
382 #define EVT_ID_ACCESS_FAULT		0x12
383 #define EVT_ID_PERMISSION_FAULT		0x13
384 
385 #define EVTQ_0_SSV			(1UL << 11)
386 #define EVTQ_0_SSID			GENMASK_ULL(31, 12)
387 #define EVTQ_0_SID			GENMASK_ULL(63, 32)
388 #define EVTQ_1_STAG			GENMASK_ULL(15, 0)
389 #define EVTQ_1_STALL			(1UL << 31)
390 #define EVTQ_1_PnU			(1UL << 33)
391 #define EVTQ_1_InD			(1UL << 34)
392 #define EVTQ_1_RnW			(1UL << 35)
393 #define EVTQ_1_S2			(1UL << 39)
394 #define EVTQ_1_CLASS			GENMASK_ULL(41, 40)
395 #define EVTQ_1_TT_READ			(1UL << 44)
396 #define EVTQ_2_ADDR			GENMASK_ULL(63, 0)
397 #define EVTQ_3_IPA			GENMASK_ULL(51, 12)
398 
399 /* PRI queue */
400 #define PRIQ_ENT_SZ_SHIFT		4
401 #define PRIQ_ENT_DWORDS			((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
402 #define PRIQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
403 
404 #define PRIQ_0_SID			GENMASK_ULL(31, 0)
405 #define PRIQ_0_SSID			GENMASK_ULL(51, 32)
406 #define PRIQ_0_PERM_PRIV		(1UL << 58)
407 #define PRIQ_0_PERM_EXEC		(1UL << 59)
408 #define PRIQ_0_PERM_READ		(1UL << 60)
409 #define PRIQ_0_PERM_WRITE		(1UL << 61)
410 #define PRIQ_0_PRG_LAST			(1UL << 62)
411 #define PRIQ_0_SSID_V			(1UL << 63)
412 
413 #define PRIQ_1_PRG_IDX			GENMASK_ULL(8, 0)
414 #define PRIQ_1_ADDR_MASK		GENMASK_ULL(63, 12)
415 
416 /* High-level queue structures */
417 #define ARM_SMMU_POLL_TIMEOUT_US	1000000 /* 1s! */
418 #define ARM_SMMU_POLL_SPIN_COUNT	10
419 
420 #define MSI_IOVA_BASE			0x8000000
421 #define MSI_IOVA_LENGTH			0x100000
422 
423 enum pri_resp {
424 	PRI_RESP_DENY = 0,
425 	PRI_RESP_FAIL = 1,
426 	PRI_RESP_SUCC = 2,
427 };
428 
429 struct arm_smmu_cmdq_ent {
430 	/* Common fields */
431 	u8				opcode;
432 	bool				substream_valid;
433 
434 	/* Command-specific fields */
435 	union {
436 		#define CMDQ_OP_PREFETCH_CFG	0x1
437 		struct {
438 			u32			sid;
439 		} prefetch;
440 
441 		#define CMDQ_OP_CFGI_STE	0x3
442 		#define CMDQ_OP_CFGI_ALL	0x4
443 		#define CMDQ_OP_CFGI_CD		0x5
444 		#define CMDQ_OP_CFGI_CD_ALL	0x6
445 		struct {
446 			u32			sid;
447 			u32			ssid;
448 			union {
449 				bool		leaf;
450 				u8		span;
451 			};
452 		} cfgi;
453 
454 		#define CMDQ_OP_TLBI_NH_ASID	0x11
455 		#define CMDQ_OP_TLBI_NH_VA	0x12
456 		#define CMDQ_OP_TLBI_EL2_ALL	0x20
457 		#define CMDQ_OP_TLBI_EL2_ASID	0x21
458 		#define CMDQ_OP_TLBI_EL2_VA	0x22
459 		#define CMDQ_OP_TLBI_S12_VMALL	0x28
460 		#define CMDQ_OP_TLBI_S2_IPA	0x2a
461 		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
462 		struct {
463 			u8			num;
464 			u8			scale;
465 			u16			asid;
466 			u16			vmid;
467 			bool			leaf;
468 			u8			ttl;
469 			u8			tg;
470 			u64			addr;
471 		} tlbi;
472 
473 		#define CMDQ_OP_ATC_INV		0x40
474 		#define ATC_INV_SIZE_ALL	52
475 		struct {
476 			u32			sid;
477 			u32			ssid;
478 			u64			addr;
479 			u8			size;
480 			bool			global;
481 		} atc;
482 
483 		#define CMDQ_OP_PRI_RESP	0x41
484 		struct {
485 			u32			sid;
486 			u32			ssid;
487 			u16			grpid;
488 			enum pri_resp		resp;
489 		} pri;
490 
491 		#define CMDQ_OP_RESUME		0x44
492 		struct {
493 			u32			sid;
494 			u16			stag;
495 			u8			resp;
496 		} resume;
497 
498 		#define CMDQ_OP_CMD_SYNC	0x46
499 		struct {
500 			u64			msiaddr;
501 		} sync;
502 	};
503 };
504 
505 struct arm_smmu_ll_queue {
506 	union {
507 		u64			val;
508 		struct {
509 			u32		prod;
510 			u32		cons;
511 		};
512 		struct {
513 			atomic_t	prod;
514 			atomic_t	cons;
515 		} atomic;
516 		u8			__pad[SMP_CACHE_BYTES];
517 	} ____cacheline_aligned_in_smp;
518 	u32				max_n_shift;
519 };
520 
521 struct arm_smmu_queue {
522 	struct arm_smmu_ll_queue	llq;
523 	int				irq; /* Wired interrupt */
524 
525 	__le64				*base;
526 	dma_addr_t			base_dma;
527 	u64				q_base;
528 
529 	size_t				ent_dwords;
530 
531 	u32 __iomem			*prod_reg;
532 	u32 __iomem			*cons_reg;
533 };
534 
535 struct arm_smmu_queue_poll {
536 	ktime_t				timeout;
537 	unsigned int			delay;
538 	unsigned int			spin_cnt;
539 	bool				wfe;
540 };
541 
542 struct arm_smmu_cmdq {
543 	struct arm_smmu_queue		q;
544 	atomic_long_t			*valid_map;
545 	atomic_t			owner_prod;
546 	atomic_t			lock;
547 };
548 
549 struct arm_smmu_cmdq_batch {
550 	u64				cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
551 	int				num;
552 };
553 
554 struct arm_smmu_evtq {
555 	struct arm_smmu_queue		q;
556 	struct iopf_queue		*iopf;
557 	u32				max_stalls;
558 };
559 
560 struct arm_smmu_priq {
561 	struct arm_smmu_queue		q;
562 };
563 
564 /* High-level stream table and context descriptor structures */
565 struct arm_smmu_strtab_l1_desc {
566 	u8				span;
567 
568 	__le64				*l2ptr;
569 	dma_addr_t			l2ptr_dma;
570 };
571 
572 struct arm_smmu_ctx_desc {
573 	u16				asid;
574 	u64				ttbr;
575 	u64				tcr;
576 	u64				mair;
577 
578 	refcount_t			refs;
579 	struct mm_struct		*mm;
580 };
581 
582 struct arm_smmu_l1_ctx_desc {
583 	__le64				*l2ptr;
584 	dma_addr_t			l2ptr_dma;
585 };
586 
587 struct arm_smmu_ctx_desc_cfg {
588 	__le64				*cdtab;
589 	dma_addr_t			cdtab_dma;
590 	struct arm_smmu_l1_ctx_desc	*l1_desc;
591 	unsigned int			num_l1_ents;
592 };
593 
594 struct arm_smmu_s1_cfg {
595 	struct arm_smmu_ctx_desc_cfg	cdcfg;
596 	struct arm_smmu_ctx_desc	cd;
597 	u8				s1fmt;
598 	u8				s1cdmax;
599 };
600 
601 struct arm_smmu_s2_cfg {
602 	u16				vmid;
603 	u64				vttbr;
604 	u64				vtcr;
605 };
606 
607 struct arm_smmu_strtab_cfg {
608 	__le64				*strtab;
609 	dma_addr_t			strtab_dma;
610 	struct arm_smmu_strtab_l1_desc	*l1_desc;
611 	unsigned int			num_l1_ents;
612 
613 	u64				strtab_base;
614 	u32				strtab_base_cfg;
615 };
616 
617 /* An SMMUv3 instance */
618 struct arm_smmu_device {
619 	struct device			*dev;
620 	void __iomem			*base;
621 	void __iomem			*page1;
622 
623 #define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
624 #define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
625 #define ARM_SMMU_FEAT_TT_LE		(1 << 2)
626 #define ARM_SMMU_FEAT_TT_BE		(1 << 3)
627 #define ARM_SMMU_FEAT_PRI		(1 << 4)
628 #define ARM_SMMU_FEAT_ATS		(1 << 5)
629 #define ARM_SMMU_FEAT_SEV		(1 << 6)
630 #define ARM_SMMU_FEAT_MSI		(1 << 7)
631 #define ARM_SMMU_FEAT_COHERENCY		(1 << 8)
632 #define ARM_SMMU_FEAT_TRANS_S1		(1 << 9)
633 #define ARM_SMMU_FEAT_TRANS_S2		(1 << 10)
634 #define ARM_SMMU_FEAT_STALLS		(1 << 11)
635 #define ARM_SMMU_FEAT_HYP		(1 << 12)
636 #define ARM_SMMU_FEAT_STALL_FORCE	(1 << 13)
637 #define ARM_SMMU_FEAT_VAX		(1 << 14)
638 #define ARM_SMMU_FEAT_RANGE_INV		(1 << 15)
639 #define ARM_SMMU_FEAT_BTM		(1 << 16)
640 #define ARM_SMMU_FEAT_SVA		(1 << 17)
641 #define ARM_SMMU_FEAT_E2H		(1 << 18)
642 	u32				features;
643 
644 #define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
645 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY	(1 << 1)
646 #define ARM_SMMU_OPT_MSIPOLL		(1 << 2)
647 	u32				options;
648 
649 	struct arm_smmu_cmdq		cmdq;
650 	struct arm_smmu_evtq		evtq;
651 	struct arm_smmu_priq		priq;
652 
653 	int				gerr_irq;
654 	int				combined_irq;
655 
656 	unsigned long			ias; /* IPA */
657 	unsigned long			oas; /* PA */
658 	unsigned long			pgsize_bitmap;
659 
660 #define ARM_SMMU_MAX_ASIDS		(1 << 16)
661 	unsigned int			asid_bits;
662 
663 #define ARM_SMMU_MAX_VMIDS		(1 << 16)
664 	unsigned int			vmid_bits;
665 	DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
666 
667 	unsigned int			ssid_bits;
668 	unsigned int			sid_bits;
669 
670 	struct arm_smmu_strtab_cfg	strtab_cfg;
671 
672 	/* IOMMU core code handle */
673 	struct iommu_device		iommu;
674 
675 	struct rb_root			streams;
676 	struct mutex			streams_mutex;
677 };
678 
679 struct arm_smmu_stream {
680 	u32				id;
681 	struct arm_smmu_master		*master;
682 	struct rb_node			node;
683 };
684 
685 /* SMMU private data for each master */
686 struct arm_smmu_master {
687 	struct arm_smmu_device		*smmu;
688 	struct device			*dev;
689 	struct arm_smmu_domain		*domain;
690 	struct list_head		domain_head;
691 	struct arm_smmu_stream		*streams;
692 	unsigned int			num_streams;
693 	bool				ats_enabled;
694 	bool				stall_enabled;
695 	bool				sva_enabled;
696 	bool				iopf_enabled;
697 	struct list_head		bonds;
698 	unsigned int			ssid_bits;
699 };
700 
701 /* SMMU private data for an IOMMU domain */
702 enum arm_smmu_domain_stage {
703 	ARM_SMMU_DOMAIN_S1 = 0,
704 	ARM_SMMU_DOMAIN_S2,
705 	ARM_SMMU_DOMAIN_NESTED,
706 	ARM_SMMU_DOMAIN_BYPASS,
707 };
708 
709 struct arm_smmu_domain {
710 	struct arm_smmu_device		*smmu;
711 	struct mutex			init_mutex; /* Protects smmu pointer */
712 
713 	struct io_pgtable_ops		*pgtbl_ops;
714 	bool				stall_enabled;
715 	atomic_t			nr_ats_masters;
716 
717 	enum arm_smmu_domain_stage	stage;
718 	union {
719 		struct arm_smmu_s1_cfg	s1_cfg;
720 		struct arm_smmu_s2_cfg	s2_cfg;
721 	};
722 
723 	struct iommu_domain		domain;
724 
725 	struct list_head		devices;
726 	spinlock_t			devices_lock;
727 
728 	struct list_head		mmu_notifiers;
729 };
730 
731 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
732 {
733 	return container_of(dom, struct arm_smmu_domain, domain);
734 }
735 
736 extern struct xarray arm_smmu_asid_xa;
737 extern struct mutex arm_smmu_asid_lock;
738 extern struct arm_smmu_ctx_desc quiet_cd;
739 
740 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
741 			    struct arm_smmu_ctx_desc *cd);
742 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
743 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
744 				 size_t granule, bool leaf,
745 				 struct arm_smmu_domain *smmu_domain);
746 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
747 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
748 			    unsigned long iova, size_t size);
749 
750 #ifdef CONFIG_ARM_SMMU_V3_SVA
751 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
752 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
753 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
754 int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
755 int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
756 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master);
757 void arm_smmu_sva_notifier_synchronize(void);
758 struct iommu_domain *arm_smmu_sva_domain_alloc(void);
759 void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
760 				   struct device *dev, ioasid_t id);
761 #else /* CONFIG_ARM_SMMU_V3_SVA */
762 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
763 {
764 	return false;
765 }
766 
767 static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
768 {
769 	return false;
770 }
771 
772 static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
773 {
774 	return false;
775 }
776 
777 static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
778 {
779 	return -ENODEV;
780 }
781 
782 static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
783 {
784 	return -ENODEV;
785 }
786 
787 static inline bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
788 {
789 	return false;
790 }
791 
792 static inline void arm_smmu_sva_notifier_synchronize(void) {}
793 
794 static inline struct iommu_domain *arm_smmu_sva_domain_alloc(void)
795 {
796 	return NULL;
797 }
798 
799 static inline void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
800 						 struct device *dev,
801 						 ioasid_t id)
802 {
803 }
804 #endif /* CONFIG_ARM_SMMU_V3_SVA */
805 #endif /* _ARM_SMMU_V3_H */
806