1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * IOMMU API for ARM architected SMMUv3 implementations.
4  *
5  * Copyright (C) 2015 ARM Limited
6  */
7 
8 #ifndef _ARM_SMMU_V3_H
9 #define _ARM_SMMU_V3_H
10 
11 #include <linux/bitfield.h>
12 #include <linux/iommu.h>
13 #include <linux/kernel.h>
14 #include <linux/mmzone.h>
15 #include <linux/sizes.h>
16 
17 /* MMIO registers */
18 #define ARM_SMMU_IDR0			0x0
19 #define IDR0_ST_LVL			GENMASK(28, 27)
20 #define IDR0_ST_LVL_2LVL		1
21 #define IDR0_STALL_MODEL		GENMASK(25, 24)
22 #define IDR0_STALL_MODEL_STALL		0
23 #define IDR0_STALL_MODEL_FORCE		2
24 #define IDR0_TTENDIAN			GENMASK(22, 21)
25 #define IDR0_TTENDIAN_MIXED		0
26 #define IDR0_TTENDIAN_LE		2
27 #define IDR0_TTENDIAN_BE		3
28 #define IDR0_CD2L			(1 << 19)
29 #define IDR0_VMID16			(1 << 18)
30 #define IDR0_PRI			(1 << 16)
31 #define IDR0_SEV			(1 << 14)
32 #define IDR0_MSI			(1 << 13)
33 #define IDR0_ASID16			(1 << 12)
34 #define IDR0_ATS			(1 << 10)
35 #define IDR0_HYP			(1 << 9)
36 #define IDR0_COHACC			(1 << 4)
37 #define IDR0_TTF			GENMASK(3, 2)
38 #define IDR0_TTF_AARCH64		2
39 #define IDR0_TTF_AARCH32_64		3
40 #define IDR0_S1P			(1 << 1)
41 #define IDR0_S2P			(1 << 0)
42 
43 #define ARM_SMMU_IDR1			0x4
44 #define IDR1_TABLES_PRESET		(1 << 30)
45 #define IDR1_QUEUES_PRESET		(1 << 29)
46 #define IDR1_REL			(1 << 28)
47 #define IDR1_CMDQS			GENMASK(25, 21)
48 #define IDR1_EVTQS			GENMASK(20, 16)
49 #define IDR1_PRIQS			GENMASK(15, 11)
50 #define IDR1_SSIDSIZE			GENMASK(10, 6)
51 #define IDR1_SIDSIZE			GENMASK(5, 0)
52 
53 #define ARM_SMMU_IDR3			0xc
54 #define IDR3_RIL			(1 << 10)
55 
56 #define ARM_SMMU_IDR5			0x14
57 #define IDR5_STALL_MAX			GENMASK(31, 16)
58 #define IDR5_GRAN64K			(1 << 6)
59 #define IDR5_GRAN16K			(1 << 5)
60 #define IDR5_GRAN4K			(1 << 4)
61 #define IDR5_OAS			GENMASK(2, 0)
62 #define IDR5_OAS_32_BIT			0
63 #define IDR5_OAS_36_BIT			1
64 #define IDR5_OAS_40_BIT			2
65 #define IDR5_OAS_42_BIT			3
66 #define IDR5_OAS_44_BIT			4
67 #define IDR5_OAS_48_BIT			5
68 #define IDR5_OAS_52_BIT			6
69 #define IDR5_VAX			GENMASK(11, 10)
70 #define IDR5_VAX_52_BIT			1
71 
72 #define ARM_SMMU_CR0			0x20
73 #define CR0_ATSCHK			(1 << 4)
74 #define CR0_CMDQEN			(1 << 3)
75 #define CR0_EVTQEN			(1 << 2)
76 #define CR0_PRIQEN			(1 << 1)
77 #define CR0_SMMUEN			(1 << 0)
78 
79 #define ARM_SMMU_CR0ACK			0x24
80 
81 #define ARM_SMMU_CR1			0x28
82 #define CR1_TABLE_SH			GENMASK(11, 10)
83 #define CR1_TABLE_OC			GENMASK(9, 8)
84 #define CR1_TABLE_IC			GENMASK(7, 6)
85 #define CR1_QUEUE_SH			GENMASK(5, 4)
86 #define CR1_QUEUE_OC			GENMASK(3, 2)
87 #define CR1_QUEUE_IC			GENMASK(1, 0)
88 /* CR1 cacheability fields don't quite follow the usual TCR-style encoding */
89 #define CR1_CACHE_NC			0
90 #define CR1_CACHE_WB			1
91 #define CR1_CACHE_WT			2
92 
93 #define ARM_SMMU_CR2			0x2c
94 #define CR2_PTM				(1 << 2)
95 #define CR2_RECINVSID			(1 << 1)
96 #define CR2_E2H				(1 << 0)
97 
98 #define ARM_SMMU_GBPA			0x44
99 #define GBPA_UPDATE			(1 << 31)
100 #define GBPA_ABORT			(1 << 20)
101 
102 #define ARM_SMMU_IRQ_CTRL		0x50
103 #define IRQ_CTRL_EVTQ_IRQEN		(1 << 2)
104 #define IRQ_CTRL_PRIQ_IRQEN		(1 << 1)
105 #define IRQ_CTRL_GERROR_IRQEN		(1 << 0)
106 
107 #define ARM_SMMU_IRQ_CTRLACK		0x54
108 
109 #define ARM_SMMU_GERROR			0x60
110 #define GERROR_SFM_ERR			(1 << 8)
111 #define GERROR_MSI_GERROR_ABT_ERR	(1 << 7)
112 #define GERROR_MSI_PRIQ_ABT_ERR		(1 << 6)
113 #define GERROR_MSI_EVTQ_ABT_ERR		(1 << 5)
114 #define GERROR_MSI_CMDQ_ABT_ERR		(1 << 4)
115 #define GERROR_PRIQ_ABT_ERR		(1 << 3)
116 #define GERROR_EVTQ_ABT_ERR		(1 << 2)
117 #define GERROR_CMDQ_ERR			(1 << 0)
118 #define GERROR_ERR_MASK			0x1fd
119 
120 #define ARM_SMMU_GERRORN		0x64
121 
122 #define ARM_SMMU_GERROR_IRQ_CFG0	0x68
123 #define ARM_SMMU_GERROR_IRQ_CFG1	0x70
124 #define ARM_SMMU_GERROR_IRQ_CFG2	0x74
125 
126 #define ARM_SMMU_STRTAB_BASE		0x80
127 #define STRTAB_BASE_RA			(1UL << 62)
128 #define STRTAB_BASE_ADDR_MASK		GENMASK_ULL(51, 6)
129 
130 #define ARM_SMMU_STRTAB_BASE_CFG	0x88
131 #define STRTAB_BASE_CFG_FMT		GENMASK(17, 16)
132 #define STRTAB_BASE_CFG_FMT_LINEAR	0
133 #define STRTAB_BASE_CFG_FMT_2LVL	1
134 #define STRTAB_BASE_CFG_SPLIT		GENMASK(10, 6)
135 #define STRTAB_BASE_CFG_LOG2SIZE	GENMASK(5, 0)
136 
137 #define ARM_SMMU_CMDQ_BASE		0x90
138 #define ARM_SMMU_CMDQ_PROD		0x98
139 #define ARM_SMMU_CMDQ_CONS		0x9c
140 
141 #define ARM_SMMU_EVTQ_BASE		0xa0
142 #define ARM_SMMU_EVTQ_PROD		0xa8
143 #define ARM_SMMU_EVTQ_CONS		0xac
144 #define ARM_SMMU_EVTQ_IRQ_CFG0		0xb0
145 #define ARM_SMMU_EVTQ_IRQ_CFG1		0xb8
146 #define ARM_SMMU_EVTQ_IRQ_CFG2		0xbc
147 
148 #define ARM_SMMU_PRIQ_BASE		0xc0
149 #define ARM_SMMU_PRIQ_PROD		0xc8
150 #define ARM_SMMU_PRIQ_CONS		0xcc
151 #define ARM_SMMU_PRIQ_IRQ_CFG0		0xd0
152 #define ARM_SMMU_PRIQ_IRQ_CFG1		0xd8
153 #define ARM_SMMU_PRIQ_IRQ_CFG2		0xdc
154 
155 #define ARM_SMMU_REG_SZ			0xe00
156 
157 /* Common MSI config fields */
158 #define MSI_CFG0_ADDR_MASK		GENMASK_ULL(51, 2)
159 #define MSI_CFG2_SH			GENMASK(5, 4)
160 #define MSI_CFG2_MEMATTR		GENMASK(3, 0)
161 
162 /* Common memory attribute values */
163 #define ARM_SMMU_SH_NSH			0
164 #define ARM_SMMU_SH_OSH			2
165 #define ARM_SMMU_SH_ISH			3
166 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE	0x1
167 #define ARM_SMMU_MEMATTR_OIWB		0xf
168 
169 #define Q_IDX(llq, p)			((p) & ((1 << (llq)->max_n_shift) - 1))
170 #define Q_WRP(llq, p)			((p) & (1 << (llq)->max_n_shift))
171 #define Q_OVERFLOW_FLAG			(1U << 31)
172 #define Q_OVF(p)			((p) & Q_OVERFLOW_FLAG)
173 #define Q_ENT(q, p)			((q)->base +			\
174 					 Q_IDX(&((q)->llq), p) *	\
175 					 (q)->ent_dwords)
176 
177 #define Q_BASE_RWA			(1UL << 62)
178 #define Q_BASE_ADDR_MASK		GENMASK_ULL(51, 5)
179 #define Q_BASE_LOG2SIZE			GENMASK(4, 0)
180 
181 /* Ensure DMA allocations are naturally aligned */
182 #ifdef CONFIG_CMA_ALIGNMENT
183 #define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
184 #else
185 #define Q_MAX_SZ_SHIFT			(PAGE_SHIFT + MAX_ORDER - 1)
186 #endif
187 
188 /*
189  * Stream table.
190  *
191  * Linear: Enough to cover 1 << IDR1.SIDSIZE entries
192  * 2lvl: 128k L1 entries,
193  *       256 lazy entries per table (each table covers a PCI bus)
194  */
195 #define STRTAB_L1_SZ_SHIFT		20
196 #define STRTAB_SPLIT			8
197 
198 #define STRTAB_L1_DESC_DWORDS		1
199 #define STRTAB_L1_DESC_SPAN		GENMASK_ULL(4, 0)
200 #define STRTAB_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 6)
201 
202 #define STRTAB_STE_DWORDS		8
203 #define STRTAB_STE_0_V			(1UL << 0)
204 #define STRTAB_STE_0_CFG		GENMASK_ULL(3, 1)
205 #define STRTAB_STE_0_CFG_ABORT		0
206 #define STRTAB_STE_0_CFG_BYPASS		4
207 #define STRTAB_STE_0_CFG_S1_TRANS	5
208 #define STRTAB_STE_0_CFG_S2_TRANS	6
209 
210 #define STRTAB_STE_0_S1FMT		GENMASK_ULL(5, 4)
211 #define STRTAB_STE_0_S1FMT_LINEAR	0
212 #define STRTAB_STE_0_S1FMT_64K_L2	2
213 #define STRTAB_STE_0_S1CTXPTR_MASK	GENMASK_ULL(51, 6)
214 #define STRTAB_STE_0_S1CDMAX		GENMASK_ULL(63, 59)
215 
216 #define STRTAB_STE_1_S1DSS		GENMASK_ULL(1, 0)
217 #define STRTAB_STE_1_S1DSS_TERMINATE	0x0
218 #define STRTAB_STE_1_S1DSS_BYPASS	0x1
219 #define STRTAB_STE_1_S1DSS_SSID0	0x2
220 
221 #define STRTAB_STE_1_S1C_CACHE_NC	0UL
222 #define STRTAB_STE_1_S1C_CACHE_WBRA	1UL
223 #define STRTAB_STE_1_S1C_CACHE_WT	2UL
224 #define STRTAB_STE_1_S1C_CACHE_WB	3UL
225 #define STRTAB_STE_1_S1CIR		GENMASK_ULL(3, 2)
226 #define STRTAB_STE_1_S1COR		GENMASK_ULL(5, 4)
227 #define STRTAB_STE_1_S1CSH		GENMASK_ULL(7, 6)
228 
229 #define STRTAB_STE_1_S1STALLD		(1UL << 27)
230 
231 #define STRTAB_STE_1_EATS		GENMASK_ULL(29, 28)
232 #define STRTAB_STE_1_EATS_ABT		0UL
233 #define STRTAB_STE_1_EATS_TRANS		1UL
234 #define STRTAB_STE_1_EATS_S1CHK		2UL
235 
236 #define STRTAB_STE_1_STRW		GENMASK_ULL(31, 30)
237 #define STRTAB_STE_1_STRW_NSEL1		0UL
238 #define STRTAB_STE_1_STRW_EL2		2UL
239 
240 #define STRTAB_STE_1_SHCFG		GENMASK_ULL(45, 44)
241 #define STRTAB_STE_1_SHCFG_INCOMING	1UL
242 
243 #define STRTAB_STE_2_S2VMID		GENMASK_ULL(15, 0)
244 #define STRTAB_STE_2_VTCR		GENMASK_ULL(50, 32)
245 #define STRTAB_STE_2_VTCR_S2T0SZ	GENMASK_ULL(5, 0)
246 #define STRTAB_STE_2_VTCR_S2SL0		GENMASK_ULL(7, 6)
247 #define STRTAB_STE_2_VTCR_S2IR0		GENMASK_ULL(9, 8)
248 #define STRTAB_STE_2_VTCR_S2OR0		GENMASK_ULL(11, 10)
249 #define STRTAB_STE_2_VTCR_S2SH0		GENMASK_ULL(13, 12)
250 #define STRTAB_STE_2_VTCR_S2TG		GENMASK_ULL(15, 14)
251 #define STRTAB_STE_2_VTCR_S2PS		GENMASK_ULL(18, 16)
252 #define STRTAB_STE_2_S2AA64		(1UL << 51)
253 #define STRTAB_STE_2_S2ENDI		(1UL << 52)
254 #define STRTAB_STE_2_S2PTW		(1UL << 54)
255 #define STRTAB_STE_2_S2R		(1UL << 58)
256 
257 #define STRTAB_STE_3_S2TTB_MASK		GENMASK_ULL(51, 4)
258 
259 /*
260  * Context descriptors.
261  *
262  * Linear: when less than 1024 SSIDs are supported
263  * 2lvl: at most 1024 L1 entries,
264  *       1024 lazy entries per table.
265  */
266 #define CTXDESC_SPLIT			10
267 #define CTXDESC_L2_ENTRIES		(1 << CTXDESC_SPLIT)
268 
269 #define CTXDESC_L1_DESC_DWORDS		1
270 #define CTXDESC_L1_DESC_V		(1UL << 0)
271 #define CTXDESC_L1_DESC_L2PTR_MASK	GENMASK_ULL(51, 12)
272 
273 #define CTXDESC_CD_DWORDS		8
274 #define CTXDESC_CD_0_TCR_T0SZ		GENMASK_ULL(5, 0)
275 #define CTXDESC_CD_0_TCR_TG0		GENMASK_ULL(7, 6)
276 #define CTXDESC_CD_0_TCR_IRGN0		GENMASK_ULL(9, 8)
277 #define CTXDESC_CD_0_TCR_ORGN0		GENMASK_ULL(11, 10)
278 #define CTXDESC_CD_0_TCR_SH0		GENMASK_ULL(13, 12)
279 #define CTXDESC_CD_0_TCR_EPD0		(1ULL << 14)
280 #define CTXDESC_CD_0_TCR_EPD1		(1ULL << 30)
281 
282 #define CTXDESC_CD_0_ENDI		(1UL << 15)
283 #define CTXDESC_CD_0_V			(1UL << 31)
284 
285 #define CTXDESC_CD_0_TCR_IPS		GENMASK_ULL(34, 32)
286 #define CTXDESC_CD_0_TCR_TBI0		(1ULL << 38)
287 
288 #define CTXDESC_CD_0_AA64		(1UL << 41)
289 #define CTXDESC_CD_0_S			(1UL << 44)
290 #define CTXDESC_CD_0_R			(1UL << 45)
291 #define CTXDESC_CD_0_A			(1UL << 46)
292 #define CTXDESC_CD_0_ASET		(1UL << 47)
293 #define CTXDESC_CD_0_ASID		GENMASK_ULL(63, 48)
294 
295 #define CTXDESC_CD_1_TTB0_MASK		GENMASK_ULL(51, 4)
296 
297 /*
298  * When the SMMU only supports linear context descriptor tables, pick a
299  * reasonable size limit (64kB).
300  */
301 #define CTXDESC_LINEAR_CDMAX		ilog2(SZ_64K / (CTXDESC_CD_DWORDS << 3))
302 
303 /* Command queue */
304 #define CMDQ_ENT_SZ_SHIFT		4
305 #define CMDQ_ENT_DWORDS			((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
306 #define CMDQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
307 
308 #define CMDQ_CONS_ERR			GENMASK(30, 24)
309 #define CMDQ_ERR_CERROR_NONE_IDX	0
310 #define CMDQ_ERR_CERROR_ILL_IDX		1
311 #define CMDQ_ERR_CERROR_ABT_IDX		2
312 #define CMDQ_ERR_CERROR_ATC_INV_IDX	3
313 
314 #define CMDQ_PROD_OWNED_FLAG		Q_OVERFLOW_FLAG
315 
316 /*
317  * This is used to size the command queue and therefore must be at least
318  * BITS_PER_LONG so that the valid_map works correctly (it relies on the
319  * total number of queue entries being a multiple of BITS_PER_LONG).
320  */
321 #define CMDQ_BATCH_ENTRIES		BITS_PER_LONG
322 
323 #define CMDQ_0_OP			GENMASK_ULL(7, 0)
324 #define CMDQ_0_SSV			(1UL << 11)
325 
326 #define CMDQ_PREFETCH_0_SID		GENMASK_ULL(63, 32)
327 #define CMDQ_PREFETCH_1_SIZE		GENMASK_ULL(4, 0)
328 #define CMDQ_PREFETCH_1_ADDR_MASK	GENMASK_ULL(63, 12)
329 
330 #define CMDQ_CFGI_0_SSID		GENMASK_ULL(31, 12)
331 #define CMDQ_CFGI_0_SID			GENMASK_ULL(63, 32)
332 #define CMDQ_CFGI_1_LEAF		(1UL << 0)
333 #define CMDQ_CFGI_1_RANGE		GENMASK_ULL(4, 0)
334 
335 #define CMDQ_TLBI_0_NUM			GENMASK_ULL(16, 12)
336 #define CMDQ_TLBI_RANGE_NUM_MAX		31
337 #define CMDQ_TLBI_0_SCALE		GENMASK_ULL(24, 20)
338 #define CMDQ_TLBI_0_VMID		GENMASK_ULL(47, 32)
339 #define CMDQ_TLBI_0_ASID		GENMASK_ULL(63, 48)
340 #define CMDQ_TLBI_1_LEAF		(1UL << 0)
341 #define CMDQ_TLBI_1_TTL			GENMASK_ULL(9, 8)
342 #define CMDQ_TLBI_1_TG			GENMASK_ULL(11, 10)
343 #define CMDQ_TLBI_1_VA_MASK		GENMASK_ULL(63, 12)
344 #define CMDQ_TLBI_1_IPA_MASK		GENMASK_ULL(51, 12)
345 
346 #define CMDQ_ATC_0_SSID			GENMASK_ULL(31, 12)
347 #define CMDQ_ATC_0_SID			GENMASK_ULL(63, 32)
348 #define CMDQ_ATC_0_GLOBAL		(1UL << 9)
349 #define CMDQ_ATC_1_SIZE			GENMASK_ULL(5, 0)
350 #define CMDQ_ATC_1_ADDR_MASK		GENMASK_ULL(63, 12)
351 
352 #define CMDQ_PRI_0_SSID			GENMASK_ULL(31, 12)
353 #define CMDQ_PRI_0_SID			GENMASK_ULL(63, 32)
354 #define CMDQ_PRI_1_GRPID		GENMASK_ULL(8, 0)
355 #define CMDQ_PRI_1_RESP			GENMASK_ULL(13, 12)
356 
357 #define CMDQ_SYNC_0_CS			GENMASK_ULL(13, 12)
358 #define CMDQ_SYNC_0_CS_NONE		0
359 #define CMDQ_SYNC_0_CS_IRQ		1
360 #define CMDQ_SYNC_0_CS_SEV		2
361 #define CMDQ_SYNC_0_MSH			GENMASK_ULL(23, 22)
362 #define CMDQ_SYNC_0_MSIATTR		GENMASK_ULL(27, 24)
363 #define CMDQ_SYNC_0_MSIDATA		GENMASK_ULL(63, 32)
364 #define CMDQ_SYNC_1_MSIADDR_MASK	GENMASK_ULL(51, 2)
365 
366 /* Event queue */
367 #define EVTQ_ENT_SZ_SHIFT		5
368 #define EVTQ_ENT_DWORDS			((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
369 #define EVTQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
370 
371 #define EVTQ_0_ID			GENMASK_ULL(7, 0)
372 
373 /* PRI queue */
374 #define PRIQ_ENT_SZ_SHIFT		4
375 #define PRIQ_ENT_DWORDS			((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
376 #define PRIQ_MAX_SZ_SHIFT		(Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
377 
378 #define PRIQ_0_SID			GENMASK_ULL(31, 0)
379 #define PRIQ_0_SSID			GENMASK_ULL(51, 32)
380 #define PRIQ_0_PERM_PRIV		(1UL << 58)
381 #define PRIQ_0_PERM_EXEC		(1UL << 59)
382 #define PRIQ_0_PERM_READ		(1UL << 60)
383 #define PRIQ_0_PERM_WRITE		(1UL << 61)
384 #define PRIQ_0_PRG_LAST			(1UL << 62)
385 #define PRIQ_0_SSID_V			(1UL << 63)
386 
387 #define PRIQ_1_PRG_IDX			GENMASK_ULL(8, 0)
388 #define PRIQ_1_ADDR_MASK		GENMASK_ULL(63, 12)
389 
390 /* High-level queue structures */
391 #define ARM_SMMU_POLL_TIMEOUT_US	1000000 /* 1s! */
392 #define ARM_SMMU_POLL_SPIN_COUNT	10
393 
394 #define MSI_IOVA_BASE			0x8000000
395 #define MSI_IOVA_LENGTH			0x100000
396 
397 enum pri_resp {
398 	PRI_RESP_DENY = 0,
399 	PRI_RESP_FAIL = 1,
400 	PRI_RESP_SUCC = 2,
401 };
402 
403 struct arm_smmu_cmdq_ent {
404 	/* Common fields */
405 	u8				opcode;
406 	bool				substream_valid;
407 
408 	/* Command-specific fields */
409 	union {
410 		#define CMDQ_OP_PREFETCH_CFG	0x1
411 		struct {
412 			u32			sid;
413 		} prefetch;
414 
415 		#define CMDQ_OP_CFGI_STE	0x3
416 		#define CMDQ_OP_CFGI_ALL	0x4
417 		#define CMDQ_OP_CFGI_CD		0x5
418 		#define CMDQ_OP_CFGI_CD_ALL	0x6
419 		struct {
420 			u32			sid;
421 			u32			ssid;
422 			union {
423 				bool		leaf;
424 				u8		span;
425 			};
426 		} cfgi;
427 
428 		#define CMDQ_OP_TLBI_NH_ASID	0x11
429 		#define CMDQ_OP_TLBI_NH_VA	0x12
430 		#define CMDQ_OP_TLBI_EL2_ALL	0x20
431 		#define CMDQ_OP_TLBI_EL2_ASID	0x21
432 		#define CMDQ_OP_TLBI_EL2_VA	0x22
433 		#define CMDQ_OP_TLBI_S12_VMALL	0x28
434 		#define CMDQ_OP_TLBI_S2_IPA	0x2a
435 		#define CMDQ_OP_TLBI_NSNH_ALL	0x30
436 		struct {
437 			u8			num;
438 			u8			scale;
439 			u16			asid;
440 			u16			vmid;
441 			bool			leaf;
442 			u8			ttl;
443 			u8			tg;
444 			u64			addr;
445 		} tlbi;
446 
447 		#define CMDQ_OP_ATC_INV		0x40
448 		#define ATC_INV_SIZE_ALL	52
449 		struct {
450 			u32			sid;
451 			u32			ssid;
452 			u64			addr;
453 			u8			size;
454 			bool			global;
455 		} atc;
456 
457 		#define CMDQ_OP_PRI_RESP	0x41
458 		struct {
459 			u32			sid;
460 			u32			ssid;
461 			u16			grpid;
462 			enum pri_resp		resp;
463 		} pri;
464 
465 		#define CMDQ_OP_CMD_SYNC	0x46
466 		struct {
467 			u64			msiaddr;
468 		} sync;
469 	};
470 };
471 
472 struct arm_smmu_ll_queue {
473 	union {
474 		u64			val;
475 		struct {
476 			u32		prod;
477 			u32		cons;
478 		};
479 		struct {
480 			atomic_t	prod;
481 			atomic_t	cons;
482 		} atomic;
483 		u8			__pad[SMP_CACHE_BYTES];
484 	} ____cacheline_aligned_in_smp;
485 	u32				max_n_shift;
486 };
487 
488 struct arm_smmu_queue {
489 	struct arm_smmu_ll_queue	llq;
490 	int				irq; /* Wired interrupt */
491 
492 	__le64				*base;
493 	dma_addr_t			base_dma;
494 	u64				q_base;
495 
496 	size_t				ent_dwords;
497 
498 	u32 __iomem			*prod_reg;
499 	u32 __iomem			*cons_reg;
500 };
501 
502 struct arm_smmu_queue_poll {
503 	ktime_t				timeout;
504 	unsigned int			delay;
505 	unsigned int			spin_cnt;
506 	bool				wfe;
507 };
508 
509 struct arm_smmu_cmdq {
510 	struct arm_smmu_queue		q;
511 	atomic_long_t			*valid_map;
512 	atomic_t			owner_prod;
513 	atomic_t			lock;
514 };
515 
516 struct arm_smmu_cmdq_batch {
517 	u64				cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
518 	int				num;
519 };
520 
521 struct arm_smmu_evtq {
522 	struct arm_smmu_queue		q;
523 	u32				max_stalls;
524 };
525 
526 struct arm_smmu_priq {
527 	struct arm_smmu_queue		q;
528 };
529 
530 /* High-level stream table and context descriptor structures */
531 struct arm_smmu_strtab_l1_desc {
532 	u8				span;
533 
534 	__le64				*l2ptr;
535 	dma_addr_t			l2ptr_dma;
536 };
537 
538 struct arm_smmu_ctx_desc {
539 	u16				asid;
540 	u64				ttbr;
541 	u64				tcr;
542 	u64				mair;
543 
544 	refcount_t			refs;
545 	struct mm_struct		*mm;
546 };
547 
548 struct arm_smmu_l1_ctx_desc {
549 	__le64				*l2ptr;
550 	dma_addr_t			l2ptr_dma;
551 };
552 
553 struct arm_smmu_ctx_desc_cfg {
554 	__le64				*cdtab;
555 	dma_addr_t			cdtab_dma;
556 	struct arm_smmu_l1_ctx_desc	*l1_desc;
557 	unsigned int			num_l1_ents;
558 };
559 
560 struct arm_smmu_s1_cfg {
561 	struct arm_smmu_ctx_desc_cfg	cdcfg;
562 	struct arm_smmu_ctx_desc	cd;
563 	u8				s1fmt;
564 	u8				s1cdmax;
565 };
566 
567 struct arm_smmu_s2_cfg {
568 	u16				vmid;
569 	u64				vttbr;
570 	u64				vtcr;
571 };
572 
573 struct arm_smmu_strtab_cfg {
574 	__le64				*strtab;
575 	dma_addr_t			strtab_dma;
576 	struct arm_smmu_strtab_l1_desc	*l1_desc;
577 	unsigned int			num_l1_ents;
578 
579 	u64				strtab_base;
580 	u32				strtab_base_cfg;
581 };
582 
583 /* An SMMUv3 instance */
584 struct arm_smmu_device {
585 	struct device			*dev;
586 	void __iomem			*base;
587 	void __iomem			*page1;
588 
589 #define ARM_SMMU_FEAT_2_LVL_STRTAB	(1 << 0)
590 #define ARM_SMMU_FEAT_2_LVL_CDTAB	(1 << 1)
591 #define ARM_SMMU_FEAT_TT_LE		(1 << 2)
592 #define ARM_SMMU_FEAT_TT_BE		(1 << 3)
593 #define ARM_SMMU_FEAT_PRI		(1 << 4)
594 #define ARM_SMMU_FEAT_ATS		(1 << 5)
595 #define ARM_SMMU_FEAT_SEV		(1 << 6)
596 #define ARM_SMMU_FEAT_MSI		(1 << 7)
597 #define ARM_SMMU_FEAT_COHERENCY		(1 << 8)
598 #define ARM_SMMU_FEAT_TRANS_S1		(1 << 9)
599 #define ARM_SMMU_FEAT_TRANS_S2		(1 << 10)
600 #define ARM_SMMU_FEAT_STALLS		(1 << 11)
601 #define ARM_SMMU_FEAT_HYP		(1 << 12)
602 #define ARM_SMMU_FEAT_STALL_FORCE	(1 << 13)
603 #define ARM_SMMU_FEAT_VAX		(1 << 14)
604 #define ARM_SMMU_FEAT_RANGE_INV		(1 << 15)
605 #define ARM_SMMU_FEAT_BTM		(1 << 16)
606 #define ARM_SMMU_FEAT_SVA		(1 << 17)
607 #define ARM_SMMU_FEAT_E2H		(1 << 18)
608 	u32				features;
609 
610 #define ARM_SMMU_OPT_SKIP_PREFETCH	(1 << 0)
611 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY	(1 << 1)
612 #define ARM_SMMU_OPT_MSIPOLL		(1 << 2)
613 	u32				options;
614 
615 	struct arm_smmu_cmdq		cmdq;
616 	struct arm_smmu_evtq		evtq;
617 	struct arm_smmu_priq		priq;
618 
619 	int				gerr_irq;
620 	int				combined_irq;
621 
622 	unsigned long			ias; /* IPA */
623 	unsigned long			oas; /* PA */
624 	unsigned long			pgsize_bitmap;
625 
626 #define ARM_SMMU_MAX_ASIDS		(1 << 16)
627 	unsigned int			asid_bits;
628 
629 #define ARM_SMMU_MAX_VMIDS		(1 << 16)
630 	unsigned int			vmid_bits;
631 	DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
632 
633 	unsigned int			ssid_bits;
634 	unsigned int			sid_bits;
635 
636 	struct arm_smmu_strtab_cfg	strtab_cfg;
637 
638 	/* IOMMU core code handle */
639 	struct iommu_device		iommu;
640 
641 	struct rb_root			streams;
642 	struct mutex			streams_mutex;
643 };
644 
645 struct arm_smmu_stream {
646 	u32				id;
647 	struct arm_smmu_master		*master;
648 	struct rb_node			node;
649 };
650 
651 /* SMMU private data for each master */
652 struct arm_smmu_master {
653 	struct arm_smmu_device		*smmu;
654 	struct device			*dev;
655 	struct arm_smmu_domain		*domain;
656 	struct list_head		domain_head;
657 	struct arm_smmu_stream		*streams;
658 	unsigned int			num_streams;
659 	bool				ats_enabled;
660 	bool				sva_enabled;
661 	struct list_head		bonds;
662 	unsigned int			ssid_bits;
663 };
664 
665 /* SMMU private data for an IOMMU domain */
666 enum arm_smmu_domain_stage {
667 	ARM_SMMU_DOMAIN_S1 = 0,
668 	ARM_SMMU_DOMAIN_S2,
669 	ARM_SMMU_DOMAIN_NESTED,
670 	ARM_SMMU_DOMAIN_BYPASS,
671 };
672 
673 struct arm_smmu_domain {
674 	struct arm_smmu_device		*smmu;
675 	struct mutex			init_mutex; /* Protects smmu pointer */
676 
677 	struct io_pgtable_ops		*pgtbl_ops;
678 	atomic_t			nr_ats_masters;
679 
680 	enum arm_smmu_domain_stage	stage;
681 	union {
682 		struct arm_smmu_s1_cfg	s1_cfg;
683 		struct arm_smmu_s2_cfg	s2_cfg;
684 	};
685 
686 	struct iommu_domain		domain;
687 
688 	struct list_head		devices;
689 	spinlock_t			devices_lock;
690 
691 	struct list_head		mmu_notifiers;
692 };
693 
694 static inline struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
695 {
696 	return container_of(dom, struct arm_smmu_domain, domain);
697 }
698 
699 extern struct xarray arm_smmu_asid_xa;
700 extern struct mutex arm_smmu_asid_lock;
701 extern struct arm_smmu_ctx_desc quiet_cd;
702 
703 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
704 			    struct arm_smmu_ctx_desc *cd);
705 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid);
706 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
707 				 size_t granule, bool leaf,
708 				 struct arm_smmu_domain *smmu_domain);
709 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
710 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
711 			    unsigned long iova, size_t size);
712 
713 #ifdef CONFIG_ARM_SMMU_V3_SVA
714 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);
715 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master);
716 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master);
717 int arm_smmu_master_enable_sva(struct arm_smmu_master *master);
718 int arm_smmu_master_disable_sva(struct arm_smmu_master *master);
719 struct iommu_sva *arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm,
720 				    void *drvdata);
721 void arm_smmu_sva_unbind(struct iommu_sva *handle);
722 u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle);
723 void arm_smmu_sva_notifier_synchronize(void);
724 #else /* CONFIG_ARM_SMMU_V3_SVA */
725 static inline bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
726 {
727 	return false;
728 }
729 
730 static inline bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
731 {
732 	return false;
733 }
734 
735 static inline bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
736 {
737 	return false;
738 }
739 
740 static inline int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
741 {
742 	return -ENODEV;
743 }
744 
745 static inline int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
746 {
747 	return -ENODEV;
748 }
749 
750 static inline struct iommu_sva *
751 arm_smmu_sva_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
752 {
753 	return ERR_PTR(-ENODEV);
754 }
755 
756 static inline void arm_smmu_sva_unbind(struct iommu_sva *handle) {}
757 
758 static inline u32 arm_smmu_sva_get_pasid(struct iommu_sva *handle)
759 {
760 	return IOMMU_PASID_INVALID;
761 }
762 
763 static inline void arm_smmu_sva_notifier_synchronize(void) {}
764 #endif /* CONFIG_ARM_SMMU_V3_SVA */
765 #endif /* _ARM_SMMU_V3_H */
766