1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * IOMMU API for ARM architected SMMU implementations.
4  *
5  * Copyright (C) 2013 ARM Limited
6  *
7  * Author: Will Deacon <will.deacon@arm.com>
8  */
9 
10 #ifndef _ARM_SMMU_H
11 #define _ARM_SMMU_H
12 
13 #include <linux/atomic.h>
14 #include <linux/bitfield.h>
15 #include <linux/bits.h>
16 #include <linux/clk.h>
17 #include <linux/device.h>
18 #include <linux/io-64-nonatomic-hi-lo.h>
19 #include <linux/io-pgtable.h>
20 #include <linux/iommu.h>
21 #include <linux/irqreturn.h>
22 #include <linux/mutex.h>
23 #include <linux/spinlock.h>
24 #include <linux/types.h>
25 
26 /* Configuration registers */
27 #define ARM_SMMU_GR0_sCR0		0x0
28 #define ARM_SMMU_sCR0_VMID16EN		BIT(31)
29 #define ARM_SMMU_sCR0_BSU		GENMASK(15, 14)
30 #define ARM_SMMU_sCR0_FB		BIT(13)
31 #define ARM_SMMU_sCR0_PTM		BIT(12)
32 #define ARM_SMMU_sCR0_VMIDPNE		BIT(11)
33 #define ARM_SMMU_sCR0_USFCFG		BIT(10)
34 #define ARM_SMMU_sCR0_GCFGFIE		BIT(5)
35 #define ARM_SMMU_sCR0_GCFGFRE		BIT(4)
36 #define ARM_SMMU_sCR0_EXIDENABLE	BIT(3)
37 #define ARM_SMMU_sCR0_GFIE		BIT(2)
38 #define ARM_SMMU_sCR0_GFRE		BIT(1)
39 #define ARM_SMMU_sCR0_CLIENTPD		BIT(0)
40 
41 /* Auxiliary Configuration register */
42 #define ARM_SMMU_GR0_sACR		0x10
43 
44 /* Identification registers */
45 #define ARM_SMMU_GR0_ID0		0x20
46 #define ARM_SMMU_ID0_S1TS		BIT(30)
47 #define ARM_SMMU_ID0_S2TS		BIT(29)
48 #define ARM_SMMU_ID0_NTS		BIT(28)
49 #define ARM_SMMU_ID0_SMS		BIT(27)
50 #define ARM_SMMU_ID0_ATOSNS		BIT(26)
51 #define ARM_SMMU_ID0_PTFS_NO_AARCH32	BIT(25)
52 #define ARM_SMMU_ID0_PTFS_NO_AARCH32S	BIT(24)
53 #define ARM_SMMU_ID0_NUMIRPT		GENMASK(23, 16)
54 #define ARM_SMMU_ID0_CTTW		BIT(14)
55 #define ARM_SMMU_ID0_NUMSIDB		GENMASK(12, 9)
56 #define ARM_SMMU_ID0_EXIDS		BIT(8)
57 #define ARM_SMMU_ID0_NUMSMRG		GENMASK(7, 0)
58 
59 #define ARM_SMMU_GR0_ID1		0x24
60 #define ARM_SMMU_ID1_PAGESIZE		BIT(31)
61 #define ARM_SMMU_ID1_NUMPAGENDXB	GENMASK(30, 28)
62 #define ARM_SMMU_ID1_NUMS2CB		GENMASK(23, 16)
63 #define ARM_SMMU_ID1_NUMCB		GENMASK(7, 0)
64 
65 #define ARM_SMMU_GR0_ID2		0x28
66 #define ARM_SMMU_ID2_VMID16		BIT(15)
67 #define ARM_SMMU_ID2_PTFS_64K		BIT(14)
68 #define ARM_SMMU_ID2_PTFS_16K		BIT(13)
69 #define ARM_SMMU_ID2_PTFS_4K		BIT(12)
70 #define ARM_SMMU_ID2_UBS		GENMASK(11, 8)
71 #define ARM_SMMU_ID2_OAS		GENMASK(7, 4)
72 #define ARM_SMMU_ID2_IAS		GENMASK(3, 0)
73 
74 #define ARM_SMMU_GR0_ID3		0x2c
75 #define ARM_SMMU_GR0_ID4		0x30
76 #define ARM_SMMU_GR0_ID5		0x34
77 #define ARM_SMMU_GR0_ID6		0x38
78 
79 #define ARM_SMMU_GR0_ID7		0x3c
80 #define ARM_SMMU_ID7_MAJOR		GENMASK(7, 4)
81 #define ARM_SMMU_ID7_MINOR		GENMASK(3, 0)
82 
83 #define ARM_SMMU_GR0_sGFSR		0x48
84 #define ARM_SMMU_sGFSR_USF		BIT(1)
85 
86 #define ARM_SMMU_GR0_sGFSYNR0		0x50
87 #define ARM_SMMU_GR0_sGFSYNR1		0x54
88 #define ARM_SMMU_GR0_sGFSYNR2		0x58
89 
90 /* Global TLB invalidation */
91 #define ARM_SMMU_GR0_TLBIVMID		0x64
92 #define ARM_SMMU_GR0_TLBIALLNSNH	0x68
93 #define ARM_SMMU_GR0_TLBIALLH		0x6c
94 #define ARM_SMMU_GR0_sTLBGSYNC		0x70
95 
96 #define ARM_SMMU_GR0_sTLBGSTATUS	0x74
97 #define ARM_SMMU_sTLBGSTATUS_GSACTIVE	BIT(0)
98 
99 /* Stream mapping registers */
100 #define ARM_SMMU_GR0_SMR(n)		(0x800 + ((n) << 2))
101 #define ARM_SMMU_SMR_VALID		BIT(31)
102 #define ARM_SMMU_SMR_MASK		GENMASK(31, 16)
103 #define ARM_SMMU_SMR_ID			GENMASK(15, 0)
104 
105 #define ARM_SMMU_GR0_S2CR(n)		(0xc00 + ((n) << 2))
106 #define ARM_SMMU_S2CR_PRIVCFG		GENMASK(25, 24)
107 enum arm_smmu_s2cr_privcfg {
108 	S2CR_PRIVCFG_DEFAULT,
109 	S2CR_PRIVCFG_DIPAN,
110 	S2CR_PRIVCFG_UNPRIV,
111 	S2CR_PRIVCFG_PRIV,
112 };
113 #define ARM_SMMU_S2CR_TYPE		GENMASK(17, 16)
114 enum arm_smmu_s2cr_type {
115 	S2CR_TYPE_TRANS,
116 	S2CR_TYPE_BYPASS,
117 	S2CR_TYPE_FAULT,
118 };
119 #define ARM_SMMU_S2CR_EXIDVALID		BIT(10)
120 #define ARM_SMMU_S2CR_CBNDX		GENMASK(7, 0)
121 
122 /* Context bank attribute registers */
123 #define ARM_SMMU_GR1_CBAR(n)		(0x0 + ((n) << 2))
124 #define ARM_SMMU_CBAR_IRPTNDX		GENMASK(31, 24)
125 #define ARM_SMMU_CBAR_TYPE		GENMASK(17, 16)
126 enum arm_smmu_cbar_type {
127 	CBAR_TYPE_S2_TRANS,
128 	CBAR_TYPE_S1_TRANS_S2_BYPASS,
129 	CBAR_TYPE_S1_TRANS_S2_FAULT,
130 	CBAR_TYPE_S1_TRANS_S2_TRANS,
131 };
132 #define ARM_SMMU_CBAR_S1_MEMATTR	GENMASK(15, 12)
133 #define ARM_SMMU_CBAR_S1_MEMATTR_WB	0xf
134 #define ARM_SMMU_CBAR_S1_BPSHCFG	GENMASK(9, 8)
135 #define ARM_SMMU_CBAR_S1_BPSHCFG_NSH	3
136 #define ARM_SMMU_CBAR_VMID		GENMASK(7, 0)
137 
138 #define ARM_SMMU_GR1_CBFRSYNRA(n)	(0x400 + ((n) << 2))
139 
140 #define ARM_SMMU_GR1_CBA2R(n)		(0x800 + ((n) << 2))
141 #define ARM_SMMU_CBA2R_VMID16		GENMASK(31, 16)
142 #define ARM_SMMU_CBA2R_VA64		BIT(0)
143 
144 #define ARM_SMMU_CB_SCTLR		0x0
145 #define ARM_SMMU_SCTLR_S1_ASIDPNE	BIT(12)
146 #define ARM_SMMU_SCTLR_CFCFG		BIT(7)
147 #define ARM_SMMU_SCTLR_HUPCF		BIT(8)
148 #define ARM_SMMU_SCTLR_CFIE		BIT(6)
149 #define ARM_SMMU_SCTLR_CFRE		BIT(5)
150 #define ARM_SMMU_SCTLR_E		BIT(4)
151 #define ARM_SMMU_SCTLR_AFE		BIT(2)
152 #define ARM_SMMU_SCTLR_TRE		BIT(1)
153 #define ARM_SMMU_SCTLR_M		BIT(0)
154 
155 #define ARM_SMMU_CB_ACTLR		0x4
156 
157 #define ARM_SMMU_CB_RESUME		0x8
158 #define ARM_SMMU_RESUME_TERMINATE	BIT(0)
159 
160 #define ARM_SMMU_CB_TCR2		0x10
161 #define ARM_SMMU_TCR2_SEP		GENMASK(17, 15)
162 #define ARM_SMMU_TCR2_SEP_UPSTREAM	0x7
163 #define ARM_SMMU_TCR2_AS		BIT(4)
164 #define ARM_SMMU_TCR2_PASIZE		GENMASK(3, 0)
165 
166 #define ARM_SMMU_CB_TTBR0		0x20
167 #define ARM_SMMU_CB_TTBR1		0x28
168 #define ARM_SMMU_TTBRn_ASID		GENMASK_ULL(63, 48)
169 
170 #define ARM_SMMU_CB_TCR			0x30
171 #define ARM_SMMU_TCR_EAE		BIT(31)
172 #define ARM_SMMU_TCR_EPD1		BIT(23)
173 #define ARM_SMMU_TCR_A1			BIT(22)
174 #define ARM_SMMU_TCR_TG0		GENMASK(15, 14)
175 #define ARM_SMMU_TCR_SH0		GENMASK(13, 12)
176 #define ARM_SMMU_TCR_ORGN0		GENMASK(11, 10)
177 #define ARM_SMMU_TCR_IRGN0		GENMASK(9, 8)
178 #define ARM_SMMU_TCR_EPD0		BIT(7)
179 #define ARM_SMMU_TCR_T0SZ		GENMASK(5, 0)
180 
181 #define ARM_SMMU_VTCR_RES1		BIT(31)
182 #define ARM_SMMU_VTCR_PS		GENMASK(18, 16)
183 #define ARM_SMMU_VTCR_TG0		ARM_SMMU_TCR_TG0
184 #define ARM_SMMU_VTCR_SH0		ARM_SMMU_TCR_SH0
185 #define ARM_SMMU_VTCR_ORGN0		ARM_SMMU_TCR_ORGN0
186 #define ARM_SMMU_VTCR_IRGN0		ARM_SMMU_TCR_IRGN0
187 #define ARM_SMMU_VTCR_SL0		GENMASK(7, 6)
188 #define ARM_SMMU_VTCR_T0SZ		ARM_SMMU_TCR_T0SZ
189 
190 #define ARM_SMMU_CB_CONTEXTIDR		0x34
191 #define ARM_SMMU_CB_S1_MAIR0		0x38
192 #define ARM_SMMU_CB_S1_MAIR1		0x3c
193 
194 #define ARM_SMMU_CB_PAR			0x50
195 #define ARM_SMMU_CB_PAR_F		BIT(0)
196 
197 #define ARM_SMMU_CB_FSR			0x58
198 #define ARM_SMMU_FSR_MULTI		BIT(31)
199 #define ARM_SMMU_FSR_SS			BIT(30)
200 #define ARM_SMMU_FSR_UUT		BIT(8)
201 #define ARM_SMMU_FSR_ASF		BIT(7)
202 #define ARM_SMMU_FSR_TLBLKF		BIT(6)
203 #define ARM_SMMU_FSR_TLBMCF		BIT(5)
204 #define ARM_SMMU_FSR_EF			BIT(4)
205 #define ARM_SMMU_FSR_PF			BIT(3)
206 #define ARM_SMMU_FSR_AFF		BIT(2)
207 #define ARM_SMMU_FSR_TF			BIT(1)
208 
209 #define ARM_SMMU_FSR_IGN		(ARM_SMMU_FSR_AFF |		\
210 					 ARM_SMMU_FSR_ASF |		\
211 					 ARM_SMMU_FSR_TLBMCF |		\
212 					 ARM_SMMU_FSR_TLBLKF)
213 
214 #define ARM_SMMU_FSR_FAULT		(ARM_SMMU_FSR_MULTI |		\
215 					 ARM_SMMU_FSR_SS |		\
216 					 ARM_SMMU_FSR_UUT |		\
217 					 ARM_SMMU_FSR_EF |		\
218 					 ARM_SMMU_FSR_PF |		\
219 					 ARM_SMMU_FSR_TF |		\
220 					 ARM_SMMU_FSR_IGN)
221 
222 #define ARM_SMMU_CB_FAR			0x60
223 
224 #define ARM_SMMU_CB_FSYNR0		0x68
225 #define ARM_SMMU_FSYNR0_WNR		BIT(4)
226 
227 #define ARM_SMMU_CB_S1_TLBIVA		0x600
228 #define ARM_SMMU_CB_S1_TLBIASID		0x610
229 #define ARM_SMMU_CB_S1_TLBIVAL		0x620
230 #define ARM_SMMU_CB_S2_TLBIIPAS2	0x630
231 #define ARM_SMMU_CB_S2_TLBIIPAS2L	0x638
232 #define ARM_SMMU_CB_TLBSYNC		0x7f0
233 #define ARM_SMMU_CB_TLBSTATUS		0x7f4
234 #define ARM_SMMU_CB_ATS1PR		0x800
235 
236 #define ARM_SMMU_CB_ATSR		0x8f0
237 #define ARM_SMMU_ATSR_ACTIVE		BIT(0)
238 
239 
240 /* Maximum number of context banks per SMMU */
241 #define ARM_SMMU_MAX_CBS		128
242 
243 #define TLB_LOOP_TIMEOUT		1000000	/* 1s! */
244 #define TLB_SPIN_COUNT			10
245 
246 /* Shared driver definitions */
247 enum arm_smmu_arch_version {
248 	ARM_SMMU_V1,
249 	ARM_SMMU_V1_64K,
250 	ARM_SMMU_V2,
251 };
252 
253 enum arm_smmu_implementation {
254 	GENERIC_SMMU,
255 	ARM_MMU500,
256 	CAVIUM_SMMUV2,
257 	QCOM_SMMUV2,
258 };
259 
260 struct arm_smmu_s2cr {
261 	struct iommu_group		*group;
262 	int				count;
263 	enum arm_smmu_s2cr_type		type;
264 	enum arm_smmu_s2cr_privcfg	privcfg;
265 	u8				cbndx;
266 };
267 
268 struct arm_smmu_smr {
269 	u16				mask;
270 	u16				id;
271 	bool				valid;
272 	bool				pinned;
273 };
274 
275 struct arm_smmu_device {
276 	struct device			*dev;
277 
278 	void __iomem			*base;
279 	unsigned int			numpage;
280 	unsigned int			pgshift;
281 
282 #define ARM_SMMU_FEAT_COHERENT_WALK	(1 << 0)
283 #define ARM_SMMU_FEAT_STREAM_MATCH	(1 << 1)
284 #define ARM_SMMU_FEAT_TRANS_S1		(1 << 2)
285 #define ARM_SMMU_FEAT_TRANS_S2		(1 << 3)
286 #define ARM_SMMU_FEAT_TRANS_NESTED	(1 << 4)
287 #define ARM_SMMU_FEAT_TRANS_OPS		(1 << 5)
288 #define ARM_SMMU_FEAT_VMID16		(1 << 6)
289 #define ARM_SMMU_FEAT_FMT_AARCH64_4K	(1 << 7)
290 #define ARM_SMMU_FEAT_FMT_AARCH64_16K	(1 << 8)
291 #define ARM_SMMU_FEAT_FMT_AARCH64_64K	(1 << 9)
292 #define ARM_SMMU_FEAT_FMT_AARCH32_L	(1 << 10)
293 #define ARM_SMMU_FEAT_FMT_AARCH32_S	(1 << 11)
294 #define ARM_SMMU_FEAT_EXIDS		(1 << 12)
295 	u32				features;
296 
297 	enum arm_smmu_arch_version	version;
298 	enum arm_smmu_implementation	model;
299 	const struct arm_smmu_impl	*impl;
300 
301 	u32				num_context_banks;
302 	u32				num_s2_context_banks;
303 	DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
304 	struct arm_smmu_cb		*cbs;
305 	atomic_t			irptndx;
306 
307 	u32				num_mapping_groups;
308 	u16				streamid_mask;
309 	u16				smr_mask_mask;
310 	struct arm_smmu_smr		*smrs;
311 	struct arm_smmu_s2cr		*s2crs;
312 	struct mutex			stream_map_mutex;
313 
314 	unsigned long			va_size;
315 	unsigned long			ipa_size;
316 	unsigned long			pa_size;
317 	unsigned long			pgsize_bitmap;
318 
319 	u32				num_global_irqs;
320 	u32				num_context_irqs;
321 	unsigned int			*irqs;
322 	struct clk_bulk_data		*clks;
323 	int				num_clks;
324 
325 	spinlock_t			global_sync_lock;
326 
327 	/* IOMMU core code handle */
328 	struct iommu_device		iommu;
329 };
330 
331 enum arm_smmu_context_fmt {
332 	ARM_SMMU_CTX_FMT_NONE,
333 	ARM_SMMU_CTX_FMT_AARCH64,
334 	ARM_SMMU_CTX_FMT_AARCH32_L,
335 	ARM_SMMU_CTX_FMT_AARCH32_S,
336 };
337 
338 struct arm_smmu_cfg {
339 	u8				cbndx;
340 	u8				irptndx;
341 	union {
342 		u16			asid;
343 		u16			vmid;
344 	};
345 	enum arm_smmu_cbar_type		cbar;
346 	enum arm_smmu_context_fmt	fmt;
347 };
348 #define ARM_SMMU_INVALID_IRPTNDX	0xff
349 
350 struct arm_smmu_cb {
351 	u64				ttbr[2];
352 	u32				tcr[2];
353 	u32				mair[2];
354 	struct arm_smmu_cfg		*cfg;
355 };
356 
357 enum arm_smmu_domain_stage {
358 	ARM_SMMU_DOMAIN_S1 = 0,
359 	ARM_SMMU_DOMAIN_S2,
360 	ARM_SMMU_DOMAIN_NESTED,
361 	ARM_SMMU_DOMAIN_BYPASS,
362 };
363 
364 struct arm_smmu_domain {
365 	struct arm_smmu_device		*smmu;
366 	struct io_pgtable_ops		*pgtbl_ops;
367 	struct io_pgtable_domain_attr	pgtbl_cfg;
368 	const struct iommu_flush_ops	*flush_ops;
369 	struct arm_smmu_cfg		cfg;
370 	enum arm_smmu_domain_stage	stage;
371 	struct mutex			init_mutex; /* Protects smmu pointer */
372 	spinlock_t			cb_lock; /* Serialises ATS1* ops and TLB syncs */
373 	struct iommu_domain		domain;
374 };
375 
376 struct arm_smmu_master_cfg {
377 	struct arm_smmu_device		*smmu;
378 	s16				smendx[];
379 };
380 
381 static inline u32 arm_smmu_lpae_tcr(const struct io_pgtable_cfg *cfg)
382 {
383 	u32 tcr = FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) |
384 		FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) |
385 		FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) |
386 		FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) |
387 		FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz);
388 
389        /*
390 	* When TTBR1 is selected shift the TCR fields by 16 bits and disable
391 	* translation in TTBR0
392 	*/
393 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
394 		tcr = (tcr << 16) & ~ARM_SMMU_TCR_A1;
395 		tcr |= ARM_SMMU_TCR_EPD0;
396 	} else
397 		tcr |= ARM_SMMU_TCR_EPD1;
398 
399 	return tcr;
400 }
401 
402 static inline u32 arm_smmu_lpae_tcr2(const struct io_pgtable_cfg *cfg)
403 {
404 	return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) |
405 	       FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM);
406 }
407 
408 static inline u32 arm_smmu_lpae_vtcr(const struct io_pgtable_cfg *cfg)
409 {
410 	return ARM_SMMU_VTCR_RES1 |
411 	       FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) |
412 	       FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) |
413 	       FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) |
414 	       FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) |
415 	       FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) |
416 	       FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) |
417 	       FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz);
418 }
419 
420 /* Implementation details, yay! */
421 struct arm_smmu_impl {
422 	u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
423 	void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
424 			  u32 val);
425 	u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset);
426 	void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset,
427 			    u64 val);
428 	int (*cfg_probe)(struct arm_smmu_device *smmu);
429 	int (*reset)(struct arm_smmu_device *smmu);
430 	int (*init_context)(struct arm_smmu_domain *smmu_domain,
431 			struct io_pgtable_cfg *cfg, struct device *dev);
432 	void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync,
433 			 int status);
434 	int (*def_domain_type)(struct device *dev);
435 	irqreturn_t (*global_fault)(int irq, void *dev);
436 	irqreturn_t (*context_fault)(int irq, void *dev);
437 	int (*alloc_context_bank)(struct arm_smmu_domain *smmu_domain,
438 				  struct arm_smmu_device *smmu,
439 				  struct device *dev, int start);
440 	void (*write_s2cr)(struct arm_smmu_device *smmu, int idx);
441 	void (*write_sctlr)(struct arm_smmu_device *smmu, int idx, u32 reg);
442 };
443 
444 #define INVALID_SMENDX			-1
445 #define cfg_smendx(cfg, fw, i) \
446 	(i >= fw->num_ids ? INVALID_SMENDX : cfg->smendx[i])
447 #define for_each_cfg_sme(cfg, fw, i, idx) \
448 	for (i = 0; idx = cfg_smendx(cfg, fw, i), i < fw->num_ids; ++i)
449 
450 static inline int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
451 {
452 	int idx;
453 
454 	do {
455 		idx = find_next_zero_bit(map, end, start);
456 		if (idx == end)
457 			return -ENOSPC;
458 	} while (test_and_set_bit(idx, map));
459 
460 	return idx;
461 }
462 
463 static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
464 {
465 	return smmu->base + (n << smmu->pgshift);
466 }
467 
468 static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
469 {
470 	if (smmu->impl && unlikely(smmu->impl->read_reg))
471 		return smmu->impl->read_reg(smmu, page, offset);
472 	return readl_relaxed(arm_smmu_page(smmu, page) + offset);
473 }
474 
475 static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
476 				   int offset, u32 val)
477 {
478 	if (smmu->impl && unlikely(smmu->impl->write_reg))
479 		smmu->impl->write_reg(smmu, page, offset, val);
480 	else
481 		writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
482 }
483 
484 static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
485 {
486 	if (smmu->impl && unlikely(smmu->impl->read_reg64))
487 		return smmu->impl->read_reg64(smmu, page, offset);
488 	return readq_relaxed(arm_smmu_page(smmu, page) + offset);
489 }
490 
491 static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
492 				   int offset, u64 val)
493 {
494 	if (smmu->impl && unlikely(smmu->impl->write_reg64))
495 		smmu->impl->write_reg64(smmu, page, offset, val);
496 	else
497 		writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
498 }
499 
500 #define ARM_SMMU_GR0		0
501 #define ARM_SMMU_GR1		1
502 #define ARM_SMMU_CB(s, n)	((s)->numpage + (n))
503 
504 #define arm_smmu_gr0_read(s, o)		\
505 	arm_smmu_readl((s), ARM_SMMU_GR0, (o))
506 #define arm_smmu_gr0_write(s, o, v)	\
507 	arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
508 
509 #define arm_smmu_gr1_read(s, o)		\
510 	arm_smmu_readl((s), ARM_SMMU_GR1, (o))
511 #define arm_smmu_gr1_write(s, o, v)	\
512 	arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
513 
514 #define arm_smmu_cb_read(s, n, o)	\
515 	arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
516 #define arm_smmu_cb_write(s, n, o, v)	\
517 	arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
518 #define arm_smmu_cb_readq(s, n, o)	\
519 	arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
520 #define arm_smmu_cb_writeq(s, n, o, v)	\
521 	arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
522 
523 struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
524 struct arm_smmu_device *nvidia_smmu_impl_init(struct arm_smmu_device *smmu);
525 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu);
526 
527 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx);
528 int arm_mmu500_reset(struct arm_smmu_device *smmu);
529 
530 #endif /* _ARM_SMMU_H */
531