1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * IOMMU API for ARM architected SMMUv3 implementations.
4  *
5  * Copyright (C) 2015 ARM Limited
6  *
7  * Author: Will Deacon <will.deacon@arm.com>
8  *
9  * This driver is powered by bad coffee and bombay mix.
10  */
11 
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitops.h>
15 #include <linux/crash_dump.h>
16 #include <linux/delay.h>
17 #include <linux/err.h>
18 #include <linux/interrupt.h>
19 #include <linux/io-pgtable.h>
20 #include <linux/iopoll.h>
21 #include <linux/module.h>
22 #include <linux/msi.h>
23 #include <linux/of.h>
24 #include <linux/of_address.h>
25 #include <linux/of_platform.h>
26 #include <linux/pci.h>
27 #include <linux/pci-ats.h>
28 #include <linux/platform_device.h>
29 
30 #include "arm-smmu-v3.h"
31 #include "../../dma-iommu.h"
32 #include "../../iommu-sva.h"
33 
34 static bool disable_bypass = true;
35 module_param(disable_bypass, bool, 0444);
36 MODULE_PARM_DESC(disable_bypass,
37 	"Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
38 
39 static bool disable_msipolling;
40 module_param(disable_msipolling, bool, 0444);
41 MODULE_PARM_DESC(disable_msipolling,
42 	"Disable MSI-based polling for CMD_SYNC completion.");
43 
44 enum arm_smmu_msi_index {
45 	EVTQ_MSI_INDEX,
46 	GERROR_MSI_INDEX,
47 	PRIQ_MSI_INDEX,
48 	ARM_SMMU_MAX_MSIS,
49 };
50 
51 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
52 	[EVTQ_MSI_INDEX] = {
53 		ARM_SMMU_EVTQ_IRQ_CFG0,
54 		ARM_SMMU_EVTQ_IRQ_CFG1,
55 		ARM_SMMU_EVTQ_IRQ_CFG2,
56 	},
57 	[GERROR_MSI_INDEX] = {
58 		ARM_SMMU_GERROR_IRQ_CFG0,
59 		ARM_SMMU_GERROR_IRQ_CFG1,
60 		ARM_SMMU_GERROR_IRQ_CFG2,
61 	},
62 	[PRIQ_MSI_INDEX] = {
63 		ARM_SMMU_PRIQ_IRQ_CFG0,
64 		ARM_SMMU_PRIQ_IRQ_CFG1,
65 		ARM_SMMU_PRIQ_IRQ_CFG2,
66 	},
67 };
68 
69 struct arm_smmu_option_prop {
70 	u32 opt;
71 	const char *prop;
72 };
73 
74 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
75 DEFINE_MUTEX(arm_smmu_asid_lock);
76 
77 /*
78  * Special value used by SVA when a process dies, to quiesce a CD without
79  * disabling it.
80  */
81 struct arm_smmu_ctx_desc quiet_cd = { 0 };
82 
83 static struct arm_smmu_option_prop arm_smmu_options[] = {
84 	{ ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
85 	{ ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
86 	{ 0, NULL},
87 };
88 
89 static void parse_driver_options(struct arm_smmu_device *smmu)
90 {
91 	int i = 0;
92 
93 	do {
94 		if (of_property_read_bool(smmu->dev->of_node,
95 						arm_smmu_options[i].prop)) {
96 			smmu->options |= arm_smmu_options[i].opt;
97 			dev_notice(smmu->dev, "option %s\n",
98 				arm_smmu_options[i].prop);
99 		}
100 	} while (arm_smmu_options[++i].opt);
101 }
102 
103 /* Low-level queue manipulation functions */
104 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
105 {
106 	u32 space, prod, cons;
107 
108 	prod = Q_IDX(q, q->prod);
109 	cons = Q_IDX(q, q->cons);
110 
111 	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
112 		space = (1 << q->max_n_shift) - (prod - cons);
113 	else
114 		space = cons - prod;
115 
116 	return space >= n;
117 }
118 
119 static bool queue_full(struct arm_smmu_ll_queue *q)
120 {
121 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
122 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
123 }
124 
125 static bool queue_empty(struct arm_smmu_ll_queue *q)
126 {
127 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
128 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
129 }
130 
131 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
132 {
133 	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
134 		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
135 	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
136 		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
137 }
138 
139 static void queue_sync_cons_out(struct arm_smmu_queue *q)
140 {
141 	/*
142 	 * Ensure that all CPU accesses (reads and writes) to the queue
143 	 * are complete before we update the cons pointer.
144 	 */
145 	__iomb();
146 	writel_relaxed(q->llq.cons, q->cons_reg);
147 }
148 
149 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
150 {
151 	u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
152 	q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
153 }
154 
155 static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
156 {
157 	struct arm_smmu_ll_queue *llq = &q->llq;
158 
159 	if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
160 		return;
161 
162 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
163 		      Q_IDX(llq, llq->cons);
164 	queue_sync_cons_out(q);
165 }
166 
167 static int queue_sync_prod_in(struct arm_smmu_queue *q)
168 {
169 	u32 prod;
170 	int ret = 0;
171 
172 	/*
173 	 * We can't use the _relaxed() variant here, as we must prevent
174 	 * speculative reads of the queue before we have determined that
175 	 * prod has indeed moved.
176 	 */
177 	prod = readl(q->prod_reg);
178 
179 	if (Q_OVF(prod) != Q_OVF(q->llq.prod))
180 		ret = -EOVERFLOW;
181 
182 	q->llq.prod = prod;
183 	return ret;
184 }
185 
186 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
187 {
188 	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
189 	return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
190 }
191 
192 static void queue_poll_init(struct arm_smmu_device *smmu,
193 			    struct arm_smmu_queue_poll *qp)
194 {
195 	qp->delay = 1;
196 	qp->spin_cnt = 0;
197 	qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
198 	qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
199 }
200 
201 static int queue_poll(struct arm_smmu_queue_poll *qp)
202 {
203 	if (ktime_compare(ktime_get(), qp->timeout) > 0)
204 		return -ETIMEDOUT;
205 
206 	if (qp->wfe) {
207 		wfe();
208 	} else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
209 		cpu_relax();
210 	} else {
211 		udelay(qp->delay);
212 		qp->delay *= 2;
213 		qp->spin_cnt = 0;
214 	}
215 
216 	return 0;
217 }
218 
219 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
220 {
221 	int i;
222 
223 	for (i = 0; i < n_dwords; ++i)
224 		*dst++ = cpu_to_le64(*src++);
225 }
226 
227 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
228 {
229 	int i;
230 
231 	for (i = 0; i < n_dwords; ++i)
232 		*dst++ = le64_to_cpu(*src++);
233 }
234 
235 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
236 {
237 	if (queue_empty(&q->llq))
238 		return -EAGAIN;
239 
240 	queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
241 	queue_inc_cons(&q->llq);
242 	queue_sync_cons_out(q);
243 	return 0;
244 }
245 
246 /* High-level queue accessors */
247 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
248 {
249 	memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
250 	cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
251 
252 	switch (ent->opcode) {
253 	case CMDQ_OP_TLBI_EL2_ALL:
254 	case CMDQ_OP_TLBI_NSNH_ALL:
255 		break;
256 	case CMDQ_OP_PREFETCH_CFG:
257 		cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
258 		break;
259 	case CMDQ_OP_CFGI_CD:
260 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
261 		fallthrough;
262 	case CMDQ_OP_CFGI_STE:
263 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
264 		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
265 		break;
266 	case CMDQ_OP_CFGI_CD_ALL:
267 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
268 		break;
269 	case CMDQ_OP_CFGI_ALL:
270 		/* Cover the entire SID range */
271 		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
272 		break;
273 	case CMDQ_OP_TLBI_NH_VA:
274 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
275 		fallthrough;
276 	case CMDQ_OP_TLBI_EL2_VA:
277 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
278 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
279 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
280 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
281 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
282 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
283 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
284 		break;
285 	case CMDQ_OP_TLBI_S2_IPA:
286 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
287 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
288 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
289 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
290 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
291 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
292 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
293 		break;
294 	case CMDQ_OP_TLBI_NH_ASID:
295 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
296 		fallthrough;
297 	case CMDQ_OP_TLBI_S12_VMALL:
298 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
299 		break;
300 	case CMDQ_OP_TLBI_EL2_ASID:
301 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
302 		break;
303 	case CMDQ_OP_ATC_INV:
304 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
305 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
306 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
307 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
308 		cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
309 		cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
310 		break;
311 	case CMDQ_OP_PRI_RESP:
312 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
313 		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
314 		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
315 		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
316 		switch (ent->pri.resp) {
317 		case PRI_RESP_DENY:
318 		case PRI_RESP_FAIL:
319 		case PRI_RESP_SUCC:
320 			break;
321 		default:
322 			return -EINVAL;
323 		}
324 		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
325 		break;
326 	case CMDQ_OP_RESUME:
327 		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
328 		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
329 		cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
330 		break;
331 	case CMDQ_OP_CMD_SYNC:
332 		if (ent->sync.msiaddr) {
333 			cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
334 			cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
335 		} else {
336 			cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
337 		}
338 		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
339 		cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
340 		break;
341 	default:
342 		return -ENOENT;
343 	}
344 
345 	return 0;
346 }
347 
348 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu)
349 {
350 	return &smmu->cmdq;
351 }
352 
353 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
354 					 struct arm_smmu_queue *q, u32 prod)
355 {
356 	struct arm_smmu_cmdq_ent ent = {
357 		.opcode = CMDQ_OP_CMD_SYNC,
358 	};
359 
360 	/*
361 	 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
362 	 * payload, so the write will zero the entire command on that platform.
363 	 */
364 	if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
365 		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
366 				   q->ent_dwords * 8;
367 	}
368 
369 	arm_smmu_cmdq_build_cmd(cmd, &ent);
370 }
371 
372 static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
373 				     struct arm_smmu_queue *q)
374 {
375 	static const char * const cerror_str[] = {
376 		[CMDQ_ERR_CERROR_NONE_IDX]	= "No error",
377 		[CMDQ_ERR_CERROR_ILL_IDX]	= "Illegal command",
378 		[CMDQ_ERR_CERROR_ABT_IDX]	= "Abort on command fetch",
379 		[CMDQ_ERR_CERROR_ATC_INV_IDX]	= "ATC invalidate timeout",
380 	};
381 
382 	int i;
383 	u64 cmd[CMDQ_ENT_DWORDS];
384 	u32 cons = readl_relaxed(q->cons_reg);
385 	u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
386 	struct arm_smmu_cmdq_ent cmd_sync = {
387 		.opcode = CMDQ_OP_CMD_SYNC,
388 	};
389 
390 	dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
391 		idx < ARRAY_SIZE(cerror_str) ?  cerror_str[idx] : "Unknown");
392 
393 	switch (idx) {
394 	case CMDQ_ERR_CERROR_ABT_IDX:
395 		dev_err(smmu->dev, "retrying command fetch\n");
396 		return;
397 	case CMDQ_ERR_CERROR_NONE_IDX:
398 		return;
399 	case CMDQ_ERR_CERROR_ATC_INV_IDX:
400 		/*
401 		 * ATC Invalidation Completion timeout. CONS is still pointing
402 		 * at the CMD_SYNC. Attempt to complete other pending commands
403 		 * by repeating the CMD_SYNC, though we might well end up back
404 		 * here since the ATC invalidation may still be pending.
405 		 */
406 		return;
407 	case CMDQ_ERR_CERROR_ILL_IDX:
408 	default:
409 		break;
410 	}
411 
412 	/*
413 	 * We may have concurrent producers, so we need to be careful
414 	 * not to touch any of the shadow cmdq state.
415 	 */
416 	queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
417 	dev_err(smmu->dev, "skipping command in error state:\n");
418 	for (i = 0; i < ARRAY_SIZE(cmd); ++i)
419 		dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
420 
421 	/* Convert the erroneous command into a CMD_SYNC */
422 	arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
423 
424 	queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
425 }
426 
427 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
428 {
429 	__arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q);
430 }
431 
432 /*
433  * Command queue locking.
434  * This is a form of bastardised rwlock with the following major changes:
435  *
436  * - The only LOCK routines are exclusive_trylock() and shared_lock().
437  *   Neither have barrier semantics, and instead provide only a control
438  *   dependency.
439  *
440  * - The UNLOCK routines are supplemented with shared_tryunlock(), which
441  *   fails if the caller appears to be the last lock holder (yes, this is
442  *   racy). All successful UNLOCK routines have RELEASE semantics.
443  */
444 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
445 {
446 	int val;
447 
448 	/*
449 	 * We can try to avoid the cmpxchg() loop by simply incrementing the
450 	 * lock counter. When held in exclusive state, the lock counter is set
451 	 * to INT_MIN so these increments won't hurt as the value will remain
452 	 * negative.
453 	 */
454 	if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
455 		return;
456 
457 	do {
458 		val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
459 	} while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
460 }
461 
462 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
463 {
464 	(void)atomic_dec_return_release(&cmdq->lock);
465 }
466 
467 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
468 {
469 	if (atomic_read(&cmdq->lock) == 1)
470 		return false;
471 
472 	arm_smmu_cmdq_shared_unlock(cmdq);
473 	return true;
474 }
475 
476 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)		\
477 ({									\
478 	bool __ret;							\
479 	local_irq_save(flags);						\
480 	__ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN);	\
481 	if (!__ret)							\
482 		local_irq_restore(flags);				\
483 	__ret;								\
484 })
485 
486 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags)		\
487 ({									\
488 	atomic_set_release(&cmdq->lock, 0);				\
489 	local_irq_restore(flags);					\
490 })
491 
492 
493 /*
494  * Command queue insertion.
495  * This is made fiddly by our attempts to achieve some sort of scalability
496  * since there is one queue shared amongst all of the CPUs in the system.  If
497  * you like mixed-size concurrency, dependency ordering and relaxed atomics,
498  * then you'll *love* this monstrosity.
499  *
500  * The basic idea is to split the queue up into ranges of commands that are
501  * owned by a given CPU; the owner may not have written all of the commands
502  * itself, but is responsible for advancing the hardware prod pointer when
503  * the time comes. The algorithm is roughly:
504  *
505  * 	1. Allocate some space in the queue. At this point we also discover
506  *	   whether the head of the queue is currently owned by another CPU,
507  *	   or whether we are the owner.
508  *
509  *	2. Write our commands into our allocated slots in the queue.
510  *
511  *	3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
512  *
513  *	4. If we are an owner:
514  *		a. Wait for the previous owner to finish.
515  *		b. Mark the queue head as unowned, which tells us the range
516  *		   that we are responsible for publishing.
517  *		c. Wait for all commands in our owned range to become valid.
518  *		d. Advance the hardware prod pointer.
519  *		e. Tell the next owner we've finished.
520  *
521  *	5. If we are inserting a CMD_SYNC (we may or may not have been an
522  *	   owner), then we need to stick around until it has completed:
523  *		a. If we have MSIs, the SMMU can write back into the CMD_SYNC
524  *		   to clear the first 4 bytes.
525  *		b. Otherwise, we spin waiting for the hardware cons pointer to
526  *		   advance past our command.
527  *
528  * The devil is in the details, particularly the use of locking for handling
529  * SYNC completion and freeing up space in the queue before we think that it is
530  * full.
531  */
532 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
533 					       u32 sprod, u32 eprod, bool set)
534 {
535 	u32 swidx, sbidx, ewidx, ebidx;
536 	struct arm_smmu_ll_queue llq = {
537 		.max_n_shift	= cmdq->q.llq.max_n_shift,
538 		.prod		= sprod,
539 	};
540 
541 	ewidx = BIT_WORD(Q_IDX(&llq, eprod));
542 	ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
543 
544 	while (llq.prod != eprod) {
545 		unsigned long mask;
546 		atomic_long_t *ptr;
547 		u32 limit = BITS_PER_LONG;
548 
549 		swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
550 		sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
551 
552 		ptr = &cmdq->valid_map[swidx];
553 
554 		if ((swidx == ewidx) && (sbidx < ebidx))
555 			limit = ebidx;
556 
557 		mask = GENMASK(limit - 1, sbidx);
558 
559 		/*
560 		 * The valid bit is the inverse of the wrap bit. This means
561 		 * that a zero-initialised queue is invalid and, after marking
562 		 * all entries as valid, they become invalid again when we
563 		 * wrap.
564 		 */
565 		if (set) {
566 			atomic_long_xor(mask, ptr);
567 		} else { /* Poll */
568 			unsigned long valid;
569 
570 			valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
571 			atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
572 		}
573 
574 		llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
575 	}
576 }
577 
578 /* Mark all entries in the range [sprod, eprod) as valid */
579 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
580 					u32 sprod, u32 eprod)
581 {
582 	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
583 }
584 
585 /* Wait for all entries in the range [sprod, eprod) to become valid */
586 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
587 					 u32 sprod, u32 eprod)
588 {
589 	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
590 }
591 
592 /* Wait for the command queue to become non-full */
593 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
594 					     struct arm_smmu_ll_queue *llq)
595 {
596 	unsigned long flags;
597 	struct arm_smmu_queue_poll qp;
598 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
599 	int ret = 0;
600 
601 	/*
602 	 * Try to update our copy of cons by grabbing exclusive cmdq access. If
603 	 * that fails, spin until somebody else updates it for us.
604 	 */
605 	if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
606 		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
607 		arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
608 		llq->val = READ_ONCE(cmdq->q.llq.val);
609 		return 0;
610 	}
611 
612 	queue_poll_init(smmu, &qp);
613 	do {
614 		llq->val = READ_ONCE(cmdq->q.llq.val);
615 		if (!queue_full(llq))
616 			break;
617 
618 		ret = queue_poll(&qp);
619 	} while (!ret);
620 
621 	return ret;
622 }
623 
624 /*
625  * Wait until the SMMU signals a CMD_SYNC completion MSI.
626  * Must be called with the cmdq lock held in some capacity.
627  */
628 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
629 					  struct arm_smmu_ll_queue *llq)
630 {
631 	int ret = 0;
632 	struct arm_smmu_queue_poll qp;
633 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
634 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
635 
636 	queue_poll_init(smmu, &qp);
637 
638 	/*
639 	 * The MSI won't generate an event, since it's being written back
640 	 * into the command queue.
641 	 */
642 	qp.wfe = false;
643 	smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
644 	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
645 	return ret;
646 }
647 
648 /*
649  * Wait until the SMMU cons index passes llq->prod.
650  * Must be called with the cmdq lock held in some capacity.
651  */
652 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
653 					       struct arm_smmu_ll_queue *llq)
654 {
655 	struct arm_smmu_queue_poll qp;
656 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
657 	u32 prod = llq->prod;
658 	int ret = 0;
659 
660 	queue_poll_init(smmu, &qp);
661 	llq->val = READ_ONCE(cmdq->q.llq.val);
662 	do {
663 		if (queue_consumed(llq, prod))
664 			break;
665 
666 		ret = queue_poll(&qp);
667 
668 		/*
669 		 * This needs to be a readl() so that our subsequent call
670 		 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
671 		 *
672 		 * Specifically, we need to ensure that we observe all
673 		 * shared_lock()s by other CMD_SYNCs that share our owner,
674 		 * so that a failing call to tryunlock() means that we're
675 		 * the last one out and therefore we can safely advance
676 		 * cmdq->q.llq.cons. Roughly speaking:
677 		 *
678 		 * CPU 0		CPU1			CPU2 (us)
679 		 *
680 		 * if (sync)
681 		 * 	shared_lock();
682 		 *
683 		 * dma_wmb();
684 		 * set_valid_map();
685 		 *
686 		 * 			if (owner) {
687 		 *				poll_valid_map();
688 		 *				<control dependency>
689 		 *				writel(prod_reg);
690 		 *
691 		 *						readl(cons_reg);
692 		 *						tryunlock();
693 		 *
694 		 * Requires us to see CPU 0's shared_lock() acquisition.
695 		 */
696 		llq->cons = readl(cmdq->q.cons_reg);
697 	} while (!ret);
698 
699 	return ret;
700 }
701 
702 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
703 					 struct arm_smmu_ll_queue *llq)
704 {
705 	if (smmu->options & ARM_SMMU_OPT_MSIPOLL)
706 		return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
707 
708 	return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
709 }
710 
711 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
712 					u32 prod, int n)
713 {
714 	int i;
715 	struct arm_smmu_ll_queue llq = {
716 		.max_n_shift	= cmdq->q.llq.max_n_shift,
717 		.prod		= prod,
718 	};
719 
720 	for (i = 0; i < n; ++i) {
721 		u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
722 
723 		prod = queue_inc_prod_n(&llq, i);
724 		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
725 	}
726 }
727 
728 /*
729  * This is the actual insertion function, and provides the following
730  * ordering guarantees to callers:
731  *
732  * - There is a dma_wmb() before publishing any commands to the queue.
733  *   This can be relied upon to order prior writes to data structures
734  *   in memory (such as a CD or an STE) before the command.
735  *
736  * - On completion of a CMD_SYNC, there is a control dependency.
737  *   This can be relied upon to order subsequent writes to memory (e.g.
738  *   freeing an IOVA) after completion of the CMD_SYNC.
739  *
740  * - Command insertion is totally ordered, so if two CPUs each race to
741  *   insert their own list of commands then all of the commands from one
742  *   CPU will appear before any of the commands from the other CPU.
743  */
744 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
745 				       u64 *cmds, int n, bool sync)
746 {
747 	u64 cmd_sync[CMDQ_ENT_DWORDS];
748 	u32 prod;
749 	unsigned long flags;
750 	bool owner;
751 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
752 	struct arm_smmu_ll_queue llq, head;
753 	int ret = 0;
754 
755 	llq.max_n_shift = cmdq->q.llq.max_n_shift;
756 
757 	/* 1. Allocate some space in the queue */
758 	local_irq_save(flags);
759 	llq.val = READ_ONCE(cmdq->q.llq.val);
760 	do {
761 		u64 old;
762 
763 		while (!queue_has_space(&llq, n + sync)) {
764 			local_irq_restore(flags);
765 			if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
766 				dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
767 			local_irq_save(flags);
768 		}
769 
770 		head.cons = llq.cons;
771 		head.prod = queue_inc_prod_n(&llq, n + sync) |
772 					     CMDQ_PROD_OWNED_FLAG;
773 
774 		old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
775 		if (old == llq.val)
776 			break;
777 
778 		llq.val = old;
779 	} while (1);
780 	owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
781 	head.prod &= ~CMDQ_PROD_OWNED_FLAG;
782 	llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
783 
784 	/*
785 	 * 2. Write our commands into the queue
786 	 * Dependency ordering from the cmpxchg() loop above.
787 	 */
788 	arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
789 	if (sync) {
790 		prod = queue_inc_prod_n(&llq, n);
791 		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod);
792 		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
793 
794 		/*
795 		 * In order to determine completion of our CMD_SYNC, we must
796 		 * ensure that the queue can't wrap twice without us noticing.
797 		 * We achieve that by taking the cmdq lock as shared before
798 		 * marking our slot as valid.
799 		 */
800 		arm_smmu_cmdq_shared_lock(cmdq);
801 	}
802 
803 	/* 3. Mark our slots as valid, ensuring commands are visible first */
804 	dma_wmb();
805 	arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
806 
807 	/* 4. If we are the owner, take control of the SMMU hardware */
808 	if (owner) {
809 		/* a. Wait for previous owner to finish */
810 		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
811 
812 		/* b. Stop gathering work by clearing the owned flag */
813 		prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
814 						   &cmdq->q.llq.atomic.prod);
815 		prod &= ~CMDQ_PROD_OWNED_FLAG;
816 
817 		/*
818 		 * c. Wait for any gathered work to be written to the queue.
819 		 * Note that we read our own entries so that we have the control
820 		 * dependency required by (d).
821 		 */
822 		arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
823 
824 		/*
825 		 * d. Advance the hardware prod pointer
826 		 * Control dependency ordering from the entries becoming valid.
827 		 */
828 		writel_relaxed(prod, cmdq->q.prod_reg);
829 
830 		/*
831 		 * e. Tell the next owner we're done
832 		 * Make sure we've updated the hardware first, so that we don't
833 		 * race to update prod and potentially move it backwards.
834 		 */
835 		atomic_set_release(&cmdq->owner_prod, prod);
836 	}
837 
838 	/* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
839 	if (sync) {
840 		llq.prod = queue_inc_prod_n(&llq, n);
841 		ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
842 		if (ret) {
843 			dev_err_ratelimited(smmu->dev,
844 					    "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
845 					    llq.prod,
846 					    readl_relaxed(cmdq->q.prod_reg),
847 					    readl_relaxed(cmdq->q.cons_reg));
848 		}
849 
850 		/*
851 		 * Try to unlock the cmdq lock. This will fail if we're the last
852 		 * reader, in which case we can safely update cmdq->q.llq.cons
853 		 */
854 		if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
855 			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
856 			arm_smmu_cmdq_shared_unlock(cmdq);
857 		}
858 	}
859 
860 	local_irq_restore(flags);
861 	return ret;
862 }
863 
864 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
865 				     struct arm_smmu_cmdq_ent *ent,
866 				     bool sync)
867 {
868 	u64 cmd[CMDQ_ENT_DWORDS];
869 
870 	if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
871 		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
872 			 ent->opcode);
873 		return -EINVAL;
874 	}
875 
876 	return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, sync);
877 }
878 
879 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
880 				   struct arm_smmu_cmdq_ent *ent)
881 {
882 	return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
883 }
884 
885 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
886 					     struct arm_smmu_cmdq_ent *ent)
887 {
888 	return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
889 }
890 
891 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
892 				    struct arm_smmu_cmdq_batch *cmds,
893 				    struct arm_smmu_cmdq_ent *cmd)
894 {
895 	int index;
896 
897 	if (cmds->num == CMDQ_BATCH_ENTRIES) {
898 		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false);
899 		cmds->num = 0;
900 	}
901 
902 	index = cmds->num * CMDQ_ENT_DWORDS;
903 	if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
904 		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
905 			 cmd->opcode);
906 		return;
907 	}
908 
909 	cmds->num++;
910 }
911 
912 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
913 				      struct arm_smmu_cmdq_batch *cmds)
914 {
915 	return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
916 }
917 
918 static int arm_smmu_page_response(struct device *dev,
919 				  struct iommu_fault_event *unused,
920 				  struct iommu_page_response *resp)
921 {
922 	struct arm_smmu_cmdq_ent cmd = {0};
923 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
924 	int sid = master->streams[0].id;
925 
926 	if (master->stall_enabled) {
927 		cmd.opcode		= CMDQ_OP_RESUME;
928 		cmd.resume.sid		= sid;
929 		cmd.resume.stag		= resp->grpid;
930 		switch (resp->code) {
931 		case IOMMU_PAGE_RESP_INVALID:
932 		case IOMMU_PAGE_RESP_FAILURE:
933 			cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
934 			break;
935 		case IOMMU_PAGE_RESP_SUCCESS:
936 			cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
937 			break;
938 		default:
939 			return -EINVAL;
940 		}
941 	} else {
942 		return -ENODEV;
943 	}
944 
945 	arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
946 	/*
947 	 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
948 	 * RESUME consumption guarantees that the stalled transaction will be
949 	 * terminated... at some point in the future. PRI_RESP is fire and
950 	 * forget.
951 	 */
952 
953 	return 0;
954 }
955 
956 /* Context descriptor manipulation functions */
957 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
958 {
959 	struct arm_smmu_cmdq_ent cmd = {
960 		.opcode	= smmu->features & ARM_SMMU_FEAT_E2H ?
961 			CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
962 		.tlbi.asid = asid,
963 	};
964 
965 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
966 }
967 
968 static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
969 			     int ssid, bool leaf)
970 {
971 	size_t i;
972 	unsigned long flags;
973 	struct arm_smmu_master *master;
974 	struct arm_smmu_cmdq_batch cmds;
975 	struct arm_smmu_device *smmu = smmu_domain->smmu;
976 	struct arm_smmu_cmdq_ent cmd = {
977 		.opcode	= CMDQ_OP_CFGI_CD,
978 		.cfgi	= {
979 			.ssid	= ssid,
980 			.leaf	= leaf,
981 		},
982 	};
983 
984 	cmds.num = 0;
985 
986 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
987 	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
988 		for (i = 0; i < master->num_streams; i++) {
989 			cmd.cfgi.sid = master->streams[i].id;
990 			arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
991 		}
992 	}
993 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
994 
995 	arm_smmu_cmdq_batch_submit(smmu, &cmds);
996 }
997 
998 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
999 					struct arm_smmu_l1_ctx_desc *l1_desc)
1000 {
1001 	size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1002 
1003 	l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
1004 					     &l1_desc->l2ptr_dma, GFP_KERNEL);
1005 	if (!l1_desc->l2ptr) {
1006 		dev_warn(smmu->dev,
1007 			 "failed to allocate context descriptor table\n");
1008 		return -ENOMEM;
1009 	}
1010 	return 0;
1011 }
1012 
1013 static void arm_smmu_write_cd_l1_desc(__le64 *dst,
1014 				      struct arm_smmu_l1_ctx_desc *l1_desc)
1015 {
1016 	u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) |
1017 		  CTXDESC_L1_DESC_V;
1018 
1019 	/* See comment in arm_smmu_write_ctx_desc() */
1020 	WRITE_ONCE(*dst, cpu_to_le64(val));
1021 }
1022 
1023 static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain,
1024 				   u32 ssid)
1025 {
1026 	__le64 *l1ptr;
1027 	unsigned int idx;
1028 	struct arm_smmu_l1_ctx_desc *l1_desc;
1029 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1030 	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1031 
1032 	if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1033 		return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
1034 
1035 	idx = ssid >> CTXDESC_SPLIT;
1036 	l1_desc = &cdcfg->l1_desc[idx];
1037 	if (!l1_desc->l2ptr) {
1038 		if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
1039 			return NULL;
1040 
1041 		l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
1042 		arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
1043 		/* An invalid L1CD can be cached */
1044 		arm_smmu_sync_cd(smmu_domain, ssid, false);
1045 	}
1046 	idx = ssid & (CTXDESC_L2_ENTRIES - 1);
1047 	return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
1048 }
1049 
1050 int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain, int ssid,
1051 			    struct arm_smmu_ctx_desc *cd)
1052 {
1053 	/*
1054 	 * This function handles the following cases:
1055 	 *
1056 	 * (1) Install primary CD, for normal DMA traffic (SSID = 0).
1057 	 * (2) Install a secondary CD, for SID+SSID traffic.
1058 	 * (3) Update ASID of a CD. Atomically write the first 64 bits of the
1059 	 *     CD, then invalidate the old entry and mappings.
1060 	 * (4) Quiesce the context without clearing the valid bit. Disable
1061 	 *     translation, and ignore any translation fault.
1062 	 * (5) Remove a secondary CD.
1063 	 */
1064 	u64 val;
1065 	bool cd_live;
1066 	__le64 *cdptr;
1067 
1068 	if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
1069 		return -E2BIG;
1070 
1071 	cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
1072 	if (!cdptr)
1073 		return -ENOMEM;
1074 
1075 	val = le64_to_cpu(cdptr[0]);
1076 	cd_live = !!(val & CTXDESC_CD_0_V);
1077 
1078 	if (!cd) { /* (5) */
1079 		val = 0;
1080 	} else if (cd == &quiet_cd) { /* (4) */
1081 		val |= CTXDESC_CD_0_TCR_EPD0;
1082 	} else if (cd_live) { /* (3) */
1083 		val &= ~CTXDESC_CD_0_ASID;
1084 		val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
1085 		/*
1086 		 * Until CD+TLB invalidation, both ASIDs may be used for tagging
1087 		 * this substream's traffic
1088 		 */
1089 	} else { /* (1) and (2) */
1090 		cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
1091 		cdptr[2] = 0;
1092 		cdptr[3] = cpu_to_le64(cd->mair);
1093 
1094 		/*
1095 		 * STE is live, and the SMMU might read dwords of this CD in any
1096 		 * order. Ensure that it observes valid values before reading
1097 		 * V=1.
1098 		 */
1099 		arm_smmu_sync_cd(smmu_domain, ssid, true);
1100 
1101 		val = cd->tcr |
1102 #ifdef __BIG_ENDIAN
1103 			CTXDESC_CD_0_ENDI |
1104 #endif
1105 			CTXDESC_CD_0_R | CTXDESC_CD_0_A |
1106 			(cd->mm ? 0 : CTXDESC_CD_0_ASET) |
1107 			CTXDESC_CD_0_AA64 |
1108 			FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
1109 			CTXDESC_CD_0_V;
1110 
1111 		if (smmu_domain->stall_enabled)
1112 			val |= CTXDESC_CD_0_S;
1113 	}
1114 
1115 	/*
1116 	 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
1117 	 * "Configuration structures and configuration invalidation completion"
1118 	 *
1119 	 *   The size of single-copy atomic reads made by the SMMU is
1120 	 *   IMPLEMENTATION DEFINED but must be at least 64 bits. Any single
1121 	 *   field within an aligned 64-bit span of a structure can be altered
1122 	 *   without first making the structure invalid.
1123 	 */
1124 	WRITE_ONCE(cdptr[0], cpu_to_le64(val));
1125 	arm_smmu_sync_cd(smmu_domain, ssid, true);
1126 	return 0;
1127 }
1128 
1129 static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
1130 {
1131 	int ret;
1132 	size_t l1size;
1133 	size_t max_contexts;
1134 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1135 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1136 	struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
1137 
1138 	max_contexts = 1 << cfg->s1cdmax;
1139 
1140 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1141 	    max_contexts <= CTXDESC_L2_ENTRIES) {
1142 		cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1143 		cdcfg->num_l1_ents = max_contexts;
1144 
1145 		l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
1146 	} else {
1147 		cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1148 		cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
1149 						  CTXDESC_L2_ENTRIES);
1150 
1151 		cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
1152 					      sizeof(*cdcfg->l1_desc),
1153 					      GFP_KERNEL);
1154 		if (!cdcfg->l1_desc)
1155 			return -ENOMEM;
1156 
1157 		l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1158 	}
1159 
1160 	cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
1161 					   GFP_KERNEL);
1162 	if (!cdcfg->cdtab) {
1163 		dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1164 		ret = -ENOMEM;
1165 		goto err_free_l1;
1166 	}
1167 
1168 	return 0;
1169 
1170 err_free_l1:
1171 	if (cdcfg->l1_desc) {
1172 		devm_kfree(smmu->dev, cdcfg->l1_desc);
1173 		cdcfg->l1_desc = NULL;
1174 	}
1175 	return ret;
1176 }
1177 
1178 static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
1179 {
1180 	int i;
1181 	size_t size, l1size;
1182 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1183 	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1184 
1185 	if (cdcfg->l1_desc) {
1186 		size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
1187 
1188 		for (i = 0; i < cdcfg->num_l1_ents; i++) {
1189 			if (!cdcfg->l1_desc[i].l2ptr)
1190 				continue;
1191 
1192 			dmam_free_coherent(smmu->dev, size,
1193 					   cdcfg->l1_desc[i].l2ptr,
1194 					   cdcfg->l1_desc[i].l2ptr_dma);
1195 		}
1196 		devm_kfree(smmu->dev, cdcfg->l1_desc);
1197 		cdcfg->l1_desc = NULL;
1198 
1199 		l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1200 	} else {
1201 		l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
1202 	}
1203 
1204 	dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
1205 	cdcfg->cdtab_dma = 0;
1206 	cdcfg->cdtab = NULL;
1207 }
1208 
1209 bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd)
1210 {
1211 	bool free;
1212 	struct arm_smmu_ctx_desc *old_cd;
1213 
1214 	if (!cd->asid)
1215 		return false;
1216 
1217 	free = refcount_dec_and_test(&cd->refs);
1218 	if (free) {
1219 		old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid);
1220 		WARN_ON(old_cd != cd);
1221 	}
1222 	return free;
1223 }
1224 
1225 /* Stream table manipulation functions */
1226 static void
1227 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1228 {
1229 	u64 val = 0;
1230 
1231 	val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1232 	val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1233 
1234 	/* See comment in arm_smmu_write_ctx_desc() */
1235 	WRITE_ONCE(*dst, cpu_to_le64(val));
1236 }
1237 
1238 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1239 {
1240 	struct arm_smmu_cmdq_ent cmd = {
1241 		.opcode	= CMDQ_OP_CFGI_STE,
1242 		.cfgi	= {
1243 			.sid	= sid,
1244 			.leaf	= true,
1245 		},
1246 	};
1247 
1248 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1249 }
1250 
1251 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1252 				      __le64 *dst)
1253 {
1254 	/*
1255 	 * This is hideously complicated, but we only really care about
1256 	 * three cases at the moment:
1257 	 *
1258 	 * 1. Invalid (all zero) -> bypass/fault (init)
1259 	 * 2. Bypass/fault -> translation/bypass (attach)
1260 	 * 3. Translation/bypass -> bypass/fault (detach)
1261 	 *
1262 	 * Given that we can't update the STE atomically and the SMMU
1263 	 * doesn't read the thing in a defined order, that leaves us
1264 	 * with the following maintenance requirements:
1265 	 *
1266 	 * 1. Update Config, return (init time STEs aren't live)
1267 	 * 2. Write everything apart from dword 0, sync, write dword 0, sync
1268 	 * 3. Update Config, sync
1269 	 */
1270 	u64 val = le64_to_cpu(dst[0]);
1271 	bool ste_live = false;
1272 	struct arm_smmu_device *smmu = NULL;
1273 	struct arm_smmu_s1_cfg *s1_cfg = NULL;
1274 	struct arm_smmu_s2_cfg *s2_cfg = NULL;
1275 	struct arm_smmu_domain *smmu_domain = NULL;
1276 	struct arm_smmu_cmdq_ent prefetch_cmd = {
1277 		.opcode		= CMDQ_OP_PREFETCH_CFG,
1278 		.prefetch	= {
1279 			.sid	= sid,
1280 		},
1281 	};
1282 
1283 	if (master) {
1284 		smmu_domain = master->domain;
1285 		smmu = master->smmu;
1286 	}
1287 
1288 	if (smmu_domain) {
1289 		switch (smmu_domain->stage) {
1290 		case ARM_SMMU_DOMAIN_S1:
1291 			s1_cfg = &smmu_domain->s1_cfg;
1292 			break;
1293 		case ARM_SMMU_DOMAIN_S2:
1294 		case ARM_SMMU_DOMAIN_NESTED:
1295 			s2_cfg = &smmu_domain->s2_cfg;
1296 			break;
1297 		default:
1298 			break;
1299 		}
1300 	}
1301 
1302 	if (val & STRTAB_STE_0_V) {
1303 		switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1304 		case STRTAB_STE_0_CFG_BYPASS:
1305 			break;
1306 		case STRTAB_STE_0_CFG_S1_TRANS:
1307 		case STRTAB_STE_0_CFG_S2_TRANS:
1308 			ste_live = true;
1309 			break;
1310 		case STRTAB_STE_0_CFG_ABORT:
1311 			BUG_ON(!disable_bypass);
1312 			break;
1313 		default:
1314 			BUG(); /* STE corruption */
1315 		}
1316 	}
1317 
1318 	/* Nuke the existing STE_0 value, as we're going to rewrite it */
1319 	val = STRTAB_STE_0_V;
1320 
1321 	/* Bypass/fault */
1322 	if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1323 		if (!smmu_domain && disable_bypass)
1324 			val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1325 		else
1326 			val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1327 
1328 		dst[0] = cpu_to_le64(val);
1329 		dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1330 						STRTAB_STE_1_SHCFG_INCOMING));
1331 		dst[2] = 0; /* Nuke the VMID */
1332 		/*
1333 		 * The SMMU can perform negative caching, so we must sync
1334 		 * the STE regardless of whether the old value was live.
1335 		 */
1336 		if (smmu)
1337 			arm_smmu_sync_ste_for_sid(smmu, sid);
1338 		return;
1339 	}
1340 
1341 	if (s1_cfg) {
1342 		u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ?
1343 			STRTAB_STE_1_STRW_EL2 : STRTAB_STE_1_STRW_NSEL1;
1344 
1345 		BUG_ON(ste_live);
1346 		dst[1] = cpu_to_le64(
1347 			 FIELD_PREP(STRTAB_STE_1_S1DSS, STRTAB_STE_1_S1DSS_SSID0) |
1348 			 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1349 			 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1350 			 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1351 			 FIELD_PREP(STRTAB_STE_1_STRW, strw));
1352 
1353 		if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1354 		    !master->stall_enabled)
1355 			dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1356 
1357 		val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1358 			FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1359 			FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) |
1360 			FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
1361 	}
1362 
1363 	if (s2_cfg) {
1364 		BUG_ON(ste_live);
1365 		dst[2] = cpu_to_le64(
1366 			 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1367 			 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1368 #ifdef __BIG_ENDIAN
1369 			 STRTAB_STE_2_S2ENDI |
1370 #endif
1371 			 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1372 			 STRTAB_STE_2_S2R);
1373 
1374 		dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1375 
1376 		val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1377 	}
1378 
1379 	if (master->ats_enabled)
1380 		dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1381 						 STRTAB_STE_1_EATS_TRANS));
1382 
1383 	arm_smmu_sync_ste_for_sid(smmu, sid);
1384 	/* See comment in arm_smmu_write_ctx_desc() */
1385 	WRITE_ONCE(dst[0], cpu_to_le64(val));
1386 	arm_smmu_sync_ste_for_sid(smmu, sid);
1387 
1388 	/* It's likely that we'll want to use the new STE soon */
1389 	if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1390 		arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1391 }
1392 
1393 static void arm_smmu_init_bypass_stes(__le64 *strtab, unsigned int nent, bool force)
1394 {
1395 	unsigned int i;
1396 	u64 val = STRTAB_STE_0_V;
1397 
1398 	if (disable_bypass && !force)
1399 		val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1400 	else
1401 		val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1402 
1403 	for (i = 0; i < nent; ++i) {
1404 		strtab[0] = cpu_to_le64(val);
1405 		strtab[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1406 						   STRTAB_STE_1_SHCFG_INCOMING));
1407 		strtab[2] = 0;
1408 		strtab += STRTAB_STE_DWORDS;
1409 	}
1410 }
1411 
1412 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1413 {
1414 	size_t size;
1415 	void *strtab;
1416 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1417 	struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1418 
1419 	if (desc->l2ptr)
1420 		return 0;
1421 
1422 	size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1423 	strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1424 
1425 	desc->span = STRTAB_SPLIT + 1;
1426 	desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1427 					  GFP_KERNEL);
1428 	if (!desc->l2ptr) {
1429 		dev_err(smmu->dev,
1430 			"failed to allocate l2 stream table for SID %u\n",
1431 			sid);
1432 		return -ENOMEM;
1433 	}
1434 
1435 	arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false);
1436 	arm_smmu_write_strtab_l1_desc(strtab, desc);
1437 	return 0;
1438 }
1439 
1440 static struct arm_smmu_master *
1441 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
1442 {
1443 	struct rb_node *node;
1444 	struct arm_smmu_stream *stream;
1445 
1446 	lockdep_assert_held(&smmu->streams_mutex);
1447 
1448 	node = smmu->streams.rb_node;
1449 	while (node) {
1450 		stream = rb_entry(node, struct arm_smmu_stream, node);
1451 		if (stream->id < sid)
1452 			node = node->rb_right;
1453 		else if (stream->id > sid)
1454 			node = node->rb_left;
1455 		else
1456 			return stream->master;
1457 	}
1458 
1459 	return NULL;
1460 }
1461 
1462 /* IRQ and event handlers */
1463 static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)
1464 {
1465 	int ret;
1466 	u32 reason;
1467 	u32 perm = 0;
1468 	struct arm_smmu_master *master;
1469 	bool ssid_valid = evt[0] & EVTQ_0_SSV;
1470 	u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]);
1471 	struct iommu_fault_event fault_evt = { };
1472 	struct iommu_fault *flt = &fault_evt.fault;
1473 
1474 	switch (FIELD_GET(EVTQ_0_ID, evt[0])) {
1475 	case EVT_ID_TRANSLATION_FAULT:
1476 		reason = IOMMU_FAULT_REASON_PTE_FETCH;
1477 		break;
1478 	case EVT_ID_ADDR_SIZE_FAULT:
1479 		reason = IOMMU_FAULT_REASON_OOR_ADDRESS;
1480 		break;
1481 	case EVT_ID_ACCESS_FAULT:
1482 		reason = IOMMU_FAULT_REASON_ACCESS;
1483 		break;
1484 	case EVT_ID_PERMISSION_FAULT:
1485 		reason = IOMMU_FAULT_REASON_PERMISSION;
1486 		break;
1487 	default:
1488 		return -EOPNOTSUPP;
1489 	}
1490 
1491 	/* Stage-2 is always pinned at the moment */
1492 	if (evt[1] & EVTQ_1_S2)
1493 		return -EFAULT;
1494 
1495 	if (evt[1] & EVTQ_1_RnW)
1496 		perm |= IOMMU_FAULT_PERM_READ;
1497 	else
1498 		perm |= IOMMU_FAULT_PERM_WRITE;
1499 
1500 	if (evt[1] & EVTQ_1_InD)
1501 		perm |= IOMMU_FAULT_PERM_EXEC;
1502 
1503 	if (evt[1] & EVTQ_1_PnU)
1504 		perm |= IOMMU_FAULT_PERM_PRIV;
1505 
1506 	if (evt[1] & EVTQ_1_STALL) {
1507 		flt->type = IOMMU_FAULT_PAGE_REQ;
1508 		flt->prm = (struct iommu_fault_page_request) {
1509 			.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
1510 			.grpid = FIELD_GET(EVTQ_1_STAG, evt[1]),
1511 			.perm = perm,
1512 			.addr = FIELD_GET(EVTQ_2_ADDR, evt[2]),
1513 		};
1514 
1515 		if (ssid_valid) {
1516 			flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1517 			flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1518 		}
1519 	} else {
1520 		flt->type = IOMMU_FAULT_DMA_UNRECOV;
1521 		flt->event = (struct iommu_fault_unrecoverable) {
1522 			.reason = reason,
1523 			.flags = IOMMU_FAULT_UNRECOV_ADDR_VALID,
1524 			.perm = perm,
1525 			.addr = FIELD_GET(EVTQ_2_ADDR, evt[2]),
1526 		};
1527 
1528 		if (ssid_valid) {
1529 			flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID;
1530 			flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1531 		}
1532 	}
1533 
1534 	mutex_lock(&smmu->streams_mutex);
1535 	master = arm_smmu_find_master(smmu, sid);
1536 	if (!master) {
1537 		ret = -EINVAL;
1538 		goto out_unlock;
1539 	}
1540 
1541 	ret = iommu_report_device_fault(master->dev, &fault_evt);
1542 	if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) {
1543 		/* Nobody cared, abort the access */
1544 		struct iommu_page_response resp = {
1545 			.pasid		= flt->prm.pasid,
1546 			.grpid		= flt->prm.grpid,
1547 			.code		= IOMMU_PAGE_RESP_FAILURE,
1548 		};
1549 		arm_smmu_page_response(master->dev, &fault_evt, &resp);
1550 	}
1551 
1552 out_unlock:
1553 	mutex_unlock(&smmu->streams_mutex);
1554 	return ret;
1555 }
1556 
1557 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1558 {
1559 	int i, ret;
1560 	struct arm_smmu_device *smmu = dev;
1561 	struct arm_smmu_queue *q = &smmu->evtq.q;
1562 	struct arm_smmu_ll_queue *llq = &q->llq;
1563 	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1564 				      DEFAULT_RATELIMIT_BURST);
1565 	u64 evt[EVTQ_ENT_DWORDS];
1566 
1567 	do {
1568 		while (!queue_remove_raw(q, evt)) {
1569 			u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1570 
1571 			ret = arm_smmu_handle_evt(smmu, evt);
1572 			if (!ret || !__ratelimit(&rs))
1573 				continue;
1574 
1575 			dev_info(smmu->dev, "event 0x%02x received:\n", id);
1576 			for (i = 0; i < ARRAY_SIZE(evt); ++i)
1577 				dev_info(smmu->dev, "\t0x%016llx\n",
1578 					 (unsigned long long)evt[i]);
1579 
1580 			cond_resched();
1581 		}
1582 
1583 		/*
1584 		 * Not much we can do on overflow, so scream and pretend we're
1585 		 * trying harder.
1586 		 */
1587 		if (queue_sync_prod_in(q) == -EOVERFLOW)
1588 			dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1589 	} while (!queue_empty(llq));
1590 
1591 	/* Sync our overflow flag, as we believe we're up to speed */
1592 	queue_sync_cons_ovf(q);
1593 	return IRQ_HANDLED;
1594 }
1595 
1596 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1597 {
1598 	u32 sid, ssid;
1599 	u16 grpid;
1600 	bool ssv, last;
1601 
1602 	sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1603 	ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1604 	ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0;
1605 	last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1606 	grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1607 
1608 	dev_info(smmu->dev, "unexpected PRI request received:\n");
1609 	dev_info(smmu->dev,
1610 		 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1611 		 sid, ssid, grpid, last ? "L" : "",
1612 		 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1613 		 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1614 		 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1615 		 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1616 		 evt[1] & PRIQ_1_ADDR_MASK);
1617 
1618 	if (last) {
1619 		struct arm_smmu_cmdq_ent cmd = {
1620 			.opcode			= CMDQ_OP_PRI_RESP,
1621 			.substream_valid	= ssv,
1622 			.pri			= {
1623 				.sid	= sid,
1624 				.ssid	= ssid,
1625 				.grpid	= grpid,
1626 				.resp	= PRI_RESP_DENY,
1627 			},
1628 		};
1629 
1630 		arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1631 	}
1632 }
1633 
1634 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1635 {
1636 	struct arm_smmu_device *smmu = dev;
1637 	struct arm_smmu_queue *q = &smmu->priq.q;
1638 	struct arm_smmu_ll_queue *llq = &q->llq;
1639 	u64 evt[PRIQ_ENT_DWORDS];
1640 
1641 	do {
1642 		while (!queue_remove_raw(q, evt))
1643 			arm_smmu_handle_ppr(smmu, evt);
1644 
1645 		if (queue_sync_prod_in(q) == -EOVERFLOW)
1646 			dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1647 	} while (!queue_empty(llq));
1648 
1649 	/* Sync our overflow flag, as we believe we're up to speed */
1650 	queue_sync_cons_ovf(q);
1651 	return IRQ_HANDLED;
1652 }
1653 
1654 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1655 
1656 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1657 {
1658 	u32 gerror, gerrorn, active;
1659 	struct arm_smmu_device *smmu = dev;
1660 
1661 	gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1662 	gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1663 
1664 	active = gerror ^ gerrorn;
1665 	if (!(active & GERROR_ERR_MASK))
1666 		return IRQ_NONE; /* No errors pending */
1667 
1668 	dev_warn(smmu->dev,
1669 		 "unexpected global error reported (0x%08x), this could be serious\n",
1670 		 active);
1671 
1672 	if (active & GERROR_SFM_ERR) {
1673 		dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1674 		arm_smmu_device_disable(smmu);
1675 	}
1676 
1677 	if (active & GERROR_MSI_GERROR_ABT_ERR)
1678 		dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1679 
1680 	if (active & GERROR_MSI_PRIQ_ABT_ERR)
1681 		dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1682 
1683 	if (active & GERROR_MSI_EVTQ_ABT_ERR)
1684 		dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1685 
1686 	if (active & GERROR_MSI_CMDQ_ABT_ERR)
1687 		dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1688 
1689 	if (active & GERROR_PRIQ_ABT_ERR)
1690 		dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1691 
1692 	if (active & GERROR_EVTQ_ABT_ERR)
1693 		dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1694 
1695 	if (active & GERROR_CMDQ_ERR)
1696 		arm_smmu_cmdq_skip_err(smmu);
1697 
1698 	writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1699 	return IRQ_HANDLED;
1700 }
1701 
1702 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1703 {
1704 	struct arm_smmu_device *smmu = dev;
1705 
1706 	arm_smmu_evtq_thread(irq, dev);
1707 	if (smmu->features & ARM_SMMU_FEAT_PRI)
1708 		arm_smmu_priq_thread(irq, dev);
1709 
1710 	return IRQ_HANDLED;
1711 }
1712 
1713 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1714 {
1715 	arm_smmu_gerror_handler(irq, dev);
1716 	return IRQ_WAKE_THREAD;
1717 }
1718 
1719 static void
1720 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1721 			struct arm_smmu_cmdq_ent *cmd)
1722 {
1723 	size_t log2_span;
1724 	size_t span_mask;
1725 	/* ATC invalidates are always on 4096-bytes pages */
1726 	size_t inval_grain_shift = 12;
1727 	unsigned long page_start, page_end;
1728 
1729 	/*
1730 	 * ATS and PASID:
1731 	 *
1732 	 * If substream_valid is clear, the PCIe TLP is sent without a PASID
1733 	 * prefix. In that case all ATC entries within the address range are
1734 	 * invalidated, including those that were requested with a PASID! There
1735 	 * is no way to invalidate only entries without PASID.
1736 	 *
1737 	 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
1738 	 * traffic), translation requests without PASID create ATC entries
1739 	 * without PASID, which must be invalidated with substream_valid clear.
1740 	 * This has the unpleasant side-effect of invalidating all PASID-tagged
1741 	 * ATC entries within the address range.
1742 	 */
1743 	*cmd = (struct arm_smmu_cmdq_ent) {
1744 		.opcode			= CMDQ_OP_ATC_INV,
1745 		.substream_valid	= !!ssid,
1746 		.atc.ssid		= ssid,
1747 	};
1748 
1749 	if (!size) {
1750 		cmd->atc.size = ATC_INV_SIZE_ALL;
1751 		return;
1752 	}
1753 
1754 	page_start	= iova >> inval_grain_shift;
1755 	page_end	= (iova + size - 1) >> inval_grain_shift;
1756 
1757 	/*
1758 	 * In an ATS Invalidate Request, the address must be aligned on the
1759 	 * range size, which must be a power of two number of page sizes. We
1760 	 * thus have to choose between grossly over-invalidating the region, or
1761 	 * splitting the invalidation into multiple commands. For simplicity
1762 	 * we'll go with the first solution, but should refine it in the future
1763 	 * if multiple commands are shown to be more efficient.
1764 	 *
1765 	 * Find the smallest power of two that covers the range. The most
1766 	 * significant differing bit between the start and end addresses,
1767 	 * fls(start ^ end), indicates the required span. For example:
1768 	 *
1769 	 * We want to invalidate pages [8; 11]. This is already the ideal range:
1770 	 *		x = 0b1000 ^ 0b1011 = 0b11
1771 	 *		span = 1 << fls(x) = 4
1772 	 *
1773 	 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
1774 	 *		x = 0b0111 ^ 0b1010 = 0b1101
1775 	 *		span = 1 << fls(x) = 16
1776 	 */
1777 	log2_span	= fls_long(page_start ^ page_end);
1778 	span_mask	= (1ULL << log2_span) - 1;
1779 
1780 	page_start	&= ~span_mask;
1781 
1782 	cmd->atc.addr	= page_start << inval_grain_shift;
1783 	cmd->atc.size	= log2_span;
1784 }
1785 
1786 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
1787 {
1788 	int i;
1789 	struct arm_smmu_cmdq_ent cmd;
1790 	struct arm_smmu_cmdq_batch cmds;
1791 
1792 	arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
1793 
1794 	cmds.num = 0;
1795 	for (i = 0; i < master->num_streams; i++) {
1796 		cmd.atc.sid = master->streams[i].id;
1797 		arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
1798 	}
1799 
1800 	return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
1801 }
1802 
1803 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
1804 			    unsigned long iova, size_t size)
1805 {
1806 	int i;
1807 	unsigned long flags;
1808 	struct arm_smmu_cmdq_ent cmd;
1809 	struct arm_smmu_master *master;
1810 	struct arm_smmu_cmdq_batch cmds;
1811 
1812 	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1813 		return 0;
1814 
1815 	/*
1816 	 * Ensure that we've completed prior invalidation of the main TLBs
1817 	 * before we read 'nr_ats_masters' in case of a concurrent call to
1818 	 * arm_smmu_enable_ats():
1819 	 *
1820 	 *	// unmap()			// arm_smmu_enable_ats()
1821 	 *	TLBI+SYNC			atomic_inc(&nr_ats_masters);
1822 	 *	smp_mb();			[...]
1823 	 *	atomic_read(&nr_ats_masters);	pci_enable_ats() // writel()
1824 	 *
1825 	 * Ensures that we always see the incremented 'nr_ats_masters' count if
1826 	 * ATS was enabled at the PCI device before completion of the TLBI.
1827 	 */
1828 	smp_mb();
1829 	if (!atomic_read(&smmu_domain->nr_ats_masters))
1830 		return 0;
1831 
1832 	arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1833 
1834 	cmds.num = 0;
1835 
1836 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1837 	list_for_each_entry(master, &smmu_domain->devices, domain_head) {
1838 		if (!master->ats_enabled)
1839 			continue;
1840 
1841 		for (i = 0; i < master->num_streams; i++) {
1842 			cmd.atc.sid = master->streams[i].id;
1843 			arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
1844 		}
1845 	}
1846 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1847 
1848 	return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
1849 }
1850 
1851 /* IO_PGTABLE API */
1852 static void arm_smmu_tlb_inv_context(void *cookie)
1853 {
1854 	struct arm_smmu_domain *smmu_domain = cookie;
1855 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1856 	struct arm_smmu_cmdq_ent cmd;
1857 
1858 	/*
1859 	 * NOTE: when io-pgtable is in non-strict mode, we may get here with
1860 	 * PTEs previously cleared by unmaps on the current CPU not yet visible
1861 	 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
1862 	 * insertion to guarantee those are observed before the TLBI. Do be
1863 	 * careful, 007.
1864 	 */
1865 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1866 		arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);
1867 	} else {
1868 		cmd.opcode	= CMDQ_OP_TLBI_S12_VMALL;
1869 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
1870 		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1871 	}
1872 	arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
1873 }
1874 
1875 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
1876 				     unsigned long iova, size_t size,
1877 				     size_t granule,
1878 				     struct arm_smmu_domain *smmu_domain)
1879 {
1880 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1881 	unsigned long end = iova + size, num_pages = 0, tg = 0;
1882 	size_t inv_range = granule;
1883 	struct arm_smmu_cmdq_batch cmds;
1884 
1885 	if (!size)
1886 		return;
1887 
1888 	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
1889 		/* Get the leaf page size */
1890 		tg = __ffs(smmu_domain->domain.pgsize_bitmap);
1891 
1892 		/* Convert page size of 12,14,16 (log2) to 1,2,3 */
1893 		cmd->tlbi.tg = (tg - 10) / 2;
1894 
1895 		/* Determine what level the granule is at */
1896 		cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
1897 
1898 		num_pages = size >> tg;
1899 	}
1900 
1901 	cmds.num = 0;
1902 
1903 	while (iova < end) {
1904 		if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
1905 			/*
1906 			 * On each iteration of the loop, the range is 5 bits
1907 			 * worth of the aligned size remaining.
1908 			 * The range in pages is:
1909 			 *
1910 			 * range = (num_pages & (0x1f << __ffs(num_pages)))
1911 			 */
1912 			unsigned long scale, num;
1913 
1914 			/* Determine the power of 2 multiple number of pages */
1915 			scale = __ffs(num_pages);
1916 			cmd->tlbi.scale = scale;
1917 
1918 			/* Determine how many chunks of 2^scale size we have */
1919 			num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
1920 			cmd->tlbi.num = num - 1;
1921 
1922 			/* range is num * 2^scale * pgsize */
1923 			inv_range = num << (scale + tg);
1924 
1925 			/* Clear out the lower order bits for the next iteration */
1926 			num_pages -= num << scale;
1927 		}
1928 
1929 		cmd->tlbi.addr = iova;
1930 		arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
1931 		iova += inv_range;
1932 	}
1933 	arm_smmu_cmdq_batch_submit(smmu, &cmds);
1934 }
1935 
1936 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
1937 					  size_t granule, bool leaf,
1938 					  struct arm_smmu_domain *smmu_domain)
1939 {
1940 	struct arm_smmu_cmdq_ent cmd = {
1941 		.tlbi = {
1942 			.leaf	= leaf,
1943 		},
1944 	};
1945 
1946 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1947 		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
1948 				  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
1949 		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;
1950 	} else {
1951 		cmd.opcode	= CMDQ_OP_TLBI_S2_IPA;
1952 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;
1953 	}
1954 	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
1955 
1956 	/*
1957 	 * Unfortunately, this can't be leaf-only since we may have
1958 	 * zapped an entire table.
1959 	 */
1960 	arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size);
1961 }
1962 
1963 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
1964 				 size_t granule, bool leaf,
1965 				 struct arm_smmu_domain *smmu_domain)
1966 {
1967 	struct arm_smmu_cmdq_ent cmd = {
1968 		.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
1969 			  CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
1970 		.tlbi = {
1971 			.asid	= asid,
1972 			.leaf	= leaf,
1973 		},
1974 	};
1975 
1976 	__arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
1977 }
1978 
1979 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
1980 					 unsigned long iova, size_t granule,
1981 					 void *cookie)
1982 {
1983 	struct arm_smmu_domain *smmu_domain = cookie;
1984 	struct iommu_domain *domain = &smmu_domain->domain;
1985 
1986 	iommu_iotlb_gather_add_page(domain, gather, iova, granule);
1987 }
1988 
1989 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
1990 				  size_t granule, void *cookie)
1991 {
1992 	arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
1993 }
1994 
1995 static const struct iommu_flush_ops arm_smmu_flush_ops = {
1996 	.tlb_flush_all	= arm_smmu_tlb_inv_context,
1997 	.tlb_flush_walk = arm_smmu_tlb_inv_walk,
1998 	.tlb_add_page	= arm_smmu_tlb_inv_page_nosync,
1999 };
2000 
2001 /* IOMMU API */
2002 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
2003 {
2004 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2005 
2006 	switch (cap) {
2007 	case IOMMU_CAP_CACHE_COHERENCY:
2008 		/* Assume that a coherent TCU implies coherent TBUs */
2009 		return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2010 	case IOMMU_CAP_NOEXEC:
2011 		return true;
2012 	default:
2013 		return false;
2014 	}
2015 }
2016 
2017 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
2018 {
2019 	struct arm_smmu_domain *smmu_domain;
2020 
2021 	if (type == IOMMU_DOMAIN_SVA)
2022 		return arm_smmu_sva_domain_alloc();
2023 
2024 	if (type != IOMMU_DOMAIN_UNMANAGED &&
2025 	    type != IOMMU_DOMAIN_DMA &&
2026 	    type != IOMMU_DOMAIN_DMA_FQ &&
2027 	    type != IOMMU_DOMAIN_IDENTITY)
2028 		return NULL;
2029 
2030 	/*
2031 	 * Allocate the domain and initialise some of its data structures.
2032 	 * We can't really do anything meaningful until we've added a
2033 	 * master.
2034 	 */
2035 	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2036 	if (!smmu_domain)
2037 		return NULL;
2038 
2039 	mutex_init(&smmu_domain->init_mutex);
2040 	INIT_LIST_HEAD(&smmu_domain->devices);
2041 	spin_lock_init(&smmu_domain->devices_lock);
2042 	INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);
2043 
2044 	return &smmu_domain->domain;
2045 }
2046 
2047 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
2048 {
2049 	int idx, size = 1 << span;
2050 
2051 	do {
2052 		idx = find_first_zero_bit(map, size);
2053 		if (idx == size)
2054 			return -ENOSPC;
2055 	} while (test_and_set_bit(idx, map));
2056 
2057 	return idx;
2058 }
2059 
2060 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
2061 {
2062 	clear_bit(idx, map);
2063 }
2064 
2065 static void arm_smmu_domain_free(struct iommu_domain *domain)
2066 {
2067 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2068 	struct arm_smmu_device *smmu = smmu_domain->smmu;
2069 
2070 	free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2071 
2072 	/* Free the CD and ASID, if we allocated them */
2073 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2074 		struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2075 
2076 		/* Prevent SVA from touching the CD while we're freeing it */
2077 		mutex_lock(&arm_smmu_asid_lock);
2078 		if (cfg->cdcfg.cdtab)
2079 			arm_smmu_free_cd_tables(smmu_domain);
2080 		arm_smmu_free_asid(&cfg->cd);
2081 		mutex_unlock(&arm_smmu_asid_lock);
2082 	} else {
2083 		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2084 		if (cfg->vmid)
2085 			arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
2086 	}
2087 
2088 	kfree(smmu_domain);
2089 }
2090 
2091 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
2092 				       struct arm_smmu_master *master,
2093 				       struct io_pgtable_cfg *pgtbl_cfg)
2094 {
2095 	int ret;
2096 	u32 asid;
2097 	struct arm_smmu_device *smmu = smmu_domain->smmu;
2098 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2099 	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2100 
2101 	refcount_set(&cfg->cd.refs, 1);
2102 
2103 	/* Prevent SVA from modifying the ASID until it is written to the CD */
2104 	mutex_lock(&arm_smmu_asid_lock);
2105 	ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd,
2106 		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2107 	if (ret)
2108 		goto out_unlock;
2109 
2110 	cfg->s1cdmax = master->ssid_bits;
2111 
2112 	smmu_domain->stall_enabled = master->stall_enabled;
2113 
2114 	ret = arm_smmu_alloc_cd_tables(smmu_domain);
2115 	if (ret)
2116 		goto out_free_asid;
2117 
2118 	cfg->cd.asid	= (u16)asid;
2119 	cfg->cd.ttbr	= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
2120 	cfg->cd.tcr	= FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
2121 			  FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
2122 			  FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
2123 			  FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
2124 			  FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
2125 			  FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
2126 			  CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
2127 	cfg->cd.mair	= pgtbl_cfg->arm_lpae_s1_cfg.mair;
2128 
2129 	/*
2130 	 * Note that this will end up calling arm_smmu_sync_cd() before
2131 	 * the master has been added to the devices list for this domain.
2132 	 * This isn't an issue because the STE hasn't been installed yet.
2133 	 */
2134 	ret = arm_smmu_write_ctx_desc(smmu_domain, 0, &cfg->cd);
2135 	if (ret)
2136 		goto out_free_cd_tables;
2137 
2138 	mutex_unlock(&arm_smmu_asid_lock);
2139 	return 0;
2140 
2141 out_free_cd_tables:
2142 	arm_smmu_free_cd_tables(smmu_domain);
2143 out_free_asid:
2144 	arm_smmu_free_asid(&cfg->cd);
2145 out_unlock:
2146 	mutex_unlock(&arm_smmu_asid_lock);
2147 	return ret;
2148 }
2149 
2150 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
2151 				       struct arm_smmu_master *master,
2152 				       struct io_pgtable_cfg *pgtbl_cfg)
2153 {
2154 	int vmid;
2155 	struct arm_smmu_device *smmu = smmu_domain->smmu;
2156 	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2157 	typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;
2158 
2159 	vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
2160 	if (vmid < 0)
2161 		return vmid;
2162 
2163 	vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2164 	cfg->vmid	= (u16)vmid;
2165 	cfg->vttbr	= pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2166 	cfg->vtcr	= FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
2167 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
2168 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
2169 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
2170 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
2171 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
2172 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
2173 	return 0;
2174 }
2175 
2176 static int arm_smmu_domain_finalise(struct iommu_domain *domain,
2177 				    struct arm_smmu_master *master)
2178 {
2179 	int ret;
2180 	unsigned long ias, oas;
2181 	enum io_pgtable_fmt fmt;
2182 	struct io_pgtable_cfg pgtbl_cfg;
2183 	struct io_pgtable_ops *pgtbl_ops;
2184 	int (*finalise_stage_fn)(struct arm_smmu_domain *,
2185 				 struct arm_smmu_master *,
2186 				 struct io_pgtable_cfg *);
2187 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2188 	struct arm_smmu_device *smmu = smmu_domain->smmu;
2189 
2190 	if (domain->type == IOMMU_DOMAIN_IDENTITY) {
2191 		smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
2192 		return 0;
2193 	}
2194 
2195 	/* Restrict the stage to what we can actually support */
2196 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2197 		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2198 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2199 		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2200 
2201 	switch (smmu_domain->stage) {
2202 	case ARM_SMMU_DOMAIN_S1:
2203 		ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
2204 		ias = min_t(unsigned long, ias, VA_BITS);
2205 		oas = smmu->ias;
2206 		fmt = ARM_64_LPAE_S1;
2207 		finalise_stage_fn = arm_smmu_domain_finalise_s1;
2208 		break;
2209 	case ARM_SMMU_DOMAIN_NESTED:
2210 	case ARM_SMMU_DOMAIN_S2:
2211 		ias = smmu->ias;
2212 		oas = smmu->oas;
2213 		fmt = ARM_64_LPAE_S2;
2214 		finalise_stage_fn = arm_smmu_domain_finalise_s2;
2215 		break;
2216 	default:
2217 		return -EINVAL;
2218 	}
2219 
2220 	pgtbl_cfg = (struct io_pgtable_cfg) {
2221 		.pgsize_bitmap	= smmu->pgsize_bitmap,
2222 		.ias		= ias,
2223 		.oas		= oas,
2224 		.coherent_walk	= smmu->features & ARM_SMMU_FEAT_COHERENCY,
2225 		.tlb		= &arm_smmu_flush_ops,
2226 		.iommu_dev	= smmu->dev,
2227 	};
2228 
2229 	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2230 	if (!pgtbl_ops)
2231 		return -ENOMEM;
2232 
2233 	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2234 	domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2235 	domain->geometry.force_aperture = true;
2236 
2237 	ret = finalise_stage_fn(smmu_domain, master, &pgtbl_cfg);
2238 	if (ret < 0) {
2239 		free_io_pgtable_ops(pgtbl_ops);
2240 		return ret;
2241 	}
2242 
2243 	smmu_domain->pgtbl_ops = pgtbl_ops;
2244 	return 0;
2245 }
2246 
2247 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2248 {
2249 	__le64 *step;
2250 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2251 
2252 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2253 		struct arm_smmu_strtab_l1_desc *l1_desc;
2254 		int idx;
2255 
2256 		/* Two-level walk */
2257 		idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
2258 		l1_desc = &cfg->l1_desc[idx];
2259 		idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
2260 		step = &l1_desc->l2ptr[idx];
2261 	} else {
2262 		/* Simple linear lookup */
2263 		step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
2264 	}
2265 
2266 	return step;
2267 }
2268 
2269 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
2270 {
2271 	int i, j;
2272 	struct arm_smmu_device *smmu = master->smmu;
2273 
2274 	for (i = 0; i < master->num_streams; ++i) {
2275 		u32 sid = master->streams[i].id;
2276 		__le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
2277 
2278 		/* Bridged PCI devices may end up with duplicated IDs */
2279 		for (j = 0; j < i; j++)
2280 			if (master->streams[j].id == sid)
2281 				break;
2282 		if (j < i)
2283 			continue;
2284 
2285 		arm_smmu_write_strtab_ent(master, sid, step);
2286 	}
2287 }
2288 
2289 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2290 {
2291 	struct device *dev = master->dev;
2292 	struct arm_smmu_device *smmu = master->smmu;
2293 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2294 
2295 	if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2296 		return false;
2297 
2298 	if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2299 		return false;
2300 
2301 	return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
2302 }
2303 
2304 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2305 {
2306 	size_t stu;
2307 	struct pci_dev *pdev;
2308 	struct arm_smmu_device *smmu = master->smmu;
2309 	struct arm_smmu_domain *smmu_domain = master->domain;
2310 
2311 	/* Don't enable ATS at the endpoint if it's not enabled in the STE */
2312 	if (!master->ats_enabled)
2313 		return;
2314 
2315 	/* Smallest Translation Unit: log2 of the smallest supported granule */
2316 	stu = __ffs(smmu->pgsize_bitmap);
2317 	pdev = to_pci_dev(master->dev);
2318 
2319 	atomic_inc(&smmu_domain->nr_ats_masters);
2320 	arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
2321 	if (pci_enable_ats(pdev, stu))
2322 		dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2323 }
2324 
2325 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
2326 {
2327 	struct arm_smmu_domain *smmu_domain = master->domain;
2328 
2329 	if (!master->ats_enabled)
2330 		return;
2331 
2332 	pci_disable_ats(to_pci_dev(master->dev));
2333 	/*
2334 	 * Ensure ATS is disabled at the endpoint before we issue the
2335 	 * ATC invalidation via the SMMU.
2336 	 */
2337 	wmb();
2338 	arm_smmu_atc_inv_master(master);
2339 	atomic_dec(&smmu_domain->nr_ats_masters);
2340 }
2341 
2342 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2343 {
2344 	int ret;
2345 	int features;
2346 	int num_pasids;
2347 	struct pci_dev *pdev;
2348 
2349 	if (!dev_is_pci(master->dev))
2350 		return -ENODEV;
2351 
2352 	pdev = to_pci_dev(master->dev);
2353 
2354 	features = pci_pasid_features(pdev);
2355 	if (features < 0)
2356 		return features;
2357 
2358 	num_pasids = pci_max_pasids(pdev);
2359 	if (num_pasids <= 0)
2360 		return num_pasids;
2361 
2362 	ret = pci_enable_pasid(pdev, features);
2363 	if (ret) {
2364 		dev_err(&pdev->dev, "Failed to enable PASID\n");
2365 		return ret;
2366 	}
2367 
2368 	master->ssid_bits = min_t(u8, ilog2(num_pasids),
2369 				  master->smmu->ssid_bits);
2370 	return 0;
2371 }
2372 
2373 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2374 {
2375 	struct pci_dev *pdev;
2376 
2377 	if (!dev_is_pci(master->dev))
2378 		return;
2379 
2380 	pdev = to_pci_dev(master->dev);
2381 
2382 	if (!pdev->pasid_enabled)
2383 		return;
2384 
2385 	master->ssid_bits = 0;
2386 	pci_disable_pasid(pdev);
2387 }
2388 
2389 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
2390 {
2391 	unsigned long flags;
2392 	struct arm_smmu_domain *smmu_domain = master->domain;
2393 
2394 	if (!smmu_domain)
2395 		return;
2396 
2397 	arm_smmu_disable_ats(master);
2398 
2399 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2400 	list_del(&master->domain_head);
2401 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2402 
2403 	master->domain = NULL;
2404 	master->ats_enabled = false;
2405 	arm_smmu_install_ste_for_dev(master);
2406 }
2407 
2408 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2409 {
2410 	int ret = 0;
2411 	unsigned long flags;
2412 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2413 	struct arm_smmu_device *smmu;
2414 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2415 	struct arm_smmu_master *master;
2416 
2417 	if (!fwspec)
2418 		return -ENOENT;
2419 
2420 	master = dev_iommu_priv_get(dev);
2421 	smmu = master->smmu;
2422 
2423 	/*
2424 	 * Checking that SVA is disabled ensures that this device isn't bound to
2425 	 * any mm, and can be safely detached from its old domain. Bonds cannot
2426 	 * be removed concurrently since we're holding the group mutex.
2427 	 */
2428 	if (arm_smmu_master_sva_enabled(master)) {
2429 		dev_err(dev, "cannot attach - SVA enabled\n");
2430 		return -EBUSY;
2431 	}
2432 
2433 	arm_smmu_detach_dev(master);
2434 
2435 	mutex_lock(&smmu_domain->init_mutex);
2436 
2437 	if (!smmu_domain->smmu) {
2438 		smmu_domain->smmu = smmu;
2439 		ret = arm_smmu_domain_finalise(domain, master);
2440 		if (ret) {
2441 			smmu_domain->smmu = NULL;
2442 			goto out_unlock;
2443 		}
2444 	} else if (smmu_domain->smmu != smmu) {
2445 		ret = -EINVAL;
2446 		goto out_unlock;
2447 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2448 		   master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
2449 		ret = -EINVAL;
2450 		goto out_unlock;
2451 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2452 		   smmu_domain->stall_enabled != master->stall_enabled) {
2453 		ret = -EINVAL;
2454 		goto out_unlock;
2455 	}
2456 
2457 	master->domain = smmu_domain;
2458 
2459 	/*
2460 	 * The SMMU does not support enabling ATS with bypass. When the STE is
2461 	 * in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and
2462 	 * Translated transactions are denied as though ATS is disabled for the
2463 	 * stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and
2464 	 * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry).
2465 	 */
2466 	if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2467 		master->ats_enabled = arm_smmu_ats_supported(master);
2468 
2469 	arm_smmu_install_ste_for_dev(master);
2470 
2471 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2472 	list_add(&master->domain_head, &smmu_domain->devices);
2473 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2474 
2475 	arm_smmu_enable_ats(master);
2476 
2477 out_unlock:
2478 	mutex_unlock(&smmu_domain->init_mutex);
2479 	return ret;
2480 }
2481 
2482 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
2483 			      phys_addr_t paddr, size_t pgsize, size_t pgcount,
2484 			      int prot, gfp_t gfp, size_t *mapped)
2485 {
2486 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2487 
2488 	if (!ops)
2489 		return -ENODEV;
2490 
2491 	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
2492 }
2493 
2494 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
2495 				   size_t pgsize, size_t pgcount,
2496 				   struct iommu_iotlb_gather *gather)
2497 {
2498 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2499 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
2500 
2501 	if (!ops)
2502 		return 0;
2503 
2504 	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
2505 }
2506 
2507 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
2508 {
2509 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2510 
2511 	if (smmu_domain->smmu)
2512 		arm_smmu_tlb_inv_context(smmu_domain);
2513 }
2514 
2515 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
2516 				struct iommu_iotlb_gather *gather)
2517 {
2518 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2519 
2520 	if (!gather->pgsize)
2521 		return;
2522 
2523 	arm_smmu_tlb_inv_range_domain(gather->start,
2524 				      gather->end - gather->start + 1,
2525 				      gather->pgsize, true, smmu_domain);
2526 }
2527 
2528 static phys_addr_t
2529 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2530 {
2531 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2532 
2533 	if (!ops)
2534 		return 0;
2535 
2536 	return ops->iova_to_phys(ops, iova);
2537 }
2538 
2539 static struct platform_driver arm_smmu_driver;
2540 
2541 static
2542 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2543 {
2544 	struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
2545 							  fwnode);
2546 	put_device(dev);
2547 	return dev ? dev_get_drvdata(dev) : NULL;
2548 }
2549 
2550 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2551 {
2552 	unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2553 
2554 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2555 		limit *= 1UL << STRTAB_SPLIT;
2556 
2557 	return sid < limit;
2558 }
2559 
2560 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
2561 {
2562 	/* Check the SIDs are in range of the SMMU and our stream table */
2563 	if (!arm_smmu_sid_in_range(smmu, sid))
2564 		return -ERANGE;
2565 
2566 	/* Ensure l2 strtab is initialised */
2567 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2568 		return arm_smmu_init_l2_strtab(smmu, sid);
2569 
2570 	return 0;
2571 }
2572 
2573 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
2574 				  struct arm_smmu_master *master)
2575 {
2576 	int i;
2577 	int ret = 0;
2578 	struct arm_smmu_stream *new_stream, *cur_stream;
2579 	struct rb_node **new_node, *parent_node = NULL;
2580 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2581 
2582 	master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
2583 				  GFP_KERNEL);
2584 	if (!master->streams)
2585 		return -ENOMEM;
2586 	master->num_streams = fwspec->num_ids;
2587 
2588 	mutex_lock(&smmu->streams_mutex);
2589 	for (i = 0; i < fwspec->num_ids; i++) {
2590 		u32 sid = fwspec->ids[i];
2591 
2592 		new_stream = &master->streams[i];
2593 		new_stream->id = sid;
2594 		new_stream->master = master;
2595 
2596 		ret = arm_smmu_init_sid_strtab(smmu, sid);
2597 		if (ret)
2598 			break;
2599 
2600 		/* Insert into SID tree */
2601 		new_node = &(smmu->streams.rb_node);
2602 		while (*new_node) {
2603 			cur_stream = rb_entry(*new_node, struct arm_smmu_stream,
2604 					      node);
2605 			parent_node = *new_node;
2606 			if (cur_stream->id > new_stream->id) {
2607 				new_node = &((*new_node)->rb_left);
2608 			} else if (cur_stream->id < new_stream->id) {
2609 				new_node = &((*new_node)->rb_right);
2610 			} else {
2611 				dev_warn(master->dev,
2612 					 "stream %u already in tree\n",
2613 					 cur_stream->id);
2614 				ret = -EINVAL;
2615 				break;
2616 			}
2617 		}
2618 		if (ret)
2619 			break;
2620 
2621 		rb_link_node(&new_stream->node, parent_node, new_node);
2622 		rb_insert_color(&new_stream->node, &smmu->streams);
2623 	}
2624 
2625 	if (ret) {
2626 		for (i--; i >= 0; i--)
2627 			rb_erase(&master->streams[i].node, &smmu->streams);
2628 		kfree(master->streams);
2629 	}
2630 	mutex_unlock(&smmu->streams_mutex);
2631 
2632 	return ret;
2633 }
2634 
2635 static void arm_smmu_remove_master(struct arm_smmu_master *master)
2636 {
2637 	int i;
2638 	struct arm_smmu_device *smmu = master->smmu;
2639 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2640 
2641 	if (!smmu || !master->streams)
2642 		return;
2643 
2644 	mutex_lock(&smmu->streams_mutex);
2645 	for (i = 0; i < fwspec->num_ids; i++)
2646 		rb_erase(&master->streams[i].node, &smmu->streams);
2647 	mutex_unlock(&smmu->streams_mutex);
2648 
2649 	kfree(master->streams);
2650 }
2651 
2652 static struct iommu_ops arm_smmu_ops;
2653 
2654 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
2655 {
2656 	int ret;
2657 	struct arm_smmu_device *smmu;
2658 	struct arm_smmu_master *master;
2659 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2660 
2661 	if (!fwspec || fwspec->ops != &arm_smmu_ops)
2662 		return ERR_PTR(-ENODEV);
2663 
2664 	if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
2665 		return ERR_PTR(-EBUSY);
2666 
2667 	smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2668 	if (!smmu)
2669 		return ERR_PTR(-ENODEV);
2670 
2671 	master = kzalloc(sizeof(*master), GFP_KERNEL);
2672 	if (!master)
2673 		return ERR_PTR(-ENOMEM);
2674 
2675 	master->dev = dev;
2676 	master->smmu = smmu;
2677 	INIT_LIST_HEAD(&master->bonds);
2678 	dev_iommu_priv_set(dev, master);
2679 
2680 	ret = arm_smmu_insert_master(smmu, master);
2681 	if (ret)
2682 		goto err_free_master;
2683 
2684 	device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
2685 	master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
2686 
2687 	/*
2688 	 * Note that PASID must be enabled before, and disabled after ATS:
2689 	 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
2690 	 *
2691 	 *   Behavior is undefined if this bit is Set and the value of the PASID
2692 	 *   Enable, Execute Requested Enable, or Privileged Mode Requested bits
2693 	 *   are changed.
2694 	 */
2695 	arm_smmu_enable_pasid(master);
2696 
2697 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
2698 		master->ssid_bits = min_t(u8, master->ssid_bits,
2699 					  CTXDESC_LINEAR_CDMAX);
2700 
2701 	if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
2702 	     device_property_read_bool(dev, "dma-can-stall")) ||
2703 	    smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
2704 		master->stall_enabled = true;
2705 
2706 	return &smmu->iommu;
2707 
2708 err_free_master:
2709 	kfree(master);
2710 	dev_iommu_priv_set(dev, NULL);
2711 	return ERR_PTR(ret);
2712 }
2713 
2714 static void arm_smmu_release_device(struct device *dev)
2715 {
2716 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2717 
2718 	if (WARN_ON(arm_smmu_master_sva_enabled(master)))
2719 		iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
2720 	arm_smmu_detach_dev(master);
2721 	arm_smmu_disable_pasid(master);
2722 	arm_smmu_remove_master(master);
2723 	kfree(master);
2724 }
2725 
2726 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2727 {
2728 	struct iommu_group *group;
2729 
2730 	/*
2731 	 * We don't support devices sharing stream IDs other than PCI RID
2732 	 * aliases, since the necessary ID-to-device lookup becomes rather
2733 	 * impractical given a potential sparse 32-bit stream ID space.
2734 	 */
2735 	if (dev_is_pci(dev))
2736 		group = pci_device_group(dev);
2737 	else
2738 		group = generic_device_group(dev);
2739 
2740 	return group;
2741 }
2742 
2743 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
2744 {
2745 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2746 	int ret = 0;
2747 
2748 	mutex_lock(&smmu_domain->init_mutex);
2749 	if (smmu_domain->smmu)
2750 		ret = -EPERM;
2751 	else
2752 		smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2753 	mutex_unlock(&smmu_domain->init_mutex);
2754 
2755 	return ret;
2756 }
2757 
2758 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2759 {
2760 	return iommu_fwspec_add_ids(dev, args->args, 1);
2761 }
2762 
2763 static void arm_smmu_get_resv_regions(struct device *dev,
2764 				      struct list_head *head)
2765 {
2766 	struct iommu_resv_region *region;
2767 	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2768 
2769 	region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2770 					 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
2771 	if (!region)
2772 		return;
2773 
2774 	list_add_tail(&region->list, head);
2775 
2776 	iommu_dma_get_resv_regions(dev, head);
2777 }
2778 
2779 static int arm_smmu_dev_enable_feature(struct device *dev,
2780 				       enum iommu_dev_features feat)
2781 {
2782 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2783 
2784 	if (!master)
2785 		return -ENODEV;
2786 
2787 	switch (feat) {
2788 	case IOMMU_DEV_FEAT_IOPF:
2789 		if (!arm_smmu_master_iopf_supported(master))
2790 			return -EINVAL;
2791 		if (master->iopf_enabled)
2792 			return -EBUSY;
2793 		master->iopf_enabled = true;
2794 		return 0;
2795 	case IOMMU_DEV_FEAT_SVA:
2796 		if (!arm_smmu_master_sva_supported(master))
2797 			return -EINVAL;
2798 		if (arm_smmu_master_sva_enabled(master))
2799 			return -EBUSY;
2800 		return arm_smmu_master_enable_sva(master);
2801 	default:
2802 		return -EINVAL;
2803 	}
2804 }
2805 
2806 static int arm_smmu_dev_disable_feature(struct device *dev,
2807 					enum iommu_dev_features feat)
2808 {
2809 	struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2810 
2811 	if (!master)
2812 		return -EINVAL;
2813 
2814 	switch (feat) {
2815 	case IOMMU_DEV_FEAT_IOPF:
2816 		if (!master->iopf_enabled)
2817 			return -EINVAL;
2818 		if (master->sva_enabled)
2819 			return -EBUSY;
2820 		master->iopf_enabled = false;
2821 		return 0;
2822 	case IOMMU_DEV_FEAT_SVA:
2823 		if (!arm_smmu_master_sva_enabled(master))
2824 			return -EINVAL;
2825 		return arm_smmu_master_disable_sva(master);
2826 	default:
2827 		return -EINVAL;
2828 	}
2829 }
2830 
2831 /*
2832  * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
2833  * PCIe link and save the data to memory by DMA. The hardware is restricted to
2834  * use identity mapping only.
2835  */
2836 #define IS_HISI_PTT_DEVICE(pdev)	((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
2837 					 (pdev)->device == 0xa12e)
2838 
2839 static int arm_smmu_def_domain_type(struct device *dev)
2840 {
2841 	if (dev_is_pci(dev)) {
2842 		struct pci_dev *pdev = to_pci_dev(dev);
2843 
2844 		if (IS_HISI_PTT_DEVICE(pdev))
2845 			return IOMMU_DOMAIN_IDENTITY;
2846 	}
2847 
2848 	return 0;
2849 }
2850 
2851 static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
2852 {
2853 	struct iommu_domain *domain;
2854 
2855 	domain = iommu_get_domain_for_dev_pasid(dev, pasid, IOMMU_DOMAIN_SVA);
2856 	if (WARN_ON(IS_ERR(domain)) || !domain)
2857 		return;
2858 
2859 	arm_smmu_sva_remove_dev_pasid(domain, dev, pasid);
2860 }
2861 
2862 static struct iommu_ops arm_smmu_ops = {
2863 	.capable		= arm_smmu_capable,
2864 	.domain_alloc		= arm_smmu_domain_alloc,
2865 	.probe_device		= arm_smmu_probe_device,
2866 	.release_device		= arm_smmu_release_device,
2867 	.device_group		= arm_smmu_device_group,
2868 	.of_xlate		= arm_smmu_of_xlate,
2869 	.get_resv_regions	= arm_smmu_get_resv_regions,
2870 	.remove_dev_pasid	= arm_smmu_remove_dev_pasid,
2871 	.dev_enable_feat	= arm_smmu_dev_enable_feature,
2872 	.dev_disable_feat	= arm_smmu_dev_disable_feature,
2873 	.page_response		= arm_smmu_page_response,
2874 	.def_domain_type	= arm_smmu_def_domain_type,
2875 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
2876 	.owner			= THIS_MODULE,
2877 	.default_domain_ops = &(const struct iommu_domain_ops) {
2878 		.attach_dev		= arm_smmu_attach_dev,
2879 		.map_pages		= arm_smmu_map_pages,
2880 		.unmap_pages		= arm_smmu_unmap_pages,
2881 		.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
2882 		.iotlb_sync		= arm_smmu_iotlb_sync,
2883 		.iova_to_phys		= arm_smmu_iova_to_phys,
2884 		.enable_nesting		= arm_smmu_enable_nesting,
2885 		.free			= arm_smmu_domain_free,
2886 	}
2887 };
2888 
2889 /* Probing and initialisation functions */
2890 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2891 				   struct arm_smmu_queue *q,
2892 				   void __iomem *page,
2893 				   unsigned long prod_off,
2894 				   unsigned long cons_off,
2895 				   size_t dwords, const char *name)
2896 {
2897 	size_t qsz;
2898 
2899 	do {
2900 		qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
2901 		q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
2902 					      GFP_KERNEL);
2903 		if (q->base || qsz < PAGE_SIZE)
2904 			break;
2905 
2906 		q->llq.max_n_shift--;
2907 	} while (1);
2908 
2909 	if (!q->base) {
2910 		dev_err(smmu->dev,
2911 			"failed to allocate queue (0x%zx bytes) for %s\n",
2912 			qsz, name);
2913 		return -ENOMEM;
2914 	}
2915 
2916 	if (!WARN_ON(q->base_dma & (qsz - 1))) {
2917 		dev_info(smmu->dev, "allocated %u entries for %s\n",
2918 			 1 << q->llq.max_n_shift, name);
2919 	}
2920 
2921 	q->prod_reg	= page + prod_off;
2922 	q->cons_reg	= page + cons_off;
2923 	q->ent_dwords	= dwords;
2924 
2925 	q->q_base  = Q_BASE_RWA;
2926 	q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2927 	q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
2928 
2929 	q->llq.prod = q->llq.cons = 0;
2930 	return 0;
2931 }
2932 
2933 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
2934 {
2935 	struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
2936 	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
2937 
2938 	atomic_set(&cmdq->owner_prod, 0);
2939 	atomic_set(&cmdq->lock, 0);
2940 
2941 	cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
2942 							      GFP_KERNEL);
2943 	if (!cmdq->valid_map)
2944 		return -ENOMEM;
2945 
2946 	return 0;
2947 }
2948 
2949 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2950 {
2951 	int ret;
2952 
2953 	/* cmdq */
2954 	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
2955 				      ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
2956 				      CMDQ_ENT_DWORDS, "cmdq");
2957 	if (ret)
2958 		return ret;
2959 
2960 	ret = arm_smmu_cmdq_init(smmu);
2961 	if (ret)
2962 		return ret;
2963 
2964 	/* evtq */
2965 	ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
2966 				      ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
2967 				      EVTQ_ENT_DWORDS, "evtq");
2968 	if (ret)
2969 		return ret;
2970 
2971 	if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
2972 	    (smmu->features & ARM_SMMU_FEAT_STALLS)) {
2973 		smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
2974 		if (!smmu->evtq.iopf)
2975 			return -ENOMEM;
2976 	}
2977 
2978 	/* priq */
2979 	if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2980 		return 0;
2981 
2982 	return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
2983 				       ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
2984 				       PRIQ_ENT_DWORDS, "priq");
2985 }
2986 
2987 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2988 {
2989 	unsigned int i;
2990 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2991 	void *strtab = smmu->strtab_cfg.strtab;
2992 
2993 	cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents,
2994 				    sizeof(*cfg->l1_desc), GFP_KERNEL);
2995 	if (!cfg->l1_desc)
2996 		return -ENOMEM;
2997 
2998 	for (i = 0; i < cfg->num_l1_ents; ++i) {
2999 		arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
3000 		strtab += STRTAB_L1_DESC_DWORDS << 3;
3001 	}
3002 
3003 	return 0;
3004 }
3005 
3006 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3007 {
3008 	void *strtab;
3009 	u64 reg;
3010 	u32 size, l1size;
3011 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3012 
3013 	/* Calculate the L1 size, capped to the SIDSIZE. */
3014 	size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
3015 	size = min(size, smmu->sid_bits - STRTAB_SPLIT);
3016 	cfg->num_l1_ents = 1 << size;
3017 
3018 	size += STRTAB_SPLIT;
3019 	if (size < smmu->sid_bits)
3020 		dev_warn(smmu->dev,
3021 			 "2-level strtab only covers %u/%u bits of SID\n",
3022 			 size, smmu->sid_bits);
3023 
3024 	l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
3025 	strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
3026 				     GFP_KERNEL);
3027 	if (!strtab) {
3028 		dev_err(smmu->dev,
3029 			"failed to allocate l1 stream table (%u bytes)\n",
3030 			l1size);
3031 		return -ENOMEM;
3032 	}
3033 	cfg->strtab = strtab;
3034 
3035 	/* Configure strtab_base_cfg for 2 levels */
3036 	reg  = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
3037 	reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
3038 	reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
3039 	cfg->strtab_base_cfg = reg;
3040 
3041 	return arm_smmu_init_l1_strtab(smmu);
3042 }
3043 
3044 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3045 {
3046 	void *strtab;
3047 	u64 reg;
3048 	u32 size;
3049 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3050 
3051 	size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
3052 	strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
3053 				     GFP_KERNEL);
3054 	if (!strtab) {
3055 		dev_err(smmu->dev,
3056 			"failed to allocate linear stream table (%u bytes)\n",
3057 			size);
3058 		return -ENOMEM;
3059 	}
3060 	cfg->strtab = strtab;
3061 	cfg->num_l1_ents = 1 << smmu->sid_bits;
3062 
3063 	/* Configure strtab_base_cfg for a linear table covering all SIDs */
3064 	reg  = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
3065 	reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
3066 	cfg->strtab_base_cfg = reg;
3067 
3068 	arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false);
3069 	return 0;
3070 }
3071 
3072 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3073 {
3074 	u64 reg;
3075 	int ret;
3076 
3077 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3078 		ret = arm_smmu_init_strtab_2lvl(smmu);
3079 	else
3080 		ret = arm_smmu_init_strtab_linear(smmu);
3081 
3082 	if (ret)
3083 		return ret;
3084 
3085 	/* Set the strtab base address */
3086 	reg  = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
3087 	reg |= STRTAB_BASE_RA;
3088 	smmu->strtab_cfg.strtab_base = reg;
3089 
3090 	/* Allocate the first VMID for stage-2 bypass STEs */
3091 	set_bit(0, smmu->vmid_map);
3092 	return 0;
3093 }
3094 
3095 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3096 {
3097 	int ret;
3098 
3099 	mutex_init(&smmu->streams_mutex);
3100 	smmu->streams = RB_ROOT;
3101 
3102 	ret = arm_smmu_init_queues(smmu);
3103 	if (ret)
3104 		return ret;
3105 
3106 	return arm_smmu_init_strtab(smmu);
3107 }
3108 
3109 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3110 				   unsigned int reg_off, unsigned int ack_off)
3111 {
3112 	u32 reg;
3113 
3114 	writel_relaxed(val, smmu->base + reg_off);
3115 	return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3116 					  1, ARM_SMMU_POLL_TIMEOUT_US);
3117 }
3118 
3119 /* GBPA is "special" */
3120 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3121 {
3122 	int ret;
3123 	u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3124 
3125 	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3126 					 1, ARM_SMMU_POLL_TIMEOUT_US);
3127 	if (ret)
3128 		return ret;
3129 
3130 	reg &= ~clr;
3131 	reg |= set;
3132 	writel_relaxed(reg | GBPA_UPDATE, gbpa);
3133 	ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3134 					 1, ARM_SMMU_POLL_TIMEOUT_US);
3135 
3136 	if (ret)
3137 		dev_err(smmu->dev, "GBPA not responding to update\n");
3138 	return ret;
3139 }
3140 
3141 static void arm_smmu_free_msis(void *data)
3142 {
3143 	struct device *dev = data;
3144 	platform_msi_domain_free_irqs(dev);
3145 }
3146 
3147 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3148 {
3149 	phys_addr_t doorbell;
3150 	struct device *dev = msi_desc_to_dev(desc);
3151 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3152 	phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
3153 
3154 	doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3155 	doorbell &= MSI_CFG0_ADDR_MASK;
3156 
3157 	writeq_relaxed(doorbell, smmu->base + cfg[0]);
3158 	writel_relaxed(msg->data, smmu->base + cfg[1]);
3159 	writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3160 }
3161 
3162 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3163 {
3164 	int ret, nvec = ARM_SMMU_MAX_MSIS;
3165 	struct device *dev = smmu->dev;
3166 
3167 	/* Clear the MSI address regs */
3168 	writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3169 	writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3170 
3171 	if (smmu->features & ARM_SMMU_FEAT_PRI)
3172 		writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3173 	else
3174 		nvec--;
3175 
3176 	if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3177 		return;
3178 
3179 	if (!dev->msi.domain) {
3180 		dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3181 		return;
3182 	}
3183 
3184 	/* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3185 	ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3186 	if (ret) {
3187 		dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3188 		return;
3189 	}
3190 
3191 	smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
3192 	smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
3193 	smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
3194 
3195 	/* Add callback to free MSIs on teardown */
3196 	devm_add_action(dev, arm_smmu_free_msis, dev);
3197 }
3198 
3199 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3200 {
3201 	int irq, ret;
3202 
3203 	arm_smmu_setup_msis(smmu);
3204 
3205 	/* Request interrupt lines */
3206 	irq = smmu->evtq.q.irq;
3207 	if (irq) {
3208 		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3209 						arm_smmu_evtq_thread,
3210 						IRQF_ONESHOT,
3211 						"arm-smmu-v3-evtq", smmu);
3212 		if (ret < 0)
3213 			dev_warn(smmu->dev, "failed to enable evtq irq\n");
3214 	} else {
3215 		dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3216 	}
3217 
3218 	irq = smmu->gerr_irq;
3219 	if (irq) {
3220 		ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3221 				       0, "arm-smmu-v3-gerror", smmu);
3222 		if (ret < 0)
3223 			dev_warn(smmu->dev, "failed to enable gerror irq\n");
3224 	} else {
3225 		dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3226 	}
3227 
3228 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
3229 		irq = smmu->priq.q.irq;
3230 		if (irq) {
3231 			ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3232 							arm_smmu_priq_thread,
3233 							IRQF_ONESHOT,
3234 							"arm-smmu-v3-priq",
3235 							smmu);
3236 			if (ret < 0)
3237 				dev_warn(smmu->dev,
3238 					 "failed to enable priq irq\n");
3239 		} else {
3240 			dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3241 		}
3242 	}
3243 }
3244 
3245 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
3246 {
3247 	int ret, irq;
3248 	u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
3249 
3250 	/* Disable IRQs first */
3251 	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
3252 				      ARM_SMMU_IRQ_CTRLACK);
3253 	if (ret) {
3254 		dev_err(smmu->dev, "failed to disable irqs\n");
3255 		return ret;
3256 	}
3257 
3258 	irq = smmu->combined_irq;
3259 	if (irq) {
3260 		/*
3261 		 * Cavium ThunderX2 implementation doesn't support unique irq
3262 		 * lines. Use a single irq line for all the SMMUv3 interrupts.
3263 		 */
3264 		ret = devm_request_threaded_irq(smmu->dev, irq,
3265 					arm_smmu_combined_irq_handler,
3266 					arm_smmu_combined_irq_thread,
3267 					IRQF_ONESHOT,
3268 					"arm-smmu-v3-combined-irq", smmu);
3269 		if (ret < 0)
3270 			dev_warn(smmu->dev, "failed to enable combined irq\n");
3271 	} else
3272 		arm_smmu_setup_unique_irqs(smmu);
3273 
3274 	if (smmu->features & ARM_SMMU_FEAT_PRI)
3275 		irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
3276 
3277 	/* Enable interrupt generation on the SMMU */
3278 	ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
3279 				      ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
3280 	if (ret)
3281 		dev_warn(smmu->dev, "failed to enable irqs\n");
3282 
3283 	return 0;
3284 }
3285 
3286 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
3287 {
3288 	int ret;
3289 
3290 	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
3291 	if (ret)
3292 		dev_err(smmu->dev, "failed to clear cr0\n");
3293 
3294 	return ret;
3295 }
3296 
3297 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
3298 {
3299 	int ret;
3300 	u32 reg, enables;
3301 	struct arm_smmu_cmdq_ent cmd;
3302 
3303 	/* Clear CR0 and sync (disables SMMU and queue processing) */
3304 	reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
3305 	if (reg & CR0_SMMUEN) {
3306 		dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3307 		WARN_ON(is_kdump_kernel() && !disable_bypass);
3308 		arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
3309 	}
3310 
3311 	ret = arm_smmu_device_disable(smmu);
3312 	if (ret)
3313 		return ret;
3314 
3315 	/* CR1 (table and queue memory attributes) */
3316 	reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
3317 	      FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
3318 	      FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
3319 	      FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
3320 	      FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
3321 	      FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
3322 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3323 
3324 	/* CR2 (random crap) */
3325 	reg = CR2_PTM | CR2_RECINVSID;
3326 
3327 	if (smmu->features & ARM_SMMU_FEAT_E2H)
3328 		reg |= CR2_E2H;
3329 
3330 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3331 
3332 	/* Stream table */
3333 	writeq_relaxed(smmu->strtab_cfg.strtab_base,
3334 		       smmu->base + ARM_SMMU_STRTAB_BASE);
3335 	writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
3336 		       smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3337 
3338 	/* Command queue */
3339 	writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3340 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3341 	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
3342 
3343 	enables = CR0_CMDQEN;
3344 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3345 				      ARM_SMMU_CR0ACK);
3346 	if (ret) {
3347 		dev_err(smmu->dev, "failed to enable command queue\n");
3348 		return ret;
3349 	}
3350 
3351 	/* Invalidate any cached configuration */
3352 	cmd.opcode = CMDQ_OP_CFGI_ALL;
3353 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3354 
3355 	/* Invalidate any stale TLB entries */
3356 	if (smmu->features & ARM_SMMU_FEAT_HYP) {
3357 		cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
3358 		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3359 	}
3360 
3361 	cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
3362 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3363 
3364 	/* Event queue */
3365 	writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3366 	writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
3367 	writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
3368 
3369 	enables |= CR0_EVTQEN;
3370 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3371 				      ARM_SMMU_CR0ACK);
3372 	if (ret) {
3373 		dev_err(smmu->dev, "failed to enable event queue\n");
3374 		return ret;
3375 	}
3376 
3377 	/* PRI queue */
3378 	if (smmu->features & ARM_SMMU_FEAT_PRI) {
3379 		writeq_relaxed(smmu->priq.q.q_base,
3380 			       smmu->base + ARM_SMMU_PRIQ_BASE);
3381 		writel_relaxed(smmu->priq.q.llq.prod,
3382 			       smmu->page1 + ARM_SMMU_PRIQ_PROD);
3383 		writel_relaxed(smmu->priq.q.llq.cons,
3384 			       smmu->page1 + ARM_SMMU_PRIQ_CONS);
3385 
3386 		enables |= CR0_PRIQEN;
3387 		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3388 					      ARM_SMMU_CR0ACK);
3389 		if (ret) {
3390 			dev_err(smmu->dev, "failed to enable PRI queue\n");
3391 			return ret;
3392 		}
3393 	}
3394 
3395 	if (smmu->features & ARM_SMMU_FEAT_ATS) {
3396 		enables |= CR0_ATSCHK;
3397 		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3398 					      ARM_SMMU_CR0ACK);
3399 		if (ret) {
3400 			dev_err(smmu->dev, "failed to enable ATS check\n");
3401 			return ret;
3402 		}
3403 	}
3404 
3405 	ret = arm_smmu_setup_irqs(smmu);
3406 	if (ret) {
3407 		dev_err(smmu->dev, "failed to setup irqs\n");
3408 		return ret;
3409 	}
3410 
3411 	if (is_kdump_kernel())
3412 		enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
3413 
3414 	/* Enable the SMMU interface, or ensure bypass */
3415 	if (!bypass || disable_bypass) {
3416 		enables |= CR0_SMMUEN;
3417 	} else {
3418 		ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
3419 		if (ret)
3420 			return ret;
3421 	}
3422 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3423 				      ARM_SMMU_CR0ACK);
3424 	if (ret) {
3425 		dev_err(smmu->dev, "failed to enable SMMU interface\n");
3426 		return ret;
3427 	}
3428 
3429 	return 0;
3430 }
3431 
3432 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
3433 {
3434 	u32 reg;
3435 	bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
3436 
3437 	/* IDR0 */
3438 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
3439 
3440 	/* 2-level structures */
3441 	if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
3442 		smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
3443 
3444 	if (reg & IDR0_CD2L)
3445 		smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
3446 
3447 	/*
3448 	 * Translation table endianness.
3449 	 * We currently require the same endianness as the CPU, but this
3450 	 * could be changed later by adding a new IO_PGTABLE_QUIRK.
3451 	 */
3452 	switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
3453 	case IDR0_TTENDIAN_MIXED:
3454 		smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
3455 		break;
3456 #ifdef __BIG_ENDIAN
3457 	case IDR0_TTENDIAN_BE:
3458 		smmu->features |= ARM_SMMU_FEAT_TT_BE;
3459 		break;
3460 #else
3461 	case IDR0_TTENDIAN_LE:
3462 		smmu->features |= ARM_SMMU_FEAT_TT_LE;
3463 		break;
3464 #endif
3465 	default:
3466 		dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
3467 		return -ENXIO;
3468 	}
3469 
3470 	/* Boolean feature flags */
3471 	if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
3472 		smmu->features |= ARM_SMMU_FEAT_PRI;
3473 
3474 	if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
3475 		smmu->features |= ARM_SMMU_FEAT_ATS;
3476 
3477 	if (reg & IDR0_SEV)
3478 		smmu->features |= ARM_SMMU_FEAT_SEV;
3479 
3480 	if (reg & IDR0_MSI) {
3481 		smmu->features |= ARM_SMMU_FEAT_MSI;
3482 		if (coherent && !disable_msipolling)
3483 			smmu->options |= ARM_SMMU_OPT_MSIPOLL;
3484 	}
3485 
3486 	if (reg & IDR0_HYP) {
3487 		smmu->features |= ARM_SMMU_FEAT_HYP;
3488 		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
3489 			smmu->features |= ARM_SMMU_FEAT_E2H;
3490 	}
3491 
3492 	/*
3493 	 * The coherency feature as set by FW is used in preference to the ID
3494 	 * register, but warn on mismatch.
3495 	 */
3496 	if (!!(reg & IDR0_COHACC) != coherent)
3497 		dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
3498 			 coherent ? "true" : "false");
3499 
3500 	switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
3501 	case IDR0_STALL_MODEL_FORCE:
3502 		smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
3503 		fallthrough;
3504 	case IDR0_STALL_MODEL_STALL:
3505 		smmu->features |= ARM_SMMU_FEAT_STALLS;
3506 	}
3507 
3508 	if (reg & IDR0_S1P)
3509 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
3510 
3511 	if (reg & IDR0_S2P)
3512 		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
3513 
3514 	if (!(reg & (IDR0_S1P | IDR0_S2P))) {
3515 		dev_err(smmu->dev, "no translation support!\n");
3516 		return -ENXIO;
3517 	}
3518 
3519 	/* We only support the AArch64 table format at present */
3520 	switch (FIELD_GET(IDR0_TTF, reg)) {
3521 	case IDR0_TTF_AARCH32_64:
3522 		smmu->ias = 40;
3523 		fallthrough;
3524 	case IDR0_TTF_AARCH64:
3525 		break;
3526 	default:
3527 		dev_err(smmu->dev, "AArch64 table format not supported!\n");
3528 		return -ENXIO;
3529 	}
3530 
3531 	/* ASID/VMID sizes */
3532 	smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
3533 	smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
3534 
3535 	/* IDR1 */
3536 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
3537 	if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
3538 		dev_err(smmu->dev, "embedded implementation not supported\n");
3539 		return -ENXIO;
3540 	}
3541 
3542 	/* Queue sizes, capped to ensure natural alignment */
3543 	smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
3544 					     FIELD_GET(IDR1_CMDQS, reg));
3545 	if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
3546 		/*
3547 		 * We don't support splitting up batches, so one batch of
3548 		 * commands plus an extra sync needs to fit inside the command
3549 		 * queue. There's also no way we can handle the weird alignment
3550 		 * restrictions on the base pointer for a unit-length queue.
3551 		 */
3552 		dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
3553 			CMDQ_BATCH_ENTRIES);
3554 		return -ENXIO;
3555 	}
3556 
3557 	smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
3558 					     FIELD_GET(IDR1_EVTQS, reg));
3559 	smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
3560 					     FIELD_GET(IDR1_PRIQS, reg));
3561 
3562 	/* SID/SSID sizes */
3563 	smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
3564 	smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3565 	smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
3566 
3567 	/*
3568 	 * If the SMMU supports fewer bits than would fill a single L2 stream
3569 	 * table, use a linear table instead.
3570 	 */
3571 	if (smmu->sid_bits <= STRTAB_SPLIT)
3572 		smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
3573 
3574 	/* IDR3 */
3575 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
3576 	if (FIELD_GET(IDR3_RIL, reg))
3577 		smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
3578 
3579 	/* IDR5 */
3580 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
3581 
3582 	/* Maximum number of outstanding stalls */
3583 	smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
3584 
3585 	/* Page sizes */
3586 	if (reg & IDR5_GRAN64K)
3587 		smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
3588 	if (reg & IDR5_GRAN16K)
3589 		smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
3590 	if (reg & IDR5_GRAN4K)
3591 		smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
3592 
3593 	/* Input address size */
3594 	if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
3595 		smmu->features |= ARM_SMMU_FEAT_VAX;
3596 
3597 	/* Output address size */
3598 	switch (FIELD_GET(IDR5_OAS, reg)) {
3599 	case IDR5_OAS_32_BIT:
3600 		smmu->oas = 32;
3601 		break;
3602 	case IDR5_OAS_36_BIT:
3603 		smmu->oas = 36;
3604 		break;
3605 	case IDR5_OAS_40_BIT:
3606 		smmu->oas = 40;
3607 		break;
3608 	case IDR5_OAS_42_BIT:
3609 		smmu->oas = 42;
3610 		break;
3611 	case IDR5_OAS_44_BIT:
3612 		smmu->oas = 44;
3613 		break;
3614 	case IDR5_OAS_52_BIT:
3615 		smmu->oas = 52;
3616 		smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
3617 		break;
3618 	default:
3619 		dev_info(smmu->dev,
3620 			"unknown output address size. Truncating to 48-bit\n");
3621 		fallthrough;
3622 	case IDR5_OAS_48_BIT:
3623 		smmu->oas = 48;
3624 	}
3625 
3626 	if (arm_smmu_ops.pgsize_bitmap == -1UL)
3627 		arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
3628 	else
3629 		arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
3630 
3631 	/* Set the DMA mask for our table walker */
3632 	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
3633 		dev_warn(smmu->dev,
3634 			 "failed to set DMA mask for table walker\n");
3635 
3636 	smmu->ias = max(smmu->ias, smmu->oas);
3637 
3638 	if (arm_smmu_sva_supported(smmu))
3639 		smmu->features |= ARM_SMMU_FEAT_SVA;
3640 
3641 	dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3642 		 smmu->ias, smmu->oas, smmu->features);
3643 	return 0;
3644 }
3645 
3646 #ifdef CONFIG_ACPI
3647 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
3648 {
3649 	switch (model) {
3650 	case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
3651 		smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
3652 		break;
3653 	case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
3654 		smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3655 		break;
3656 	}
3657 
3658 	dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3659 }
3660 
3661 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3662 				      struct arm_smmu_device *smmu)
3663 {
3664 	struct acpi_iort_smmu_v3 *iort_smmu;
3665 	struct device *dev = smmu->dev;
3666 	struct acpi_iort_node *node;
3667 
3668 	node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3669 
3670 	/* Retrieve SMMUv3 specific data */
3671 	iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3672 
3673 	acpi_smmu_get_options(iort_smmu->model, smmu);
3674 
3675 	if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3676 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3677 
3678 	return 0;
3679 }
3680 #else
3681 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3682 					     struct arm_smmu_device *smmu)
3683 {
3684 	return -ENODEV;
3685 }
3686 #endif
3687 
3688 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3689 				    struct arm_smmu_device *smmu)
3690 {
3691 	struct device *dev = &pdev->dev;
3692 	u32 cells;
3693 	int ret = -EINVAL;
3694 
3695 	if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3696 		dev_err(dev, "missing #iommu-cells property\n");
3697 	else if (cells != 1)
3698 		dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3699 	else
3700 		ret = 0;
3701 
3702 	parse_driver_options(smmu);
3703 
3704 	if (of_dma_is_coherent(dev->of_node))
3705 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3706 
3707 	return ret;
3708 }
3709 
3710 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3711 {
3712 	if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3713 		return SZ_64K;
3714 	else
3715 		return SZ_128K;
3716 }
3717 
3718 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
3719 				      resource_size_t size)
3720 {
3721 	struct resource res = DEFINE_RES_MEM(start, size);
3722 
3723 	return devm_ioremap_resource(dev, &res);
3724 }
3725 
3726 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
3727 {
3728 	struct list_head rmr_list;
3729 	struct iommu_resv_region *e;
3730 
3731 	INIT_LIST_HEAD(&rmr_list);
3732 	iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
3733 
3734 	list_for_each_entry(e, &rmr_list, list) {
3735 		__le64 *step;
3736 		struct iommu_iort_rmr_data *rmr;
3737 		int ret, i;
3738 
3739 		rmr = container_of(e, struct iommu_iort_rmr_data, rr);
3740 		for (i = 0; i < rmr->num_sids; i++) {
3741 			ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
3742 			if (ret) {
3743 				dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
3744 					rmr->sids[i]);
3745 				continue;
3746 			}
3747 
3748 			step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]);
3749 			arm_smmu_init_bypass_stes(step, 1, true);
3750 		}
3751 	}
3752 
3753 	iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
3754 }
3755 
3756 static int arm_smmu_device_probe(struct platform_device *pdev)
3757 {
3758 	int irq, ret;
3759 	struct resource *res;
3760 	resource_size_t ioaddr;
3761 	struct arm_smmu_device *smmu;
3762 	struct device *dev = &pdev->dev;
3763 	bool bypass;
3764 
3765 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3766 	if (!smmu)
3767 		return -ENOMEM;
3768 	smmu->dev = dev;
3769 
3770 	if (dev->of_node) {
3771 		ret = arm_smmu_device_dt_probe(pdev, smmu);
3772 	} else {
3773 		ret = arm_smmu_device_acpi_probe(pdev, smmu);
3774 		if (ret == -ENODEV)
3775 			return ret;
3776 	}
3777 
3778 	/* Set bypass mode according to firmware probing result */
3779 	bypass = !!ret;
3780 
3781 	/* Base address */
3782 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3783 	if (!res)
3784 		return -EINVAL;
3785 	if (resource_size(res) < arm_smmu_resource_size(smmu)) {
3786 		dev_err(dev, "MMIO region too small (%pr)\n", res);
3787 		return -EINVAL;
3788 	}
3789 	ioaddr = res->start;
3790 
3791 	/*
3792 	 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
3793 	 * the PMCG registers which are reserved by the PMU driver.
3794 	 */
3795 	smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
3796 	if (IS_ERR(smmu->base))
3797 		return PTR_ERR(smmu->base);
3798 
3799 	if (arm_smmu_resource_size(smmu) > SZ_64K) {
3800 		smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
3801 					       ARM_SMMU_REG_SZ);
3802 		if (IS_ERR(smmu->page1))
3803 			return PTR_ERR(smmu->page1);
3804 	} else {
3805 		smmu->page1 = smmu->base;
3806 	}
3807 
3808 	/* Interrupt lines */
3809 
3810 	irq = platform_get_irq_byname_optional(pdev, "combined");
3811 	if (irq > 0)
3812 		smmu->combined_irq = irq;
3813 	else {
3814 		irq = platform_get_irq_byname_optional(pdev, "eventq");
3815 		if (irq > 0)
3816 			smmu->evtq.q.irq = irq;
3817 
3818 		irq = platform_get_irq_byname_optional(pdev, "priq");
3819 		if (irq > 0)
3820 			smmu->priq.q.irq = irq;
3821 
3822 		irq = platform_get_irq_byname_optional(pdev, "gerror");
3823 		if (irq > 0)
3824 			smmu->gerr_irq = irq;
3825 	}
3826 	/* Probe the h/w */
3827 	ret = arm_smmu_device_hw_probe(smmu);
3828 	if (ret)
3829 		return ret;
3830 
3831 	/* Initialise in-memory data structures */
3832 	ret = arm_smmu_init_structures(smmu);
3833 	if (ret)
3834 		return ret;
3835 
3836 	/* Record our private device structure */
3837 	platform_set_drvdata(pdev, smmu);
3838 
3839 	/* Check for RMRs and install bypass STEs if any */
3840 	arm_smmu_rmr_install_bypass_ste(smmu);
3841 
3842 	/* Reset the device */
3843 	ret = arm_smmu_device_reset(smmu, bypass);
3844 	if (ret)
3845 		return ret;
3846 
3847 	/* And we're up. Go go go! */
3848 	ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3849 				     "smmu3.%pa", &ioaddr);
3850 	if (ret)
3851 		return ret;
3852 
3853 	ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
3854 	if (ret) {
3855 		dev_err(dev, "Failed to register iommu\n");
3856 		iommu_device_sysfs_remove(&smmu->iommu);
3857 		return ret;
3858 	}
3859 
3860 	return 0;
3861 }
3862 
3863 static void arm_smmu_device_remove(struct platform_device *pdev)
3864 {
3865 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3866 
3867 	iommu_device_unregister(&smmu->iommu);
3868 	iommu_device_sysfs_remove(&smmu->iommu);
3869 	arm_smmu_device_disable(smmu);
3870 	iopf_queue_free(smmu->evtq.iopf);
3871 }
3872 
3873 static void arm_smmu_device_shutdown(struct platform_device *pdev)
3874 {
3875 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3876 
3877 	arm_smmu_device_disable(smmu);
3878 }
3879 
3880 static const struct of_device_id arm_smmu_of_match[] = {
3881 	{ .compatible = "arm,smmu-v3", },
3882 	{ },
3883 };
3884 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
3885 
3886 static void arm_smmu_driver_unregister(struct platform_driver *drv)
3887 {
3888 	arm_smmu_sva_notifier_synchronize();
3889 	platform_driver_unregister(drv);
3890 }
3891 
3892 static struct platform_driver arm_smmu_driver = {
3893 	.driver	= {
3894 		.name			= "arm-smmu-v3",
3895 		.of_match_table		= arm_smmu_of_match,
3896 		.suppress_bind_attrs	= true,
3897 	},
3898 	.probe	= arm_smmu_device_probe,
3899 	.remove_new = arm_smmu_device_remove,
3900 	.shutdown = arm_smmu_device_shutdown,
3901 };
3902 module_driver(arm_smmu_driver, platform_driver_register,
3903 	      arm_smmu_driver_unregister);
3904 
3905 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
3906 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
3907 MODULE_ALIAS("platform:arm-smmu-v3");
3908 MODULE_LICENSE("GPL v2");
3909