1 // SPDX-License-Identifier: GPL-2.0
19 #include <linux/io-pgtable.h>
27 #include <linux/pci-ats.h>
30 #include "arm-smmu-v3.h"
31 #include "../../dma-iommu.h"
32 #include "../../iommu-sva.h"
37 …domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
42 	"Disable MSI-based polling for CMD_SYNC completion.");
84 	{ ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
85 	{ ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
89 static void parse_driver_options(struct arm_smmu_device *smmu)  in parse_driver_options()  argument
94 		if (of_property_read_bool(smmu->dev->of_node,  in parse_driver_options()
96 			smmu->options |= arm_smmu_options[i].opt;  in parse_driver_options()
97 			dev_notice(smmu->dev, "option %s\n",  in parse_driver_options()
103 /* Low-level queue manipulation functions */
108 	prod = Q_IDX(q, q->prod);  in queue_has_space()
109 	cons = Q_IDX(q, q->cons);  in queue_has_space()
111 	if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))  in queue_has_space()
112 		space = (1 << q->max_n_shift) - (prod - cons);  in queue_has_space()
114 		space = cons - prod;  in queue_has_space()
121 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_full()
122 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);  in queue_full()
127 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_empty()
128 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);  in queue_empty()
133 	return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&  in queue_consumed()
134 		(Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||  in queue_consumed()
135 	       ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&  in queue_consumed()
136 		(Q_IDX(q, q->cons) <= Q_IDX(q, prod)));  in queue_consumed()
146 	writel_relaxed(q->llq.cons, q->cons_reg);  in queue_sync_cons_out()
151 	u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;  in queue_inc_cons()
152 	q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);  in queue_inc_cons()
157 	struct arm_smmu_ll_queue *llq = &q->llq;  in queue_sync_cons_ovf()
159 	if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))  in queue_sync_cons_ovf()
162 	llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |  in queue_sync_cons_ovf()
163 		      Q_IDX(llq, llq->cons);  in queue_sync_cons_ovf()
177 	prod = readl(q->prod_reg);  in queue_sync_prod_in()
179 	if (Q_OVF(prod) != Q_OVF(q->llq.prod))  in queue_sync_prod_in()
180 		ret = -EOVERFLOW;  in queue_sync_prod_in()
182 	q->llq.prod = prod;  in queue_sync_prod_in()
188 	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;  in queue_inc_prod_n()
189 	return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);  in queue_inc_prod_n()
192 static void queue_poll_init(struct arm_smmu_device *smmu,  in queue_poll_init()  argument
195 	qp->delay = 1;  in queue_poll_init()
196 	qp->spin_cnt = 0;  in queue_poll_init()
197 	qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);  in queue_poll_init()
198 	qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);  in queue_poll_init()
203 	if (ktime_compare(ktime_get(), qp->timeout) > 0)  in queue_poll()
204 		return -ETIMEDOUT;  in queue_poll()
206 	if (qp->wfe) {  in queue_poll()
208 	} else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {  in queue_poll()
211 		udelay(qp->delay);  in queue_poll()
212 		qp->delay *= 2;  in queue_poll()
213 		qp->spin_cnt = 0;  in queue_poll()
237 	if (queue_empty(&q->llq))  in queue_remove_raw()
238 		return -EAGAIN;  in queue_remove_raw()
240 	queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);  in queue_remove_raw()
241 	queue_inc_cons(&q->llq);  in queue_remove_raw()
246 /* High-level queue accessors */
250 	cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);  in arm_smmu_cmdq_build_cmd()
252 	switch (ent->opcode) {  in arm_smmu_cmdq_build_cmd()
257 		cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);  in arm_smmu_cmdq_build_cmd()
260 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);  in arm_smmu_cmdq_build_cmd()
263 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);  in arm_smmu_cmdq_build_cmd()
264 		cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);  in arm_smmu_cmdq_build_cmd()
267 		cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);  in arm_smmu_cmdq_build_cmd()
274 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);  in arm_smmu_cmdq_build_cmd()
277 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);  in arm_smmu_cmdq_build_cmd()
278 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);  in arm_smmu_cmdq_build_cmd()
279 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);  in arm_smmu_cmdq_build_cmd()
280 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);  in arm_smmu_cmdq_build_cmd()
281 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);  in arm_smmu_cmdq_build_cmd()
282 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);  in arm_smmu_cmdq_build_cmd()
283 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;  in arm_smmu_cmdq_build_cmd()
286 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);  in arm_smmu_cmdq_build_cmd()
287 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);  in arm_smmu_cmdq_build_cmd()
288 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);  in arm_smmu_cmdq_build_cmd()
289 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);  in arm_smmu_cmdq_build_cmd()
290 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);  in arm_smmu_cmdq_build_cmd()
291 		cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);  in arm_smmu_cmdq_build_cmd()
292 		cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;  in arm_smmu_cmdq_build_cmd()
295 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);  in arm_smmu_cmdq_build_cmd()
298 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);  in arm_smmu_cmdq_build_cmd()
301 		cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);  in arm_smmu_cmdq_build_cmd()
304 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);  in arm_smmu_cmdq_build_cmd()
305 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);  in arm_smmu_cmdq_build_cmd()
306 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);  in arm_smmu_cmdq_build_cmd()
307 		cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);  in arm_smmu_cmdq_build_cmd()
308 		cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);  in arm_smmu_cmdq_build_cmd()
309 		cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;  in arm_smmu_cmdq_build_cmd()
312 		cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);  in arm_smmu_cmdq_build_cmd()
313 		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);  in arm_smmu_cmdq_build_cmd()
314 		cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);  in arm_smmu_cmdq_build_cmd()
315 		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);  in arm_smmu_cmdq_build_cmd()
316 		switch (ent->pri.resp) {  in arm_smmu_cmdq_build_cmd()
322 			return -EINVAL;  in arm_smmu_cmdq_build_cmd()
324 		cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);  in arm_smmu_cmdq_build_cmd()
327 		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);  in arm_smmu_cmdq_build_cmd()
328 		cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);  in arm_smmu_cmdq_build_cmd()
329 		cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);  in arm_smmu_cmdq_build_cmd()
332 		if (ent->sync.msiaddr) {  in arm_smmu_cmdq_build_cmd()
334 			cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;  in arm_smmu_cmdq_build_cmd()
342 		return -ENOENT;  in arm_smmu_cmdq_build_cmd()
348 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu)  in arm_smmu_get_cmdq()  argument
350 	return &smmu->cmdq;  in arm_smmu_get_cmdq()
353 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,  in arm_smmu_cmdq_build_sync_cmd()  argument
364 	if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {  in arm_smmu_cmdq_build_sync_cmd()
365 		ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *  in arm_smmu_cmdq_build_sync_cmd()
366 				   q->ent_dwords * 8;  in arm_smmu_cmdq_build_sync_cmd()
372 static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,  in __arm_smmu_cmdq_skip_err()  argument
384 	u32 cons = readl_relaxed(q->cons_reg);  in __arm_smmu_cmdq_skip_err()
390 	dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,  in __arm_smmu_cmdq_skip_err()
395 		dev_err(smmu->dev, "retrying command fetch\n");  in __arm_smmu_cmdq_skip_err()
416 	queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);  in __arm_smmu_cmdq_skip_err()
417 	dev_err(smmu->dev, "skipping command in error state:\n");  in __arm_smmu_cmdq_skip_err()
419 		dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);  in __arm_smmu_cmdq_skip_err()
424 	queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);  in __arm_smmu_cmdq_skip_err()
427 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)  in arm_smmu_cmdq_skip_err()  argument
429 	__arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q);  in arm_smmu_cmdq_skip_err()
436  * - The only LOCK routines are exclusive_trylock() and shared_lock().
440  * - The UNLOCK routines are supplemented with shared_tryunlock(), which
454 	if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)  in arm_smmu_cmdq_shared_lock()
458 		val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);  in arm_smmu_cmdq_shared_lock()
459 	} while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);  in arm_smmu_cmdq_shared_lock()
464 	(void)atomic_dec_return_release(&cmdq->lock);  in arm_smmu_cmdq_shared_unlock()
469 	if (atomic_read(&cmdq->lock) == 1)  in arm_smmu_cmdq_shared_tryunlock()
480 	__ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN);	\
488 	atomic_set_release(&cmdq->lock, 0);				\
497  * you like mixed-size concurrency, dependency ordering and relaxed atomics,
523  *		a. If we have MSIs, the SMMU can write back into the CMD_SYNC
537 		.max_n_shift	= cmdq->q.llq.max_n_shift,  in __arm_smmu_cmdq_poll_set_valid_map()
552 		ptr = &cmdq->valid_map[swidx];  in __arm_smmu_cmdq_poll_set_valid_map()
557 		mask = GENMASK(limit - 1, sbidx);  in __arm_smmu_cmdq_poll_set_valid_map()
561 		 * that a zero-initialised queue is invalid and, after marking  in __arm_smmu_cmdq_poll_set_valid_map()
574 		llq.prod = queue_inc_prod_n(&llq, limit - sbidx);  in __arm_smmu_cmdq_poll_set_valid_map()
592 /* Wait for the command queue to become non-full */
593 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,  in arm_smmu_cmdq_poll_until_not_full()  argument
598 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);  in arm_smmu_cmdq_poll_until_not_full()
606 		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));  in arm_smmu_cmdq_poll_until_not_full()
608 		llq->val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_poll_until_not_full()
612 	queue_poll_init(smmu, &qp);  in arm_smmu_cmdq_poll_until_not_full()
614 		llq->val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_poll_until_not_full()
625  * Wait until the SMMU signals a CMD_SYNC completion MSI.
628 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,  in __arm_smmu_cmdq_poll_until_msi()  argument
633 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);  in __arm_smmu_cmdq_poll_until_msi()
634 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));  in __arm_smmu_cmdq_poll_until_msi()
636 	queue_poll_init(smmu, &qp);  in __arm_smmu_cmdq_poll_until_msi()
644 	llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);  in __arm_smmu_cmdq_poll_until_msi()
649  * Wait until the SMMU cons index passes llq->prod.
652 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,  in __arm_smmu_cmdq_poll_until_consumed()  argument
656 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);  in __arm_smmu_cmdq_poll_until_consumed()
657 	u32 prod = llq->prod;  in __arm_smmu_cmdq_poll_until_consumed()
660 	queue_poll_init(smmu, &qp);  in __arm_smmu_cmdq_poll_until_consumed()
661 	llq->val = READ_ONCE(cmdq->q.llq.val);  in __arm_smmu_cmdq_poll_until_consumed()
676 		 * cmdq->q.llq.cons. Roughly speaking:  in __arm_smmu_cmdq_poll_until_consumed()
696 		llq->cons = readl(cmdq->q.cons_reg);  in __arm_smmu_cmdq_poll_until_consumed()
702 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,  in arm_smmu_cmdq_poll_until_sync()  argument
705 	if (smmu->options & ARM_SMMU_OPT_MSIPOLL)  in arm_smmu_cmdq_poll_until_sync()
706 		return __arm_smmu_cmdq_poll_until_msi(smmu, llq);  in arm_smmu_cmdq_poll_until_sync()
708 	return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);  in arm_smmu_cmdq_poll_until_sync()
716 		.max_n_shift	= cmdq->q.llq.max_n_shift,  in arm_smmu_cmdq_write_entries()
724 		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_write_entries()
732  * - There is a dma_wmb() before publishing any commands to the queue.
736  * - On completion of a CMD_SYNC, there is a control dependency.
740  * - Command insertion is totally ordered, so if two CPUs each race to
744 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,  in arm_smmu_cmdq_issue_cmdlist()  argument
751 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);  in arm_smmu_cmdq_issue_cmdlist()
755 	llq.max_n_shift = cmdq->q.llq.max_n_shift;  in arm_smmu_cmdq_issue_cmdlist()
759 	llq.val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_issue_cmdlist()
765 			if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))  in arm_smmu_cmdq_issue_cmdlist()
766 				dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");  in arm_smmu_cmdq_issue_cmdlist()
774 		old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);  in arm_smmu_cmdq_issue_cmdlist()
791 		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod);  in arm_smmu_cmdq_issue_cmdlist()
792 		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_issue_cmdlist()
807 	/* 4. If we are the owner, take control of the SMMU hardware */  in arm_smmu_cmdq_issue_cmdlist()
810 		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);  in arm_smmu_cmdq_issue_cmdlist()
814 						   &cmdq->q.llq.atomic.prod);  in arm_smmu_cmdq_issue_cmdlist()
828 		writel_relaxed(prod, cmdq->q.prod_reg);  in arm_smmu_cmdq_issue_cmdlist()
835 		atomic_set_release(&cmdq->owner_prod, prod);  in arm_smmu_cmdq_issue_cmdlist()
841 		ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);  in arm_smmu_cmdq_issue_cmdlist()
843 			dev_err_ratelimited(smmu->dev,  in arm_smmu_cmdq_issue_cmdlist()
846 					    readl_relaxed(cmdq->q.prod_reg),  in arm_smmu_cmdq_issue_cmdlist()
847 					    readl_relaxed(cmdq->q.cons_reg));  in arm_smmu_cmdq_issue_cmdlist()
852 		 * reader, in which case we can safely update cmdq->q.llq.cons  in arm_smmu_cmdq_issue_cmdlist()
855 			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);  in arm_smmu_cmdq_issue_cmdlist()
864 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,  in __arm_smmu_cmdq_issue_cmd()  argument
871 		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",  in __arm_smmu_cmdq_issue_cmd()
872 			 ent->opcode);  in __arm_smmu_cmdq_issue_cmd()
873 		return -EINVAL;  in __arm_smmu_cmdq_issue_cmd()
876 	return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, sync);  in __arm_smmu_cmdq_issue_cmd()
879 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,  in arm_smmu_cmdq_issue_cmd()  argument
882 	return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);  in arm_smmu_cmdq_issue_cmd()
885 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,  in arm_smmu_cmdq_issue_cmd_with_sync()  argument
888 	return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);  in arm_smmu_cmdq_issue_cmd_with_sync()
891 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,  in arm_smmu_cmdq_batch_add()  argument
897 	if (cmds->num == CMDQ_BATCH_ENTRIES - 1 &&  in arm_smmu_cmdq_batch_add()
898 	    (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC)) {  in arm_smmu_cmdq_batch_add()
899 		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);  in arm_smmu_cmdq_batch_add()
900 		cmds->num = 0;  in arm_smmu_cmdq_batch_add()
903 	if (cmds->num == CMDQ_BATCH_ENTRIES) {  in arm_smmu_cmdq_batch_add()
904 		arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false);  in arm_smmu_cmdq_batch_add()
905 		cmds->num = 0;  in arm_smmu_cmdq_batch_add()
908 	index = cmds->num * CMDQ_ENT_DWORDS;  in arm_smmu_cmdq_batch_add()
909 	if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {  in arm_smmu_cmdq_batch_add()
910 		dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",  in arm_smmu_cmdq_batch_add()
911 			 cmd->opcode);  in arm_smmu_cmdq_batch_add()
915 	cmds->num++;  in arm_smmu_cmdq_batch_add()
918 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,  in arm_smmu_cmdq_batch_submit()  argument
921 	return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);  in arm_smmu_cmdq_batch_submit()
930 	int sid = master->streams[0].id;  in arm_smmu_page_response()
932 	if (master->stall_enabled) {  in arm_smmu_page_response()
935 		cmd.resume.stag		= resp->grpid;  in arm_smmu_page_response()
936 		switch (resp->code) {  in arm_smmu_page_response()
945 			return -EINVAL;  in arm_smmu_page_response()
948 		return -ENODEV;  in arm_smmu_page_response()
951 	arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);  in arm_smmu_page_response()
963 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)  in arm_smmu_tlb_inv_asid()  argument
966 		.opcode	= smmu->features & ARM_SMMU_FEAT_E2H ?  in arm_smmu_tlb_inv_asid()
971 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);  in arm_smmu_tlb_inv_asid()
981 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_sync_cd()  local
992 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);  in arm_smmu_sync_cd()
993 	list_for_each_entry(master, &smmu_domain->devices, domain_head) {  in arm_smmu_sync_cd()
994 		for (i = 0; i < master->num_streams; i++) {  in arm_smmu_sync_cd()
995 			cmd.cfgi.sid = master->streams[i].id;  in arm_smmu_sync_cd()
996 			arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);  in arm_smmu_sync_cd()
999 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);  in arm_smmu_sync_cd()
1001 	arm_smmu_cmdq_batch_submit(smmu, &cmds);  in arm_smmu_sync_cd()
1004 static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,  in arm_smmu_alloc_cd_leaf_table()  argument
1009 	l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,  in arm_smmu_alloc_cd_leaf_table()
1010 					     &l1_desc->l2ptr_dma, GFP_KERNEL);  in arm_smmu_alloc_cd_leaf_table()
1011 	if (!l1_desc->l2ptr) {  in arm_smmu_alloc_cd_leaf_table()
1012 		dev_warn(smmu->dev,  in arm_smmu_alloc_cd_leaf_table()
1014 		return -ENOMEM;  in arm_smmu_alloc_cd_leaf_table()
1022 	u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) |  in arm_smmu_write_cd_l1_desc()
1035 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_get_cd_ptr()  local
1036 	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;  in arm_smmu_get_cd_ptr()
1038 	if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)  in arm_smmu_get_cd_ptr()
1039 		return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;  in arm_smmu_get_cd_ptr()
1042 	l1_desc = &cdcfg->l1_desc[idx];  in arm_smmu_get_cd_ptr()
1043 	if (!l1_desc->l2ptr) {  in arm_smmu_get_cd_ptr()
1044 		if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))  in arm_smmu_get_cd_ptr()
1047 		l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;  in arm_smmu_get_cd_ptr()
1052 	idx = ssid & (CTXDESC_L2_ENTRIES - 1);  in arm_smmu_get_cd_ptr()
1053 	return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;  in arm_smmu_get_cd_ptr()
1074 	if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))  in arm_smmu_write_ctx_desc()
1075 		return -E2BIG;  in arm_smmu_write_ctx_desc()
1079 		return -ENOMEM;  in arm_smmu_write_ctx_desc()
1090 		val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);  in arm_smmu_write_ctx_desc()
1096 		cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);  in arm_smmu_write_ctx_desc()
1098 		cdptr[3] = cpu_to_le64(cd->mair);  in arm_smmu_write_ctx_desc()
1101 		 * STE is live, and the SMMU might read dwords of this CD in any  in arm_smmu_write_ctx_desc()
1107 		val = cd->tcr |  in arm_smmu_write_ctx_desc()
1112 			(cd->mm ? 0 : CTXDESC_CD_0_ASET) |  in arm_smmu_write_ctx_desc()
1114 			FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |  in arm_smmu_write_ctx_desc()
1117 		if (smmu_domain->stall_enabled)  in arm_smmu_write_ctx_desc()
1122 	 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3  in arm_smmu_write_ctx_desc()
1125 	 *   The size of single-copy atomic reads made by the SMMU is  in arm_smmu_write_ctx_desc()
1127 	 *   field within an aligned 64-bit span of a structure can be altered  in arm_smmu_write_ctx_desc()
1140 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_alloc_cd_tables()  local
1141 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;  in arm_smmu_alloc_cd_tables()
1142 	struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;  in arm_smmu_alloc_cd_tables()
1144 	max_contexts = 1 << cfg->s1cdmax;  in arm_smmu_alloc_cd_tables()
1146 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||  in arm_smmu_alloc_cd_tables()
1148 		cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;  in arm_smmu_alloc_cd_tables()
1149 		cdcfg->num_l1_ents = max_contexts;  in arm_smmu_alloc_cd_tables()
1153 		cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;  in arm_smmu_alloc_cd_tables()
1154 		cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,  in arm_smmu_alloc_cd_tables()
1157 		cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,  in arm_smmu_alloc_cd_tables()
1158 					      sizeof(*cdcfg->l1_desc),  in arm_smmu_alloc_cd_tables()
1160 		if (!cdcfg->l1_desc)  in arm_smmu_alloc_cd_tables()
1161 			return -ENOMEM;  in arm_smmu_alloc_cd_tables()
1163 		l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);  in arm_smmu_alloc_cd_tables()
1166 	cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,  in arm_smmu_alloc_cd_tables()
1168 	if (!cdcfg->cdtab) {  in arm_smmu_alloc_cd_tables()
1169 		dev_warn(smmu->dev, "failed to allocate context descriptor\n");  in arm_smmu_alloc_cd_tables()
1170 		ret = -ENOMEM;  in arm_smmu_alloc_cd_tables()
1177 	if (cdcfg->l1_desc) {  in arm_smmu_alloc_cd_tables()
1178 		devm_kfree(smmu->dev, cdcfg->l1_desc);  in arm_smmu_alloc_cd_tables()
1179 		cdcfg->l1_desc = NULL;  in arm_smmu_alloc_cd_tables()
1188 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_free_cd_tables()  local
1189 	struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;  in arm_smmu_free_cd_tables()
1191 	if (cdcfg->l1_desc) {  in arm_smmu_free_cd_tables()
1194 		for (i = 0; i < cdcfg->num_l1_ents; i++) {  in arm_smmu_free_cd_tables()
1195 			if (!cdcfg->l1_desc[i].l2ptr)  in arm_smmu_free_cd_tables()
1198 			dmam_free_coherent(smmu->dev, size,  in arm_smmu_free_cd_tables()
1199 					   cdcfg->l1_desc[i].l2ptr,  in arm_smmu_free_cd_tables()
1200 					   cdcfg->l1_desc[i].l2ptr_dma);  in arm_smmu_free_cd_tables()
1202 		devm_kfree(smmu->dev, cdcfg->l1_desc);  in arm_smmu_free_cd_tables()
1203 		cdcfg->l1_desc = NULL;  in arm_smmu_free_cd_tables()
1205 		l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);  in arm_smmu_free_cd_tables()
1207 		l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);  in arm_smmu_free_cd_tables()
1210 	dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);  in arm_smmu_free_cd_tables()
1211 	cdcfg->cdtab_dma = 0;  in arm_smmu_free_cd_tables()
1212 	cdcfg->cdtab = NULL;  in arm_smmu_free_cd_tables()
1220 	if (!cd->asid)  in arm_smmu_free_asid()
1223 	free = refcount_dec_and_test(&cd->refs);  in arm_smmu_free_asid()
1225 		old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid);  in arm_smmu_free_asid()
1237 	val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);  in arm_smmu_write_strtab_l1_desc()
1238 	val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;  in arm_smmu_write_strtab_l1_desc()
1244 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)  in arm_smmu_sync_ste_for_sid()  argument
1254 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);  in arm_smmu_sync_ste_for_sid()
1264 	 * 1. Invalid (all zero) -> bypass/fault (init)  in arm_smmu_write_strtab_ent()
1265 	 * 2. Bypass/fault -> translation/bypass (attach)  in arm_smmu_write_strtab_ent()
1266 	 * 3. Translation/bypass -> bypass/fault (detach)  in arm_smmu_write_strtab_ent()
1268 	 * Given that we can't update the STE atomically and the SMMU  in arm_smmu_write_strtab_ent()
1278 	struct arm_smmu_device *smmu = NULL;  in arm_smmu_write_strtab_ent()  local
1290 		smmu_domain = master->domain;  in arm_smmu_write_strtab_ent()
1291 		smmu = master->smmu;  in arm_smmu_write_strtab_ent()
1295 		switch (smmu_domain->stage) {  in arm_smmu_write_strtab_ent()
1297 			s1_cfg = &smmu_domain->s1_cfg;  in arm_smmu_write_strtab_ent()
1301 			s2_cfg = &smmu_domain->s2_cfg;  in arm_smmu_write_strtab_ent()
1339 		 * The SMMU can perform negative caching, so we must sync  in arm_smmu_write_strtab_ent()
1342 		if (smmu)  in arm_smmu_write_strtab_ent()
1343 			arm_smmu_sync_ste_for_sid(smmu, sid);  in arm_smmu_write_strtab_ent()
1348 		u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ?  in arm_smmu_write_strtab_ent()
1359 		if (smmu->features & ARM_SMMU_FEAT_STALLS &&  in arm_smmu_write_strtab_ent()
1360 		    !master->stall_enabled)  in arm_smmu_write_strtab_ent()
1363 		val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |  in arm_smmu_write_strtab_ent()
1365 			FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) |  in arm_smmu_write_strtab_ent()
1366 			FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);  in arm_smmu_write_strtab_ent()
1372 			 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |  in arm_smmu_write_strtab_ent()
1373 			 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |  in arm_smmu_write_strtab_ent()
1380 		dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);  in arm_smmu_write_strtab_ent()
1385 	if (master->ats_enabled)  in arm_smmu_write_strtab_ent()
1389 	arm_smmu_sync_ste_for_sid(smmu, sid);  in arm_smmu_write_strtab_ent()
1392 	arm_smmu_sync_ste_for_sid(smmu, sid);  in arm_smmu_write_strtab_ent()
1395 	if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))  in arm_smmu_write_strtab_ent()
1396 		arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);  in arm_smmu_write_strtab_ent()
1418 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)  in arm_smmu_init_l2_strtab()  argument
1422 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;  in arm_smmu_init_l2_strtab()
1423 	struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];  in arm_smmu_init_l2_strtab()
1425 	if (desc->l2ptr)  in arm_smmu_init_l2_strtab()
1429 	strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];  in arm_smmu_init_l2_strtab()
1431 	desc->span = STRTAB_SPLIT + 1;  in arm_smmu_init_l2_strtab()
1432 	desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,  in arm_smmu_init_l2_strtab()
1434 	if (!desc->l2ptr) {  in arm_smmu_init_l2_strtab()
1435 		dev_err(smmu->dev,  in arm_smmu_init_l2_strtab()
1438 		return -ENOMEM;  in arm_smmu_init_l2_strtab()
1441 	arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false);  in arm_smmu_init_l2_strtab()
1447 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)  in arm_smmu_find_master()  argument
1452 	lockdep_assert_held(&smmu->streams_mutex);  in arm_smmu_find_master()
1454 	node = smmu->streams.rb_node;  in arm_smmu_find_master()
1457 		if (stream->id < sid)  in arm_smmu_find_master()
1458 			node = node->rb_right;  in arm_smmu_find_master()
1459 		else if (stream->id > sid)  in arm_smmu_find_master()
1460 			node = node->rb_left;  in arm_smmu_find_master()
1462 			return stream->master;  in arm_smmu_find_master()
1469 static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)  in arm_smmu_handle_evt()  argument
1494 		return -EOPNOTSUPP;  in arm_smmu_handle_evt()
1497 	/* Stage-2 is always pinned at the moment */  in arm_smmu_handle_evt()
1499 		return -EFAULT;  in arm_smmu_handle_evt()
1513 		flt->type = IOMMU_FAULT_PAGE_REQ;  in arm_smmu_handle_evt()
1514 		flt->prm = (struct iommu_fault_page_request) {  in arm_smmu_handle_evt()
1522 			flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;  in arm_smmu_handle_evt()
1523 			flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);  in arm_smmu_handle_evt()
1526 		flt->type = IOMMU_FAULT_DMA_UNRECOV;  in arm_smmu_handle_evt()
1527 		flt->event = (struct iommu_fault_unrecoverable) {  in arm_smmu_handle_evt()
1535 			flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID;  in arm_smmu_handle_evt()
1536 			flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);  in arm_smmu_handle_evt()
1540 	mutex_lock(&smmu->streams_mutex);  in arm_smmu_handle_evt()
1541 	master = arm_smmu_find_master(smmu, sid);  in arm_smmu_handle_evt()
1543 		ret = -EINVAL;  in arm_smmu_handle_evt()
1547 	ret = iommu_report_device_fault(master->dev, &fault_evt);  in arm_smmu_handle_evt()
1548 	if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) {  in arm_smmu_handle_evt()
1551 			.pasid		= flt->prm.pasid,  in arm_smmu_handle_evt()
1552 			.grpid		= flt->prm.grpid,  in arm_smmu_handle_evt()
1555 		arm_smmu_page_response(master->dev, &fault_evt, &resp);  in arm_smmu_handle_evt()
1559 	mutex_unlock(&smmu->streams_mutex);  in arm_smmu_handle_evt()
1566 	struct arm_smmu_device *smmu = dev;  in arm_smmu_evtq_thread()  local
1567 	struct arm_smmu_queue *q = &smmu->evtq.q;  in arm_smmu_evtq_thread()
1568 	struct arm_smmu_ll_queue *llq = &q->llq;  in arm_smmu_evtq_thread()
1577 			ret = arm_smmu_handle_evt(smmu, evt);  in arm_smmu_evtq_thread()
1581 			dev_info(smmu->dev, "event 0x%02x received:\n", id);  in arm_smmu_evtq_thread()
1583 				dev_info(smmu->dev, "\t0x%016llx\n",  in arm_smmu_evtq_thread()
1593 		if (queue_sync_prod_in(q) == -EOVERFLOW)  in arm_smmu_evtq_thread()
1594 			dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");  in arm_smmu_evtq_thread()
1602 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)  in arm_smmu_handle_ppr()  argument
1614 	dev_info(smmu->dev, "unexpected PRI request received:\n");  in arm_smmu_handle_ppr()
1615 	dev_info(smmu->dev,  in arm_smmu_handle_ppr()
1636 		arm_smmu_cmdq_issue_cmd(smmu, &cmd);  in arm_smmu_handle_ppr()
1642 	struct arm_smmu_device *smmu = dev;  in arm_smmu_priq_thread()  local
1643 	struct arm_smmu_queue *q = &smmu->priq.q;  in arm_smmu_priq_thread()
1644 	struct arm_smmu_ll_queue *llq = &q->llq;  in arm_smmu_priq_thread()
1649 			arm_smmu_handle_ppr(smmu, evt);  in arm_smmu_priq_thread()
1651 		if (queue_sync_prod_in(q) == -EOVERFLOW)  in arm_smmu_priq_thread()
1652 			dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");  in arm_smmu_priq_thread()
1660 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1665 	struct arm_smmu_device *smmu = dev;  in arm_smmu_gerror_handler()  local
1667 	gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);  in arm_smmu_gerror_handler()
1668 	gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);  in arm_smmu_gerror_handler()
1674 	dev_warn(smmu->dev,  in arm_smmu_gerror_handler()
1679 		dev_err(smmu->dev, "device has entered Service Failure Mode!\n");  in arm_smmu_gerror_handler()
1680 		arm_smmu_device_disable(smmu);  in arm_smmu_gerror_handler()
1684 		dev_warn(smmu->dev, "GERROR MSI write aborted\n");  in arm_smmu_gerror_handler()
1687 		dev_warn(smmu->dev, "PRIQ MSI write aborted\n");  in arm_smmu_gerror_handler()
1690 		dev_warn(smmu->dev, "EVTQ MSI write aborted\n");  in arm_smmu_gerror_handler()
1693 		dev_warn(smmu->dev, "CMDQ MSI write aborted\n");  in arm_smmu_gerror_handler()
1696 		dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");  in arm_smmu_gerror_handler()
1699 		dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");  in arm_smmu_gerror_handler()
1702 		arm_smmu_cmdq_skip_err(smmu);  in arm_smmu_gerror_handler()
1704 	writel(gerror, smmu->base + ARM_SMMU_GERRORN);  in arm_smmu_gerror_handler()
1710 	struct arm_smmu_device *smmu = dev;  in arm_smmu_combined_irq_thread()  local
1713 	if (smmu->features & ARM_SMMU_FEAT_PRI)  in arm_smmu_combined_irq_thread()
1731 	/* ATC invalidates are always on 4096-bytes pages */  in arm_smmu_atc_inv_to_cmd()
1743 	 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID  in arm_smmu_atc_inv_to_cmd()
1746 	 * This has the unpleasant side-effect of invalidating all PASID-tagged  in arm_smmu_atc_inv_to_cmd()
1756 		cmd->atc.size = ATC_INV_SIZE_ALL;  in arm_smmu_atc_inv_to_cmd()
1761 	page_end	= (iova + size - 1) >> inval_grain_shift;  in arm_smmu_atc_inv_to_cmd()
1766 	 * thus have to choose between grossly over-invalidating the region, or  in arm_smmu_atc_inv_to_cmd()
1784 	span_mask	= (1ULL << log2_span) - 1;  in arm_smmu_atc_inv_to_cmd()
1788 	cmd->atc.addr	= page_start << inval_grain_shift;  in arm_smmu_atc_inv_to_cmd()
1789 	cmd->atc.size	= log2_span;  in arm_smmu_atc_inv_to_cmd()
1801 	for (i = 0; i < master->num_streams; i++) {  in arm_smmu_atc_inv_master()
1802 		cmd.atc.sid = master->streams[i].id;  in arm_smmu_atc_inv_master()
1803 		arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);  in arm_smmu_atc_inv_master()
1806 	return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);  in arm_smmu_atc_inv_master()
1818 	if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))  in arm_smmu_atc_inv_domain()
1835 	if (!atomic_read(&smmu_domain->nr_ats_masters))  in arm_smmu_atc_inv_domain()
1842 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);  in arm_smmu_atc_inv_domain()
1843 	list_for_each_entry(master, &smmu_domain->devices, domain_head) {  in arm_smmu_atc_inv_domain()
1844 		if (!master->ats_enabled)  in arm_smmu_atc_inv_domain()
1847 		for (i = 0; i < master->num_streams; i++) {  in arm_smmu_atc_inv_domain()
1848 			cmd.atc.sid = master->streams[i].id;  in arm_smmu_atc_inv_domain()
1849 			arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);  in arm_smmu_atc_inv_domain()
1852 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);  in arm_smmu_atc_inv_domain()
1854 	return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);  in arm_smmu_atc_inv_domain()
1861 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_context()  local
1865 	 * NOTE: when io-pgtable is in non-strict mode, we may get here with  in arm_smmu_tlb_inv_context()
1867 	 * to the SMMU. We are relying on the dma_wmb() implicit during cmd  in arm_smmu_tlb_inv_context()
1871 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_tlb_inv_context()
1872 		arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);  in arm_smmu_tlb_inv_context()
1875 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;  in arm_smmu_tlb_inv_context()
1876 		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);  in arm_smmu_tlb_inv_context()
1886 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in __arm_smmu_tlb_inv_range()  local
1894 	if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {  in __arm_smmu_tlb_inv_range()
1896 		tg = __ffs(smmu_domain->domain.pgsize_bitmap);  in __arm_smmu_tlb_inv_range()
1901 		cmd->tlbi.tg = (tg - 10) / 2;  in __arm_smmu_tlb_inv_range()
1904 		 * Determine what level the granule is at. For non-leaf, both  in __arm_smmu_tlb_inv_range()
1905 		 * io-pgtable and SVA pass a nominal last-level granule because  in __arm_smmu_tlb_inv_range()
1911 		if (cmd->tlbi.leaf)  in __arm_smmu_tlb_inv_range()
1912 			cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));  in __arm_smmu_tlb_inv_range()
1920 		if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {  in __arm_smmu_tlb_inv_range()
1932 			cmd->tlbi.scale = scale;  in __arm_smmu_tlb_inv_range()
1936 			cmd->tlbi.num = num - 1;  in __arm_smmu_tlb_inv_range()
1942 			num_pages -= num << scale;  in __arm_smmu_tlb_inv_range()
1945 		cmd->tlbi.addr = iova;  in __arm_smmu_tlb_inv_range()
1946 		arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);  in __arm_smmu_tlb_inv_range()
1949 	arm_smmu_cmdq_batch_submit(smmu, &cmds);  in __arm_smmu_tlb_inv_range()
1962 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_tlb_inv_range_domain()
1963 		cmd.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?  in arm_smmu_tlb_inv_range_domain()
1965 		cmd.tlbi.asid	= smmu_domain->s1_cfg.cd.asid;  in arm_smmu_tlb_inv_range_domain()
1968 		cmd.tlbi.vmid	= smmu_domain->s2_cfg.vmid;  in arm_smmu_tlb_inv_range_domain()
1973 	 * Unfortunately, this can't be leaf-only since we may have  in arm_smmu_tlb_inv_range_domain()
1984 		.opcode	= smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?  in arm_smmu_tlb_inv_range_asid()
2000 	struct iommu_domain *domain = &smmu_domain->domain;  in arm_smmu_tlb_inv_page_nosync()
2025 		return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;  in arm_smmu_capable()
2055 	mutex_init(&smmu_domain->init_mutex);  in arm_smmu_domain_alloc()
2056 	INIT_LIST_HEAD(&smmu_domain->devices);  in arm_smmu_domain_alloc()
2057 	spin_lock_init(&smmu_domain->devices_lock);  in arm_smmu_domain_alloc()
2058 	INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);  in arm_smmu_domain_alloc()
2060 	return &smmu_domain->domain;  in arm_smmu_domain_alloc()
2066 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_free()  local
2068 	free_io_pgtable_ops(smmu_domain->pgtbl_ops);  in arm_smmu_domain_free()
2071 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {  in arm_smmu_domain_free()
2072 		struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;  in arm_smmu_domain_free()
2076 		if (cfg->cdcfg.cdtab)  in arm_smmu_domain_free()
2078 		arm_smmu_free_asid(&cfg->cd);  in arm_smmu_domain_free()
2081 		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;  in arm_smmu_domain_free()
2082 		if (cfg->vmid)  in arm_smmu_domain_free()
2083 			ida_free(&smmu->vmid_map, cfg->vmid);  in arm_smmu_domain_free()
2095 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise_s1()  local
2096 	struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;  in arm_smmu_domain_finalise_s1()
2097 	typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;  in arm_smmu_domain_finalise_s1()
2099 	refcount_set(&cfg->cd.refs, 1);  in arm_smmu_domain_finalise_s1()
2103 	ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd,  in arm_smmu_domain_finalise_s1()
2104 		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);  in arm_smmu_domain_finalise_s1()
2108 	cfg->s1cdmax = master->ssid_bits;  in arm_smmu_domain_finalise_s1()
2110 	smmu_domain->stall_enabled = master->stall_enabled;  in arm_smmu_domain_finalise_s1()
2116 	cfg->cd.asid	= (u16)asid;  in arm_smmu_domain_finalise_s1()
2117 	cfg->cd.ttbr	= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;  in arm_smmu_domain_finalise_s1()
2118 	cfg->cd.tcr	= FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |  in arm_smmu_domain_finalise_s1()
2119 			  FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |  in arm_smmu_domain_finalise_s1()
2120 			  FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |  in arm_smmu_domain_finalise_s1()
2121 			  FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |  in arm_smmu_domain_finalise_s1()
2122 			  FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |  in arm_smmu_domain_finalise_s1()
2123 			  FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |  in arm_smmu_domain_finalise_s1()
2125 	cfg->cd.mair	= pgtbl_cfg->arm_lpae_s1_cfg.mair;  in arm_smmu_domain_finalise_s1()
2132 	ret = arm_smmu_write_ctx_desc(smmu_domain, IOMMU_NO_PASID, &cfg->cd);  in arm_smmu_domain_finalise_s1()
2142 	arm_smmu_free_asid(&cfg->cd);  in arm_smmu_domain_finalise_s1()
2153 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise_s2()  local
2154 	struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;  in arm_smmu_domain_finalise_s2()
2155 	typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;  in arm_smmu_domain_finalise_s2()
2157 	/* Reserve VMID 0 for stage-2 bypass STEs */  in arm_smmu_domain_finalise_s2()
2158 	vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,  in arm_smmu_domain_finalise_s2()
2163 	vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;  in arm_smmu_domain_finalise_s2()
2164 	cfg->vmid	= (u16)vmid;  in arm_smmu_domain_finalise_s2()
2165 	cfg->vttbr	= pgtbl_cfg->arm_lpae_s2_cfg.vttbr;  in arm_smmu_domain_finalise_s2()
2166 	cfg->vtcr	= FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |  in arm_smmu_domain_finalise_s2()
2167 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |  in arm_smmu_domain_finalise_s2()
2168 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |  in arm_smmu_domain_finalise_s2()
2169 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |  in arm_smmu_domain_finalise_s2()
2170 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |  in arm_smmu_domain_finalise_s2()
2171 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |  in arm_smmu_domain_finalise_s2()
2172 			  FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);  in arm_smmu_domain_finalise_s2()
2188 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_finalise()  local
2190 	if (domain->type == IOMMU_DOMAIN_IDENTITY) {  in arm_smmu_domain_finalise()
2191 		smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;  in arm_smmu_domain_finalise()
2196 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))  in arm_smmu_domain_finalise()
2197 		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;  in arm_smmu_domain_finalise()
2198 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))  in arm_smmu_domain_finalise()
2199 		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;  in arm_smmu_domain_finalise()
2201 	switch (smmu_domain->stage) {  in arm_smmu_domain_finalise()
2203 		ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;  in arm_smmu_domain_finalise()
2205 		oas = smmu->ias;  in arm_smmu_domain_finalise()
2211 		ias = smmu->ias;  in arm_smmu_domain_finalise()
2212 		oas = smmu->oas;  in arm_smmu_domain_finalise()
2217 		return -EINVAL;  in arm_smmu_domain_finalise()
2221 		.pgsize_bitmap	= smmu->pgsize_bitmap,  in arm_smmu_domain_finalise()
2224 		.coherent_walk	= smmu->features & ARM_SMMU_FEAT_COHERENCY,  in arm_smmu_domain_finalise()
2226 		.iommu_dev	= smmu->dev,  in arm_smmu_domain_finalise()
2231 		return -ENOMEM;  in arm_smmu_domain_finalise()
2233 	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;  in arm_smmu_domain_finalise()
2234 	domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;  in arm_smmu_domain_finalise()
2235 	domain->geometry.force_aperture = true;  in arm_smmu_domain_finalise()
2243 	smmu_domain->pgtbl_ops = pgtbl_ops;  in arm_smmu_domain_finalise()
2247 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)  in arm_smmu_get_step_for_sid()  argument
2250 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;  in arm_smmu_get_step_for_sid()
2252 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {  in arm_smmu_get_step_for_sid()
2256 		/* Two-level walk */  in arm_smmu_get_step_for_sid()
2258 		l1_desc = &cfg->l1_desc[idx];  in arm_smmu_get_step_for_sid()
2259 		idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;  in arm_smmu_get_step_for_sid()
2260 		step = &l1_desc->l2ptr[idx];  in arm_smmu_get_step_for_sid()
2263 		step = &cfg->strtab[sid * STRTAB_STE_DWORDS];  in arm_smmu_get_step_for_sid()
2272 	struct arm_smmu_device *smmu = master->smmu;  in arm_smmu_install_ste_for_dev()  local
2274 	for (i = 0; i < master->num_streams; ++i) {  in arm_smmu_install_ste_for_dev()
2275 		u32 sid = master->streams[i].id;  in arm_smmu_install_ste_for_dev()
2276 		__le64 *step = arm_smmu_get_step_for_sid(smmu, sid);  in arm_smmu_install_ste_for_dev()
2280 			if (master->streams[j].id == sid)  in arm_smmu_install_ste_for_dev()
2291 	struct device *dev = master->dev;  in arm_smmu_ats_supported()
2292 	struct arm_smmu_device *smmu = master->smmu;  in arm_smmu_ats_supported()  local
2295 	if (!(smmu->features & ARM_SMMU_FEAT_ATS))  in arm_smmu_ats_supported()
2298 	if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))  in arm_smmu_ats_supported()
2308 	struct arm_smmu_device *smmu = master->smmu;  in arm_smmu_enable_ats()  local
2309 	struct arm_smmu_domain *smmu_domain = master->domain;  in arm_smmu_enable_ats()
2312 	if (!master->ats_enabled)  in arm_smmu_enable_ats()
2316 	stu = __ffs(smmu->pgsize_bitmap);  in arm_smmu_enable_ats()
2317 	pdev = to_pci_dev(master->dev);  in arm_smmu_enable_ats()
2319 	atomic_inc(&smmu_domain->nr_ats_masters);  in arm_smmu_enable_ats()
2322 		dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);  in arm_smmu_enable_ats()
2327 	struct arm_smmu_domain *smmu_domain = master->domain;  in arm_smmu_disable_ats()
2329 	if (!master->ats_enabled)  in arm_smmu_disable_ats()
2332 	pci_disable_ats(to_pci_dev(master->dev));  in arm_smmu_disable_ats()
2335 	 * ATC invalidation via the SMMU.  in arm_smmu_disable_ats()
2339 	atomic_dec(&smmu_domain->nr_ats_masters);  in arm_smmu_disable_ats()
2349 	if (!dev_is_pci(master->dev))  in arm_smmu_enable_pasid()
2350 		return -ENODEV;  in arm_smmu_enable_pasid()
2352 	pdev = to_pci_dev(master->dev);  in arm_smmu_enable_pasid()
2364 		dev_err(&pdev->dev, "Failed to enable PASID\n");  in arm_smmu_enable_pasid()
2368 	master->ssid_bits = min_t(u8, ilog2(num_pasids),  in arm_smmu_enable_pasid()
2369 				  master->smmu->ssid_bits);  in arm_smmu_enable_pasid()
2377 	if (!dev_is_pci(master->dev))  in arm_smmu_disable_pasid()
2380 	pdev = to_pci_dev(master->dev);  in arm_smmu_disable_pasid()
2382 	if (!pdev->pasid_enabled)  in arm_smmu_disable_pasid()
2385 	master->ssid_bits = 0;  in arm_smmu_disable_pasid()
2392 	struct arm_smmu_domain *smmu_domain = master->domain;  in arm_smmu_detach_dev()
2399 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);  in arm_smmu_detach_dev()
2400 	list_del(&master->domain_head);  in arm_smmu_detach_dev()
2401 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);  in arm_smmu_detach_dev()
2403 	master->domain = NULL;  in arm_smmu_detach_dev()
2404 	master->ats_enabled = false;  in arm_smmu_detach_dev()
2413 	struct arm_smmu_device *smmu;  in arm_smmu_attach_dev()  local
2418 		return -ENOENT;  in arm_smmu_attach_dev()
2421 	smmu = master->smmu;  in arm_smmu_attach_dev()
2429 		dev_err(dev, "cannot attach - SVA enabled\n");  in arm_smmu_attach_dev()
2430 		return -EBUSY;  in arm_smmu_attach_dev()
2435 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_attach_dev()
2437 	if (!smmu_domain->smmu) {  in arm_smmu_attach_dev()
2438 		smmu_domain->smmu = smmu;  in arm_smmu_attach_dev()
2441 			smmu_domain->smmu = NULL;  in arm_smmu_attach_dev()
2444 	} else if (smmu_domain->smmu != smmu) {  in arm_smmu_attach_dev()
2445 		ret = -EINVAL;  in arm_smmu_attach_dev()
2447 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&  in arm_smmu_attach_dev()
2448 		   master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {  in arm_smmu_attach_dev()
2449 		ret = -EINVAL;  in arm_smmu_attach_dev()
2451 	} else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&  in arm_smmu_attach_dev()
2452 		   smmu_domain->stall_enabled != master->stall_enabled) {  in arm_smmu_attach_dev()
2453 		ret = -EINVAL;  in arm_smmu_attach_dev()
2457 	master->domain = smmu_domain;  in arm_smmu_attach_dev()
2460 	 * The SMMU does not support enabling ATS with bypass. When the STE is  in arm_smmu_attach_dev()
2466 	if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)  in arm_smmu_attach_dev()
2467 		master->ats_enabled = arm_smmu_ats_supported(master);  in arm_smmu_attach_dev()
2471 	spin_lock_irqsave(&smmu_domain->devices_lock, flags);  in arm_smmu_attach_dev()
2472 	list_add(&master->domain_head, &smmu_domain->devices);  in arm_smmu_attach_dev()
2473 	spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);  in arm_smmu_attach_dev()
2478 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_attach_dev()
2486 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;  in arm_smmu_map_pages()
2489 		return -ENODEV;  in arm_smmu_map_pages()
2491 	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);  in arm_smmu_map_pages()
2499 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;  in arm_smmu_unmap_pages()
2504 	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);  in arm_smmu_unmap_pages()
2511 	if (smmu_domain->smmu)  in arm_smmu_flush_iotlb_all()
2520 	if (!gather->pgsize)  in arm_smmu_iotlb_sync()
2523 	arm_smmu_tlb_inv_range_domain(gather->start,  in arm_smmu_iotlb_sync()
2524 				      gather->end - gather->start + 1,  in arm_smmu_iotlb_sync()
2525 				      gather->pgsize, true, smmu_domain);  in arm_smmu_iotlb_sync()
2531 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;  in arm_smmu_iova_to_phys()
2536 	return ops->iova_to_phys(ops, iova);  in arm_smmu_iova_to_phys()
2550 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)  in arm_smmu_sid_in_range()  argument
2552 	unsigned long limit = smmu->strtab_cfg.num_l1_ents;  in arm_smmu_sid_in_range()
2554 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)  in arm_smmu_sid_in_range()
2560 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)  in arm_smmu_init_sid_strtab()  argument
2562 	/* Check the SIDs are in range of the SMMU and our stream table */  in arm_smmu_init_sid_strtab()
2563 	if (!arm_smmu_sid_in_range(smmu, sid))  in arm_smmu_init_sid_strtab()
2564 		return -ERANGE;  in arm_smmu_init_sid_strtab()
2567 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)  in arm_smmu_init_sid_strtab()
2568 		return arm_smmu_init_l2_strtab(smmu, sid);  in arm_smmu_init_sid_strtab()
2573 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,  in arm_smmu_insert_master()  argument
2580 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);  in arm_smmu_insert_master()
2582 	master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),  in arm_smmu_insert_master()
2584 	if (!master->streams)  in arm_smmu_insert_master()
2585 		return -ENOMEM;  in arm_smmu_insert_master()
2586 	master->num_streams = fwspec->num_ids;  in arm_smmu_insert_master()
2588 	mutex_lock(&smmu->streams_mutex);  in arm_smmu_insert_master()
2589 	for (i = 0; i < fwspec->num_ids; i++) {  in arm_smmu_insert_master()
2590 		u32 sid = fwspec->ids[i];  in arm_smmu_insert_master()
2592 		new_stream = &master->streams[i];  in arm_smmu_insert_master()
2593 		new_stream->id = sid;  in arm_smmu_insert_master()
2594 		new_stream->master = master;  in arm_smmu_insert_master()
2596 		ret = arm_smmu_init_sid_strtab(smmu, sid);  in arm_smmu_insert_master()
2601 		new_node = &(smmu->streams.rb_node);  in arm_smmu_insert_master()
2606 			if (cur_stream->id > new_stream->id) {  in arm_smmu_insert_master()
2607 				new_node = &((*new_node)->rb_left);  in arm_smmu_insert_master()
2608 			} else if (cur_stream->id < new_stream->id) {  in arm_smmu_insert_master()
2609 				new_node = &((*new_node)->rb_right);  in arm_smmu_insert_master()
2611 				dev_warn(master->dev,  in arm_smmu_insert_master()
2613 					 cur_stream->id);  in arm_smmu_insert_master()
2614 				ret = -EINVAL;  in arm_smmu_insert_master()
2621 		rb_link_node(&new_stream->node, parent_node, new_node);  in arm_smmu_insert_master()
2622 		rb_insert_color(&new_stream->node, &smmu->streams);  in arm_smmu_insert_master()
2626 		for (i--; i >= 0; i--)  in arm_smmu_insert_master()
2627 			rb_erase(&master->streams[i].node, &smmu->streams);  in arm_smmu_insert_master()
2628 		kfree(master->streams);  in arm_smmu_insert_master()
2630 	mutex_unlock(&smmu->streams_mutex);  in arm_smmu_insert_master()
2638 	struct arm_smmu_device *smmu = master->smmu;  in arm_smmu_remove_master()  local
2639 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);  in arm_smmu_remove_master()
2641 	if (!smmu || !master->streams)  in arm_smmu_remove_master()
2644 	mutex_lock(&smmu->streams_mutex);  in arm_smmu_remove_master()
2645 	for (i = 0; i < fwspec->num_ids; i++)  in arm_smmu_remove_master()
2646 		rb_erase(&master->streams[i].node, &smmu->streams);  in arm_smmu_remove_master()
2647 	mutex_unlock(&smmu->streams_mutex);  in arm_smmu_remove_master()
2649 	kfree(master->streams);  in arm_smmu_remove_master()
2657 	struct arm_smmu_device *smmu;  in arm_smmu_probe_device()  local
2661 	if (!fwspec || fwspec->ops != &arm_smmu_ops)  in arm_smmu_probe_device()
2662 		return ERR_PTR(-ENODEV);  in arm_smmu_probe_device()
2665 		return ERR_PTR(-EBUSY);  in arm_smmu_probe_device()
2667 	smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);  in arm_smmu_probe_device()
2668 	if (!smmu)  in arm_smmu_probe_device()
2669 		return ERR_PTR(-ENODEV);  in arm_smmu_probe_device()
2673 		return ERR_PTR(-ENOMEM);  in arm_smmu_probe_device()
2675 	master->dev = dev;  in arm_smmu_probe_device()
2676 	master->smmu = smmu;  in arm_smmu_probe_device()
2677 	INIT_LIST_HEAD(&master->bonds);  in arm_smmu_probe_device()
2680 	ret = arm_smmu_insert_master(smmu, master);  in arm_smmu_probe_device()
2684 	device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);  in arm_smmu_probe_device()
2685 	master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);  in arm_smmu_probe_device()
2689 	 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register  in arm_smmu_probe_device()
2697 	if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))  in arm_smmu_probe_device()
2698 		master->ssid_bits = min_t(u8, master->ssid_bits,  in arm_smmu_probe_device()
2701 	if ((smmu->features & ARM_SMMU_FEAT_STALLS &&  in arm_smmu_probe_device()
2702 	     device_property_read_bool(dev, "dma-can-stall")) ||  in arm_smmu_probe_device()
2703 	    smmu->features & ARM_SMMU_FEAT_STALL_FORCE)  in arm_smmu_probe_device()
2704 		master->stall_enabled = true;  in arm_smmu_probe_device()
2706 	return &smmu->iommu;  in arm_smmu_probe_device()
2719 		iopf_queue_remove_device(master->smmu->evtq.iopf, dev);  in arm_smmu_release_device()
2732 	 * aliases, since the necessary ID-to-device lookup becomes rather  in arm_smmu_device_group()
2733 	 * impractical given a potential sparse 32-bit stream ID space.  in arm_smmu_device_group()
2748 	mutex_lock(&smmu_domain->init_mutex);  in arm_smmu_enable_nesting()
2749 	if (smmu_domain->smmu)  in arm_smmu_enable_nesting()
2750 		ret = -EPERM;  in arm_smmu_enable_nesting()
2752 		smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;  in arm_smmu_enable_nesting()
2753 	mutex_unlock(&smmu_domain->init_mutex);  in arm_smmu_enable_nesting()
2760 	return iommu_fwspec_add_ids(dev, args->args, 1);  in arm_smmu_of_xlate()
2774 	list_add_tail(&region->list, head);  in arm_smmu_get_resv_regions()
2785 		return -ENODEV;  in arm_smmu_dev_enable_feature()
2790 			return -EINVAL;  in arm_smmu_dev_enable_feature()
2791 		if (master->iopf_enabled)  in arm_smmu_dev_enable_feature()
2792 			return -EBUSY;  in arm_smmu_dev_enable_feature()
2793 		master->iopf_enabled = true;  in arm_smmu_dev_enable_feature()
2797 			return -EINVAL;  in arm_smmu_dev_enable_feature()
2799 			return -EBUSY;  in arm_smmu_dev_enable_feature()
2802 		return -EINVAL;  in arm_smmu_dev_enable_feature()
2812 		return -EINVAL;  in arm_smmu_dev_disable_feature()
2816 		if (!master->iopf_enabled)  in arm_smmu_dev_disable_feature()
2817 			return -EINVAL;  in arm_smmu_dev_disable_feature()
2818 		if (master->sva_enabled)  in arm_smmu_dev_disable_feature()
2819 			return -EBUSY;  in arm_smmu_dev_disable_feature()
2820 		master->iopf_enabled = false;  in arm_smmu_dev_disable_feature()
2824 			return -EINVAL;  in arm_smmu_dev_disable_feature()
2827 		return -EINVAL;  in arm_smmu_dev_disable_feature()
2836 #define IS_HISI_PTT_DEVICE(pdev)	((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
2837 					 (pdev)->device == 0xa12e)
2875 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
2890 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,  in arm_smmu_init_one_queue()  argument
2900 		qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;  in arm_smmu_init_one_queue()
2901 		q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,  in arm_smmu_init_one_queue()
2903 		if (q->base || qsz < PAGE_SIZE)  in arm_smmu_init_one_queue()
2906 		q->llq.max_n_shift--;  in arm_smmu_init_one_queue()
2909 	if (!q->base) {  in arm_smmu_init_one_queue()
2910 		dev_err(smmu->dev,  in arm_smmu_init_one_queue()
2913 		return -ENOMEM;  in arm_smmu_init_one_queue()
2916 	if (!WARN_ON(q->base_dma & (qsz - 1))) {  in arm_smmu_init_one_queue()
2917 		dev_info(smmu->dev, "allocated %u entries for %s\n",  in arm_smmu_init_one_queue()
2918 			 1 << q->llq.max_n_shift, name);  in arm_smmu_init_one_queue()
2921 	q->prod_reg	= page + prod_off;  in arm_smmu_init_one_queue()
2922 	q->cons_reg	= page + cons_off;  in arm_smmu_init_one_queue()
2923 	q->ent_dwords	= dwords;  in arm_smmu_init_one_queue()
2925 	q->q_base  = Q_BASE_RWA;  in arm_smmu_init_one_queue()
2926 	q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;  in arm_smmu_init_one_queue()
2927 	q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);  in arm_smmu_init_one_queue()
2929 	q->llq.prod = q->llq.cons = 0;  in arm_smmu_init_one_queue()
2933 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)  in arm_smmu_cmdq_init()  argument
2935 	struct arm_smmu_cmdq *cmdq = &smmu->cmdq;  in arm_smmu_cmdq_init()
2936 	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;  in arm_smmu_cmdq_init()
2938 	atomic_set(&cmdq->owner_prod, 0);  in arm_smmu_cmdq_init()
2939 	atomic_set(&cmdq->lock, 0);  in arm_smmu_cmdq_init()
2941 	cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,  in arm_smmu_cmdq_init()
2943 	if (!cmdq->valid_map)  in arm_smmu_cmdq_init()
2944 		return -ENOMEM;  in arm_smmu_cmdq_init()
2949 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)  in arm_smmu_init_queues()  argument
2954 	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,  in arm_smmu_init_queues()
2960 	ret = arm_smmu_cmdq_init(smmu);  in arm_smmu_init_queues()
2965 	ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,  in arm_smmu_init_queues()
2971 	if ((smmu->features & ARM_SMMU_FEAT_SVA) &&  in arm_smmu_init_queues()
2972 	    (smmu->features & ARM_SMMU_FEAT_STALLS)) {  in arm_smmu_init_queues()
2973 		smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));  in arm_smmu_init_queues()
2974 		if (!smmu->evtq.iopf)  in arm_smmu_init_queues()
2975 			return -ENOMEM;  in arm_smmu_init_queues()
2979 	if (!(smmu->features & ARM_SMMU_FEAT_PRI))  in arm_smmu_init_queues()
2982 	return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,  in arm_smmu_init_queues()
2987 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)  in arm_smmu_init_l1_strtab()  argument
2990 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;  in arm_smmu_init_l1_strtab()
2991 	void *strtab = smmu->strtab_cfg.strtab;  in arm_smmu_init_l1_strtab()
2993 	cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents,  in arm_smmu_init_l1_strtab()
2994 				    sizeof(*cfg->l1_desc), GFP_KERNEL);  in arm_smmu_init_l1_strtab()
2995 	if (!cfg->l1_desc)  in arm_smmu_init_l1_strtab()
2996 		return -ENOMEM;  in arm_smmu_init_l1_strtab()
2998 	for (i = 0; i < cfg->num_l1_ents; ++i) {  in arm_smmu_init_l1_strtab()
2999 		arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);  in arm_smmu_init_l1_strtab()
3006 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)  in arm_smmu_init_strtab_2lvl()  argument
3011 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;  in arm_smmu_init_strtab_2lvl()
3014 	size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);  in arm_smmu_init_strtab_2lvl()
3015 	size = min(size, smmu->sid_bits - STRTAB_SPLIT);  in arm_smmu_init_strtab_2lvl()
3016 	cfg->num_l1_ents = 1 << size;  in arm_smmu_init_strtab_2lvl()
3019 	if (size < smmu->sid_bits)  in arm_smmu_init_strtab_2lvl()
3020 		dev_warn(smmu->dev,  in arm_smmu_init_strtab_2lvl()
3021 			 "2-level strtab only covers %u/%u bits of SID\n",  in arm_smmu_init_strtab_2lvl()
3022 			 size, smmu->sid_bits);  in arm_smmu_init_strtab_2lvl()
3024 	l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);  in arm_smmu_init_strtab_2lvl()
3025 	strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,  in arm_smmu_init_strtab_2lvl()
3028 		dev_err(smmu->dev,  in arm_smmu_init_strtab_2lvl()
3031 		return -ENOMEM;  in arm_smmu_init_strtab_2lvl()
3033 	cfg->strtab = strtab;  in arm_smmu_init_strtab_2lvl()
3039 	cfg->strtab_base_cfg = reg;  in arm_smmu_init_strtab_2lvl()
3041 	return arm_smmu_init_l1_strtab(smmu);  in arm_smmu_init_strtab_2lvl()
3044 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)  in arm_smmu_init_strtab_linear()  argument
3049 	struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;  in arm_smmu_init_strtab_linear()
3051 	size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);  in arm_smmu_init_strtab_linear()
3052 	strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,  in arm_smmu_init_strtab_linear()
3055 		dev_err(smmu->dev,  in arm_smmu_init_strtab_linear()
3058 		return -ENOMEM;  in arm_smmu_init_strtab_linear()
3060 	cfg->strtab = strtab;  in arm_smmu_init_strtab_linear()
3061 	cfg->num_l1_ents = 1 << smmu->sid_bits;  in arm_smmu_init_strtab_linear()
3065 	reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);  in arm_smmu_init_strtab_linear()
3066 	cfg->strtab_base_cfg = reg;  in arm_smmu_init_strtab_linear()
3068 	arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false);  in arm_smmu_init_strtab_linear()
3072 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)  in arm_smmu_init_strtab()  argument
3077 	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)  in arm_smmu_init_strtab()
3078 		ret = arm_smmu_init_strtab_2lvl(smmu);  in arm_smmu_init_strtab()
3080 		ret = arm_smmu_init_strtab_linear(smmu);  in arm_smmu_init_strtab()
3086 	reg  = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;  in arm_smmu_init_strtab()
3088 	smmu->strtab_cfg.strtab_base = reg;  in arm_smmu_init_strtab()
3090 	ida_init(&smmu->vmid_map);  in arm_smmu_init_strtab()
3095 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)  in arm_smmu_init_structures()  argument
3099 	mutex_init(&smmu->streams_mutex);  in arm_smmu_init_structures()
3100 	smmu->streams = RB_ROOT;  in arm_smmu_init_structures()
3102 	ret = arm_smmu_init_queues(smmu);  in arm_smmu_init_structures()
3106 	return arm_smmu_init_strtab(smmu);  in arm_smmu_init_structures()
3109 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,  in arm_smmu_write_reg_sync()  argument
3114 	writel_relaxed(val, smmu->base + reg_off);  in arm_smmu_write_reg_sync()
3115 	return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,  in arm_smmu_write_reg_sync()
3120 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)  in arm_smmu_update_gbpa()  argument
3123 	u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;  in arm_smmu_update_gbpa()
3137 		dev_err(smmu->dev, "GBPA not responding to update\n");  in arm_smmu_update_gbpa()
3151 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);  in arm_smmu_write_msi_msg()  local
3152 	phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];  in arm_smmu_write_msi_msg()
3154 	doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;  in arm_smmu_write_msi_msg()
3157 	writeq_relaxed(doorbell, smmu->base + cfg[0]);  in arm_smmu_write_msi_msg()
3158 	writel_relaxed(msg->data, smmu->base + cfg[1]);  in arm_smmu_write_msi_msg()
3159 	writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);  in arm_smmu_write_msi_msg()
3162 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)  in arm_smmu_setup_msis()  argument
3165 	struct device *dev = smmu->dev;  in arm_smmu_setup_msis()
3168 	writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);  in arm_smmu_setup_msis()
3169 	writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);  in arm_smmu_setup_msis()
3171 	if (smmu->features & ARM_SMMU_FEAT_PRI)  in arm_smmu_setup_msis()
3172 		writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);  in arm_smmu_setup_msis()
3174 		nvec--;  in arm_smmu_setup_msis()
3176 	if (!(smmu->features & ARM_SMMU_FEAT_MSI))  in arm_smmu_setup_msis()
3179 	if (!dev->msi.domain) {  in arm_smmu_setup_msis()
3180 		dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");  in arm_smmu_setup_msis()
3187 		dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");  in arm_smmu_setup_msis()
3191 	smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);  in arm_smmu_setup_msis()
3192 	smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);  in arm_smmu_setup_msis()
3193 	smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);  in arm_smmu_setup_msis()
3199 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)  in arm_smmu_setup_unique_irqs()  argument
3203 	arm_smmu_setup_msis(smmu);  in arm_smmu_setup_unique_irqs()
3206 	irq = smmu->evtq.q.irq;  in arm_smmu_setup_unique_irqs()
3208 		ret = devm_request_threaded_irq(smmu->dev, irq, NULL,  in arm_smmu_setup_unique_irqs()
3211 						"arm-smmu-v3-evtq", smmu);  in arm_smmu_setup_unique_irqs()
3213 			dev_warn(smmu->dev, "failed to enable evtq irq\n");  in arm_smmu_setup_unique_irqs()
3215 		dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");  in arm_smmu_setup_unique_irqs()
3218 	irq = smmu->gerr_irq;  in arm_smmu_setup_unique_irqs()
3220 		ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,  in arm_smmu_setup_unique_irqs()
3221 				       0, "arm-smmu-v3-gerror", smmu);  in arm_smmu_setup_unique_irqs()
3223 			dev_warn(smmu->dev, "failed to enable gerror irq\n");  in arm_smmu_setup_unique_irqs()
3225 		dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");  in arm_smmu_setup_unique_irqs()
3228 	if (smmu->features & ARM_SMMU_FEAT_PRI) {  in arm_smmu_setup_unique_irqs()
3229 		irq = smmu->priq.q.irq;  in arm_smmu_setup_unique_irqs()
3231 			ret = devm_request_threaded_irq(smmu->dev, irq, NULL,  in arm_smmu_setup_unique_irqs()
3234 							"arm-smmu-v3-priq",  in arm_smmu_setup_unique_irqs()
3235 							smmu);  in arm_smmu_setup_unique_irqs()
3237 				dev_warn(smmu->dev,  in arm_smmu_setup_unique_irqs()
3240 			dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");  in arm_smmu_setup_unique_irqs()
3245 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)  in arm_smmu_setup_irqs()  argument
3251 	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,  in arm_smmu_setup_irqs()
3254 		dev_err(smmu->dev, "failed to disable irqs\n");  in arm_smmu_setup_irqs()
3258 	irq = smmu->combined_irq;  in arm_smmu_setup_irqs()
3264 		ret = devm_request_threaded_irq(smmu->dev, irq,  in arm_smmu_setup_irqs()
3268 					"arm-smmu-v3-combined-irq", smmu);  in arm_smmu_setup_irqs()
3270 			dev_warn(smmu->dev, "failed to enable combined irq\n");  in arm_smmu_setup_irqs()
3272 		arm_smmu_setup_unique_irqs(smmu);  in arm_smmu_setup_irqs()
3274 	if (smmu->features & ARM_SMMU_FEAT_PRI)  in arm_smmu_setup_irqs()
3277 	/* Enable interrupt generation on the SMMU */  in arm_smmu_setup_irqs()
3278 	ret = arm_smmu_write_reg_sync(smmu, irqen_flags,  in arm_smmu_setup_irqs()
3281 		dev_warn(smmu->dev, "failed to enable irqs\n");  in arm_smmu_setup_irqs()
3286 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)  in arm_smmu_device_disable()  argument
3290 	ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);  in arm_smmu_device_disable()
3292 		dev_err(smmu->dev, "failed to clear cr0\n");  in arm_smmu_device_disable()
3297 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)  in arm_smmu_device_reset()  argument
3303 	/* Clear CR0 and sync (disables SMMU and queue processing) */  in arm_smmu_device_reset()
3304 	reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);  in arm_smmu_device_reset()
3306 		dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");  in arm_smmu_device_reset()
3308 		arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);  in arm_smmu_device_reset()
3311 	ret = arm_smmu_device_disable(smmu);  in arm_smmu_device_reset()
3322 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);  in arm_smmu_device_reset()
3327 	if (smmu->features & ARM_SMMU_FEAT_E2H)  in arm_smmu_device_reset()
3330 	writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);  in arm_smmu_device_reset()
3333 	writeq_relaxed(smmu->strtab_cfg.strtab_base,  in arm_smmu_device_reset()
3334 		       smmu->base + ARM_SMMU_STRTAB_BASE);  in arm_smmu_device_reset()
3335 	writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,  in arm_smmu_device_reset()
3336 		       smmu->base + ARM_SMMU_STRTAB_BASE_CFG);  in arm_smmu_device_reset()
3339 	writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);  in arm_smmu_device_reset()
3340 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);  in arm_smmu_device_reset()
3341 	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);  in arm_smmu_device_reset()
3344 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,  in arm_smmu_device_reset()
3347 		dev_err(smmu->dev, "failed to enable command queue\n");  in arm_smmu_device_reset()
3353 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);  in arm_smmu_device_reset()
3356 	if (smmu->features & ARM_SMMU_FEAT_HYP) {  in arm_smmu_device_reset()
3358 		arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);  in arm_smmu_device_reset()
3362 	arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);  in arm_smmu_device_reset()
3365 	writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);  in arm_smmu_device_reset()
3366 	writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);  in arm_smmu_device_reset()
3367 	writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);  in arm_smmu_device_reset()
3370 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,  in arm_smmu_device_reset()
3373 		dev_err(smmu->dev, "failed to enable event queue\n");  in arm_smmu_device_reset()
3378 	if (smmu->features & ARM_SMMU_FEAT_PRI) {  in arm_smmu_device_reset()
3379 		writeq_relaxed(smmu->priq.q.q_base,  in arm_smmu_device_reset()
3380 			       smmu->base + ARM_SMMU_PRIQ_BASE);  in arm_smmu_device_reset()
3381 		writel_relaxed(smmu->priq.q.llq.prod,  in arm_smmu_device_reset()
3382 			       smmu->page1 + ARM_SMMU_PRIQ_PROD);  in arm_smmu_device_reset()
3383 		writel_relaxed(smmu->priq.q.llq.cons,  in arm_smmu_device_reset()
3384 			       smmu->page1 + ARM_SMMU_PRIQ_CONS);  in arm_smmu_device_reset()
3387 		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,  in arm_smmu_device_reset()
3390 			dev_err(smmu->dev, "failed to enable PRI queue\n");  in arm_smmu_device_reset()
3395 	if (smmu->features & ARM_SMMU_FEAT_ATS) {  in arm_smmu_device_reset()
3397 		ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,  in arm_smmu_device_reset()
3400 			dev_err(smmu->dev, "failed to enable ATS check\n");  in arm_smmu_device_reset()
3405 	ret = arm_smmu_setup_irqs(smmu);  in arm_smmu_device_reset()
3407 		dev_err(smmu->dev, "failed to setup irqs\n");  in arm_smmu_device_reset()
3414 	/* Enable the SMMU interface, or ensure bypass */  in arm_smmu_device_reset()
3418 		ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);  in arm_smmu_device_reset()
3422 	ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,  in arm_smmu_device_reset()
3425 		dev_err(smmu->dev, "failed to enable SMMU interface\n");  in arm_smmu_device_reset()
3436 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)  in arm_smmu_device_iidr_probe()  argument
3441 	reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);  in arm_smmu_device_iidr_probe()
3453 				smmu->features &= ~ARM_SMMU_FEAT_SEV;  in arm_smmu_device_iidr_probe()
3456 				smmu->features &= ~ARM_SMMU_FEAT_NESTING;  in arm_smmu_device_iidr_probe()
3460 			smmu->features &= ~ARM_SMMU_FEAT_BTM;  in arm_smmu_device_iidr_probe()
3461 			smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;  in arm_smmu_device_iidr_probe()
3463 			smmu->features &= ~ARM_SMMU_FEAT_NESTING;  in arm_smmu_device_iidr_probe()
3470 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)  in arm_smmu_device_hw_probe()  argument
3473 	bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;  in arm_smmu_device_hw_probe()
3476 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);  in arm_smmu_device_hw_probe()
3478 	/* 2-level structures */  in arm_smmu_device_hw_probe()
3480 		smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;  in arm_smmu_device_hw_probe()
3483 		smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;  in arm_smmu_device_hw_probe()
3492 		smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;  in arm_smmu_device_hw_probe()
3496 		smmu->features |= ARM_SMMU_FEAT_TT_BE;  in arm_smmu_device_hw_probe()
3500 		smmu->features |= ARM_SMMU_FEAT_TT_LE;  in arm_smmu_device_hw_probe()
3504 		dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");  in arm_smmu_device_hw_probe()
3505 		return -ENXIO;  in arm_smmu_device_hw_probe()
3510 		smmu->features |= ARM_SMMU_FEAT_PRI;  in arm_smmu_device_hw_probe()
3513 		smmu->features |= ARM_SMMU_FEAT_ATS;  in arm_smmu_device_hw_probe()
3516 		smmu->features |= ARM_SMMU_FEAT_SEV;  in arm_smmu_device_hw_probe()
3519 		smmu->features |= ARM_SMMU_FEAT_MSI;  in arm_smmu_device_hw_probe()
3521 			smmu->options |= ARM_SMMU_OPT_MSIPOLL;  in arm_smmu_device_hw_probe()
3525 		smmu->features |= ARM_SMMU_FEAT_HYP;  in arm_smmu_device_hw_probe()
3527 			smmu->features |= ARM_SMMU_FEAT_E2H;  in arm_smmu_device_hw_probe()
3535 		dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",  in arm_smmu_device_hw_probe()
3540 		smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;  in arm_smmu_device_hw_probe()
3543 		smmu->features |= ARM_SMMU_FEAT_STALLS;  in arm_smmu_device_hw_probe()
3547 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;  in arm_smmu_device_hw_probe()
3550 		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;  in arm_smmu_device_hw_probe()
3553 		dev_err(smmu->dev, "no translation support!\n");  in arm_smmu_device_hw_probe()
3554 		return -ENXIO;  in arm_smmu_device_hw_probe()
3560 		smmu->ias = 40;  in arm_smmu_device_hw_probe()
3565 		dev_err(smmu->dev, "AArch64 table format not supported!\n");  in arm_smmu_device_hw_probe()
3566 		return -ENXIO;  in arm_smmu_device_hw_probe()
3570 	smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;  in arm_smmu_device_hw_probe()
3571 	smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;  in arm_smmu_device_hw_probe()
3574 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);  in arm_smmu_device_hw_probe()
3576 		dev_err(smmu->dev, "embedded implementation not supported\n");  in arm_smmu_device_hw_probe()
3577 		return -ENXIO;  in arm_smmu_device_hw_probe()
3581 	smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3583 	if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {  in arm_smmu_device_hw_probe()
3588 		 * restrictions on the base pointer for a unit-length queue.  in arm_smmu_device_hw_probe()
3590 		dev_err(smmu->dev, "command queue size <= %d entries not supported\n",  in arm_smmu_device_hw_probe()
3592 		return -ENXIO;  in arm_smmu_device_hw_probe()
3595 	smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3597 	smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3601 	smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);  in arm_smmu_device_hw_probe()
3602 	smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);  in arm_smmu_device_hw_probe()
3603 	smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;  in arm_smmu_device_hw_probe()
3606 	 * If the SMMU supports fewer bits than would fill a single L2 stream  in arm_smmu_device_hw_probe()
3609 	if (smmu->sid_bits <= STRTAB_SPLIT)  in arm_smmu_device_hw_probe()
3610 		smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;  in arm_smmu_device_hw_probe()
3613 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);  in arm_smmu_device_hw_probe()
3615 		smmu->features |= ARM_SMMU_FEAT_RANGE_INV;  in arm_smmu_device_hw_probe()
3618 	reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);  in arm_smmu_device_hw_probe()
3621 	smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);  in arm_smmu_device_hw_probe()
3625 		smmu->pgsize_bitmap |= SZ_64K | SZ_512M;  in arm_smmu_device_hw_probe()
3627 		smmu->pgsize_bitmap |= SZ_16K | SZ_32M;  in arm_smmu_device_hw_probe()
3629 		smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;  in arm_smmu_device_hw_probe()
3633 		smmu->features |= ARM_SMMU_FEAT_VAX;  in arm_smmu_device_hw_probe()
3638 		smmu->oas = 32;  in arm_smmu_device_hw_probe()
3641 		smmu->oas = 36;  in arm_smmu_device_hw_probe()
3644 		smmu->oas = 40;  in arm_smmu_device_hw_probe()
3647 		smmu->oas = 42;  in arm_smmu_device_hw_probe()
3650 		smmu->oas = 44;  in arm_smmu_device_hw_probe()
3653 		smmu->oas = 52;  in arm_smmu_device_hw_probe()
3654 		smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */  in arm_smmu_device_hw_probe()
3657 		dev_info(smmu->dev,  in arm_smmu_device_hw_probe()
3658 			"unknown output address size. Truncating to 48-bit\n");  in arm_smmu_device_hw_probe()
3661 		smmu->oas = 48;  in arm_smmu_device_hw_probe()
3664 	if (arm_smmu_ops.pgsize_bitmap == -1UL)  in arm_smmu_device_hw_probe()
3665 		arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;  in arm_smmu_device_hw_probe()
3667 		arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;  in arm_smmu_device_hw_probe()
3670 	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))  in arm_smmu_device_hw_probe()
3671 		dev_warn(smmu->dev,  in arm_smmu_device_hw_probe()
3674 	smmu->ias = max(smmu->ias, smmu->oas);  in arm_smmu_device_hw_probe()
3676 	if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&  in arm_smmu_device_hw_probe()
3677 	    (smmu->features & ARM_SMMU_FEAT_TRANS_S2))  in arm_smmu_device_hw_probe()
3678 		smmu->features |= ARM_SMMU_FEAT_NESTING;  in arm_smmu_device_hw_probe()
3680 	arm_smmu_device_iidr_probe(smmu);  in arm_smmu_device_hw_probe()
3682 	if (arm_smmu_sva_supported(smmu))  in arm_smmu_device_hw_probe()
3683 		smmu->features |= ARM_SMMU_FEAT_SVA;  in arm_smmu_device_hw_probe()
3685 	dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",  in arm_smmu_device_hw_probe()
3686 		 smmu->ias, smmu->oas, smmu->features);  in arm_smmu_device_hw_probe()
3691 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)  in acpi_smmu_get_options()  argument
3695 		smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;  in acpi_smmu_get_options()
3698 		smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;  in acpi_smmu_get_options()
3702 	dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);  in acpi_smmu_get_options()
3706 				      struct arm_smmu_device *smmu)  in arm_smmu_device_acpi_probe()  argument
3709 	struct device *dev = smmu->dev;  in arm_smmu_device_acpi_probe()
3715 	iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;  in arm_smmu_device_acpi_probe()
3717 	acpi_smmu_get_options(iort_smmu->model, smmu);  in arm_smmu_device_acpi_probe()
3719 	if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)  in arm_smmu_device_acpi_probe()
3720 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;  in arm_smmu_device_acpi_probe()
3726 					     struct arm_smmu_device *smmu)  in arm_smmu_device_acpi_probe()  argument
3728 	return -ENODEV;  in arm_smmu_device_acpi_probe()
3733 				    struct arm_smmu_device *smmu)  in arm_smmu_device_dt_probe()  argument
3735 	struct device *dev = &pdev->dev;  in arm_smmu_device_dt_probe()
3737 	int ret = -EINVAL;  in arm_smmu_device_dt_probe()
3739 	if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))  in arm_smmu_device_dt_probe()
3740 		dev_err(dev, "missing #iommu-cells property\n");  in arm_smmu_device_dt_probe()
3742 		dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);  in arm_smmu_device_dt_probe()
3746 	parse_driver_options(smmu);  in arm_smmu_device_dt_probe()
3748 	if (of_dma_is_coherent(dev->of_node))  in arm_smmu_device_dt_probe()
3749 		smmu->features |= ARM_SMMU_FEAT_COHERENCY;  in arm_smmu_device_dt_probe()
3754 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)  in arm_smmu_resource_size()  argument
3756 	if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)  in arm_smmu_resource_size()
3770 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)  in arm_smmu_rmr_install_bypass_ste()  argument
3776 	iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);  in arm_smmu_rmr_install_bypass_ste()
3784 		for (i = 0; i < rmr->num_sids; i++) {  in arm_smmu_rmr_install_bypass_ste()
3785 			ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);  in arm_smmu_rmr_install_bypass_ste()
3787 				dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",  in arm_smmu_rmr_install_bypass_ste()
3788 					rmr->sids[i]);  in arm_smmu_rmr_install_bypass_ste()
3792 			step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]);  in arm_smmu_rmr_install_bypass_ste()
3797 	iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);  in arm_smmu_rmr_install_bypass_ste()
3805 	struct arm_smmu_device *smmu;  in arm_smmu_device_probe()  local
3806 	struct device *dev = &pdev->dev;  in arm_smmu_device_probe()
3809 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);  in arm_smmu_device_probe()
3810 	if (!smmu)  in arm_smmu_device_probe()
3811 		return -ENOMEM;  in arm_smmu_device_probe()
3812 	smmu->dev = dev;  in arm_smmu_device_probe()
3814 	if (dev->of_node) {  in arm_smmu_device_probe()
3815 		ret = arm_smmu_device_dt_probe(pdev, smmu);  in arm_smmu_device_probe()
3817 		ret = arm_smmu_device_acpi_probe(pdev, smmu);  in arm_smmu_device_probe()
3818 		if (ret == -ENODEV)  in arm_smmu_device_probe()
3828 		return -EINVAL;  in arm_smmu_device_probe()
3829 	if (resource_size(res) < arm_smmu_resource_size(smmu)) {  in arm_smmu_device_probe()
3831 		return -EINVAL;  in arm_smmu_device_probe()
3833 	ioaddr = res->start;  in arm_smmu_device_probe()
3839 	smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);  in arm_smmu_device_probe()
3840 	if (IS_ERR(smmu->base))  in arm_smmu_device_probe()
3841 		return PTR_ERR(smmu->base);  in arm_smmu_device_probe()
3843 	if (arm_smmu_resource_size(smmu) > SZ_64K) {  in arm_smmu_device_probe()
3844 		smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,  in arm_smmu_device_probe()
3846 		if (IS_ERR(smmu->page1))  in arm_smmu_device_probe()
3847 			return PTR_ERR(smmu->page1);  in arm_smmu_device_probe()
3849 		smmu->page1 = smmu->base;  in arm_smmu_device_probe()
3856 		smmu->combined_irq = irq;  in arm_smmu_device_probe()
3860 			smmu->evtq.q.irq = irq;  in arm_smmu_device_probe()
3864 			smmu->priq.q.irq = irq;  in arm_smmu_device_probe()
3868 			smmu->gerr_irq = irq;  in arm_smmu_device_probe()
3871 	ret = arm_smmu_device_hw_probe(smmu);  in arm_smmu_device_probe()
3875 	/* Initialise in-memory data structures */  in arm_smmu_device_probe()
3876 	ret = arm_smmu_init_structures(smmu);  in arm_smmu_device_probe()
3881 	platform_set_drvdata(pdev, smmu);  in arm_smmu_device_probe()
3884 	arm_smmu_rmr_install_bypass_ste(smmu);  in arm_smmu_device_probe()
3887 	ret = arm_smmu_device_reset(smmu, bypass);  in arm_smmu_device_probe()
3892 	ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,  in arm_smmu_device_probe()
3897 	ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);  in arm_smmu_device_probe()
3900 		iommu_device_sysfs_remove(&smmu->iommu);  in arm_smmu_device_probe()
3909 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);  in arm_smmu_device_remove()  local
3911 	iommu_device_unregister(&smmu->iommu);  in arm_smmu_device_remove()
3912 	iommu_device_sysfs_remove(&smmu->iommu);  in arm_smmu_device_remove()
3913 	arm_smmu_device_disable(smmu);  in arm_smmu_device_remove()
3914 	iopf_queue_free(smmu->evtq.iopf);  in arm_smmu_device_remove()
3915 	ida_destroy(&smmu->vmid_map);  in arm_smmu_device_remove()
3920 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);  in arm_smmu_device_shutdown()  local
3922 	arm_smmu_device_disable(smmu);  in arm_smmu_device_shutdown()
3926 	{ .compatible = "arm,smmu-v3", },
3939 		.name			= "arm-smmu-v3",
3952 MODULE_ALIAS("platform:arm-smmu-v3");
3953 MODULE_LICENSE("GPL v2");