1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * IOMMU API for ARM architected SMMU implementations.
4  *
5  * Copyright (C) 2013 ARM Limited
6  *
7  * Author: Will Deacon <will.deacon@arm.com>
8  *
9  * This driver currently supports:
10  *	- SMMUv1 and v2 implementations
11  *	- Stream-matching and stream-indexing
12  *	- v7/v8 long-descriptor format
13  *	- Non-secure access to the SMMU
14  *	- Context fault reporting
15  *	- Extended Stream ID (16 bit)
16  */
17 
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19 
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37 
38 #include <linux/fsl/mc.h>
39 
40 #include "arm-smmu.h"
41 #include "../../dma-iommu.h"
42 
43 /*
44  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
45  * global register space are still, in fact, using a hypervisor to mediate it
46  * by trapping and emulating register accesses. Sadly, some deployed versions
47  * of said trapping code have bugs wherein they go horribly wrong for stores
48  * using r31 (i.e. XZR/WZR) as the source register.
49  */
50 #define QCOM_DUMMY_VAL -1
51 
52 #define MSI_IOVA_BASE			0x8000000
53 #define MSI_IOVA_LENGTH			0x100000
54 
55 static int force_stage;
56 module_param(force_stage, int, S_IRUGO);
57 MODULE_PARM_DESC(force_stage,
58 	"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
59 static bool disable_bypass =
60 	IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
61 module_param(disable_bypass, bool, S_IRUGO);
62 MODULE_PARM_DESC(disable_bypass,
63 	"Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
64 
65 #define s2cr_init_val (struct arm_smmu_s2cr){				\
66 	.type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,	\
67 }
68 
69 static bool using_legacy_binding, using_generic_binding;
70 
71 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
72 {
73 	if (pm_runtime_enabled(smmu->dev))
74 		return pm_runtime_resume_and_get(smmu->dev);
75 
76 	return 0;
77 }
78 
79 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
80 {
81 	if (pm_runtime_enabled(smmu->dev))
82 		pm_runtime_put_autosuspend(smmu->dev);
83 }
84 
85 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
86 {
87 	return container_of(dom, struct arm_smmu_domain, domain);
88 }
89 
90 static struct platform_driver arm_smmu_driver;
91 static struct iommu_ops arm_smmu_ops;
92 
93 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
94 static struct device_node *dev_get_dev_node(struct device *dev)
95 {
96 	if (dev_is_pci(dev)) {
97 		struct pci_bus *bus = to_pci_dev(dev)->bus;
98 
99 		while (!pci_is_root_bus(bus))
100 			bus = bus->parent;
101 		return of_node_get(bus->bridge->parent->of_node);
102 	}
103 
104 	return of_node_get(dev->of_node);
105 }
106 
107 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
108 {
109 	*((__be32 *)data) = cpu_to_be32(alias);
110 	return 0; /* Continue walking */
111 }
112 
113 static int __find_legacy_master_phandle(struct device *dev, void *data)
114 {
115 	struct of_phandle_iterator *it = *(void **)data;
116 	struct device_node *np = it->node;
117 	int err;
118 
119 	of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
120 			    "#stream-id-cells", -1)
121 		if (it->node == np) {
122 			*(void **)data = dev;
123 			return 1;
124 		}
125 	it->node = np;
126 	return err == -ENOENT ? 0 : err;
127 }
128 
129 static int arm_smmu_register_legacy_master(struct device *dev,
130 					   struct arm_smmu_device **smmu)
131 {
132 	struct device *smmu_dev;
133 	struct device_node *np;
134 	struct of_phandle_iterator it;
135 	void *data = &it;
136 	u32 *sids;
137 	__be32 pci_sid;
138 	int err;
139 
140 	np = dev_get_dev_node(dev);
141 	if (!np || !of_property_present(np, "#stream-id-cells")) {
142 		of_node_put(np);
143 		return -ENODEV;
144 	}
145 
146 	it.node = np;
147 	err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
148 				     __find_legacy_master_phandle);
149 	smmu_dev = data;
150 	of_node_put(np);
151 	if (err == 0)
152 		return -ENODEV;
153 	if (err < 0)
154 		return err;
155 
156 	if (dev_is_pci(dev)) {
157 		/* "mmu-masters" assumes Stream ID == Requester ID */
158 		pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
159 				       &pci_sid);
160 		it.cur = &pci_sid;
161 		it.cur_count = 1;
162 	}
163 
164 	err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
165 				&arm_smmu_ops);
166 	if (err)
167 		return err;
168 
169 	sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
170 	if (!sids)
171 		return -ENOMEM;
172 
173 	*smmu = dev_get_drvdata(smmu_dev);
174 	of_phandle_iterator_args(&it, sids, it.cur_count);
175 	err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
176 	kfree(sids);
177 	return err;
178 }
179 #else
180 static int arm_smmu_register_legacy_master(struct device *dev,
181 					   struct arm_smmu_device **smmu)
182 {
183 	return -ENODEV;
184 }
185 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
186 
187 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
188 {
189 	clear_bit(idx, map);
190 }
191 
192 /* Wait for any pending TLB invalidations to complete */
193 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
194 				int sync, int status)
195 {
196 	unsigned int spin_cnt, delay;
197 	u32 reg;
198 
199 	if (smmu->impl && unlikely(smmu->impl->tlb_sync))
200 		return smmu->impl->tlb_sync(smmu, page, sync, status);
201 
202 	arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
203 	for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
204 		for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
205 			reg = arm_smmu_readl(smmu, page, status);
206 			if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
207 				return;
208 			cpu_relax();
209 		}
210 		udelay(delay);
211 	}
212 	dev_err_ratelimited(smmu->dev,
213 			    "TLB sync timed out -- SMMU may be deadlocked\n");
214 }
215 
216 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
217 {
218 	unsigned long flags;
219 
220 	spin_lock_irqsave(&smmu->global_sync_lock, flags);
221 	__arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
222 			    ARM_SMMU_GR0_sTLBGSTATUS);
223 	spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
224 }
225 
226 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
227 {
228 	struct arm_smmu_device *smmu = smmu_domain->smmu;
229 	unsigned long flags;
230 
231 	spin_lock_irqsave(&smmu_domain->cb_lock, flags);
232 	__arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
233 			    ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
234 	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
235 }
236 
237 static void arm_smmu_tlb_inv_context_s1(void *cookie)
238 {
239 	struct arm_smmu_domain *smmu_domain = cookie;
240 	/*
241 	 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
242 	 * current CPU are visible beforehand.
243 	 */
244 	wmb();
245 	arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
246 			  ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
247 	arm_smmu_tlb_sync_context(smmu_domain);
248 }
249 
250 static void arm_smmu_tlb_inv_context_s2(void *cookie)
251 {
252 	struct arm_smmu_domain *smmu_domain = cookie;
253 	struct arm_smmu_device *smmu = smmu_domain->smmu;
254 
255 	/* See above */
256 	wmb();
257 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
258 	arm_smmu_tlb_sync_global(smmu);
259 }
260 
261 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
262 				      size_t granule, void *cookie, int reg)
263 {
264 	struct arm_smmu_domain *smmu_domain = cookie;
265 	struct arm_smmu_device *smmu = smmu_domain->smmu;
266 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
267 	int idx = cfg->cbndx;
268 
269 	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
270 		wmb();
271 
272 	if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
273 		iova = (iova >> 12) << 12;
274 		iova |= cfg->asid;
275 		do {
276 			arm_smmu_cb_write(smmu, idx, reg, iova);
277 			iova += granule;
278 		} while (size -= granule);
279 	} else {
280 		iova >>= 12;
281 		iova |= (u64)cfg->asid << 48;
282 		do {
283 			arm_smmu_cb_writeq(smmu, idx, reg, iova);
284 			iova += granule >> 12;
285 		} while (size -= granule);
286 	}
287 }
288 
289 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
290 				      size_t granule, void *cookie, int reg)
291 {
292 	struct arm_smmu_domain *smmu_domain = cookie;
293 	struct arm_smmu_device *smmu = smmu_domain->smmu;
294 	int idx = smmu_domain->cfg.cbndx;
295 
296 	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
297 		wmb();
298 
299 	iova >>= 12;
300 	do {
301 		if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
302 			arm_smmu_cb_writeq(smmu, idx, reg, iova);
303 		else
304 			arm_smmu_cb_write(smmu, idx, reg, iova);
305 		iova += granule >> 12;
306 	} while (size -= granule);
307 }
308 
309 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
310 				     size_t granule, void *cookie)
311 {
312 	struct arm_smmu_domain *smmu_domain = cookie;
313 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
314 
315 	if (cfg->flush_walk_prefer_tlbiasid) {
316 		arm_smmu_tlb_inv_context_s1(cookie);
317 	} else {
318 		arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
319 					  ARM_SMMU_CB_S1_TLBIVA);
320 		arm_smmu_tlb_sync_context(cookie);
321 	}
322 }
323 
324 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
325 				     unsigned long iova, size_t granule,
326 				     void *cookie)
327 {
328 	arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
329 				  ARM_SMMU_CB_S1_TLBIVAL);
330 }
331 
332 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
333 				     size_t granule, void *cookie)
334 {
335 	arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
336 				  ARM_SMMU_CB_S2_TLBIIPAS2);
337 	arm_smmu_tlb_sync_context(cookie);
338 }
339 
340 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
341 				     unsigned long iova, size_t granule,
342 				     void *cookie)
343 {
344 	arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
345 				  ARM_SMMU_CB_S2_TLBIIPAS2L);
346 }
347 
348 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
349 					size_t granule, void *cookie)
350 {
351 	arm_smmu_tlb_inv_context_s2(cookie);
352 }
353 /*
354  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
355  * almost negligible, but the benefit of getting the first one in as far ahead
356  * of the sync as possible is significant, hence we don't just make this a
357  * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
358  * think.
359  */
360 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
361 					unsigned long iova, size_t granule,
362 					void *cookie)
363 {
364 	struct arm_smmu_domain *smmu_domain = cookie;
365 	struct arm_smmu_device *smmu = smmu_domain->smmu;
366 
367 	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
368 		wmb();
369 
370 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
371 }
372 
373 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
374 	.tlb_flush_all	= arm_smmu_tlb_inv_context_s1,
375 	.tlb_flush_walk	= arm_smmu_tlb_inv_walk_s1,
376 	.tlb_add_page	= arm_smmu_tlb_add_page_s1,
377 };
378 
379 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
380 	.tlb_flush_all	= arm_smmu_tlb_inv_context_s2,
381 	.tlb_flush_walk	= arm_smmu_tlb_inv_walk_s2,
382 	.tlb_add_page	= arm_smmu_tlb_add_page_s2,
383 };
384 
385 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
386 	.tlb_flush_all	= arm_smmu_tlb_inv_context_s2,
387 	.tlb_flush_walk	= arm_smmu_tlb_inv_walk_s2_v1,
388 	.tlb_add_page	= arm_smmu_tlb_add_page_s2_v1,
389 };
390 
391 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
392 {
393 	u32 fsr, fsynr, cbfrsynra;
394 	unsigned long iova;
395 	struct iommu_domain *domain = dev;
396 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
397 	struct arm_smmu_device *smmu = smmu_domain->smmu;
398 	int idx = smmu_domain->cfg.cbndx;
399 	int ret;
400 
401 	fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
402 	if (!(fsr & ARM_SMMU_FSR_FAULT))
403 		return IRQ_NONE;
404 
405 	fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
406 	iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
407 	cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
408 
409 	ret = report_iommu_fault(domain, NULL, iova,
410 		fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
411 
412 	if (ret == -ENOSYS)
413 		dev_err_ratelimited(smmu->dev,
414 		"Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
415 			    fsr, iova, fsynr, cbfrsynra, idx);
416 
417 	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
418 	return IRQ_HANDLED;
419 }
420 
421 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
422 {
423 	u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
424 	struct arm_smmu_device *smmu = dev;
425 	static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
426 				      DEFAULT_RATELIMIT_BURST);
427 
428 	gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
429 	gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
430 	gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
431 	gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
432 
433 	if (!gfsr)
434 		return IRQ_NONE;
435 
436 	if (__ratelimit(&rs)) {
437 		if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
438 		    (gfsr & ARM_SMMU_sGFSR_USF))
439 			dev_err(smmu->dev,
440 				"Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
441 				(u16)gfsynr1);
442 		else
443 			dev_err(smmu->dev,
444 				"Unexpected global fault, this could be serious\n");
445 		dev_err(smmu->dev,
446 			"\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
447 			gfsr, gfsynr0, gfsynr1, gfsynr2);
448 	}
449 
450 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
451 	return IRQ_HANDLED;
452 }
453 
454 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
455 				       struct io_pgtable_cfg *pgtbl_cfg)
456 {
457 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
458 	struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
459 	bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
460 
461 	cb->cfg = cfg;
462 
463 	/* TCR */
464 	if (stage1) {
465 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
466 			cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
467 		} else {
468 			cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
469 			cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
470 			if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
471 				cb->tcr[1] |= ARM_SMMU_TCR2_AS;
472 			else
473 				cb->tcr[0] |= ARM_SMMU_TCR_EAE;
474 		}
475 	} else {
476 		cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
477 	}
478 
479 	/* TTBRs */
480 	if (stage1) {
481 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
482 			cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
483 			cb->ttbr[1] = 0;
484 		} else {
485 			cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
486 						 cfg->asid);
487 			cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
488 						 cfg->asid);
489 
490 			if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
491 				cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
492 			else
493 				cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
494 		}
495 	} else {
496 		cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
497 	}
498 
499 	/* MAIRs (stage-1 only) */
500 	if (stage1) {
501 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
502 			cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
503 			cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
504 		} else {
505 			cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
506 			cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
507 		}
508 	}
509 }
510 
511 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
512 {
513 	u32 reg;
514 	bool stage1;
515 	struct arm_smmu_cb *cb = &smmu->cbs[idx];
516 	struct arm_smmu_cfg *cfg = cb->cfg;
517 
518 	/* Unassigned context banks only need disabling */
519 	if (!cfg) {
520 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
521 		return;
522 	}
523 
524 	stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
525 
526 	/* CBA2R */
527 	if (smmu->version > ARM_SMMU_V1) {
528 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
529 			reg = ARM_SMMU_CBA2R_VA64;
530 		else
531 			reg = 0;
532 		/* 16-bit VMIDs live in CBA2R */
533 		if (smmu->features & ARM_SMMU_FEAT_VMID16)
534 			reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
535 
536 		arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
537 	}
538 
539 	/* CBAR */
540 	reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
541 	if (smmu->version < ARM_SMMU_V2)
542 		reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
543 
544 	/*
545 	 * Use the weakest shareability/memory types, so they are
546 	 * overridden by the ttbcr/pte.
547 	 */
548 	if (stage1) {
549 		reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
550 				  ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
551 		       FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
552 				  ARM_SMMU_CBAR_S1_MEMATTR_WB);
553 	} else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
554 		/* 8-bit VMIDs live in CBAR */
555 		reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
556 	}
557 	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
558 
559 	/*
560 	 * TCR
561 	 * We must write this before the TTBRs, since it determines the
562 	 * access behaviour of some fields (in particular, ASID[15:8]).
563 	 */
564 	if (stage1 && smmu->version > ARM_SMMU_V1)
565 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
566 	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
567 
568 	/* TTBRs */
569 	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
570 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
571 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
572 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
573 	} else {
574 		arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
575 		if (stage1)
576 			arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
577 					   cb->ttbr[1]);
578 	}
579 
580 	/* MAIRs (stage-1 only) */
581 	if (stage1) {
582 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
583 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
584 	}
585 
586 	/* SCTLR */
587 	reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
588 	      ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
589 	if (stage1)
590 		reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
591 	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
592 		reg |= ARM_SMMU_SCTLR_E;
593 
594 	if (smmu->impl && smmu->impl->write_sctlr)
595 		smmu->impl->write_sctlr(smmu, idx, reg);
596 	else
597 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
598 }
599 
600 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
601 				       struct arm_smmu_device *smmu,
602 				       struct device *dev, unsigned int start)
603 {
604 	if (smmu->impl && smmu->impl->alloc_context_bank)
605 		return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
606 
607 	return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
608 }
609 
610 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
611 					struct arm_smmu_device *smmu,
612 					struct device *dev)
613 {
614 	int irq, start, ret = 0;
615 	unsigned long ias, oas;
616 	struct io_pgtable_ops *pgtbl_ops;
617 	struct io_pgtable_cfg pgtbl_cfg;
618 	enum io_pgtable_fmt fmt;
619 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
620 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
621 	irqreturn_t (*context_fault)(int irq, void *dev);
622 
623 	mutex_lock(&smmu_domain->init_mutex);
624 	if (smmu_domain->smmu)
625 		goto out_unlock;
626 
627 	if (domain->type == IOMMU_DOMAIN_IDENTITY) {
628 		smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
629 		smmu_domain->smmu = smmu;
630 		goto out_unlock;
631 	}
632 
633 	/*
634 	 * Mapping the requested stage onto what we support is surprisingly
635 	 * complicated, mainly because the spec allows S1+S2 SMMUs without
636 	 * support for nested translation. That means we end up with the
637 	 * following table:
638 	 *
639 	 * Requested        Supported        Actual
640 	 *     S1               N              S1
641 	 *     S1             S1+S2            S1
642 	 *     S1               S2             S2
643 	 *     S1               S1             S1
644 	 *     N                N              N
645 	 *     N              S1+S2            S2
646 	 *     N                S2             S2
647 	 *     N                S1             S1
648 	 *
649 	 * Note that you can't actually request stage-2 mappings.
650 	 */
651 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
652 		smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
653 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
654 		smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
655 
656 	/*
657 	 * Choosing a suitable context format is even more fiddly. Until we
658 	 * grow some way for the caller to express a preference, and/or move
659 	 * the decision into the io-pgtable code where it arguably belongs,
660 	 * just aim for the closest thing to the rest of the system, and hope
661 	 * that the hardware isn't esoteric enough that we can't assume AArch64
662 	 * support to be a superset of AArch32 support...
663 	 */
664 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
665 		cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
666 	if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
667 	    !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
668 	    (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
669 	    (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
670 		cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
671 	if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
672 	    (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
673 			       ARM_SMMU_FEAT_FMT_AARCH64_16K |
674 			       ARM_SMMU_FEAT_FMT_AARCH64_4K)))
675 		cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
676 
677 	if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
678 		ret = -EINVAL;
679 		goto out_unlock;
680 	}
681 
682 	switch (smmu_domain->stage) {
683 	case ARM_SMMU_DOMAIN_S1:
684 		cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
685 		start = smmu->num_s2_context_banks;
686 		ias = smmu->va_size;
687 		oas = smmu->ipa_size;
688 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
689 			fmt = ARM_64_LPAE_S1;
690 		} else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
691 			fmt = ARM_32_LPAE_S1;
692 			ias = min(ias, 32UL);
693 			oas = min(oas, 40UL);
694 		} else {
695 			fmt = ARM_V7S;
696 			ias = min(ias, 32UL);
697 			oas = min(oas, 32UL);
698 		}
699 		smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
700 		break;
701 	case ARM_SMMU_DOMAIN_NESTED:
702 		/*
703 		 * We will likely want to change this if/when KVM gets
704 		 * involved.
705 		 */
706 	case ARM_SMMU_DOMAIN_S2:
707 		cfg->cbar = CBAR_TYPE_S2_TRANS;
708 		start = 0;
709 		ias = smmu->ipa_size;
710 		oas = smmu->pa_size;
711 		if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
712 			fmt = ARM_64_LPAE_S2;
713 		} else {
714 			fmt = ARM_32_LPAE_S2;
715 			ias = min(ias, 40UL);
716 			oas = min(oas, 40UL);
717 		}
718 		if (smmu->version == ARM_SMMU_V2)
719 			smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
720 		else
721 			smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
722 		break;
723 	default:
724 		ret = -EINVAL;
725 		goto out_unlock;
726 	}
727 
728 	ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
729 	if (ret < 0) {
730 		goto out_unlock;
731 	}
732 
733 	smmu_domain->smmu = smmu;
734 
735 	cfg->cbndx = ret;
736 	if (smmu->version < ARM_SMMU_V2) {
737 		cfg->irptndx = atomic_inc_return(&smmu->irptndx);
738 		cfg->irptndx %= smmu->num_context_irqs;
739 	} else {
740 		cfg->irptndx = cfg->cbndx;
741 	}
742 
743 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
744 		cfg->vmid = cfg->cbndx + 1;
745 	else
746 		cfg->asid = cfg->cbndx;
747 
748 	pgtbl_cfg = (struct io_pgtable_cfg) {
749 		.pgsize_bitmap	= smmu->pgsize_bitmap,
750 		.ias		= ias,
751 		.oas		= oas,
752 		.coherent_walk	= smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
753 		.tlb		= smmu_domain->flush_ops,
754 		.iommu_dev	= smmu->dev,
755 	};
756 
757 	if (smmu->impl && smmu->impl->init_context) {
758 		ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
759 		if (ret)
760 			goto out_clear_smmu;
761 	}
762 
763 	if (smmu_domain->pgtbl_quirks)
764 		pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
765 
766 	pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
767 	if (!pgtbl_ops) {
768 		ret = -ENOMEM;
769 		goto out_clear_smmu;
770 	}
771 
772 	/* Update the domain's page sizes to reflect the page table format */
773 	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
774 
775 	if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
776 		domain->geometry.aperture_start = ~0UL << ias;
777 		domain->geometry.aperture_end = ~0UL;
778 	} else {
779 		domain->geometry.aperture_end = (1UL << ias) - 1;
780 	}
781 
782 	domain->geometry.force_aperture = true;
783 
784 	/* Initialise the context bank with our page table cfg */
785 	arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
786 	arm_smmu_write_context_bank(smmu, cfg->cbndx);
787 
788 	/*
789 	 * Request context fault interrupt. Do this last to avoid the
790 	 * handler seeing a half-initialised domain state.
791 	 */
792 	irq = smmu->irqs[cfg->irptndx];
793 
794 	if (smmu->impl && smmu->impl->context_fault)
795 		context_fault = smmu->impl->context_fault;
796 	else
797 		context_fault = arm_smmu_context_fault;
798 
799 	ret = devm_request_irq(smmu->dev, irq, context_fault,
800 			       IRQF_SHARED, "arm-smmu-context-fault", domain);
801 	if (ret < 0) {
802 		dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
803 			cfg->irptndx, irq);
804 		cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
805 	}
806 
807 	mutex_unlock(&smmu_domain->init_mutex);
808 
809 	/* Publish page table ops for map/unmap */
810 	smmu_domain->pgtbl_ops = pgtbl_ops;
811 	return 0;
812 
813 out_clear_smmu:
814 	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
815 	smmu_domain->smmu = NULL;
816 out_unlock:
817 	mutex_unlock(&smmu_domain->init_mutex);
818 	return ret;
819 }
820 
821 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
822 {
823 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
824 	struct arm_smmu_device *smmu = smmu_domain->smmu;
825 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
826 	int ret, irq;
827 
828 	if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
829 		return;
830 
831 	ret = arm_smmu_rpm_get(smmu);
832 	if (ret < 0)
833 		return;
834 
835 	/*
836 	 * Disable the context bank and free the page tables before freeing
837 	 * it.
838 	 */
839 	smmu->cbs[cfg->cbndx].cfg = NULL;
840 	arm_smmu_write_context_bank(smmu, cfg->cbndx);
841 
842 	if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
843 		irq = smmu->irqs[cfg->irptndx];
844 		devm_free_irq(smmu->dev, irq, domain);
845 	}
846 
847 	free_io_pgtable_ops(smmu_domain->pgtbl_ops);
848 	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
849 
850 	arm_smmu_rpm_put(smmu);
851 }
852 
853 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
854 {
855 	struct arm_smmu_domain *smmu_domain;
856 
857 	if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) {
858 		if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
859 			return NULL;
860 	}
861 	/*
862 	 * Allocate the domain and initialise some of its data structures.
863 	 * We can't really do anything meaningful until we've added a
864 	 * master.
865 	 */
866 	smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
867 	if (!smmu_domain)
868 		return NULL;
869 
870 	mutex_init(&smmu_domain->init_mutex);
871 	spin_lock_init(&smmu_domain->cb_lock);
872 
873 	return &smmu_domain->domain;
874 }
875 
876 static void arm_smmu_domain_free(struct iommu_domain *domain)
877 {
878 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
879 
880 	/*
881 	 * Free the domain resources. We assume that all devices have
882 	 * already been detached.
883 	 */
884 	arm_smmu_destroy_domain_context(domain);
885 	kfree(smmu_domain);
886 }
887 
888 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
889 {
890 	struct arm_smmu_smr *smr = smmu->smrs + idx;
891 	u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
892 		  FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
893 
894 	if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
895 		reg |= ARM_SMMU_SMR_VALID;
896 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
897 }
898 
899 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
900 {
901 	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
902 	u32 reg;
903 
904 	if (smmu->impl && smmu->impl->write_s2cr) {
905 		smmu->impl->write_s2cr(smmu, idx);
906 		return;
907 	}
908 
909 	reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
910 	      FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
911 	      FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
912 
913 	if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
914 	    smmu->smrs[idx].valid)
915 		reg |= ARM_SMMU_S2CR_EXIDVALID;
916 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
917 }
918 
919 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
920 {
921 	arm_smmu_write_s2cr(smmu, idx);
922 	if (smmu->smrs)
923 		arm_smmu_write_smr(smmu, idx);
924 }
925 
926 /*
927  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
928  * should be called after sCR0 is written.
929  */
930 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
931 {
932 	u32 smr;
933 	int i;
934 
935 	if (!smmu->smrs)
936 		return;
937 	/*
938 	 * If we've had to accommodate firmware memory regions, we may
939 	 * have live SMRs by now; tread carefully...
940 	 *
941 	 * Somewhat perversely, not having a free SMR for this test implies we
942 	 * can get away without it anyway, as we'll only be able to 'allocate'
943 	 * these SMRs for the ID/mask values we're already trusting to be OK.
944 	 */
945 	for (i = 0; i < smmu->num_mapping_groups; i++)
946 		if (!smmu->smrs[i].valid)
947 			goto smr_ok;
948 	return;
949 smr_ok:
950 	/*
951 	 * SMR.ID bits may not be preserved if the corresponding MASK
952 	 * bits are set, so check each one separately. We can reject
953 	 * masters later if they try to claim IDs outside these masks.
954 	 */
955 	smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
956 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
957 	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
958 	smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
959 
960 	smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
961 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
962 	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
963 	smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
964 }
965 
966 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
967 {
968 	struct arm_smmu_smr *smrs = smmu->smrs;
969 	int i, free_idx = -ENOSPC;
970 
971 	/* Stream indexing is blissfully easy */
972 	if (!smrs)
973 		return id;
974 
975 	/* Validating SMRs is... less so */
976 	for (i = 0; i < smmu->num_mapping_groups; ++i) {
977 		if (!smrs[i].valid) {
978 			/*
979 			 * Note the first free entry we come across, which
980 			 * we'll claim in the end if nothing else matches.
981 			 */
982 			if (free_idx < 0)
983 				free_idx = i;
984 			continue;
985 		}
986 		/*
987 		 * If the new entry is _entirely_ matched by an existing entry,
988 		 * then reuse that, with the guarantee that there also cannot
989 		 * be any subsequent conflicting entries. In normal use we'd
990 		 * expect simply identical entries for this case, but there's
991 		 * no harm in accommodating the generalisation.
992 		 */
993 		if ((mask & smrs[i].mask) == mask &&
994 		    !((id ^ smrs[i].id) & ~smrs[i].mask))
995 			return i;
996 		/*
997 		 * If the new entry has any other overlap with an existing one,
998 		 * though, then there always exists at least one stream ID
999 		 * which would cause a conflict, and we can't allow that risk.
1000 		 */
1001 		if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1002 			return -EINVAL;
1003 	}
1004 
1005 	return free_idx;
1006 }
1007 
1008 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1009 {
1010 	if (--smmu->s2crs[idx].count)
1011 		return false;
1012 
1013 	smmu->s2crs[idx] = s2cr_init_val;
1014 	if (smmu->smrs)
1015 		smmu->smrs[idx].valid = false;
1016 
1017 	return true;
1018 }
1019 
1020 static int arm_smmu_master_alloc_smes(struct device *dev)
1021 {
1022 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1023 	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1024 	struct arm_smmu_device *smmu = cfg->smmu;
1025 	struct arm_smmu_smr *smrs = smmu->smrs;
1026 	int i, idx, ret;
1027 
1028 	mutex_lock(&smmu->stream_map_mutex);
1029 	/* Figure out a viable stream map entry allocation */
1030 	for_each_cfg_sme(cfg, fwspec, i, idx) {
1031 		u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1032 		u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1033 
1034 		if (idx != INVALID_SMENDX) {
1035 			ret = -EEXIST;
1036 			goto out_err;
1037 		}
1038 
1039 		ret = arm_smmu_find_sme(smmu, sid, mask);
1040 		if (ret < 0)
1041 			goto out_err;
1042 
1043 		idx = ret;
1044 		if (smrs && smmu->s2crs[idx].count == 0) {
1045 			smrs[idx].id = sid;
1046 			smrs[idx].mask = mask;
1047 			smrs[idx].valid = true;
1048 		}
1049 		smmu->s2crs[idx].count++;
1050 		cfg->smendx[i] = (s16)idx;
1051 	}
1052 
1053 	/* It worked! Now, poke the actual hardware */
1054 	for_each_cfg_sme(cfg, fwspec, i, idx)
1055 		arm_smmu_write_sme(smmu, idx);
1056 
1057 	mutex_unlock(&smmu->stream_map_mutex);
1058 	return 0;
1059 
1060 out_err:
1061 	while (i--) {
1062 		arm_smmu_free_sme(smmu, cfg->smendx[i]);
1063 		cfg->smendx[i] = INVALID_SMENDX;
1064 	}
1065 	mutex_unlock(&smmu->stream_map_mutex);
1066 	return ret;
1067 }
1068 
1069 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1070 				      struct iommu_fwspec *fwspec)
1071 {
1072 	struct arm_smmu_device *smmu = cfg->smmu;
1073 	int i, idx;
1074 
1075 	mutex_lock(&smmu->stream_map_mutex);
1076 	for_each_cfg_sme(cfg, fwspec, i, idx) {
1077 		if (arm_smmu_free_sme(smmu, idx))
1078 			arm_smmu_write_sme(smmu, idx);
1079 		cfg->smendx[i] = INVALID_SMENDX;
1080 	}
1081 	mutex_unlock(&smmu->stream_map_mutex);
1082 }
1083 
1084 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1085 				      struct arm_smmu_master_cfg *cfg,
1086 				      struct iommu_fwspec *fwspec)
1087 {
1088 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1089 	struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1090 	u8 cbndx = smmu_domain->cfg.cbndx;
1091 	enum arm_smmu_s2cr_type type;
1092 	int i, idx;
1093 
1094 	if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1095 		type = S2CR_TYPE_BYPASS;
1096 	else
1097 		type = S2CR_TYPE_TRANS;
1098 
1099 	for_each_cfg_sme(cfg, fwspec, i, idx) {
1100 		if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1101 			continue;
1102 
1103 		s2cr[idx].type = type;
1104 		s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1105 		s2cr[idx].cbndx = cbndx;
1106 		arm_smmu_write_s2cr(smmu, idx);
1107 	}
1108 	return 0;
1109 }
1110 
1111 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1112 {
1113 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1114 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1115 	struct arm_smmu_master_cfg *cfg;
1116 	struct arm_smmu_device *smmu;
1117 	int ret;
1118 
1119 	if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1120 		dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1121 		return -ENXIO;
1122 	}
1123 
1124 	/*
1125 	 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1126 	 * domains between of_xlate() and probe_device() - we have no way to cope
1127 	 * with that, so until ARM gets converted to rely on groups and default
1128 	 * domains, just say no (but more politely than by dereferencing NULL).
1129 	 * This should be at least a WARN_ON once that's sorted.
1130 	 */
1131 	cfg = dev_iommu_priv_get(dev);
1132 	if (!cfg)
1133 		return -ENODEV;
1134 
1135 	smmu = cfg->smmu;
1136 
1137 	ret = arm_smmu_rpm_get(smmu);
1138 	if (ret < 0)
1139 		return ret;
1140 
1141 	/* Ensure that the domain is finalised */
1142 	ret = arm_smmu_init_domain_context(domain, smmu, dev);
1143 	if (ret < 0)
1144 		goto rpm_put;
1145 
1146 	/*
1147 	 * Sanity check the domain. We don't support domains across
1148 	 * different SMMUs.
1149 	 */
1150 	if (smmu_domain->smmu != smmu) {
1151 		ret = -EINVAL;
1152 		goto rpm_put;
1153 	}
1154 
1155 	/* Looks ok, so add the device to the domain */
1156 	ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1157 
1158 	/*
1159 	 * Setup an autosuspend delay to avoid bouncing runpm state.
1160 	 * Otherwise, if a driver for a suspended consumer device
1161 	 * unmaps buffers, it will runpm resume/suspend for each one.
1162 	 *
1163 	 * For example, when used by a GPU device, when an application
1164 	 * or game exits, it can trigger unmapping 100s or 1000s of
1165 	 * buffers.  With a runpm cycle for each buffer, that adds up
1166 	 * to 5-10sec worth of reprogramming the context bank, while
1167 	 * the system appears to be locked up to the user.
1168 	 */
1169 	pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1170 	pm_runtime_use_autosuspend(smmu->dev);
1171 
1172 rpm_put:
1173 	arm_smmu_rpm_put(smmu);
1174 	return ret;
1175 }
1176 
1177 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1178 			      phys_addr_t paddr, size_t pgsize, size_t pgcount,
1179 			      int prot, gfp_t gfp, size_t *mapped)
1180 {
1181 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1182 	struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1183 	int ret;
1184 
1185 	if (!ops)
1186 		return -ENODEV;
1187 
1188 	arm_smmu_rpm_get(smmu);
1189 	ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1190 	arm_smmu_rpm_put(smmu);
1191 
1192 	return ret;
1193 }
1194 
1195 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1196 				   size_t pgsize, size_t pgcount,
1197 				   struct iommu_iotlb_gather *iotlb_gather)
1198 {
1199 	struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1200 	struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1201 	size_t ret;
1202 
1203 	if (!ops)
1204 		return 0;
1205 
1206 	arm_smmu_rpm_get(smmu);
1207 	ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1208 	arm_smmu_rpm_put(smmu);
1209 
1210 	return ret;
1211 }
1212 
1213 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1214 {
1215 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1216 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1217 
1218 	if (smmu_domain->flush_ops) {
1219 		arm_smmu_rpm_get(smmu);
1220 		smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1221 		arm_smmu_rpm_put(smmu);
1222 	}
1223 }
1224 
1225 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1226 				struct iommu_iotlb_gather *gather)
1227 {
1228 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1229 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1230 
1231 	if (!smmu)
1232 		return;
1233 
1234 	arm_smmu_rpm_get(smmu);
1235 	if (smmu->version == ARM_SMMU_V2 ||
1236 	    smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1237 		arm_smmu_tlb_sync_context(smmu_domain);
1238 	else
1239 		arm_smmu_tlb_sync_global(smmu);
1240 	arm_smmu_rpm_put(smmu);
1241 }
1242 
1243 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1244 					      dma_addr_t iova)
1245 {
1246 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1247 	struct arm_smmu_device *smmu = smmu_domain->smmu;
1248 	struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1249 	struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1250 	struct device *dev = smmu->dev;
1251 	void __iomem *reg;
1252 	u32 tmp;
1253 	u64 phys;
1254 	unsigned long va, flags;
1255 	int ret, idx = cfg->cbndx;
1256 	phys_addr_t addr = 0;
1257 
1258 	ret = arm_smmu_rpm_get(smmu);
1259 	if (ret < 0)
1260 		return 0;
1261 
1262 	spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1263 	va = iova & ~0xfffUL;
1264 	if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1265 		arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1266 	else
1267 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1268 
1269 	reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1270 	if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1271 				      5, 50)) {
1272 		spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1273 		dev_err(dev,
1274 			"iova to phys timed out on %pad. Falling back to software table walk.\n",
1275 			&iova);
1276 		arm_smmu_rpm_put(smmu);
1277 		return ops->iova_to_phys(ops, iova);
1278 	}
1279 
1280 	phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1281 	spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1282 	if (phys & ARM_SMMU_CB_PAR_F) {
1283 		dev_err(dev, "translation fault!\n");
1284 		dev_err(dev, "PAR = 0x%llx\n", phys);
1285 		goto out;
1286 	}
1287 
1288 	addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1289 out:
1290 	arm_smmu_rpm_put(smmu);
1291 
1292 	return addr;
1293 }
1294 
1295 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1296 					dma_addr_t iova)
1297 {
1298 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1299 	struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1300 
1301 	if (!ops)
1302 		return 0;
1303 
1304 	if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1305 			smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1306 		return arm_smmu_iova_to_phys_hard(domain, iova);
1307 
1308 	return ops->iova_to_phys(ops, iova);
1309 }
1310 
1311 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1312 {
1313 	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1314 
1315 	switch (cap) {
1316 	case IOMMU_CAP_CACHE_COHERENCY:
1317 		/*
1318 		 * It's overwhelmingly the case in practice that when the pagetable
1319 		 * walk interface is connected to a coherent interconnect, all the
1320 		 * translation interfaces are too. Furthermore if the device is
1321 		 * natively coherent, then its translation interface must also be.
1322 		 */
1323 		return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1324 			device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1325 	case IOMMU_CAP_NOEXEC:
1326 	case IOMMU_CAP_DEFERRED_FLUSH:
1327 		return true;
1328 	default:
1329 		return false;
1330 	}
1331 }
1332 
1333 static
1334 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1335 {
1336 	struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1337 							  fwnode);
1338 	put_device(dev);
1339 	return dev ? dev_get_drvdata(dev) : NULL;
1340 }
1341 
1342 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1343 {
1344 	struct arm_smmu_device *smmu = NULL;
1345 	struct arm_smmu_master_cfg *cfg;
1346 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1347 	int i, ret;
1348 
1349 	if (using_legacy_binding) {
1350 		ret = arm_smmu_register_legacy_master(dev, &smmu);
1351 
1352 		/*
1353 		 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1354 		 * will allocate/initialise a new one. Thus we need to update fwspec for
1355 		 * later use.
1356 		 */
1357 		fwspec = dev_iommu_fwspec_get(dev);
1358 		if (ret)
1359 			goto out_free;
1360 	} else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1361 		smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1362 	} else {
1363 		return ERR_PTR(-ENODEV);
1364 	}
1365 
1366 	ret = -EINVAL;
1367 	for (i = 0; i < fwspec->num_ids; i++) {
1368 		u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1369 		u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1370 
1371 		if (sid & ~smmu->streamid_mask) {
1372 			dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1373 				sid, smmu->streamid_mask);
1374 			goto out_free;
1375 		}
1376 		if (mask & ~smmu->smr_mask_mask) {
1377 			dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1378 				mask, smmu->smr_mask_mask);
1379 			goto out_free;
1380 		}
1381 	}
1382 
1383 	ret = -ENOMEM;
1384 	cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1385 		      GFP_KERNEL);
1386 	if (!cfg)
1387 		goto out_free;
1388 
1389 	cfg->smmu = smmu;
1390 	dev_iommu_priv_set(dev, cfg);
1391 	while (i--)
1392 		cfg->smendx[i] = INVALID_SMENDX;
1393 
1394 	ret = arm_smmu_rpm_get(smmu);
1395 	if (ret < 0)
1396 		goto out_cfg_free;
1397 
1398 	ret = arm_smmu_master_alloc_smes(dev);
1399 	arm_smmu_rpm_put(smmu);
1400 
1401 	if (ret)
1402 		goto out_cfg_free;
1403 
1404 	device_link_add(dev, smmu->dev,
1405 			DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1406 
1407 	return &smmu->iommu;
1408 
1409 out_cfg_free:
1410 	kfree(cfg);
1411 out_free:
1412 	iommu_fwspec_free(dev);
1413 	return ERR_PTR(ret);
1414 }
1415 
1416 static void arm_smmu_release_device(struct device *dev)
1417 {
1418 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1419 	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1420 	int ret;
1421 
1422 	ret = arm_smmu_rpm_get(cfg->smmu);
1423 	if (ret < 0)
1424 		return;
1425 
1426 	arm_smmu_master_free_smes(cfg, fwspec);
1427 
1428 	arm_smmu_rpm_put(cfg->smmu);
1429 
1430 	dev_iommu_priv_set(dev, NULL);
1431 	kfree(cfg);
1432 }
1433 
1434 static void arm_smmu_probe_finalize(struct device *dev)
1435 {
1436 	struct arm_smmu_master_cfg *cfg;
1437 	struct arm_smmu_device *smmu;
1438 
1439 	cfg = dev_iommu_priv_get(dev);
1440 	smmu = cfg->smmu;
1441 
1442 	if (smmu->impl && smmu->impl->probe_finalize)
1443 		smmu->impl->probe_finalize(smmu, dev);
1444 }
1445 
1446 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1447 {
1448 	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1449 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1450 	struct arm_smmu_device *smmu = cfg->smmu;
1451 	struct iommu_group *group = NULL;
1452 	int i, idx;
1453 
1454 	mutex_lock(&smmu->stream_map_mutex);
1455 	for_each_cfg_sme(cfg, fwspec, i, idx) {
1456 		if (group && smmu->s2crs[idx].group &&
1457 		    group != smmu->s2crs[idx].group) {
1458 			mutex_unlock(&smmu->stream_map_mutex);
1459 			return ERR_PTR(-EINVAL);
1460 		}
1461 
1462 		group = smmu->s2crs[idx].group;
1463 	}
1464 
1465 	if (group) {
1466 		mutex_unlock(&smmu->stream_map_mutex);
1467 		return iommu_group_ref_get(group);
1468 	}
1469 
1470 	if (dev_is_pci(dev))
1471 		group = pci_device_group(dev);
1472 	else if (dev_is_fsl_mc(dev))
1473 		group = fsl_mc_device_group(dev);
1474 	else
1475 		group = generic_device_group(dev);
1476 
1477 	/* Remember group for faster lookups */
1478 	if (!IS_ERR(group))
1479 		for_each_cfg_sme(cfg, fwspec, i, idx)
1480 			smmu->s2crs[idx].group = group;
1481 
1482 	mutex_unlock(&smmu->stream_map_mutex);
1483 	return group;
1484 }
1485 
1486 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1487 {
1488 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1489 	int ret = 0;
1490 
1491 	mutex_lock(&smmu_domain->init_mutex);
1492 	if (smmu_domain->smmu)
1493 		ret = -EPERM;
1494 	else
1495 		smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1496 	mutex_unlock(&smmu_domain->init_mutex);
1497 
1498 	return ret;
1499 }
1500 
1501 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1502 		unsigned long quirks)
1503 {
1504 	struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1505 	int ret = 0;
1506 
1507 	mutex_lock(&smmu_domain->init_mutex);
1508 	if (smmu_domain->smmu)
1509 		ret = -EPERM;
1510 	else
1511 		smmu_domain->pgtbl_quirks = quirks;
1512 	mutex_unlock(&smmu_domain->init_mutex);
1513 
1514 	return ret;
1515 }
1516 
1517 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1518 {
1519 	u32 mask, fwid = 0;
1520 
1521 	if (args->args_count > 0)
1522 		fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1523 
1524 	if (args->args_count > 1)
1525 		fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1526 	else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1527 		fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1528 
1529 	return iommu_fwspec_add_ids(dev, &fwid, 1);
1530 }
1531 
1532 static void arm_smmu_get_resv_regions(struct device *dev,
1533 				      struct list_head *head)
1534 {
1535 	struct iommu_resv_region *region;
1536 	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1537 
1538 	region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1539 					 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1540 	if (!region)
1541 		return;
1542 
1543 	list_add_tail(&region->list, head);
1544 
1545 	iommu_dma_get_resv_regions(dev, head);
1546 }
1547 
1548 static int arm_smmu_def_domain_type(struct device *dev)
1549 {
1550 	struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1551 	const struct arm_smmu_impl *impl = cfg->smmu->impl;
1552 
1553 	if (using_legacy_binding)
1554 		return IOMMU_DOMAIN_IDENTITY;
1555 
1556 	if (impl && impl->def_domain_type)
1557 		return impl->def_domain_type(dev);
1558 
1559 	return 0;
1560 }
1561 
1562 static struct iommu_ops arm_smmu_ops = {
1563 	.capable		= arm_smmu_capable,
1564 	.domain_alloc		= arm_smmu_domain_alloc,
1565 	.probe_device		= arm_smmu_probe_device,
1566 	.release_device		= arm_smmu_release_device,
1567 	.probe_finalize		= arm_smmu_probe_finalize,
1568 	.device_group		= arm_smmu_device_group,
1569 	.of_xlate		= arm_smmu_of_xlate,
1570 	.get_resv_regions	= arm_smmu_get_resv_regions,
1571 	.def_domain_type	= arm_smmu_def_domain_type,
1572 	.pgsize_bitmap		= -1UL, /* Restricted during device attach */
1573 	.owner			= THIS_MODULE,
1574 	.default_domain_ops = &(const struct iommu_domain_ops) {
1575 		.attach_dev		= arm_smmu_attach_dev,
1576 		.map_pages		= arm_smmu_map_pages,
1577 		.unmap_pages		= arm_smmu_unmap_pages,
1578 		.flush_iotlb_all	= arm_smmu_flush_iotlb_all,
1579 		.iotlb_sync		= arm_smmu_iotlb_sync,
1580 		.iova_to_phys		= arm_smmu_iova_to_phys,
1581 		.enable_nesting		= arm_smmu_enable_nesting,
1582 		.set_pgtable_quirks	= arm_smmu_set_pgtable_quirks,
1583 		.free			= arm_smmu_domain_free,
1584 	}
1585 };
1586 
1587 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1588 {
1589 	int i;
1590 	u32 reg;
1591 
1592 	/* clear global FSR */
1593 	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1594 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1595 
1596 	/*
1597 	 * Reset stream mapping groups: Initial values mark all SMRn as
1598 	 * invalid and all S2CRn as bypass unless overridden.
1599 	 */
1600 	for (i = 0; i < smmu->num_mapping_groups; ++i)
1601 		arm_smmu_write_sme(smmu, i);
1602 
1603 	/* Make sure all context banks are disabled and clear CB_FSR  */
1604 	for (i = 0; i < smmu->num_context_banks; ++i) {
1605 		arm_smmu_write_context_bank(smmu, i);
1606 		arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1607 	}
1608 
1609 	/* Invalidate the TLB, just in case */
1610 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1611 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1612 
1613 	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1614 
1615 	/* Enable fault reporting */
1616 	reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1617 		ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1618 
1619 	/* Disable TLB broadcasting. */
1620 	reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1621 
1622 	/* Enable client access, handling unmatched streams as appropriate */
1623 	reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1624 	if (disable_bypass)
1625 		reg |= ARM_SMMU_sCR0_USFCFG;
1626 	else
1627 		reg &= ~ARM_SMMU_sCR0_USFCFG;
1628 
1629 	/* Disable forced broadcasting */
1630 	reg &= ~ARM_SMMU_sCR0_FB;
1631 
1632 	/* Don't upgrade barriers */
1633 	reg &= ~(ARM_SMMU_sCR0_BSU);
1634 
1635 	if (smmu->features & ARM_SMMU_FEAT_VMID16)
1636 		reg |= ARM_SMMU_sCR0_VMID16EN;
1637 
1638 	if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1639 		reg |= ARM_SMMU_sCR0_EXIDENABLE;
1640 
1641 	if (smmu->impl && smmu->impl->reset)
1642 		smmu->impl->reset(smmu);
1643 
1644 	/* Push the button */
1645 	arm_smmu_tlb_sync_global(smmu);
1646 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1647 }
1648 
1649 static int arm_smmu_id_size_to_bits(int size)
1650 {
1651 	switch (size) {
1652 	case 0:
1653 		return 32;
1654 	case 1:
1655 		return 36;
1656 	case 2:
1657 		return 40;
1658 	case 3:
1659 		return 42;
1660 	case 4:
1661 		return 44;
1662 	case 5:
1663 	default:
1664 		return 48;
1665 	}
1666 }
1667 
1668 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1669 {
1670 	unsigned int size;
1671 	u32 id;
1672 	bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1673 	int i, ret;
1674 
1675 	dev_notice(smmu->dev, "probing hardware configuration...\n");
1676 	dev_notice(smmu->dev, "SMMUv%d with:\n",
1677 			smmu->version == ARM_SMMU_V2 ? 2 : 1);
1678 
1679 	/* ID0 */
1680 	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1681 
1682 	/* Restrict available stages based on module parameter */
1683 	if (force_stage == 1)
1684 		id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1685 	else if (force_stage == 2)
1686 		id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1687 
1688 	if (id & ARM_SMMU_ID0_S1TS) {
1689 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1690 		dev_notice(smmu->dev, "\tstage 1 translation\n");
1691 	}
1692 
1693 	if (id & ARM_SMMU_ID0_S2TS) {
1694 		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1695 		dev_notice(smmu->dev, "\tstage 2 translation\n");
1696 	}
1697 
1698 	if (id & ARM_SMMU_ID0_NTS) {
1699 		smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1700 		dev_notice(smmu->dev, "\tnested translation\n");
1701 	}
1702 
1703 	if (!(smmu->features &
1704 		(ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1705 		dev_err(smmu->dev, "\tno translation support!\n");
1706 		return -ENODEV;
1707 	}
1708 
1709 	if ((id & ARM_SMMU_ID0_S1TS) &&
1710 	    ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1711 		smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1712 		dev_notice(smmu->dev, "\taddress translation ops\n");
1713 	}
1714 
1715 	/*
1716 	 * In order for DMA API calls to work properly, we must defer to what
1717 	 * the FW says about coherency, regardless of what the hardware claims.
1718 	 * Fortunately, this also opens up a workaround for systems where the
1719 	 * ID register value has ended up configured incorrectly.
1720 	 */
1721 	cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1722 	if (cttw_fw || cttw_reg)
1723 		dev_notice(smmu->dev, "\t%scoherent table walk\n",
1724 			   cttw_fw ? "" : "non-");
1725 	if (cttw_fw != cttw_reg)
1726 		dev_notice(smmu->dev,
1727 			   "\t(IDR0.CTTW overridden by FW configuration)\n");
1728 
1729 	/* Max. number of entries we have for stream matching/indexing */
1730 	if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1731 		smmu->features |= ARM_SMMU_FEAT_EXIDS;
1732 		size = 1 << 16;
1733 	} else {
1734 		size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1735 	}
1736 	smmu->streamid_mask = size - 1;
1737 	if (id & ARM_SMMU_ID0_SMS) {
1738 		smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1739 		size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1740 		if (size == 0) {
1741 			dev_err(smmu->dev,
1742 				"stream-matching supported, but no SMRs present!\n");
1743 			return -ENODEV;
1744 		}
1745 
1746 		/* Zero-initialised to mark as invalid */
1747 		smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1748 					  GFP_KERNEL);
1749 		if (!smmu->smrs)
1750 			return -ENOMEM;
1751 
1752 		dev_notice(smmu->dev,
1753 			   "\tstream matching with %u register groups", size);
1754 	}
1755 	/* s2cr->type == 0 means translation, so initialise explicitly */
1756 	smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1757 					 GFP_KERNEL);
1758 	if (!smmu->s2crs)
1759 		return -ENOMEM;
1760 	for (i = 0; i < size; i++)
1761 		smmu->s2crs[i] = s2cr_init_val;
1762 
1763 	smmu->num_mapping_groups = size;
1764 	mutex_init(&smmu->stream_map_mutex);
1765 	spin_lock_init(&smmu->global_sync_lock);
1766 
1767 	if (smmu->version < ARM_SMMU_V2 ||
1768 	    !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1769 		smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1770 		if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1771 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1772 	}
1773 
1774 	/* ID1 */
1775 	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1776 	smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1777 
1778 	/* Check for size mismatch of SMMU address space from mapped region */
1779 	size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1780 	if (smmu->numpage != 2 * size << smmu->pgshift)
1781 		dev_warn(smmu->dev,
1782 			"SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1783 			2 * size << smmu->pgshift, smmu->numpage);
1784 	/* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1785 	smmu->numpage = size;
1786 
1787 	smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1788 	smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1789 	if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1790 		dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1791 		return -ENODEV;
1792 	}
1793 	dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1794 		   smmu->num_context_banks, smmu->num_s2_context_banks);
1795 	smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1796 				 sizeof(*smmu->cbs), GFP_KERNEL);
1797 	if (!smmu->cbs)
1798 		return -ENOMEM;
1799 
1800 	/* ID2 */
1801 	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1802 	size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1803 	smmu->ipa_size = size;
1804 
1805 	/* The output mask is also applied for bypass */
1806 	size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1807 	smmu->pa_size = size;
1808 
1809 	if (id & ARM_SMMU_ID2_VMID16)
1810 		smmu->features |= ARM_SMMU_FEAT_VMID16;
1811 
1812 	/*
1813 	 * What the page table walker can address actually depends on which
1814 	 * descriptor format is in use, but since a) we don't know that yet,
1815 	 * and b) it can vary per context bank, this will have to do...
1816 	 */
1817 	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1818 		dev_warn(smmu->dev,
1819 			 "failed to set DMA mask for table walker\n");
1820 
1821 	if (smmu->version < ARM_SMMU_V2) {
1822 		smmu->va_size = smmu->ipa_size;
1823 		if (smmu->version == ARM_SMMU_V1_64K)
1824 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1825 	} else {
1826 		size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1827 		smmu->va_size = arm_smmu_id_size_to_bits(size);
1828 		if (id & ARM_SMMU_ID2_PTFS_4K)
1829 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1830 		if (id & ARM_SMMU_ID2_PTFS_16K)
1831 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1832 		if (id & ARM_SMMU_ID2_PTFS_64K)
1833 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1834 	}
1835 
1836 	if (smmu->impl && smmu->impl->cfg_probe) {
1837 		ret = smmu->impl->cfg_probe(smmu);
1838 		if (ret)
1839 			return ret;
1840 	}
1841 
1842 	/* Now we've corralled the various formats, what'll it do? */
1843 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1844 		smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1845 	if (smmu->features &
1846 	    (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1847 		smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1848 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1849 		smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1850 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1851 		smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1852 
1853 	if (arm_smmu_ops.pgsize_bitmap == -1UL)
1854 		arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1855 	else
1856 		arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1857 	dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1858 		   smmu->pgsize_bitmap);
1859 
1860 
1861 	if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1862 		dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1863 			   smmu->va_size, smmu->ipa_size);
1864 
1865 	if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1866 		dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1867 			   smmu->ipa_size, smmu->pa_size);
1868 
1869 	return 0;
1870 }
1871 
1872 struct arm_smmu_match_data {
1873 	enum arm_smmu_arch_version version;
1874 	enum arm_smmu_implementation model;
1875 };
1876 
1877 #define ARM_SMMU_MATCH_DATA(name, ver, imp)	\
1878 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1879 
1880 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1881 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1882 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1883 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1884 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1885 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1886 
1887 static const struct of_device_id arm_smmu_of_match[] = {
1888 	{ .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1889 	{ .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1890 	{ .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1891 	{ .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1892 	{ .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1893 	{ .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1894 	{ .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1895 	{ .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1896 	{ },
1897 };
1898 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1899 
1900 #ifdef CONFIG_ACPI
1901 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1902 {
1903 	int ret = 0;
1904 
1905 	switch (model) {
1906 	case ACPI_IORT_SMMU_V1:
1907 	case ACPI_IORT_SMMU_CORELINK_MMU400:
1908 		smmu->version = ARM_SMMU_V1;
1909 		smmu->model = GENERIC_SMMU;
1910 		break;
1911 	case ACPI_IORT_SMMU_CORELINK_MMU401:
1912 		smmu->version = ARM_SMMU_V1_64K;
1913 		smmu->model = GENERIC_SMMU;
1914 		break;
1915 	case ACPI_IORT_SMMU_V2:
1916 		smmu->version = ARM_SMMU_V2;
1917 		smmu->model = GENERIC_SMMU;
1918 		break;
1919 	case ACPI_IORT_SMMU_CORELINK_MMU500:
1920 		smmu->version = ARM_SMMU_V2;
1921 		smmu->model = ARM_MMU500;
1922 		break;
1923 	case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1924 		smmu->version = ARM_SMMU_V2;
1925 		smmu->model = CAVIUM_SMMUV2;
1926 		break;
1927 	default:
1928 		ret = -ENODEV;
1929 	}
1930 
1931 	return ret;
1932 }
1933 
1934 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1935 				      u32 *global_irqs, u32 *pmu_irqs)
1936 {
1937 	struct device *dev = smmu->dev;
1938 	struct acpi_iort_node *node =
1939 		*(struct acpi_iort_node **)dev_get_platdata(dev);
1940 	struct acpi_iort_smmu *iort_smmu;
1941 	int ret;
1942 
1943 	/* Retrieve SMMU1/2 specific data */
1944 	iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1945 
1946 	ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1947 	if (ret < 0)
1948 		return ret;
1949 
1950 	/* Ignore the configuration access interrupt */
1951 	*global_irqs = 1;
1952 	*pmu_irqs = 0;
1953 
1954 	if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1955 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1956 
1957 	return 0;
1958 }
1959 #else
1960 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1961 					     u32 *global_irqs, u32 *pmu_irqs)
1962 {
1963 	return -ENODEV;
1964 }
1965 #endif
1966 
1967 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
1968 				    u32 *global_irqs, u32 *pmu_irqs)
1969 {
1970 	const struct arm_smmu_match_data *data;
1971 	struct device *dev = smmu->dev;
1972 	bool legacy_binding;
1973 
1974 	if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
1975 		return dev_err_probe(dev, -ENODEV,
1976 				     "missing #global-interrupts property\n");
1977 	*pmu_irqs = 0;
1978 
1979 	data = of_device_get_match_data(dev);
1980 	smmu->version = data->version;
1981 	smmu->model = data->model;
1982 
1983 	legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1984 	if (legacy_binding && !using_generic_binding) {
1985 		if (!using_legacy_binding) {
1986 			pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
1987 				  IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
1988 		}
1989 		using_legacy_binding = true;
1990 	} else if (!legacy_binding && !using_legacy_binding) {
1991 		using_generic_binding = true;
1992 	} else {
1993 		dev_err(dev, "not probing due to mismatched DT properties\n");
1994 		return -ENODEV;
1995 	}
1996 
1997 	if (of_dma_is_coherent(dev->of_node))
1998 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1999 
2000 	return 0;
2001 }
2002 
2003 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2004 {
2005 	struct list_head rmr_list;
2006 	struct iommu_resv_region *e;
2007 	int idx, cnt = 0;
2008 	u32 reg;
2009 
2010 	INIT_LIST_HEAD(&rmr_list);
2011 	iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2012 
2013 	/*
2014 	 * Rather than trying to look at existing mappings that
2015 	 * are setup by the firmware and then invalidate the ones
2016 	 * that do no have matching RMR entries, just disable the
2017 	 * SMMU until it gets enabled again in the reset routine.
2018 	 */
2019 	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2020 	reg |= ARM_SMMU_sCR0_CLIENTPD;
2021 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2022 
2023 	list_for_each_entry(e, &rmr_list, list) {
2024 		struct iommu_iort_rmr_data *rmr;
2025 		int i;
2026 
2027 		rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2028 		for (i = 0; i < rmr->num_sids; i++) {
2029 			idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2030 			if (idx < 0)
2031 				continue;
2032 
2033 			if (smmu->s2crs[idx].count == 0) {
2034 				smmu->smrs[idx].id = rmr->sids[i];
2035 				smmu->smrs[idx].mask = 0;
2036 				smmu->smrs[idx].valid = true;
2037 			}
2038 			smmu->s2crs[idx].count++;
2039 			smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2040 			smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2041 
2042 			cnt++;
2043 		}
2044 	}
2045 
2046 	dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2047 		   cnt == 1 ? "" : "s");
2048 	iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2049 }
2050 
2051 static int arm_smmu_device_probe(struct platform_device *pdev)
2052 {
2053 	struct resource *res;
2054 	struct arm_smmu_device *smmu;
2055 	struct device *dev = &pdev->dev;
2056 	int num_irqs, i, err;
2057 	u32 global_irqs, pmu_irqs;
2058 	irqreturn_t (*global_fault)(int irq, void *dev);
2059 
2060 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2061 	if (!smmu) {
2062 		dev_err(dev, "failed to allocate arm_smmu_device\n");
2063 		return -ENOMEM;
2064 	}
2065 	smmu->dev = dev;
2066 
2067 	if (dev->of_node)
2068 		err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2069 	else
2070 		err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2071 	if (err)
2072 		return err;
2073 
2074 	smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2075 	if (IS_ERR(smmu->base))
2076 		return PTR_ERR(smmu->base);
2077 	smmu->ioaddr = res->start;
2078 
2079 	/*
2080 	 * The resource size should effectively match the value of SMMU_TOP;
2081 	 * stash that temporarily until we know PAGESIZE to validate it with.
2082 	 */
2083 	smmu->numpage = resource_size(res);
2084 
2085 	smmu = arm_smmu_impl_init(smmu);
2086 	if (IS_ERR(smmu))
2087 		return PTR_ERR(smmu);
2088 
2089 	num_irqs = platform_irq_count(pdev);
2090 
2091 	smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2092 	if (smmu->num_context_irqs <= 0)
2093 		return dev_err_probe(dev, -ENODEV,
2094 				"found %d interrupts but expected at least %d\n",
2095 				num_irqs, global_irqs + pmu_irqs + 1);
2096 
2097 	smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2098 				  sizeof(*smmu->irqs), GFP_KERNEL);
2099 	if (!smmu->irqs)
2100 		return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2101 				     smmu->num_context_irqs);
2102 
2103 	for (i = 0; i < smmu->num_context_irqs; i++) {
2104 		int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2105 
2106 		if (irq < 0)
2107 			return irq;
2108 		smmu->irqs[i] = irq;
2109 	}
2110 
2111 	err = devm_clk_bulk_get_all(dev, &smmu->clks);
2112 	if (err < 0) {
2113 		dev_err(dev, "failed to get clocks %d\n", err);
2114 		return err;
2115 	}
2116 	smmu->num_clks = err;
2117 
2118 	err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2119 	if (err)
2120 		return err;
2121 
2122 	err = arm_smmu_device_cfg_probe(smmu);
2123 	if (err)
2124 		return err;
2125 
2126 	if (smmu->version == ARM_SMMU_V2) {
2127 		if (smmu->num_context_banks > smmu->num_context_irqs) {
2128 			dev_err(dev,
2129 			      "found only %d context irq(s) but %d required\n",
2130 			      smmu->num_context_irqs, smmu->num_context_banks);
2131 			return -ENODEV;
2132 		}
2133 
2134 		/* Ignore superfluous interrupts */
2135 		smmu->num_context_irqs = smmu->num_context_banks;
2136 	}
2137 
2138 	if (smmu->impl && smmu->impl->global_fault)
2139 		global_fault = smmu->impl->global_fault;
2140 	else
2141 		global_fault = arm_smmu_global_fault;
2142 
2143 	for (i = 0; i < global_irqs; i++) {
2144 		int irq = platform_get_irq(pdev, i);
2145 
2146 		if (irq < 0)
2147 			return irq;
2148 
2149 		err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2150 				       "arm-smmu global fault", smmu);
2151 		if (err)
2152 			return dev_err_probe(dev, err,
2153 					"failed to request global IRQ %d (%u)\n",
2154 					i, irq);
2155 	}
2156 
2157 	err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2158 				     "smmu.%pa", &smmu->ioaddr);
2159 	if (err) {
2160 		dev_err(dev, "Failed to register iommu in sysfs\n");
2161 		return err;
2162 	}
2163 
2164 	err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
2165 	if (err) {
2166 		dev_err(dev, "Failed to register iommu\n");
2167 		iommu_device_sysfs_remove(&smmu->iommu);
2168 		return err;
2169 	}
2170 
2171 	platform_set_drvdata(pdev, smmu);
2172 
2173 	/* Check for RMRs and install bypass SMRs if any */
2174 	arm_smmu_rmr_install_bypass_smr(smmu);
2175 
2176 	arm_smmu_device_reset(smmu);
2177 	arm_smmu_test_smr_masks(smmu);
2178 
2179 	/*
2180 	 * We want to avoid touching dev->power.lock in fastpaths unless
2181 	 * it's really going to do something useful - pm_runtime_enabled()
2182 	 * can serve as an ideal proxy for that decision. So, conditionally
2183 	 * enable pm_runtime.
2184 	 */
2185 	if (dev->pm_domain) {
2186 		pm_runtime_set_active(dev);
2187 		pm_runtime_enable(dev);
2188 	}
2189 
2190 	return 0;
2191 }
2192 
2193 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2194 {
2195 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2196 
2197 	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2198 		dev_notice(&pdev->dev, "disabling translation\n");
2199 
2200 	arm_smmu_rpm_get(smmu);
2201 	/* Turn the thing off */
2202 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2203 	arm_smmu_rpm_put(smmu);
2204 
2205 	if (pm_runtime_enabled(smmu->dev))
2206 		pm_runtime_force_suspend(smmu->dev);
2207 	else
2208 		clk_bulk_disable(smmu->num_clks, smmu->clks);
2209 
2210 	clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2211 }
2212 
2213 static void arm_smmu_device_remove(struct platform_device *pdev)
2214 {
2215 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2216 
2217 	iommu_device_unregister(&smmu->iommu);
2218 	iommu_device_sysfs_remove(&smmu->iommu);
2219 
2220 	arm_smmu_device_shutdown(pdev);
2221 }
2222 
2223 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2224 {
2225 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2226 	int ret;
2227 
2228 	ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2229 	if (ret)
2230 		return ret;
2231 
2232 	arm_smmu_device_reset(smmu);
2233 
2234 	return 0;
2235 }
2236 
2237 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2238 {
2239 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2240 
2241 	clk_bulk_disable(smmu->num_clks, smmu->clks);
2242 
2243 	return 0;
2244 }
2245 
2246 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2247 {
2248 	int ret;
2249 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2250 
2251 	ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2252 	if (ret)
2253 		return ret;
2254 
2255 	if (pm_runtime_suspended(dev))
2256 		return 0;
2257 
2258 	ret = arm_smmu_runtime_resume(dev);
2259 	if (ret)
2260 		clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2261 
2262 	return ret;
2263 }
2264 
2265 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2266 {
2267 	int ret = 0;
2268 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2269 
2270 	if (pm_runtime_suspended(dev))
2271 		goto clk_unprepare;
2272 
2273 	ret = arm_smmu_runtime_suspend(dev);
2274 	if (ret)
2275 		return ret;
2276 
2277 clk_unprepare:
2278 	clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2279 	return ret;
2280 }
2281 
2282 static const struct dev_pm_ops arm_smmu_pm_ops = {
2283 	SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2284 	SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2285 			   arm_smmu_runtime_resume, NULL)
2286 };
2287 
2288 static struct platform_driver arm_smmu_driver = {
2289 	.driver	= {
2290 		.name			= "arm-smmu",
2291 		.of_match_table		= arm_smmu_of_match,
2292 		.pm			= &arm_smmu_pm_ops,
2293 		.suppress_bind_attrs    = true,
2294 	},
2295 	.probe	= arm_smmu_device_probe,
2296 	.remove_new = arm_smmu_device_remove,
2297 	.shutdown = arm_smmu_device_shutdown,
2298 };
2299 module_platform_driver(arm_smmu_driver);
2300 
2301 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2302 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2303 MODULE_ALIAS("platform:arm-smmu");
2304 MODULE_LICENSE("GPL v2");
2305