1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37
38 #include <linux/fsl/mc.h>
39
40 #include "arm-smmu.h"
41 #include "../../dma-iommu.h"
42
43 /*
44 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
45 * global register space are still, in fact, using a hypervisor to mediate it
46 * by trapping and emulating register accesses. Sadly, some deployed versions
47 * of said trapping code have bugs wherein they go horribly wrong for stores
48 * using r31 (i.e. XZR/WZR) as the source register.
49 */
50 #define QCOM_DUMMY_VAL -1
51
52 #define MSI_IOVA_BASE 0x8000000
53 #define MSI_IOVA_LENGTH 0x100000
54
55 static int force_stage;
56 module_param(force_stage, int, S_IRUGO);
57 MODULE_PARM_DESC(force_stage,
58 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
59 static bool disable_bypass =
60 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
61 module_param(disable_bypass, bool, S_IRUGO);
62 MODULE_PARM_DESC(disable_bypass,
63 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
64
65 #define s2cr_init_val (struct arm_smmu_s2cr){ \
66 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
67 }
68
69 static bool using_legacy_binding, using_generic_binding;
70
arm_smmu_rpm_get(struct arm_smmu_device * smmu)71 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
72 {
73 if (pm_runtime_enabled(smmu->dev))
74 return pm_runtime_resume_and_get(smmu->dev);
75
76 return 0;
77 }
78
arm_smmu_rpm_put(struct arm_smmu_device * smmu)79 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
80 {
81 if (pm_runtime_enabled(smmu->dev))
82 pm_runtime_put_autosuspend(smmu->dev);
83 }
84
to_smmu_domain(struct iommu_domain * dom)85 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
86 {
87 return container_of(dom, struct arm_smmu_domain, domain);
88 }
89
90 static struct platform_driver arm_smmu_driver;
91 static struct iommu_ops arm_smmu_ops;
92
93 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)94 static struct device_node *dev_get_dev_node(struct device *dev)
95 {
96 if (dev_is_pci(dev)) {
97 struct pci_bus *bus = to_pci_dev(dev)->bus;
98
99 while (!pci_is_root_bus(bus))
100 bus = bus->parent;
101 return of_node_get(bus->bridge->parent->of_node);
102 }
103
104 return of_node_get(dev->of_node);
105 }
106
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)107 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
108 {
109 *((__be32 *)data) = cpu_to_be32(alias);
110 return 0; /* Continue walking */
111 }
112
__find_legacy_master_phandle(struct device * dev,void * data)113 static int __find_legacy_master_phandle(struct device *dev, void *data)
114 {
115 struct of_phandle_iterator *it = *(void **)data;
116 struct device_node *np = it->node;
117 int err;
118
119 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
120 "#stream-id-cells", -1)
121 if (it->node == np) {
122 *(void **)data = dev;
123 return 1;
124 }
125 it->node = np;
126 return err == -ENOENT ? 0 : err;
127 }
128
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)129 static int arm_smmu_register_legacy_master(struct device *dev,
130 struct arm_smmu_device **smmu)
131 {
132 struct device *smmu_dev;
133 struct device_node *np;
134 struct of_phandle_iterator it;
135 void *data = ⁢
136 u32 *sids;
137 __be32 pci_sid;
138 int err;
139
140 np = dev_get_dev_node(dev);
141 if (!np || !of_property_present(np, "#stream-id-cells")) {
142 of_node_put(np);
143 return -ENODEV;
144 }
145
146 it.node = np;
147 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
148 __find_legacy_master_phandle);
149 smmu_dev = data;
150 of_node_put(np);
151 if (err == 0)
152 return -ENODEV;
153 if (err < 0)
154 return err;
155
156 if (dev_is_pci(dev)) {
157 /* "mmu-masters" assumes Stream ID == Requester ID */
158 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
159 &pci_sid);
160 it.cur = &pci_sid;
161 it.cur_count = 1;
162 }
163
164 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
165 &arm_smmu_ops);
166 if (err)
167 return err;
168
169 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
170 if (!sids)
171 return -ENOMEM;
172
173 *smmu = dev_get_drvdata(smmu_dev);
174 of_phandle_iterator_args(&it, sids, it.cur_count);
175 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
176 kfree(sids);
177 return err;
178 }
179 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)180 static int arm_smmu_register_legacy_master(struct device *dev,
181 struct arm_smmu_device **smmu)
182 {
183 return -ENODEV;
184 }
185 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
186
__arm_smmu_free_bitmap(unsigned long * map,int idx)187 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
188 {
189 clear_bit(idx, map);
190 }
191
192 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)193 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
194 int sync, int status)
195 {
196 unsigned int spin_cnt, delay;
197 u32 reg;
198
199 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
200 return smmu->impl->tlb_sync(smmu, page, sync, status);
201
202 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
203 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
204 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
205 reg = arm_smmu_readl(smmu, page, status);
206 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
207 return;
208 cpu_relax();
209 }
210 udelay(delay);
211 }
212 dev_err_ratelimited(smmu->dev,
213 "TLB sync timed out -- SMMU may be deadlocked\n");
214 }
215
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)216 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
217 {
218 unsigned long flags;
219
220 spin_lock_irqsave(&smmu->global_sync_lock, flags);
221 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
222 ARM_SMMU_GR0_sTLBGSTATUS);
223 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
224 }
225
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)226 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
227 {
228 struct arm_smmu_device *smmu = smmu_domain->smmu;
229 unsigned long flags;
230
231 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
232 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
233 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
234 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
235 }
236
arm_smmu_tlb_inv_context_s1(void * cookie)237 static void arm_smmu_tlb_inv_context_s1(void *cookie)
238 {
239 struct arm_smmu_domain *smmu_domain = cookie;
240 /*
241 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
242 * current CPU are visible beforehand.
243 */
244 wmb();
245 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
246 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
247 arm_smmu_tlb_sync_context(smmu_domain);
248 }
249
arm_smmu_tlb_inv_context_s2(void * cookie)250 static void arm_smmu_tlb_inv_context_s2(void *cookie)
251 {
252 struct arm_smmu_domain *smmu_domain = cookie;
253 struct arm_smmu_device *smmu = smmu_domain->smmu;
254
255 /* See above */
256 wmb();
257 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
258 arm_smmu_tlb_sync_global(smmu);
259 }
260
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)261 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
262 size_t granule, void *cookie, int reg)
263 {
264 struct arm_smmu_domain *smmu_domain = cookie;
265 struct arm_smmu_device *smmu = smmu_domain->smmu;
266 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
267 int idx = cfg->cbndx;
268
269 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
270 wmb();
271
272 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
273 iova = (iova >> 12) << 12;
274 iova |= cfg->asid;
275 do {
276 arm_smmu_cb_write(smmu, idx, reg, iova);
277 iova += granule;
278 } while (size -= granule);
279 } else {
280 iova >>= 12;
281 iova |= (u64)cfg->asid << 48;
282 do {
283 arm_smmu_cb_writeq(smmu, idx, reg, iova);
284 iova += granule >> 12;
285 } while (size -= granule);
286 }
287 }
288
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)289 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
290 size_t granule, void *cookie, int reg)
291 {
292 struct arm_smmu_domain *smmu_domain = cookie;
293 struct arm_smmu_device *smmu = smmu_domain->smmu;
294 int idx = smmu_domain->cfg.cbndx;
295
296 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
297 wmb();
298
299 iova >>= 12;
300 do {
301 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
302 arm_smmu_cb_writeq(smmu, idx, reg, iova);
303 else
304 arm_smmu_cb_write(smmu, idx, reg, iova);
305 iova += granule >> 12;
306 } while (size -= granule);
307 }
308
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)309 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
310 size_t granule, void *cookie)
311 {
312 struct arm_smmu_domain *smmu_domain = cookie;
313 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
314
315 if (cfg->flush_walk_prefer_tlbiasid) {
316 arm_smmu_tlb_inv_context_s1(cookie);
317 } else {
318 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
319 ARM_SMMU_CB_S1_TLBIVA);
320 arm_smmu_tlb_sync_context(cookie);
321 }
322 }
323
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)324 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
325 unsigned long iova, size_t granule,
326 void *cookie)
327 {
328 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
329 ARM_SMMU_CB_S1_TLBIVAL);
330 }
331
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)332 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
333 size_t granule, void *cookie)
334 {
335 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
336 ARM_SMMU_CB_S2_TLBIIPAS2);
337 arm_smmu_tlb_sync_context(cookie);
338 }
339
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)340 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
341 unsigned long iova, size_t granule,
342 void *cookie)
343 {
344 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
345 ARM_SMMU_CB_S2_TLBIIPAS2L);
346 }
347
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)348 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
349 size_t granule, void *cookie)
350 {
351 arm_smmu_tlb_inv_context_s2(cookie);
352 }
353 /*
354 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
355 * almost negligible, but the benefit of getting the first one in as far ahead
356 * of the sync as possible is significant, hence we don't just make this a
357 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
358 * think.
359 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)360 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
361 unsigned long iova, size_t granule,
362 void *cookie)
363 {
364 struct arm_smmu_domain *smmu_domain = cookie;
365 struct arm_smmu_device *smmu = smmu_domain->smmu;
366
367 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
368 wmb();
369
370 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
371 }
372
373 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
374 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
375 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
376 .tlb_add_page = arm_smmu_tlb_add_page_s1,
377 };
378
379 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
380 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
381 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
382 .tlb_add_page = arm_smmu_tlb_add_page_s2,
383 };
384
385 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
386 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
387 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
388 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
389 };
390
arm_smmu_context_fault(int irq,void * dev)391 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
392 {
393 u32 fsr, fsynr, cbfrsynra;
394 unsigned long iova;
395 struct iommu_domain *domain = dev;
396 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
397 struct arm_smmu_device *smmu = smmu_domain->smmu;
398 int idx = smmu_domain->cfg.cbndx;
399 int ret;
400
401 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
402 if (!(fsr & ARM_SMMU_FSR_FAULT))
403 return IRQ_NONE;
404
405 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
406 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
407 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
408
409 ret = report_iommu_fault(domain, NULL, iova,
410 fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
411
412 if (ret == -ENOSYS)
413 dev_err_ratelimited(smmu->dev,
414 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
415 fsr, iova, fsynr, cbfrsynra, idx);
416
417 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
418 return IRQ_HANDLED;
419 }
420
arm_smmu_global_fault(int irq,void * dev)421 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
422 {
423 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
424 struct arm_smmu_device *smmu = dev;
425 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
426 DEFAULT_RATELIMIT_BURST);
427
428 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
429 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
430 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
431 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
432
433 if (!gfsr)
434 return IRQ_NONE;
435
436 if (__ratelimit(&rs)) {
437 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
438 (gfsr & ARM_SMMU_sGFSR_USF))
439 dev_err(smmu->dev,
440 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
441 (u16)gfsynr1);
442 else
443 dev_err(smmu->dev,
444 "Unexpected global fault, this could be serious\n");
445 dev_err(smmu->dev,
446 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
447 gfsr, gfsynr0, gfsynr1, gfsynr2);
448 }
449
450 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
451 return IRQ_HANDLED;
452 }
453
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)454 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
455 struct io_pgtable_cfg *pgtbl_cfg)
456 {
457 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
458 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
459 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
460
461 cb->cfg = cfg;
462
463 /* TCR */
464 if (stage1) {
465 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
466 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
467 } else {
468 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
469 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
470 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
471 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
472 else
473 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
474 }
475 } else {
476 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
477 }
478
479 /* TTBRs */
480 if (stage1) {
481 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
482 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
483 cb->ttbr[1] = 0;
484 } else {
485 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
486 cfg->asid);
487 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
488 cfg->asid);
489
490 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
491 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
492 else
493 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
494 }
495 } else {
496 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
497 }
498
499 /* MAIRs (stage-1 only) */
500 if (stage1) {
501 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
502 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
503 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
504 } else {
505 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
506 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
507 }
508 }
509 }
510
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)511 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
512 {
513 u32 reg;
514 bool stage1;
515 struct arm_smmu_cb *cb = &smmu->cbs[idx];
516 struct arm_smmu_cfg *cfg = cb->cfg;
517
518 /* Unassigned context banks only need disabling */
519 if (!cfg) {
520 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
521 return;
522 }
523
524 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
525
526 /* CBA2R */
527 if (smmu->version > ARM_SMMU_V1) {
528 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
529 reg = ARM_SMMU_CBA2R_VA64;
530 else
531 reg = 0;
532 /* 16-bit VMIDs live in CBA2R */
533 if (smmu->features & ARM_SMMU_FEAT_VMID16)
534 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
535
536 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
537 }
538
539 /* CBAR */
540 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
541 if (smmu->version < ARM_SMMU_V2)
542 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
543
544 /*
545 * Use the weakest shareability/memory types, so they are
546 * overridden by the ttbcr/pte.
547 */
548 if (stage1) {
549 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
550 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
551 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
552 ARM_SMMU_CBAR_S1_MEMATTR_WB);
553 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
554 /* 8-bit VMIDs live in CBAR */
555 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
556 }
557 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
558
559 /*
560 * TCR
561 * We must write this before the TTBRs, since it determines the
562 * access behaviour of some fields (in particular, ASID[15:8]).
563 */
564 if (stage1 && smmu->version > ARM_SMMU_V1)
565 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
566 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
567
568 /* TTBRs */
569 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
570 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
571 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
572 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
573 } else {
574 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
575 if (stage1)
576 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
577 cb->ttbr[1]);
578 }
579
580 /* MAIRs (stage-1 only) */
581 if (stage1) {
582 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
583 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
584 }
585
586 /* SCTLR */
587 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
588 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
589 if (stage1)
590 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
591 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
592 reg |= ARM_SMMU_SCTLR_E;
593
594 if (smmu->impl && smmu->impl->write_sctlr)
595 smmu->impl->write_sctlr(smmu, idx, reg);
596 else
597 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
598 }
599
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)600 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
601 struct arm_smmu_device *smmu,
602 struct device *dev, unsigned int start)
603 {
604 if (smmu->impl && smmu->impl->alloc_context_bank)
605 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
606
607 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
608 }
609
arm_smmu_init_domain_context(struct iommu_domain * domain,struct arm_smmu_device * smmu,struct device * dev)610 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
611 struct arm_smmu_device *smmu,
612 struct device *dev)
613 {
614 int irq, start, ret = 0;
615 unsigned long ias, oas;
616 struct io_pgtable_ops *pgtbl_ops;
617 struct io_pgtable_cfg pgtbl_cfg;
618 enum io_pgtable_fmt fmt;
619 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
620 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
621 irqreturn_t (*context_fault)(int irq, void *dev);
622
623 mutex_lock(&smmu_domain->init_mutex);
624 if (smmu_domain->smmu)
625 goto out_unlock;
626
627 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
628 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
629 smmu_domain->smmu = smmu;
630 goto out_unlock;
631 }
632
633 /*
634 * Mapping the requested stage onto what we support is surprisingly
635 * complicated, mainly because the spec allows S1+S2 SMMUs without
636 * support for nested translation. That means we end up with the
637 * following table:
638 *
639 * Requested Supported Actual
640 * S1 N S1
641 * S1 S1+S2 S1
642 * S1 S2 S2
643 * S1 S1 S1
644 * N N N
645 * N S1+S2 S2
646 * N S2 S2
647 * N S1 S1
648 *
649 * Note that you can't actually request stage-2 mappings.
650 */
651 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
652 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
653 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
654 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
655
656 /*
657 * Choosing a suitable context format is even more fiddly. Until we
658 * grow some way for the caller to express a preference, and/or move
659 * the decision into the io-pgtable code where it arguably belongs,
660 * just aim for the closest thing to the rest of the system, and hope
661 * that the hardware isn't esoteric enough that we can't assume AArch64
662 * support to be a superset of AArch32 support...
663 */
664 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
665 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
666 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
667 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
668 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
669 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
670 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
671 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
672 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
673 ARM_SMMU_FEAT_FMT_AARCH64_16K |
674 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
675 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
676
677 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
678 ret = -EINVAL;
679 goto out_unlock;
680 }
681
682 switch (smmu_domain->stage) {
683 case ARM_SMMU_DOMAIN_S1:
684 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
685 start = smmu->num_s2_context_banks;
686 ias = smmu->va_size;
687 oas = smmu->ipa_size;
688 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
689 fmt = ARM_64_LPAE_S1;
690 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
691 fmt = ARM_32_LPAE_S1;
692 ias = min(ias, 32UL);
693 oas = min(oas, 40UL);
694 } else {
695 fmt = ARM_V7S;
696 ias = min(ias, 32UL);
697 oas = min(oas, 32UL);
698 }
699 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
700 break;
701 case ARM_SMMU_DOMAIN_NESTED:
702 /*
703 * We will likely want to change this if/when KVM gets
704 * involved.
705 */
706 case ARM_SMMU_DOMAIN_S2:
707 cfg->cbar = CBAR_TYPE_S2_TRANS;
708 start = 0;
709 ias = smmu->ipa_size;
710 oas = smmu->pa_size;
711 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
712 fmt = ARM_64_LPAE_S2;
713 } else {
714 fmt = ARM_32_LPAE_S2;
715 ias = min(ias, 40UL);
716 oas = min(oas, 40UL);
717 }
718 if (smmu->version == ARM_SMMU_V2)
719 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
720 else
721 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
722 break;
723 default:
724 ret = -EINVAL;
725 goto out_unlock;
726 }
727
728 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
729 if (ret < 0) {
730 goto out_unlock;
731 }
732
733 smmu_domain->smmu = smmu;
734
735 cfg->cbndx = ret;
736 if (smmu->version < ARM_SMMU_V2) {
737 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
738 cfg->irptndx %= smmu->num_context_irqs;
739 } else {
740 cfg->irptndx = cfg->cbndx;
741 }
742
743 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
744 cfg->vmid = cfg->cbndx + 1;
745 else
746 cfg->asid = cfg->cbndx;
747
748 pgtbl_cfg = (struct io_pgtable_cfg) {
749 .pgsize_bitmap = smmu->pgsize_bitmap,
750 .ias = ias,
751 .oas = oas,
752 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
753 .tlb = smmu_domain->flush_ops,
754 .iommu_dev = smmu->dev,
755 };
756
757 if (smmu->impl && smmu->impl->init_context) {
758 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
759 if (ret)
760 goto out_clear_smmu;
761 }
762
763 if (smmu_domain->pgtbl_quirks)
764 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
765
766 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
767 if (!pgtbl_ops) {
768 ret = -ENOMEM;
769 goto out_clear_smmu;
770 }
771
772 /* Update the domain's page sizes to reflect the page table format */
773 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
774
775 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
776 domain->geometry.aperture_start = ~0UL << ias;
777 domain->geometry.aperture_end = ~0UL;
778 } else {
779 domain->geometry.aperture_end = (1UL << ias) - 1;
780 }
781
782 domain->geometry.force_aperture = true;
783
784 /* Initialise the context bank with our page table cfg */
785 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
786 arm_smmu_write_context_bank(smmu, cfg->cbndx);
787
788 /*
789 * Request context fault interrupt. Do this last to avoid the
790 * handler seeing a half-initialised domain state.
791 */
792 irq = smmu->irqs[cfg->irptndx];
793
794 if (smmu->impl && smmu->impl->context_fault)
795 context_fault = smmu->impl->context_fault;
796 else
797 context_fault = arm_smmu_context_fault;
798
799 ret = devm_request_irq(smmu->dev, irq, context_fault,
800 IRQF_SHARED, "arm-smmu-context-fault", domain);
801 if (ret < 0) {
802 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
803 cfg->irptndx, irq);
804 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
805 }
806
807 mutex_unlock(&smmu_domain->init_mutex);
808
809 /* Publish page table ops for map/unmap */
810 smmu_domain->pgtbl_ops = pgtbl_ops;
811 return 0;
812
813 out_clear_smmu:
814 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
815 smmu_domain->smmu = NULL;
816 out_unlock:
817 mutex_unlock(&smmu_domain->init_mutex);
818 return ret;
819 }
820
arm_smmu_destroy_domain_context(struct iommu_domain * domain)821 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
822 {
823 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
824 struct arm_smmu_device *smmu = smmu_domain->smmu;
825 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
826 int ret, irq;
827
828 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
829 return;
830
831 ret = arm_smmu_rpm_get(smmu);
832 if (ret < 0)
833 return;
834
835 /*
836 * Disable the context bank and free the page tables before freeing
837 * it.
838 */
839 smmu->cbs[cfg->cbndx].cfg = NULL;
840 arm_smmu_write_context_bank(smmu, cfg->cbndx);
841
842 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
843 irq = smmu->irqs[cfg->irptndx];
844 devm_free_irq(smmu->dev, irq, domain);
845 }
846
847 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
848 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
849
850 arm_smmu_rpm_put(smmu);
851 }
852
arm_smmu_domain_alloc(unsigned type)853 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
854 {
855 struct arm_smmu_domain *smmu_domain;
856
857 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) {
858 if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
859 return NULL;
860 }
861 /*
862 * Allocate the domain and initialise some of its data structures.
863 * We can't really do anything meaningful until we've added a
864 * master.
865 */
866 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
867 if (!smmu_domain)
868 return NULL;
869
870 mutex_init(&smmu_domain->init_mutex);
871 spin_lock_init(&smmu_domain->cb_lock);
872
873 return &smmu_domain->domain;
874 }
875
arm_smmu_domain_free(struct iommu_domain * domain)876 static void arm_smmu_domain_free(struct iommu_domain *domain)
877 {
878 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
879
880 /*
881 * Free the domain resources. We assume that all devices have
882 * already been detached.
883 */
884 arm_smmu_destroy_domain_context(domain);
885 kfree(smmu_domain);
886 }
887
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)888 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
889 {
890 struct arm_smmu_smr *smr = smmu->smrs + idx;
891 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
892 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
893
894 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
895 reg |= ARM_SMMU_SMR_VALID;
896 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
897 }
898
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)899 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
900 {
901 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
902 u32 reg;
903
904 if (smmu->impl && smmu->impl->write_s2cr) {
905 smmu->impl->write_s2cr(smmu, idx);
906 return;
907 }
908
909 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
910 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
911 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
912
913 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
914 smmu->smrs[idx].valid)
915 reg |= ARM_SMMU_S2CR_EXIDVALID;
916 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
917 }
918
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)919 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
920 {
921 arm_smmu_write_s2cr(smmu, idx);
922 if (smmu->smrs)
923 arm_smmu_write_smr(smmu, idx);
924 }
925
926 /*
927 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
928 * should be called after sCR0 is written.
929 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)930 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
931 {
932 u32 smr;
933 int i;
934
935 if (!smmu->smrs)
936 return;
937 /*
938 * If we've had to accommodate firmware memory regions, we may
939 * have live SMRs by now; tread carefully...
940 *
941 * Somewhat perversely, not having a free SMR for this test implies we
942 * can get away without it anyway, as we'll only be able to 'allocate'
943 * these SMRs for the ID/mask values we're already trusting to be OK.
944 */
945 for (i = 0; i < smmu->num_mapping_groups; i++)
946 if (!smmu->smrs[i].valid)
947 goto smr_ok;
948 return;
949 smr_ok:
950 /*
951 * SMR.ID bits may not be preserved if the corresponding MASK
952 * bits are set, so check each one separately. We can reject
953 * masters later if they try to claim IDs outside these masks.
954 */
955 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
956 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
957 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
958 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
959
960 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
961 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
962 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
963 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
964 }
965
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)966 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
967 {
968 struct arm_smmu_smr *smrs = smmu->smrs;
969 int i, free_idx = -ENOSPC;
970
971 /* Stream indexing is blissfully easy */
972 if (!smrs)
973 return id;
974
975 /* Validating SMRs is... less so */
976 for (i = 0; i < smmu->num_mapping_groups; ++i) {
977 if (!smrs[i].valid) {
978 /*
979 * Note the first free entry we come across, which
980 * we'll claim in the end if nothing else matches.
981 */
982 if (free_idx < 0)
983 free_idx = i;
984 continue;
985 }
986 /*
987 * If the new entry is _entirely_ matched by an existing entry,
988 * then reuse that, with the guarantee that there also cannot
989 * be any subsequent conflicting entries. In normal use we'd
990 * expect simply identical entries for this case, but there's
991 * no harm in accommodating the generalisation.
992 */
993 if ((mask & smrs[i].mask) == mask &&
994 !((id ^ smrs[i].id) & ~smrs[i].mask))
995 return i;
996 /*
997 * If the new entry has any other overlap with an existing one,
998 * though, then there always exists at least one stream ID
999 * which would cause a conflict, and we can't allow that risk.
1000 */
1001 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1002 return -EINVAL;
1003 }
1004
1005 return free_idx;
1006 }
1007
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1008 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1009 {
1010 if (--smmu->s2crs[idx].count)
1011 return false;
1012
1013 smmu->s2crs[idx] = s2cr_init_val;
1014 if (smmu->smrs)
1015 smmu->smrs[idx].valid = false;
1016
1017 return true;
1018 }
1019
arm_smmu_master_alloc_smes(struct device * dev)1020 static int arm_smmu_master_alloc_smes(struct device *dev)
1021 {
1022 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1023 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1024 struct arm_smmu_device *smmu = cfg->smmu;
1025 struct arm_smmu_smr *smrs = smmu->smrs;
1026 int i, idx, ret;
1027
1028 mutex_lock(&smmu->stream_map_mutex);
1029 /* Figure out a viable stream map entry allocation */
1030 for_each_cfg_sme(cfg, fwspec, i, idx) {
1031 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1032 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1033
1034 if (idx != INVALID_SMENDX) {
1035 ret = -EEXIST;
1036 goto out_err;
1037 }
1038
1039 ret = arm_smmu_find_sme(smmu, sid, mask);
1040 if (ret < 0)
1041 goto out_err;
1042
1043 idx = ret;
1044 if (smrs && smmu->s2crs[idx].count == 0) {
1045 smrs[idx].id = sid;
1046 smrs[idx].mask = mask;
1047 smrs[idx].valid = true;
1048 }
1049 smmu->s2crs[idx].count++;
1050 cfg->smendx[i] = (s16)idx;
1051 }
1052
1053 /* It worked! Now, poke the actual hardware */
1054 for_each_cfg_sme(cfg, fwspec, i, idx)
1055 arm_smmu_write_sme(smmu, idx);
1056
1057 mutex_unlock(&smmu->stream_map_mutex);
1058 return 0;
1059
1060 out_err:
1061 while (i--) {
1062 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1063 cfg->smendx[i] = INVALID_SMENDX;
1064 }
1065 mutex_unlock(&smmu->stream_map_mutex);
1066 return ret;
1067 }
1068
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1069 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1070 struct iommu_fwspec *fwspec)
1071 {
1072 struct arm_smmu_device *smmu = cfg->smmu;
1073 int i, idx;
1074
1075 mutex_lock(&smmu->stream_map_mutex);
1076 for_each_cfg_sme(cfg, fwspec, i, idx) {
1077 if (arm_smmu_free_sme(smmu, idx))
1078 arm_smmu_write_sme(smmu, idx);
1079 cfg->smendx[i] = INVALID_SMENDX;
1080 }
1081 mutex_unlock(&smmu->stream_map_mutex);
1082 }
1083
arm_smmu_domain_add_master(struct arm_smmu_domain * smmu_domain,struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1084 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1085 struct arm_smmu_master_cfg *cfg,
1086 struct iommu_fwspec *fwspec)
1087 {
1088 struct arm_smmu_device *smmu = smmu_domain->smmu;
1089 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1090 u8 cbndx = smmu_domain->cfg.cbndx;
1091 enum arm_smmu_s2cr_type type;
1092 int i, idx;
1093
1094 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1095 type = S2CR_TYPE_BYPASS;
1096 else
1097 type = S2CR_TYPE_TRANS;
1098
1099 for_each_cfg_sme(cfg, fwspec, i, idx) {
1100 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1101 continue;
1102
1103 s2cr[idx].type = type;
1104 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1105 s2cr[idx].cbndx = cbndx;
1106 arm_smmu_write_s2cr(smmu, idx);
1107 }
1108 return 0;
1109 }
1110
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1111 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1112 {
1113 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1114 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1115 struct arm_smmu_master_cfg *cfg;
1116 struct arm_smmu_device *smmu;
1117 int ret;
1118
1119 if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1120 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1121 return -ENXIO;
1122 }
1123
1124 /*
1125 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1126 * domains between of_xlate() and probe_device() - we have no way to cope
1127 * with that, so until ARM gets converted to rely on groups and default
1128 * domains, just say no (but more politely than by dereferencing NULL).
1129 * This should be at least a WARN_ON once that's sorted.
1130 */
1131 cfg = dev_iommu_priv_get(dev);
1132 if (!cfg)
1133 return -ENODEV;
1134
1135 smmu = cfg->smmu;
1136
1137 ret = arm_smmu_rpm_get(smmu);
1138 if (ret < 0)
1139 return ret;
1140
1141 /* Ensure that the domain is finalised */
1142 ret = arm_smmu_init_domain_context(domain, smmu, dev);
1143 if (ret < 0)
1144 goto rpm_put;
1145
1146 /*
1147 * Sanity check the domain. We don't support domains across
1148 * different SMMUs.
1149 */
1150 if (smmu_domain->smmu != smmu) {
1151 ret = -EINVAL;
1152 goto rpm_put;
1153 }
1154
1155 /* Looks ok, so add the device to the domain */
1156 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1157
1158 /*
1159 * Setup an autosuspend delay to avoid bouncing runpm state.
1160 * Otherwise, if a driver for a suspended consumer device
1161 * unmaps buffers, it will runpm resume/suspend for each one.
1162 *
1163 * For example, when used by a GPU device, when an application
1164 * or game exits, it can trigger unmapping 100s or 1000s of
1165 * buffers. With a runpm cycle for each buffer, that adds up
1166 * to 5-10sec worth of reprogramming the context bank, while
1167 * the system appears to be locked up to the user.
1168 */
1169 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1170 pm_runtime_use_autosuspend(smmu->dev);
1171
1172 rpm_put:
1173 arm_smmu_rpm_put(smmu);
1174 return ret;
1175 }
1176
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1177 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1178 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1179 int prot, gfp_t gfp, size_t *mapped)
1180 {
1181 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1182 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1183 int ret;
1184
1185 if (!ops)
1186 return -ENODEV;
1187
1188 arm_smmu_rpm_get(smmu);
1189 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1190 arm_smmu_rpm_put(smmu);
1191
1192 return ret;
1193 }
1194
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1195 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1196 size_t pgsize, size_t pgcount,
1197 struct iommu_iotlb_gather *iotlb_gather)
1198 {
1199 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1200 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1201 size_t ret;
1202
1203 if (!ops)
1204 return 0;
1205
1206 arm_smmu_rpm_get(smmu);
1207 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1208 arm_smmu_rpm_put(smmu);
1209
1210 return ret;
1211 }
1212
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1213 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1214 {
1215 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1216 struct arm_smmu_device *smmu = smmu_domain->smmu;
1217
1218 if (smmu_domain->flush_ops) {
1219 arm_smmu_rpm_get(smmu);
1220 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1221 arm_smmu_rpm_put(smmu);
1222 }
1223 }
1224
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1225 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1226 struct iommu_iotlb_gather *gather)
1227 {
1228 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1229 struct arm_smmu_device *smmu = smmu_domain->smmu;
1230
1231 if (!smmu)
1232 return;
1233
1234 arm_smmu_rpm_get(smmu);
1235 if (smmu->version == ARM_SMMU_V2 ||
1236 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1237 arm_smmu_tlb_sync_context(smmu_domain);
1238 else
1239 arm_smmu_tlb_sync_global(smmu);
1240 arm_smmu_rpm_put(smmu);
1241 }
1242
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1243 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1244 dma_addr_t iova)
1245 {
1246 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1247 struct arm_smmu_device *smmu = smmu_domain->smmu;
1248 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1249 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1250 struct device *dev = smmu->dev;
1251 void __iomem *reg;
1252 u32 tmp;
1253 u64 phys;
1254 unsigned long va, flags;
1255 int ret, idx = cfg->cbndx;
1256 phys_addr_t addr = 0;
1257
1258 ret = arm_smmu_rpm_get(smmu);
1259 if (ret < 0)
1260 return 0;
1261
1262 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1263 va = iova & ~0xfffUL;
1264 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1265 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1266 else
1267 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1268
1269 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1270 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1271 5, 50)) {
1272 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1273 dev_err(dev,
1274 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1275 &iova);
1276 arm_smmu_rpm_put(smmu);
1277 return ops->iova_to_phys(ops, iova);
1278 }
1279
1280 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1281 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1282 if (phys & ARM_SMMU_CB_PAR_F) {
1283 dev_err(dev, "translation fault!\n");
1284 dev_err(dev, "PAR = 0x%llx\n", phys);
1285 goto out;
1286 }
1287
1288 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1289 out:
1290 arm_smmu_rpm_put(smmu);
1291
1292 return addr;
1293 }
1294
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1295 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1296 dma_addr_t iova)
1297 {
1298 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1299 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1300
1301 if (!ops)
1302 return 0;
1303
1304 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1305 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1306 return arm_smmu_iova_to_phys_hard(domain, iova);
1307
1308 return ops->iova_to_phys(ops, iova);
1309 }
1310
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1311 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1312 {
1313 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1314
1315 switch (cap) {
1316 case IOMMU_CAP_CACHE_COHERENCY:
1317 /*
1318 * It's overwhelmingly the case in practice that when the pagetable
1319 * walk interface is connected to a coherent interconnect, all the
1320 * translation interfaces are too. Furthermore if the device is
1321 * natively coherent, then its translation interface must also be.
1322 */
1323 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1324 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1325 case IOMMU_CAP_NOEXEC:
1326 case IOMMU_CAP_DEFERRED_FLUSH:
1327 return true;
1328 default:
1329 return false;
1330 }
1331 }
1332
1333 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1334 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1335 {
1336 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1337 fwnode);
1338 put_device(dev);
1339 return dev ? dev_get_drvdata(dev) : NULL;
1340 }
1341
arm_smmu_probe_device(struct device * dev)1342 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1343 {
1344 struct arm_smmu_device *smmu = NULL;
1345 struct arm_smmu_master_cfg *cfg;
1346 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1347 int i, ret;
1348
1349 if (using_legacy_binding) {
1350 ret = arm_smmu_register_legacy_master(dev, &smmu);
1351
1352 /*
1353 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1354 * will allocate/initialise a new one. Thus we need to update fwspec for
1355 * later use.
1356 */
1357 fwspec = dev_iommu_fwspec_get(dev);
1358 if (ret)
1359 goto out_free;
1360 } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1361 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1362
1363 /*
1364 * Defer probe if the relevant SMMU instance hasn't finished
1365 * probing yet. This is a fragile hack and we'd ideally
1366 * avoid this race in the core code. Until that's ironed
1367 * out, however, this is the most pragmatic option on the
1368 * table.
1369 */
1370 if (!smmu)
1371 return ERR_PTR(dev_err_probe(dev, -EPROBE_DEFER,
1372 "smmu dev has not bound yet\n"));
1373 } else {
1374 return ERR_PTR(-ENODEV);
1375 }
1376
1377 ret = -EINVAL;
1378 for (i = 0; i < fwspec->num_ids; i++) {
1379 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1380 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1381
1382 if (sid & ~smmu->streamid_mask) {
1383 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1384 sid, smmu->streamid_mask);
1385 goto out_free;
1386 }
1387 if (mask & ~smmu->smr_mask_mask) {
1388 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1389 mask, smmu->smr_mask_mask);
1390 goto out_free;
1391 }
1392 }
1393
1394 ret = -ENOMEM;
1395 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1396 GFP_KERNEL);
1397 if (!cfg)
1398 goto out_free;
1399
1400 cfg->smmu = smmu;
1401 dev_iommu_priv_set(dev, cfg);
1402 while (i--)
1403 cfg->smendx[i] = INVALID_SMENDX;
1404
1405 ret = arm_smmu_rpm_get(smmu);
1406 if (ret < 0)
1407 goto out_cfg_free;
1408
1409 ret = arm_smmu_master_alloc_smes(dev);
1410 arm_smmu_rpm_put(smmu);
1411
1412 if (ret)
1413 goto out_cfg_free;
1414
1415 device_link_add(dev, smmu->dev,
1416 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1417
1418 return &smmu->iommu;
1419
1420 out_cfg_free:
1421 kfree(cfg);
1422 out_free:
1423 iommu_fwspec_free(dev);
1424 return ERR_PTR(ret);
1425 }
1426
arm_smmu_release_device(struct device * dev)1427 static void arm_smmu_release_device(struct device *dev)
1428 {
1429 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1430 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1431 int ret;
1432
1433 ret = arm_smmu_rpm_get(cfg->smmu);
1434 if (ret < 0)
1435 return;
1436
1437 arm_smmu_master_free_smes(cfg, fwspec);
1438
1439 arm_smmu_rpm_put(cfg->smmu);
1440
1441 dev_iommu_priv_set(dev, NULL);
1442 kfree(cfg);
1443 }
1444
arm_smmu_probe_finalize(struct device * dev)1445 static void arm_smmu_probe_finalize(struct device *dev)
1446 {
1447 struct arm_smmu_master_cfg *cfg;
1448 struct arm_smmu_device *smmu;
1449
1450 cfg = dev_iommu_priv_get(dev);
1451 smmu = cfg->smmu;
1452
1453 if (smmu->impl && smmu->impl->probe_finalize)
1454 smmu->impl->probe_finalize(smmu, dev);
1455 }
1456
arm_smmu_device_group(struct device * dev)1457 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1458 {
1459 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1460 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1461 struct arm_smmu_device *smmu = cfg->smmu;
1462 struct iommu_group *group = NULL;
1463 int i, idx;
1464
1465 mutex_lock(&smmu->stream_map_mutex);
1466 for_each_cfg_sme(cfg, fwspec, i, idx) {
1467 if (group && smmu->s2crs[idx].group &&
1468 group != smmu->s2crs[idx].group) {
1469 mutex_unlock(&smmu->stream_map_mutex);
1470 return ERR_PTR(-EINVAL);
1471 }
1472
1473 group = smmu->s2crs[idx].group;
1474 }
1475
1476 if (group) {
1477 mutex_unlock(&smmu->stream_map_mutex);
1478 return iommu_group_ref_get(group);
1479 }
1480
1481 if (dev_is_pci(dev))
1482 group = pci_device_group(dev);
1483 else if (dev_is_fsl_mc(dev))
1484 group = fsl_mc_device_group(dev);
1485 else
1486 group = generic_device_group(dev);
1487
1488 /* Remember group for faster lookups */
1489 if (!IS_ERR(group))
1490 for_each_cfg_sme(cfg, fwspec, i, idx)
1491 smmu->s2crs[idx].group = group;
1492
1493 mutex_unlock(&smmu->stream_map_mutex);
1494 return group;
1495 }
1496
arm_smmu_enable_nesting(struct iommu_domain * domain)1497 static int arm_smmu_enable_nesting(struct iommu_domain *domain)
1498 {
1499 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1500 int ret = 0;
1501
1502 mutex_lock(&smmu_domain->init_mutex);
1503 if (smmu_domain->smmu)
1504 ret = -EPERM;
1505 else
1506 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1507 mutex_unlock(&smmu_domain->init_mutex);
1508
1509 return ret;
1510 }
1511
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1512 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1513 unsigned long quirks)
1514 {
1515 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1516 int ret = 0;
1517
1518 mutex_lock(&smmu_domain->init_mutex);
1519 if (smmu_domain->smmu)
1520 ret = -EPERM;
1521 else
1522 smmu_domain->pgtbl_quirks = quirks;
1523 mutex_unlock(&smmu_domain->init_mutex);
1524
1525 return ret;
1526 }
1527
arm_smmu_of_xlate(struct device * dev,struct of_phandle_args * args)1528 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1529 {
1530 u32 mask, fwid = 0;
1531
1532 if (args->args_count > 0)
1533 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1534
1535 if (args->args_count > 1)
1536 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1537 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1538 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1539
1540 return iommu_fwspec_add_ids(dev, &fwid, 1);
1541 }
1542
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1543 static void arm_smmu_get_resv_regions(struct device *dev,
1544 struct list_head *head)
1545 {
1546 struct iommu_resv_region *region;
1547 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1548
1549 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1550 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1551 if (!region)
1552 return;
1553
1554 list_add_tail(®ion->list, head);
1555
1556 iommu_dma_get_resv_regions(dev, head);
1557 }
1558
arm_smmu_def_domain_type(struct device * dev)1559 static int arm_smmu_def_domain_type(struct device *dev)
1560 {
1561 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1562 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1563
1564 if (using_legacy_binding)
1565 return IOMMU_DOMAIN_IDENTITY;
1566
1567 if (impl && impl->def_domain_type)
1568 return impl->def_domain_type(dev);
1569
1570 return 0;
1571 }
1572
1573 static struct iommu_ops arm_smmu_ops = {
1574 .capable = arm_smmu_capable,
1575 .domain_alloc = arm_smmu_domain_alloc,
1576 .probe_device = arm_smmu_probe_device,
1577 .release_device = arm_smmu_release_device,
1578 .probe_finalize = arm_smmu_probe_finalize,
1579 .device_group = arm_smmu_device_group,
1580 .of_xlate = arm_smmu_of_xlate,
1581 .get_resv_regions = arm_smmu_get_resv_regions,
1582 .def_domain_type = arm_smmu_def_domain_type,
1583 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1584 .owner = THIS_MODULE,
1585 .default_domain_ops = &(const struct iommu_domain_ops) {
1586 .attach_dev = arm_smmu_attach_dev,
1587 .map_pages = arm_smmu_map_pages,
1588 .unmap_pages = arm_smmu_unmap_pages,
1589 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1590 .iotlb_sync = arm_smmu_iotlb_sync,
1591 .iova_to_phys = arm_smmu_iova_to_phys,
1592 .enable_nesting = arm_smmu_enable_nesting,
1593 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1594 .free = arm_smmu_domain_free,
1595 }
1596 };
1597
arm_smmu_device_reset(struct arm_smmu_device * smmu)1598 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1599 {
1600 int i;
1601 u32 reg;
1602
1603 /* clear global FSR */
1604 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1605 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1606
1607 /*
1608 * Reset stream mapping groups: Initial values mark all SMRn as
1609 * invalid and all S2CRn as bypass unless overridden.
1610 */
1611 for (i = 0; i < smmu->num_mapping_groups; ++i)
1612 arm_smmu_write_sme(smmu, i);
1613
1614 /* Make sure all context banks are disabled and clear CB_FSR */
1615 for (i = 0; i < smmu->num_context_banks; ++i) {
1616 arm_smmu_write_context_bank(smmu, i);
1617 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1618 }
1619
1620 /* Invalidate the TLB, just in case */
1621 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1622 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1623
1624 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1625
1626 /* Enable fault reporting */
1627 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1628 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1629
1630 /* Disable TLB broadcasting. */
1631 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1632
1633 /* Enable client access, handling unmatched streams as appropriate */
1634 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1635 if (disable_bypass)
1636 reg |= ARM_SMMU_sCR0_USFCFG;
1637 else
1638 reg &= ~ARM_SMMU_sCR0_USFCFG;
1639
1640 /* Disable forced broadcasting */
1641 reg &= ~ARM_SMMU_sCR0_FB;
1642
1643 /* Don't upgrade barriers */
1644 reg &= ~(ARM_SMMU_sCR0_BSU);
1645
1646 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1647 reg |= ARM_SMMU_sCR0_VMID16EN;
1648
1649 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1650 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1651
1652 if (smmu->impl && smmu->impl->reset)
1653 smmu->impl->reset(smmu);
1654
1655 /* Push the button */
1656 arm_smmu_tlb_sync_global(smmu);
1657 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1658 }
1659
arm_smmu_id_size_to_bits(int size)1660 static int arm_smmu_id_size_to_bits(int size)
1661 {
1662 switch (size) {
1663 case 0:
1664 return 32;
1665 case 1:
1666 return 36;
1667 case 2:
1668 return 40;
1669 case 3:
1670 return 42;
1671 case 4:
1672 return 44;
1673 case 5:
1674 default:
1675 return 48;
1676 }
1677 }
1678
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1679 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1680 {
1681 unsigned int size;
1682 u32 id;
1683 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1684 int i, ret;
1685
1686 dev_notice(smmu->dev, "probing hardware configuration...\n");
1687 dev_notice(smmu->dev, "SMMUv%d with:\n",
1688 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1689
1690 /* ID0 */
1691 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1692
1693 /* Restrict available stages based on module parameter */
1694 if (force_stage == 1)
1695 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1696 else if (force_stage == 2)
1697 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1698
1699 if (id & ARM_SMMU_ID0_S1TS) {
1700 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1701 dev_notice(smmu->dev, "\tstage 1 translation\n");
1702 }
1703
1704 if (id & ARM_SMMU_ID0_S2TS) {
1705 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1706 dev_notice(smmu->dev, "\tstage 2 translation\n");
1707 }
1708
1709 if (id & ARM_SMMU_ID0_NTS) {
1710 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1711 dev_notice(smmu->dev, "\tnested translation\n");
1712 }
1713
1714 if (!(smmu->features &
1715 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1716 dev_err(smmu->dev, "\tno translation support!\n");
1717 return -ENODEV;
1718 }
1719
1720 if ((id & ARM_SMMU_ID0_S1TS) &&
1721 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1722 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1723 dev_notice(smmu->dev, "\taddress translation ops\n");
1724 }
1725
1726 /*
1727 * In order for DMA API calls to work properly, we must defer to what
1728 * the FW says about coherency, regardless of what the hardware claims.
1729 * Fortunately, this also opens up a workaround for systems where the
1730 * ID register value has ended up configured incorrectly.
1731 */
1732 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1733 if (cttw_fw || cttw_reg)
1734 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1735 cttw_fw ? "" : "non-");
1736 if (cttw_fw != cttw_reg)
1737 dev_notice(smmu->dev,
1738 "\t(IDR0.CTTW overridden by FW configuration)\n");
1739
1740 /* Max. number of entries we have for stream matching/indexing */
1741 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1742 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1743 size = 1 << 16;
1744 } else {
1745 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1746 }
1747 smmu->streamid_mask = size - 1;
1748 if (id & ARM_SMMU_ID0_SMS) {
1749 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1750 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1751 if (size == 0) {
1752 dev_err(smmu->dev,
1753 "stream-matching supported, but no SMRs present!\n");
1754 return -ENODEV;
1755 }
1756
1757 /* Zero-initialised to mark as invalid */
1758 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1759 GFP_KERNEL);
1760 if (!smmu->smrs)
1761 return -ENOMEM;
1762
1763 dev_notice(smmu->dev,
1764 "\tstream matching with %u register groups", size);
1765 }
1766 /* s2cr->type == 0 means translation, so initialise explicitly */
1767 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1768 GFP_KERNEL);
1769 if (!smmu->s2crs)
1770 return -ENOMEM;
1771 for (i = 0; i < size; i++)
1772 smmu->s2crs[i] = s2cr_init_val;
1773
1774 smmu->num_mapping_groups = size;
1775 mutex_init(&smmu->stream_map_mutex);
1776 spin_lock_init(&smmu->global_sync_lock);
1777
1778 if (smmu->version < ARM_SMMU_V2 ||
1779 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1780 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1781 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1782 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1783 }
1784
1785 /* ID1 */
1786 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1787 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1788
1789 /* Check for size mismatch of SMMU address space from mapped region */
1790 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1791 if (smmu->numpage != 2 * size << smmu->pgshift)
1792 dev_warn(smmu->dev,
1793 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1794 2 * size << smmu->pgshift, smmu->numpage);
1795 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1796 smmu->numpage = size;
1797
1798 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1799 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1800 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1801 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1802 return -ENODEV;
1803 }
1804 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1805 smmu->num_context_banks, smmu->num_s2_context_banks);
1806 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1807 sizeof(*smmu->cbs), GFP_KERNEL);
1808 if (!smmu->cbs)
1809 return -ENOMEM;
1810
1811 /* ID2 */
1812 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1813 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1814 smmu->ipa_size = size;
1815
1816 /* The output mask is also applied for bypass */
1817 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1818 smmu->pa_size = size;
1819
1820 if (id & ARM_SMMU_ID2_VMID16)
1821 smmu->features |= ARM_SMMU_FEAT_VMID16;
1822
1823 /*
1824 * What the page table walker can address actually depends on which
1825 * descriptor format is in use, but since a) we don't know that yet,
1826 * and b) it can vary per context bank, this will have to do...
1827 */
1828 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1829 dev_warn(smmu->dev,
1830 "failed to set DMA mask for table walker\n");
1831
1832 if (smmu->version < ARM_SMMU_V2) {
1833 smmu->va_size = smmu->ipa_size;
1834 if (smmu->version == ARM_SMMU_V1_64K)
1835 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1836 } else {
1837 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1838 smmu->va_size = arm_smmu_id_size_to_bits(size);
1839 if (id & ARM_SMMU_ID2_PTFS_4K)
1840 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1841 if (id & ARM_SMMU_ID2_PTFS_16K)
1842 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1843 if (id & ARM_SMMU_ID2_PTFS_64K)
1844 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1845 }
1846
1847 if (smmu->impl && smmu->impl->cfg_probe) {
1848 ret = smmu->impl->cfg_probe(smmu);
1849 if (ret)
1850 return ret;
1851 }
1852
1853 /* Now we've corralled the various formats, what'll it do? */
1854 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1855 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1856 if (smmu->features &
1857 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1858 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1859 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1860 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1861 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1862 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1863
1864 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1865 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1866 else
1867 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1868 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1869 smmu->pgsize_bitmap);
1870
1871
1872 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1873 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1874 smmu->va_size, smmu->ipa_size);
1875
1876 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1877 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1878 smmu->ipa_size, smmu->pa_size);
1879
1880 return 0;
1881 }
1882
1883 struct arm_smmu_match_data {
1884 enum arm_smmu_arch_version version;
1885 enum arm_smmu_implementation model;
1886 };
1887
1888 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1889 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1890
1891 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1892 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1893 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1894 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1895 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1896 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1897
1898 static const struct of_device_id arm_smmu_of_match[] = {
1899 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1900 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1901 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1902 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1903 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1904 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1905 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1906 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1907 { },
1908 };
1909 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1910
1911 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1912 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1913 {
1914 int ret = 0;
1915
1916 switch (model) {
1917 case ACPI_IORT_SMMU_V1:
1918 case ACPI_IORT_SMMU_CORELINK_MMU400:
1919 smmu->version = ARM_SMMU_V1;
1920 smmu->model = GENERIC_SMMU;
1921 break;
1922 case ACPI_IORT_SMMU_CORELINK_MMU401:
1923 smmu->version = ARM_SMMU_V1_64K;
1924 smmu->model = GENERIC_SMMU;
1925 break;
1926 case ACPI_IORT_SMMU_V2:
1927 smmu->version = ARM_SMMU_V2;
1928 smmu->model = GENERIC_SMMU;
1929 break;
1930 case ACPI_IORT_SMMU_CORELINK_MMU500:
1931 smmu->version = ARM_SMMU_V2;
1932 smmu->model = ARM_MMU500;
1933 break;
1934 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1935 smmu->version = ARM_SMMU_V2;
1936 smmu->model = CAVIUM_SMMUV2;
1937 break;
1938 default:
1939 ret = -ENODEV;
1940 }
1941
1942 return ret;
1943 }
1944
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1945 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1946 u32 *global_irqs, u32 *pmu_irqs)
1947 {
1948 struct device *dev = smmu->dev;
1949 struct acpi_iort_node *node =
1950 *(struct acpi_iort_node **)dev_get_platdata(dev);
1951 struct acpi_iort_smmu *iort_smmu;
1952 int ret;
1953
1954 /* Retrieve SMMU1/2 specific data */
1955 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1956
1957 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1958 if (ret < 0)
1959 return ret;
1960
1961 /* Ignore the configuration access interrupt */
1962 *global_irqs = 1;
1963 *pmu_irqs = 0;
1964
1965 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1966 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1967
1968 return 0;
1969 }
1970 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1971 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1972 u32 *global_irqs, u32 *pmu_irqs)
1973 {
1974 return -ENODEV;
1975 }
1976 #endif
1977
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1978 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
1979 u32 *global_irqs, u32 *pmu_irqs)
1980 {
1981 const struct arm_smmu_match_data *data;
1982 struct device *dev = smmu->dev;
1983 bool legacy_binding;
1984
1985 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
1986 return dev_err_probe(dev, -ENODEV,
1987 "missing #global-interrupts property\n");
1988 *pmu_irqs = 0;
1989
1990 data = of_device_get_match_data(dev);
1991 smmu->version = data->version;
1992 smmu->model = data->model;
1993
1994 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1995 if (legacy_binding && !using_generic_binding) {
1996 if (!using_legacy_binding) {
1997 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
1998 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
1999 }
2000 using_legacy_binding = true;
2001 } else if (!legacy_binding && !using_legacy_binding) {
2002 using_generic_binding = true;
2003 } else {
2004 dev_err(dev, "not probing due to mismatched DT properties\n");
2005 return -ENODEV;
2006 }
2007
2008 if (of_dma_is_coherent(dev->of_node))
2009 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2010
2011 return 0;
2012 }
2013
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2014 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2015 {
2016 struct list_head rmr_list;
2017 struct iommu_resv_region *e;
2018 int idx, cnt = 0;
2019 u32 reg;
2020
2021 INIT_LIST_HEAD(&rmr_list);
2022 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2023
2024 /*
2025 * Rather than trying to look at existing mappings that
2026 * are setup by the firmware and then invalidate the ones
2027 * that do no have matching RMR entries, just disable the
2028 * SMMU until it gets enabled again in the reset routine.
2029 */
2030 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2031 reg |= ARM_SMMU_sCR0_CLIENTPD;
2032 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2033
2034 list_for_each_entry(e, &rmr_list, list) {
2035 struct iommu_iort_rmr_data *rmr;
2036 int i;
2037
2038 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2039 for (i = 0; i < rmr->num_sids; i++) {
2040 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2041 if (idx < 0)
2042 continue;
2043
2044 if (smmu->s2crs[idx].count == 0) {
2045 smmu->smrs[idx].id = rmr->sids[i];
2046 smmu->smrs[idx].mask = 0;
2047 smmu->smrs[idx].valid = true;
2048 }
2049 smmu->s2crs[idx].count++;
2050 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2051 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2052
2053 cnt++;
2054 }
2055 }
2056
2057 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2058 cnt == 1 ? "" : "s");
2059 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2060 }
2061
arm_smmu_device_probe(struct platform_device * pdev)2062 static int arm_smmu_device_probe(struct platform_device *pdev)
2063 {
2064 struct resource *res;
2065 struct arm_smmu_device *smmu;
2066 struct device *dev = &pdev->dev;
2067 int num_irqs, i, err;
2068 u32 global_irqs, pmu_irqs;
2069 irqreturn_t (*global_fault)(int irq, void *dev);
2070
2071 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2072 if (!smmu) {
2073 dev_err(dev, "failed to allocate arm_smmu_device\n");
2074 return -ENOMEM;
2075 }
2076 smmu->dev = dev;
2077
2078 if (dev->of_node)
2079 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2080 else
2081 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2082 if (err)
2083 return err;
2084
2085 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2086 if (IS_ERR(smmu->base))
2087 return PTR_ERR(smmu->base);
2088 smmu->ioaddr = res->start;
2089
2090 /*
2091 * The resource size should effectively match the value of SMMU_TOP;
2092 * stash that temporarily until we know PAGESIZE to validate it with.
2093 */
2094 smmu->numpage = resource_size(res);
2095
2096 smmu = arm_smmu_impl_init(smmu);
2097 if (IS_ERR(smmu))
2098 return PTR_ERR(smmu);
2099
2100 num_irqs = platform_irq_count(pdev);
2101
2102 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2103 if (smmu->num_context_irqs <= 0)
2104 return dev_err_probe(dev, -ENODEV,
2105 "found %d interrupts but expected at least %d\n",
2106 num_irqs, global_irqs + pmu_irqs + 1);
2107
2108 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2109 sizeof(*smmu->irqs), GFP_KERNEL);
2110 if (!smmu->irqs)
2111 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2112 smmu->num_context_irqs);
2113
2114 for (i = 0; i < smmu->num_context_irqs; i++) {
2115 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2116
2117 if (irq < 0)
2118 return irq;
2119 smmu->irqs[i] = irq;
2120 }
2121
2122 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2123 if (err < 0) {
2124 dev_err(dev, "failed to get clocks %d\n", err);
2125 return err;
2126 }
2127 smmu->num_clks = err;
2128
2129 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2130 if (err)
2131 return err;
2132
2133 err = arm_smmu_device_cfg_probe(smmu);
2134 if (err)
2135 return err;
2136
2137 if (smmu->version == ARM_SMMU_V2) {
2138 if (smmu->num_context_banks > smmu->num_context_irqs) {
2139 dev_err(dev,
2140 "found only %d context irq(s) but %d required\n",
2141 smmu->num_context_irqs, smmu->num_context_banks);
2142 return -ENODEV;
2143 }
2144
2145 /* Ignore superfluous interrupts */
2146 smmu->num_context_irqs = smmu->num_context_banks;
2147 }
2148
2149 if (smmu->impl && smmu->impl->global_fault)
2150 global_fault = smmu->impl->global_fault;
2151 else
2152 global_fault = arm_smmu_global_fault;
2153
2154 for (i = 0; i < global_irqs; i++) {
2155 int irq = platform_get_irq(pdev, i);
2156
2157 if (irq < 0)
2158 return irq;
2159
2160 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2161 "arm-smmu global fault", smmu);
2162 if (err)
2163 return dev_err_probe(dev, err,
2164 "failed to request global IRQ %d (%u)\n",
2165 i, irq);
2166 }
2167
2168 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2169 "smmu.%pa", &smmu->ioaddr);
2170 if (err) {
2171 dev_err(dev, "Failed to register iommu in sysfs\n");
2172 return err;
2173 }
2174
2175 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
2176 if (err) {
2177 dev_err(dev, "Failed to register iommu\n");
2178 iommu_device_sysfs_remove(&smmu->iommu);
2179 return err;
2180 }
2181
2182 platform_set_drvdata(pdev, smmu);
2183
2184 /* Check for RMRs and install bypass SMRs if any */
2185 arm_smmu_rmr_install_bypass_smr(smmu);
2186
2187 arm_smmu_device_reset(smmu);
2188 arm_smmu_test_smr_masks(smmu);
2189
2190 /*
2191 * We want to avoid touching dev->power.lock in fastpaths unless
2192 * it's really going to do something useful - pm_runtime_enabled()
2193 * can serve as an ideal proxy for that decision. So, conditionally
2194 * enable pm_runtime.
2195 */
2196 if (dev->pm_domain) {
2197 pm_runtime_set_active(dev);
2198 pm_runtime_enable(dev);
2199 }
2200
2201 return 0;
2202 }
2203
arm_smmu_device_shutdown(struct platform_device * pdev)2204 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2205 {
2206 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2207
2208 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2209 dev_notice(&pdev->dev, "disabling translation\n");
2210
2211 arm_smmu_rpm_get(smmu);
2212 /* Turn the thing off */
2213 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2214 arm_smmu_rpm_put(smmu);
2215
2216 if (pm_runtime_enabled(smmu->dev))
2217 pm_runtime_force_suspend(smmu->dev);
2218 else
2219 clk_bulk_disable(smmu->num_clks, smmu->clks);
2220
2221 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2222 }
2223
arm_smmu_device_remove(struct platform_device * pdev)2224 static void arm_smmu_device_remove(struct platform_device *pdev)
2225 {
2226 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2227
2228 iommu_device_unregister(&smmu->iommu);
2229 iommu_device_sysfs_remove(&smmu->iommu);
2230
2231 arm_smmu_device_shutdown(pdev);
2232 }
2233
arm_smmu_runtime_resume(struct device * dev)2234 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2235 {
2236 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2237 int ret;
2238
2239 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2240 if (ret)
2241 return ret;
2242
2243 arm_smmu_device_reset(smmu);
2244
2245 return 0;
2246 }
2247
arm_smmu_runtime_suspend(struct device * dev)2248 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2249 {
2250 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2251
2252 clk_bulk_disable(smmu->num_clks, smmu->clks);
2253
2254 return 0;
2255 }
2256
arm_smmu_pm_resume(struct device * dev)2257 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2258 {
2259 int ret;
2260 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2261
2262 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2263 if (ret)
2264 return ret;
2265
2266 if (pm_runtime_suspended(dev))
2267 return 0;
2268
2269 ret = arm_smmu_runtime_resume(dev);
2270 if (ret)
2271 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2272
2273 return ret;
2274 }
2275
arm_smmu_pm_suspend(struct device * dev)2276 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2277 {
2278 int ret = 0;
2279 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2280
2281 if (pm_runtime_suspended(dev))
2282 goto clk_unprepare;
2283
2284 ret = arm_smmu_runtime_suspend(dev);
2285 if (ret)
2286 return ret;
2287
2288 clk_unprepare:
2289 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2290 return ret;
2291 }
2292
2293 static const struct dev_pm_ops arm_smmu_pm_ops = {
2294 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2295 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2296 arm_smmu_runtime_resume, NULL)
2297 };
2298
2299 static struct platform_driver arm_smmu_driver = {
2300 .driver = {
2301 .name = "arm-smmu",
2302 .of_match_table = arm_smmu_of_match,
2303 .pm = &arm_smmu_pm_ops,
2304 .suppress_bind_attrs = true,
2305 },
2306 .probe = arm_smmu_device_probe,
2307 .remove_new = arm_smmu_device_remove,
2308 .shutdown = arm_smmu_device_shutdown,
2309 };
2310 module_platform_driver(arm_smmu_driver);
2311
2312 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2313 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2314 MODULE_ALIAS("platform:arm-smmu");
2315 MODULE_LICENSE("GPL v2");
2316