1e86d1aa8SWill Deacon // SPDX-License-Identifier: GPL-2.0-only
2e86d1aa8SWill Deacon /*
3e86d1aa8SWill Deacon * IOMMU API for ARM architected SMMU implementations.
4e86d1aa8SWill Deacon *
5e86d1aa8SWill Deacon * Copyright (C) 2013 ARM Limited
6e86d1aa8SWill Deacon *
7e86d1aa8SWill Deacon * Author: Will Deacon <will.deacon@arm.com>
8e86d1aa8SWill Deacon *
9e86d1aa8SWill Deacon * This driver currently supports:
10e86d1aa8SWill Deacon * - SMMUv1 and v2 implementations
11e86d1aa8SWill Deacon * - Stream-matching and stream-indexing
12e86d1aa8SWill Deacon * - v7/v8 long-descriptor format
13e86d1aa8SWill Deacon * - Non-secure access to the SMMU
14e86d1aa8SWill Deacon * - Context fault reporting
15e86d1aa8SWill Deacon * - Extended Stream ID (16 bit)
16e86d1aa8SWill Deacon */
17e86d1aa8SWill Deacon
18e86d1aa8SWill Deacon #define pr_fmt(fmt) "arm-smmu: " fmt
19e86d1aa8SWill Deacon
20e86d1aa8SWill Deacon #include <linux/acpi.h>
21e86d1aa8SWill Deacon #include <linux/acpi_iort.h>
22e86d1aa8SWill Deacon #include <linux/bitfield.h>
23e86d1aa8SWill Deacon #include <linux/delay.h>
24e86d1aa8SWill Deacon #include <linux/dma-mapping.h>
25e86d1aa8SWill Deacon #include <linux/err.h>
26e86d1aa8SWill Deacon #include <linux/interrupt.h>
27e86d1aa8SWill Deacon #include <linux/io.h>
28e86d1aa8SWill Deacon #include <linux/iopoll.h>
29e86d1aa8SWill Deacon #include <linux/module.h>
30e86d1aa8SWill Deacon #include <linux/of.h>
31e86d1aa8SWill Deacon #include <linux/of_address.h>
32e86d1aa8SWill Deacon #include <linux/pci.h>
33e86d1aa8SWill Deacon #include <linux/platform_device.h>
34e86d1aa8SWill Deacon #include <linux/pm_runtime.h>
35e86d1aa8SWill Deacon #include <linux/ratelimit.h>
36e86d1aa8SWill Deacon #include <linux/slab.h>
37e86d1aa8SWill Deacon
38e86d1aa8SWill Deacon #include <linux/fsl/mc.h>
39e86d1aa8SWill Deacon
40e86d1aa8SWill Deacon #include "arm-smmu.h"
41f2042ed2SRobin Murphy #include "../../dma-iommu.h"
42e86d1aa8SWill Deacon
43e86d1aa8SWill Deacon /*
44e86d1aa8SWill Deacon * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
45e86d1aa8SWill Deacon * global register space are still, in fact, using a hypervisor to mediate it
46e86d1aa8SWill Deacon * by trapping and emulating register accesses. Sadly, some deployed versions
47e86d1aa8SWill Deacon * of said trapping code have bugs wherein they go horribly wrong for stores
48e86d1aa8SWill Deacon * using r31 (i.e. XZR/WZR) as the source register.
49e86d1aa8SWill Deacon */
50e86d1aa8SWill Deacon #define QCOM_DUMMY_VAL -1
51e86d1aa8SWill Deacon
52e86d1aa8SWill Deacon #define MSI_IOVA_BASE 0x8000000
53e86d1aa8SWill Deacon #define MSI_IOVA_LENGTH 0x100000
54e86d1aa8SWill Deacon
55e86d1aa8SWill Deacon static int force_stage;
56e86d1aa8SWill Deacon module_param(force_stage, int, S_IRUGO);
57e86d1aa8SWill Deacon MODULE_PARM_DESC(force_stage,
58e86d1aa8SWill Deacon "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
59e86d1aa8SWill Deacon static bool disable_bypass =
60e86d1aa8SWill Deacon IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
61e86d1aa8SWill Deacon module_param(disable_bypass, bool, S_IRUGO);
62e86d1aa8SWill Deacon MODULE_PARM_DESC(disable_bypass,
63e86d1aa8SWill Deacon "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
64e86d1aa8SWill Deacon
65e86d1aa8SWill Deacon #define s2cr_init_val (struct arm_smmu_s2cr){ \
66e86d1aa8SWill Deacon .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
67e86d1aa8SWill Deacon }
68e86d1aa8SWill Deacon
69e86d1aa8SWill Deacon static bool using_legacy_binding, using_generic_binding;
70e86d1aa8SWill Deacon
arm_smmu_rpm_get(struct arm_smmu_device * smmu)71e86d1aa8SWill Deacon static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
72e86d1aa8SWill Deacon {
73e86d1aa8SWill Deacon if (pm_runtime_enabled(smmu->dev))
741adf30f1SXiyu Yang return pm_runtime_resume_and_get(smmu->dev);
75e86d1aa8SWill Deacon
76e86d1aa8SWill Deacon return 0;
77e86d1aa8SWill Deacon }
78e86d1aa8SWill Deacon
arm_smmu_rpm_put(struct arm_smmu_device * smmu)79e86d1aa8SWill Deacon static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
80e86d1aa8SWill Deacon {
81e86d1aa8SWill Deacon if (pm_runtime_enabled(smmu->dev))
82e86d1aa8SWill Deacon pm_runtime_put_autosuspend(smmu->dev);
83e86d1aa8SWill Deacon }
84e86d1aa8SWill Deacon
to_smmu_domain(struct iommu_domain * dom)85e86d1aa8SWill Deacon static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
86e86d1aa8SWill Deacon {
87e86d1aa8SWill Deacon return container_of(dom, struct arm_smmu_domain, domain);
88e86d1aa8SWill Deacon }
89e86d1aa8SWill Deacon
90e86d1aa8SWill Deacon static struct platform_driver arm_smmu_driver;
91e86d1aa8SWill Deacon static struct iommu_ops arm_smmu_ops;
92e86d1aa8SWill Deacon
93e86d1aa8SWill Deacon #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)94e86d1aa8SWill Deacon static struct device_node *dev_get_dev_node(struct device *dev)
95e86d1aa8SWill Deacon {
96e86d1aa8SWill Deacon if (dev_is_pci(dev)) {
97e86d1aa8SWill Deacon struct pci_bus *bus = to_pci_dev(dev)->bus;
98e86d1aa8SWill Deacon
99e86d1aa8SWill Deacon while (!pci_is_root_bus(bus))
100e86d1aa8SWill Deacon bus = bus->parent;
101e86d1aa8SWill Deacon return of_node_get(bus->bridge->parent->of_node);
102e86d1aa8SWill Deacon }
103e86d1aa8SWill Deacon
104e86d1aa8SWill Deacon return of_node_get(dev->of_node);
105e86d1aa8SWill Deacon }
106e86d1aa8SWill Deacon
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)107e86d1aa8SWill Deacon static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
108e86d1aa8SWill Deacon {
109e86d1aa8SWill Deacon *((__be32 *)data) = cpu_to_be32(alias);
110e86d1aa8SWill Deacon return 0; /* Continue walking */
111e86d1aa8SWill Deacon }
112e86d1aa8SWill Deacon
__find_legacy_master_phandle(struct device * dev,void * data)113e86d1aa8SWill Deacon static int __find_legacy_master_phandle(struct device *dev, void *data)
114e86d1aa8SWill Deacon {
115e86d1aa8SWill Deacon struct of_phandle_iterator *it = *(void **)data;
116e86d1aa8SWill Deacon struct device_node *np = it->node;
117e86d1aa8SWill Deacon int err;
118e86d1aa8SWill Deacon
119e86d1aa8SWill Deacon of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
120e86d1aa8SWill Deacon "#stream-id-cells", -1)
121e86d1aa8SWill Deacon if (it->node == np) {
122e86d1aa8SWill Deacon *(void **)data = dev;
123e86d1aa8SWill Deacon return 1;
124e86d1aa8SWill Deacon }
125e86d1aa8SWill Deacon it->node = np;
126e86d1aa8SWill Deacon return err == -ENOENT ? 0 : err;
127e86d1aa8SWill Deacon }
128e86d1aa8SWill Deacon
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)129e86d1aa8SWill Deacon static int arm_smmu_register_legacy_master(struct device *dev,
130e86d1aa8SWill Deacon struct arm_smmu_device **smmu)
131e86d1aa8SWill Deacon {
132e86d1aa8SWill Deacon struct device *smmu_dev;
133e86d1aa8SWill Deacon struct device_node *np;
134e86d1aa8SWill Deacon struct of_phandle_iterator it;
135e86d1aa8SWill Deacon void *data = ⁢
136e86d1aa8SWill Deacon u32 *sids;
137e86d1aa8SWill Deacon __be32 pci_sid;
138e86d1aa8SWill Deacon int err;
139e86d1aa8SWill Deacon
140e86d1aa8SWill Deacon np = dev_get_dev_node(dev);
141a6c9e387SRob Herring if (!np || !of_property_present(np, "#stream-id-cells")) {
142e86d1aa8SWill Deacon of_node_put(np);
143e86d1aa8SWill Deacon return -ENODEV;
144e86d1aa8SWill Deacon }
145e86d1aa8SWill Deacon
146e86d1aa8SWill Deacon it.node = np;
147e86d1aa8SWill Deacon err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
148e86d1aa8SWill Deacon __find_legacy_master_phandle);
149e86d1aa8SWill Deacon smmu_dev = data;
150e86d1aa8SWill Deacon of_node_put(np);
151e86d1aa8SWill Deacon if (err == 0)
152e86d1aa8SWill Deacon return -ENODEV;
153e86d1aa8SWill Deacon if (err < 0)
154e86d1aa8SWill Deacon return err;
155e86d1aa8SWill Deacon
156e86d1aa8SWill Deacon if (dev_is_pci(dev)) {
157e86d1aa8SWill Deacon /* "mmu-masters" assumes Stream ID == Requester ID */
158e86d1aa8SWill Deacon pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
159e86d1aa8SWill Deacon &pci_sid);
160e86d1aa8SWill Deacon it.cur = &pci_sid;
161e86d1aa8SWill Deacon it.cur_count = 1;
162e86d1aa8SWill Deacon }
163e86d1aa8SWill Deacon
164e86d1aa8SWill Deacon err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
165e86d1aa8SWill Deacon &arm_smmu_ops);
166e86d1aa8SWill Deacon if (err)
167e86d1aa8SWill Deacon return err;
168e86d1aa8SWill Deacon
169e86d1aa8SWill Deacon sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
170e86d1aa8SWill Deacon if (!sids)
171e86d1aa8SWill Deacon return -ENOMEM;
172e86d1aa8SWill Deacon
173e86d1aa8SWill Deacon *smmu = dev_get_drvdata(smmu_dev);
174e86d1aa8SWill Deacon of_phandle_iterator_args(&it, sids, it.cur_count);
175e86d1aa8SWill Deacon err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
176e86d1aa8SWill Deacon kfree(sids);
177e86d1aa8SWill Deacon return err;
178e86d1aa8SWill Deacon }
179e86d1aa8SWill Deacon #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)180e86d1aa8SWill Deacon static int arm_smmu_register_legacy_master(struct device *dev,
181e86d1aa8SWill Deacon struct arm_smmu_device **smmu)
182e86d1aa8SWill Deacon {
183e86d1aa8SWill Deacon return -ENODEV;
184e86d1aa8SWill Deacon }
185e86d1aa8SWill Deacon #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
186e86d1aa8SWill Deacon
__arm_smmu_free_bitmap(unsigned long * map,int idx)187e86d1aa8SWill Deacon static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
188e86d1aa8SWill Deacon {
189e86d1aa8SWill Deacon clear_bit(idx, map);
190e86d1aa8SWill Deacon }
191e86d1aa8SWill Deacon
192e86d1aa8SWill Deacon /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)193e86d1aa8SWill Deacon static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
194e86d1aa8SWill Deacon int sync, int status)
195e86d1aa8SWill Deacon {
196e86d1aa8SWill Deacon unsigned int spin_cnt, delay;
197e86d1aa8SWill Deacon u32 reg;
198e86d1aa8SWill Deacon
199e86d1aa8SWill Deacon if (smmu->impl && unlikely(smmu->impl->tlb_sync))
200e86d1aa8SWill Deacon return smmu->impl->tlb_sync(smmu, page, sync, status);
201e86d1aa8SWill Deacon
202e86d1aa8SWill Deacon arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
203e86d1aa8SWill Deacon for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
204e86d1aa8SWill Deacon for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
205e86d1aa8SWill Deacon reg = arm_smmu_readl(smmu, page, status);
206e86d1aa8SWill Deacon if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
207e86d1aa8SWill Deacon return;
208e86d1aa8SWill Deacon cpu_relax();
209e86d1aa8SWill Deacon }
210e86d1aa8SWill Deacon udelay(delay);
211e86d1aa8SWill Deacon }
212e86d1aa8SWill Deacon dev_err_ratelimited(smmu->dev,
213e86d1aa8SWill Deacon "TLB sync timed out -- SMMU may be deadlocked\n");
214e86d1aa8SWill Deacon }
215e86d1aa8SWill Deacon
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)216e86d1aa8SWill Deacon static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
217e86d1aa8SWill Deacon {
218e86d1aa8SWill Deacon unsigned long flags;
219e86d1aa8SWill Deacon
220e86d1aa8SWill Deacon spin_lock_irqsave(&smmu->global_sync_lock, flags);
221e86d1aa8SWill Deacon __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
222e86d1aa8SWill Deacon ARM_SMMU_GR0_sTLBGSTATUS);
223e86d1aa8SWill Deacon spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
224e86d1aa8SWill Deacon }
225e86d1aa8SWill Deacon
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)226e86d1aa8SWill Deacon static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
227e86d1aa8SWill Deacon {
228e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
229e86d1aa8SWill Deacon unsigned long flags;
230e86d1aa8SWill Deacon
231e86d1aa8SWill Deacon spin_lock_irqsave(&smmu_domain->cb_lock, flags);
232e86d1aa8SWill Deacon __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
233e86d1aa8SWill Deacon ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
234e86d1aa8SWill Deacon spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
235e86d1aa8SWill Deacon }
236e86d1aa8SWill Deacon
arm_smmu_tlb_inv_context_s1(void * cookie)237e86d1aa8SWill Deacon static void arm_smmu_tlb_inv_context_s1(void *cookie)
238e86d1aa8SWill Deacon {
239e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = cookie;
240e86d1aa8SWill Deacon /*
241e86d1aa8SWill Deacon * The TLBI write may be relaxed, so ensure that PTEs cleared by the
242e86d1aa8SWill Deacon * current CPU are visible beforehand.
243e86d1aa8SWill Deacon */
244e86d1aa8SWill Deacon wmb();
245e86d1aa8SWill Deacon arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
246e86d1aa8SWill Deacon ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
247e86d1aa8SWill Deacon arm_smmu_tlb_sync_context(smmu_domain);
248e86d1aa8SWill Deacon }
249e86d1aa8SWill Deacon
arm_smmu_tlb_inv_context_s2(void * cookie)250e86d1aa8SWill Deacon static void arm_smmu_tlb_inv_context_s2(void *cookie)
251e86d1aa8SWill Deacon {
252e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = cookie;
253e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
254e86d1aa8SWill Deacon
255e86d1aa8SWill Deacon /* See above */
256e86d1aa8SWill Deacon wmb();
257e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
258e86d1aa8SWill Deacon arm_smmu_tlb_sync_global(smmu);
259e86d1aa8SWill Deacon }
260e86d1aa8SWill Deacon
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)261e86d1aa8SWill Deacon static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
262e86d1aa8SWill Deacon size_t granule, void *cookie, int reg)
263e86d1aa8SWill Deacon {
264e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = cookie;
265e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
266e86d1aa8SWill Deacon struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
267e86d1aa8SWill Deacon int idx = cfg->cbndx;
268e86d1aa8SWill Deacon
269e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
270e86d1aa8SWill Deacon wmb();
271e86d1aa8SWill Deacon
272e86d1aa8SWill Deacon if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
273e86d1aa8SWill Deacon iova = (iova >> 12) << 12;
274e86d1aa8SWill Deacon iova |= cfg->asid;
275e86d1aa8SWill Deacon do {
276e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, reg, iova);
277e86d1aa8SWill Deacon iova += granule;
278e86d1aa8SWill Deacon } while (size -= granule);
279e86d1aa8SWill Deacon } else {
280e86d1aa8SWill Deacon iova >>= 12;
281e86d1aa8SWill Deacon iova |= (u64)cfg->asid << 48;
282e86d1aa8SWill Deacon do {
283e86d1aa8SWill Deacon arm_smmu_cb_writeq(smmu, idx, reg, iova);
284e86d1aa8SWill Deacon iova += granule >> 12;
285e86d1aa8SWill Deacon } while (size -= granule);
286e86d1aa8SWill Deacon }
287e86d1aa8SWill Deacon }
288e86d1aa8SWill Deacon
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)289e86d1aa8SWill Deacon static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
290e86d1aa8SWill Deacon size_t granule, void *cookie, int reg)
291e86d1aa8SWill Deacon {
292e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = cookie;
293e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
294e86d1aa8SWill Deacon int idx = smmu_domain->cfg.cbndx;
295e86d1aa8SWill Deacon
296e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
297e86d1aa8SWill Deacon wmb();
298e86d1aa8SWill Deacon
299e86d1aa8SWill Deacon iova >>= 12;
300e86d1aa8SWill Deacon do {
301e86d1aa8SWill Deacon if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
302e86d1aa8SWill Deacon arm_smmu_cb_writeq(smmu, idx, reg, iova);
303e86d1aa8SWill Deacon else
304e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, reg, iova);
305e86d1aa8SWill Deacon iova += granule >> 12;
306e86d1aa8SWill Deacon } while (size -= granule);
307e86d1aa8SWill Deacon }
308e86d1aa8SWill Deacon
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)309e86d1aa8SWill Deacon static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
310e86d1aa8SWill Deacon size_t granule, void *cookie)
311e86d1aa8SWill Deacon {
312ef75702dSSai Prakash Ranjan struct arm_smmu_domain *smmu_domain = cookie;
313ef75702dSSai Prakash Ranjan struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
314ef75702dSSai Prakash Ranjan
315ef75702dSSai Prakash Ranjan if (cfg->flush_walk_prefer_tlbiasid) {
316ef75702dSSai Prakash Ranjan arm_smmu_tlb_inv_context_s1(cookie);
317ef75702dSSai Prakash Ranjan } else {
318e86d1aa8SWill Deacon arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
319e86d1aa8SWill Deacon ARM_SMMU_CB_S1_TLBIVA);
320e86d1aa8SWill Deacon arm_smmu_tlb_sync_context(cookie);
321e86d1aa8SWill Deacon }
322ef75702dSSai Prakash Ranjan }
323e86d1aa8SWill Deacon
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)324e86d1aa8SWill Deacon static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
325e86d1aa8SWill Deacon unsigned long iova, size_t granule,
326e86d1aa8SWill Deacon void *cookie)
327e86d1aa8SWill Deacon {
328e86d1aa8SWill Deacon arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
329e86d1aa8SWill Deacon ARM_SMMU_CB_S1_TLBIVAL);
330e86d1aa8SWill Deacon }
331e86d1aa8SWill Deacon
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)332e86d1aa8SWill Deacon static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
333e86d1aa8SWill Deacon size_t granule, void *cookie)
334e86d1aa8SWill Deacon {
335e86d1aa8SWill Deacon arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
336e86d1aa8SWill Deacon ARM_SMMU_CB_S2_TLBIIPAS2);
337e86d1aa8SWill Deacon arm_smmu_tlb_sync_context(cookie);
338e86d1aa8SWill Deacon }
339e86d1aa8SWill Deacon
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)340e86d1aa8SWill Deacon static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
341e86d1aa8SWill Deacon unsigned long iova, size_t granule,
342e86d1aa8SWill Deacon void *cookie)
343e86d1aa8SWill Deacon {
344e86d1aa8SWill Deacon arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
345e86d1aa8SWill Deacon ARM_SMMU_CB_S2_TLBIIPAS2L);
346e86d1aa8SWill Deacon }
347e86d1aa8SWill Deacon
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)348fefe8527SRobin Murphy static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
349e86d1aa8SWill Deacon size_t granule, void *cookie)
350e86d1aa8SWill Deacon {
351e86d1aa8SWill Deacon arm_smmu_tlb_inv_context_s2(cookie);
352e86d1aa8SWill Deacon }
353e86d1aa8SWill Deacon /*
354e86d1aa8SWill Deacon * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
355e86d1aa8SWill Deacon * almost negligible, but the benefit of getting the first one in as far ahead
356e86d1aa8SWill Deacon * of the sync as possible is significant, hence we don't just make this a
357e86d1aa8SWill Deacon * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
358e86d1aa8SWill Deacon * think.
359e86d1aa8SWill Deacon */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)360e86d1aa8SWill Deacon static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
361e86d1aa8SWill Deacon unsigned long iova, size_t granule,
362e86d1aa8SWill Deacon void *cookie)
363e86d1aa8SWill Deacon {
364e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = cookie;
365e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
366e86d1aa8SWill Deacon
367e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
368e86d1aa8SWill Deacon wmb();
369e86d1aa8SWill Deacon
370e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
371e86d1aa8SWill Deacon }
372e86d1aa8SWill Deacon
373e86d1aa8SWill Deacon static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
374e86d1aa8SWill Deacon .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
375e86d1aa8SWill Deacon .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
376e86d1aa8SWill Deacon .tlb_add_page = arm_smmu_tlb_add_page_s1,
377e86d1aa8SWill Deacon };
378e86d1aa8SWill Deacon
379e86d1aa8SWill Deacon static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
380e86d1aa8SWill Deacon .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
381e86d1aa8SWill Deacon .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
382e86d1aa8SWill Deacon .tlb_add_page = arm_smmu_tlb_add_page_s2,
383e86d1aa8SWill Deacon };
384e86d1aa8SWill Deacon
385e86d1aa8SWill Deacon static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
386e86d1aa8SWill Deacon .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
387fefe8527SRobin Murphy .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
388e86d1aa8SWill Deacon .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
389e86d1aa8SWill Deacon };
390e86d1aa8SWill Deacon
arm_smmu_context_fault(int irq,void * dev)391e86d1aa8SWill Deacon static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
392e86d1aa8SWill Deacon {
393e86d1aa8SWill Deacon u32 fsr, fsynr, cbfrsynra;
394e86d1aa8SWill Deacon unsigned long iova;
395e86d1aa8SWill Deacon struct iommu_domain *domain = dev;
396e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
397e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
398e86d1aa8SWill Deacon int idx = smmu_domain->cfg.cbndx;
399f8f934c1SJordan Crouse int ret;
400e86d1aa8SWill Deacon
401e86d1aa8SWill Deacon fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
402e86d1aa8SWill Deacon if (!(fsr & ARM_SMMU_FSR_FAULT))
403e86d1aa8SWill Deacon return IRQ_NONE;
404e86d1aa8SWill Deacon
405e86d1aa8SWill Deacon fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
406e86d1aa8SWill Deacon iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
407e86d1aa8SWill Deacon cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
408e86d1aa8SWill Deacon
409f8f934c1SJordan Crouse ret = report_iommu_fault(domain, NULL, iova,
410f8f934c1SJordan Crouse fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
411f8f934c1SJordan Crouse
412f8f934c1SJordan Crouse if (ret == -ENOSYS)
413e86d1aa8SWill Deacon dev_err_ratelimited(smmu->dev,
414e86d1aa8SWill Deacon "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
415e86d1aa8SWill Deacon fsr, iova, fsynr, cbfrsynra, idx);
416e86d1aa8SWill Deacon
417e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
418e86d1aa8SWill Deacon return IRQ_HANDLED;
419e86d1aa8SWill Deacon }
420e86d1aa8SWill Deacon
arm_smmu_global_fault(int irq,void * dev)421e86d1aa8SWill Deacon static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
422e86d1aa8SWill Deacon {
423e86d1aa8SWill Deacon u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
424e86d1aa8SWill Deacon struct arm_smmu_device *smmu = dev;
425e86d1aa8SWill Deacon static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
426e86d1aa8SWill Deacon DEFAULT_RATELIMIT_BURST);
427e86d1aa8SWill Deacon
428e86d1aa8SWill Deacon gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
429e86d1aa8SWill Deacon gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
430e86d1aa8SWill Deacon gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
431e86d1aa8SWill Deacon gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
432e86d1aa8SWill Deacon
433e86d1aa8SWill Deacon if (!gfsr)
434e86d1aa8SWill Deacon return IRQ_NONE;
435e86d1aa8SWill Deacon
436e86d1aa8SWill Deacon if (__ratelimit(&rs)) {
437e86d1aa8SWill Deacon if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
438e86d1aa8SWill Deacon (gfsr & ARM_SMMU_sGFSR_USF))
439e86d1aa8SWill Deacon dev_err(smmu->dev,
440e86d1aa8SWill Deacon "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
441e86d1aa8SWill Deacon (u16)gfsynr1);
442e86d1aa8SWill Deacon else
443e86d1aa8SWill Deacon dev_err(smmu->dev,
444e86d1aa8SWill Deacon "Unexpected global fault, this could be serious\n");
445e86d1aa8SWill Deacon dev_err(smmu->dev,
446e86d1aa8SWill Deacon "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
447e86d1aa8SWill Deacon gfsr, gfsynr0, gfsynr1, gfsynr2);
448e86d1aa8SWill Deacon }
449e86d1aa8SWill Deacon
450e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
451e86d1aa8SWill Deacon return IRQ_HANDLED;
452e86d1aa8SWill Deacon }
453e86d1aa8SWill Deacon
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)454e86d1aa8SWill Deacon static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
455e86d1aa8SWill Deacon struct io_pgtable_cfg *pgtbl_cfg)
456e86d1aa8SWill Deacon {
457e86d1aa8SWill Deacon struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
458e86d1aa8SWill Deacon struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
459e86d1aa8SWill Deacon bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
460e86d1aa8SWill Deacon
461e86d1aa8SWill Deacon cb->cfg = cfg;
462e86d1aa8SWill Deacon
463e86d1aa8SWill Deacon /* TCR */
464e86d1aa8SWill Deacon if (stage1) {
465e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
466e86d1aa8SWill Deacon cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
467e86d1aa8SWill Deacon } else {
468e86d1aa8SWill Deacon cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
469e86d1aa8SWill Deacon cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
470e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
471e86d1aa8SWill Deacon cb->tcr[1] |= ARM_SMMU_TCR2_AS;
472e86d1aa8SWill Deacon else
473e86d1aa8SWill Deacon cb->tcr[0] |= ARM_SMMU_TCR_EAE;
474e86d1aa8SWill Deacon }
475e86d1aa8SWill Deacon } else {
476e86d1aa8SWill Deacon cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
477e86d1aa8SWill Deacon }
478e86d1aa8SWill Deacon
479e86d1aa8SWill Deacon /* TTBRs */
480e86d1aa8SWill Deacon if (stage1) {
481e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
482e86d1aa8SWill Deacon cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
483e86d1aa8SWill Deacon cb->ttbr[1] = 0;
484e86d1aa8SWill Deacon } else {
48567f1a7a3SJordan Crouse cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
486e86d1aa8SWill Deacon cfg->asid);
487e86d1aa8SWill Deacon cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
488e86d1aa8SWill Deacon cfg->asid);
48967f1a7a3SJordan Crouse
49067f1a7a3SJordan Crouse if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
49167f1a7a3SJordan Crouse cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
49267f1a7a3SJordan Crouse else
49367f1a7a3SJordan Crouse cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
494e86d1aa8SWill Deacon }
495e86d1aa8SWill Deacon } else {
496e86d1aa8SWill Deacon cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
497e86d1aa8SWill Deacon }
498e86d1aa8SWill Deacon
499e86d1aa8SWill Deacon /* MAIRs (stage-1 only) */
500e86d1aa8SWill Deacon if (stage1) {
501e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
502e86d1aa8SWill Deacon cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
503e86d1aa8SWill Deacon cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
504e86d1aa8SWill Deacon } else {
505e86d1aa8SWill Deacon cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
506e86d1aa8SWill Deacon cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
507e86d1aa8SWill Deacon }
508e86d1aa8SWill Deacon }
509e86d1aa8SWill Deacon }
510e86d1aa8SWill Deacon
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)511556db53aSJordan Crouse void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
512e86d1aa8SWill Deacon {
513e86d1aa8SWill Deacon u32 reg;
514e86d1aa8SWill Deacon bool stage1;
515e86d1aa8SWill Deacon struct arm_smmu_cb *cb = &smmu->cbs[idx];
516e86d1aa8SWill Deacon struct arm_smmu_cfg *cfg = cb->cfg;
517e86d1aa8SWill Deacon
518e86d1aa8SWill Deacon /* Unassigned context banks only need disabling */
519e86d1aa8SWill Deacon if (!cfg) {
520e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
521e86d1aa8SWill Deacon return;
522e86d1aa8SWill Deacon }
523e86d1aa8SWill Deacon
524e86d1aa8SWill Deacon stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
525e86d1aa8SWill Deacon
526e86d1aa8SWill Deacon /* CBA2R */
527e86d1aa8SWill Deacon if (smmu->version > ARM_SMMU_V1) {
528e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
529e86d1aa8SWill Deacon reg = ARM_SMMU_CBA2R_VA64;
530e86d1aa8SWill Deacon else
531e86d1aa8SWill Deacon reg = 0;
532e86d1aa8SWill Deacon /* 16-bit VMIDs live in CBA2R */
533e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_VMID16)
534e86d1aa8SWill Deacon reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
535e86d1aa8SWill Deacon
536e86d1aa8SWill Deacon arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
537e86d1aa8SWill Deacon }
538e86d1aa8SWill Deacon
539e86d1aa8SWill Deacon /* CBAR */
540e86d1aa8SWill Deacon reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
541e86d1aa8SWill Deacon if (smmu->version < ARM_SMMU_V2)
542e86d1aa8SWill Deacon reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
543e86d1aa8SWill Deacon
544e86d1aa8SWill Deacon /*
545e86d1aa8SWill Deacon * Use the weakest shareability/memory types, so they are
546e86d1aa8SWill Deacon * overridden by the ttbcr/pte.
547e86d1aa8SWill Deacon */
548e86d1aa8SWill Deacon if (stage1) {
549e86d1aa8SWill Deacon reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
550e86d1aa8SWill Deacon ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
551e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
552e86d1aa8SWill Deacon ARM_SMMU_CBAR_S1_MEMATTR_WB);
553e86d1aa8SWill Deacon } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
554e86d1aa8SWill Deacon /* 8-bit VMIDs live in CBAR */
555e86d1aa8SWill Deacon reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
556e86d1aa8SWill Deacon }
557e86d1aa8SWill Deacon arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
558e86d1aa8SWill Deacon
559e86d1aa8SWill Deacon /*
560e86d1aa8SWill Deacon * TCR
561e86d1aa8SWill Deacon * We must write this before the TTBRs, since it determines the
562e86d1aa8SWill Deacon * access behaviour of some fields (in particular, ASID[15:8]).
563e86d1aa8SWill Deacon */
564e86d1aa8SWill Deacon if (stage1 && smmu->version > ARM_SMMU_V1)
565e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
566e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
567e86d1aa8SWill Deacon
568e86d1aa8SWill Deacon /* TTBRs */
569e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
570e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
571e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
572e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
573e86d1aa8SWill Deacon } else {
574e86d1aa8SWill Deacon arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
575e86d1aa8SWill Deacon if (stage1)
576e86d1aa8SWill Deacon arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
577e86d1aa8SWill Deacon cb->ttbr[1]);
578e86d1aa8SWill Deacon }
579e86d1aa8SWill Deacon
580e86d1aa8SWill Deacon /* MAIRs (stage-1 only) */
581e86d1aa8SWill Deacon if (stage1) {
582e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
583e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
584e86d1aa8SWill Deacon }
585e86d1aa8SWill Deacon
586e86d1aa8SWill Deacon /* SCTLR */
587e86d1aa8SWill Deacon reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
588e86d1aa8SWill Deacon ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
589e86d1aa8SWill Deacon if (stage1)
590e86d1aa8SWill Deacon reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
591e86d1aa8SWill Deacon if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
592e86d1aa8SWill Deacon reg |= ARM_SMMU_SCTLR_E;
593e86d1aa8SWill Deacon
594bffb2eafSRob Clark if (smmu->impl && smmu->impl->write_sctlr)
595bffb2eafSRob Clark smmu->impl->write_sctlr(smmu, idx, reg);
596bffb2eafSRob Clark else
597e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
598e86d1aa8SWill Deacon }
599e86d1aa8SWill Deacon
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)600556db53aSJordan Crouse static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
601556db53aSJordan Crouse struct arm_smmu_device *smmu,
602556db53aSJordan Crouse struct device *dev, unsigned int start)
603556db53aSJordan Crouse {
604556db53aSJordan Crouse if (smmu->impl && smmu->impl->alloc_context_bank)
605556db53aSJordan Crouse return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
606556db53aSJordan Crouse
607556db53aSJordan Crouse return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
608556db53aSJordan Crouse }
609556db53aSJordan Crouse
arm_smmu_init_domain_context(struct iommu_domain * domain,struct arm_smmu_device * smmu,struct device * dev)610e86d1aa8SWill Deacon static int arm_smmu_init_domain_context(struct iommu_domain *domain,
611556db53aSJordan Crouse struct arm_smmu_device *smmu,
612556db53aSJordan Crouse struct device *dev)
613e86d1aa8SWill Deacon {
614e86d1aa8SWill Deacon int irq, start, ret = 0;
615e86d1aa8SWill Deacon unsigned long ias, oas;
616e86d1aa8SWill Deacon struct io_pgtable_ops *pgtbl_ops;
617e86d1aa8SWill Deacon struct io_pgtable_cfg pgtbl_cfg;
618e86d1aa8SWill Deacon enum io_pgtable_fmt fmt;
619e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
620e86d1aa8SWill Deacon struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
621e86d1aa8SWill Deacon irqreturn_t (*context_fault)(int irq, void *dev);
622e86d1aa8SWill Deacon
623e86d1aa8SWill Deacon mutex_lock(&smmu_domain->init_mutex);
624e86d1aa8SWill Deacon if (smmu_domain->smmu)
625e86d1aa8SWill Deacon goto out_unlock;
626e86d1aa8SWill Deacon
627e86d1aa8SWill Deacon if (domain->type == IOMMU_DOMAIN_IDENTITY) {
628e86d1aa8SWill Deacon smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
629e86d1aa8SWill Deacon smmu_domain->smmu = smmu;
630e86d1aa8SWill Deacon goto out_unlock;
631e86d1aa8SWill Deacon }
632e86d1aa8SWill Deacon
633e86d1aa8SWill Deacon /*
634e86d1aa8SWill Deacon * Mapping the requested stage onto what we support is surprisingly
635e86d1aa8SWill Deacon * complicated, mainly because the spec allows S1+S2 SMMUs without
636e86d1aa8SWill Deacon * support for nested translation. That means we end up with the
637e86d1aa8SWill Deacon * following table:
638e86d1aa8SWill Deacon *
639e86d1aa8SWill Deacon * Requested Supported Actual
640e86d1aa8SWill Deacon * S1 N S1
641e86d1aa8SWill Deacon * S1 S1+S2 S1
642e86d1aa8SWill Deacon * S1 S2 S2
643e86d1aa8SWill Deacon * S1 S1 S1
644e86d1aa8SWill Deacon * N N N
645e86d1aa8SWill Deacon * N S1+S2 S2
646e86d1aa8SWill Deacon * N S2 S2
647e86d1aa8SWill Deacon * N S1 S1
648e86d1aa8SWill Deacon *
649e86d1aa8SWill Deacon * Note that you can't actually request stage-2 mappings.
650e86d1aa8SWill Deacon */
651e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
652e86d1aa8SWill Deacon smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
653e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
654e86d1aa8SWill Deacon smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
655e86d1aa8SWill Deacon
656e86d1aa8SWill Deacon /*
657e86d1aa8SWill Deacon * Choosing a suitable context format is even more fiddly. Until we
658e86d1aa8SWill Deacon * grow some way for the caller to express a preference, and/or move
659e86d1aa8SWill Deacon * the decision into the io-pgtable code where it arguably belongs,
660e86d1aa8SWill Deacon * just aim for the closest thing to the rest of the system, and hope
661e86d1aa8SWill Deacon * that the hardware isn't esoteric enough that we can't assume AArch64
662e86d1aa8SWill Deacon * support to be a superset of AArch32 support...
663e86d1aa8SWill Deacon */
664e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
665e86d1aa8SWill Deacon cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
666e86d1aa8SWill Deacon if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
667e86d1aa8SWill Deacon !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
668e86d1aa8SWill Deacon (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
669e86d1aa8SWill Deacon (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
670e86d1aa8SWill Deacon cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
671e86d1aa8SWill Deacon if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
672e86d1aa8SWill Deacon (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
673e86d1aa8SWill Deacon ARM_SMMU_FEAT_FMT_AARCH64_16K |
674e86d1aa8SWill Deacon ARM_SMMU_FEAT_FMT_AARCH64_4K)))
675e86d1aa8SWill Deacon cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
676e86d1aa8SWill Deacon
677e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
678e86d1aa8SWill Deacon ret = -EINVAL;
679e86d1aa8SWill Deacon goto out_unlock;
680e86d1aa8SWill Deacon }
681e86d1aa8SWill Deacon
682e86d1aa8SWill Deacon switch (smmu_domain->stage) {
683e86d1aa8SWill Deacon case ARM_SMMU_DOMAIN_S1:
684e86d1aa8SWill Deacon cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
685e86d1aa8SWill Deacon start = smmu->num_s2_context_banks;
686e86d1aa8SWill Deacon ias = smmu->va_size;
687e86d1aa8SWill Deacon oas = smmu->ipa_size;
688e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
689e86d1aa8SWill Deacon fmt = ARM_64_LPAE_S1;
690e86d1aa8SWill Deacon } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
691e86d1aa8SWill Deacon fmt = ARM_32_LPAE_S1;
692e86d1aa8SWill Deacon ias = min(ias, 32UL);
693e86d1aa8SWill Deacon oas = min(oas, 40UL);
694e86d1aa8SWill Deacon } else {
695e86d1aa8SWill Deacon fmt = ARM_V7S;
696e86d1aa8SWill Deacon ias = min(ias, 32UL);
697e86d1aa8SWill Deacon oas = min(oas, 32UL);
698e86d1aa8SWill Deacon }
699e86d1aa8SWill Deacon smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
700e86d1aa8SWill Deacon break;
701e86d1aa8SWill Deacon case ARM_SMMU_DOMAIN_NESTED:
702e86d1aa8SWill Deacon /*
703e86d1aa8SWill Deacon * We will likely want to change this if/when KVM gets
704e86d1aa8SWill Deacon * involved.
705e86d1aa8SWill Deacon */
706e86d1aa8SWill Deacon case ARM_SMMU_DOMAIN_S2:
707e86d1aa8SWill Deacon cfg->cbar = CBAR_TYPE_S2_TRANS;
708e86d1aa8SWill Deacon start = 0;
709e86d1aa8SWill Deacon ias = smmu->ipa_size;
710e86d1aa8SWill Deacon oas = smmu->pa_size;
711e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
712e86d1aa8SWill Deacon fmt = ARM_64_LPAE_S2;
713e86d1aa8SWill Deacon } else {
714e86d1aa8SWill Deacon fmt = ARM_32_LPAE_S2;
715e86d1aa8SWill Deacon ias = min(ias, 40UL);
716e86d1aa8SWill Deacon oas = min(oas, 40UL);
717e86d1aa8SWill Deacon }
718e86d1aa8SWill Deacon if (smmu->version == ARM_SMMU_V2)
719e86d1aa8SWill Deacon smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
720e86d1aa8SWill Deacon else
721e86d1aa8SWill Deacon smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
722e86d1aa8SWill Deacon break;
723e86d1aa8SWill Deacon default:
724e86d1aa8SWill Deacon ret = -EINVAL;
725e86d1aa8SWill Deacon goto out_unlock;
726e86d1aa8SWill Deacon }
727556db53aSJordan Crouse
728556db53aSJordan Crouse ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
729556db53aSJordan Crouse if (ret < 0) {
730e86d1aa8SWill Deacon goto out_unlock;
731556db53aSJordan Crouse }
732556db53aSJordan Crouse
733556db53aSJordan Crouse smmu_domain->smmu = smmu;
734e86d1aa8SWill Deacon
735e86d1aa8SWill Deacon cfg->cbndx = ret;
736e86d1aa8SWill Deacon if (smmu->version < ARM_SMMU_V2) {
737e86d1aa8SWill Deacon cfg->irptndx = atomic_inc_return(&smmu->irptndx);
738e86d1aa8SWill Deacon cfg->irptndx %= smmu->num_context_irqs;
739e86d1aa8SWill Deacon } else {
740e86d1aa8SWill Deacon cfg->irptndx = cfg->cbndx;
741e86d1aa8SWill Deacon }
742e86d1aa8SWill Deacon
743e86d1aa8SWill Deacon if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
744e86d1aa8SWill Deacon cfg->vmid = cfg->cbndx + 1;
745e86d1aa8SWill Deacon else
746e86d1aa8SWill Deacon cfg->asid = cfg->cbndx;
747e86d1aa8SWill Deacon
748e86d1aa8SWill Deacon pgtbl_cfg = (struct io_pgtable_cfg) {
749e86d1aa8SWill Deacon .pgsize_bitmap = smmu->pgsize_bitmap,
750e86d1aa8SWill Deacon .ias = ias,
751e86d1aa8SWill Deacon .oas = oas,
752e86d1aa8SWill Deacon .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
753e86d1aa8SWill Deacon .tlb = smmu_domain->flush_ops,
754e86d1aa8SWill Deacon .iommu_dev = smmu->dev,
755e86d1aa8SWill Deacon };
756e86d1aa8SWill Deacon
757dd147a89SJordan Crouse if (smmu->impl && smmu->impl->init_context) {
758556db53aSJordan Crouse ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
759dd147a89SJordan Crouse if (ret)
760dd147a89SJordan Crouse goto out_clear_smmu;
761dd147a89SJordan Crouse }
762dd147a89SJordan Crouse
7634fc52b81SChristoph Hellwig if (smmu_domain->pgtbl_quirks)
7644fc52b81SChristoph Hellwig pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
765c99110a8SSai Prakash Ranjan
766e86d1aa8SWill Deacon pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
767e86d1aa8SWill Deacon if (!pgtbl_ops) {
768e86d1aa8SWill Deacon ret = -ENOMEM;
769e86d1aa8SWill Deacon goto out_clear_smmu;
770e86d1aa8SWill Deacon }
771e86d1aa8SWill Deacon
772e86d1aa8SWill Deacon /* Update the domain's page sizes to reflect the page table format */
773e86d1aa8SWill Deacon domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
77467f1a7a3SJordan Crouse
77567f1a7a3SJordan Crouse if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
77667f1a7a3SJordan Crouse domain->geometry.aperture_start = ~0UL << ias;
77767f1a7a3SJordan Crouse domain->geometry.aperture_end = ~0UL;
77867f1a7a3SJordan Crouse } else {
779e86d1aa8SWill Deacon domain->geometry.aperture_end = (1UL << ias) - 1;
78067f1a7a3SJordan Crouse }
78167f1a7a3SJordan Crouse
782e86d1aa8SWill Deacon domain->geometry.force_aperture = true;
783e86d1aa8SWill Deacon
784e86d1aa8SWill Deacon /* Initialise the context bank with our page table cfg */
785e86d1aa8SWill Deacon arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
786e86d1aa8SWill Deacon arm_smmu_write_context_bank(smmu, cfg->cbndx);
787e86d1aa8SWill Deacon
788e86d1aa8SWill Deacon /*
789e86d1aa8SWill Deacon * Request context fault interrupt. Do this last to avoid the
790e86d1aa8SWill Deacon * handler seeing a half-initialised domain state.
791e86d1aa8SWill Deacon */
79297dfad19SRobin Murphy irq = smmu->irqs[cfg->irptndx];
793e86d1aa8SWill Deacon
794e86d1aa8SWill Deacon if (smmu->impl && smmu->impl->context_fault)
795e86d1aa8SWill Deacon context_fault = smmu->impl->context_fault;
796e86d1aa8SWill Deacon else
797e86d1aa8SWill Deacon context_fault = arm_smmu_context_fault;
798e86d1aa8SWill Deacon
799e86d1aa8SWill Deacon ret = devm_request_irq(smmu->dev, irq, context_fault,
800e86d1aa8SWill Deacon IRQF_SHARED, "arm-smmu-context-fault", domain);
801e86d1aa8SWill Deacon if (ret < 0) {
802e86d1aa8SWill Deacon dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
803e86d1aa8SWill Deacon cfg->irptndx, irq);
804e86d1aa8SWill Deacon cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
805e86d1aa8SWill Deacon }
806e86d1aa8SWill Deacon
807e86d1aa8SWill Deacon mutex_unlock(&smmu_domain->init_mutex);
808e86d1aa8SWill Deacon
809e86d1aa8SWill Deacon /* Publish page table ops for map/unmap */
810e86d1aa8SWill Deacon smmu_domain->pgtbl_ops = pgtbl_ops;
811e86d1aa8SWill Deacon return 0;
812e86d1aa8SWill Deacon
813e86d1aa8SWill Deacon out_clear_smmu:
814e86d1aa8SWill Deacon __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
815e86d1aa8SWill Deacon smmu_domain->smmu = NULL;
816e86d1aa8SWill Deacon out_unlock:
817e86d1aa8SWill Deacon mutex_unlock(&smmu_domain->init_mutex);
818e86d1aa8SWill Deacon return ret;
819e86d1aa8SWill Deacon }
820e86d1aa8SWill Deacon
arm_smmu_destroy_domain_context(struct iommu_domain * domain)821e86d1aa8SWill Deacon static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
822e86d1aa8SWill Deacon {
823e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
824e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
825e86d1aa8SWill Deacon struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
826e86d1aa8SWill Deacon int ret, irq;
827e86d1aa8SWill Deacon
828e86d1aa8SWill Deacon if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
829e86d1aa8SWill Deacon return;
830e86d1aa8SWill Deacon
831e86d1aa8SWill Deacon ret = arm_smmu_rpm_get(smmu);
832e86d1aa8SWill Deacon if (ret < 0)
833e86d1aa8SWill Deacon return;
834e86d1aa8SWill Deacon
835e86d1aa8SWill Deacon /*
836e86d1aa8SWill Deacon * Disable the context bank and free the page tables before freeing
837e86d1aa8SWill Deacon * it.
838e86d1aa8SWill Deacon */
839e86d1aa8SWill Deacon smmu->cbs[cfg->cbndx].cfg = NULL;
840e86d1aa8SWill Deacon arm_smmu_write_context_bank(smmu, cfg->cbndx);
841e86d1aa8SWill Deacon
842e86d1aa8SWill Deacon if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
84397dfad19SRobin Murphy irq = smmu->irqs[cfg->irptndx];
844e86d1aa8SWill Deacon devm_free_irq(smmu->dev, irq, domain);
845e86d1aa8SWill Deacon }
846e86d1aa8SWill Deacon
847e86d1aa8SWill Deacon free_io_pgtable_ops(smmu_domain->pgtbl_ops);
848e86d1aa8SWill Deacon __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
849e86d1aa8SWill Deacon
850e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
851e86d1aa8SWill Deacon }
852e86d1aa8SWill Deacon
arm_smmu_domain_alloc(unsigned type)853e86d1aa8SWill Deacon static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
854e86d1aa8SWill Deacon {
855e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain;
856e86d1aa8SWill Deacon
857229496a0SRobin Murphy if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) {
858*a4fdd976SRobin Murphy if (using_legacy_binding || type != IOMMU_DOMAIN_DMA)
859e86d1aa8SWill Deacon return NULL;
860229496a0SRobin Murphy }
861e86d1aa8SWill Deacon /*
862e86d1aa8SWill Deacon * Allocate the domain and initialise some of its data structures.
863e86d1aa8SWill Deacon * We can't really do anything meaningful until we've added a
864e86d1aa8SWill Deacon * master.
865e86d1aa8SWill Deacon */
866e86d1aa8SWill Deacon smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
867e86d1aa8SWill Deacon if (!smmu_domain)
868e86d1aa8SWill Deacon return NULL;
869e86d1aa8SWill Deacon
870e86d1aa8SWill Deacon mutex_init(&smmu_domain->init_mutex);
871e86d1aa8SWill Deacon spin_lock_init(&smmu_domain->cb_lock);
872e86d1aa8SWill Deacon
873e86d1aa8SWill Deacon return &smmu_domain->domain;
874e86d1aa8SWill Deacon }
875e86d1aa8SWill Deacon
arm_smmu_domain_free(struct iommu_domain * domain)876e86d1aa8SWill Deacon static void arm_smmu_domain_free(struct iommu_domain *domain)
877e86d1aa8SWill Deacon {
878e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
879e86d1aa8SWill Deacon
880e86d1aa8SWill Deacon /*
881e86d1aa8SWill Deacon * Free the domain resources. We assume that all devices have
882e86d1aa8SWill Deacon * already been detached.
883e86d1aa8SWill Deacon */
884e86d1aa8SWill Deacon arm_smmu_destroy_domain_context(domain);
885e86d1aa8SWill Deacon kfree(smmu_domain);
886e86d1aa8SWill Deacon }
887e86d1aa8SWill Deacon
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)888e86d1aa8SWill Deacon static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
889e86d1aa8SWill Deacon {
890e86d1aa8SWill Deacon struct arm_smmu_smr *smr = smmu->smrs + idx;
891e86d1aa8SWill Deacon u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
892e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
893e86d1aa8SWill Deacon
894e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
895e86d1aa8SWill Deacon reg |= ARM_SMMU_SMR_VALID;
896e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
897e86d1aa8SWill Deacon }
898e86d1aa8SWill Deacon
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)899e86d1aa8SWill Deacon static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
900e86d1aa8SWill Deacon {
901e86d1aa8SWill Deacon struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
90256b75b51SBjorn Andersson u32 reg;
90356b75b51SBjorn Andersson
90456b75b51SBjorn Andersson if (smmu->impl && smmu->impl->write_s2cr) {
90556b75b51SBjorn Andersson smmu->impl->write_s2cr(smmu, idx);
90656b75b51SBjorn Andersson return;
90756b75b51SBjorn Andersson }
90856b75b51SBjorn Andersson
90956b75b51SBjorn Andersson reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
910e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
911e86d1aa8SWill Deacon FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
912e86d1aa8SWill Deacon
913e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
914e86d1aa8SWill Deacon smmu->smrs[idx].valid)
915e86d1aa8SWill Deacon reg |= ARM_SMMU_S2CR_EXIDVALID;
916e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
917e86d1aa8SWill Deacon }
918e86d1aa8SWill Deacon
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)919e86d1aa8SWill Deacon static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
920e86d1aa8SWill Deacon {
921e86d1aa8SWill Deacon arm_smmu_write_s2cr(smmu, idx);
922e86d1aa8SWill Deacon if (smmu->smrs)
923e86d1aa8SWill Deacon arm_smmu_write_smr(smmu, idx);
924e86d1aa8SWill Deacon }
925e86d1aa8SWill Deacon
926e86d1aa8SWill Deacon /*
927e86d1aa8SWill Deacon * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
928e86d1aa8SWill Deacon * should be called after sCR0 is written.
929e86d1aa8SWill Deacon */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)930e86d1aa8SWill Deacon static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
931e86d1aa8SWill Deacon {
932e86d1aa8SWill Deacon u32 smr;
933e86d1aa8SWill Deacon int i;
934e86d1aa8SWill Deacon
935e86d1aa8SWill Deacon if (!smmu->smrs)
936e86d1aa8SWill Deacon return;
937e86d1aa8SWill Deacon /*
938e86d1aa8SWill Deacon * If we've had to accommodate firmware memory regions, we may
939e86d1aa8SWill Deacon * have live SMRs by now; tread carefully...
940e86d1aa8SWill Deacon *
941e86d1aa8SWill Deacon * Somewhat perversely, not having a free SMR for this test implies we
942e86d1aa8SWill Deacon * can get away without it anyway, as we'll only be able to 'allocate'
943e86d1aa8SWill Deacon * these SMRs for the ID/mask values we're already trusting to be OK.
944e86d1aa8SWill Deacon */
945e86d1aa8SWill Deacon for (i = 0; i < smmu->num_mapping_groups; i++)
946e86d1aa8SWill Deacon if (!smmu->smrs[i].valid)
947e86d1aa8SWill Deacon goto smr_ok;
948e86d1aa8SWill Deacon return;
949e86d1aa8SWill Deacon smr_ok:
950e86d1aa8SWill Deacon /*
951e86d1aa8SWill Deacon * SMR.ID bits may not be preserved if the corresponding MASK
952e86d1aa8SWill Deacon * bits are set, so check each one separately. We can reject
953e86d1aa8SWill Deacon * masters later if they try to claim IDs outside these masks.
954e86d1aa8SWill Deacon */
955e86d1aa8SWill Deacon smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
956e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
957e86d1aa8SWill Deacon smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
958e86d1aa8SWill Deacon smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
959e86d1aa8SWill Deacon
960e86d1aa8SWill Deacon smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
961e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
962e86d1aa8SWill Deacon smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
963e86d1aa8SWill Deacon smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
964e86d1aa8SWill Deacon }
965e86d1aa8SWill Deacon
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)966e86d1aa8SWill Deacon static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
967e86d1aa8SWill Deacon {
968e86d1aa8SWill Deacon struct arm_smmu_smr *smrs = smmu->smrs;
969e86d1aa8SWill Deacon int i, free_idx = -ENOSPC;
970e86d1aa8SWill Deacon
971e86d1aa8SWill Deacon /* Stream indexing is blissfully easy */
972e86d1aa8SWill Deacon if (!smrs)
973e86d1aa8SWill Deacon return id;
974e86d1aa8SWill Deacon
975e86d1aa8SWill Deacon /* Validating SMRs is... less so */
976e86d1aa8SWill Deacon for (i = 0; i < smmu->num_mapping_groups; ++i) {
977e86d1aa8SWill Deacon if (!smrs[i].valid) {
978e86d1aa8SWill Deacon /*
979e86d1aa8SWill Deacon * Note the first free entry we come across, which
980e86d1aa8SWill Deacon * we'll claim in the end if nothing else matches.
981e86d1aa8SWill Deacon */
982e86d1aa8SWill Deacon if (free_idx < 0)
983e86d1aa8SWill Deacon free_idx = i;
984e86d1aa8SWill Deacon continue;
985e86d1aa8SWill Deacon }
986e86d1aa8SWill Deacon /*
987e86d1aa8SWill Deacon * If the new entry is _entirely_ matched by an existing entry,
988e86d1aa8SWill Deacon * then reuse that, with the guarantee that there also cannot
989e86d1aa8SWill Deacon * be any subsequent conflicting entries. In normal use we'd
990e86d1aa8SWill Deacon * expect simply identical entries for this case, but there's
991e86d1aa8SWill Deacon * no harm in accommodating the generalisation.
992e86d1aa8SWill Deacon */
993e86d1aa8SWill Deacon if ((mask & smrs[i].mask) == mask &&
994e86d1aa8SWill Deacon !((id ^ smrs[i].id) & ~smrs[i].mask))
995e86d1aa8SWill Deacon return i;
996e86d1aa8SWill Deacon /*
997e86d1aa8SWill Deacon * If the new entry has any other overlap with an existing one,
998e86d1aa8SWill Deacon * though, then there always exists at least one stream ID
999e86d1aa8SWill Deacon * which would cause a conflict, and we can't allow that risk.
1000e86d1aa8SWill Deacon */
1001e86d1aa8SWill Deacon if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1002e86d1aa8SWill Deacon return -EINVAL;
1003e86d1aa8SWill Deacon }
1004e86d1aa8SWill Deacon
1005e86d1aa8SWill Deacon return free_idx;
1006e86d1aa8SWill Deacon }
1007e86d1aa8SWill Deacon
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1008e86d1aa8SWill Deacon static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1009e86d1aa8SWill Deacon {
1010e86d1aa8SWill Deacon if (--smmu->s2crs[idx].count)
1011e86d1aa8SWill Deacon return false;
1012e86d1aa8SWill Deacon
1013e86d1aa8SWill Deacon smmu->s2crs[idx] = s2cr_init_val;
1014e86d1aa8SWill Deacon if (smmu->smrs)
1015e86d1aa8SWill Deacon smmu->smrs[idx].valid = false;
1016e86d1aa8SWill Deacon
1017e86d1aa8SWill Deacon return true;
1018e86d1aa8SWill Deacon }
1019e86d1aa8SWill Deacon
arm_smmu_master_alloc_smes(struct device * dev)1020e86d1aa8SWill Deacon static int arm_smmu_master_alloc_smes(struct device *dev)
1021e86d1aa8SWill Deacon {
1022e86d1aa8SWill Deacon struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1023e86d1aa8SWill Deacon struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1024e86d1aa8SWill Deacon struct arm_smmu_device *smmu = cfg->smmu;
1025e86d1aa8SWill Deacon struct arm_smmu_smr *smrs = smmu->smrs;
1026e86d1aa8SWill Deacon int i, idx, ret;
1027e86d1aa8SWill Deacon
1028e86d1aa8SWill Deacon mutex_lock(&smmu->stream_map_mutex);
1029e86d1aa8SWill Deacon /* Figure out a viable stream map entry allocation */
1030e86d1aa8SWill Deacon for_each_cfg_sme(cfg, fwspec, i, idx) {
1031e86d1aa8SWill Deacon u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1032e86d1aa8SWill Deacon u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1033e86d1aa8SWill Deacon
1034e86d1aa8SWill Deacon if (idx != INVALID_SMENDX) {
1035e86d1aa8SWill Deacon ret = -EEXIST;
1036e86d1aa8SWill Deacon goto out_err;
1037e86d1aa8SWill Deacon }
1038e86d1aa8SWill Deacon
1039e86d1aa8SWill Deacon ret = arm_smmu_find_sme(smmu, sid, mask);
1040e86d1aa8SWill Deacon if (ret < 0)
1041e86d1aa8SWill Deacon goto out_err;
1042e86d1aa8SWill Deacon
1043e86d1aa8SWill Deacon idx = ret;
1044e86d1aa8SWill Deacon if (smrs && smmu->s2crs[idx].count == 0) {
1045e86d1aa8SWill Deacon smrs[idx].id = sid;
1046e86d1aa8SWill Deacon smrs[idx].mask = mask;
1047e86d1aa8SWill Deacon smrs[idx].valid = true;
1048e86d1aa8SWill Deacon }
1049e86d1aa8SWill Deacon smmu->s2crs[idx].count++;
1050e86d1aa8SWill Deacon cfg->smendx[i] = (s16)idx;
1051e86d1aa8SWill Deacon }
1052e86d1aa8SWill Deacon
1053e86d1aa8SWill Deacon /* It worked! Now, poke the actual hardware */
1054e86d1aa8SWill Deacon for_each_cfg_sme(cfg, fwspec, i, idx)
1055e86d1aa8SWill Deacon arm_smmu_write_sme(smmu, idx);
1056e86d1aa8SWill Deacon
1057e86d1aa8SWill Deacon mutex_unlock(&smmu->stream_map_mutex);
1058e86d1aa8SWill Deacon return 0;
1059e86d1aa8SWill Deacon
1060e86d1aa8SWill Deacon out_err:
1061e86d1aa8SWill Deacon while (i--) {
1062e86d1aa8SWill Deacon arm_smmu_free_sme(smmu, cfg->smendx[i]);
1063e86d1aa8SWill Deacon cfg->smendx[i] = INVALID_SMENDX;
1064e86d1aa8SWill Deacon }
1065e86d1aa8SWill Deacon mutex_unlock(&smmu->stream_map_mutex);
1066e86d1aa8SWill Deacon return ret;
1067e86d1aa8SWill Deacon }
1068e86d1aa8SWill Deacon
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1069e86d1aa8SWill Deacon static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1070e86d1aa8SWill Deacon struct iommu_fwspec *fwspec)
1071e86d1aa8SWill Deacon {
1072e86d1aa8SWill Deacon struct arm_smmu_device *smmu = cfg->smmu;
1073e86d1aa8SWill Deacon int i, idx;
1074e86d1aa8SWill Deacon
1075e86d1aa8SWill Deacon mutex_lock(&smmu->stream_map_mutex);
1076e86d1aa8SWill Deacon for_each_cfg_sme(cfg, fwspec, i, idx) {
1077e86d1aa8SWill Deacon if (arm_smmu_free_sme(smmu, idx))
1078e86d1aa8SWill Deacon arm_smmu_write_sme(smmu, idx);
1079e86d1aa8SWill Deacon cfg->smendx[i] = INVALID_SMENDX;
1080e86d1aa8SWill Deacon }
1081e86d1aa8SWill Deacon mutex_unlock(&smmu->stream_map_mutex);
1082e86d1aa8SWill Deacon }
1083e86d1aa8SWill Deacon
arm_smmu_domain_add_master(struct arm_smmu_domain * smmu_domain,struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1084e86d1aa8SWill Deacon static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1085e86d1aa8SWill Deacon struct arm_smmu_master_cfg *cfg,
1086e86d1aa8SWill Deacon struct iommu_fwspec *fwspec)
1087e86d1aa8SWill Deacon {
1088e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
1089e86d1aa8SWill Deacon struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1090e86d1aa8SWill Deacon u8 cbndx = smmu_domain->cfg.cbndx;
1091e86d1aa8SWill Deacon enum arm_smmu_s2cr_type type;
1092e86d1aa8SWill Deacon int i, idx;
1093e86d1aa8SWill Deacon
1094e86d1aa8SWill Deacon if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1095e86d1aa8SWill Deacon type = S2CR_TYPE_BYPASS;
1096e86d1aa8SWill Deacon else
1097e86d1aa8SWill Deacon type = S2CR_TYPE_TRANS;
1098e86d1aa8SWill Deacon
1099e86d1aa8SWill Deacon for_each_cfg_sme(cfg, fwspec, i, idx) {
1100e86d1aa8SWill Deacon if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1101e86d1aa8SWill Deacon continue;
1102e86d1aa8SWill Deacon
1103e86d1aa8SWill Deacon s2cr[idx].type = type;
1104e86d1aa8SWill Deacon s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1105e86d1aa8SWill Deacon s2cr[idx].cbndx = cbndx;
1106e86d1aa8SWill Deacon arm_smmu_write_s2cr(smmu, idx);
1107e86d1aa8SWill Deacon }
1108e86d1aa8SWill Deacon return 0;
1109e86d1aa8SWill Deacon }
1110e86d1aa8SWill Deacon
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1111e86d1aa8SWill Deacon static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1112e86d1aa8SWill Deacon {
1113e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1114e86d1aa8SWill Deacon struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1115e86d1aa8SWill Deacon struct arm_smmu_master_cfg *cfg;
1116e86d1aa8SWill Deacon struct arm_smmu_device *smmu;
1117e86d1aa8SWill Deacon int ret;
1118e86d1aa8SWill Deacon
1119e86d1aa8SWill Deacon if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1120e86d1aa8SWill Deacon dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1121e86d1aa8SWill Deacon return -ENXIO;
1122e86d1aa8SWill Deacon }
1123e86d1aa8SWill Deacon
1124e86d1aa8SWill Deacon /*
1125e86d1aa8SWill Deacon * FIXME: The arch/arm DMA API code tries to attach devices to its own
1126e86d1aa8SWill Deacon * domains between of_xlate() and probe_device() - we have no way to cope
1127e86d1aa8SWill Deacon * with that, so until ARM gets converted to rely on groups and default
1128e86d1aa8SWill Deacon * domains, just say no (but more politely than by dereferencing NULL).
1129e86d1aa8SWill Deacon * This should be at least a WARN_ON once that's sorted.
1130e86d1aa8SWill Deacon */
1131e86d1aa8SWill Deacon cfg = dev_iommu_priv_get(dev);
1132e86d1aa8SWill Deacon if (!cfg)
1133e86d1aa8SWill Deacon return -ENODEV;
1134e86d1aa8SWill Deacon
1135e86d1aa8SWill Deacon smmu = cfg->smmu;
1136e86d1aa8SWill Deacon
1137e86d1aa8SWill Deacon ret = arm_smmu_rpm_get(smmu);
1138e86d1aa8SWill Deacon if (ret < 0)
1139e86d1aa8SWill Deacon return ret;
1140e86d1aa8SWill Deacon
1141e86d1aa8SWill Deacon /* Ensure that the domain is finalised */
1142556db53aSJordan Crouse ret = arm_smmu_init_domain_context(domain, smmu, dev);
1143e86d1aa8SWill Deacon if (ret < 0)
1144e86d1aa8SWill Deacon goto rpm_put;
1145e86d1aa8SWill Deacon
1146e86d1aa8SWill Deacon /*
1147e86d1aa8SWill Deacon * Sanity check the domain. We don't support domains across
1148e86d1aa8SWill Deacon * different SMMUs.
1149e86d1aa8SWill Deacon */
1150e86d1aa8SWill Deacon if (smmu_domain->smmu != smmu) {
1151e86d1aa8SWill Deacon ret = -EINVAL;
1152e86d1aa8SWill Deacon goto rpm_put;
1153e86d1aa8SWill Deacon }
1154e86d1aa8SWill Deacon
1155e86d1aa8SWill Deacon /* Looks ok, so add the device to the domain */
1156e86d1aa8SWill Deacon ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec);
1157e86d1aa8SWill Deacon
1158e86d1aa8SWill Deacon /*
1159e86d1aa8SWill Deacon * Setup an autosuspend delay to avoid bouncing runpm state.
1160e86d1aa8SWill Deacon * Otherwise, if a driver for a suspended consumer device
1161e86d1aa8SWill Deacon * unmaps buffers, it will runpm resume/suspend for each one.
1162e86d1aa8SWill Deacon *
1163e86d1aa8SWill Deacon * For example, when used by a GPU device, when an application
1164e86d1aa8SWill Deacon * or game exits, it can trigger unmapping 100s or 1000s of
1165e86d1aa8SWill Deacon * buffers. With a runpm cycle for each buffer, that adds up
1166e86d1aa8SWill Deacon * to 5-10sec worth of reprogramming the context bank, while
1167e86d1aa8SWill Deacon * the system appears to be locked up to the user.
1168e86d1aa8SWill Deacon */
1169e86d1aa8SWill Deacon pm_runtime_set_autosuspend_delay(smmu->dev, 20);
1170e86d1aa8SWill Deacon pm_runtime_use_autosuspend(smmu->dev);
1171e86d1aa8SWill Deacon
1172e86d1aa8SWill Deacon rpm_put:
1173e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
1174e86d1aa8SWill Deacon return ret;
1175e86d1aa8SWill Deacon }
1176e86d1aa8SWill Deacon
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)117780803531SIsaac J. Manjarres static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
117880803531SIsaac J. Manjarres phys_addr_t paddr, size_t pgsize, size_t pgcount,
117980803531SIsaac J. Manjarres int prot, gfp_t gfp, size_t *mapped)
1180e86d1aa8SWill Deacon {
1181e86d1aa8SWill Deacon struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1182e86d1aa8SWill Deacon struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1183e86d1aa8SWill Deacon int ret;
1184e86d1aa8SWill Deacon
1185e86d1aa8SWill Deacon if (!ops)
1186e86d1aa8SWill Deacon return -ENODEV;
1187e86d1aa8SWill Deacon
1188e86d1aa8SWill Deacon arm_smmu_rpm_get(smmu);
118980803531SIsaac J. Manjarres ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1190e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
1191e86d1aa8SWill Deacon
1192e86d1aa8SWill Deacon return ret;
1193e86d1aa8SWill Deacon }
1194e86d1aa8SWill Deacon
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)11959ea1a2c4SIsaac J. Manjarres static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
11969ea1a2c4SIsaac J. Manjarres size_t pgsize, size_t pgcount,
11979ea1a2c4SIsaac J. Manjarres struct iommu_iotlb_gather *iotlb_gather)
1198e86d1aa8SWill Deacon {
1199e86d1aa8SWill Deacon struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1200e86d1aa8SWill Deacon struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1201e86d1aa8SWill Deacon size_t ret;
1202e86d1aa8SWill Deacon
1203e86d1aa8SWill Deacon if (!ops)
1204e86d1aa8SWill Deacon return 0;
1205e86d1aa8SWill Deacon
1206e86d1aa8SWill Deacon arm_smmu_rpm_get(smmu);
12079ea1a2c4SIsaac J. Manjarres ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1208e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
1209e86d1aa8SWill Deacon
1210e86d1aa8SWill Deacon return ret;
1211e86d1aa8SWill Deacon }
1212e86d1aa8SWill Deacon
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1213e86d1aa8SWill Deacon static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1214e86d1aa8SWill Deacon {
1215e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1216e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
1217e86d1aa8SWill Deacon
1218e86d1aa8SWill Deacon if (smmu_domain->flush_ops) {
1219e86d1aa8SWill Deacon arm_smmu_rpm_get(smmu);
1220e86d1aa8SWill Deacon smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1221e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
1222e86d1aa8SWill Deacon }
1223e86d1aa8SWill Deacon }
1224e86d1aa8SWill Deacon
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1225e86d1aa8SWill Deacon static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1226e86d1aa8SWill Deacon struct iommu_iotlb_gather *gather)
1227e86d1aa8SWill Deacon {
1228e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1229e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
1230e86d1aa8SWill Deacon
1231e86d1aa8SWill Deacon if (!smmu)
1232e86d1aa8SWill Deacon return;
1233e86d1aa8SWill Deacon
1234e86d1aa8SWill Deacon arm_smmu_rpm_get(smmu);
1235e86d1aa8SWill Deacon if (smmu->version == ARM_SMMU_V2 ||
1236e86d1aa8SWill Deacon smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1237e86d1aa8SWill Deacon arm_smmu_tlb_sync_context(smmu_domain);
1238e86d1aa8SWill Deacon else
1239e86d1aa8SWill Deacon arm_smmu_tlb_sync_global(smmu);
1240e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
1241e86d1aa8SWill Deacon }
1242e86d1aa8SWill Deacon
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1243e86d1aa8SWill Deacon static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1244e86d1aa8SWill Deacon dma_addr_t iova)
1245e86d1aa8SWill Deacon {
1246e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1247e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
1248e86d1aa8SWill Deacon struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1249e86d1aa8SWill Deacon struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1250e86d1aa8SWill Deacon struct device *dev = smmu->dev;
1251e86d1aa8SWill Deacon void __iomem *reg;
1252e86d1aa8SWill Deacon u32 tmp;
1253e86d1aa8SWill Deacon u64 phys;
1254e86d1aa8SWill Deacon unsigned long va, flags;
1255e86d1aa8SWill Deacon int ret, idx = cfg->cbndx;
12567c8f176dSXiyu Yang phys_addr_t addr = 0;
1257e86d1aa8SWill Deacon
1258e86d1aa8SWill Deacon ret = arm_smmu_rpm_get(smmu);
1259e86d1aa8SWill Deacon if (ret < 0)
1260e86d1aa8SWill Deacon return 0;
1261e86d1aa8SWill Deacon
1262e86d1aa8SWill Deacon spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1263e86d1aa8SWill Deacon va = iova & ~0xfffUL;
1264e86d1aa8SWill Deacon if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1265e86d1aa8SWill Deacon arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1266e86d1aa8SWill Deacon else
1267e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1268e86d1aa8SWill Deacon
1269e86d1aa8SWill Deacon reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1270e86d1aa8SWill Deacon if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE),
1271e86d1aa8SWill Deacon 5, 50)) {
1272e86d1aa8SWill Deacon spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1273e86d1aa8SWill Deacon dev_err(dev,
1274e86d1aa8SWill Deacon "iova to phys timed out on %pad. Falling back to software table walk.\n",
1275e86d1aa8SWill Deacon &iova);
12767c8f176dSXiyu Yang arm_smmu_rpm_put(smmu);
1277e86d1aa8SWill Deacon return ops->iova_to_phys(ops, iova);
1278e86d1aa8SWill Deacon }
1279e86d1aa8SWill Deacon
1280e86d1aa8SWill Deacon phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1281e86d1aa8SWill Deacon spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1282e86d1aa8SWill Deacon if (phys & ARM_SMMU_CB_PAR_F) {
1283e86d1aa8SWill Deacon dev_err(dev, "translation fault!\n");
1284e86d1aa8SWill Deacon dev_err(dev, "PAR = 0x%llx\n", phys);
12857c8f176dSXiyu Yang goto out;
1286e86d1aa8SWill Deacon }
1287e86d1aa8SWill Deacon
12887c8f176dSXiyu Yang addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
12897c8f176dSXiyu Yang out:
1290e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
1291e86d1aa8SWill Deacon
12927c8f176dSXiyu Yang return addr;
1293e86d1aa8SWill Deacon }
1294e86d1aa8SWill Deacon
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1295e86d1aa8SWill Deacon static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1296e86d1aa8SWill Deacon dma_addr_t iova)
1297e86d1aa8SWill Deacon {
1298e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1299e86d1aa8SWill Deacon struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1300e86d1aa8SWill Deacon
1301e86d1aa8SWill Deacon if (!ops)
1302e86d1aa8SWill Deacon return 0;
1303e86d1aa8SWill Deacon
1304e86d1aa8SWill Deacon if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1305e86d1aa8SWill Deacon smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1306e86d1aa8SWill Deacon return arm_smmu_iova_to_phys_hard(domain, iova);
1307e86d1aa8SWill Deacon
1308e86d1aa8SWill Deacon return ops->iova_to_phys(ops, iova);
1309e86d1aa8SWill Deacon }
1310e86d1aa8SWill Deacon
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1311359ad157SRobin Murphy static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1312e86d1aa8SWill Deacon {
1313df198b37SRobin Murphy struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1314df198b37SRobin Murphy
1315e86d1aa8SWill Deacon switch (cap) {
1316e86d1aa8SWill Deacon case IOMMU_CAP_CACHE_COHERENCY:
1317ac9c5e92SRobin Murphy /*
1318ac9c5e92SRobin Murphy * It's overwhelmingly the case in practice that when the pagetable
1319ac9c5e92SRobin Murphy * walk interface is connected to a coherent interconnect, all the
1320ac9c5e92SRobin Murphy * translation interfaces are too. Furthermore if the device is
1321ac9c5e92SRobin Murphy * natively coherent, then its translation interface must also be.
1322ac9c5e92SRobin Murphy */
1323ac9c5e92SRobin Murphy return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1324ac9c5e92SRobin Murphy device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1325e86d1aa8SWill Deacon case IOMMU_CAP_NOEXEC:
13264a20ce0fSRobin Murphy case IOMMU_CAP_DEFERRED_FLUSH:
1327e86d1aa8SWill Deacon return true;
1328e86d1aa8SWill Deacon default:
1329e86d1aa8SWill Deacon return false;
1330e86d1aa8SWill Deacon }
1331e86d1aa8SWill Deacon }
1332e86d1aa8SWill Deacon
1333e86d1aa8SWill Deacon static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1334e86d1aa8SWill Deacon struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1335e86d1aa8SWill Deacon {
1336e86d1aa8SWill Deacon struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1337e86d1aa8SWill Deacon fwnode);
1338e86d1aa8SWill Deacon put_device(dev);
1339e86d1aa8SWill Deacon return dev ? dev_get_drvdata(dev) : NULL;
1340e86d1aa8SWill Deacon }
1341e86d1aa8SWill Deacon
arm_smmu_probe_device(struct device * dev)1342e86d1aa8SWill Deacon static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1343e86d1aa8SWill Deacon {
1344e86d1aa8SWill Deacon struct arm_smmu_device *smmu = NULL;
1345e86d1aa8SWill Deacon struct arm_smmu_master_cfg *cfg;
1346e86d1aa8SWill Deacon struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1347e86d1aa8SWill Deacon int i, ret;
1348e86d1aa8SWill Deacon
1349e86d1aa8SWill Deacon if (using_legacy_binding) {
1350e86d1aa8SWill Deacon ret = arm_smmu_register_legacy_master(dev, &smmu);
1351e86d1aa8SWill Deacon
1352e86d1aa8SWill Deacon /*
1353e86d1aa8SWill Deacon * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1354e86d1aa8SWill Deacon * will allocate/initialise a new one. Thus we need to update fwspec for
1355e86d1aa8SWill Deacon * later use.
1356e86d1aa8SWill Deacon */
1357e86d1aa8SWill Deacon fwspec = dev_iommu_fwspec_get(dev);
1358e86d1aa8SWill Deacon if (ret)
1359e86d1aa8SWill Deacon goto out_free;
1360e86d1aa8SWill Deacon } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1361e86d1aa8SWill Deacon smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1362e86d1aa8SWill Deacon } else {
1363e86d1aa8SWill Deacon return ERR_PTR(-ENODEV);
1364e86d1aa8SWill Deacon }
1365e86d1aa8SWill Deacon
1366e86d1aa8SWill Deacon ret = -EINVAL;
1367e86d1aa8SWill Deacon for (i = 0; i < fwspec->num_ids; i++) {
1368e86d1aa8SWill Deacon u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1369e86d1aa8SWill Deacon u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1370e86d1aa8SWill Deacon
1371e86d1aa8SWill Deacon if (sid & ~smmu->streamid_mask) {
1372e86d1aa8SWill Deacon dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1373e86d1aa8SWill Deacon sid, smmu->streamid_mask);
1374e86d1aa8SWill Deacon goto out_free;
1375e86d1aa8SWill Deacon }
1376e86d1aa8SWill Deacon if (mask & ~smmu->smr_mask_mask) {
1377e86d1aa8SWill Deacon dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1378e86d1aa8SWill Deacon mask, smmu->smr_mask_mask);
1379e86d1aa8SWill Deacon goto out_free;
1380e86d1aa8SWill Deacon }
1381e86d1aa8SWill Deacon }
1382e86d1aa8SWill Deacon
1383e86d1aa8SWill Deacon ret = -ENOMEM;
1384e86d1aa8SWill Deacon cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1385e86d1aa8SWill Deacon GFP_KERNEL);
1386e86d1aa8SWill Deacon if (!cfg)
1387e86d1aa8SWill Deacon goto out_free;
1388e86d1aa8SWill Deacon
1389e86d1aa8SWill Deacon cfg->smmu = smmu;
1390e86d1aa8SWill Deacon dev_iommu_priv_set(dev, cfg);
1391e86d1aa8SWill Deacon while (i--)
1392e86d1aa8SWill Deacon cfg->smendx[i] = INVALID_SMENDX;
1393e86d1aa8SWill Deacon
1394e86d1aa8SWill Deacon ret = arm_smmu_rpm_get(smmu);
1395e86d1aa8SWill Deacon if (ret < 0)
1396e86d1aa8SWill Deacon goto out_cfg_free;
1397e86d1aa8SWill Deacon
1398e86d1aa8SWill Deacon ret = arm_smmu_master_alloc_smes(dev);
1399e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
1400e86d1aa8SWill Deacon
1401e86d1aa8SWill Deacon if (ret)
1402e86d1aa8SWill Deacon goto out_cfg_free;
1403e86d1aa8SWill Deacon
1404e86d1aa8SWill Deacon device_link_add(dev, smmu->dev,
1405e86d1aa8SWill Deacon DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1406e86d1aa8SWill Deacon
1407e86d1aa8SWill Deacon return &smmu->iommu;
1408e86d1aa8SWill Deacon
1409e86d1aa8SWill Deacon out_cfg_free:
1410e86d1aa8SWill Deacon kfree(cfg);
1411e86d1aa8SWill Deacon out_free:
1412e86d1aa8SWill Deacon iommu_fwspec_free(dev);
1413e86d1aa8SWill Deacon return ERR_PTR(ret);
1414e86d1aa8SWill Deacon }
1415e86d1aa8SWill Deacon
arm_smmu_release_device(struct device * dev)1416e86d1aa8SWill Deacon static void arm_smmu_release_device(struct device *dev)
1417e86d1aa8SWill Deacon {
1418e86d1aa8SWill Deacon struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
14194d26ba67SRobin Murphy struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1420e86d1aa8SWill Deacon int ret;
1421e86d1aa8SWill Deacon
14224d26ba67SRobin Murphy ret = arm_smmu_rpm_get(cfg->smmu);
1423e86d1aa8SWill Deacon if (ret < 0)
1424e86d1aa8SWill Deacon return;
1425e86d1aa8SWill Deacon
1426e86d1aa8SWill Deacon arm_smmu_master_free_smes(cfg, fwspec);
1427e86d1aa8SWill Deacon
14284d26ba67SRobin Murphy arm_smmu_rpm_put(cfg->smmu);
1429e86d1aa8SWill Deacon
1430e86d1aa8SWill Deacon dev_iommu_priv_set(dev, NULL);
1431e86d1aa8SWill Deacon kfree(cfg);
1432e86d1aa8SWill Deacon }
1433e86d1aa8SWill Deacon
arm_smmu_probe_finalize(struct device * dev)14340d97174aSThierry Reding static void arm_smmu_probe_finalize(struct device *dev)
14350d97174aSThierry Reding {
14360d97174aSThierry Reding struct arm_smmu_master_cfg *cfg;
14370d97174aSThierry Reding struct arm_smmu_device *smmu;
14380d97174aSThierry Reding
14390d97174aSThierry Reding cfg = dev_iommu_priv_get(dev);
14400d97174aSThierry Reding smmu = cfg->smmu;
14410d97174aSThierry Reding
1442b472191fSWill Deacon if (smmu->impl && smmu->impl->probe_finalize)
14430d97174aSThierry Reding smmu->impl->probe_finalize(smmu, dev);
14440d97174aSThierry Reding }
14450d97174aSThierry Reding
arm_smmu_device_group(struct device * dev)1446e86d1aa8SWill Deacon static struct iommu_group *arm_smmu_device_group(struct device *dev)
1447e86d1aa8SWill Deacon {
1448e86d1aa8SWill Deacon struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1449e86d1aa8SWill Deacon struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1450e86d1aa8SWill Deacon struct arm_smmu_device *smmu = cfg->smmu;
1451e86d1aa8SWill Deacon struct iommu_group *group = NULL;
1452e86d1aa8SWill Deacon int i, idx;
1453e86d1aa8SWill Deacon
1454b1a13479SKrishna Reddy mutex_lock(&smmu->stream_map_mutex);
1455e86d1aa8SWill Deacon for_each_cfg_sme(cfg, fwspec, i, idx) {
1456e86d1aa8SWill Deacon if (group && smmu->s2crs[idx].group &&
145753f61313SYang Yingliang group != smmu->s2crs[idx].group) {
145853f61313SYang Yingliang mutex_unlock(&smmu->stream_map_mutex);
1459e86d1aa8SWill Deacon return ERR_PTR(-EINVAL);
146053f61313SYang Yingliang }
1461e86d1aa8SWill Deacon
1462e86d1aa8SWill Deacon group = smmu->s2crs[idx].group;
1463e86d1aa8SWill Deacon }
1464e86d1aa8SWill Deacon
1465b1a13479SKrishna Reddy if (group) {
1466b1a13479SKrishna Reddy mutex_unlock(&smmu->stream_map_mutex);
1467e86d1aa8SWill Deacon return iommu_group_ref_get(group);
1468b1a13479SKrishna Reddy }
1469e86d1aa8SWill Deacon
1470e86d1aa8SWill Deacon if (dev_is_pci(dev))
1471e86d1aa8SWill Deacon group = pci_device_group(dev);
1472e86d1aa8SWill Deacon else if (dev_is_fsl_mc(dev))
1473e86d1aa8SWill Deacon group = fsl_mc_device_group(dev);
1474e86d1aa8SWill Deacon else
1475e86d1aa8SWill Deacon group = generic_device_group(dev);
1476e86d1aa8SWill Deacon
1477e86d1aa8SWill Deacon /* Remember group for faster lookups */
1478e86d1aa8SWill Deacon if (!IS_ERR(group))
1479e86d1aa8SWill Deacon for_each_cfg_sme(cfg, fwspec, i, idx)
1480e86d1aa8SWill Deacon smmu->s2crs[idx].group = group;
1481e86d1aa8SWill Deacon
1482b1a13479SKrishna Reddy mutex_unlock(&smmu->stream_map_mutex);
1483e86d1aa8SWill Deacon return group;
1484e86d1aa8SWill Deacon }
1485e86d1aa8SWill Deacon
arm_smmu_enable_nesting(struct iommu_domain * domain)14867e147547SChristoph Hellwig static int arm_smmu_enable_nesting(struct iommu_domain *domain)
14877e147547SChristoph Hellwig {
14887e147547SChristoph Hellwig struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
14897e147547SChristoph Hellwig int ret = 0;
14907e147547SChristoph Hellwig
14917e147547SChristoph Hellwig mutex_lock(&smmu_domain->init_mutex);
14927e147547SChristoph Hellwig if (smmu_domain->smmu)
14937e147547SChristoph Hellwig ret = -EPERM;
14947e147547SChristoph Hellwig else
14957e147547SChristoph Hellwig smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
14967e147547SChristoph Hellwig mutex_unlock(&smmu_domain->init_mutex);
14977e147547SChristoph Hellwig
14987e147547SChristoph Hellwig return ret;
14997e147547SChristoph Hellwig }
15007e147547SChristoph Hellwig
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)15014fc52b81SChristoph Hellwig static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
15024fc52b81SChristoph Hellwig unsigned long quirks)
1503e86d1aa8SWill Deacon {
1504e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
15054fc52b81SChristoph Hellwig int ret = 0;
1506e86d1aa8SWill Deacon
1507e86d1aa8SWill Deacon mutex_lock(&smmu_domain->init_mutex);
15084fc52b81SChristoph Hellwig if (smmu_domain->smmu)
1509c99110a8SSai Prakash Ranjan ret = -EPERM;
15104fc52b81SChristoph Hellwig else
15114fc52b81SChristoph Hellwig smmu_domain->pgtbl_quirks = quirks;
1512e86d1aa8SWill Deacon mutex_unlock(&smmu_domain->init_mutex);
15134fc52b81SChristoph Hellwig
1514e86d1aa8SWill Deacon return ret;
1515e86d1aa8SWill Deacon }
1516e86d1aa8SWill Deacon
arm_smmu_of_xlate(struct device * dev,struct of_phandle_args * args)1517e86d1aa8SWill Deacon static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1518e86d1aa8SWill Deacon {
1519e86d1aa8SWill Deacon u32 mask, fwid = 0;
1520e86d1aa8SWill Deacon
1521e86d1aa8SWill Deacon if (args->args_count > 0)
1522e86d1aa8SWill Deacon fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1523e86d1aa8SWill Deacon
1524e86d1aa8SWill Deacon if (args->args_count > 1)
1525e86d1aa8SWill Deacon fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1526e86d1aa8SWill Deacon else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1527e86d1aa8SWill Deacon fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1528e86d1aa8SWill Deacon
1529e86d1aa8SWill Deacon return iommu_fwspec_add_ids(dev, &fwid, 1);
1530e86d1aa8SWill Deacon }
1531e86d1aa8SWill Deacon
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1532e86d1aa8SWill Deacon static void arm_smmu_get_resv_regions(struct device *dev,
1533e86d1aa8SWill Deacon struct list_head *head)
1534e86d1aa8SWill Deacon {
1535e86d1aa8SWill Deacon struct iommu_resv_region *region;
1536e86d1aa8SWill Deacon int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1537e86d1aa8SWill Deacon
1538e86d1aa8SWill Deacon region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
15390251d010SLu Baolu prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1540e86d1aa8SWill Deacon if (!region)
1541e86d1aa8SWill Deacon return;
1542e86d1aa8SWill Deacon
1543e86d1aa8SWill Deacon list_add_tail(®ion->list, head);
1544e86d1aa8SWill Deacon
1545e86d1aa8SWill Deacon iommu_dma_get_resv_regions(dev, head);
1546e86d1aa8SWill Deacon }
1547e86d1aa8SWill Deacon
arm_smmu_def_domain_type(struct device * dev)1548e86d1aa8SWill Deacon static int arm_smmu_def_domain_type(struct device *dev)
1549e86d1aa8SWill Deacon {
1550e86d1aa8SWill Deacon struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1551e86d1aa8SWill Deacon const struct arm_smmu_impl *impl = cfg->smmu->impl;
1552e86d1aa8SWill Deacon
1553628bf55bSRobin Murphy if (using_legacy_binding)
1554628bf55bSRobin Murphy return IOMMU_DOMAIN_IDENTITY;
1555628bf55bSRobin Murphy
1556e86d1aa8SWill Deacon if (impl && impl->def_domain_type)
1557e86d1aa8SWill Deacon return impl->def_domain_type(dev);
1558e86d1aa8SWill Deacon
1559e86d1aa8SWill Deacon return 0;
1560e86d1aa8SWill Deacon }
1561e86d1aa8SWill Deacon
1562e86d1aa8SWill Deacon static struct iommu_ops arm_smmu_ops = {
1563e86d1aa8SWill Deacon .capable = arm_smmu_capable,
1564e86d1aa8SWill Deacon .domain_alloc = arm_smmu_domain_alloc,
1565e86d1aa8SWill Deacon .probe_device = arm_smmu_probe_device,
1566e86d1aa8SWill Deacon .release_device = arm_smmu_release_device,
15670d97174aSThierry Reding .probe_finalize = arm_smmu_probe_finalize,
1568e86d1aa8SWill Deacon .device_group = arm_smmu_device_group,
1569e86d1aa8SWill Deacon .of_xlate = arm_smmu_of_xlate,
1570e86d1aa8SWill Deacon .get_resv_regions = arm_smmu_get_resv_regions,
1571e86d1aa8SWill Deacon .def_domain_type = arm_smmu_def_domain_type,
1572e86d1aa8SWill Deacon .pgsize_bitmap = -1UL, /* Restricted during device attach */
1573c0aec668SRobin Murphy .owner = THIS_MODULE,
15749a630a4bSLu Baolu .default_domain_ops = &(const struct iommu_domain_ops) {
15759a630a4bSLu Baolu .attach_dev = arm_smmu_attach_dev,
15769a630a4bSLu Baolu .map_pages = arm_smmu_map_pages,
15779a630a4bSLu Baolu .unmap_pages = arm_smmu_unmap_pages,
15789a630a4bSLu Baolu .flush_iotlb_all = arm_smmu_flush_iotlb_all,
15799a630a4bSLu Baolu .iotlb_sync = arm_smmu_iotlb_sync,
15809a630a4bSLu Baolu .iova_to_phys = arm_smmu_iova_to_phys,
15819a630a4bSLu Baolu .enable_nesting = arm_smmu_enable_nesting,
15829a630a4bSLu Baolu .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
15839a630a4bSLu Baolu .free = arm_smmu_domain_free,
15849a630a4bSLu Baolu }
1585e86d1aa8SWill Deacon };
1586e86d1aa8SWill Deacon
arm_smmu_device_reset(struct arm_smmu_device * smmu)1587e86d1aa8SWill Deacon static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1588e86d1aa8SWill Deacon {
1589e86d1aa8SWill Deacon int i;
1590e86d1aa8SWill Deacon u32 reg;
1591e86d1aa8SWill Deacon
1592e86d1aa8SWill Deacon /* clear global FSR */
1593e86d1aa8SWill Deacon reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1594e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1595e86d1aa8SWill Deacon
1596e86d1aa8SWill Deacon /*
1597e86d1aa8SWill Deacon * Reset stream mapping groups: Initial values mark all SMRn as
1598e86d1aa8SWill Deacon * invalid and all S2CRn as bypass unless overridden.
1599e86d1aa8SWill Deacon */
1600e86d1aa8SWill Deacon for (i = 0; i < smmu->num_mapping_groups; ++i)
1601e86d1aa8SWill Deacon arm_smmu_write_sme(smmu, i);
1602e86d1aa8SWill Deacon
1603e86d1aa8SWill Deacon /* Make sure all context banks are disabled and clear CB_FSR */
1604e86d1aa8SWill Deacon for (i = 0; i < smmu->num_context_banks; ++i) {
1605e86d1aa8SWill Deacon arm_smmu_write_context_bank(smmu, i);
1606e86d1aa8SWill Deacon arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);
1607e86d1aa8SWill Deacon }
1608e86d1aa8SWill Deacon
1609e86d1aa8SWill Deacon /* Invalidate the TLB, just in case */
1610e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1611e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1612e86d1aa8SWill Deacon
1613e86d1aa8SWill Deacon reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1614e86d1aa8SWill Deacon
1615e86d1aa8SWill Deacon /* Enable fault reporting */
1616e86d1aa8SWill Deacon reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1617e86d1aa8SWill Deacon ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1618e86d1aa8SWill Deacon
1619e86d1aa8SWill Deacon /* Disable TLB broadcasting. */
1620e86d1aa8SWill Deacon reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1621e86d1aa8SWill Deacon
1622e86d1aa8SWill Deacon /* Enable client access, handling unmatched streams as appropriate */
1623e86d1aa8SWill Deacon reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1624e86d1aa8SWill Deacon if (disable_bypass)
1625e86d1aa8SWill Deacon reg |= ARM_SMMU_sCR0_USFCFG;
1626e86d1aa8SWill Deacon else
1627e86d1aa8SWill Deacon reg &= ~ARM_SMMU_sCR0_USFCFG;
1628e86d1aa8SWill Deacon
1629e86d1aa8SWill Deacon /* Disable forced broadcasting */
1630e86d1aa8SWill Deacon reg &= ~ARM_SMMU_sCR0_FB;
1631e86d1aa8SWill Deacon
1632e86d1aa8SWill Deacon /* Don't upgrade barriers */
1633e86d1aa8SWill Deacon reg &= ~(ARM_SMMU_sCR0_BSU);
1634e86d1aa8SWill Deacon
1635e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_VMID16)
1636e86d1aa8SWill Deacon reg |= ARM_SMMU_sCR0_VMID16EN;
1637e86d1aa8SWill Deacon
1638e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1639e86d1aa8SWill Deacon reg |= ARM_SMMU_sCR0_EXIDENABLE;
1640e86d1aa8SWill Deacon
1641e86d1aa8SWill Deacon if (smmu->impl && smmu->impl->reset)
1642e86d1aa8SWill Deacon smmu->impl->reset(smmu);
1643e86d1aa8SWill Deacon
1644e86d1aa8SWill Deacon /* Push the button */
1645e86d1aa8SWill Deacon arm_smmu_tlb_sync_global(smmu);
1646e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1647e86d1aa8SWill Deacon }
1648e86d1aa8SWill Deacon
arm_smmu_id_size_to_bits(int size)1649e86d1aa8SWill Deacon static int arm_smmu_id_size_to_bits(int size)
1650e86d1aa8SWill Deacon {
1651e86d1aa8SWill Deacon switch (size) {
1652e86d1aa8SWill Deacon case 0:
1653e86d1aa8SWill Deacon return 32;
1654e86d1aa8SWill Deacon case 1:
1655e86d1aa8SWill Deacon return 36;
1656e86d1aa8SWill Deacon case 2:
1657e86d1aa8SWill Deacon return 40;
1658e86d1aa8SWill Deacon case 3:
1659e86d1aa8SWill Deacon return 42;
1660e86d1aa8SWill Deacon case 4:
1661e86d1aa8SWill Deacon return 44;
1662e86d1aa8SWill Deacon case 5:
1663e86d1aa8SWill Deacon default:
1664e86d1aa8SWill Deacon return 48;
1665e86d1aa8SWill Deacon }
1666e86d1aa8SWill Deacon }
1667e86d1aa8SWill Deacon
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1668e86d1aa8SWill Deacon static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1669e86d1aa8SWill Deacon {
1670e86d1aa8SWill Deacon unsigned int size;
1671e86d1aa8SWill Deacon u32 id;
1672e86d1aa8SWill Deacon bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1673e86d1aa8SWill Deacon int i, ret;
1674e86d1aa8SWill Deacon
1675e86d1aa8SWill Deacon dev_notice(smmu->dev, "probing hardware configuration...\n");
1676e86d1aa8SWill Deacon dev_notice(smmu->dev, "SMMUv%d with:\n",
1677e86d1aa8SWill Deacon smmu->version == ARM_SMMU_V2 ? 2 : 1);
1678e86d1aa8SWill Deacon
1679e86d1aa8SWill Deacon /* ID0 */
1680e86d1aa8SWill Deacon id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1681e86d1aa8SWill Deacon
1682e86d1aa8SWill Deacon /* Restrict available stages based on module parameter */
1683e86d1aa8SWill Deacon if (force_stage == 1)
1684e86d1aa8SWill Deacon id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1685e86d1aa8SWill Deacon else if (force_stage == 2)
1686e86d1aa8SWill Deacon id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1687e86d1aa8SWill Deacon
1688e86d1aa8SWill Deacon if (id & ARM_SMMU_ID0_S1TS) {
1689e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1690e86d1aa8SWill Deacon dev_notice(smmu->dev, "\tstage 1 translation\n");
1691e86d1aa8SWill Deacon }
1692e86d1aa8SWill Deacon
1693e86d1aa8SWill Deacon if (id & ARM_SMMU_ID0_S2TS) {
1694e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1695e86d1aa8SWill Deacon dev_notice(smmu->dev, "\tstage 2 translation\n");
1696e86d1aa8SWill Deacon }
1697e86d1aa8SWill Deacon
1698e86d1aa8SWill Deacon if (id & ARM_SMMU_ID0_NTS) {
1699e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1700e86d1aa8SWill Deacon dev_notice(smmu->dev, "\tnested translation\n");
1701e86d1aa8SWill Deacon }
1702e86d1aa8SWill Deacon
1703e86d1aa8SWill Deacon if (!(smmu->features &
1704e86d1aa8SWill Deacon (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1705e86d1aa8SWill Deacon dev_err(smmu->dev, "\tno translation support!\n");
1706e86d1aa8SWill Deacon return -ENODEV;
1707e86d1aa8SWill Deacon }
1708e86d1aa8SWill Deacon
1709e86d1aa8SWill Deacon if ((id & ARM_SMMU_ID0_S1TS) &&
1710e86d1aa8SWill Deacon ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1711e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1712e86d1aa8SWill Deacon dev_notice(smmu->dev, "\taddress translation ops\n");
1713e86d1aa8SWill Deacon }
1714e86d1aa8SWill Deacon
1715e86d1aa8SWill Deacon /*
1716e86d1aa8SWill Deacon * In order for DMA API calls to work properly, we must defer to what
1717e86d1aa8SWill Deacon * the FW says about coherency, regardless of what the hardware claims.
1718e86d1aa8SWill Deacon * Fortunately, this also opens up a workaround for systems where the
1719e86d1aa8SWill Deacon * ID register value has ended up configured incorrectly.
1720e86d1aa8SWill Deacon */
1721e86d1aa8SWill Deacon cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1722e86d1aa8SWill Deacon if (cttw_fw || cttw_reg)
1723e86d1aa8SWill Deacon dev_notice(smmu->dev, "\t%scoherent table walk\n",
1724e86d1aa8SWill Deacon cttw_fw ? "" : "non-");
1725e86d1aa8SWill Deacon if (cttw_fw != cttw_reg)
1726e86d1aa8SWill Deacon dev_notice(smmu->dev,
1727e86d1aa8SWill Deacon "\t(IDR0.CTTW overridden by FW configuration)\n");
1728e86d1aa8SWill Deacon
1729e86d1aa8SWill Deacon /* Max. number of entries we have for stream matching/indexing */
1730e86d1aa8SWill Deacon if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1731e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_EXIDS;
1732e86d1aa8SWill Deacon size = 1 << 16;
1733e86d1aa8SWill Deacon } else {
1734e86d1aa8SWill Deacon size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1735e86d1aa8SWill Deacon }
1736e86d1aa8SWill Deacon smmu->streamid_mask = size - 1;
1737e86d1aa8SWill Deacon if (id & ARM_SMMU_ID0_SMS) {
1738e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1739e86d1aa8SWill Deacon size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1740e86d1aa8SWill Deacon if (size == 0) {
1741e86d1aa8SWill Deacon dev_err(smmu->dev,
1742e86d1aa8SWill Deacon "stream-matching supported, but no SMRs present!\n");
1743e86d1aa8SWill Deacon return -ENODEV;
1744e86d1aa8SWill Deacon }
1745e86d1aa8SWill Deacon
1746e86d1aa8SWill Deacon /* Zero-initialised to mark as invalid */
1747e86d1aa8SWill Deacon smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1748e86d1aa8SWill Deacon GFP_KERNEL);
1749e86d1aa8SWill Deacon if (!smmu->smrs)
1750e86d1aa8SWill Deacon return -ENOMEM;
1751e86d1aa8SWill Deacon
1752e86d1aa8SWill Deacon dev_notice(smmu->dev,
1753e86d1aa8SWill Deacon "\tstream matching with %u register groups", size);
1754e86d1aa8SWill Deacon }
1755e86d1aa8SWill Deacon /* s2cr->type == 0 means translation, so initialise explicitly */
1756e86d1aa8SWill Deacon smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1757e86d1aa8SWill Deacon GFP_KERNEL);
1758e86d1aa8SWill Deacon if (!smmu->s2crs)
1759e86d1aa8SWill Deacon return -ENOMEM;
1760e86d1aa8SWill Deacon for (i = 0; i < size; i++)
1761e86d1aa8SWill Deacon smmu->s2crs[i] = s2cr_init_val;
1762e86d1aa8SWill Deacon
1763e86d1aa8SWill Deacon smmu->num_mapping_groups = size;
1764e86d1aa8SWill Deacon mutex_init(&smmu->stream_map_mutex);
1765e86d1aa8SWill Deacon spin_lock_init(&smmu->global_sync_lock);
1766e86d1aa8SWill Deacon
1767e86d1aa8SWill Deacon if (smmu->version < ARM_SMMU_V2 ||
1768e86d1aa8SWill Deacon !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1769e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1770e86d1aa8SWill Deacon if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1771e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1772e86d1aa8SWill Deacon }
1773e86d1aa8SWill Deacon
1774e86d1aa8SWill Deacon /* ID1 */
1775e86d1aa8SWill Deacon id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1776e86d1aa8SWill Deacon smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1777e86d1aa8SWill Deacon
1778e86d1aa8SWill Deacon /* Check for size mismatch of SMMU address space from mapped region */
1779e86d1aa8SWill Deacon size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1780e86d1aa8SWill Deacon if (smmu->numpage != 2 * size << smmu->pgshift)
1781e86d1aa8SWill Deacon dev_warn(smmu->dev,
1782e86d1aa8SWill Deacon "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1783e86d1aa8SWill Deacon 2 * size << smmu->pgshift, smmu->numpage);
1784e86d1aa8SWill Deacon /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1785e86d1aa8SWill Deacon smmu->numpage = size;
1786e86d1aa8SWill Deacon
1787e86d1aa8SWill Deacon smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1788e86d1aa8SWill Deacon smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1789e86d1aa8SWill Deacon if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1790e86d1aa8SWill Deacon dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1791e86d1aa8SWill Deacon return -ENODEV;
1792e86d1aa8SWill Deacon }
1793e86d1aa8SWill Deacon dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1794e86d1aa8SWill Deacon smmu->num_context_banks, smmu->num_s2_context_banks);
1795e86d1aa8SWill Deacon smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1796e86d1aa8SWill Deacon sizeof(*smmu->cbs), GFP_KERNEL);
1797e86d1aa8SWill Deacon if (!smmu->cbs)
1798e86d1aa8SWill Deacon return -ENOMEM;
1799e86d1aa8SWill Deacon
1800e86d1aa8SWill Deacon /* ID2 */
1801e86d1aa8SWill Deacon id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1802e86d1aa8SWill Deacon size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1803e86d1aa8SWill Deacon smmu->ipa_size = size;
1804e86d1aa8SWill Deacon
1805e86d1aa8SWill Deacon /* The output mask is also applied for bypass */
1806e86d1aa8SWill Deacon size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1807e86d1aa8SWill Deacon smmu->pa_size = size;
1808e86d1aa8SWill Deacon
1809e86d1aa8SWill Deacon if (id & ARM_SMMU_ID2_VMID16)
1810e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_VMID16;
1811e86d1aa8SWill Deacon
1812e86d1aa8SWill Deacon /*
1813e86d1aa8SWill Deacon * What the page table walker can address actually depends on which
1814e86d1aa8SWill Deacon * descriptor format is in use, but since a) we don't know that yet,
1815e86d1aa8SWill Deacon * and b) it can vary per context bank, this will have to do...
1816e86d1aa8SWill Deacon */
1817e86d1aa8SWill Deacon if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1818e86d1aa8SWill Deacon dev_warn(smmu->dev,
1819e86d1aa8SWill Deacon "failed to set DMA mask for table walker\n");
1820e86d1aa8SWill Deacon
1821e86d1aa8SWill Deacon if (smmu->version < ARM_SMMU_V2) {
1822e86d1aa8SWill Deacon smmu->va_size = smmu->ipa_size;
1823e86d1aa8SWill Deacon if (smmu->version == ARM_SMMU_V1_64K)
1824e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1825e86d1aa8SWill Deacon } else {
1826e86d1aa8SWill Deacon size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1827e86d1aa8SWill Deacon smmu->va_size = arm_smmu_id_size_to_bits(size);
1828e86d1aa8SWill Deacon if (id & ARM_SMMU_ID2_PTFS_4K)
1829e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1830e86d1aa8SWill Deacon if (id & ARM_SMMU_ID2_PTFS_16K)
1831e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1832e86d1aa8SWill Deacon if (id & ARM_SMMU_ID2_PTFS_64K)
1833e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1834e86d1aa8SWill Deacon }
1835e86d1aa8SWill Deacon
1836e86d1aa8SWill Deacon if (smmu->impl && smmu->impl->cfg_probe) {
1837e86d1aa8SWill Deacon ret = smmu->impl->cfg_probe(smmu);
1838e86d1aa8SWill Deacon if (ret)
1839e86d1aa8SWill Deacon return ret;
1840e86d1aa8SWill Deacon }
1841e86d1aa8SWill Deacon
1842e86d1aa8SWill Deacon /* Now we've corralled the various formats, what'll it do? */
1843e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1844e86d1aa8SWill Deacon smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1845e86d1aa8SWill Deacon if (smmu->features &
1846e86d1aa8SWill Deacon (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1847e86d1aa8SWill Deacon smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1848e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1849e86d1aa8SWill Deacon smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1850e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1851e86d1aa8SWill Deacon smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1852e86d1aa8SWill Deacon
1853e86d1aa8SWill Deacon if (arm_smmu_ops.pgsize_bitmap == -1UL)
1854e86d1aa8SWill Deacon arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1855e86d1aa8SWill Deacon else
1856e86d1aa8SWill Deacon arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1857e86d1aa8SWill Deacon dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1858e86d1aa8SWill Deacon smmu->pgsize_bitmap);
1859e86d1aa8SWill Deacon
1860e86d1aa8SWill Deacon
1861e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1862e86d1aa8SWill Deacon dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1863e86d1aa8SWill Deacon smmu->va_size, smmu->ipa_size);
1864e86d1aa8SWill Deacon
1865e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1866e86d1aa8SWill Deacon dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1867e86d1aa8SWill Deacon smmu->ipa_size, smmu->pa_size);
1868e86d1aa8SWill Deacon
1869e86d1aa8SWill Deacon return 0;
1870e86d1aa8SWill Deacon }
1871e86d1aa8SWill Deacon
1872e86d1aa8SWill Deacon struct arm_smmu_match_data {
1873e86d1aa8SWill Deacon enum arm_smmu_arch_version version;
1874e86d1aa8SWill Deacon enum arm_smmu_implementation model;
1875e86d1aa8SWill Deacon };
1876e86d1aa8SWill Deacon
1877e86d1aa8SWill Deacon #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1878e86d1aa8SWill Deacon static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1879e86d1aa8SWill Deacon
1880e86d1aa8SWill Deacon ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1881e86d1aa8SWill Deacon ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1882e86d1aa8SWill Deacon ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1883e86d1aa8SWill Deacon ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1884e86d1aa8SWill Deacon ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1885e86d1aa8SWill Deacon ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1886e86d1aa8SWill Deacon
1887e86d1aa8SWill Deacon static const struct of_device_id arm_smmu_of_match[] = {
1888e86d1aa8SWill Deacon { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1889e86d1aa8SWill Deacon { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1890e86d1aa8SWill Deacon { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1891e86d1aa8SWill Deacon { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1892e86d1aa8SWill Deacon { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1893e86d1aa8SWill Deacon { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1894e86d1aa8SWill Deacon { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1895e86d1aa8SWill Deacon { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1896e86d1aa8SWill Deacon { },
1897e86d1aa8SWill Deacon };
1898e86d1aa8SWill Deacon MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1899e86d1aa8SWill Deacon
1900e86d1aa8SWill Deacon #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1901e86d1aa8SWill Deacon static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1902e86d1aa8SWill Deacon {
1903e86d1aa8SWill Deacon int ret = 0;
1904e86d1aa8SWill Deacon
1905e86d1aa8SWill Deacon switch (model) {
1906e86d1aa8SWill Deacon case ACPI_IORT_SMMU_V1:
1907e86d1aa8SWill Deacon case ACPI_IORT_SMMU_CORELINK_MMU400:
1908e86d1aa8SWill Deacon smmu->version = ARM_SMMU_V1;
1909e86d1aa8SWill Deacon smmu->model = GENERIC_SMMU;
1910e86d1aa8SWill Deacon break;
1911e86d1aa8SWill Deacon case ACPI_IORT_SMMU_CORELINK_MMU401:
1912e86d1aa8SWill Deacon smmu->version = ARM_SMMU_V1_64K;
1913e86d1aa8SWill Deacon smmu->model = GENERIC_SMMU;
1914e86d1aa8SWill Deacon break;
1915e86d1aa8SWill Deacon case ACPI_IORT_SMMU_V2:
1916e86d1aa8SWill Deacon smmu->version = ARM_SMMU_V2;
1917e86d1aa8SWill Deacon smmu->model = GENERIC_SMMU;
1918e86d1aa8SWill Deacon break;
1919e86d1aa8SWill Deacon case ACPI_IORT_SMMU_CORELINK_MMU500:
1920e86d1aa8SWill Deacon smmu->version = ARM_SMMU_V2;
1921e86d1aa8SWill Deacon smmu->model = ARM_MMU500;
1922e86d1aa8SWill Deacon break;
1923e86d1aa8SWill Deacon case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1924e86d1aa8SWill Deacon smmu->version = ARM_SMMU_V2;
1925e86d1aa8SWill Deacon smmu->model = CAVIUM_SMMUV2;
1926e86d1aa8SWill Deacon break;
1927e86d1aa8SWill Deacon default:
1928e86d1aa8SWill Deacon ret = -ENODEV;
1929e86d1aa8SWill Deacon }
1930e86d1aa8SWill Deacon
1931e86d1aa8SWill Deacon return ret;
1932e86d1aa8SWill Deacon }
1933e86d1aa8SWill Deacon
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)193497dfad19SRobin Murphy static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
193597dfad19SRobin Murphy u32 *global_irqs, u32 *pmu_irqs)
1936e86d1aa8SWill Deacon {
1937e86d1aa8SWill Deacon struct device *dev = smmu->dev;
1938e86d1aa8SWill Deacon struct acpi_iort_node *node =
1939e86d1aa8SWill Deacon *(struct acpi_iort_node **)dev_get_platdata(dev);
1940e86d1aa8SWill Deacon struct acpi_iort_smmu *iort_smmu;
1941e86d1aa8SWill Deacon int ret;
1942e86d1aa8SWill Deacon
1943e86d1aa8SWill Deacon /* Retrieve SMMU1/2 specific data */
1944e86d1aa8SWill Deacon iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1945e86d1aa8SWill Deacon
1946e86d1aa8SWill Deacon ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1947e86d1aa8SWill Deacon if (ret < 0)
1948e86d1aa8SWill Deacon return ret;
1949e86d1aa8SWill Deacon
1950e86d1aa8SWill Deacon /* Ignore the configuration access interrupt */
195197dfad19SRobin Murphy *global_irqs = 1;
195297dfad19SRobin Murphy *pmu_irqs = 0;
1953e86d1aa8SWill Deacon
1954e86d1aa8SWill Deacon if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1955e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1956e86d1aa8SWill Deacon
1957e86d1aa8SWill Deacon return 0;
1958e86d1aa8SWill Deacon }
1959e86d1aa8SWill Deacon #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)196097dfad19SRobin Murphy static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
196197dfad19SRobin Murphy u32 *global_irqs, u32 *pmu_irqs)
1962e86d1aa8SWill Deacon {
1963e86d1aa8SWill Deacon return -ENODEV;
1964e86d1aa8SWill Deacon }
1965e86d1aa8SWill Deacon #endif
1966e86d1aa8SWill Deacon
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)196797dfad19SRobin Murphy static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
196897dfad19SRobin Murphy u32 *global_irqs, u32 *pmu_irqs)
1969e86d1aa8SWill Deacon {
1970e86d1aa8SWill Deacon const struct arm_smmu_match_data *data;
197197dfad19SRobin Murphy struct device *dev = smmu->dev;
1972e86d1aa8SWill Deacon bool legacy_binding;
1973e86d1aa8SWill Deacon
197497dfad19SRobin Murphy if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
197597dfad19SRobin Murphy return dev_err_probe(dev, -ENODEV,
197697dfad19SRobin Murphy "missing #global-interrupts property\n");
197797dfad19SRobin Murphy *pmu_irqs = 0;
1978e86d1aa8SWill Deacon
1979e86d1aa8SWill Deacon data = of_device_get_match_data(dev);
1980e86d1aa8SWill Deacon smmu->version = data->version;
1981e86d1aa8SWill Deacon smmu->model = data->model;
1982e86d1aa8SWill Deacon
1983e86d1aa8SWill Deacon legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1984e86d1aa8SWill Deacon if (legacy_binding && !using_generic_binding) {
1985e86d1aa8SWill Deacon if (!using_legacy_binding) {
1986e86d1aa8SWill Deacon pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
1987e86d1aa8SWill Deacon IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
1988e86d1aa8SWill Deacon }
1989e86d1aa8SWill Deacon using_legacy_binding = true;
1990e86d1aa8SWill Deacon } else if (!legacy_binding && !using_legacy_binding) {
1991e86d1aa8SWill Deacon using_generic_binding = true;
1992e86d1aa8SWill Deacon } else {
1993e86d1aa8SWill Deacon dev_err(dev, "not probing due to mismatched DT properties\n");
1994e86d1aa8SWill Deacon return -ENODEV;
1995e86d1aa8SWill Deacon }
1996e86d1aa8SWill Deacon
1997e86d1aa8SWill Deacon if (of_dma_is_coherent(dev->of_node))
1998e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1999e86d1aa8SWill Deacon
2000e86d1aa8SWill Deacon return 0;
2001e86d1aa8SWill Deacon }
2002e86d1aa8SWill Deacon
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)20030bec0557SJon Nettleton static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
20040bec0557SJon Nettleton {
20050bec0557SJon Nettleton struct list_head rmr_list;
20060bec0557SJon Nettleton struct iommu_resv_region *e;
20070bec0557SJon Nettleton int idx, cnt = 0;
20080bec0557SJon Nettleton u32 reg;
20090bec0557SJon Nettleton
20100bec0557SJon Nettleton INIT_LIST_HEAD(&rmr_list);
20110bec0557SJon Nettleton iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
20120bec0557SJon Nettleton
20130bec0557SJon Nettleton /*
20140bec0557SJon Nettleton * Rather than trying to look at existing mappings that
20150bec0557SJon Nettleton * are setup by the firmware and then invalidate the ones
20160bec0557SJon Nettleton * that do no have matching RMR entries, just disable the
20170bec0557SJon Nettleton * SMMU until it gets enabled again in the reset routine.
20180bec0557SJon Nettleton */
20190bec0557SJon Nettleton reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
20200bec0557SJon Nettleton reg |= ARM_SMMU_sCR0_CLIENTPD;
20210bec0557SJon Nettleton arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
20220bec0557SJon Nettleton
20230bec0557SJon Nettleton list_for_each_entry(e, &rmr_list, list) {
20240bec0557SJon Nettleton struct iommu_iort_rmr_data *rmr;
20250bec0557SJon Nettleton int i;
20260bec0557SJon Nettleton
20270bec0557SJon Nettleton rmr = container_of(e, struct iommu_iort_rmr_data, rr);
20280bec0557SJon Nettleton for (i = 0; i < rmr->num_sids; i++) {
20290bec0557SJon Nettleton idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
20300bec0557SJon Nettleton if (idx < 0)
20310bec0557SJon Nettleton continue;
20320bec0557SJon Nettleton
20330bec0557SJon Nettleton if (smmu->s2crs[idx].count == 0) {
20340bec0557SJon Nettleton smmu->smrs[idx].id = rmr->sids[i];
20350bec0557SJon Nettleton smmu->smrs[idx].mask = 0;
20360bec0557SJon Nettleton smmu->smrs[idx].valid = true;
20370bec0557SJon Nettleton }
20380bec0557SJon Nettleton smmu->s2crs[idx].count++;
20390bec0557SJon Nettleton smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
20400bec0557SJon Nettleton smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
20410bec0557SJon Nettleton
20420bec0557SJon Nettleton cnt++;
20430bec0557SJon Nettleton }
20440bec0557SJon Nettleton }
20450bec0557SJon Nettleton
20460bec0557SJon Nettleton dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
20470bec0557SJon Nettleton cnt == 1 ? "" : "s");
20480bec0557SJon Nettleton iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
20490bec0557SJon Nettleton }
20500bec0557SJon Nettleton
arm_smmu_device_probe(struct platform_device * pdev)2051e86d1aa8SWill Deacon static int arm_smmu_device_probe(struct platform_device *pdev)
2052e86d1aa8SWill Deacon {
2053e86d1aa8SWill Deacon struct resource *res;
2054e86d1aa8SWill Deacon struct arm_smmu_device *smmu;
2055e86d1aa8SWill Deacon struct device *dev = &pdev->dev;
2056e86d1aa8SWill Deacon int num_irqs, i, err;
205797dfad19SRobin Murphy u32 global_irqs, pmu_irqs;
2058e86d1aa8SWill Deacon irqreturn_t (*global_fault)(int irq, void *dev);
2059e86d1aa8SWill Deacon
2060e86d1aa8SWill Deacon smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2061e86d1aa8SWill Deacon if (!smmu) {
2062e86d1aa8SWill Deacon dev_err(dev, "failed to allocate arm_smmu_device\n");
2063e86d1aa8SWill Deacon return -ENOMEM;
2064e86d1aa8SWill Deacon }
2065e86d1aa8SWill Deacon smmu->dev = dev;
2066e86d1aa8SWill Deacon
2067e86d1aa8SWill Deacon if (dev->of_node)
206897dfad19SRobin Murphy err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2069e86d1aa8SWill Deacon else
207097dfad19SRobin Murphy err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2071e86d1aa8SWill Deacon if (err)
2072e86d1aa8SWill Deacon return err;
2073e86d1aa8SWill Deacon
2074d9ed8af1SYang Yingliang smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2075e86d1aa8SWill Deacon if (IS_ERR(smmu->base))
2076e86d1aa8SWill Deacon return PTR_ERR(smmu->base);
2077b9b721d1SSai Prakash Ranjan smmu->ioaddr = res->start;
2078b9b721d1SSai Prakash Ranjan
2079e86d1aa8SWill Deacon /*
2080e86d1aa8SWill Deacon * The resource size should effectively match the value of SMMU_TOP;
2081e86d1aa8SWill Deacon * stash that temporarily until we know PAGESIZE to validate it with.
2082e86d1aa8SWill Deacon */
2083e86d1aa8SWill Deacon smmu->numpage = resource_size(res);
2084e86d1aa8SWill Deacon
2085e86d1aa8SWill Deacon smmu = arm_smmu_impl_init(smmu);
2086e86d1aa8SWill Deacon if (IS_ERR(smmu))
2087e86d1aa8SWill Deacon return PTR_ERR(smmu);
2088e86d1aa8SWill Deacon
208997dfad19SRobin Murphy num_irqs = platform_irq_count(pdev);
2090e86d1aa8SWill Deacon
209197dfad19SRobin Murphy smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
209297dfad19SRobin Murphy if (smmu->num_context_irqs <= 0)
209397dfad19SRobin Murphy return dev_err_probe(dev, -ENODEV,
209497dfad19SRobin Murphy "found %d interrupts but expected at least %d\n",
209597dfad19SRobin Murphy num_irqs, global_irqs + pmu_irqs + 1);
2096e86d1aa8SWill Deacon
209797dfad19SRobin Murphy smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
209897dfad19SRobin Murphy sizeof(*smmu->irqs), GFP_KERNEL);
209997dfad19SRobin Murphy if (!smmu->irqs)
210097dfad19SRobin Murphy return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
210197dfad19SRobin Murphy smmu->num_context_irqs);
2102e86d1aa8SWill Deacon
210397dfad19SRobin Murphy for (i = 0; i < smmu->num_context_irqs; i++) {
210497dfad19SRobin Murphy int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2105e86d1aa8SWill Deacon
2106e86d1aa8SWill Deacon if (irq < 0)
210797dfad19SRobin Murphy return irq;
2108e86d1aa8SWill Deacon smmu->irqs[i] = irq;
2109e86d1aa8SWill Deacon }
2110e86d1aa8SWill Deacon
2111e86d1aa8SWill Deacon err = devm_clk_bulk_get_all(dev, &smmu->clks);
2112e86d1aa8SWill Deacon if (err < 0) {
2113e86d1aa8SWill Deacon dev_err(dev, "failed to get clocks %d\n", err);
2114e86d1aa8SWill Deacon return err;
2115e86d1aa8SWill Deacon }
2116e86d1aa8SWill Deacon smmu->num_clks = err;
2117e86d1aa8SWill Deacon
2118e86d1aa8SWill Deacon err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2119e86d1aa8SWill Deacon if (err)
2120e86d1aa8SWill Deacon return err;
2121e86d1aa8SWill Deacon
2122e86d1aa8SWill Deacon err = arm_smmu_device_cfg_probe(smmu);
2123e86d1aa8SWill Deacon if (err)
2124e86d1aa8SWill Deacon return err;
2125e86d1aa8SWill Deacon
2126e86d1aa8SWill Deacon if (smmu->version == ARM_SMMU_V2) {
2127e86d1aa8SWill Deacon if (smmu->num_context_banks > smmu->num_context_irqs) {
2128e86d1aa8SWill Deacon dev_err(dev,
2129e86d1aa8SWill Deacon "found only %d context irq(s) but %d required\n",
2130e86d1aa8SWill Deacon smmu->num_context_irqs, smmu->num_context_banks);
2131e86d1aa8SWill Deacon return -ENODEV;
2132e86d1aa8SWill Deacon }
2133e86d1aa8SWill Deacon
2134e86d1aa8SWill Deacon /* Ignore superfluous interrupts */
2135e86d1aa8SWill Deacon smmu->num_context_irqs = smmu->num_context_banks;
2136e86d1aa8SWill Deacon }
2137e86d1aa8SWill Deacon
2138e86d1aa8SWill Deacon if (smmu->impl && smmu->impl->global_fault)
2139e86d1aa8SWill Deacon global_fault = smmu->impl->global_fault;
2140e86d1aa8SWill Deacon else
2141e86d1aa8SWill Deacon global_fault = arm_smmu_global_fault;
2142e86d1aa8SWill Deacon
214397dfad19SRobin Murphy for (i = 0; i < global_irqs; i++) {
214497dfad19SRobin Murphy int irq = platform_get_irq(pdev, i);
214597dfad19SRobin Murphy
214697dfad19SRobin Murphy if (irq < 0)
214797dfad19SRobin Murphy return irq;
214897dfad19SRobin Murphy
214997dfad19SRobin Murphy err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
215097dfad19SRobin Murphy "arm-smmu global fault", smmu);
215197dfad19SRobin Murphy if (err)
215297dfad19SRobin Murphy return dev_err_probe(dev, err,
215397dfad19SRobin Murphy "failed to request global IRQ %d (%u)\n",
215497dfad19SRobin Murphy i, irq);
2155e86d1aa8SWill Deacon }
2156e86d1aa8SWill Deacon
2157e86d1aa8SWill Deacon err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2158b9b721d1SSai Prakash Ranjan "smmu.%pa", &smmu->ioaddr);
2159e86d1aa8SWill Deacon if (err) {
2160e86d1aa8SWill Deacon dev_err(dev, "Failed to register iommu in sysfs\n");
2161e86d1aa8SWill Deacon return err;
2162e86d1aa8SWill Deacon }
2163e86d1aa8SWill Deacon
21642d471b20SRobin Murphy err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
2165e86d1aa8SWill Deacon if (err) {
2166e86d1aa8SWill Deacon dev_err(dev, "Failed to register iommu\n");
21673c34d1c2SRobin Murphy iommu_device_sysfs_remove(&smmu->iommu);
21683c34d1c2SRobin Murphy return err;
2169e86d1aa8SWill Deacon }
2170e86d1aa8SWill Deacon
2171e86d1aa8SWill Deacon platform_set_drvdata(pdev, smmu);
21720bec0557SJon Nettleton
21730bec0557SJon Nettleton /* Check for RMRs and install bypass SMRs if any */
21740bec0557SJon Nettleton arm_smmu_rmr_install_bypass_smr(smmu);
21750bec0557SJon Nettleton
2176e86d1aa8SWill Deacon arm_smmu_device_reset(smmu);
2177e86d1aa8SWill Deacon arm_smmu_test_smr_masks(smmu);
2178e86d1aa8SWill Deacon
2179e86d1aa8SWill Deacon /*
2180e86d1aa8SWill Deacon * We want to avoid touching dev->power.lock in fastpaths unless
2181e86d1aa8SWill Deacon * it's really going to do something useful - pm_runtime_enabled()
2182e86d1aa8SWill Deacon * can serve as an ideal proxy for that decision. So, conditionally
2183e86d1aa8SWill Deacon * enable pm_runtime.
2184e86d1aa8SWill Deacon */
2185e86d1aa8SWill Deacon if (dev->pm_domain) {
2186e86d1aa8SWill Deacon pm_runtime_set_active(dev);
2187e86d1aa8SWill Deacon pm_runtime_enable(dev);
2188e86d1aa8SWill Deacon }
2189e86d1aa8SWill Deacon
2190e86d1aa8SWill Deacon return 0;
2191e86d1aa8SWill Deacon }
2192e86d1aa8SWill Deacon
arm_smmu_device_shutdown(struct platform_device * pdev)2193ce31e6caSVladimir Oltean static void arm_smmu_device_shutdown(struct platform_device *pdev)
2194e86d1aa8SWill Deacon {
2195e86d1aa8SWill Deacon struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2196e86d1aa8SWill Deacon
2197e86d1aa8SWill Deacon if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2198e86d1aa8SWill Deacon dev_notice(&pdev->dev, "disabling translation\n");
2199e86d1aa8SWill Deacon
2200e86d1aa8SWill Deacon arm_smmu_rpm_get(smmu);
2201e86d1aa8SWill Deacon /* Turn the thing off */
2202e86d1aa8SWill Deacon arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2203e86d1aa8SWill Deacon arm_smmu_rpm_put(smmu);
2204e86d1aa8SWill Deacon
2205e86d1aa8SWill Deacon if (pm_runtime_enabled(smmu->dev))
2206e86d1aa8SWill Deacon pm_runtime_force_suspend(smmu->dev);
2207e86d1aa8SWill Deacon else
2208e86d1aa8SWill Deacon clk_bulk_disable(smmu->num_clks, smmu->clks);
2209e86d1aa8SWill Deacon
2210e86d1aa8SWill Deacon clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2211e86d1aa8SWill Deacon }
2212e86d1aa8SWill Deacon
arm_smmu_device_remove(struct platform_device * pdev)221362565a77SUwe Kleine-König static void arm_smmu_device_remove(struct platform_device *pdev)
2214e86d1aa8SWill Deacon {
2215ce31e6caSVladimir Oltean struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2216ce31e6caSVladimir Oltean
2217ce31e6caSVladimir Oltean iommu_device_unregister(&smmu->iommu);
2218ce31e6caSVladimir Oltean iommu_device_sysfs_remove(&smmu->iommu);
2219ce31e6caSVladimir Oltean
2220ce31e6caSVladimir Oltean arm_smmu_device_shutdown(pdev);
2221e86d1aa8SWill Deacon }
2222e86d1aa8SWill Deacon
arm_smmu_runtime_resume(struct device * dev)2223e86d1aa8SWill Deacon static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2224e86d1aa8SWill Deacon {
2225e86d1aa8SWill Deacon struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2226e86d1aa8SWill Deacon int ret;
2227e86d1aa8SWill Deacon
2228e86d1aa8SWill Deacon ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2229e86d1aa8SWill Deacon if (ret)
2230e86d1aa8SWill Deacon return ret;
2231e86d1aa8SWill Deacon
2232e86d1aa8SWill Deacon arm_smmu_device_reset(smmu);
2233e86d1aa8SWill Deacon
2234e86d1aa8SWill Deacon return 0;
2235e86d1aa8SWill Deacon }
2236e86d1aa8SWill Deacon
arm_smmu_runtime_suspend(struct device * dev)2237e86d1aa8SWill Deacon static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2238e86d1aa8SWill Deacon {
2239e86d1aa8SWill Deacon struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2240e86d1aa8SWill Deacon
2241e86d1aa8SWill Deacon clk_bulk_disable(smmu->num_clks, smmu->clks);
2242e86d1aa8SWill Deacon
2243e86d1aa8SWill Deacon return 0;
2244e86d1aa8SWill Deacon }
2245e86d1aa8SWill Deacon
arm_smmu_pm_resume(struct device * dev)2246e86d1aa8SWill Deacon static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2247e86d1aa8SWill Deacon {
2248afefe67eSSai Prakash Ranjan int ret;
2249afefe67eSSai Prakash Ranjan struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2250afefe67eSSai Prakash Ranjan
2251afefe67eSSai Prakash Ranjan ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2252afefe67eSSai Prakash Ranjan if (ret)
2253afefe67eSSai Prakash Ranjan return ret;
2254afefe67eSSai Prakash Ranjan
2255e86d1aa8SWill Deacon if (pm_runtime_suspended(dev))
2256e86d1aa8SWill Deacon return 0;
2257e86d1aa8SWill Deacon
2258afefe67eSSai Prakash Ranjan ret = arm_smmu_runtime_resume(dev);
2259afefe67eSSai Prakash Ranjan if (ret)
2260afefe67eSSai Prakash Ranjan clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2261afefe67eSSai Prakash Ranjan
2262afefe67eSSai Prakash Ranjan return ret;
2263e86d1aa8SWill Deacon }
2264e86d1aa8SWill Deacon
arm_smmu_pm_suspend(struct device * dev)2265e86d1aa8SWill Deacon static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2266e86d1aa8SWill Deacon {
2267afefe67eSSai Prakash Ranjan int ret = 0;
2268afefe67eSSai Prakash Ranjan struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2269e86d1aa8SWill Deacon
2270afefe67eSSai Prakash Ranjan if (pm_runtime_suspended(dev))
2271afefe67eSSai Prakash Ranjan goto clk_unprepare;
2272afefe67eSSai Prakash Ranjan
2273afefe67eSSai Prakash Ranjan ret = arm_smmu_runtime_suspend(dev);
2274afefe67eSSai Prakash Ranjan if (ret)
2275afefe67eSSai Prakash Ranjan return ret;
2276afefe67eSSai Prakash Ranjan
2277afefe67eSSai Prakash Ranjan clk_unprepare:
2278afefe67eSSai Prakash Ranjan clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2279afefe67eSSai Prakash Ranjan return ret;
2280e86d1aa8SWill Deacon }
2281e86d1aa8SWill Deacon
2282e86d1aa8SWill Deacon static const struct dev_pm_ops arm_smmu_pm_ops = {
2283e86d1aa8SWill Deacon SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2284e86d1aa8SWill Deacon SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2285e86d1aa8SWill Deacon arm_smmu_runtime_resume, NULL)
2286e86d1aa8SWill Deacon };
2287e86d1aa8SWill Deacon
2288e86d1aa8SWill Deacon static struct platform_driver arm_smmu_driver = {
2289e86d1aa8SWill Deacon .driver = {
2290e86d1aa8SWill Deacon .name = "arm-smmu",
2291e86d1aa8SWill Deacon .of_match_table = arm_smmu_of_match,
2292e86d1aa8SWill Deacon .pm = &arm_smmu_pm_ops,
2293e86d1aa8SWill Deacon .suppress_bind_attrs = true,
2294e86d1aa8SWill Deacon },
2295e86d1aa8SWill Deacon .probe = arm_smmu_device_probe,
229662565a77SUwe Kleine-König .remove_new = arm_smmu_device_remove,
2297e86d1aa8SWill Deacon .shutdown = arm_smmu_device_shutdown,
2298e86d1aa8SWill Deacon };
2299e86d1aa8SWill Deacon module_platform_driver(arm_smmu_driver);
2300e86d1aa8SWill Deacon
2301e86d1aa8SWill Deacon MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2302e86d1aa8SWill Deacon MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2303e86d1aa8SWill Deacon MODULE_ALIAS("platform:arm-smmu");
2304e86d1aa8SWill Deacon MODULE_LICENSE("GPL v2");
2305