Lines Matching +full:cn9900 +full:- +full:broken +full:- +full:page1 +full:- +full:regspace
1 // SPDX-License-Identifier: GPL-2.0
19 #include <linux/io-pgtable.h>
27 #include <linux/pci-ats.h>
30 #include "arm-smmu-v3.h"
31 #include "../../dma-iommu.h"
32 #include "../../iommu-sva.h"
42 "Disable MSI-based polling for CMD_SYNC completion.");
84 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
85 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
94 if (of_property_read_bool(smmu->dev->of_node,
96 smmu->options |= arm_smmu_options[i].opt;
97 dev_notice(smmu->dev, "option %s\n",
103 /* Low-level queue manipulation functions */
108 prod = Q_IDX(q, q->prod);
109 cons = Q_IDX(q, q->cons);
111 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
112 space = (1 << q->max_n_shift) - (prod - cons);
114 space = cons - prod;
121 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
122 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
127 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
128 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
133 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
134 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
135 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
136 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
146 writel_relaxed(q->llq.cons, q->cons_reg);
151 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
152 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
157 struct arm_smmu_ll_queue *llq = &q->llq;
159 if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
162 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
163 Q_IDX(llq, llq->cons);
177 prod = readl(q->prod_reg);
179 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
180 ret = -EOVERFLOW;
182 q->llq.prod = prod;
188 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
189 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
195 qp->delay = 1;
196 qp->spin_cnt = 0;
197 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
198 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
203 if (ktime_compare(ktime_get(), qp->timeout) > 0)
204 return -ETIMEDOUT;
206 if (qp->wfe) {
208 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
211 udelay(qp->delay);
212 qp->delay *= 2;
213 qp->spin_cnt = 0;
237 if (queue_empty(&q->llq))
238 return -EAGAIN;
240 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
241 queue_inc_cons(&q->llq);
246 /* High-level queue accessors */
250 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
252 switch (ent->opcode) {
257 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
260 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
263 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
264 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
267 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
274 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
277 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
278 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
279 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
280 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
281 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
282 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
283 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
286 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
287 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
288 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
289 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
290 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
291 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
292 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
295 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
298 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
301 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
304 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
305 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
306 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
307 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
308 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
309 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
312 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
313 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
314 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
315 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
316 switch (ent->pri.resp) {
322 return -EINVAL;
324 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
327 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
328 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
329 cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
332 if (ent->sync.msiaddr) {
334 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
342 return -ENOENT;
350 return &smmu->cmdq;
364 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
365 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
366 q->ent_dwords * 8;
384 u32 cons = readl_relaxed(q->cons_reg);
390 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
395 dev_err(smmu->dev, "retrying command fetch\n");
416 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
417 dev_err(smmu->dev, "skipping command in error state:\n");
419 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
424 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
429 __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q);
436 * - The only LOCK routines are exclusive_trylock() and shared_lock().
440 * - The UNLOCK routines are supplemented with shared_tryunlock(), which
454 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
458 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
459 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
464 (void)atomic_dec_return_release(&cmdq->lock);
469 if (atomic_read(&cmdq->lock) == 1)
480 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
488 atomic_set_release(&cmdq->lock, 0); \
497 * you like mixed-size concurrency, dependency ordering and relaxed atomics,
537 .max_n_shift = cmdq->q.llq.max_n_shift,
552 ptr = &cmdq->valid_map[swidx];
557 mask = GENMASK(limit - 1, sbidx);
561 * that a zero-initialised queue is invalid and, after marking
574 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
592 /* Wait for the command queue to become non-full */
606 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
608 llq->val = READ_ONCE(cmdq->q.llq.val);
614 llq->val = READ_ONCE(cmdq->q.llq.val);
634 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
644 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
649 * Wait until the SMMU cons index passes llq->prod.
657 u32 prod = llq->prod;
661 llq->val = READ_ONCE(cmdq->q.llq.val);
676 * cmdq->q.llq.cons. Roughly speaking:
696 llq->cons = readl(cmdq->q.cons_reg);
705 if (smmu->options & ARM_SMMU_OPT_MSIPOLL)
716 .max_n_shift = cmdq->q.llq.max_n_shift,
724 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
732 * - There is a dma_wmb() before publishing any commands to the queue.
736 * - On completion of a CMD_SYNC, there is a control dependency.
740 * - Command insertion is totally ordered, so if two CPUs each race to
755 llq.max_n_shift = cmdq->q.llq.max_n_shift;
759 llq.val = READ_ONCE(cmdq->q.llq.val);
766 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
774 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
791 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod);
792 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
810 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
814 &cmdq->q.llq.atomic.prod);
828 writel_relaxed(prod, cmdq->q.prod_reg);
835 atomic_set_release(&cmdq->owner_prod, prod);
843 dev_err_ratelimited(smmu->dev,
846 readl_relaxed(cmdq->q.prod_reg),
847 readl_relaxed(cmdq->q.cons_reg));
852 * reader, in which case we can safely update cmdq->q.llq.cons
855 WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
871 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
872 ent->opcode);
873 return -EINVAL;
897 if (cmds->num == CMDQ_BATCH_ENTRIES - 1 &&
898 (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC)) {
899 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
900 cmds->num = 0;
903 if (cmds->num == CMDQ_BATCH_ENTRIES) {
904 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false);
905 cmds->num = 0;
908 index = cmds->num * CMDQ_ENT_DWORDS;
909 if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
910 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
911 cmd->opcode);
915 cmds->num++;
921 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, true);
930 int sid = master->streams[0].id;
932 if (master->stall_enabled) {
935 cmd.resume.stag = resp->grpid;
936 switch (resp->code) {
945 return -EINVAL;
948 return -ENODEV;
951 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
966 .opcode = smmu->features & ARM_SMMU_FEAT_E2H ?
981 struct arm_smmu_device *smmu = smmu_domain->smmu;
992 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
993 list_for_each_entry(master, &smmu_domain->devices, domain_head) {
994 for (i = 0; i < master->num_streams; i++) {
995 cmd.cfgi.sid = master->streams[i].id;
999 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1009 l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
1010 &l1_desc->l2ptr_dma, GFP_KERNEL);
1011 if (!l1_desc->l2ptr) {
1012 dev_warn(smmu->dev,
1014 return -ENOMEM;
1022 u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) |
1035 struct arm_smmu_device *smmu = smmu_domain->smmu;
1036 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1038 if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1039 return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
1042 l1_desc = &cdcfg->l1_desc[idx];
1043 if (!l1_desc->l2ptr) {
1047 l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
1052 idx = ssid & (CTXDESC_L2_ENTRIES - 1);
1053 return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
1074 if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
1075 return -E2BIG;
1079 return -ENOMEM;
1090 val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
1096 cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
1098 cdptr[3] = cpu_to_le64(cd->mair);
1107 val = cd->tcr |
1112 (cd->mm ? 0 : CTXDESC_CD_0_ASET) |
1114 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
1117 if (smmu_domain->stall_enabled)
1122 * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
1125 * The size of single-copy atomic reads made by the SMMU is
1127 * field within an aligned 64-bit span of a structure can be altered
1140 struct arm_smmu_device *smmu = smmu_domain->smmu;
1141 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
1142 struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
1144 max_contexts = 1 << cfg->s1cdmax;
1146 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1148 cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1149 cdcfg->num_l1_ents = max_contexts;
1153 cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1154 cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
1157 cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
1158 sizeof(*cdcfg->l1_desc),
1160 if (!cdcfg->l1_desc)
1161 return -ENOMEM;
1163 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1166 cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
1168 if (!cdcfg->cdtab) {
1169 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
1170 ret = -ENOMEM;
1177 if (cdcfg->l1_desc) {
1178 devm_kfree(smmu->dev, cdcfg->l1_desc);
1179 cdcfg->l1_desc = NULL;
1188 struct arm_smmu_device *smmu = smmu_domain->smmu;
1189 struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
1191 if (cdcfg->l1_desc) {
1194 for (i = 0; i < cdcfg->num_l1_ents; i++) {
1195 if (!cdcfg->l1_desc[i].l2ptr)
1198 dmam_free_coherent(smmu->dev, size,
1199 cdcfg->l1_desc[i].l2ptr,
1200 cdcfg->l1_desc[i].l2ptr_dma);
1202 devm_kfree(smmu->dev, cdcfg->l1_desc);
1203 cdcfg->l1_desc = NULL;
1205 l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
1207 l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
1210 dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
1211 cdcfg->cdtab_dma = 0;
1212 cdcfg->cdtab = NULL;
1220 if (!cd->asid)
1223 free = refcount_dec_and_test(&cd->refs);
1225 old_cd = xa_erase(&arm_smmu_asid_xa, cd->asid);
1237 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1238 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1264 * 1. Invalid (all zero) -> bypass/fault (init)
1265 * 2. Bypass/fault -> translation/bypass (attach)
1266 * 3. Translation/bypass -> bypass/fault (detach)
1290 smmu_domain = master->domain;
1291 smmu = master->smmu;
1295 switch (smmu_domain->stage) {
1297 s1_cfg = &smmu_domain->s1_cfg;
1301 s2_cfg = &smmu_domain->s2_cfg;
1348 u64 strw = smmu->features & ARM_SMMU_FEAT_E2H ?
1359 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1360 !master->stall_enabled)
1363 val |= (s1_cfg->cdcfg.cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1365 FIELD_PREP(STRTAB_STE_0_S1CDMAX, s1_cfg->s1cdmax) |
1366 FIELD_PREP(STRTAB_STE_0_S1FMT, s1_cfg->s1fmt);
1372 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1373 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1380 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1385 if (master->ats_enabled)
1395 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1422 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1423 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1425 if (desc->l2ptr)
1429 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1431 desc->span = STRTAB_SPLIT + 1;
1432 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1434 if (!desc->l2ptr) {
1435 dev_err(smmu->dev,
1438 return -ENOMEM;
1441 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT, false);
1452 if (*sid_lhs < stream_rhs->id)
1453 return -1;
1454 if (*sid_lhs > stream_rhs->id)
1463 &rb_entry(lhs, struct arm_smmu_stream, node)->id, rhs);
1471 lockdep_assert_held(&smmu->streams_mutex);
1473 node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key);
1476 return rb_entry(node, struct arm_smmu_stream, node)->master;
1505 return -EOPNOTSUPP;
1508 /* Stage-2 is always pinned at the moment */
1510 return -EFAULT;
1524 flt->type = IOMMU_FAULT_PAGE_REQ;
1525 flt->prm = (struct iommu_fault_page_request) {
1533 flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1534 flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1537 flt->type = IOMMU_FAULT_DMA_UNRECOV;
1538 flt->event = (struct iommu_fault_unrecoverable) {
1546 flt->event.flags |= IOMMU_FAULT_UNRECOV_PASID_VALID;
1547 flt->event.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1551 mutex_lock(&smmu->streams_mutex);
1554 ret = -EINVAL;
1558 ret = iommu_report_device_fault(master->dev, &fault_evt);
1559 if (ret && flt->type == IOMMU_FAULT_PAGE_REQ) {
1562 .pasid = flt->prm.pasid,
1563 .grpid = flt->prm.grpid,
1566 arm_smmu_page_response(master->dev, &fault_evt, &resp);
1570 mutex_unlock(&smmu->streams_mutex);
1578 struct arm_smmu_queue *q = &smmu->evtq.q;
1579 struct arm_smmu_ll_queue *llq = &q->llq;
1592 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1594 dev_info(smmu->dev, "\t0x%016llx\n",
1604 if (queue_sync_prod_in(q) == -EOVERFLOW)
1605 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1625 dev_info(smmu->dev, "unexpected PRI request received:\n");
1626 dev_info(smmu->dev,
1654 struct arm_smmu_queue *q = &smmu->priq.q;
1655 struct arm_smmu_ll_queue *llq = &q->llq;
1662 if (queue_sync_prod_in(q) == -EOVERFLOW)
1663 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1678 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1679 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1685 dev_warn(smmu->dev,
1690 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1695 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1698 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1701 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1704 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1707 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1710 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1715 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1724 if (smmu->features & ARM_SMMU_FEAT_PRI)
1742 /* ATC invalidates are always on 4096-bytes pages */
1754 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
1757 * This has the unpleasant side-effect of invalidating all PASID-tagged
1767 cmd->atc.size = ATC_INV_SIZE_ALL;
1772 page_end = (iova + size - 1) >> inval_grain_shift;
1777 * thus have to choose between grossly over-invalidating the region, or
1795 span_mask = (1ULL << log2_span) - 1;
1799 cmd->atc.addr = page_start << inval_grain_shift;
1800 cmd->atc.size = log2_span;
1812 for (i = 0; i < master->num_streams; i++) {
1813 cmd.atc.sid = master->streams[i].id;
1814 arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
1817 return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
1829 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1846 if (!atomic_read(&smmu_domain->nr_ats_masters))
1853 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1854 list_for_each_entry(master, &smmu_domain->devices, domain_head) {
1855 if (!master->ats_enabled)
1858 for (i = 0; i < master->num_streams; i++) {
1859 cmd.atc.sid = master->streams[i].id;
1860 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
1863 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1865 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
1872 struct arm_smmu_device *smmu = smmu_domain->smmu;
1876 * NOTE: when io-pgtable is in non-strict mode, we may get here with
1882 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1883 arm_smmu_tlb_inv_asid(smmu, smmu_domain->s1_cfg.cd.asid);
1886 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1897 struct arm_smmu_device *smmu = smmu_domain->smmu;
1905 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
1907 tg = __ffs(smmu_domain->domain.pgsize_bitmap);
1912 cmd->tlbi.tg = (tg - 10) / 2;
1915 * Determine what level the granule is at. For non-leaf, both
1916 * io-pgtable and SVA pass a nominal last-level granule because
1922 if (cmd->tlbi.leaf)
1923 cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
1931 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
1943 cmd->tlbi.scale = scale;
1947 cmd->tlbi.num = num - 1;
1953 num_pages -= num << scale;
1956 cmd->tlbi.addr = iova;
1973 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1974 cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
1976 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1979 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1984 * Unfortunately, this can't be leaf-only since we may have
1995 .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2011 struct iommu_domain *domain = &smmu_domain->domain;
2036 return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2066 mutex_init(&smmu_domain->init_mutex);
2067 INIT_LIST_HEAD(&smmu_domain->devices);
2068 spin_lock_init(&smmu_domain->devices_lock);
2069 INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);
2071 return &smmu_domain->domain;
2077 struct arm_smmu_device *smmu = smmu_domain->smmu;
2079 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2082 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2083 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2087 if (cfg->cdcfg.cdtab)
2089 arm_smmu_free_asid(&cfg->cd);
2092 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2093 if (cfg->vmid)
2094 ida_free(&smmu->vmid_map, cfg->vmid);
2106 struct arm_smmu_device *smmu = smmu_domain->smmu;
2107 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2108 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr = &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2110 refcount_set(&cfg->cd.refs, 1);
2114 ret = xa_alloc(&arm_smmu_asid_xa, &asid, &cfg->cd,
2115 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2119 cfg->s1cdmax = master->ssid_bits;
2121 smmu_domain->stall_enabled = master->stall_enabled;
2127 cfg->cd.asid = (u16)asid;
2128 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
2129 cfg->cd.tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
2130 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
2131 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
2132 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
2133 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
2134 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
2136 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair;
2143 ret = arm_smmu_write_ctx_desc(smmu_domain, IOMMU_NO_PASID, &cfg->cd);
2153 arm_smmu_free_asid(&cfg->cd);
2164 struct arm_smmu_device *smmu = smmu_domain->smmu;
2165 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2166 typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr;
2168 /* Reserve VMID 0 for stage-2 bypass STEs */
2169 vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
2174 vtcr = &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2175 cfg->vmid = (u16)vmid;
2176 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2177 cfg->vtcr = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
2178 FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
2179 FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
2180 FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
2181 FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
2182 FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
2183 FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
2199 struct arm_smmu_device *smmu = smmu_domain->smmu;
2201 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
2202 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
2207 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2208 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2209 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2210 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2212 switch (smmu_domain->stage) {
2214 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
2216 oas = smmu->ias;
2222 ias = smmu->ias;
2223 oas = smmu->oas;
2228 return -EINVAL;
2232 .pgsize_bitmap = smmu->pgsize_bitmap,
2235 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2237 .iommu_dev = smmu->dev,
2242 return -ENOMEM;
2244 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2245 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2246 domain->geometry.force_aperture = true;
2254 smmu_domain->pgtbl_ops = pgtbl_ops;
2261 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2263 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2267 /* Two-level walk */
2269 l1_desc = &cfg->l1_desc[idx];
2270 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
2271 step = &l1_desc->l2ptr[idx];
2274 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
2283 struct arm_smmu_device *smmu = master->smmu;
2285 for (i = 0; i < master->num_streams; ++i) {
2286 u32 sid = master->streams[i].id;
2291 if (master->streams[j].id == sid)
2302 struct device *dev = master->dev;
2303 struct arm_smmu_device *smmu = master->smmu;
2306 if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2309 if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2319 struct arm_smmu_device *smmu = master->smmu;
2320 struct arm_smmu_domain *smmu_domain = master->domain;
2323 if (!master->ats_enabled)
2327 stu = __ffs(smmu->pgsize_bitmap);
2328 pdev = to_pci_dev(master->dev);
2330 atomic_inc(&smmu_domain->nr_ats_masters);
2333 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2338 struct arm_smmu_domain *smmu_domain = master->domain;
2340 if (!master->ats_enabled)
2343 pci_disable_ats(to_pci_dev(master->dev));
2350 atomic_dec(&smmu_domain->nr_ats_masters);
2360 if (!dev_is_pci(master->dev))
2361 return -ENODEV;
2363 pdev = to_pci_dev(master->dev);
2375 dev_err(&pdev->dev, "Failed to enable PASID\n");
2379 master->ssid_bits = min_t(u8, ilog2(num_pasids),
2380 master->smmu->ssid_bits);
2388 if (!dev_is_pci(master->dev))
2391 pdev = to_pci_dev(master->dev);
2393 if (!pdev->pasid_enabled)
2396 master->ssid_bits = 0;
2403 struct arm_smmu_domain *smmu_domain = master->domain;
2410 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2411 list_del(&master->domain_head);
2412 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2414 master->domain = NULL;
2415 master->ats_enabled = false;
2429 return -ENOENT;
2432 smmu = master->smmu;
2440 dev_err(dev, "cannot attach - SVA enabled\n");
2441 return -EBUSY;
2446 mutex_lock(&smmu_domain->init_mutex);
2448 if (!smmu_domain->smmu) {
2449 smmu_domain->smmu = smmu;
2452 smmu_domain->smmu = NULL;
2455 } else if (smmu_domain->smmu != smmu) {
2456 ret = -EINVAL;
2458 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2459 master->ssid_bits != smmu_domain->s1_cfg.s1cdmax) {
2460 ret = -EINVAL;
2462 } else if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1 &&
2463 smmu_domain->stall_enabled != master->stall_enabled) {
2464 ret = -EINVAL;
2468 master->domain = smmu_domain;
2477 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2478 master->ats_enabled = arm_smmu_ats_supported(master);
2482 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2483 list_add(&master->domain_head, &smmu_domain->devices);
2484 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2489 mutex_unlock(&smmu_domain->init_mutex);
2497 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2500 return -ENODEV;
2502 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
2510 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
2515 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
2522 if (smmu_domain->smmu)
2531 if (!gather->pgsize)
2534 arm_smmu_tlb_inv_range_domain(gather->start,
2535 gather->end - gather->start + 1,
2536 gather->pgsize, true, smmu_domain);
2542 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2547 return ops->iova_to_phys(ops, iova);
2563 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2565 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2575 return -ERANGE;
2578 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2589 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2591 master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
2593 if (!master->streams)
2594 return -ENOMEM;
2595 master->num_streams = fwspec->num_ids;
2597 mutex_lock(&smmu->streams_mutex);
2598 for (i = 0; i < fwspec->num_ids; i++) {
2599 struct arm_smmu_stream *new_stream = &master->streams[i];
2601 u32 sid = fwspec->ids[i];
2603 new_stream->id = sid;
2604 new_stream->master = master;
2611 existing = rb_find_add(&new_stream->node, &smmu->streams,
2616 ->master;
2622 dev_warn(master->dev,
2624 dev_name(existing_master->dev));
2625 ret = -EINVAL;
2631 for (i--; i >= 0; i--)
2632 rb_erase(&master->streams[i].node, &smmu->streams);
2633 kfree(master->streams);
2635 mutex_unlock(&smmu->streams_mutex);
2643 struct arm_smmu_device *smmu = master->smmu;
2644 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2646 if (!smmu || !master->streams)
2649 mutex_lock(&smmu->streams_mutex);
2650 for (i = 0; i < fwspec->num_ids; i++)
2651 rb_erase(&master->streams[i].node, &smmu->streams);
2652 mutex_unlock(&smmu->streams_mutex);
2654 kfree(master->streams);
2666 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2667 return ERR_PTR(-ENODEV);
2670 return ERR_PTR(-EBUSY);
2672 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2674 return ERR_PTR(-ENODEV);
2678 return ERR_PTR(-ENOMEM);
2680 master->dev = dev;
2681 master->smmu = smmu;
2682 INIT_LIST_HEAD(&master->bonds);
2689 device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
2690 master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
2694 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
2702 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
2703 master->ssid_bits = min_t(u8, master->ssid_bits,
2706 if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
2707 device_property_read_bool(dev, "dma-can-stall")) ||
2708 smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
2709 master->stall_enabled = true;
2711 return &smmu->iommu;
2724 iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
2737 * aliases, since the necessary ID-to-device lookup becomes rather
2738 * impractical given a potential sparse 32-bit stream ID space.
2753 mutex_lock(&smmu_domain->init_mutex);
2754 if (smmu_domain->smmu)
2755 ret = -EPERM;
2757 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2758 mutex_unlock(&smmu_domain->init_mutex);
2765 return iommu_fwspec_add_ids(dev, args->args, 1);
2779 list_add_tail(®ion->list, head);
2790 return -ENODEV;
2795 return -EINVAL;
2796 if (master->iopf_enabled)
2797 return -EBUSY;
2798 master->iopf_enabled = true;
2802 return -EINVAL;
2804 return -EBUSY;
2807 return -EINVAL;
2817 return -EINVAL;
2821 if (!master->iopf_enabled)
2822 return -EINVAL;
2823 if (master->sva_enabled)
2824 return -EBUSY;
2825 master->iopf_enabled = false;
2829 return -EINVAL;
2832 return -EINVAL;
2841 #define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
2842 (pdev)->device == 0xa12e)
2880 .pgsize_bitmap = -1UL, /* Restricted during device attach */
2905 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
2906 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
2908 if (q->base || qsz < PAGE_SIZE)
2911 q->llq.max_n_shift--;
2914 if (!q->base) {
2915 dev_err(smmu->dev,
2918 return -ENOMEM;
2921 if (!WARN_ON(q->base_dma & (qsz - 1))) {
2922 dev_info(smmu->dev, "allocated %u entries for %s\n",
2923 1 << q->llq.max_n_shift, name);
2926 q->prod_reg = page + prod_off;
2927 q->cons_reg = page + cons_off;
2928 q->ent_dwords = dwords;
2930 q->q_base = Q_BASE_RWA;
2931 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2932 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
2934 q->llq.prod = q->llq.cons = 0;
2940 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
2941 unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
2943 atomic_set(&cmdq->owner_prod, 0);
2944 atomic_set(&cmdq->lock, 0);
2946 cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
2948 if (!cmdq->valid_map)
2949 return -ENOMEM;
2959 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
2970 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
2976 if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
2977 (smmu->features & ARM_SMMU_FEAT_STALLS)) {
2978 smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
2979 if (!smmu->evtq.iopf)
2980 return -ENOMEM;
2984 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2987 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
2995 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2996 void *strtab = smmu->strtab_cfg.strtab;
2998 cfg->l1_desc = devm_kcalloc(smmu->dev, cfg->num_l1_ents,
2999 sizeof(*cfg->l1_desc), GFP_KERNEL);
3000 if (!cfg->l1_desc)
3001 return -ENOMEM;
3003 for (i = 0; i < cfg->num_l1_ents; ++i) {
3004 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
3016 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3019 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
3020 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
3021 cfg->num_l1_ents = 1 << size;
3024 if (size < smmu->sid_bits)
3025 dev_warn(smmu->dev,
3026 "2-level strtab only covers %u/%u bits of SID\n",
3027 size, smmu->sid_bits);
3029 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
3030 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
3033 dev_err(smmu->dev,
3036 return -ENOMEM;
3038 cfg->strtab = strtab;
3044 cfg->strtab_base_cfg = reg;
3054 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3056 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
3057 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
3060 dev_err(smmu->dev,
3063 return -ENOMEM;
3065 cfg->strtab = strtab;
3066 cfg->num_l1_ents = 1 << smmu->sid_bits;
3070 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
3071 cfg->strtab_base_cfg = reg;
3073 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents, false);
3082 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3091 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
3093 smmu->strtab_cfg.strtab_base = reg;
3095 ida_init(&smmu->vmid_map);
3104 mutex_init(&smmu->streams_mutex);
3105 smmu->streams = RB_ROOT;
3119 writel_relaxed(val, smmu->base + reg_off);
3120 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3128 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3142 dev_err(smmu->dev, "GBPA not responding to update\n");
3157 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
3159 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3162 writeq_relaxed(doorbell, smmu->base + cfg[0]);
3163 writel_relaxed(msg->data, smmu->base + cfg[1]);
3164 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3170 struct device *dev = smmu->dev;
3173 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3174 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3176 if (smmu->features & ARM_SMMU_FEAT_PRI)
3177 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3179 nvec--;
3181 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3184 if (!dev->msi.domain) {
3185 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3192 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3196 smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
3197 smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
3198 smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
3211 irq = smmu->evtq.q.irq;
3213 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3216 "arm-smmu-v3-evtq", smmu);
3218 dev_warn(smmu->dev, "failed to enable evtq irq\n");
3220 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3223 irq = smmu->gerr_irq;
3225 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3226 0, "arm-smmu-v3-gerror", smmu);
3228 dev_warn(smmu->dev, "failed to enable gerror irq\n");
3230 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3233 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3234 irq = smmu->priq.q.irq;
3236 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3239 "arm-smmu-v3-priq",
3242 dev_warn(smmu->dev,
3245 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3259 dev_err(smmu->dev, "failed to disable irqs\n");
3263 irq = smmu->combined_irq;
3269 ret = devm_request_threaded_irq(smmu->dev, irq,
3273 "arm-smmu-v3-combined-irq", smmu);
3275 dev_warn(smmu->dev, "failed to enable combined irq\n");
3279 if (smmu->features & ARM_SMMU_FEAT_PRI)
3286 dev_warn(smmu->dev, "failed to enable irqs\n");
3297 dev_err(smmu->dev, "failed to clear cr0\n");
3309 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
3311 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3327 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3332 if (smmu->features & ARM_SMMU_FEAT_E2H)
3335 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3338 writeq_relaxed(smmu->strtab_cfg.strtab_base,
3339 smmu->base + ARM_SMMU_STRTAB_BASE);
3340 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
3341 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3344 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3345 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3346 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
3352 dev_err(smmu->dev, "failed to enable command queue\n");
3361 if (smmu->features & ARM_SMMU_FEAT_HYP) {
3370 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3371 writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
3372 writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
3378 dev_err(smmu->dev, "failed to enable event queue\n");
3383 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3384 writeq_relaxed(smmu->priq.q.q_base,
3385 smmu->base + ARM_SMMU_PRIQ_BASE);
3386 writel_relaxed(smmu->priq.q.llq.prod,
3387 smmu->page1 + ARM_SMMU_PRIQ_PROD);
3388 writel_relaxed(smmu->priq.q.llq.cons,
3389 smmu->page1 + ARM_SMMU_PRIQ_CONS);
3395 dev_err(smmu->dev, "failed to enable PRI queue\n");
3400 if (smmu->features & ARM_SMMU_FEAT_ATS) {
3405 dev_err(smmu->dev, "failed to enable ATS check\n");
3412 dev_err(smmu->dev, "failed to setup irqs\n");
3430 dev_err(smmu->dev, "failed to enable SMMU interface\n");
3446 reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
3458 smmu->features &= ~ARM_SMMU_FEAT_SEV;
3461 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
3465 smmu->features &= ~ARM_SMMU_FEAT_BTM;
3466 smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
3468 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
3478 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
3481 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
3483 /* 2-level structures */
3485 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
3488 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
3497 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
3501 smmu->features |= ARM_SMMU_FEAT_TT_BE;
3505 smmu->features |= ARM_SMMU_FEAT_TT_LE;
3509 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
3510 return -ENXIO;
3515 smmu->features |= ARM_SMMU_FEAT_PRI;
3518 smmu->features |= ARM_SMMU_FEAT_ATS;
3521 smmu->features |= ARM_SMMU_FEAT_SEV;
3524 smmu->features |= ARM_SMMU_FEAT_MSI;
3526 smmu->options |= ARM_SMMU_OPT_MSIPOLL;
3530 smmu->features |= ARM_SMMU_FEAT_HYP;
3532 smmu->features |= ARM_SMMU_FEAT_E2H;
3540 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
3545 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
3548 smmu->features |= ARM_SMMU_FEAT_STALLS;
3552 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
3555 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
3558 dev_err(smmu->dev, "no translation support!\n");
3559 return -ENXIO;
3565 smmu->ias = 40;
3570 dev_err(smmu->dev, "AArch64 table format not supported!\n");
3571 return -ENXIO;
3575 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
3576 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
3579 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
3581 dev_err(smmu->dev, "embedded implementation not supported\n");
3582 return -ENXIO;
3586 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
3588 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
3593 * restrictions on the base pointer for a unit-length queue.
3595 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
3597 return -ENXIO;
3600 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
3602 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
3606 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
3607 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3608 smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
3614 if (smmu->sid_bits <= STRTAB_SPLIT)
3615 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
3618 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
3620 smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
3623 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
3626 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
3630 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
3632 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
3634 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
3638 smmu->features |= ARM_SMMU_FEAT_VAX;
3643 smmu->oas = 32;
3646 smmu->oas = 36;
3649 smmu->oas = 40;
3652 smmu->oas = 42;
3655 smmu->oas = 44;
3658 smmu->oas = 52;
3659 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
3662 dev_info(smmu->dev,
3663 "unknown output address size. Truncating to 48-bit\n");
3666 smmu->oas = 48;
3669 if (arm_smmu_ops.pgsize_bitmap == -1UL)
3670 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
3672 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
3675 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
3676 dev_warn(smmu->dev,
3679 smmu->ias = max(smmu->ias, smmu->oas);
3681 if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
3682 (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
3683 smmu->features |= ARM_SMMU_FEAT_NESTING;
3688 smmu->features |= ARM_SMMU_FEAT_SVA;
3690 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3691 smmu->ias, smmu->oas, smmu->features);
3700 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
3703 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3707 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3714 struct device *dev = smmu->dev;
3720 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3722 acpi_smmu_get_options(iort_smmu->model, smmu);
3724 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3725 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3733 return -ENODEV;
3740 struct device *dev = &pdev->dev;
3742 int ret = -EINVAL;
3744 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3745 dev_err(dev, "missing #iommu-cells property\n");
3747 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3753 if (of_dma_is_coherent(dev->of_node))
3754 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3761 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3781 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
3789 for (i = 0; i < rmr->num_sids; i++) {
3790 ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
3792 dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
3793 rmr->sids[i]);
3797 step = arm_smmu_get_step_for_sid(smmu, rmr->sids[i]);
3802 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
3811 struct device *dev = &pdev->dev;
3816 return -ENOMEM;
3817 smmu->dev = dev;
3819 if (dev->of_node) {
3823 if (ret == -ENODEV)
3833 return -EINVAL;
3836 return -EINVAL;
3838 ioaddr = res->start;
3844 smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
3845 if (IS_ERR(smmu->base))
3846 return PTR_ERR(smmu->base);
3849 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
3851 if (IS_ERR(smmu->page1))
3852 return PTR_ERR(smmu->page1);
3854 smmu->page1 = smmu->base;
3861 smmu->combined_irq = irq;
3865 smmu->evtq.q.irq = irq;
3869 smmu->priq.q.irq = irq;
3873 smmu->gerr_irq = irq;
3880 /* Initialise in-memory data structures */
3897 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3902 ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
3911 iommu_device_sysfs_remove(&smmu->iommu);
3915 iopf_queue_free(smmu->evtq.iopf);
3923 iommu_device_unregister(&smmu->iommu);
3924 iommu_device_sysfs_remove(&smmu->iommu);
3926 iopf_queue_free(smmu->evtq.iopf);
3927 ida_destroy(&smmu->vmid_map);
3938 { .compatible = "arm,smmu-v3", },
3951 .name = "arm-smmu-v3",
3964 MODULE_ALIAS("platform:arm-smmu-v3");