Lines Matching +full:pass +full:- +full:1
2 * QEMU emulation of an RISC-V IOMMU
4 * Copyright (C) 2021-2023, Rivos Inc.
23 #include "hw/qdev-properties.h"
30 #include "riscv-iommu.h"
31 #include "riscv-iommu-bits.h"
32 #include "riscv-iommu-hpm.h"
35 #define LIMIT_CACHE_CTX (1U << 7)
36 #define LIMIT_CACHE_IOT (1U << 20)
57 RISCV_IOMMU_TRANS_TAG_VG, /* G-stage only */
65 uint64_t pscid:20; /* Process Soft-Context identifier */
67 uint64_t gscid:16; /* Guest Soft-Context identifier */
94 if (!s->notify) { in riscv_iommu_notify()
99 ipsr = riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_IPSR, (1 << vec_type), 0); in riscv_iommu_notify()
101 if (!(ipsr & (1 << vec_type))) { in riscv_iommu_notify()
103 s->notify(s, vector); in riscv_iommu_notify()
112 uint32_t head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQH) & s->fq_mask; in riscv_iommu_fault()
113 uint32_t tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQT) & s->fq_mask; in riscv_iommu_fault()
114 uint32_t next = (tail + 1) & s->fq_mask; in riscv_iommu_fault()
115 uint32_t devid = get_field(ev->hdr, RISCV_IOMMU_FQ_HDR_DID); in riscv_iommu_fault()
117 trace_riscv_iommu_flt(s->parent_obj.id, PCI_BUS_NUM(devid), PCI_SLOT(devid), in riscv_iommu_fault()
118 PCI_FUNC(devid), ev->hdr, ev->iotval); in riscv_iommu_fault()
129 dma_addr_t addr = s->fq_addr + tail * sizeof(*ev); in riscv_iommu_fault()
130 if (dma_memory_write(s->target_as, addr, ev, sizeof(*ev), in riscv_iommu_fault()
148 uint32_t head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQH) & s->pq_mask; in riscv_iommu_pri()
149 uint32_t tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQT) & s->pq_mask; in riscv_iommu_pri()
150 uint32_t next = (tail + 1) & s->pq_mask; in riscv_iommu_pri()
151 uint32_t devid = get_field(pr->hdr, RISCV_IOMMU_PREQ_HDR_DID); in riscv_iommu_pri()
153 trace_riscv_iommu_pri(s->parent_obj.id, PCI_BUS_NUM(devid), PCI_SLOT(devid), in riscv_iommu_pri()
154 PCI_FUNC(devid), pr->payload); in riscv_iommu_pri()
165 dma_addr_t addr = s->pq_addr + tail * sizeof(*pr); in riscv_iommu_pri()
166 if (dma_memory_write(s->target_as, addr, pr, sizeof(*pr), in riscv_iommu_pri()
183 * bits from 'val' contiguously at the least-significant end of the
185 * other bits at the most-significant end of the result with zeros.
191 * ext = 1 0 1 0 0 1 1 0
194 * This function, taken from the riscv-iommu 1.0 spec, section 2.3.3
201 uint64_t rot = 1; in riscv_iommu_pext_u64()
204 if (ext & 1) { in riscv_iommu_pext_u64()
205 if (val & 1) { in riscv_iommu_pext_u64()
208 rot <<= 1; in riscv_iommu_pext_u64()
210 val >>= 1; in riscv_iommu_pext_u64()
211 ext >>= 1; in riscv_iommu_pext_u64()
221 if (!s->enable_msi) { in riscv_iommu_msi_check()
225 if (get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_MODE) != in riscv_iommu_msi_check()
230 if ((PPN_DOWN(gpa) ^ ctx->msi_addr_pattern) & ~ctx->msi_addr_mask) { in riscv_iommu_msi_check()
238 * RISCV IOMMU Address Translation Lookup - Page Table Walk
265 G_STAGE = 1, in riscv_iommu_spa_fetch()
266 } pass; in riscv_iommu_spa_fetch() local
269 satp = get_field(ctx->satp, RISCV_IOMMU_ATP_MODE_FIELD); in riscv_iommu_spa_fetch()
270 gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD); in riscv_iommu_spa_fetch()
279 * mapped using the g-stage page table, whether single- in riscv_iommu_spa_fetch()
280 * or two-stage paging is enabled. It's unavoidable though, in riscv_iommu_spa_fetch()
281 * because the spec mandates that we do a first-stage in riscv_iommu_spa_fetch()
286 if (!en_s && (iotlb->perm & IOMMU_WO) && in riscv_iommu_spa_fetch()
287 riscv_iommu_msi_check(s, ctx, iotlb->iova)) { in riscv_iommu_spa_fetch()
288 iotlb->target_as = &s->trap_as; in riscv_iommu_spa_fetch()
289 iotlb->translated_addr = iotlb->iova; in riscv_iommu_spa_fetch()
290 iotlb->addr_mask = ~TARGET_PAGE_MASK; in riscv_iommu_spa_fetch()
294 /* Exit early for pass-through mode. */ in riscv_iommu_spa_fetch()
296 iotlb->translated_addr = iotlb->iova; in riscv_iommu_spa_fetch()
297 iotlb->addr_mask = ~TARGET_PAGE_MASK; in riscv_iommu_spa_fetch()
298 /* Allow R/W in pass-through mode */ in riscv_iommu_spa_fetch()
299 iotlb->perm = IOMMU_RW; in riscv_iommu_spa_fetch()
304 for (pass = 0; pass < 2; pass++) { in riscv_iommu_spa_fetch()
307 sc[pass].step = 0; in riscv_iommu_spa_fetch()
308 if (pass ? (s->fctl & RISCV_IOMMU_FCTL_GXL) : in riscv_iommu_spa_fetch()
309 (ctx->tc & RISCV_IOMMU_DC_TC_SXL)) { in riscv_iommu_spa_fetch()
310 /* 32bit mode for GXL/SXL == 1 */ in riscv_iommu_spa_fetch()
311 switch (pass ? gatp : satp) { in riscv_iommu_spa_fetch()
313 sc[pass].levels = 0; in riscv_iommu_spa_fetch()
314 sc[pass].ptidxbits = 0; in riscv_iommu_spa_fetch()
315 sc[pass].ptesize = 0; in riscv_iommu_spa_fetch()
318 sv_mode = pass ? RISCV_IOMMU_CAP_SV32X4 : RISCV_IOMMU_CAP_SV32; in riscv_iommu_spa_fetch()
319 if (!(s->cap & sv_mode)) { in riscv_iommu_spa_fetch()
322 sc[pass].levels = 2; in riscv_iommu_spa_fetch()
323 sc[pass].ptidxbits = 10; in riscv_iommu_spa_fetch()
324 sc[pass].ptesize = 4; in riscv_iommu_spa_fetch()
331 switch (pass ? gatp : satp) { in riscv_iommu_spa_fetch()
333 sc[pass].levels = 0; in riscv_iommu_spa_fetch()
334 sc[pass].ptidxbits = 0; in riscv_iommu_spa_fetch()
335 sc[pass].ptesize = 0; in riscv_iommu_spa_fetch()
338 sv_mode = pass ? RISCV_IOMMU_CAP_SV39X4 : RISCV_IOMMU_CAP_SV39; in riscv_iommu_spa_fetch()
339 if (!(s->cap & sv_mode)) { in riscv_iommu_spa_fetch()
342 sc[pass].levels = 3; in riscv_iommu_spa_fetch()
343 sc[pass].ptidxbits = 9; in riscv_iommu_spa_fetch()
344 sc[pass].ptesize = 8; in riscv_iommu_spa_fetch()
347 sv_mode = pass ? RISCV_IOMMU_CAP_SV48X4 : RISCV_IOMMU_CAP_SV48; in riscv_iommu_spa_fetch()
348 if (!(s->cap & sv_mode)) { in riscv_iommu_spa_fetch()
351 sc[pass].levels = 4; in riscv_iommu_spa_fetch()
352 sc[pass].ptidxbits = 9; in riscv_iommu_spa_fetch()
353 sc[pass].ptesize = 8; in riscv_iommu_spa_fetch()
356 sv_mode = pass ? RISCV_IOMMU_CAP_SV57X4 : RISCV_IOMMU_CAP_SV57; in riscv_iommu_spa_fetch()
357 if (!(s->cap & sv_mode)) { in riscv_iommu_spa_fetch()
360 sc[pass].levels = 5; in riscv_iommu_spa_fetch()
361 sc[pass].ptidxbits = 9; in riscv_iommu_spa_fetch()
362 sc[pass].ptesize = 8; in riscv_iommu_spa_fetch()
371 gatp = PPN_PHYS(get_field(ctx->gatp, RISCV_IOMMU_ATP_PPN_FIELD)); in riscv_iommu_spa_fetch()
372 satp = PPN_PHYS(get_field(ctx->satp, RISCV_IOMMU_ATP_PPN_FIELD)); in riscv_iommu_spa_fetch()
373 addr = (en_s && en_g) ? satp : iotlb->iova; in riscv_iommu_spa_fetch()
375 pass = en_g ? G_STAGE : S_STAGE; in riscv_iommu_spa_fetch()
378 const unsigned widened = (pass && !sc[pass].step) ? 2 : 0; in riscv_iommu_spa_fetch()
379 const unsigned va_bits = widened + sc[pass].ptidxbits; in riscv_iommu_spa_fetch()
380 const unsigned va_skip = TARGET_PAGE_BITS + sc[pass].ptidxbits * in riscv_iommu_spa_fetch()
381 (sc[pass].levels - 1 - sc[pass].step); in riscv_iommu_spa_fetch()
382 const unsigned idx = (addr >> va_skip) & ((1 << va_bits) - 1); in riscv_iommu_spa_fetch()
383 const dma_addr_t pte_addr = base + idx * sc[pass].ptesize; in riscv_iommu_spa_fetch()
385 ctx->tc & (pass ? RISCV_IOMMU_DC_TC_GADE : RISCV_IOMMU_DC_TC_SADE); in riscv_iommu_spa_fetch()
388 if (!sc[pass].step) { in riscv_iommu_spa_fetch()
390 const uint64_t va_mask = (1ULL << va_len) - 1; in riscv_iommu_spa_fetch()
392 if (pass == S_STAGE && va_len > 32) { in riscv_iommu_spa_fetch()
395 mask = (1L << (TARGET_LONG_BITS - (va_len - 1))) - 1; in riscv_iommu_spa_fetch()
396 masked_msbs = (addr >> (va_len - 1)) & mask; in riscv_iommu_spa_fetch()
399 return (iotlb->perm & IOMMU_WO) ? in riscv_iommu_spa_fetch()
405 return (iotlb->perm & IOMMU_WO) ? in riscv_iommu_spa_fetch()
413 if (pass == S_STAGE) { in riscv_iommu_spa_fetch()
420 if (sc[pass].ptesize == 4) { in riscv_iommu_spa_fetch()
422 ret = ldl_le_dma(s->target_as, pte_addr, &pte32, in riscv_iommu_spa_fetch()
426 ret = ldq_le_dma(s->target_as, pte_addr, &pte, in riscv_iommu_spa_fetch()
430 return (iotlb->perm & IOMMU_WO) ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT in riscv_iommu_spa_fetch()
434 sc[pass].step++; in riscv_iommu_spa_fetch()
445 } else if (ppn & ((1ULL << (va_skip - TARGET_PAGE_BITS)) - 1)) { in riscv_iommu_spa_fetch()
447 } else if ((iotlb->perm & IOMMU_RO) && !(pte & PTE_R)) { in riscv_iommu_spa_fetch()
449 } else if ((iotlb->perm & IOMMU_WO) && !(pte & PTE_W)) { in riscv_iommu_spa_fetch()
451 } else if ((iotlb->perm & IOMMU_RO) && !ade && !(pte & PTE_A)) { in riscv_iommu_spa_fetch()
453 } else if ((iotlb->perm & IOMMU_WO) && !ade && !(pte & PTE_D)) { in riscv_iommu_spa_fetch()
457 sc[pass].step = sc[pass].levels; in riscv_iommu_spa_fetch()
458 base = PPN_PHYS(ppn) | (addr & ((1ULL << va_skip) - 1)); in riscv_iommu_spa_fetch()
460 iotlb->addr_mask &= (1ULL << va_skip) - 1; in riscv_iommu_spa_fetch()
461 /* Continue with S-Stage translation? */ in riscv_iommu_spa_fetch()
462 if (pass && sc[0].step != sc[0].levels) { in riscv_iommu_spa_fetch()
463 pass = S_STAGE; in riscv_iommu_spa_fetch()
464 addr = iotlb->iova; in riscv_iommu_spa_fetch()
468 iotlb->translated_addr = base; in riscv_iommu_spa_fetch()
469 iotlb->perm = (pte & PTE_W) ? ((pte & PTE_R) ? IOMMU_RW : IOMMU_WO) in riscv_iommu_spa_fetch()
473 if (pass == S_STAGE && (iotlb->perm & IOMMU_WO) && in riscv_iommu_spa_fetch()
476 iotlb->target_as = &s->trap_as; in riscv_iommu_spa_fetch()
477 iotlb->addr_mask = ~TARGET_PAGE_MASK; in riscv_iommu_spa_fetch()
481 /* Continue with G-Stage translation? */ in riscv_iommu_spa_fetch()
482 if (!pass && en_g) { in riscv_iommu_spa_fetch()
483 pass = G_STAGE; in riscv_iommu_spa_fetch()
486 sc[pass].step = 0; in riscv_iommu_spa_fetch()
493 if (sc[pass].step == sc[pass].levels) { in riscv_iommu_spa_fetch()
497 /* Continue with G-Stage translation? */ in riscv_iommu_spa_fetch()
498 if (!pass && en_g) { in riscv_iommu_spa_fetch()
499 pass = G_STAGE; in riscv_iommu_spa_fetch()
502 sc[pass].step = 0; in riscv_iommu_spa_fetch()
504 } while (1); in riscv_iommu_spa_fetch()
506 return (iotlb->perm & IOMMU_WO) ? in riscv_iommu_spa_fetch()
507 (pass ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS : in riscv_iommu_spa_fetch()
509 (pass ? RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS : in riscv_iommu_spa_fetch()
521 if (ctx->tc & RISCV_IOMMU_DC_TC_DTF) { in riscv_iommu_report_fault()
539 ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_DID, ctx->devid); in riscv_iommu_report_fault()
543 ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_PID, ctx->process_id); in riscv_iommu_report_fault()
566 intn = riscv_iommu_pext_u64(PPN_DOWN(gpa), ctx->msi_addr_mask); in riscv_iommu_msi_write()
575 addr = PPN_PHYS(get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_PPN)); in riscv_iommu_msi_write()
577 res = dma_memory_read(s->target_as, addr, &pte, sizeof(pte), in riscv_iommu_msi_write()
589 le64_to_cpus(&pte[1]); in riscv_iommu_msi_write()
593 * The spec mentions that: "If msipte.C == 1, then further in riscv_iommu_msi_write()
605 /* MSI Pass-through mode */ in riscv_iommu_msi_write()
608 trace_riscv_iommu_msi(s->parent_obj.id, PCI_BUS_NUM(ctx->devid), in riscv_iommu_msi_write()
609 PCI_SLOT(ctx->devid), PCI_FUNC(ctx->devid), in riscv_iommu_msi_write()
612 res = dma_memory_write(s->target_as, addr, &data, size, attrs); in riscv_iommu_msi_write()
630 * for an IMSIC interrupt file (2047) or destination address is not 32-bit in riscv_iommu_msi_write()
645 trace_riscv_iommu_msi(s->parent_obj.id, PCI_BUS_NUM(ctx->devid), in riscv_iommu_msi_write()
646 PCI_SLOT(ctx->devid), PCI_FUNC(ctx->devid), in riscv_iommu_msi_write()
650 data = 1ULL << (data & 0x03f); in riscv_iommu_msi_write()
651 res = dma_memory_read(s->target_as, addr, &intn, sizeof(intn), attrs); in riscv_iommu_msi_write()
658 res = dma_memory_write(s->target_as, addr, &intn, sizeof(intn), attrs); in riscv_iommu_msi_write()
666 res = dma_memory_read(s->target_as, addr, &intn, sizeof(intn), attrs); in riscv_iommu_msi_write()
678 addr = PPN_PHYS(get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NPPN)); in riscv_iommu_msi_write()
679 n190 = get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NID) | in riscv_iommu_msi_write()
680 (get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NID_MSB) << 10); in riscv_iommu_msi_write()
682 res = dma_memory_write(s->target_as, addr, &n190, sizeof(n190), attrs); in riscv_iommu_msi_write()
688 trace_riscv_iommu_mrif_notification(s->parent_obj.id, n190, addr); in riscv_iommu_msi_write()
694 !!ctx->process_id, 0, 0); in riscv_iommu_msi_write()
700 * riscv-iommu spec section "Device-context configuration
709 if (!(s->cap & RISCV_IOMMU_CAP_ATS) && in riscv_iommu_validate_device_ctx()
710 (ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS || in riscv_iommu_validate_device_ctx()
711 ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI || in riscv_iommu_validate_device_ctx()
712 ctx->tc & RISCV_IOMMU_DC_TC_PRPR)) { in riscv_iommu_validate_device_ctx()
716 if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS) && in riscv_iommu_validate_device_ctx()
717 (ctx->tc & RISCV_IOMMU_DC_TC_T2GPA || in riscv_iommu_validate_device_ctx()
718 ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI)) { in riscv_iommu_validate_device_ctx()
722 if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI) && in riscv_iommu_validate_device_ctx()
723 ctx->tc & RISCV_IOMMU_DC_TC_PRPR) { in riscv_iommu_validate_device_ctx()
727 if (!(s->cap & RISCV_IOMMU_CAP_T2GPA) && in riscv_iommu_validate_device_ctx()
728 ctx->tc & RISCV_IOMMU_DC_TC_T2GPA) { in riscv_iommu_validate_device_ctx()
732 if (s->cap & RISCV_IOMMU_CAP_MSI_FLAT) { in riscv_iommu_validate_device_ctx()
733 msi_mode = get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_MODE); in riscv_iommu_validate_device_ctx()
741 gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD); in riscv_iommu_validate_device_ctx()
742 if (ctx->tc & RISCV_IOMMU_DC_TC_T2GPA && in riscv_iommu_validate_device_ctx()
747 fsc_mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE); in riscv_iommu_validate_device_ctx()
749 if (ctx->tc & RISCV_IOMMU_DC_TC_PDTV) { in riscv_iommu_validate_device_ctx()
752 if (!(s->cap & RISCV_IOMMU_CAP_PD8)) { in riscv_iommu_validate_device_ctx()
757 if (!(s->cap & RISCV_IOMMU_CAP_PD17)) { in riscv_iommu_validate_device_ctx()
762 if (!(s->cap & RISCV_IOMMU_CAP_PD20)) { in riscv_iommu_validate_device_ctx()
769 if (ctx->tc & RISCV_IOMMU_DC_TC_DPE) { in riscv_iommu_validate_device_ctx()
773 if (ctx->tc & RISCV_IOMMU_DC_TC_SXL) { in riscv_iommu_validate_device_ctx()
775 !(s->cap & RISCV_IOMMU_CAP_SV32)) { in riscv_iommu_validate_device_ctx()
781 if (!(s->cap & RISCV_IOMMU_CAP_SV39)) { in riscv_iommu_validate_device_ctx()
786 if (!(s->cap & RISCV_IOMMU_CAP_SV48)) { in riscv_iommu_validate_device_ctx()
791 if (!(s->cap & RISCV_IOMMU_CAP_SV57)) { in riscv_iommu_validate_device_ctx()
801 * always zero (little-endian accesses). Thus TC_SBE must in riscv_iommu_validate_device_ctx()
804 if (ctx->tc & RISCV_IOMMU_DC_TC_SBE) { in riscv_iommu_validate_device_ctx()
813 * "Process-context configuration checks".
820 if (get_field(ctx->ta, RISCV_IOMMU_PC_TA_RESERVED)) { in riscv_iommu_validate_process_ctx()
824 if (get_field(ctx->satp, RISCV_IOMMU_PC_FSC_RESERVED)) { in riscv_iommu_validate_process_ctx()
828 mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE); in riscv_iommu_validate_process_ctx()
840 if (ctx->tc & RISCV_IOMMU_DC_TC_SXL) { in riscv_iommu_validate_process_ctx()
842 !(s->cap & RISCV_IOMMU_CAP_SV32)) { in riscv_iommu_validate_process_ctx()
848 if (!(s->cap & RISCV_IOMMU_CAP_SV39)) { in riscv_iommu_validate_process_ctx()
853 if (!(s->cap & RISCV_IOMMU_CAP_SV48)) { in riscv_iommu_validate_process_ctx()
858 if (!(s->cap & RISCV_IOMMU_CAP_SV57)) { in riscv_iommu_validate_process_ctx()
869 * RISC-V IOMMU Device Context Loopkup - Device Directory Tree Walk
877 const uint64_t ddtp = s->ddtp; in riscv_iommu_ctx_fetch()
881 /* Device Context format: 0: extended (64 bytes) | 1: base (32 bytes) */ in riscv_iommu_ctx_fetch()
882 const int dc_fmt = !s->enable_msi; in riscv_iommu_ctx_fetch()
892 /* mock up pass-through translation context */ in riscv_iommu_ctx_fetch()
893 ctx->gatp = set_field(0, RISCV_IOMMU_ATP_MODE_FIELD, in riscv_iommu_ctx_fetch()
895 ctx->satp = set_field(0, RISCV_IOMMU_ATP_MODE_FIELD, in riscv_iommu_ctx_fetch()
898 ctx->tc = RISCV_IOMMU_DC_TC_V; in riscv_iommu_ctx_fetch()
899 if (s->enable_ats) { in riscv_iommu_ctx_fetch()
900 ctx->tc |= RISCV_IOMMU_DC_TC_EN_ATS; in riscv_iommu_ctx_fetch()
903 ctx->ta = 0; in riscv_iommu_ctx_fetch()
904 ctx->msiptp = 0; in riscv_iommu_ctx_fetch()
912 depth = 1; in riscv_iommu_ctx_fetch()
926 * - if extended device-context format is used: in riscv_iommu_ctx_fetch()
927 * 1LVL: 6, 2LVL: 15, 3LVL: 24 in riscv_iommu_ctx_fetch()
928 * - if base device-context format is used: in riscv_iommu_ctx_fetch()
929 * 1LVL: 7, 2LVL: 16, 3LVL: 24 in riscv_iommu_ctx_fetch()
931 if (ctx->devid >= (1 << (depth * 9 + 6 + (dc_fmt && depth != 2)))) { in riscv_iommu_ctx_fetch()
936 for (; depth-- > 0; ) { in riscv_iommu_ctx_fetch()
942 * - if extended device-context format is used: in riscv_iommu_ctx_fetch()
944 * - if base device-context format is used: in riscv_iommu_ctx_fetch()
948 addr |= ((ctx->devid >> split) << 3) & ~TARGET_PAGE_MASK; in riscv_iommu_ctx_fetch()
949 if (dma_memory_read(s->target_as, addr, &de, sizeof(de), in riscv_iommu_ctx_fetch()
968 addr |= (ctx->devid * dc_len) & ~TARGET_PAGE_MASK; in riscv_iommu_ctx_fetch()
971 if (dma_memory_read(s->target_as, addr, &dc, dc_len, in riscv_iommu_ctx_fetch()
977 ctx->tc = le64_to_cpu(dc.tc); in riscv_iommu_ctx_fetch()
978 ctx->gatp = le64_to_cpu(dc.iohgatp); in riscv_iommu_ctx_fetch()
979 ctx->satp = le64_to_cpu(dc.fsc); in riscv_iommu_ctx_fetch()
980 ctx->ta = le64_to_cpu(dc.ta); in riscv_iommu_ctx_fetch()
981 ctx->msiptp = le64_to_cpu(dc.msiptp); in riscv_iommu_ctx_fetch()
982 ctx->msi_addr_mask = le64_to_cpu(dc.msi_addr_mask); in riscv_iommu_ctx_fetch()
983 ctx->msi_addr_pattern = le64_to_cpu(dc.msi_addr_pattern); in riscv_iommu_ctx_fetch()
985 if (!(ctx->tc & RISCV_IOMMU_DC_TC_V)) { in riscv_iommu_ctx_fetch()
994 mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE); in riscv_iommu_ctx_fetch()
995 addr = PPN_PHYS(get_field(ctx->satp, RISCV_IOMMU_DC_FSC_PPN)); in riscv_iommu_ctx_fetch()
997 if (!(ctx->tc & RISCV_IOMMU_DC_TC_PDTV)) { in riscv_iommu_ctx_fetch()
998 if (ctx->process_id != RISCV_IOMMU_NOPROCID) { in riscv_iommu_ctx_fetch()
1009 if (ctx->process_id == RISCV_IOMMU_NOPROCID) { in riscv_iommu_ctx_fetch()
1010 if (!(ctx->tc & RISCV_IOMMU_DC_TC_DPE)) { in riscv_iommu_ctx_fetch()
1012 ctx->satp = 0ULL; in riscv_iommu_ctx_fetch()
1016 ctx->process_id = 0; in riscv_iommu_ctx_fetch()
1021 /* No S-Stage translation, done. */ in riscv_iommu_ctx_fetch()
1031 for (depth = mode - RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8; depth-- > 0; ) { in riscv_iommu_ctx_fetch()
1036 * level. See IOMMU Specification, 2.2. Process-Directory-Table. in riscv_iommu_ctx_fetch()
1039 addr |= ((ctx->process_id >> split) << 3) & ~TARGET_PAGE_MASK; in riscv_iommu_ctx_fetch()
1040 if (dma_memory_read(s->target_as, addr, &de, sizeof(de), in riscv_iommu_ctx_fetch()
1054 addr |= (ctx->process_id << 4) & ~TARGET_PAGE_MASK; in riscv_iommu_ctx_fetch()
1055 if (dma_memory_read(s->target_as, addr, &dc.ta, sizeof(uint64_t) * 2, in riscv_iommu_ctx_fetch()
1061 ctx->ta = le64_to_cpu(dc.ta); in riscv_iommu_ctx_fetch()
1062 ctx->satp = le64_to_cpu(dc.fsc); in riscv_iommu_ctx_fetch()
1064 if (!(ctx->ta & RISCV_IOMMU_PC_TA_V)) { in riscv_iommu_ctx_fetch()
1080 return c1->devid == c2->devid && in riscv_iommu_ctx_equal()
1081 c1->process_id == c2->process_id; in riscv_iommu_ctx_equal()
1089 * assuming 24-bit wide devid. in riscv_iommu_ctx_hash()
1091 return (guint)(ctx->devid) + ((guint)(ctx->process_id) << 24); in riscv_iommu_ctx_hash()
1099 if (ctx->tc & RISCV_IOMMU_DC_TC_V && in riscv_iommu_ctx_inval_devid_procid()
1100 ctx->devid == arg->devid && in riscv_iommu_ctx_inval_devid_procid()
1101 ctx->process_id == arg->process_id) { in riscv_iommu_ctx_inval_devid_procid()
1102 ctx->tc &= ~RISCV_IOMMU_DC_TC_V; in riscv_iommu_ctx_inval_devid_procid()
1111 if (ctx->tc & RISCV_IOMMU_DC_TC_V && in riscv_iommu_ctx_inval_devid()
1112 ctx->devid == arg->devid) { in riscv_iommu_ctx_inval_devid()
1113 ctx->tc &= ~RISCV_IOMMU_DC_TC_V; in riscv_iommu_ctx_inval_devid()
1121 if (ctx->tc & RISCV_IOMMU_DC_TC_V) { in riscv_iommu_ctx_inval_all()
1122 ctx->tc &= ~RISCV_IOMMU_DC_TC_V; in riscv_iommu_ctx_inval_all()
1134 ctx_cache = g_hash_table_ref(s->ctx_cache); in riscv_iommu_ctx_inval()
1151 ctx_cache = g_hash_table_ref(s->ctx_cache); in riscv_iommu_ctx()
1154 if (ctx && (ctx->tc & RISCV_IOMMU_DC_TC_V)) { in riscv_iommu_ctx()
1159 ctx = g_new0(RISCVIOMMUContext, 1); in riscv_iommu_ctx()
1160 ctx->devid = devid; in riscv_iommu_ctx()
1161 ctx->process_id = process_id; in riscv_iommu_ctx()
1171 g_hash_table_unref(qatomic_xchg(&s->ctx_cache, ctx_cache)); in riscv_iommu_ctx()
1201 devid |= s->bus << 8; in riscv_iommu_space()
1203 QLIST_FOREACH(as, &s->spaces, list) { in riscv_iommu_space()
1204 if (as->devid == devid) { in riscv_iommu_space()
1211 as = g_new0(RISCVIOMMUSpace, 1); in riscv_iommu_space()
1213 as->iommu = s; in riscv_iommu_space()
1214 as->devid = devid; in riscv_iommu_space()
1216 snprintf(name, sizeof(name), "riscv-iommu-%04x:%02x.%d-iova", in riscv_iommu_space()
1217 PCI_BUS_NUM(as->devid), PCI_SLOT(as->devid), PCI_FUNC(as->devid)); in riscv_iommu_space()
1220 memory_region_init_iommu(&as->iova_mr, sizeof(as->iova_mr), in riscv_iommu_space()
1223 address_space_init(&as->iova_as, MEMORY_REGION(&as->iova_mr), name); in riscv_iommu_space()
1225 QLIST_INSERT_HEAD(&s->spaces, as, list); in riscv_iommu_space()
1227 trace_riscv_iommu_new(s->parent_obj.id, PCI_BUS_NUM(as->devid), in riscv_iommu_space()
1228 PCI_SLOT(as->devid), PCI_FUNC(as->devid)); in riscv_iommu_space()
1230 return &as->iova_as; in riscv_iommu_space()
1238 return t1->gscid == t2->gscid && t1->pscid == t2->pscid && in riscv_iommu_iot_equal()
1239 t1->iova == t2->iova && t1->tag == t2->tag; in riscv_iommu_iot_equal()
1245 return (guint)t->iova; in riscv_iommu_iot_hash()
1249 /* GV: 0 AV: 0 GVMA: 1 */
1255 if (iot->tag == arg->tag) { in riscv_iommu_iot_inval_all()
1256 iot->perm = IOMMU_NONE; in riscv_iommu_iot_inval_all()
1260 /* GV: 0 AV: 0 PSCV: 1 GVMA: 0 */
1266 if (iot->tag == arg->tag && in riscv_iommu_iot_inval_pscid()
1267 iot->pscid == arg->pscid) { in riscv_iommu_iot_inval_pscid()
1268 iot->perm = IOMMU_NONE; in riscv_iommu_iot_inval_pscid()
1272 /* GV: 0 AV: 1 PSCV: 0 GVMA: 0 */
1278 if (iot->tag == arg->tag && in riscv_iommu_iot_inval_iova()
1279 iot->iova == arg->iova) { in riscv_iommu_iot_inval_iova()
1280 iot->perm = IOMMU_NONE; in riscv_iommu_iot_inval_iova()
1284 /* GV: 0 AV: 1 PSCV: 1 GVMA: 0 */
1290 if (iot->tag == arg->tag && in riscv_iommu_iot_inval_pscid_iova()
1291 iot->pscid == arg->pscid && in riscv_iommu_iot_inval_pscid_iova()
1292 iot->iova == arg->iova) { in riscv_iommu_iot_inval_pscid_iova()
1293 iot->perm = IOMMU_NONE; in riscv_iommu_iot_inval_pscid_iova()
1297 /* GV: 1 AV: 0 PSCV: 0 GVMA: 0 */
1298 /* GV: 1 AV: 0 GVMA: 1 */
1304 if (iot->tag == arg->tag && in riscv_iommu_iot_inval_gscid()
1305 iot->gscid == arg->gscid) { in riscv_iommu_iot_inval_gscid()
1306 iot->perm = IOMMU_NONE; in riscv_iommu_iot_inval_gscid()
1310 /* GV: 1 AV: 0 PSCV: 1 GVMA: 0 */
1316 if (iot->tag == arg->tag && in riscv_iommu_iot_inval_gscid_pscid()
1317 iot->gscid == arg->gscid && in riscv_iommu_iot_inval_gscid_pscid()
1318 iot->pscid == arg->pscid) { in riscv_iommu_iot_inval_gscid_pscid()
1319 iot->perm = IOMMU_NONE; in riscv_iommu_iot_inval_gscid_pscid()
1323 /* GV: 1 AV: 1 PSCV: 0 GVMA: 0 */
1324 /* GV: 1 AV: 1 GVMA: 1 */
1330 if (iot->tag == arg->tag && in riscv_iommu_iot_inval_gscid_iova()
1331 iot->gscid == arg->gscid && in riscv_iommu_iot_inval_gscid_iova()
1332 iot->iova == arg->iova) { in riscv_iommu_iot_inval_gscid_iova()
1333 iot->perm = IOMMU_NONE; in riscv_iommu_iot_inval_gscid_iova()
1337 /* GV: 1 AV: 1 PSCV: 1 GVMA: 0 */
1343 if (iot->tag == arg->tag && in riscv_iommu_iot_inval_gscid_pscid_iova()
1344 iot->gscid == arg->gscid && in riscv_iommu_iot_inval_gscid_pscid_iova()
1345 iot->pscid == arg->pscid && in riscv_iommu_iot_inval_gscid_pscid_iova()
1346 iot->iova == arg->iova) { in riscv_iommu_iot_inval_gscid_pscid_iova()
1347 iot->perm = IOMMU_NONE; in riscv_iommu_iot_inval_gscid_pscid_iova()
1351 /* caller should keep ref-count for iot_cache object */
1357 .gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID), in riscv_iommu_iot_lookup()
1358 .pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID), in riscv_iommu_iot_lookup()
1364 /* caller should keep ref-count for iot_cache object */
1368 if (!s->iot_limit) { in riscv_iommu_iot_update()
1372 if (g_hash_table_size(s->iot_cache) >= s->iot_limit) { in riscv_iommu_iot_update()
1376 g_hash_table_unref(qatomic_xchg(&s->iot_cache, iot_cache)); in riscv_iommu_iot_update()
1392 iot_cache = g_hash_table_ref(s->iot_cache); in riscv_iommu_iot_inval()
1399 uint64_t satp = get_field(ctx->satp, RISCV_IOMMU_ATP_MODE_FIELD); in riscv_iommu_get_transtag()
1400 uint64_t gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD); in riscv_iommu_get_transtag()
1424 iot_cache = g_hash_table_ref(s->iot_cache); in riscv_iommu_translate()
1427 * enable automatic page-request generation for ATS queries. in riscv_iommu_translate()
1429 enable_pri = (iotlb->perm == IOMMU_NONE) && (ctx->tc & BIT_ULL(32)); in riscv_iommu_translate()
1430 enable_pid = (ctx->tc & RISCV_IOMMU_DC_TC_PDTV); in riscv_iommu_translate()
1433 if (iotlb->perm == IOMMU_NONE) { in riscv_iommu_translate()
1436 if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS)) { in riscv_iommu_translate()
1443 iot = riscv_iommu_iot_lookup(ctx, iot_cache, iotlb->iova, transtag); in riscv_iommu_translate()
1444 perm = iot ? iot->perm : IOMMU_NONE; in riscv_iommu_translate()
1446 iotlb->translated_addr = PPN_PHYS(iot->phys); in riscv_iommu_translate()
1447 iotlb->addr_mask = ~TARGET_PAGE_MASK; in riscv_iommu_translate()
1448 iotlb->perm = perm; in riscv_iommu_translate()
1458 if (!fault && iotlb->target_as == &s->trap_as) { in riscv_iommu_translate()
1464 * We made an implementation choice to not cache identity-mapped in riscv_iommu_translate()
1469 if (!fault && iotlb->translated_addr != iotlb->iova && enable_cache) { in riscv_iommu_translate()
1470 iot = g_new0(RISCVIOMMUEntry, 1); in riscv_iommu_translate()
1471 iot->iova = PPN_DOWN(iotlb->iova); in riscv_iommu_translate()
1472 iot->phys = PPN_DOWN(iotlb->translated_addr); in riscv_iommu_translate()
1473 iot->gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID); in riscv_iommu_translate()
1474 iot->pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID); in riscv_iommu_translate()
1475 iot->perm = iotlb->perm; in riscv_iommu_translate()
1476 iot->tag = transtag; in riscv_iommu_translate()
1487 RISCV_IOMMU_PREQ_HDR_PID, ctx->process_id); in riscv_iommu_translate()
1489 pr.hdr = set_field(pr.hdr, RISCV_IOMMU_PREQ_HDR_DID, ctx->devid); in riscv_iommu_translate()
1490 pr.payload = (iotlb->iova & TARGET_PAGE_MASK) | in riscv_iommu_translate()
1499 if (iotlb->perm & IOMMU_RW) { in riscv_iommu_translate()
1501 } else if (iotlb->perm & IOMMU_RO) { in riscv_iommu_translate()
1506 iotlb->iova, iotlb->translated_addr); in riscv_iommu_translate()
1525 return dma_memory_write(s->target_as, addr, &data, sizeof(data), in riscv_iommu_iofence()
1539 const bool pv = cmd->dword0 & RISCV_IOMMU_CMD_ATS_PV; in riscv_iommu_ats()
1541 if (cmd->dword0 & RISCV_IOMMU_CMD_ATS_DSV) { in riscv_iommu_ats()
1543 devid = get_field(cmd->dword0, in riscv_iommu_ats()
1546 devid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_RID); in riscv_iommu_ats()
1549 pid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_PID); in riscv_iommu_ats()
1551 QLIST_FOREACH(as, &s->spaces, list) { in riscv_iommu_ats()
1552 if (as->devid == devid) { in riscv_iommu_ats()
1557 if (!as || !as->notifier) { in riscv_iommu_ats()
1563 event.entry.target_as = s->target_as; in riscv_iommu_ats()
1565 IOMMU_NOTIFIER_FOREACH(n, &as->iova_mr) { in riscv_iommu_ats()
1566 if (!pv || n->iommu_idx == pid) { in riscv_iommu_ats()
1567 event.entry.iova = n->start; in riscv_iommu_ats()
1568 event.entry.addr_mask = n->end - n->start; in riscv_iommu_ats()
1569 trace_fn(as->iova_mr.parent_obj.name); in riscv_iommu_ats()
1585 unsigned resp_code = get_field(cmd->dword1, in riscv_iommu_ats_prgr()
1596 uint64_t old_ddtp = s->ddtp; in riscv_iommu_process_ddtp()
1604 * {OFF, BARE} -> {OFF, BARE, 1LVL, 2LVL, 3LVL} in riscv_iommu_process_ddtp()
1605 * {1LVL, 2LVL, 3LVL} -> {OFF, BARE} in riscv_iommu_process_ddtp()
1625 s->ddtp = new_ddtp; in riscv_iommu_process_ddtp()
1643 tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQT) & s->cq_mask; in riscv_iommu_process_cq_tail()
1644 head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQH) & s->cq_mask; in riscv_iommu_process_cq_tail()
1653 addr = s->cq_addr + head * sizeof(cmd); in riscv_iommu_process_cq_tail()
1654 res = dma_memory_read(s->target_as, addr, &cmd, sizeof(cmd), in riscv_iommu_process_cq_tail()
1663 trace_riscv_iommu_cmd(s->parent_obj.id, cmd.dword0, cmd.dword1); in riscv_iommu_process_cq_tail()
1695 /* illegal command arguments IOTINVAL.GVMA & PSCV == 1 */ in riscv_iommu_process_cq_tail()
1780 if (!s->enable_ats) { in riscv_iommu_process_cq_tail()
1789 if (!s->enable_ats) { in riscv_iommu_process_cq_tail()
1805 head = (head + 1) & s->cq_mask; in riscv_iommu_process_cq_tail()
1826 s->cq_mask = (2ULL << get_field(base, RISCV_IOMMU_CQB_LOG2SZ)) - 1; in riscv_iommu_process_cq_control()
1827 s->cq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_CQB_PPN)); in riscv_iommu_process_cq_control()
1828 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQT], ~s->cq_mask); in riscv_iommu_process_cq_control()
1829 stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_CQH], 0); in riscv_iommu_process_cq_control()
1830 stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_CQT], 0); in riscv_iommu_process_cq_control()
1836 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQT], ~0); in riscv_iommu_process_cq_control()
1857 s->fq_mask = (2ULL << get_field(base, RISCV_IOMMU_FQB_LOG2SZ)) - 1; in riscv_iommu_process_fq_control()
1858 s->fq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_FQB_PPN)); in riscv_iommu_process_fq_control()
1859 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQH], ~s->fq_mask); in riscv_iommu_process_fq_control()
1860 stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_FQH], 0); in riscv_iommu_process_fq_control()
1861 stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_FQT], 0); in riscv_iommu_process_fq_control()
1866 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQH], ~0); in riscv_iommu_process_fq_control()
1887 s->pq_mask = (2ULL << get_field(base, RISCV_IOMMU_PQB_LOG2SZ)) - 1; in riscv_iommu_process_pq_control()
1888 s->pq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_PQB_PPN)); in riscv_iommu_process_pq_control()
1889 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQH], ~s->pq_mask); in riscv_iommu_process_pq_control()
1890 stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_PQH], 0); in riscv_iommu_process_pq_control()
1891 stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_PQT], 0); in riscv_iommu_process_pq_control()
1896 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQH], ~0); in riscv_iommu_process_pq_control()
1958 s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_CIV); in riscv_iommu_update_icvec()
1961 s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_FIV); in riscv_iommu_update_icvec()
1964 s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_PMIV); in riscv_iommu_update_icvec()
1967 s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_PIV); in riscv_iommu_update_icvec()
2050 * by 'reg_addr', after considering read-only/read-write/write-clear
2053 * The result is written in little-endian.
2059 uint64_t ro = ldn_le_p(&s->regs_ro[reg_addr], size); in riscv_iommu_write_reg_val()
2060 uint64_t wc = ldn_le_p(&s->regs_wc[reg_addr], size); in riscv_iommu_write_reg_val()
2061 uint64_t rw = ldn_le_p(&s->regs_rw[reg_addr], size); in riscv_iommu_write_reg_val()
2077 if ((addr & (size - 1)) != 0) { in riscv_iommu_mmio_write()
2163 riscv_iommu_write_reg_val(s, &s->regs_rw[addr], addr, size, data); in riscv_iommu_mmio_write()
2165 /* Busy flag update, MSB 4-byte register. */ in riscv_iommu_mmio_write()
2167 uint32_t rw = ldl_le_p(&s->regs_rw[regb]); in riscv_iommu_mmio_write()
2168 stl_le_p(&s->regs_rw[regb], rw | busy); in riscv_iommu_mmio_write()
2188 uint64_t val = -1; in riscv_iommu_mmio_read()
2191 if ((addr & (size - 1)) != 0) { in riscv_iommu_mmio_read()
2213 val = ldq_le_p(&s->regs_rw[addr]); in riscv_iommu_mmio_read()
2219 ptr = &s->regs_rw[addr]; in riscv_iommu_mmio_read()
2245 * Translations matching MSI pattern check are redirected to "riscv-iommu-trap"
2265 devid |= s->bus << 8; in riscv_iommu_trap_write()
2300 s->cap = set_field(s->cap, RISCV_IOMMU_CAP_IGS, mode); in riscv_iommu_set_cap_igs()
2308 s->cap = RISCV_IOMMU_CAP_DBG; in riscv_iommu_instance_init()
2311 s->cap = set_field(s->cap, RISCV_IOMMU_CAP_PAS, in riscv_iommu_instance_init()
2315 s->pid_bits = 8; /* restricted to size of MemTxAttrs.pid */ in riscv_iommu_instance_init()
2316 s->cap |= RISCV_IOMMU_CAP_PD8; in riscv_iommu_instance_init()
2319 s->regs_rw = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE); in riscv_iommu_instance_init()
2320 s->regs_ro = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE); in riscv_iommu_instance_init()
2321 s->regs_wc = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE); in riscv_iommu_instance_init()
2323 /* Mark all registers read-only */ in riscv_iommu_instance_init()
2324 memset(s->regs_ro, 0xff, RISCV_IOMMU_REG_SIZE); in riscv_iommu_instance_init()
2327 s->ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash, in riscv_iommu_instance_init()
2331 s->iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash, in riscv_iommu_instance_init()
2335 s->iommus.le_next = NULL; in riscv_iommu_instance_init()
2336 s->iommus.le_prev = NULL; in riscv_iommu_instance_init()
2337 QLIST_INIT(&s->spaces); in riscv_iommu_instance_init()
2344 s->cap |= s->version & RISCV_IOMMU_CAP_VERSION; in riscv_iommu_realize()
2345 if (s->enable_msi) { in riscv_iommu_realize()
2346 s->cap |= RISCV_IOMMU_CAP_MSI_FLAT | RISCV_IOMMU_CAP_MSI_MRIF; in riscv_iommu_realize()
2348 if (s->enable_ats) { in riscv_iommu_realize()
2349 s->cap |= RISCV_IOMMU_CAP_ATS; in riscv_iommu_realize()
2351 if (s->enable_s_stage) { in riscv_iommu_realize()
2352 s->cap |= RISCV_IOMMU_CAP_SV32 | RISCV_IOMMU_CAP_SV39 | in riscv_iommu_realize()
2355 if (s->enable_g_stage) { in riscv_iommu_realize()
2356 s->cap |= RISCV_IOMMU_CAP_SV32X4 | RISCV_IOMMU_CAP_SV39X4 | in riscv_iommu_realize()
2360 if (s->hpm_cntrs > 0) { in riscv_iommu_realize()
2362 if (s->hpm_cntrs > RISCV_IOMMU_IOCOUNT_NUM) { in riscv_iommu_realize()
2363 s->hpm_cntrs = RISCV_IOMMU_IOCOUNT_NUM; in riscv_iommu_realize()
2366 s->cap |= RISCV_IOMMU_CAP_HPM; in riscv_iommu_realize()
2369 /* Out-of-reset translation mode: OFF (DMA disabled) BARE (passthrough) */ in riscv_iommu_realize()
2370 s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, s->enable_off ? in riscv_iommu_realize()
2378 memory_region_init_io(&s->regs_mr, OBJECT(dev), &riscv_iommu_mmio_ops, s, in riscv_iommu_realize()
2379 "riscv-iommu-regs", RISCV_IOMMU_REG_SIZE); in riscv_iommu_realize()
2381 /* Set power-on register state */ in riscv_iommu_realize()
2382 stq_le_p(&s->regs_rw[RISCV_IOMMU_REG_CAP], s->cap); in riscv_iommu_realize()
2383 stq_le_p(&s->regs_rw[RISCV_IOMMU_REG_FCTL], 0); in riscv_iommu_realize()
2384 stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_FCTL], in riscv_iommu_realize()
2386 stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_DDTP], in riscv_iommu_realize()
2388 stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQB], in riscv_iommu_realize()
2390 stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQB], in riscv_iommu_realize()
2392 stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQB], in riscv_iommu_realize()
2394 stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_CQCSR], RISCV_IOMMU_CQCSR_CQMF | in riscv_iommu_realize()
2396 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQCSR], RISCV_IOMMU_CQCSR_CQON | in riscv_iommu_realize()
2398 stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_FQCSR], RISCV_IOMMU_FQCSR_FQMF | in riscv_iommu_realize()
2400 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQCSR], RISCV_IOMMU_FQCSR_FQON | in riscv_iommu_realize()
2402 stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_PQCSR], RISCV_IOMMU_PQCSR_PQMF | in riscv_iommu_realize()
2404 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQCSR], RISCV_IOMMU_PQCSR_PQON | in riscv_iommu_realize()
2406 stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_IPSR], ~0); in riscv_iommu_realize()
2407 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_ICVEC], 0); in riscv_iommu_realize()
2408 stq_le_p(&s->regs_rw[RISCV_IOMMU_REG_DDTP], s->ddtp); in riscv_iommu_realize()
2410 if (s->cap & RISCV_IOMMU_CAP_DBG) { in riscv_iommu_realize()
2411 stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_TR_REQ_IOVA], 0); in riscv_iommu_realize()
2412 stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_TR_REQ_CTL], in riscv_iommu_realize()
2417 if (s->cap & RISCV_IOMMU_CAP_HPM) { in riscv_iommu_realize()
2418 /* +1 for cycle counter bit. */ in riscv_iommu_realize()
2419 stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_IOCOUNTINH], in riscv_iommu_realize()
2420 ~((2 << s->hpm_cntrs) - 1)); in riscv_iommu_realize()
2421 stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_IOHPMCYCLES], 0); in riscv_iommu_realize()
2422 memset(&s->regs_ro[RISCV_IOMMU_REG_IOHPMCTR_BASE], in riscv_iommu_realize()
2423 0x00, s->hpm_cntrs * 8); in riscv_iommu_realize()
2424 memset(&s->regs_ro[RISCV_IOMMU_REG_IOHPMEVT_BASE], in riscv_iommu_realize()
2425 0x00, s->hpm_cntrs * 8); in riscv_iommu_realize()
2429 if (s->target_mr) { in riscv_iommu_realize()
2430 s->target_as = g_new0(AddressSpace, 1); in riscv_iommu_realize()
2431 address_space_init(s->target_as, s->target_mr, in riscv_iommu_realize()
2432 "riscv-iommu-downstream"); in riscv_iommu_realize()
2435 s->target_as = &address_space_memory; in riscv_iommu_realize()
2439 memory_region_init_io(&s->trap_mr, OBJECT(dev), &riscv_iommu_trap_ops, s, in riscv_iommu_realize()
2440 "riscv-iommu-trap", ~0ULL); in riscv_iommu_realize()
2441 address_space_init(&s->trap_as, &s->trap_mr, "riscv-iommu-trap-as"); in riscv_iommu_realize()
2443 if (s->cap & RISCV_IOMMU_CAP_HPM) { in riscv_iommu_realize()
2444 s->hpm_timer = in riscv_iommu_realize()
2446 s->hpm_event_ctr_map = g_hash_table_new(g_direct_hash, g_direct_equal); in riscv_iommu_realize()
2454 g_hash_table_unref(s->iot_cache); in riscv_iommu_unrealize()
2455 g_hash_table_unref(s->ctx_cache); in riscv_iommu_unrealize()
2457 if (s->cap & RISCV_IOMMU_CAP_HPM) { in riscv_iommu_unrealize()
2458 g_hash_table_unref(s->hpm_event_ctr_map); in riscv_iommu_unrealize()
2459 timer_free(s->hpm_timer); in riscv_iommu_unrealize()
2472 ddtp_mode = s->enable_off ? in riscv_iommu_reset()
2474 s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, ddtp_mode); in riscv_iommu_reset()
2475 riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_DDTP, s->ddtp); in riscv_iommu_reset()
2494 g_hash_table_remove_all(s->ctx_cache); in riscv_iommu_reset()
2495 g_hash_table_remove_all(s->iot_cache); in riscv_iommu_reset()
2502 DEFINE_PROP_UINT32("ioatc-limit", RISCVIOMMUState, iot_limit,
2507 DEFINE_PROP_BOOL("s-stage", RISCVIOMMUState, enable_s_stage, TRUE),
2508 DEFINE_PROP_BOOL("g-stage", RISCVIOMMUState, enable_g_stage, TRUE),
2509 DEFINE_PROP_LINK("downstream-mr", RISCVIOMMUState, target_mr,
2511 DEFINE_PROP_UINT8("hpm-counters", RISCVIOMMUState, hpm_cntrs,
2519 /* internal device for riscv-iommu-{pci/sys}, not user-creatable */ in riscv_iommu_class_init()
2520 dc->user_creatable = false; in riscv_iommu_class_init()
2521 dc->realize = riscv_iommu_realize; in riscv_iommu_class_init()
2522 dc->unrealize = riscv_iommu_unrealize; in riscv_iommu_class_init()
2541 /* RISC-V IOMMU Memory Region - Address Translation Space */
2551 .target_as = as->iommu->target_as, in riscv_iommu_memory_region_translate()
2556 ctx = riscv_iommu_ctx(as->iommu, as->devid, iommu_idx, &ref); in riscv_iommu_memory_region_translate()
2561 } else if (riscv_iommu_translate(as->iommu, ctx, &iotlb, true)) { in riscv_iommu_memory_region_translate()
2568 trace_riscv_iommu_dma(as->iommu->parent_obj.id, PCI_BUS_NUM(as->devid), in riscv_iommu_memory_region_translate()
2569 PCI_SLOT(as->devid), PCI_FUNC(as->devid), iommu_idx, in riscv_iommu_memory_region_translate()
2573 riscv_iommu_ctx_put(as->iommu, ref); in riscv_iommu_memory_region_translate()
2585 as->notifier = true; in riscv_iommu_memory_region_notify()
2586 trace_riscv_iommu_notifier_add(iommu_mr->parent_obj.name); in riscv_iommu_memory_region_notify()
2588 as->notifier = false; in riscv_iommu_memory_region_notify()
2589 trace_riscv_iommu_notifier_del(iommu_mr->parent_obj.name); in riscv_iommu_memory_region_notify()
2597 return pci_get_word(pdev->config + PCI_CLASS_DEVICE) == 0x0806; in pci_is_iommu()
2607 return s->target_as; in riscv_iommu_find_as()
2611 while (s->iommus.le_prev) { in riscv_iommu_find_as()
2612 s = *(s->iommus.le_prev); in riscv_iommu_find_as()
2618 s = s->iommus.le_next; in riscv_iommu_find_as()
2631 if (bus->iommu_ops && in riscv_iommu_pci_setup_iommu()
2632 bus->iommu_ops->get_address_space == riscv_iommu_find_as) { in riscv_iommu_pci_setup_iommu()
2634 RISCVIOMMUState *last = (RISCVIOMMUState *)bus->iommu_opaque; in riscv_iommu_pci_setup_iommu()
2636 } else if (!bus->iommu_ops && !bus->iommu_opaque) { in riscv_iommu_pci_setup_iommu()
2653 return 1 << as->iommu->pid_bits; in riscv_iommu_memory_region_index_len()
2660 imrc->translate = riscv_iommu_memory_region_translate; in riscv_iommu_memory_region_init()
2661 imrc->notify_flag_changed = riscv_iommu_memory_region_notify; in riscv_iommu_memory_region_init()
2662 imrc->attrs_to_index = riscv_iommu_memory_region_index; in riscv_iommu_memory_region_init()
2663 imrc->num_indexes = riscv_iommu_memory_region_index_len; in riscv_iommu_memory_region_init()