/*
* QEMU emulation of an RISC-V IOMMU
*
* Copyright (C) 2021-2023, Rivos Inc.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2 or later, as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include "qemu/osdep.h"
#include "qom/object.h"
#include "hw/pci/pci_bus.h"
#include "hw/pci/pci_device.h"
#include "hw/qdev-properties.h"
#include "hw/riscv/riscv_hart.h"
#include "migration/vmstate.h"
#include "qapi/error.h"
#include "qemu/timer.h"
#include "cpu_bits.h"
#include "riscv-iommu.h"
#include "riscv-iommu-bits.h"
#include "trace.h"
#define LIMIT_CACHE_CTX (1U << 7)
#define LIMIT_CACHE_IOT (1U << 20)
/* Physical page number coversions */
#define PPN_PHYS(ppn) ((ppn) << TARGET_PAGE_BITS)
#define PPN_DOWN(phy) ((phy) >> TARGET_PAGE_BITS)
typedef struct RISCVIOMMUContext RISCVIOMMUContext;
typedef struct RISCVIOMMUEntry RISCVIOMMUEntry;
/* Device assigned I/O address space */
struct RISCVIOMMUSpace {
IOMMUMemoryRegion iova_mr; /* IOVA memory region for attached device */
AddressSpace iova_as; /* IOVA address space for attached device */
RISCVIOMMUState *iommu; /* Managing IOMMU device state */
uint32_t devid; /* Requester identifier, AKA device_id */
bool notifier; /* IOMMU unmap notifier enabled */
QLIST_ENTRY(RISCVIOMMUSpace) list;
};
/* Device translation context state. */
struct RISCVIOMMUContext {
uint64_t devid:24; /* Requester Id, AKA device_id */
uint64_t process_id:20; /* Process ID. PASID for PCIe */
uint64_t tc; /* Translation Control */
uint64_t ta; /* Translation Attributes */
uint64_t satp; /* S-Stage address translation and protection */
uint64_t gatp; /* G-Stage address translation and protection */
uint64_t msi_addr_mask; /* MSI filtering - address mask */
uint64_t msi_addr_pattern; /* MSI filtering - address pattern */
uint64_t msiptp; /* MSI redirection page table pointer */
};
/* Address translation cache entry */
struct RISCVIOMMUEntry {
uint64_t iova:44; /* IOVA Page Number */
uint64_t pscid:20; /* Process Soft-Context identifier */
uint64_t phys:44; /* Physical Page Number */
uint64_t gscid:16; /* Guest Soft-Context identifier */
uint64_t perm:2; /* IOMMU_RW flags */
};
/* IOMMU index for transactions without process_id specified. */
#define RISCV_IOMMU_NOPROCID 0
static uint8_t riscv_iommu_get_icvec_vector(uint32_t icvec, uint32_t vec_type)
{
switch (vec_type) {
case RISCV_IOMMU_INTR_CQ:
return icvec & RISCV_IOMMU_ICVEC_CIV;
case RISCV_IOMMU_INTR_FQ:
return (icvec & RISCV_IOMMU_ICVEC_FIV) >> 4;
case RISCV_IOMMU_INTR_PM:
return (icvec & RISCV_IOMMU_ICVEC_PMIV) >> 8;
case RISCV_IOMMU_INTR_PQ:
return (icvec & RISCV_IOMMU_ICVEC_PIV) >> 12;
default:
g_assert_not_reached();
}
}
static void riscv_iommu_notify(RISCVIOMMUState *s, int vec_type)
{
const uint32_t fctl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FCTL);
uint32_t ipsr, icvec, vector;
if (fctl & RISCV_IOMMU_FCTL_WSI || !s->notify) {
return;
}
icvec = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_ICVEC);
ipsr = riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_IPSR, (1 << vec_type), 0);
if (!(ipsr & (1 << vec_type))) {
vector = riscv_iommu_get_icvec_vector(icvec, vec_type);
s->notify(s, vector);
trace_riscv_iommu_notify_int_vector(vec_type, vector);
}
}
static void riscv_iommu_fault(RISCVIOMMUState *s,
struct riscv_iommu_fq_record *ev)
{
uint32_t ctrl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQCSR);
uint32_t head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQH) & s->fq_mask;
uint32_t tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQT) & s->fq_mask;
uint32_t next = (tail + 1) & s->fq_mask;
uint32_t devid = get_field(ev->hdr, RISCV_IOMMU_FQ_HDR_DID);
trace_riscv_iommu_flt(s->parent_obj.id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
PCI_FUNC(devid), ev->hdr, ev->iotval);
if (!(ctrl & RISCV_IOMMU_FQCSR_FQON) ||
!!(ctrl & (RISCV_IOMMU_FQCSR_FQOF | RISCV_IOMMU_FQCSR_FQMF))) {
return;
}
if (head == next) {
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR,
RISCV_IOMMU_FQCSR_FQOF, 0);
} else {
dma_addr_t addr = s->fq_addr + tail * sizeof(*ev);
if (dma_memory_write(s->target_as, addr, ev, sizeof(*ev),
MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR,
RISCV_IOMMU_FQCSR_FQMF, 0);
} else {
riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_FQT, next);
}
}
if (ctrl & RISCV_IOMMU_FQCSR_FIE) {
riscv_iommu_notify(s, RISCV_IOMMU_INTR_FQ);
}
}
static void riscv_iommu_pri(RISCVIOMMUState *s,
struct riscv_iommu_pq_record *pr)
{
uint32_t ctrl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQCSR);
uint32_t head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQH) & s->pq_mask;
uint32_t tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQT) & s->pq_mask;
uint32_t next = (tail + 1) & s->pq_mask;
uint32_t devid = get_field(pr->hdr, RISCV_IOMMU_PREQ_HDR_DID);
trace_riscv_iommu_pri(s->parent_obj.id, PCI_BUS_NUM(devid), PCI_SLOT(devid),
PCI_FUNC(devid), pr->payload);
if (!(ctrl & RISCV_IOMMU_PQCSR_PQON) ||
!!(ctrl & (RISCV_IOMMU_PQCSR_PQOF | RISCV_IOMMU_PQCSR_PQMF))) {
return;
}
if (head == next) {
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR,
RISCV_IOMMU_PQCSR_PQOF, 0);
} else {
dma_addr_t addr = s->pq_addr + tail * sizeof(*pr);
if (dma_memory_write(s->target_as, addr, pr, sizeof(*pr),
MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR,
RISCV_IOMMU_PQCSR_PQMF, 0);
} else {
riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_PQT, next);
}
}
if (ctrl & RISCV_IOMMU_PQCSR_PIE) {
riscv_iommu_notify(s, RISCV_IOMMU_INTR_PQ);
}
}
/*
* Discards all bits from 'val' whose matching bits in the same
* positions in the mask 'ext' are zeros, and packs the remaining
* bits from 'val' contiguously at the least-significant end of the
* result, keeping the same bit order as 'val' and filling any
* other bits at the most-significant end of the result with zeros.
*
* For example, for the following 'val' and 'ext', the return 'ret'
* will be:
*
* val = a b c d e f g h
* ext = 1 0 1 0 0 1 1 0
* ret = 0 0 0 0 a c f g
*
* This function, taken from the riscv-iommu 1.0 spec, section 2.3.3
* "Process to translate addresses of MSIs", is similar to bit manip
* function PEXT (Parallel bits extract) from x86.
*/
static uint64_t riscv_iommu_pext_u64(uint64_t val, uint64_t ext)
{
uint64_t ret = 0;
uint64_t rot = 1;
while (ext) {
if (ext & 1) {
if (val & 1) {
ret |= rot;
}
rot <<= 1;
}
val >>= 1;
ext >>= 1;
}
return ret;
}
/* Check if GPA matches MSI/MRIF pattern. */
static bool riscv_iommu_msi_check(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,
dma_addr_t gpa)
{
if (!s->enable_msi) {
return false;
}
if (get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_MODE) !=
RISCV_IOMMU_DC_MSIPTP_MODE_FLAT) {
return false; /* Invalid MSI/MRIF mode */
}
if ((PPN_DOWN(gpa) ^ ctx->msi_addr_pattern) & ~ctx->msi_addr_mask) {
return false; /* GPA not in MSI range defined by AIA IMSIC rules. */
}
return true;
}
/*
* RISCV IOMMU Address Translation Lookup - Page Table Walk
*
* Note: Code is based on get_physical_address() from target/riscv/cpu_helper.c
* Both implementation can be merged into single helper function in future.
* Keeping them separate for now, as error reporting and flow specifics are
* sufficiently different for separate implementation.
*
* @s : IOMMU Device State
* @ctx : Translation context for device id and process address space id.
* @iotlb : translation data: physical address and access mode.
* @return : success or fault cause code.
*/
static int riscv_iommu_spa_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,
IOMMUTLBEntry *iotlb)
{
dma_addr_t addr, base;
uint64_t satp, gatp, pte;
bool en_s, en_g;
struct {
unsigned char step;
unsigned char levels;
unsigned char ptidxbits;
unsigned char ptesize;
} sc[2];
/* Translation stage phase */
enum {
S_STAGE = 0,
G_STAGE = 1,
} pass;
MemTxResult ret;
satp = get_field(ctx->satp, RISCV_IOMMU_ATP_MODE_FIELD);
gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
en_s = satp != RISCV_IOMMU_DC_FSC_MODE_BARE;
en_g = gatp != RISCV_IOMMU_DC_IOHGATP_MODE_BARE;
/*
* Early check for MSI address match when IOVA == GPA.
* Note that the (!en_s) condition means that the MSI
* page table may only be used when guest pages are
* mapped using the g-stage page table, whether single-
* or two-stage paging is enabled. It's unavoidable though,
* because the spec mandates that we do a first-stage
* translation before we check the MSI page table, which
* means we can't do an early MSI check unless we have
* strictly !en_s.
*/
if (!en_s && (iotlb->perm & IOMMU_WO) &&
riscv_iommu_msi_check(s, ctx, iotlb->iova)) {
iotlb->target_as = &s->trap_as;
iotlb->translated_addr = iotlb->iova;
iotlb->addr_mask = ~TARGET_PAGE_MASK;
return 0;
}
/* Exit early for pass-through mode. */
if (!(en_s || en_g)) {
iotlb->translated_addr = iotlb->iova;
iotlb->addr_mask = ~TARGET_PAGE_MASK;
/* Allow R/W in pass-through mode */
iotlb->perm = IOMMU_RW;
return 0;
}
/* S/G translation parameters. */
for (pass = 0; pass < 2; pass++) {
uint32_t sv_mode;
sc[pass].step = 0;
if (pass ? (s->fctl & RISCV_IOMMU_FCTL_GXL) :
(ctx->tc & RISCV_IOMMU_DC_TC_SXL)) {
/* 32bit mode for GXL/SXL == 1 */
switch (pass ? gatp : satp) {
case RISCV_IOMMU_DC_IOHGATP_MODE_BARE:
sc[pass].levels = 0;
sc[pass].ptidxbits = 0;
sc[pass].ptesize = 0;
break;
case RISCV_IOMMU_DC_IOHGATP_MODE_SV32X4:
sv_mode = pass ? RISCV_IOMMU_CAP_SV32X4 : RISCV_IOMMU_CAP_SV32;
if (!(s->cap & sv_mode)) {
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
sc[pass].levels = 2;
sc[pass].ptidxbits = 10;
sc[pass].ptesize = 4;
break;
default:
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
} else {
/* 64bit mode for GXL/SXL == 0 */
switch (pass ? gatp : satp) {
case RISCV_IOMMU_DC_IOHGATP_MODE_BARE:
sc[pass].levels = 0;
sc[pass].ptidxbits = 0;
sc[pass].ptesize = 0;
break;
case RISCV_IOMMU_DC_IOHGATP_MODE_SV39X4:
sv_mode = pass ? RISCV_IOMMU_CAP_SV39X4 : RISCV_IOMMU_CAP_SV39;
if (!(s->cap & sv_mode)) {
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
sc[pass].levels = 3;
sc[pass].ptidxbits = 9;
sc[pass].ptesize = 8;
break;
case RISCV_IOMMU_DC_IOHGATP_MODE_SV48X4:
sv_mode = pass ? RISCV_IOMMU_CAP_SV48X4 : RISCV_IOMMU_CAP_SV48;
if (!(s->cap & sv_mode)) {
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
sc[pass].levels = 4;
sc[pass].ptidxbits = 9;
sc[pass].ptesize = 8;
break;
case RISCV_IOMMU_DC_IOHGATP_MODE_SV57X4:
sv_mode = pass ? RISCV_IOMMU_CAP_SV57X4 : RISCV_IOMMU_CAP_SV57;
if (!(s->cap & sv_mode)) {
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
sc[pass].levels = 5;
sc[pass].ptidxbits = 9;
sc[pass].ptesize = 8;
break;
default:
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
}
};
/* S/G stages translation tables root pointers */
gatp = PPN_PHYS(get_field(ctx->gatp, RISCV_IOMMU_ATP_PPN_FIELD));
satp = PPN_PHYS(get_field(ctx->satp, RISCV_IOMMU_ATP_PPN_FIELD));
addr = (en_s && en_g) ? satp : iotlb->iova;
base = en_g ? gatp : satp;
pass = en_g ? G_STAGE : S_STAGE;
do {
const unsigned widened = (pass && !sc[pass].step) ? 2 : 0;
const unsigned va_bits = widened + sc[pass].ptidxbits;
const unsigned va_skip = TARGET_PAGE_BITS + sc[pass].ptidxbits *
(sc[pass].levels - 1 - sc[pass].step);
const unsigned idx = (addr >> va_skip) & ((1 << va_bits) - 1);
const dma_addr_t pte_addr = base + idx * sc[pass].ptesize;
const bool ade =
ctx->tc & (pass ? RISCV_IOMMU_DC_TC_GADE : RISCV_IOMMU_DC_TC_SADE);
/* Address range check before first level lookup */
if (!sc[pass].step) {
const uint64_t va_mask = (1ULL << (va_skip + va_bits)) - 1;
if ((addr & va_mask) != addr) {
return RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED;
}
}
/* Read page table entry */
if (sc[pass].ptesize == 4) {
uint32_t pte32 = 0;
ret = ldl_le_dma(s->target_as, pte_addr, &pte32,
MEMTXATTRS_UNSPECIFIED);
pte = pte32;
} else {
ret = ldq_le_dma(s->target_as, pte_addr, &pte,
MEMTXATTRS_UNSPECIFIED);
}
if (ret != MEMTX_OK) {
return (iotlb->perm & IOMMU_WO) ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT
: RISCV_IOMMU_FQ_CAUSE_RD_FAULT;
}
sc[pass].step++;
hwaddr ppn = pte >> PTE_PPN_SHIFT;
if (!(pte & PTE_V)) {
break; /* Invalid PTE */
} else if (!(pte & (PTE_R | PTE_W | PTE_X))) {
base = PPN_PHYS(ppn); /* Inner PTE, continue walking */
} else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) {
break; /* Reserved leaf PTE flags: PTE_W */
} else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) {
break; /* Reserved leaf PTE flags: PTE_W + PTE_X */
} else if (ppn & ((1ULL << (va_skip - TARGET_PAGE_BITS)) - 1)) {
break; /* Misaligned PPN */
} else if ((iotlb->perm & IOMMU_RO) && !(pte & PTE_R)) {
break; /* Read access check failed */
} else if ((iotlb->perm & IOMMU_WO) && !(pte & PTE_W)) {
break; /* Write access check failed */
} else if ((iotlb->perm & IOMMU_RO) && !ade && !(pte & PTE_A)) {
break; /* Access bit not set */
} else if ((iotlb->perm & IOMMU_WO) && !ade && !(pte & PTE_D)) {
break; /* Dirty bit not set */
} else {
/* Leaf PTE, translation completed. */
sc[pass].step = sc[pass].levels;
base = PPN_PHYS(ppn) | (addr & ((1ULL << va_skip) - 1));
/* Update address mask based on smallest translation granularity */
iotlb->addr_mask &= (1ULL << va_skip) - 1;
/* Continue with S-Stage translation? */
if (pass && sc[0].step != sc[0].levels) {
pass = S_STAGE;
addr = iotlb->iova;
continue;
}
/* Translation phase completed (GPA or SPA) */
iotlb->translated_addr = base;
iotlb->perm = (pte & PTE_W) ? ((pte & PTE_R) ? IOMMU_RW : IOMMU_WO)
: IOMMU_RO;
/* Check MSI GPA address match */
if (pass == S_STAGE && (iotlb->perm & IOMMU_WO) &&
riscv_iommu_msi_check(s, ctx, base)) {
/* Trap MSI writes and return GPA address. */
iotlb->target_as = &s->trap_as;
iotlb->addr_mask = ~TARGET_PAGE_MASK;
return 0;
}
/* Continue with G-Stage translation? */
if (!pass && en_g) {
pass = G_STAGE;
addr = base;
base = gatp;
sc[pass].step = 0;
continue;
}
return 0;
}
if (sc[pass].step == sc[pass].levels) {
break; /* Can't find leaf PTE */
}
/* Continue with G-Stage translation? */
if (!pass && en_g) {
pass = G_STAGE;
addr = base;
base = gatp;
sc[pass].step = 0;
}
} while (1);
return (iotlb->perm & IOMMU_WO) ?
(pass ? RISCV_IOMMU_FQ_CAUSE_WR_FAULT_VS :
RISCV_IOMMU_FQ_CAUSE_WR_FAULT_S) :
(pass ? RISCV_IOMMU_FQ_CAUSE_RD_FAULT_VS :
RISCV_IOMMU_FQ_CAUSE_RD_FAULT_S);
}
static void riscv_iommu_report_fault(RISCVIOMMUState *s,
RISCVIOMMUContext *ctx,
uint32_t fault_type, uint32_t cause,
bool pv,
uint64_t iotval, uint64_t iotval2)
{
struct riscv_iommu_fq_record ev = { 0 };
if (ctx->tc & RISCV_IOMMU_DC_TC_DTF) {
switch (cause) {
case RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED:
case RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT:
case RISCV_IOMMU_FQ_CAUSE_DDT_INVALID:
case RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED:
case RISCV_IOMMU_FQ_CAUSE_DDT_CORRUPTED:
case RISCV_IOMMU_FQ_CAUSE_INTERNAL_DP_ERROR:
case RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT:
break;
default:
/* DTF prevents reporting a fault for this given cause */
return;
}
}
ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_CAUSE, cause);
ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_TTYPE, fault_type);
ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_DID, ctx->devid);
ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_PV, true);
if (pv) {
ev.hdr = set_field(ev.hdr, RISCV_IOMMU_FQ_HDR_PID, ctx->process_id);
}
ev.iotval = iotval;
ev.iotval2 = iotval2;
riscv_iommu_fault(s, &ev);
}
/* Redirect MSI write for given GPA. */
static MemTxResult riscv_iommu_msi_write(RISCVIOMMUState *s,
RISCVIOMMUContext *ctx, uint64_t gpa, uint64_t data,
unsigned size, MemTxAttrs attrs)
{
MemTxResult res;
dma_addr_t addr;
uint64_t intn;
uint32_t n190;
uint64_t pte[2];
int fault_type = RISCV_IOMMU_FQ_TTYPE_UADDR_WR;
int cause;
/* Interrupt File Number */
intn = riscv_iommu_pext_u64(PPN_DOWN(gpa), ctx->msi_addr_mask);
if (intn >= 256) {
/* Interrupt file number out of range */
res = MEMTX_ACCESS_ERROR;
cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT;
goto err;
}
/* fetch MSI PTE */
addr = PPN_PHYS(get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_PPN));
addr = addr | (intn * sizeof(pte));
res = dma_memory_read(s->target_as, addr, &pte, sizeof(pte),
MEMTXATTRS_UNSPECIFIED);
if (res != MEMTX_OK) {
if (res == MEMTX_DECODE_ERROR) {
cause = RISCV_IOMMU_FQ_CAUSE_MSI_PT_CORRUPTED;
} else {
cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT;
}
goto err;
}
le64_to_cpus(&pte[0]);
le64_to_cpus(&pte[1]);
if (!(pte[0] & RISCV_IOMMU_MSI_PTE_V) || (pte[0] & RISCV_IOMMU_MSI_PTE_C)) {
/*
* The spec mentions that: "If msipte.C == 1, then further
* processing to interpret the PTE is implementation
* defined.". We'll abort with cause = 262 for this
* case too.
*/
res = MEMTX_ACCESS_ERROR;
cause = RISCV_IOMMU_FQ_CAUSE_MSI_INVALID;
goto err;
}
switch (get_field(pte[0], RISCV_IOMMU_MSI_PTE_M)) {
case RISCV_IOMMU_MSI_PTE_M_BASIC:
/* MSI Pass-through mode */
addr = PPN_PHYS(get_field(pte[0], RISCV_IOMMU_MSI_PTE_PPN));
trace_riscv_iommu_msi(s->parent_obj.id, PCI_BUS_NUM(ctx->devid),
PCI_SLOT(ctx->devid), PCI_FUNC(ctx->devid),
gpa, addr);
res = dma_memory_write(s->target_as, addr, &data, size, attrs);
if (res != MEMTX_OK) {
cause = RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT;
goto err;
}
return MEMTX_OK;
case RISCV_IOMMU_MSI_PTE_M_MRIF:
/* MRIF mode, continue. */
break;
default:
res = MEMTX_ACCESS_ERROR;
cause = RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED;
goto err;
}
/*
* Report an error for interrupt identities exceeding the maximum allowed
* for an IMSIC interrupt file (2047) or destination address is not 32-bit
* aligned. See IOMMU Specification, Chapter 2.3. MSI page tables.
*/
if ((data > 2047) || (gpa & 3)) {
res = MEMTX_ACCESS_ERROR;
cause = RISCV_IOMMU_FQ_CAUSE_MSI_MISCONFIGURED;
goto err;
}
/* MSI MRIF mode, non atomic pending bit update */
/* MRIF pending bit address */
addr = get_field(pte[0], RISCV_IOMMU_MSI_PTE_MRIF_ADDR) << 9;
addr = addr | ((data & 0x7c0) >> 3);
trace_riscv_iommu_msi(s->parent_obj.id, PCI_BUS_NUM(ctx->devid),
PCI_SLOT(ctx->devid), PCI_FUNC(ctx->devid),
gpa, addr);
/* MRIF pending bit mask */
data = 1ULL << (data & 0x03f);
res = dma_memory_read(s->target_as, addr, &intn, sizeof(intn), attrs);
if (res != MEMTX_OK) {
cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT;
goto err;
}
intn = intn | data;
res = dma_memory_write(s->target_as, addr, &intn, sizeof(intn), attrs);
if (res != MEMTX_OK) {
cause = RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT;
goto err;
}
/* Get MRIF enable bits */
addr = addr + sizeof(intn);
res = dma_memory_read(s->target_as, addr, &intn, sizeof(intn), attrs);
if (res != MEMTX_OK) {
cause = RISCV_IOMMU_FQ_CAUSE_MSI_LOAD_FAULT;
goto err;
}
if (!(intn & data)) {
/* notification disabled, MRIF update completed. */
return MEMTX_OK;
}
/* Send notification message */
addr = PPN_PHYS(get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NPPN));
n190 = get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NID) |
(get_field(pte[1], RISCV_IOMMU_MSI_MRIF_NID_MSB) << 10);
res = dma_memory_write(s->target_as, addr, &n190, sizeof(n190), attrs);
if (res != MEMTX_OK) {
cause = RISCV_IOMMU_FQ_CAUSE_MSI_WR_FAULT;
goto err;
}
trace_riscv_iommu_mrif_notification(s->parent_obj.id, n190, addr);
return MEMTX_OK;
err:
riscv_iommu_report_fault(s, ctx, fault_type, cause,
!!ctx->process_id, 0, 0);
return res;
}
/*
* Check device context configuration as described by the
* riscv-iommu spec section "Device-context configuration
* checks".
*/
static bool riscv_iommu_validate_device_ctx(RISCVIOMMUState *s,
RISCVIOMMUContext *ctx)
{
uint32_t fsc_mode, msi_mode;
uint64_t gatp;
if (!(s->cap & RISCV_IOMMU_CAP_ATS) &&
(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS ||
ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI ||
ctx->tc & RISCV_IOMMU_DC_TC_PRPR)) {
return false;
}
if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS) &&
(ctx->tc & RISCV_IOMMU_DC_TC_T2GPA ||
ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI)) {
return false;
}
if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_PRI) &&
ctx->tc & RISCV_IOMMU_DC_TC_PRPR) {
return false;
}
if (!(s->cap & RISCV_IOMMU_CAP_T2GPA) &&
ctx->tc & RISCV_IOMMU_DC_TC_T2GPA) {
return false;
}
if (s->cap & RISCV_IOMMU_CAP_MSI_FLAT) {
msi_mode = get_field(ctx->msiptp, RISCV_IOMMU_DC_MSIPTP_MODE);
if (msi_mode != RISCV_IOMMU_DC_MSIPTP_MODE_OFF &&
msi_mode != RISCV_IOMMU_DC_MSIPTP_MODE_FLAT) {
return false;
}
}
gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
if (ctx->tc & RISCV_IOMMU_DC_TC_T2GPA &&
gatp == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
return false;
}
fsc_mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE);
if (ctx->tc & RISCV_IOMMU_DC_TC_PDTV) {
switch (fsc_mode) {
case RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8:
if (!(s->cap & RISCV_IOMMU_CAP_PD8)) {
return false;
}
break;
case RISCV_IOMMU_DC_FSC_PDTP_MODE_PD17:
if (!(s->cap & RISCV_IOMMU_CAP_PD17)) {
return false;
}
break;
case RISCV_IOMMU_DC_FSC_PDTP_MODE_PD20:
if (!(s->cap & RISCV_IOMMU_CAP_PD20)) {
return false;
}
break;
}
} else {
/* DC.tc.PDTV is 0 */
if (ctx->tc & RISCV_IOMMU_DC_TC_DPE) {
return false;
}
if (ctx->tc & RISCV_IOMMU_DC_TC_SXL) {
if (fsc_mode == RISCV_IOMMU_CAP_SV32 &&
!(s->cap & RISCV_IOMMU_CAP_SV32)) {
return false;
}
} else {
switch (fsc_mode) {
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39:
if (!(s->cap & RISCV_IOMMU_CAP_SV39)) {
return false;
}
break;
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48:
if (!(s->cap & RISCV_IOMMU_CAP_SV48)) {
return false;
}
break;
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57:
if (!(s->cap & RISCV_IOMMU_CAP_SV57)) {
return false;
}
break;
}
}
}
/*
* CAP_END is always zero (only one endianess). FCTL_BE is
* always zero (little-endian accesses). Thus TC_SBE must
* always be LE, i.e. zero.
*/
if (ctx->tc & RISCV_IOMMU_DC_TC_SBE) {
return false;
}
return true;
}
/*
* Validate process context (PC) according to section
* "Process-context configuration checks".
*/
static bool riscv_iommu_validate_process_ctx(RISCVIOMMUState *s,
RISCVIOMMUContext *ctx)
{
uint32_t mode;
if (get_field(ctx->ta, RISCV_IOMMU_PC_TA_RESERVED)) {
return false;
}
if (get_field(ctx->satp, RISCV_IOMMU_PC_FSC_RESERVED)) {
return false;
}
mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE);
switch (mode) {
case RISCV_IOMMU_DC_FSC_MODE_BARE:
/* sv39 and sv32 modes have the same value (8) */
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39:
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48:
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57:
break;
default:
return false;
}
if (ctx->tc & RISCV_IOMMU_DC_TC_SXL) {
if (mode == RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV32 &&
!(s->cap & RISCV_IOMMU_CAP_SV32)) {
return false;
}
} else {
switch (mode) {
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39:
if (!(s->cap & RISCV_IOMMU_CAP_SV39)) {
return false;
}
break;
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48:
if (!(s->cap & RISCV_IOMMU_CAP_SV48)) {
return false;
}
break;
case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57:
if (!(s->cap & RISCV_IOMMU_CAP_SV57)) {
return false;
}
break;
}
}
return true;
}
/*
* RISC-V IOMMU Device Context Loopkup - Device Directory Tree Walk
*
* @s : IOMMU Device State
* @ctx : Device Translation Context with devid and process_id set.
* @return : success or fault code.
*/
static int riscv_iommu_ctx_fetch(RISCVIOMMUState *s, RISCVIOMMUContext *ctx)
{
const uint64_t ddtp = s->ddtp;
unsigned mode = get_field(ddtp, RISCV_IOMMU_DDTP_MODE);
dma_addr_t addr = PPN_PHYS(get_field(ddtp, RISCV_IOMMU_DDTP_PPN));
struct riscv_iommu_dc dc;
/* Device Context format: 0: extended (64 bytes) | 1: base (32 bytes) */
const int dc_fmt = !s->enable_msi;
const size_t dc_len = sizeof(dc) >> dc_fmt;
int depth;
uint64_t de;
switch (mode) {
case RISCV_IOMMU_DDTP_MODE_OFF:
return RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED;
case RISCV_IOMMU_DDTP_MODE_BARE:
/* mock up pass-through translation context */
ctx->gatp = set_field(0, RISCV_IOMMU_ATP_MODE_FIELD,
RISCV_IOMMU_DC_IOHGATP_MODE_BARE);
ctx->satp = set_field(0, RISCV_IOMMU_ATP_MODE_FIELD,
RISCV_IOMMU_DC_FSC_MODE_BARE);
ctx->tc = RISCV_IOMMU_DC_TC_V;
if (s->enable_ats) {
ctx->tc |= RISCV_IOMMU_DC_TC_EN_ATS;
}
ctx->ta = 0;
ctx->msiptp = 0;
return 0;
case RISCV_IOMMU_DDTP_MODE_1LVL:
depth = 0;
break;
case RISCV_IOMMU_DDTP_MODE_2LVL:
depth = 1;
break;
case RISCV_IOMMU_DDTP_MODE_3LVL:
depth = 2;
break;
default:
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
/*
* Check supported device id width (in bits).
* See IOMMU Specification, Chapter 6. Software guidelines.
* - if extended device-context format is used:
* 1LVL: 6, 2LVL: 15, 3LVL: 24
* - if base device-context format is used:
* 1LVL: 7, 2LVL: 16, 3LVL: 24
*/
if (ctx->devid >= (1 << (depth * 9 + 6 + (dc_fmt && depth != 2)))) {
return RISCV_IOMMU_FQ_CAUSE_TTYPE_BLOCKED;
}
/* Device directory tree walk */
for (; depth-- > 0; ) {
/*
* Select device id index bits based on device directory tree level
* and device context format.
* See IOMMU Specification, Chapter 2. Data Structures.
* - if extended device-context format is used:
* device index: [23:15][14:6][5:0]
* - if base device-context format is used:
* device index: [23:16][15:7][6:0]
*/
const int split = depth * 9 + 6 + dc_fmt;
addr |= ((ctx->devid >> split) << 3) & ~TARGET_PAGE_MASK;
if (dma_memory_read(s->target_as, addr, &de, sizeof(de),
MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
return RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT;
}
le64_to_cpus(&de);
if (!(de & RISCV_IOMMU_DDTE_VALID)) {
/* invalid directory entry */
return RISCV_IOMMU_FQ_CAUSE_DDT_INVALID;
}
if (de & ~(RISCV_IOMMU_DDTE_PPN | RISCV_IOMMU_DDTE_VALID)) {
/* reserved bits set */
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
addr = PPN_PHYS(get_field(de, RISCV_IOMMU_DDTE_PPN));
}
/* index into device context entry page */
addr |= (ctx->devid * dc_len) & ~TARGET_PAGE_MASK;
memset(&dc, 0, sizeof(dc));
if (dma_memory_read(s->target_as, addr, &dc, dc_len,
MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
return RISCV_IOMMU_FQ_CAUSE_DDT_LOAD_FAULT;
}
/* Set translation context. */
ctx->tc = le64_to_cpu(dc.tc);
ctx->gatp = le64_to_cpu(dc.iohgatp);
ctx->satp = le64_to_cpu(dc.fsc);
ctx->ta = le64_to_cpu(dc.ta);
ctx->msiptp = le64_to_cpu(dc.msiptp);
ctx->msi_addr_mask = le64_to_cpu(dc.msi_addr_mask);
ctx->msi_addr_pattern = le64_to_cpu(dc.msi_addr_pattern);
if (!(ctx->tc & RISCV_IOMMU_DC_TC_V)) {
return RISCV_IOMMU_FQ_CAUSE_DDT_INVALID;
}
if (!riscv_iommu_validate_device_ctx(s, ctx)) {
return RISCV_IOMMU_FQ_CAUSE_DDT_MISCONFIGURED;
}
/* FSC field checks */
mode = get_field(ctx->satp, RISCV_IOMMU_DC_FSC_MODE);
addr = PPN_PHYS(get_field(ctx->satp, RISCV_IOMMU_DC_FSC_PPN));
if (!(ctx->tc & RISCV_IOMMU_DC_TC_PDTV)) {
if (ctx->process_id != RISCV_IOMMU_NOPROCID) {
/* PID is disabled */
return RISCV_IOMMU_FQ_CAUSE_TTYPE_BLOCKED;
}
if (mode > RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57) {
/* Invalid translation mode */
return RISCV_IOMMU_FQ_CAUSE_DDT_INVALID;
}
return 0;
}
if (ctx->process_id == RISCV_IOMMU_NOPROCID) {
if (!(ctx->tc & RISCV_IOMMU_DC_TC_DPE)) {
/* No default process_id enabled, set BARE mode */
ctx->satp = 0ULL;
return 0;
} else {
/* Use default process_id #0 */
ctx->process_id = 0;
}
}
if (mode == RISCV_IOMMU_DC_FSC_MODE_BARE) {
/* No S-Stage translation, done. */
return 0;
}
/* FSC.TC.PDTV enabled */
if (mode > RISCV_IOMMU_DC_FSC_PDTP_MODE_PD20) {
/* Invalid PDTP.MODE */
return RISCV_IOMMU_FQ_CAUSE_PDT_MISCONFIGURED;
}
for (depth = mode - RISCV_IOMMU_DC_FSC_PDTP_MODE_PD8; depth-- > 0; ) {
/*
* Select process id index bits based on process directory tree
* level. See IOMMU Specification, 2.2. Process-Directory-Table.
*/
const int split = depth * 9 + 8;
addr |= ((ctx->process_id >> split) << 3) & ~TARGET_PAGE_MASK;
if (dma_memory_read(s->target_as, addr, &de, sizeof(de),
MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
return RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT;
}
le64_to_cpus(&de);
if (!(de & RISCV_IOMMU_PC_TA_V)) {
return RISCV_IOMMU_FQ_CAUSE_PDT_INVALID;
}
addr = PPN_PHYS(get_field(de, RISCV_IOMMU_PC_FSC_PPN));
}
/* Leaf entry in PDT */
addr |= (ctx->process_id << 4) & ~TARGET_PAGE_MASK;
if (dma_memory_read(s->target_as, addr, &dc.ta, sizeof(uint64_t) * 2,
MEMTXATTRS_UNSPECIFIED) != MEMTX_OK) {
return RISCV_IOMMU_FQ_CAUSE_PDT_LOAD_FAULT;
}
/* Use FSC and TA from process directory entry. */
ctx->ta = le64_to_cpu(dc.ta);
ctx->satp = le64_to_cpu(dc.fsc);
if (!(ctx->ta & RISCV_IOMMU_PC_TA_V)) {
return RISCV_IOMMU_FQ_CAUSE_PDT_INVALID;
}
if (!riscv_iommu_validate_process_ctx(s, ctx)) {
return RISCV_IOMMU_FQ_CAUSE_PDT_MISCONFIGURED;
}
return 0;
}
/* Translation Context cache support */
static gboolean riscv_iommu_ctx_equal(gconstpointer v1, gconstpointer v2)
{
RISCVIOMMUContext *c1 = (RISCVIOMMUContext *) v1;
RISCVIOMMUContext *c2 = (RISCVIOMMUContext *) v2;
return c1->devid == c2->devid &&
c1->process_id == c2->process_id;
}
static guint riscv_iommu_ctx_hash(gconstpointer v)
{
RISCVIOMMUContext *ctx = (RISCVIOMMUContext *) v;
/*
* Generate simple hash of (process_id, devid)
* assuming 24-bit wide devid.
*/
return (guint)(ctx->devid) + ((guint)(ctx->process_id) << 24);
}
static void riscv_iommu_ctx_inval_devid_procid(gpointer key, gpointer value,
gpointer data)
{
RISCVIOMMUContext *ctx = (RISCVIOMMUContext *) value;
RISCVIOMMUContext *arg = (RISCVIOMMUContext *) data;
if (ctx->tc & RISCV_IOMMU_DC_TC_V &&
ctx->devid == arg->devid &&
ctx->process_id == arg->process_id) {
ctx->tc &= ~RISCV_IOMMU_DC_TC_V;
}
}
static void riscv_iommu_ctx_inval_devid(gpointer key, gpointer value,
gpointer data)
{
RISCVIOMMUContext *ctx = (RISCVIOMMUContext *) value;
RISCVIOMMUContext *arg = (RISCVIOMMUContext *) data;
if (ctx->tc & RISCV_IOMMU_DC_TC_V &&
ctx->devid == arg->devid) {
ctx->tc &= ~RISCV_IOMMU_DC_TC_V;
}
}
static void riscv_iommu_ctx_inval_all(gpointer key, gpointer value,
gpointer data)
{
RISCVIOMMUContext *ctx = (RISCVIOMMUContext *) value;
if (ctx->tc & RISCV_IOMMU_DC_TC_V) {
ctx->tc &= ~RISCV_IOMMU_DC_TC_V;
}
}
static void riscv_iommu_ctx_inval(RISCVIOMMUState *s, GHFunc func,
uint32_t devid, uint32_t process_id)
{
GHashTable *ctx_cache;
RISCVIOMMUContext key = {
.devid = devid,
.process_id = process_id,
};
ctx_cache = g_hash_table_ref(s->ctx_cache);
g_hash_table_foreach(ctx_cache, func, &key);
g_hash_table_unref(ctx_cache);
}
/* Find or allocate translation context for a given {device_id, process_id} */
static RISCVIOMMUContext *riscv_iommu_ctx(RISCVIOMMUState *s,
unsigned devid, unsigned process_id,
void **ref)
{
GHashTable *ctx_cache;
RISCVIOMMUContext *ctx;
RISCVIOMMUContext key = {
.devid = devid,
.process_id = process_id,
};
ctx_cache = g_hash_table_ref(s->ctx_cache);
ctx = g_hash_table_lookup(ctx_cache, &key);
if (ctx && (ctx->tc & RISCV_IOMMU_DC_TC_V)) {
*ref = ctx_cache;
return ctx;
}
ctx = g_new0(RISCVIOMMUContext, 1);
ctx->devid = devid;
ctx->process_id = process_id;
int fault = riscv_iommu_ctx_fetch(s, ctx);
if (!fault) {
if (g_hash_table_size(ctx_cache) >= LIMIT_CACHE_CTX) {
g_hash_table_unref(ctx_cache);
ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash,
riscv_iommu_ctx_equal,
g_free, NULL);
g_hash_table_ref(ctx_cache);
g_hash_table_unref(qatomic_xchg(&s->ctx_cache, ctx_cache));
}
g_hash_table_add(ctx_cache, ctx);
*ref = ctx_cache;
return ctx;
}
g_hash_table_unref(ctx_cache);
*ref = NULL;
riscv_iommu_report_fault(s, ctx, RISCV_IOMMU_FQ_TTYPE_UADDR_RD,
fault, !!process_id, 0, 0);
g_free(ctx);
return NULL;
}
static void riscv_iommu_ctx_put(RISCVIOMMUState *s, void *ref)
{
if (ref) {
g_hash_table_unref((GHashTable *)ref);
}
}
/* Find or allocate address space for a given device */
static AddressSpace *riscv_iommu_space(RISCVIOMMUState *s, uint32_t devid)
{
RISCVIOMMUSpace *as;
/* FIXME: PCIe bus remapping for attached endpoints. */
devid |= s->bus << 8;
QLIST_FOREACH(as, &s->spaces, list) {
if (as->devid == devid) {
break;
}
}
if (as == NULL) {
char name[64];
as = g_new0(RISCVIOMMUSpace, 1);
as->iommu = s;
as->devid = devid;
snprintf(name, sizeof(name), "riscv-iommu-%04x:%02x.%d-iova",
PCI_BUS_NUM(as->devid), PCI_SLOT(as->devid), PCI_FUNC(as->devid));
/* IOVA address space, untranslated addresses */
memory_region_init_iommu(&as->iova_mr, sizeof(as->iova_mr),
TYPE_RISCV_IOMMU_MEMORY_REGION,
OBJECT(as), "riscv_iommu", UINT64_MAX);
address_space_init(&as->iova_as, MEMORY_REGION(&as->iova_mr), name);
QLIST_INSERT_HEAD(&s->spaces, as, list);
trace_riscv_iommu_new(s->parent_obj.id, PCI_BUS_NUM(as->devid),
PCI_SLOT(as->devid), PCI_FUNC(as->devid));
}
return &as->iova_as;
}
/* Translation Object cache support */
static gboolean riscv_iommu_iot_equal(gconstpointer v1, gconstpointer v2)
{
RISCVIOMMUEntry *t1 = (RISCVIOMMUEntry *) v1;
RISCVIOMMUEntry *t2 = (RISCVIOMMUEntry *) v2;
return t1->gscid == t2->gscid && t1->pscid == t2->pscid &&
t1->iova == t2->iova;
}
static guint riscv_iommu_iot_hash(gconstpointer v)
{
RISCVIOMMUEntry *t = (RISCVIOMMUEntry *) v;
return (guint)t->iova;
}
/* GV: 1 PSCV: 1 AV: 1 */
static void riscv_iommu_iot_inval_pscid_iova(gpointer key, gpointer value,
gpointer data)
{
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
if (iot->gscid == arg->gscid &&
iot->pscid == arg->pscid &&
iot->iova == arg->iova) {
iot->perm = IOMMU_NONE;
}
}
/* GV: 1 PSCV: 1 AV: 0 */
static void riscv_iommu_iot_inval_pscid(gpointer key, gpointer value,
gpointer data)
{
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
if (iot->gscid == arg->gscid &&
iot->pscid == arg->pscid) {
iot->perm = IOMMU_NONE;
}
}
/* GV: 1 GVMA: 1 */
static void riscv_iommu_iot_inval_gscid_gpa(gpointer key, gpointer value,
gpointer data)
{
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
if (iot->gscid == arg->gscid) {
/* simplified cache, no GPA matching */
iot->perm = IOMMU_NONE;
}
}
/* GV: 1 GVMA: 0 */
static void riscv_iommu_iot_inval_gscid(gpointer key, gpointer value,
gpointer data)
{
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
if (iot->gscid == arg->gscid) {
iot->perm = IOMMU_NONE;
}
}
/* GV: 0 */
static void riscv_iommu_iot_inval_all(gpointer key, gpointer value,
gpointer data)
{
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
iot->perm = IOMMU_NONE;
}
/* caller should keep ref-count for iot_cache object */
static RISCVIOMMUEntry *riscv_iommu_iot_lookup(RISCVIOMMUContext *ctx,
GHashTable *iot_cache, hwaddr iova)
{
RISCVIOMMUEntry key = {
.gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID),
.pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID),
.iova = PPN_DOWN(iova),
};
return g_hash_table_lookup(iot_cache, &key);
}
/* caller should keep ref-count for iot_cache object */
static void riscv_iommu_iot_update(RISCVIOMMUState *s,
GHashTable *iot_cache, RISCVIOMMUEntry *iot)
{
if (!s->iot_limit) {
return;
}
if (g_hash_table_size(s->iot_cache) >= s->iot_limit) {
iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash,
riscv_iommu_iot_equal,
g_free, NULL);
g_hash_table_unref(qatomic_xchg(&s->iot_cache, iot_cache));
}
g_hash_table_add(iot_cache, iot);
}
static void riscv_iommu_iot_inval(RISCVIOMMUState *s, GHFunc func,
uint32_t gscid, uint32_t pscid, hwaddr iova)
{
GHashTable *iot_cache;
RISCVIOMMUEntry key = {
.gscid = gscid,
.pscid = pscid,
.iova = PPN_DOWN(iova),
};
iot_cache = g_hash_table_ref(s->iot_cache);
g_hash_table_foreach(iot_cache, func, &key);
g_hash_table_unref(iot_cache);
}
static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,
IOMMUTLBEntry *iotlb, bool enable_cache)
{
RISCVIOMMUEntry *iot;
IOMMUAccessFlags perm;
bool enable_pid;
bool enable_pri;
GHashTable *iot_cache;
int fault;
iot_cache = g_hash_table_ref(s->iot_cache);
/*
* TC[32] is reserved for custom extensions, used here to temporarily
* enable automatic page-request generation for ATS queries.
*/
enable_pri = (iotlb->perm == IOMMU_NONE) && (ctx->tc & BIT_ULL(32));
enable_pid = (ctx->tc & RISCV_IOMMU_DC_TC_PDTV);
/* Check for ATS request. */
if (iotlb->perm == IOMMU_NONE) {
/* Check if ATS is disabled. */
if (!(ctx->tc & RISCV_IOMMU_DC_TC_EN_ATS)) {
enable_pri = false;
fault = RISCV_IOMMU_FQ_CAUSE_TTYPE_BLOCKED;
goto done;
}
}
iot = riscv_iommu_iot_lookup(ctx, iot_cache, iotlb->iova);
perm = iot ? iot->perm : IOMMU_NONE;
if (perm != IOMMU_NONE) {
iotlb->translated_addr = PPN_PHYS(iot->phys);
iotlb->addr_mask = ~TARGET_PAGE_MASK;
iotlb->perm = perm;
fault = 0;
goto done;
}
/* Translate using device directory / page table information. */
fault = riscv_iommu_spa_fetch(s, ctx, iotlb);
if (!fault && iotlb->target_as == &s->trap_as) {
/* Do not cache trapped MSI translations */
goto done;
}
/*
* We made an implementation choice to not cache identity-mapped
* translations, as allowed by the specification, to avoid
* translation cache evictions for other devices sharing the
* IOMMU hardware model.
*/
if (!fault && iotlb->translated_addr != iotlb->iova && enable_cache) {
iot = g_new0(RISCVIOMMUEntry, 1);
iot->iova = PPN_DOWN(iotlb->iova);
iot->phys = PPN_DOWN(iotlb->translated_addr);
iot->gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID);
iot->pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID);
iot->perm = iotlb->perm;
riscv_iommu_iot_update(s, iot_cache, iot);
}
done:
g_hash_table_unref(iot_cache);
if (enable_pri && fault) {
struct riscv_iommu_pq_record pr = {0};
if (enable_pid) {
pr.hdr = set_field(RISCV_IOMMU_PREQ_HDR_PV,
RISCV_IOMMU_PREQ_HDR_PID, ctx->process_id);
}
pr.hdr = set_field(pr.hdr, RISCV_IOMMU_PREQ_HDR_DID, ctx->devid);
pr.payload = (iotlb->iova & TARGET_PAGE_MASK) |
RISCV_IOMMU_PREQ_PAYLOAD_M;
riscv_iommu_pri(s, &pr);
return fault;
}
if (fault) {
unsigned ttype = RISCV_IOMMU_FQ_TTYPE_PCIE_ATS_REQ;
if (iotlb->perm & IOMMU_RW) {
ttype = RISCV_IOMMU_FQ_TTYPE_UADDR_WR;
} else if (iotlb->perm & IOMMU_RO) {
ttype = RISCV_IOMMU_FQ_TTYPE_UADDR_RD;
}
riscv_iommu_report_fault(s, ctx, ttype, fault, enable_pid,
iotlb->iova, iotlb->translated_addr);
return fault;
}
return 0;
}
/* IOMMU Command Interface */
static MemTxResult riscv_iommu_iofence(RISCVIOMMUState *s, bool notify,
uint64_t addr, uint32_t data)
{
/*
* ATS processing in this implementation of the IOMMU is synchronous,
* no need to wait for completions here.
*/
if (!notify) {
return MEMTX_OK;
}
return dma_memory_write(s->target_as, addr, &data, sizeof(data),
MEMTXATTRS_UNSPECIFIED);
}
static void riscv_iommu_ats(RISCVIOMMUState *s,
struct riscv_iommu_command *cmd, IOMMUNotifierFlag flag,
IOMMUAccessFlags perm,
void (*trace_fn)(const char *id))
{
RISCVIOMMUSpace *as = NULL;
IOMMUNotifier *n;
IOMMUTLBEvent event;
uint32_t pid;
uint32_t devid;
const bool pv = cmd->dword0 & RISCV_IOMMU_CMD_ATS_PV;
if (cmd->dword0 & RISCV_IOMMU_CMD_ATS_DSV) {
/* Use device segment and requester id */
devid = get_field(cmd->dword0,
RISCV_IOMMU_CMD_ATS_DSEG | RISCV_IOMMU_CMD_ATS_RID);
} else {
devid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_RID);
}
pid = get_field(cmd->dword0, RISCV_IOMMU_CMD_ATS_PID);
QLIST_FOREACH(as, &s->spaces, list) {
if (as->devid == devid) {
break;
}
}
if (!as || !as->notifier) {
return;
}
event.type = flag;
event.entry.perm = perm;
event.entry.target_as = s->target_as;
IOMMU_NOTIFIER_FOREACH(n, &as->iova_mr) {
if (!pv || n->iommu_idx == pid) {
event.entry.iova = n->start;
event.entry.addr_mask = n->end - n->start;
trace_fn(as->iova_mr.parent_obj.name);
memory_region_notify_iommu_one(n, &event);
}
}
}
static void riscv_iommu_ats_inval(RISCVIOMMUState *s,
struct riscv_iommu_command *cmd)
{
return riscv_iommu_ats(s, cmd, IOMMU_NOTIFIER_DEVIOTLB_UNMAP, IOMMU_NONE,
trace_riscv_iommu_ats_inval);
}
static void riscv_iommu_ats_prgr(RISCVIOMMUState *s,
struct riscv_iommu_command *cmd)
{
unsigned resp_code = get_field(cmd->dword1,
RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE);
/* Using the access flag to carry response code information */
IOMMUAccessFlags perm = resp_code ? IOMMU_NONE : IOMMU_RW;
return riscv_iommu_ats(s, cmd, IOMMU_NOTIFIER_MAP, perm,
trace_riscv_iommu_ats_prgr);
}
static void riscv_iommu_process_ddtp(RISCVIOMMUState *s)
{
uint64_t old_ddtp = s->ddtp;
uint64_t new_ddtp = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_DDTP);
unsigned new_mode = get_field(new_ddtp, RISCV_IOMMU_DDTP_MODE);
unsigned old_mode = get_field(old_ddtp, RISCV_IOMMU_DDTP_MODE);
bool ok = false;
/*
* Check for allowed DDTP.MODE transitions:
* {OFF, BARE} -> {OFF, BARE, 1LVL, 2LVL, 3LVL}
* {1LVL, 2LVL, 3LVL} -> {OFF, BARE}
*/
if (new_mode == old_mode ||
new_mode == RISCV_IOMMU_DDTP_MODE_OFF ||
new_mode == RISCV_IOMMU_DDTP_MODE_BARE) {
ok = true;
} else if (new_mode == RISCV_IOMMU_DDTP_MODE_1LVL ||
new_mode == RISCV_IOMMU_DDTP_MODE_2LVL ||
new_mode == RISCV_IOMMU_DDTP_MODE_3LVL) {
ok = old_mode == RISCV_IOMMU_DDTP_MODE_OFF ||
old_mode == RISCV_IOMMU_DDTP_MODE_BARE;
}
if (ok) {
/* clear reserved and busy bits, report back sanitized version */
new_ddtp = set_field(new_ddtp & RISCV_IOMMU_DDTP_PPN,
RISCV_IOMMU_DDTP_MODE, new_mode);
} else {
new_ddtp = old_ddtp;
}
s->ddtp = new_ddtp;
riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_DDTP, new_ddtp);
}
/* Command function and opcode field. */
#define RISCV_IOMMU_CMD(func, op) (((func) << 7) | (op))
static void riscv_iommu_process_cq_tail(RISCVIOMMUState *s)
{
struct riscv_iommu_command cmd;
MemTxResult res;
dma_addr_t addr;
uint32_t tail, head, ctrl;
uint64_t cmd_opcode;
GHFunc func;
ctrl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQCSR);
tail = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQT) & s->cq_mask;
head = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQH) & s->cq_mask;
/* Check for pending error or queue processing disabled */
if (!(ctrl & RISCV_IOMMU_CQCSR_CQON) ||
!!(ctrl & (RISCV_IOMMU_CQCSR_CMD_ILL | RISCV_IOMMU_CQCSR_CQMF))) {
return;
}
while (tail != head) {
addr = s->cq_addr + head * sizeof(cmd);
res = dma_memory_read(s->target_as, addr, &cmd, sizeof(cmd),
MEMTXATTRS_UNSPECIFIED);
if (res != MEMTX_OK) {
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR,
RISCV_IOMMU_CQCSR_CQMF, 0);
goto fault;
}
trace_riscv_iommu_cmd(s->parent_obj.id, cmd.dword0, cmd.dword1);
cmd_opcode = get_field(cmd.dword0,
RISCV_IOMMU_CMD_OPCODE | RISCV_IOMMU_CMD_FUNC);
switch (cmd_opcode) {
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IOFENCE_FUNC_C,
RISCV_IOMMU_CMD_IOFENCE_OPCODE):
res = riscv_iommu_iofence(s,
cmd.dword0 & RISCV_IOMMU_CMD_IOFENCE_AV, cmd.dword1 << 2,
get_field(cmd.dword0, RISCV_IOMMU_CMD_IOFENCE_DATA));
if (res != MEMTX_OK) {
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR,
RISCV_IOMMU_CQCSR_CQMF, 0);
goto fault;
}
break;
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IOTINVAL_FUNC_GVMA,
RISCV_IOMMU_CMD_IOTINVAL_OPCODE):
if (cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_PSCV) {
/* illegal command arguments IOTINVAL.GVMA & PSCV == 1 */
goto cmd_ill;
} else if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_GV)) {
/* invalidate all cache mappings */
func = riscv_iommu_iot_inval_all;
} else if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_AV)) {
/* invalidate cache matching GSCID */
func = riscv_iommu_iot_inval_gscid;
} else {
/* invalidate cache matching GSCID and ADDR (GPA) */
func = riscv_iommu_iot_inval_gscid_gpa;
}
riscv_iommu_iot_inval(s, func,
get_field(cmd.dword0, RISCV_IOMMU_CMD_IOTINVAL_GSCID), 0,
cmd.dword1 << 2 & TARGET_PAGE_MASK);
break;
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA,
RISCV_IOMMU_CMD_IOTINVAL_OPCODE):
if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_GV)) {
/* invalidate all cache mappings, simplified model */
func = riscv_iommu_iot_inval_all;
} else if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_PSCV)) {
/* invalidate cache matching GSCID, simplified model */
func = riscv_iommu_iot_inval_gscid;
} else if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_AV)) {
/* invalidate cache matching GSCID and PSCID */
func = riscv_iommu_iot_inval_pscid;
} else {
/* invalidate cache matching GSCID and PSCID and ADDR (IOVA) */
func = riscv_iommu_iot_inval_pscid_iova;
}
riscv_iommu_iot_inval(s, func,
get_field(cmd.dword0, RISCV_IOMMU_CMD_IOTINVAL_GSCID),
get_field(cmd.dword0, RISCV_IOMMU_CMD_IOTINVAL_PSCID),
cmd.dword1 << 2 & TARGET_PAGE_MASK);
break;
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT,
RISCV_IOMMU_CMD_IODIR_OPCODE):
if (!(cmd.dword0 & RISCV_IOMMU_CMD_IODIR_DV)) {
/* invalidate all device context cache mappings */
func = riscv_iommu_ctx_inval_all;
} else {
/* invalidate all device context matching DID */
func = riscv_iommu_ctx_inval_devid;
}
riscv_iommu_ctx_inval(s, func,
get_field(cmd.dword0, RISCV_IOMMU_CMD_IODIR_DID), 0);
break;
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT,
RISCV_IOMMU_CMD_IODIR_OPCODE):
if (!(cmd.dword0 & RISCV_IOMMU_CMD_IODIR_DV)) {
/* illegal command arguments IODIR_PDT & DV == 0 */
goto cmd_ill;
} else {
func = riscv_iommu_ctx_inval_devid_procid;
}
riscv_iommu_ctx_inval(s, func,
get_field(cmd.dword0, RISCV_IOMMU_CMD_IODIR_DID),
get_field(cmd.dword0, RISCV_IOMMU_CMD_IODIR_PID));
break;
/* ATS commands */
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_ATS_FUNC_INVAL,
RISCV_IOMMU_CMD_ATS_OPCODE):
if (!s->enable_ats) {
goto cmd_ill;
}
riscv_iommu_ats_inval(s, &cmd);
break;
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_ATS_FUNC_PRGR,
RISCV_IOMMU_CMD_ATS_OPCODE):
if (!s->enable_ats) {
goto cmd_ill;
}
riscv_iommu_ats_prgr(s, &cmd);
break;
default:
cmd_ill:
/* Invalid instruction, do not advance instruction index. */
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR,
RISCV_IOMMU_CQCSR_CMD_ILL, 0);
goto fault;
}
/* Advance and update head pointer after command completes. */
head = (head + 1) & s->cq_mask;
riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_CQH, head);
}
return;
fault:
if (ctrl & RISCV_IOMMU_CQCSR_CIE) {
riscv_iommu_notify(s, RISCV_IOMMU_INTR_CQ);
}
}
static void riscv_iommu_process_cq_control(RISCVIOMMUState *s)
{
uint64_t base;
uint32_t ctrl_set = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQCSR);
uint32_t ctrl_clr;
bool enable = !!(ctrl_set & RISCV_IOMMU_CQCSR_CQEN);
bool active = !!(ctrl_set & RISCV_IOMMU_CQCSR_CQON);
if (enable && !active) {
base = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_CQB);
s->cq_mask = (2ULL << get_field(base, RISCV_IOMMU_CQB_LOG2SZ)) - 1;
s->cq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_CQB_PPN));
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQT], ~s->cq_mask);
stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_CQH], 0);
stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_CQT], 0);
ctrl_set = RISCV_IOMMU_CQCSR_CQON;
ctrl_clr = RISCV_IOMMU_CQCSR_BUSY | RISCV_IOMMU_CQCSR_CQMF |
RISCV_IOMMU_CQCSR_CMD_ILL | RISCV_IOMMU_CQCSR_CMD_TO |
RISCV_IOMMU_CQCSR_FENCE_W_IP;
} else if (!enable && active) {
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQT], ~0);
ctrl_set = 0;
ctrl_clr = RISCV_IOMMU_CQCSR_BUSY | RISCV_IOMMU_CQCSR_CQON;
} else {
ctrl_set = 0;
ctrl_clr = RISCV_IOMMU_CQCSR_BUSY;
}
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR, ctrl_set, ctrl_clr);
}
static void riscv_iommu_process_fq_control(RISCVIOMMUState *s)
{
uint64_t base;
uint32_t ctrl_set = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQCSR);
uint32_t ctrl_clr;
bool enable = !!(ctrl_set & RISCV_IOMMU_FQCSR_FQEN);
bool active = !!(ctrl_set & RISCV_IOMMU_FQCSR_FQON);
if (enable && !active) {
base = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_FQB);
s->fq_mask = (2ULL << get_field(base, RISCV_IOMMU_FQB_LOG2SZ)) - 1;
s->fq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_FQB_PPN));
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQH], ~s->fq_mask);
stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_FQH], 0);
stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_FQT], 0);
ctrl_set = RISCV_IOMMU_FQCSR_FQON;
ctrl_clr = RISCV_IOMMU_FQCSR_BUSY | RISCV_IOMMU_FQCSR_FQMF |
RISCV_IOMMU_FQCSR_FQOF;
} else if (!enable && active) {
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQH], ~0);
ctrl_set = 0;
ctrl_clr = RISCV_IOMMU_FQCSR_BUSY | RISCV_IOMMU_FQCSR_FQON;
} else {
ctrl_set = 0;
ctrl_clr = RISCV_IOMMU_FQCSR_BUSY;
}
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR, ctrl_set, ctrl_clr);
}
static void riscv_iommu_process_pq_control(RISCVIOMMUState *s)
{
uint64_t base;
uint32_t ctrl_set = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQCSR);
uint32_t ctrl_clr;
bool enable = !!(ctrl_set & RISCV_IOMMU_PQCSR_PQEN);
bool active = !!(ctrl_set & RISCV_IOMMU_PQCSR_PQON);
if (enable && !active) {
base = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_PQB);
s->pq_mask = (2ULL << get_field(base, RISCV_IOMMU_PQB_LOG2SZ)) - 1;
s->pq_addr = PPN_PHYS(get_field(base, RISCV_IOMMU_PQB_PPN));
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQH], ~s->pq_mask);
stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_PQH], 0);
stl_le_p(&s->regs_rw[RISCV_IOMMU_REG_PQT], 0);
ctrl_set = RISCV_IOMMU_PQCSR_PQON;
ctrl_clr = RISCV_IOMMU_PQCSR_BUSY | RISCV_IOMMU_PQCSR_PQMF |
RISCV_IOMMU_PQCSR_PQOF;
} else if (!enable && active) {
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQH], ~0);
ctrl_set = 0;
ctrl_clr = RISCV_IOMMU_PQCSR_BUSY | RISCV_IOMMU_PQCSR_PQON;
} else {
ctrl_set = 0;
ctrl_clr = RISCV_IOMMU_PQCSR_BUSY;
}
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR, ctrl_set, ctrl_clr);
}
static void riscv_iommu_process_dbg(RISCVIOMMUState *s)
{
uint64_t iova = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_TR_REQ_IOVA);
uint64_t ctrl = riscv_iommu_reg_get64(s, RISCV_IOMMU_REG_TR_REQ_CTL);
unsigned devid = get_field(ctrl, RISCV_IOMMU_TR_REQ_CTL_DID);
unsigned pid = get_field(ctrl, RISCV_IOMMU_TR_REQ_CTL_PID);
RISCVIOMMUContext *ctx;
void *ref;
if (!(ctrl & RISCV_IOMMU_TR_REQ_CTL_GO_BUSY)) {
return;
}
ctx = riscv_iommu_ctx(s, devid, pid, &ref);
if (ctx == NULL) {
riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_TR_RESPONSE,
RISCV_IOMMU_TR_RESPONSE_FAULT |
(RISCV_IOMMU_FQ_CAUSE_DMA_DISABLED << 10));
} else {
IOMMUTLBEntry iotlb = {
.iova = iova,
.perm = ctrl & RISCV_IOMMU_TR_REQ_CTL_NW ? IOMMU_RO : IOMMU_RW,
.addr_mask = ~0,
.target_as = NULL,
};
int fault = riscv_iommu_translate(s, ctx, &iotlb, false);
if (fault) {
iova = RISCV_IOMMU_TR_RESPONSE_FAULT | (((uint64_t) fault) << 10);
} else {
iova = iotlb.translated_addr & ~iotlb.addr_mask;
iova >>= TARGET_PAGE_BITS;
iova &= RISCV_IOMMU_TR_RESPONSE_PPN;
/* We do not support superpages (> 4kbs) for now */
iova &= ~RISCV_IOMMU_TR_RESPONSE_S;
}
riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_TR_RESPONSE, iova);
}
riscv_iommu_reg_mod64(s, RISCV_IOMMU_REG_TR_REQ_CTL, 0,
RISCV_IOMMU_TR_REQ_CTL_GO_BUSY);
riscv_iommu_ctx_put(s, ref);
}
typedef void riscv_iommu_process_fn(RISCVIOMMUState *s);
static void riscv_iommu_update_icvec(RISCVIOMMUState *s, uint64_t data)
{
uint64_t icvec = 0;
icvec |= MIN(data & RISCV_IOMMU_ICVEC_CIV,
s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_CIV);
icvec |= MIN(data & RISCV_IOMMU_ICVEC_FIV,
s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_FIV);
icvec |= MIN(data & RISCV_IOMMU_ICVEC_PMIV,
s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_PMIV);
icvec |= MIN(data & RISCV_IOMMU_ICVEC_PIV,
s->icvec_avail_vectors & RISCV_IOMMU_ICVEC_PIV);
trace_riscv_iommu_icvec_write(data, icvec);
riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_ICVEC, icvec);
}
static void riscv_iommu_update_ipsr(RISCVIOMMUState *s, uint64_t data)
{
uint32_t cqcsr, fqcsr, pqcsr;
uint32_t ipsr_set = 0;
uint32_t ipsr_clr = 0;
if (data & RISCV_IOMMU_IPSR_CIP) {
cqcsr = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_CQCSR);
if (cqcsr & RISCV_IOMMU_CQCSR_CIE &&
(cqcsr & RISCV_IOMMU_CQCSR_FENCE_W_IP ||
cqcsr & RISCV_IOMMU_CQCSR_CMD_ILL ||
cqcsr & RISCV_IOMMU_CQCSR_CMD_TO ||
cqcsr & RISCV_IOMMU_CQCSR_CQMF)) {
ipsr_set |= RISCV_IOMMU_IPSR_CIP;
} else {
ipsr_clr |= RISCV_IOMMU_IPSR_CIP;
}
} else {
ipsr_clr |= RISCV_IOMMU_IPSR_CIP;
}
if (data & RISCV_IOMMU_IPSR_FIP) {
fqcsr = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FQCSR);
if (fqcsr & RISCV_IOMMU_FQCSR_FIE &&
(fqcsr & RISCV_IOMMU_FQCSR_FQOF ||
fqcsr & RISCV_IOMMU_FQCSR_FQMF)) {
ipsr_set |= RISCV_IOMMU_IPSR_FIP;
} else {
ipsr_clr |= RISCV_IOMMU_IPSR_FIP;
}
} else {
ipsr_clr |= RISCV_IOMMU_IPSR_FIP;
}
if (data & RISCV_IOMMU_IPSR_PIP) {
pqcsr = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_PQCSR);
if (pqcsr & RISCV_IOMMU_PQCSR_PIE &&
(pqcsr & RISCV_IOMMU_PQCSR_PQOF ||
pqcsr & RISCV_IOMMU_PQCSR_PQMF)) {
ipsr_set |= RISCV_IOMMU_IPSR_PIP;
} else {
ipsr_clr |= RISCV_IOMMU_IPSR_PIP;
}
} else {
ipsr_clr |= RISCV_IOMMU_IPSR_PIP;
}
riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_IPSR, ipsr_set, ipsr_clr);
}
/*
* Write the resulting value of 'data' for the reg specified
* by 'reg_addr', after considering read-only/read-write/write-clear
* bits, in the pointer 'dest'.
*
* The result is written in little-endian.
*/
static void riscv_iommu_write_reg_val(RISCVIOMMUState *s,
void *dest, hwaddr reg_addr,
int size, uint64_t data)
{
uint64_t ro = ldn_le_p(&s->regs_ro[reg_addr], size);
uint64_t wc = ldn_le_p(&s->regs_wc[reg_addr], size);
uint64_t rw = ldn_le_p(&s->regs_rw[reg_addr], size);
stn_le_p(dest, size, ((rw & ro) | (data & ~ro)) & ~(data & wc));
}
static MemTxResult riscv_iommu_mmio_write(void *opaque, hwaddr addr,
uint64_t data, unsigned size,
MemTxAttrs attrs)
{
riscv_iommu_process_fn *process_fn = NULL;
RISCVIOMMUState *s = opaque;
uint32_t regb = addr & ~3;
uint32_t busy = 0;
uint64_t val = 0;
if ((addr & (size - 1)) != 0) {
/* Unsupported MMIO alignment or access size */
return MEMTX_ERROR;
}
if (addr + size > RISCV_IOMMU_REG_MSI_CONFIG) {
/* Unsupported MMIO access location. */
return MEMTX_ACCESS_ERROR;
}
/* Track actionable MMIO write. */
switch (regb) {
case RISCV_IOMMU_REG_DDTP:
case RISCV_IOMMU_REG_DDTP + 4:
process_fn = riscv_iommu_process_ddtp;
regb = RISCV_IOMMU_REG_DDTP;
busy = RISCV_IOMMU_DDTP_BUSY;
break;
case RISCV_IOMMU_REG_CQT:
process_fn = riscv_iommu_process_cq_tail;
break;
case RISCV_IOMMU_REG_CQCSR:
process_fn = riscv_iommu_process_cq_control;
busy = RISCV_IOMMU_CQCSR_BUSY;
break;
case RISCV_IOMMU_REG_FQCSR:
process_fn = riscv_iommu_process_fq_control;
busy = RISCV_IOMMU_FQCSR_BUSY;
break;
case RISCV_IOMMU_REG_PQCSR:
process_fn = riscv_iommu_process_pq_control;
busy = RISCV_IOMMU_PQCSR_BUSY;
break;
case RISCV_IOMMU_REG_ICVEC:
case RISCV_IOMMU_REG_IPSR:
/*
* ICVEC and IPSR have special read/write procedures. We'll
* call their respective helpers and exit.
*/
riscv_iommu_write_reg_val(s, &val, addr, size, data);
/*
* 'val' is stored as LE. Switch to host endianess
* before using it.
*/
val = le64_to_cpu(val);
if (regb == RISCV_IOMMU_REG_ICVEC) {
riscv_iommu_update_icvec(s, val);
} else {
riscv_iommu_update_ipsr(s, val);
}
return MEMTX_OK;
case RISCV_IOMMU_REG_TR_REQ_CTL:
process_fn = riscv_iommu_process_dbg;
regb = RISCV_IOMMU_REG_TR_REQ_CTL;
busy = RISCV_IOMMU_TR_REQ_CTL_GO_BUSY;
break;
default:
break;
}
/*
* Registers update might be not synchronized with core logic.
* If system software updates register when relevant BUSY bit
* is set IOMMU behavior of additional writes to the register
* is UNSPECIFIED.
*/
riscv_iommu_write_reg_val(s, &s->regs_rw[addr], addr, size, data);
/* Busy flag update, MSB 4-byte register. */
if (busy) {
uint32_t rw = ldl_le_p(&s->regs_rw[regb]);
stl_le_p(&s->regs_rw[regb], rw | busy);
}
if (process_fn) {
process_fn(s);
}
return MEMTX_OK;
}
static MemTxResult riscv_iommu_mmio_read(void *opaque, hwaddr addr,
uint64_t *data, unsigned size, MemTxAttrs attrs)
{
RISCVIOMMUState *s = opaque;
uint64_t val = -1;
uint8_t *ptr;
if ((addr & (size - 1)) != 0) {
/* Unsupported MMIO alignment. */
return MEMTX_ERROR;
}
if (addr + size > RISCV_IOMMU_REG_MSI_CONFIG) {
return MEMTX_ACCESS_ERROR;
}
ptr = &s->regs_rw[addr];
val = ldn_le_p(ptr, size);
*data = val;
return MEMTX_OK;
}
static const MemoryRegionOps riscv_iommu_mmio_ops = {
.read_with_attrs = riscv_iommu_mmio_read,
.write_with_attrs = riscv_iommu_mmio_write,
.endianness = DEVICE_NATIVE_ENDIAN,
.impl = {
.min_access_size = 4,
.max_access_size = 8,
.unaligned = false,
},
.valid = {
.min_access_size = 4,
.max_access_size = 8,
}
};
/*
* Translations matching MSI pattern check are redirected to "riscv-iommu-trap"
* memory region as untranslated address, for additional MSI/MRIF interception
* by IOMMU interrupt remapping implementation.
* Note: Device emulation code generating an MSI is expected to provide a valid
* memory transaction attributes with requested_id set.
*/
static MemTxResult riscv_iommu_trap_write(void *opaque, hwaddr addr,
uint64_t data, unsigned size, MemTxAttrs attrs)
{
RISCVIOMMUState* s = (RISCVIOMMUState *)opaque;
RISCVIOMMUContext *ctx;
MemTxResult res;
void *ref;
uint32_t devid = attrs.requester_id;
if (attrs.unspecified) {
return MEMTX_ACCESS_ERROR;
}
/* FIXME: PCIe bus remapping for attached endpoints. */
devid |= s->bus << 8;
ctx = riscv_iommu_ctx(s, devid, 0, &ref);
if (ctx == NULL) {
res = MEMTX_ACCESS_ERROR;
} else {
res = riscv_iommu_msi_write(s, ctx, addr, data, size, attrs);
}
riscv_iommu_ctx_put(s, ref);
return res;
}
static MemTxResult riscv_iommu_trap_read(void *opaque, hwaddr addr,
uint64_t *data, unsigned size, MemTxAttrs attrs)
{
return MEMTX_ACCESS_ERROR;
}
static const MemoryRegionOps riscv_iommu_trap_ops = {
.read_with_attrs = riscv_iommu_trap_read,
.write_with_attrs = riscv_iommu_trap_write,
.endianness = DEVICE_LITTLE_ENDIAN,
.impl = {
.min_access_size = 4,
.max_access_size = 8,
.unaligned = true,
},
.valid = {
.min_access_size = 4,
.max_access_size = 8,
}
};
static void riscv_iommu_realize(DeviceState *dev, Error **errp)
{
RISCVIOMMUState *s = RISCV_IOMMU(dev);
s->cap = s->version & RISCV_IOMMU_CAP_VERSION;
if (s->enable_msi) {
s->cap |= RISCV_IOMMU_CAP_MSI_FLAT | RISCV_IOMMU_CAP_MSI_MRIF;
}
if (s->enable_ats) {
s->cap |= RISCV_IOMMU_CAP_ATS;
}
if (s->enable_s_stage) {
s->cap |= RISCV_IOMMU_CAP_SV32 | RISCV_IOMMU_CAP_SV39 |
RISCV_IOMMU_CAP_SV48 | RISCV_IOMMU_CAP_SV57;
}
if (s->enable_g_stage) {
s->cap |= RISCV_IOMMU_CAP_SV32X4 | RISCV_IOMMU_CAP_SV39X4 |
RISCV_IOMMU_CAP_SV48X4 | RISCV_IOMMU_CAP_SV57X4;
}
/* Enable translation debug interface */
s->cap |= RISCV_IOMMU_CAP_DBG;
/* Report QEMU target physical address space limits */
s->cap = set_field(s->cap, RISCV_IOMMU_CAP_PAS,
TARGET_PHYS_ADDR_SPACE_BITS);
/* TODO: method to report supported PID bits */
s->pid_bits = 8; /* restricted to size of MemTxAttrs.pid */
s->cap |= RISCV_IOMMU_CAP_PD8;
/* Out-of-reset translation mode: OFF (DMA disabled) BARE (passthrough) */
s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, s->enable_off ?
RISCV_IOMMU_DDTP_MODE_OFF : RISCV_IOMMU_DDTP_MODE_BARE);
/* register storage */
s->regs_rw = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
s->regs_ro = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
s->regs_wc = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
/* Mark all registers read-only */
memset(s->regs_ro, 0xff, RISCV_IOMMU_REG_SIZE);
/*
* Register complete MMIO space, including MSI/PBA registers.
* Note, PCIDevice implementation will add overlapping MR for MSI/PBA,
* managed directly by the PCIDevice implementation.
*/
memory_region_init_io(&s->regs_mr, OBJECT(dev), &riscv_iommu_mmio_ops, s,
"riscv-iommu-regs", RISCV_IOMMU_REG_SIZE);
/* Set power-on register state */
stq_le_p(&s->regs_rw[RISCV_IOMMU_REG_CAP], s->cap);
stq_le_p(&s->regs_rw[RISCV_IOMMU_REG_FCTL], 0);
stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_FCTL],
~(RISCV_IOMMU_FCTL_BE | RISCV_IOMMU_FCTL_WSI));
stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_DDTP],
~(RISCV_IOMMU_DDTP_PPN | RISCV_IOMMU_DDTP_MODE));
stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQB],
~(RISCV_IOMMU_CQB_LOG2SZ | RISCV_IOMMU_CQB_PPN));
stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQB],
~(RISCV_IOMMU_FQB_LOG2SZ | RISCV_IOMMU_FQB_PPN));
stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQB],
~(RISCV_IOMMU_PQB_LOG2SZ | RISCV_IOMMU_PQB_PPN));
stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_CQCSR], RISCV_IOMMU_CQCSR_CQMF |
RISCV_IOMMU_CQCSR_CMD_TO | RISCV_IOMMU_CQCSR_CMD_ILL);
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_CQCSR], RISCV_IOMMU_CQCSR_CQON |
RISCV_IOMMU_CQCSR_BUSY);
stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_FQCSR], RISCV_IOMMU_FQCSR_FQMF |
RISCV_IOMMU_FQCSR_FQOF);
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_FQCSR], RISCV_IOMMU_FQCSR_FQON |
RISCV_IOMMU_FQCSR_BUSY);
stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_PQCSR], RISCV_IOMMU_PQCSR_PQMF |
RISCV_IOMMU_PQCSR_PQOF);
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_PQCSR], RISCV_IOMMU_PQCSR_PQON |
RISCV_IOMMU_PQCSR_BUSY);
stl_le_p(&s->regs_wc[RISCV_IOMMU_REG_IPSR], ~0);
stl_le_p(&s->regs_ro[RISCV_IOMMU_REG_ICVEC], 0);
stq_le_p(&s->regs_rw[RISCV_IOMMU_REG_DDTP], s->ddtp);
/* If debug registers enabled. */
if (s->cap & RISCV_IOMMU_CAP_DBG) {
stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_TR_REQ_IOVA], 0);
stq_le_p(&s->regs_ro[RISCV_IOMMU_REG_TR_REQ_CTL],
RISCV_IOMMU_TR_REQ_CTL_GO_BUSY);
}
/* Memory region for downstream access, if specified. */
if (s->target_mr) {
s->target_as = g_new0(AddressSpace, 1);
address_space_init(s->target_as, s->target_mr,
"riscv-iommu-downstream");
} else {
/* Fallback to global system memory. */
s->target_as = &address_space_memory;
}
/* Memory region for untranslated MRIF/MSI writes */
memory_region_init_io(&s->trap_mr, OBJECT(dev), &riscv_iommu_trap_ops, s,
"riscv-iommu-trap", ~0ULL);
address_space_init(&s->trap_as, &s->trap_mr, "riscv-iommu-trap-as");
/* Device translation context cache */
s->ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash,
riscv_iommu_ctx_equal,
g_free, NULL);
s->iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash,
riscv_iommu_iot_equal,
g_free, NULL);
s->iommus.le_next = NULL;
s->iommus.le_prev = NULL;
QLIST_INIT(&s->spaces);
}
static void riscv_iommu_unrealize(DeviceState *dev)
{
RISCVIOMMUState *s = RISCV_IOMMU(dev);
g_hash_table_unref(s->iot_cache);
g_hash_table_unref(s->ctx_cache);
}
static Property riscv_iommu_properties[] = {
DEFINE_PROP_UINT32("version", RISCVIOMMUState, version,
RISCV_IOMMU_SPEC_DOT_VER),
DEFINE_PROP_UINT32("bus", RISCVIOMMUState, bus, 0x0),
DEFINE_PROP_UINT32("ioatc-limit", RISCVIOMMUState, iot_limit,
LIMIT_CACHE_IOT),
DEFINE_PROP_BOOL("intremap", RISCVIOMMUState, enable_msi, TRUE),
DEFINE_PROP_BOOL("ats", RISCVIOMMUState, enable_ats, TRUE),
DEFINE_PROP_BOOL("off", RISCVIOMMUState, enable_off, TRUE),
DEFINE_PROP_BOOL("s-stage", RISCVIOMMUState, enable_s_stage, TRUE),
DEFINE_PROP_BOOL("g-stage", RISCVIOMMUState, enable_g_stage, TRUE),
DEFINE_PROP_LINK("downstream-mr", RISCVIOMMUState, target_mr,
TYPE_MEMORY_REGION, MemoryRegion *),
DEFINE_PROP_END_OF_LIST(),
};
static void riscv_iommu_class_init(ObjectClass *klass, void* data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
/* internal device for riscv-iommu-{pci/sys}, not user-creatable */
dc->user_creatable = false;
dc->realize = riscv_iommu_realize;
dc->unrealize = riscv_iommu_unrealize;
device_class_set_props(dc, riscv_iommu_properties);
}
static const TypeInfo riscv_iommu_info = {
.name = TYPE_RISCV_IOMMU,
.parent = TYPE_DEVICE,
.instance_size = sizeof(RISCVIOMMUState),
.class_init = riscv_iommu_class_init,
};
static const char *IOMMU_FLAG_STR[] = {
"NA",
"RO",
"WR",
"RW",
};
/* RISC-V IOMMU Memory Region - Address Translation Space */
static IOMMUTLBEntry riscv_iommu_memory_region_translate(
IOMMUMemoryRegion *iommu_mr, hwaddr addr,
IOMMUAccessFlags flag, int iommu_idx)
{
RISCVIOMMUSpace *as = container_of(iommu_mr, RISCVIOMMUSpace, iova_mr);
RISCVIOMMUContext *ctx;
void *ref;
IOMMUTLBEntry iotlb = {
.iova = addr,
.target_as = as->iommu->target_as,
.addr_mask = ~0ULL,
.perm = flag,
};
ctx = riscv_iommu_ctx(as->iommu, as->devid, iommu_idx, &ref);
if (ctx == NULL) {
/* Translation disabled or invalid. */
iotlb.addr_mask = 0;
iotlb.perm = IOMMU_NONE;
} else if (riscv_iommu_translate(as->iommu, ctx, &iotlb, true)) {
/* Translation disabled or fault reported. */
iotlb.addr_mask = 0;
iotlb.perm = IOMMU_NONE;
}
/* Trace all dma translations with original access flags. */
trace_riscv_iommu_dma(as->iommu->parent_obj.id, PCI_BUS_NUM(as->devid),
PCI_SLOT(as->devid), PCI_FUNC(as->devid), iommu_idx,
IOMMU_FLAG_STR[flag & IOMMU_RW], iotlb.iova,
iotlb.translated_addr);
riscv_iommu_ctx_put(as->iommu, ref);
return iotlb;
}
static int riscv_iommu_memory_region_notify(
IOMMUMemoryRegion *iommu_mr, IOMMUNotifierFlag old,
IOMMUNotifierFlag new, Error **errp)
{
RISCVIOMMUSpace *as = container_of(iommu_mr, RISCVIOMMUSpace, iova_mr);
if (old == IOMMU_NOTIFIER_NONE) {
as->notifier = true;
trace_riscv_iommu_notifier_add(iommu_mr->parent_obj.name);
} else if (new == IOMMU_NOTIFIER_NONE) {
as->notifier = false;
trace_riscv_iommu_notifier_del(iommu_mr->parent_obj.name);
}
return 0;
}
static inline bool pci_is_iommu(PCIDevice *pdev)
{
return pci_get_word(pdev->config + PCI_CLASS_DEVICE) == 0x0806;
}
static AddressSpace *riscv_iommu_find_as(PCIBus *bus, void *opaque, int devfn)
{
RISCVIOMMUState *s = (RISCVIOMMUState *) opaque;
PCIDevice *pdev = pci_find_device(bus, pci_bus_num(bus), devfn);
AddressSpace *as = NULL;
if (pdev && pci_is_iommu(pdev)) {
return s->target_as;
}
/* Find first registered IOMMU device */
while (s->iommus.le_prev) {
s = *(s->iommus.le_prev);
}
/* Find first matching IOMMU */
while (s != NULL && as == NULL) {
as = riscv_iommu_space(s, PCI_BUILD_BDF(pci_bus_num(bus), devfn));
s = s->iommus.le_next;
}
return as ? as : &address_space_memory;
}
static const PCIIOMMUOps riscv_iommu_ops = {
.get_address_space = riscv_iommu_find_as,
};
void riscv_iommu_pci_setup_iommu(RISCVIOMMUState *iommu, PCIBus *bus,
Error **errp)
{
if (bus->iommu_ops &&
bus->iommu_ops->get_address_space == riscv_iommu_find_as) {
/* Allow multiple IOMMUs on the same PCIe bus, link known devices */
RISCVIOMMUState *last = (RISCVIOMMUState *)bus->iommu_opaque;
QLIST_INSERT_AFTER(last, iommu, iommus);
} else if (!bus->iommu_ops && !bus->iommu_opaque) {
pci_setup_iommu(bus, &riscv_iommu_ops, iommu);
} else {
error_setg(errp, "can't register secondary IOMMU for PCI bus #%d",
pci_bus_num(bus));
}
}
static int riscv_iommu_memory_region_index(IOMMUMemoryRegion *iommu_mr,
MemTxAttrs attrs)
{
return attrs.unspecified ? RISCV_IOMMU_NOPROCID : (int)attrs.pid;
}
static int riscv_iommu_memory_region_index_len(IOMMUMemoryRegion *iommu_mr)
{
RISCVIOMMUSpace *as = container_of(iommu_mr, RISCVIOMMUSpace, iova_mr);
return 1 << as->iommu->pid_bits;
}
static void riscv_iommu_memory_region_init(ObjectClass *klass, void *data)
{
IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
imrc->translate = riscv_iommu_memory_region_translate;
imrc->notify_flag_changed = riscv_iommu_memory_region_notify;
imrc->attrs_to_index = riscv_iommu_memory_region_index;
imrc->num_indexes = riscv_iommu_memory_region_index_len;
}
static const TypeInfo riscv_iommu_memory_region_info = {
.parent = TYPE_IOMMU_MEMORY_REGION,
.name = TYPE_RISCV_IOMMU_MEMORY_REGION,
.class_init = riscv_iommu_memory_region_init,
};
static void riscv_iommu_register_mr_types(void)
{
type_register_static(&riscv_iommu_memory_region_info);
type_register_static(&riscv_iommu_info);
}
type_init(riscv_iommu_register_mr_types);