11da12ec4SLe Tan /* 21da12ec4SLe Tan * QEMU emulation of an Intel IOMMU (VT-d) 31da12ec4SLe Tan * (DMA Remapping device) 41da12ec4SLe Tan * 51da12ec4SLe Tan * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 61da12ec4SLe Tan * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 71da12ec4SLe Tan * 81da12ec4SLe Tan * This program is free software; you can redistribute it and/or modify 91da12ec4SLe Tan * it under the terms of the GNU General Public License as published by 101da12ec4SLe Tan * the Free Software Foundation; either version 2 of the License, or 111da12ec4SLe Tan * (at your option) any later version. 121da12ec4SLe Tan 131da12ec4SLe Tan * This program is distributed in the hope that it will be useful, 141da12ec4SLe Tan * but WITHOUT ANY WARRANTY; without even the implied warranty of 151da12ec4SLe Tan * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 161da12ec4SLe Tan * GNU General Public License for more details. 171da12ec4SLe Tan 181da12ec4SLe Tan * You should have received a copy of the GNU General Public License along 191da12ec4SLe Tan * with this program; if not, see <http://www.gnu.org/licenses/>. 201da12ec4SLe Tan */ 211da12ec4SLe Tan 22b6a0aa05SPeter Maydell #include "qemu/osdep.h" 234684a204SPeter Xu #include "qemu/error-report.h" 24db725815SMarkus Armbruster #include "qemu/main-loop.h" 256333e93cSRadim Krčmář #include "qapi/error.h" 261da12ec4SLe Tan #include "hw/sysbus.h" 271da12ec4SLe Tan #include "exec/address-spaces.h" 281da12ec4SLe Tan #include "intel_iommu_internal.h" 297df953bdSKnut Omang #include "hw/pci/pci.h" 303cb3b154SAlex Williamson #include "hw/pci/pci_bus.h" 31a27bd6c7SMarkus Armbruster #include "hw/qdev-properties.h" 32621d983aSMarcel Apfelbaum #include "hw/i386/pc.h" 33dea651a9SFeng Wu #include "hw/i386/apic-msidef.h" 3404af0e18SPeter Xu #include "hw/boards.h" 3504af0e18SPeter Xu #include "hw/i386/x86-iommu.h" 36cb135f59SPeter Xu #include "hw/pci-host/q35.h" 374684a204SPeter Xu #include "sysemu/kvm.h" 3828cf553aSPeter Xu #include "sysemu/sysemu.h" 3932946019SRadim Krčmář #include "hw/i386/apic_internal.h" 40fb506e70SRadim Krčmář #include "kvm_i386.h" 41d6454270SMarkus Armbruster #include "migration/vmstate.h" 42bc535e59SPeter Xu #include "trace.h" 431da12ec4SLe Tan 44fb43cf73SLiu, Yi L /* context entry operations */ 45fb43cf73SLiu, Yi L #define VTD_CE_GET_RID2PASID(ce) \ 46fb43cf73SLiu, Yi L ((ce)->val[1] & VTD_SM_CONTEXT_ENTRY_RID2PASID_MASK) 47fb43cf73SLiu, Yi L #define VTD_CE_GET_PASID_DIR_TABLE(ce) \ 48fb43cf73SLiu, Yi L ((ce)->val[0] & VTD_PASID_DIR_BASE_ADDR_MASK) 49fb43cf73SLiu, Yi L 50fb43cf73SLiu, Yi L /* pe operations */ 51fb43cf73SLiu, Yi L #define VTD_PE_GET_TYPE(pe) ((pe)->val[0] & VTD_SM_PASID_ENTRY_PGTT) 52fb43cf73SLiu, Yi L #define VTD_PE_GET_LEVEL(pe) (2 + (((pe)->val[0] >> 2) & VTD_SM_PASID_ENTRY_AW)) 53fb43cf73SLiu, Yi L #define VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write) {\ 54fb43cf73SLiu, Yi L if (ret_fr) { \ 55fb43cf73SLiu, Yi L ret_fr = -ret_fr; \ 56fb43cf73SLiu, Yi L if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { \ 57fb43cf73SLiu, Yi L trace_vtd_fault_disabled(); \ 58fb43cf73SLiu, Yi L } else { \ 59fb43cf73SLiu, Yi L vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); \ 60fb43cf73SLiu, Yi L } \ 61fb43cf73SLiu, Yi L goto error; \ 62fb43cf73SLiu, Yi L } \ 63fb43cf73SLiu, Yi L } 64fb43cf73SLiu, Yi L 652cc9ddccSPeter Xu static void vtd_address_space_refresh_all(IntelIOMMUState *s); 66c28b535dSPeter Xu static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n); 672cc9ddccSPeter Xu 6828cf553aSPeter Xu static void vtd_panic_require_caching_mode(void) 6928cf553aSPeter Xu { 7028cf553aSPeter Xu error_report("We need to set caching-mode=on for intel-iommu to enable " 7128cf553aSPeter Xu "device assignment with IOMMU protection."); 7228cf553aSPeter Xu exit(1); 7328cf553aSPeter Xu } 7428cf553aSPeter Xu 751da12ec4SLe Tan static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 761da12ec4SLe Tan uint64_t wmask, uint64_t w1cmask) 771da12ec4SLe Tan { 781da12ec4SLe Tan stq_le_p(&s->csr[addr], val); 791da12ec4SLe Tan stq_le_p(&s->wmask[addr], wmask); 801da12ec4SLe Tan stq_le_p(&s->w1cmask[addr], w1cmask); 811da12ec4SLe Tan } 821da12ec4SLe Tan 831da12ec4SLe Tan static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 841da12ec4SLe Tan { 851da12ec4SLe Tan stq_le_p(&s->womask[addr], mask); 861da12ec4SLe Tan } 871da12ec4SLe Tan 881da12ec4SLe Tan static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 891da12ec4SLe Tan uint32_t wmask, uint32_t w1cmask) 901da12ec4SLe Tan { 911da12ec4SLe Tan stl_le_p(&s->csr[addr], val); 921da12ec4SLe Tan stl_le_p(&s->wmask[addr], wmask); 931da12ec4SLe Tan stl_le_p(&s->w1cmask[addr], w1cmask); 941da12ec4SLe Tan } 951da12ec4SLe Tan 961da12ec4SLe Tan static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 971da12ec4SLe Tan { 981da12ec4SLe Tan stl_le_p(&s->womask[addr], mask); 991da12ec4SLe Tan } 1001da12ec4SLe Tan 1011da12ec4SLe Tan /* "External" get/set operations */ 1021da12ec4SLe Tan static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 1031da12ec4SLe Tan { 1041da12ec4SLe Tan uint64_t oldval = ldq_le_p(&s->csr[addr]); 1051da12ec4SLe Tan uint64_t wmask = ldq_le_p(&s->wmask[addr]); 1061da12ec4SLe Tan uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 1071da12ec4SLe Tan stq_le_p(&s->csr[addr], 1081da12ec4SLe Tan ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 1091da12ec4SLe Tan } 1101da12ec4SLe Tan 1111da12ec4SLe Tan static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 1121da12ec4SLe Tan { 1131da12ec4SLe Tan uint32_t oldval = ldl_le_p(&s->csr[addr]); 1141da12ec4SLe Tan uint32_t wmask = ldl_le_p(&s->wmask[addr]); 1151da12ec4SLe Tan uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 1161da12ec4SLe Tan stl_le_p(&s->csr[addr], 1171da12ec4SLe Tan ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 1181da12ec4SLe Tan } 1191da12ec4SLe Tan 1201da12ec4SLe Tan static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 1211da12ec4SLe Tan { 1221da12ec4SLe Tan uint64_t val = ldq_le_p(&s->csr[addr]); 1231da12ec4SLe Tan uint64_t womask = ldq_le_p(&s->womask[addr]); 1241da12ec4SLe Tan return val & ~womask; 1251da12ec4SLe Tan } 1261da12ec4SLe Tan 1271da12ec4SLe Tan static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 1281da12ec4SLe Tan { 1291da12ec4SLe Tan uint32_t val = ldl_le_p(&s->csr[addr]); 1301da12ec4SLe Tan uint32_t womask = ldl_le_p(&s->womask[addr]); 1311da12ec4SLe Tan return val & ~womask; 1321da12ec4SLe Tan } 1331da12ec4SLe Tan 1341da12ec4SLe Tan /* "Internal" get/set operations */ 1351da12ec4SLe Tan static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 1361da12ec4SLe Tan { 1371da12ec4SLe Tan return ldq_le_p(&s->csr[addr]); 1381da12ec4SLe Tan } 1391da12ec4SLe Tan 1401da12ec4SLe Tan static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 1411da12ec4SLe Tan { 1421da12ec4SLe Tan return ldl_le_p(&s->csr[addr]); 1431da12ec4SLe Tan } 1441da12ec4SLe Tan 1451da12ec4SLe Tan static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 1461da12ec4SLe Tan { 1471da12ec4SLe Tan stq_le_p(&s->csr[addr], val); 1481da12ec4SLe Tan } 1491da12ec4SLe Tan 1501da12ec4SLe Tan static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 1511da12ec4SLe Tan uint32_t clear, uint32_t mask) 1521da12ec4SLe Tan { 1531da12ec4SLe Tan uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 1541da12ec4SLe Tan stl_le_p(&s->csr[addr], new_val); 1551da12ec4SLe Tan return new_val; 1561da12ec4SLe Tan } 1571da12ec4SLe Tan 1581da12ec4SLe Tan static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 1591da12ec4SLe Tan uint64_t clear, uint64_t mask) 1601da12ec4SLe Tan { 1611da12ec4SLe Tan uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 1621da12ec4SLe Tan stq_le_p(&s->csr[addr], new_val); 1631da12ec4SLe Tan return new_val; 1641da12ec4SLe Tan } 1651da12ec4SLe Tan 1661d9efa73SPeter Xu static inline void vtd_iommu_lock(IntelIOMMUState *s) 1671d9efa73SPeter Xu { 1681d9efa73SPeter Xu qemu_mutex_lock(&s->iommu_lock); 1691d9efa73SPeter Xu } 1701d9efa73SPeter Xu 1711d9efa73SPeter Xu static inline void vtd_iommu_unlock(IntelIOMMUState *s) 1721d9efa73SPeter Xu { 1731d9efa73SPeter Xu qemu_mutex_unlock(&s->iommu_lock); 1741d9efa73SPeter Xu } 1751d9efa73SPeter Xu 1762811af3bSPeter Xu static void vtd_update_scalable_state(IntelIOMMUState *s) 1772811af3bSPeter Xu { 1782811af3bSPeter Xu uint64_t val = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 1792811af3bSPeter Xu 1802811af3bSPeter Xu if (s->scalable_mode) { 1812811af3bSPeter Xu s->root_scalable = val & VTD_RTADDR_SMT; 1822811af3bSPeter Xu } 1832811af3bSPeter Xu } 1842811af3bSPeter Xu 1854f8a62a9SPeter Xu /* Whether the address space needs to notify new mappings */ 1864f8a62a9SPeter Xu static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as) 1874f8a62a9SPeter Xu { 1884f8a62a9SPeter Xu return as->notifier_flags & IOMMU_NOTIFIER_MAP; 1894f8a62a9SPeter Xu } 1904f8a62a9SPeter Xu 191b5a280c0SLe Tan /* GHashTable functions */ 192b5a280c0SLe Tan static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2) 193b5a280c0SLe Tan { 194b5a280c0SLe Tan return *((const uint64_t *)v1) == *((const uint64_t *)v2); 195b5a280c0SLe Tan } 196b5a280c0SLe Tan 197b5a280c0SLe Tan static guint vtd_uint64_hash(gconstpointer v) 198b5a280c0SLe Tan { 199b5a280c0SLe Tan return (guint)*(const uint64_t *)v; 200b5a280c0SLe Tan } 201b5a280c0SLe Tan 202b5a280c0SLe Tan static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 203b5a280c0SLe Tan gpointer user_data) 204b5a280c0SLe Tan { 205b5a280c0SLe Tan VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 206b5a280c0SLe Tan uint16_t domain_id = *(uint16_t *)user_data; 207b5a280c0SLe Tan return entry->domain_id == domain_id; 208b5a280c0SLe Tan } 209b5a280c0SLe Tan 210d66b969bSJason Wang /* The shift of an addr for a certain level of paging structure */ 211d66b969bSJason Wang static inline uint32_t vtd_slpt_level_shift(uint32_t level) 212d66b969bSJason Wang { 2137e58326aSPeter Xu assert(level != 0); 214d66b969bSJason Wang return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 215d66b969bSJason Wang } 216d66b969bSJason Wang 217d66b969bSJason Wang static inline uint64_t vtd_slpt_level_page_mask(uint32_t level) 218d66b969bSJason Wang { 219d66b969bSJason Wang return ~((1ULL << vtd_slpt_level_shift(level)) - 1); 220d66b969bSJason Wang } 221d66b969bSJason Wang 222b5a280c0SLe Tan static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 223b5a280c0SLe Tan gpointer user_data) 224b5a280c0SLe Tan { 225b5a280c0SLe Tan VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 226b5a280c0SLe Tan VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 227d66b969bSJason Wang uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask; 228d66b969bSJason Wang uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K; 229b5a280c0SLe Tan return (entry->domain_id == info->domain_id) && 230d66b969bSJason Wang (((entry->gfn & info->mask) == gfn) || 231d66b969bSJason Wang (entry->gfn == gfn_tlb)); 232b5a280c0SLe Tan } 233b5a280c0SLe Tan 234d92fa2dcSLe Tan /* Reset all the gen of VTDAddressSpace to zero and set the gen of 2351d9efa73SPeter Xu * IntelIOMMUState to 1. Must be called with IOMMU lock held. 236d92fa2dcSLe Tan */ 2371d9efa73SPeter Xu static void vtd_reset_context_cache_locked(IntelIOMMUState *s) 238d92fa2dcSLe Tan { 239d92fa2dcSLe Tan VTDAddressSpace *vtd_as; 2407df953bdSKnut Omang VTDBus *vtd_bus; 2417df953bdSKnut Omang GHashTableIter bus_it; 242d92fa2dcSLe Tan uint32_t devfn_it; 243d92fa2dcSLe Tan 2447feb51b7SPeter Xu trace_vtd_context_cache_reset(); 2457feb51b7SPeter Xu 2467df953bdSKnut Omang g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr); 2477df953bdSKnut Omang 2487df953bdSKnut Omang while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) { 249bf33cc75SPeter Xu for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) { 2507df953bdSKnut Omang vtd_as = vtd_bus->dev_as[devfn_it]; 251d92fa2dcSLe Tan if (!vtd_as) { 252d92fa2dcSLe Tan continue; 253d92fa2dcSLe Tan } 254d92fa2dcSLe Tan vtd_as->context_cache_entry.context_cache_gen = 0; 255d92fa2dcSLe Tan } 256d92fa2dcSLe Tan } 257d92fa2dcSLe Tan s->context_cache_gen = 1; 258d92fa2dcSLe Tan } 259d92fa2dcSLe Tan 2601d9efa73SPeter Xu /* Must be called with IOMMU lock held. */ 2611d9efa73SPeter Xu static void vtd_reset_iotlb_locked(IntelIOMMUState *s) 262b5a280c0SLe Tan { 263b5a280c0SLe Tan assert(s->iotlb); 264b5a280c0SLe Tan g_hash_table_remove_all(s->iotlb); 265b5a280c0SLe Tan } 266b5a280c0SLe Tan 2671d9efa73SPeter Xu static void vtd_reset_iotlb(IntelIOMMUState *s) 2681d9efa73SPeter Xu { 2691d9efa73SPeter Xu vtd_iommu_lock(s); 2701d9efa73SPeter Xu vtd_reset_iotlb_locked(s); 2711d9efa73SPeter Xu vtd_iommu_unlock(s); 2721d9efa73SPeter Xu } 2731d9efa73SPeter Xu 27406aba4caSPeter Xu static void vtd_reset_caches(IntelIOMMUState *s) 27506aba4caSPeter Xu { 27606aba4caSPeter Xu vtd_iommu_lock(s); 27706aba4caSPeter Xu vtd_reset_iotlb_locked(s); 27806aba4caSPeter Xu vtd_reset_context_cache_locked(s); 27906aba4caSPeter Xu vtd_iommu_unlock(s); 28006aba4caSPeter Xu } 28106aba4caSPeter Xu 282bacabb0aSJason Wang static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint16_t source_id, 283d66b969bSJason Wang uint32_t level) 284d66b969bSJason Wang { 285d66b969bSJason Wang return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) | 286d66b969bSJason Wang ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT); 287d66b969bSJason Wang } 288d66b969bSJason Wang 289d66b969bSJason Wang static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) 290d66b969bSJason Wang { 291d66b969bSJason Wang return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K; 292d66b969bSJason Wang } 293d66b969bSJason Wang 2941d9efa73SPeter Xu /* Must be called with IOMMU lock held */ 295b5a280c0SLe Tan static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 296b5a280c0SLe Tan hwaddr addr) 297b5a280c0SLe Tan { 298d66b969bSJason Wang VTDIOTLBEntry *entry; 299b5a280c0SLe Tan uint64_t key; 300d66b969bSJason Wang int level; 301b5a280c0SLe Tan 302d66b969bSJason Wang for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) { 303d66b969bSJason Wang key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level), 304d66b969bSJason Wang source_id, level); 305d66b969bSJason Wang entry = g_hash_table_lookup(s->iotlb, &key); 306d66b969bSJason Wang if (entry) { 307d66b969bSJason Wang goto out; 308d66b969bSJason Wang } 309d66b969bSJason Wang } 310b5a280c0SLe Tan 311d66b969bSJason Wang out: 312d66b969bSJason Wang return entry; 313b5a280c0SLe Tan } 314b5a280c0SLe Tan 3151d9efa73SPeter Xu /* Must be with IOMMU lock held */ 316b5a280c0SLe Tan static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 317b5a280c0SLe Tan uint16_t domain_id, hwaddr addr, uint64_t slpte, 31807f7b733SPeter Xu uint8_t access_flags, uint32_t level) 319b5a280c0SLe Tan { 320b5a280c0SLe Tan VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 321b5a280c0SLe Tan uint64_t *key = g_malloc(sizeof(*key)); 322d66b969bSJason Wang uint64_t gfn = vtd_get_iotlb_gfn(addr, level); 323b5a280c0SLe Tan 3246c441e1dSPeter Xu trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id); 325b5a280c0SLe Tan if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 3266c441e1dSPeter Xu trace_vtd_iotlb_reset("iotlb exceeds size limit"); 3271d9efa73SPeter Xu vtd_reset_iotlb_locked(s); 328b5a280c0SLe Tan } 329b5a280c0SLe Tan 330b5a280c0SLe Tan entry->gfn = gfn; 331b5a280c0SLe Tan entry->domain_id = domain_id; 332b5a280c0SLe Tan entry->slpte = slpte; 33307f7b733SPeter Xu entry->access_flags = access_flags; 334d66b969bSJason Wang entry->mask = vtd_slpt_level_page_mask(level); 335d66b969bSJason Wang *key = vtd_get_iotlb_key(gfn, source_id, level); 336b5a280c0SLe Tan g_hash_table_replace(s->iotlb, key, entry); 337b5a280c0SLe Tan } 338b5a280c0SLe Tan 3391da12ec4SLe Tan /* Given the reg addr of both the message data and address, generate an 3401da12ec4SLe Tan * interrupt via MSI. 3411da12ec4SLe Tan */ 3421da12ec4SLe Tan static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 3431da12ec4SLe Tan hwaddr mesg_data_reg) 3441da12ec4SLe Tan { 34532946019SRadim Krčmář MSIMessage msi; 3461da12ec4SLe Tan 3471da12ec4SLe Tan assert(mesg_data_reg < DMAR_REG_SIZE); 3481da12ec4SLe Tan assert(mesg_addr_reg < DMAR_REG_SIZE); 3491da12ec4SLe Tan 35032946019SRadim Krčmář msi.address = vtd_get_long_raw(s, mesg_addr_reg); 35132946019SRadim Krčmář msi.data = vtd_get_long_raw(s, mesg_data_reg); 3521da12ec4SLe Tan 3537feb51b7SPeter Xu trace_vtd_irq_generate(msi.address, msi.data); 3547feb51b7SPeter Xu 35532946019SRadim Krčmář apic_get_class()->send_msi(&msi); 3561da12ec4SLe Tan } 3571da12ec4SLe Tan 3581da12ec4SLe Tan /* Generate a fault event to software via MSI if conditions are met. 3591da12ec4SLe Tan * Notice that the value of FSTS_REG being passed to it should be the one 3601da12ec4SLe Tan * before any update. 3611da12ec4SLe Tan */ 3621da12ec4SLe Tan static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 3631da12ec4SLe Tan { 3641da12ec4SLe Tan if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 3651da12ec4SLe Tan pre_fsts & VTD_FSTS_IQE) { 3661376211fSPeter Xu error_report_once("There are previous interrupt conditions " 3677feb51b7SPeter Xu "to be serviced by software, fault event " 3681376211fSPeter Xu "is not generated"); 3691da12ec4SLe Tan return; 3701da12ec4SLe Tan } 3711da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 3721da12ec4SLe Tan if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 3731376211fSPeter Xu error_report_once("Interrupt Mask set, irq is not generated"); 3741da12ec4SLe Tan } else { 3751da12ec4SLe Tan vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 3761da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 3771da12ec4SLe Tan } 3781da12ec4SLe Tan } 3791da12ec4SLe Tan 3801da12ec4SLe Tan /* Check if the Fault (F) field of the Fault Recording Register referenced by 3811da12ec4SLe Tan * @index is Set. 3821da12ec4SLe Tan */ 3831da12ec4SLe Tan static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 3841da12ec4SLe Tan { 3851da12ec4SLe Tan /* Each reg is 128-bit */ 3861da12ec4SLe Tan hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 3871da12ec4SLe Tan addr += 8; /* Access the high 64-bit half */ 3881da12ec4SLe Tan 3891da12ec4SLe Tan assert(index < DMAR_FRCD_REG_NR); 3901da12ec4SLe Tan 3911da12ec4SLe Tan return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 3921da12ec4SLe Tan } 3931da12ec4SLe Tan 3941da12ec4SLe Tan /* Update the PPF field of Fault Status Register. 3951da12ec4SLe Tan * Should be called whenever change the F field of any fault recording 3961da12ec4SLe Tan * registers. 3971da12ec4SLe Tan */ 3981da12ec4SLe Tan static void vtd_update_fsts_ppf(IntelIOMMUState *s) 3991da12ec4SLe Tan { 4001da12ec4SLe Tan uint32_t i; 4011da12ec4SLe Tan uint32_t ppf_mask = 0; 4021da12ec4SLe Tan 4031da12ec4SLe Tan for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 4041da12ec4SLe Tan if (vtd_is_frcd_set(s, i)) { 4051da12ec4SLe Tan ppf_mask = VTD_FSTS_PPF; 4061da12ec4SLe Tan break; 4071da12ec4SLe Tan } 4081da12ec4SLe Tan } 4091da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 4107feb51b7SPeter Xu trace_vtd_fsts_ppf(!!ppf_mask); 4111da12ec4SLe Tan } 4121da12ec4SLe Tan 4131da12ec4SLe Tan static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 4141da12ec4SLe Tan { 4151da12ec4SLe Tan /* Each reg is 128-bit */ 4161da12ec4SLe Tan hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 4171da12ec4SLe Tan addr += 8; /* Access the high 64-bit half */ 4181da12ec4SLe Tan 4191da12ec4SLe Tan assert(index < DMAR_FRCD_REG_NR); 4201da12ec4SLe Tan 4211da12ec4SLe Tan vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 4221da12ec4SLe Tan vtd_update_fsts_ppf(s); 4231da12ec4SLe Tan } 4241da12ec4SLe Tan 4251da12ec4SLe Tan /* Must not update F field now, should be done later */ 4261da12ec4SLe Tan static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 4271da12ec4SLe Tan uint16_t source_id, hwaddr addr, 4281da12ec4SLe Tan VTDFaultReason fault, bool is_write) 4291da12ec4SLe Tan { 4301da12ec4SLe Tan uint64_t hi = 0, lo; 4311da12ec4SLe Tan hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 4321da12ec4SLe Tan 4331da12ec4SLe Tan assert(index < DMAR_FRCD_REG_NR); 4341da12ec4SLe Tan 4351da12ec4SLe Tan lo = VTD_FRCD_FI(addr); 4361da12ec4SLe Tan hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 4371da12ec4SLe Tan if (!is_write) { 4381da12ec4SLe Tan hi |= VTD_FRCD_T; 4391da12ec4SLe Tan } 4401da12ec4SLe Tan vtd_set_quad_raw(s, frcd_reg_addr, lo); 4411da12ec4SLe Tan vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 4427feb51b7SPeter Xu 4437feb51b7SPeter Xu trace_vtd_frr_new(index, hi, lo); 4441da12ec4SLe Tan } 4451da12ec4SLe Tan 4461da12ec4SLe Tan /* Try to collapse multiple pending faults from the same requester */ 4471da12ec4SLe Tan static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 4481da12ec4SLe Tan { 4491da12ec4SLe Tan uint32_t i; 4501da12ec4SLe Tan uint64_t frcd_reg; 4511da12ec4SLe Tan hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 4521da12ec4SLe Tan 4531da12ec4SLe Tan for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 4541da12ec4SLe Tan frcd_reg = vtd_get_quad_raw(s, addr); 4551da12ec4SLe Tan if ((frcd_reg & VTD_FRCD_F) && 4561da12ec4SLe Tan ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 4571da12ec4SLe Tan return true; 4581da12ec4SLe Tan } 4591da12ec4SLe Tan addr += 16; /* 128-bit for each */ 4601da12ec4SLe Tan } 4611da12ec4SLe Tan return false; 4621da12ec4SLe Tan } 4631da12ec4SLe Tan 4641da12ec4SLe Tan /* Log and report an DMAR (address translation) fault to software */ 4651da12ec4SLe Tan static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 4661da12ec4SLe Tan hwaddr addr, VTDFaultReason fault, 4671da12ec4SLe Tan bool is_write) 4681da12ec4SLe Tan { 4691da12ec4SLe Tan uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 4701da12ec4SLe Tan 4711da12ec4SLe Tan assert(fault < VTD_FR_MAX); 4721da12ec4SLe Tan 4731da12ec4SLe Tan if (fault == VTD_FR_RESERVED_ERR) { 4741da12ec4SLe Tan /* This is not a normal fault reason case. Drop it. */ 4751da12ec4SLe Tan return; 4761da12ec4SLe Tan } 4777feb51b7SPeter Xu 4787feb51b7SPeter Xu trace_vtd_dmar_fault(source_id, fault, addr, is_write); 4797feb51b7SPeter Xu 4801da12ec4SLe Tan if (fsts_reg & VTD_FSTS_PFO) { 4811376211fSPeter Xu error_report_once("New fault is not recorded due to " 4821376211fSPeter Xu "Primary Fault Overflow"); 4831da12ec4SLe Tan return; 4841da12ec4SLe Tan } 4857feb51b7SPeter Xu 4861da12ec4SLe Tan if (vtd_try_collapse_fault(s, source_id)) { 4871376211fSPeter Xu error_report_once("New fault is not recorded due to " 4881376211fSPeter Xu "compression of faults"); 4891da12ec4SLe Tan return; 4901da12ec4SLe Tan } 4917feb51b7SPeter Xu 4921da12ec4SLe Tan if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 4931376211fSPeter Xu error_report_once("Next Fault Recording Reg is used, " 4941376211fSPeter Xu "new fault is not recorded, set PFO field"); 4951da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 4961da12ec4SLe Tan return; 4971da12ec4SLe Tan } 4981da12ec4SLe Tan 4991da12ec4SLe Tan vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write); 5001da12ec4SLe Tan 5011da12ec4SLe Tan if (fsts_reg & VTD_FSTS_PPF) { 5021376211fSPeter Xu error_report_once("There are pending faults already, " 5031376211fSPeter Xu "fault event is not generated"); 5041da12ec4SLe Tan vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 5051da12ec4SLe Tan s->next_frcd_reg++; 5061da12ec4SLe Tan if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 5071da12ec4SLe Tan s->next_frcd_reg = 0; 5081da12ec4SLe Tan } 5091da12ec4SLe Tan } else { 5101da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 5111da12ec4SLe Tan VTD_FSTS_FRI(s->next_frcd_reg)); 5121da12ec4SLe Tan vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 5131da12ec4SLe Tan s->next_frcd_reg++; 5141da12ec4SLe Tan if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 5151da12ec4SLe Tan s->next_frcd_reg = 0; 5161da12ec4SLe Tan } 5171da12ec4SLe Tan /* This case actually cause the PPF to be Set. 5181da12ec4SLe Tan * So generate fault event (interrupt). 5191da12ec4SLe Tan */ 5201da12ec4SLe Tan vtd_generate_fault_event(s, fsts_reg); 5211da12ec4SLe Tan } 5221da12ec4SLe Tan } 5231da12ec4SLe Tan 524ed7b8fbcSLe Tan /* Handle Invalidation Queue Errors of queued invalidation interface error 525ed7b8fbcSLe Tan * conditions. 526ed7b8fbcSLe Tan */ 527ed7b8fbcSLe Tan static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 528ed7b8fbcSLe Tan { 529ed7b8fbcSLe Tan uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 530ed7b8fbcSLe Tan 531ed7b8fbcSLe Tan vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 532ed7b8fbcSLe Tan vtd_generate_fault_event(s, fsts_reg); 533ed7b8fbcSLe Tan } 534ed7b8fbcSLe Tan 535ed7b8fbcSLe Tan /* Set the IWC field and try to generate an invalidation completion interrupt */ 536ed7b8fbcSLe Tan static void vtd_generate_completion_event(IntelIOMMUState *s) 537ed7b8fbcSLe Tan { 538ed7b8fbcSLe Tan if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 539bc535e59SPeter Xu trace_vtd_inv_desc_wait_irq("One pending, skip current"); 540ed7b8fbcSLe Tan return; 541ed7b8fbcSLe Tan } 542ed7b8fbcSLe Tan vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 543ed7b8fbcSLe Tan vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 544ed7b8fbcSLe Tan if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 545bc535e59SPeter Xu trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, " 546bc535e59SPeter Xu "new event not generated"); 547ed7b8fbcSLe Tan return; 548ed7b8fbcSLe Tan } else { 549ed7b8fbcSLe Tan /* Generate the interrupt event */ 550bc535e59SPeter Xu trace_vtd_inv_desc_wait_irq("Generating complete event"); 551ed7b8fbcSLe Tan vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 552ed7b8fbcSLe Tan vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 553ed7b8fbcSLe Tan } 554ed7b8fbcSLe Tan } 555ed7b8fbcSLe Tan 556fb43cf73SLiu, Yi L static inline bool vtd_root_entry_present(IntelIOMMUState *s, 557fb43cf73SLiu, Yi L VTDRootEntry *re, 558fb43cf73SLiu, Yi L uint8_t devfn) 5591da12ec4SLe Tan { 560fb43cf73SLiu, Yi L if (s->root_scalable && devfn > UINT8_MAX / 2) { 561fb43cf73SLiu, Yi L return re->hi & VTD_ROOT_ENTRY_P; 562fb43cf73SLiu, Yi L } 563fb43cf73SLiu, Yi L 564fb43cf73SLiu, Yi L return re->lo & VTD_ROOT_ENTRY_P; 5651da12ec4SLe Tan } 5661da12ec4SLe Tan 5671da12ec4SLe Tan static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 5681da12ec4SLe Tan VTDRootEntry *re) 5691da12ec4SLe Tan { 5701da12ec4SLe Tan dma_addr_t addr; 5711da12ec4SLe Tan 5721da12ec4SLe Tan addr = s->root + index * sizeof(*re); 5731da12ec4SLe Tan if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) { 574fb43cf73SLiu, Yi L re->lo = 0; 5751da12ec4SLe Tan return -VTD_FR_ROOT_TABLE_INV; 5761da12ec4SLe Tan } 577fb43cf73SLiu, Yi L re->lo = le64_to_cpu(re->lo); 578fb43cf73SLiu, Yi L re->hi = le64_to_cpu(re->hi); 5791da12ec4SLe Tan return 0; 5801da12ec4SLe Tan } 5811da12ec4SLe Tan 5828f7d7161SPeter Xu static inline bool vtd_ce_present(VTDContextEntry *context) 5831da12ec4SLe Tan { 5841da12ec4SLe Tan return context->lo & VTD_CONTEXT_ENTRY_P; 5851da12ec4SLe Tan } 5861da12ec4SLe Tan 587fb43cf73SLiu, Yi L static int vtd_get_context_entry_from_root(IntelIOMMUState *s, 588fb43cf73SLiu, Yi L VTDRootEntry *re, 589fb43cf73SLiu, Yi L uint8_t index, 5901da12ec4SLe Tan VTDContextEntry *ce) 5911da12ec4SLe Tan { 592fb43cf73SLiu, Yi L dma_addr_t addr, ce_size; 5931da12ec4SLe Tan 5946c441e1dSPeter Xu /* we have checked that root entry is present */ 595fb43cf73SLiu, Yi L ce_size = s->root_scalable ? VTD_CTX_ENTRY_SCALABLE_SIZE : 596fb43cf73SLiu, Yi L VTD_CTX_ENTRY_LEGACY_SIZE; 597fb43cf73SLiu, Yi L 598fb43cf73SLiu, Yi L if (s->root_scalable && index > UINT8_MAX / 2) { 599fb43cf73SLiu, Yi L index = index & (~VTD_DEVFN_CHECK_MASK); 600fb43cf73SLiu, Yi L addr = re->hi & VTD_ROOT_ENTRY_CTP; 601fb43cf73SLiu, Yi L } else { 602fb43cf73SLiu, Yi L addr = re->lo & VTD_ROOT_ENTRY_CTP; 603fb43cf73SLiu, Yi L } 604fb43cf73SLiu, Yi L 605fb43cf73SLiu, Yi L addr = addr + index * ce_size; 606fb43cf73SLiu, Yi L if (dma_memory_read(&address_space_memory, addr, ce, ce_size)) { 6071da12ec4SLe Tan return -VTD_FR_CONTEXT_TABLE_INV; 6081da12ec4SLe Tan } 609fb43cf73SLiu, Yi L 6101da12ec4SLe Tan ce->lo = le64_to_cpu(ce->lo); 6111da12ec4SLe Tan ce->hi = le64_to_cpu(ce->hi); 612fb43cf73SLiu, Yi L if (ce_size == VTD_CTX_ENTRY_SCALABLE_SIZE) { 613fb43cf73SLiu, Yi L ce->val[2] = le64_to_cpu(ce->val[2]); 614fb43cf73SLiu, Yi L ce->val[3] = le64_to_cpu(ce->val[3]); 615fb43cf73SLiu, Yi L } 6161da12ec4SLe Tan return 0; 6171da12ec4SLe Tan } 6181da12ec4SLe Tan 6198f7d7161SPeter Xu static inline dma_addr_t vtd_ce_get_slpt_base(VTDContextEntry *ce) 6201da12ec4SLe Tan { 6211da12ec4SLe Tan return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 6221da12ec4SLe Tan } 6231da12ec4SLe Tan 62437f51384SPrasad Singamsetty static inline uint64_t vtd_get_slpte_addr(uint64_t slpte, uint8_t aw) 6251da12ec4SLe Tan { 62637f51384SPrasad Singamsetty return slpte & VTD_SL_PT_BASE_ADDR_MASK(aw); 6271da12ec4SLe Tan } 6281da12ec4SLe Tan 6291da12ec4SLe Tan /* Whether the pte indicates the address of the page frame */ 6301da12ec4SLe Tan static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 6311da12ec4SLe Tan { 6321da12ec4SLe Tan return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 6331da12ec4SLe Tan } 6341da12ec4SLe Tan 6351da12ec4SLe Tan /* Get the content of a spte located in @base_addr[@index] */ 6361da12ec4SLe Tan static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 6371da12ec4SLe Tan { 6381da12ec4SLe Tan uint64_t slpte; 6391da12ec4SLe Tan 6401da12ec4SLe Tan assert(index < VTD_SL_PT_ENTRY_NR); 6411da12ec4SLe Tan 6421da12ec4SLe Tan if (dma_memory_read(&address_space_memory, 6431da12ec4SLe Tan base_addr + index * sizeof(slpte), &slpte, 6441da12ec4SLe Tan sizeof(slpte))) { 6451da12ec4SLe Tan slpte = (uint64_t)-1; 6461da12ec4SLe Tan return slpte; 6471da12ec4SLe Tan } 6481da12ec4SLe Tan slpte = le64_to_cpu(slpte); 6491da12ec4SLe Tan return slpte; 6501da12ec4SLe Tan } 6511da12ec4SLe Tan 6526e905564SPeter Xu /* Given an iova and the level of paging structure, return the offset 6536e905564SPeter Xu * of current level. 6541da12ec4SLe Tan */ 6556e905564SPeter Xu static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level) 6561da12ec4SLe Tan { 6576e905564SPeter Xu return (iova >> vtd_slpt_level_shift(level)) & 6581da12ec4SLe Tan ((1ULL << VTD_SL_LEVEL_BITS) - 1); 6591da12ec4SLe Tan } 6601da12ec4SLe Tan 6611da12ec4SLe Tan /* Check Capability Register to see if the @level of page-table is supported */ 6621da12ec4SLe Tan static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 6631da12ec4SLe Tan { 6641da12ec4SLe Tan return VTD_CAP_SAGAW_MASK & s->cap & 6651da12ec4SLe Tan (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 6661da12ec4SLe Tan } 6671da12ec4SLe Tan 668fb43cf73SLiu, Yi L /* Return true if check passed, otherwise false */ 669fb43cf73SLiu, Yi L static inline bool vtd_pe_type_check(X86IOMMUState *x86_iommu, 670fb43cf73SLiu, Yi L VTDPASIDEntry *pe) 671fb43cf73SLiu, Yi L { 672fb43cf73SLiu, Yi L switch (VTD_PE_GET_TYPE(pe)) { 673fb43cf73SLiu, Yi L case VTD_SM_PASID_ENTRY_FLT: 674fb43cf73SLiu, Yi L case VTD_SM_PASID_ENTRY_SLT: 675fb43cf73SLiu, Yi L case VTD_SM_PASID_ENTRY_NESTED: 676fb43cf73SLiu, Yi L break; 677fb43cf73SLiu, Yi L case VTD_SM_PASID_ENTRY_PT: 678fb43cf73SLiu, Yi L if (!x86_iommu->pt_supported) { 679fb43cf73SLiu, Yi L return false; 680fb43cf73SLiu, Yi L } 681fb43cf73SLiu, Yi L break; 682fb43cf73SLiu, Yi L default: 683fb43cf73SLiu, Yi L /* Unknwon type */ 684fb43cf73SLiu, Yi L return false; 685fb43cf73SLiu, Yi L } 686fb43cf73SLiu, Yi L return true; 687fb43cf73SLiu, Yi L } 688fb43cf73SLiu, Yi L 689*56fc1e6aSLiu Yi L static inline bool vtd_pdire_present(VTDPASIDDirEntry *pdire) 690*56fc1e6aSLiu Yi L { 691*56fc1e6aSLiu Yi L return pdire->val & 1; 692*56fc1e6aSLiu Yi L } 693*56fc1e6aSLiu Yi L 694*56fc1e6aSLiu Yi L /** 695*56fc1e6aSLiu Yi L * Caller of this function should check present bit if wants 696*56fc1e6aSLiu Yi L * to use pdir entry for futher usage except for fpd bit check. 697*56fc1e6aSLiu Yi L */ 698*56fc1e6aSLiu Yi L static int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, 699fb43cf73SLiu, Yi L uint32_t pasid, 700fb43cf73SLiu, Yi L VTDPASIDDirEntry *pdire) 701fb43cf73SLiu, Yi L { 702fb43cf73SLiu, Yi L uint32_t index; 703fb43cf73SLiu, Yi L dma_addr_t addr, entry_size; 704fb43cf73SLiu, Yi L 705fb43cf73SLiu, Yi L index = VTD_PASID_DIR_INDEX(pasid); 706fb43cf73SLiu, Yi L entry_size = VTD_PASID_DIR_ENTRY_SIZE; 707fb43cf73SLiu, Yi L addr = pasid_dir_base + index * entry_size; 708fb43cf73SLiu, Yi L if (dma_memory_read(&address_space_memory, addr, pdire, entry_size)) { 709fb43cf73SLiu, Yi L return -VTD_FR_PASID_TABLE_INV; 710fb43cf73SLiu, Yi L } 711fb43cf73SLiu, Yi L 712fb43cf73SLiu, Yi L return 0; 713fb43cf73SLiu, Yi L } 714fb43cf73SLiu, Yi L 715*56fc1e6aSLiu Yi L static inline bool vtd_pe_present(VTDPASIDEntry *pe) 716*56fc1e6aSLiu Yi L { 717*56fc1e6aSLiu Yi L return pe->val[0] & VTD_PASID_ENTRY_P; 718*56fc1e6aSLiu Yi L } 719*56fc1e6aSLiu Yi L 720*56fc1e6aSLiu Yi L static int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s, 721fb43cf73SLiu, Yi L uint32_t pasid, 722*56fc1e6aSLiu Yi L dma_addr_t addr, 723fb43cf73SLiu, Yi L VTDPASIDEntry *pe) 724fb43cf73SLiu, Yi L { 725fb43cf73SLiu, Yi L uint32_t index; 726*56fc1e6aSLiu Yi L dma_addr_t entry_size; 727fb43cf73SLiu, Yi L X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 728fb43cf73SLiu, Yi L 729fb43cf73SLiu, Yi L index = VTD_PASID_TABLE_INDEX(pasid); 730fb43cf73SLiu, Yi L entry_size = VTD_PASID_ENTRY_SIZE; 731fb43cf73SLiu, Yi L addr = addr + index * entry_size; 732fb43cf73SLiu, Yi L if (dma_memory_read(&address_space_memory, addr, pe, entry_size)) { 733fb43cf73SLiu, Yi L return -VTD_FR_PASID_TABLE_INV; 734fb43cf73SLiu, Yi L } 735fb43cf73SLiu, Yi L 736fb43cf73SLiu, Yi L /* Do translation type check */ 737fb43cf73SLiu, Yi L if (!vtd_pe_type_check(x86_iommu, pe)) { 738fb43cf73SLiu, Yi L return -VTD_FR_PASID_TABLE_INV; 739fb43cf73SLiu, Yi L } 740fb43cf73SLiu, Yi L 741fb43cf73SLiu, Yi L if (!vtd_is_level_supported(s, VTD_PE_GET_LEVEL(pe))) { 742fb43cf73SLiu, Yi L return -VTD_FR_PASID_TABLE_INV; 743fb43cf73SLiu, Yi L } 744fb43cf73SLiu, Yi L 745fb43cf73SLiu, Yi L return 0; 746fb43cf73SLiu, Yi L } 747fb43cf73SLiu, Yi L 748*56fc1e6aSLiu Yi L /** 749*56fc1e6aSLiu Yi L * Caller of this function should check present bit if wants 750*56fc1e6aSLiu Yi L * to use pasid entry for futher usage except for fpd bit check. 751*56fc1e6aSLiu Yi L */ 752*56fc1e6aSLiu Yi L static int vtd_get_pe_from_pdire(IntelIOMMUState *s, 753*56fc1e6aSLiu Yi L uint32_t pasid, 754*56fc1e6aSLiu Yi L VTDPASIDDirEntry *pdire, 755*56fc1e6aSLiu Yi L VTDPASIDEntry *pe) 756*56fc1e6aSLiu Yi L { 757*56fc1e6aSLiu Yi L dma_addr_t addr = pdire->val & VTD_PASID_TABLE_BASE_ADDR_MASK; 758*56fc1e6aSLiu Yi L 759*56fc1e6aSLiu Yi L return vtd_get_pe_in_pasid_leaf_table(s, pasid, addr, pe); 760*56fc1e6aSLiu Yi L } 761*56fc1e6aSLiu Yi L 762*56fc1e6aSLiu Yi L /** 763*56fc1e6aSLiu Yi L * This function gets a pasid entry from a specified pasid 764*56fc1e6aSLiu Yi L * table (includes dir and leaf table) with a specified pasid. 765*56fc1e6aSLiu Yi L * Sanity check should be done to ensure return a present 766*56fc1e6aSLiu Yi L * pasid entry to caller. 767*56fc1e6aSLiu Yi L */ 768*56fc1e6aSLiu Yi L static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s, 769fb43cf73SLiu, Yi L dma_addr_t pasid_dir_base, 770fb43cf73SLiu, Yi L uint32_t pasid, 771fb43cf73SLiu, Yi L VTDPASIDEntry *pe) 772fb43cf73SLiu, Yi L { 773fb43cf73SLiu, Yi L int ret; 774fb43cf73SLiu, Yi L VTDPASIDDirEntry pdire; 775fb43cf73SLiu, Yi L 776*56fc1e6aSLiu Yi L ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, 777*56fc1e6aSLiu Yi L pasid, &pdire); 778fb43cf73SLiu, Yi L if (ret) { 779fb43cf73SLiu, Yi L return ret; 780fb43cf73SLiu, Yi L } 781fb43cf73SLiu, Yi L 782*56fc1e6aSLiu Yi L if (!vtd_pdire_present(&pdire)) { 783*56fc1e6aSLiu Yi L return -VTD_FR_PASID_TABLE_INV; 784*56fc1e6aSLiu Yi L } 785*56fc1e6aSLiu Yi L 786*56fc1e6aSLiu Yi L ret = vtd_get_pe_from_pdire(s, pasid, &pdire, pe); 787fb43cf73SLiu, Yi L if (ret) { 788fb43cf73SLiu, Yi L return ret; 789fb43cf73SLiu, Yi L } 790fb43cf73SLiu, Yi L 791*56fc1e6aSLiu Yi L if (!vtd_pe_present(pe)) { 792*56fc1e6aSLiu Yi L return -VTD_FR_PASID_TABLE_INV; 793*56fc1e6aSLiu Yi L } 794*56fc1e6aSLiu Yi L 795*56fc1e6aSLiu Yi L return 0; 796fb43cf73SLiu, Yi L } 797fb43cf73SLiu, Yi L 798fb43cf73SLiu, Yi L static int vtd_ce_get_rid2pasid_entry(IntelIOMMUState *s, 799fb43cf73SLiu, Yi L VTDContextEntry *ce, 800fb43cf73SLiu, Yi L VTDPASIDEntry *pe) 801fb43cf73SLiu, Yi L { 802fb43cf73SLiu, Yi L uint32_t pasid; 803fb43cf73SLiu, Yi L dma_addr_t pasid_dir_base; 804fb43cf73SLiu, Yi L int ret = 0; 805fb43cf73SLiu, Yi L 806fb43cf73SLiu, Yi L pasid = VTD_CE_GET_RID2PASID(ce); 807fb43cf73SLiu, Yi L pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); 808*56fc1e6aSLiu Yi L ret = vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe); 809fb43cf73SLiu, Yi L 810fb43cf73SLiu, Yi L return ret; 811fb43cf73SLiu, Yi L } 812fb43cf73SLiu, Yi L 813fb43cf73SLiu, Yi L static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s, 814fb43cf73SLiu, Yi L VTDContextEntry *ce, 815fb43cf73SLiu, Yi L bool *pe_fpd_set) 816fb43cf73SLiu, Yi L { 817fb43cf73SLiu, Yi L int ret; 818fb43cf73SLiu, Yi L uint32_t pasid; 819fb43cf73SLiu, Yi L dma_addr_t pasid_dir_base; 820fb43cf73SLiu, Yi L VTDPASIDDirEntry pdire; 821fb43cf73SLiu, Yi L VTDPASIDEntry pe; 822fb43cf73SLiu, Yi L 823fb43cf73SLiu, Yi L pasid = VTD_CE_GET_RID2PASID(ce); 824fb43cf73SLiu, Yi L pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); 825fb43cf73SLiu, Yi L 826*56fc1e6aSLiu Yi L /* 827*56fc1e6aSLiu Yi L * No present bit check since fpd is meaningful even 828*56fc1e6aSLiu Yi L * if the present bit is clear. 829*56fc1e6aSLiu Yi L */ 830*56fc1e6aSLiu Yi L ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, pasid, &pdire); 831fb43cf73SLiu, Yi L if (ret) { 832fb43cf73SLiu, Yi L return ret; 833fb43cf73SLiu, Yi L } 834fb43cf73SLiu, Yi L 835fb43cf73SLiu, Yi L if (pdire.val & VTD_PASID_DIR_FPD) { 836fb43cf73SLiu, Yi L *pe_fpd_set = true; 837fb43cf73SLiu, Yi L return 0; 838fb43cf73SLiu, Yi L } 839fb43cf73SLiu, Yi L 840*56fc1e6aSLiu Yi L if (!vtd_pdire_present(&pdire)) { 841*56fc1e6aSLiu Yi L return -VTD_FR_PASID_TABLE_INV; 842*56fc1e6aSLiu Yi L } 843*56fc1e6aSLiu Yi L 844*56fc1e6aSLiu Yi L /* 845*56fc1e6aSLiu Yi L * No present bit check since fpd is meaningful even 846*56fc1e6aSLiu Yi L * if the present bit is clear. 847*56fc1e6aSLiu Yi L */ 848*56fc1e6aSLiu Yi L ret = vtd_get_pe_from_pdire(s, pasid, &pdire, &pe); 849fb43cf73SLiu, Yi L if (ret) { 850fb43cf73SLiu, Yi L return ret; 851fb43cf73SLiu, Yi L } 852fb43cf73SLiu, Yi L 853fb43cf73SLiu, Yi L if (pe.val[0] & VTD_PASID_ENTRY_FPD) { 854fb43cf73SLiu, Yi L *pe_fpd_set = true; 855fb43cf73SLiu, Yi L } 856fb43cf73SLiu, Yi L 857fb43cf73SLiu, Yi L return 0; 858fb43cf73SLiu, Yi L } 859fb43cf73SLiu, Yi L 8601da12ec4SLe Tan /* Get the page-table level that hardware should use for the second-level 8611da12ec4SLe Tan * page-table walk from the Address Width field of context-entry. 8621da12ec4SLe Tan */ 8638f7d7161SPeter Xu static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce) 8641da12ec4SLe Tan { 8651da12ec4SLe Tan return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 8661da12ec4SLe Tan } 8671da12ec4SLe Tan 868fb43cf73SLiu, Yi L static uint32_t vtd_get_iova_level(IntelIOMMUState *s, 869fb43cf73SLiu, Yi L VTDContextEntry *ce) 870fb43cf73SLiu, Yi L { 871fb43cf73SLiu, Yi L VTDPASIDEntry pe; 872fb43cf73SLiu, Yi L 873fb43cf73SLiu, Yi L if (s->root_scalable) { 874fb43cf73SLiu, Yi L vtd_ce_get_rid2pasid_entry(s, ce, &pe); 875fb43cf73SLiu, Yi L return VTD_PE_GET_LEVEL(&pe); 876fb43cf73SLiu, Yi L } 877fb43cf73SLiu, Yi L 878fb43cf73SLiu, Yi L return vtd_ce_get_level(ce); 879fb43cf73SLiu, Yi L } 880fb43cf73SLiu, Yi L 8818f7d7161SPeter Xu static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce) 8821da12ec4SLe Tan { 8831da12ec4SLe Tan return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 8841da12ec4SLe Tan } 8851da12ec4SLe Tan 886fb43cf73SLiu, Yi L static uint32_t vtd_get_iova_agaw(IntelIOMMUState *s, 887fb43cf73SLiu, Yi L VTDContextEntry *ce) 888fb43cf73SLiu, Yi L { 889fb43cf73SLiu, Yi L VTDPASIDEntry pe; 890fb43cf73SLiu, Yi L 891fb43cf73SLiu, Yi L if (s->root_scalable) { 892fb43cf73SLiu, Yi L vtd_ce_get_rid2pasid_entry(s, ce, &pe); 893fb43cf73SLiu, Yi L return 30 + ((pe.val[0] >> 2) & VTD_SM_PASID_ENTRY_AW) * 9; 894fb43cf73SLiu, Yi L } 895fb43cf73SLiu, Yi L 896fb43cf73SLiu, Yi L return vtd_ce_get_agaw(ce); 897fb43cf73SLiu, Yi L } 898fb43cf73SLiu, Yi L 899127ff5c3SPeter Xu static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce) 900127ff5c3SPeter Xu { 901127ff5c3SPeter Xu return ce->lo & VTD_CONTEXT_ENTRY_TT; 902127ff5c3SPeter Xu } 903127ff5c3SPeter Xu 904fb43cf73SLiu, Yi L /* Only for Legacy Mode. Return true if check passed, otherwise false */ 905f80c9874SPeter Xu static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu, 906f80c9874SPeter Xu VTDContextEntry *ce) 907f80c9874SPeter Xu { 908f80c9874SPeter Xu switch (vtd_ce_get_type(ce)) { 909f80c9874SPeter Xu case VTD_CONTEXT_TT_MULTI_LEVEL: 910f80c9874SPeter Xu /* Always supported */ 911f80c9874SPeter Xu break; 912f80c9874SPeter Xu case VTD_CONTEXT_TT_DEV_IOTLB: 913f80c9874SPeter Xu if (!x86_iommu->dt_supported) { 914095955b2SPeter Xu error_report_once("%s: DT specified but not supported", __func__); 915f80c9874SPeter Xu return false; 916f80c9874SPeter Xu } 917f80c9874SPeter Xu break; 918dbaabb25SPeter Xu case VTD_CONTEXT_TT_PASS_THROUGH: 919dbaabb25SPeter Xu if (!x86_iommu->pt_supported) { 920095955b2SPeter Xu error_report_once("%s: PT specified but not supported", __func__); 921dbaabb25SPeter Xu return false; 922dbaabb25SPeter Xu } 923dbaabb25SPeter Xu break; 924f80c9874SPeter Xu default: 925fb43cf73SLiu, Yi L /* Unknown type */ 926095955b2SPeter Xu error_report_once("%s: unknown ce type: %"PRIu32, __func__, 927095955b2SPeter Xu vtd_ce_get_type(ce)); 928f80c9874SPeter Xu return false; 929f80c9874SPeter Xu } 930f80c9874SPeter Xu return true; 931f80c9874SPeter Xu } 932f80c9874SPeter Xu 933fb43cf73SLiu, Yi L static inline uint64_t vtd_iova_limit(IntelIOMMUState *s, 934fb43cf73SLiu, Yi L VTDContextEntry *ce, uint8_t aw) 935f06a696dSPeter Xu { 936fb43cf73SLiu, Yi L uint32_t ce_agaw = vtd_get_iova_agaw(s, ce); 93737f51384SPrasad Singamsetty return 1ULL << MIN(ce_agaw, aw); 938f06a696dSPeter Xu } 939f06a696dSPeter Xu 940f06a696dSPeter Xu /* Return true if IOVA passes range check, otherwise false. */ 941fb43cf73SLiu, Yi L static inline bool vtd_iova_range_check(IntelIOMMUState *s, 942fb43cf73SLiu, Yi L uint64_t iova, VTDContextEntry *ce, 94337f51384SPrasad Singamsetty uint8_t aw) 944f06a696dSPeter Xu { 945f06a696dSPeter Xu /* 946f06a696dSPeter Xu * Check if @iova is above 2^X-1, where X is the minimum of MGAW 947f06a696dSPeter Xu * in CAP_REG and AW in context-entry. 948f06a696dSPeter Xu */ 949fb43cf73SLiu, Yi L return !(iova & ~(vtd_iova_limit(s, ce, aw) - 1)); 950fb43cf73SLiu, Yi L } 951fb43cf73SLiu, Yi L 952fb43cf73SLiu, Yi L static dma_addr_t vtd_get_iova_pgtbl_base(IntelIOMMUState *s, 953fb43cf73SLiu, Yi L VTDContextEntry *ce) 954fb43cf73SLiu, Yi L { 955fb43cf73SLiu, Yi L VTDPASIDEntry pe; 956fb43cf73SLiu, Yi L 957fb43cf73SLiu, Yi L if (s->root_scalable) { 958fb43cf73SLiu, Yi L vtd_ce_get_rid2pasid_entry(s, ce, &pe); 959fb43cf73SLiu, Yi L return pe.val[0] & VTD_SM_PASID_ENTRY_SLPTPTR; 960fb43cf73SLiu, Yi L } 961fb43cf73SLiu, Yi L 962fb43cf73SLiu, Yi L return vtd_ce_get_slpt_base(ce); 963f06a696dSPeter Xu } 964f06a696dSPeter Xu 96592e5d85eSPrasad Singamsetty /* 96692e5d85eSPrasad Singamsetty * Rsvd field masks for spte: 967ce586f3bSQi, Yadong * vtd_spte_rsvd 4k pages 968ce586f3bSQi, Yadong * vtd_spte_rsvd_large large pages 96992e5d85eSPrasad Singamsetty */ 970ce586f3bSQi, Yadong static uint64_t vtd_spte_rsvd[5]; 971ce586f3bSQi, Yadong static uint64_t vtd_spte_rsvd_large[5]; 9721da12ec4SLe Tan 9731da12ec4SLe Tan static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 9741da12ec4SLe Tan { 975ce586f3bSQi, Yadong uint64_t rsvd_mask = vtd_spte_rsvd[level]; 976ce586f3bSQi, Yadong 977ce586f3bSQi, Yadong if ((level == VTD_SL_PD_LEVEL || level == VTD_SL_PDP_LEVEL) && 978ce586f3bSQi, Yadong (slpte & VTD_SL_PT_PAGE_SIZE_MASK)) { 979ce586f3bSQi, Yadong /* large page */ 980ce586f3bSQi, Yadong rsvd_mask = vtd_spte_rsvd_large[level]; 9811da12ec4SLe Tan } 982ce586f3bSQi, Yadong 983ce586f3bSQi, Yadong return slpte & rsvd_mask; 9841da12ec4SLe Tan } 9851da12ec4SLe Tan 986dbaabb25SPeter Xu /* Find the VTD address space associated with a given bus number */ 987dbaabb25SPeter Xu static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num) 988dbaabb25SPeter Xu { 989dbaabb25SPeter Xu VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num]; 990dbaabb25SPeter Xu if (!vtd_bus) { 991dbaabb25SPeter Xu /* 992dbaabb25SPeter Xu * Iterate over the registered buses to find the one which 993dbaabb25SPeter Xu * currently hold this bus number, and update the bus_num 994dbaabb25SPeter Xu * lookup table: 995dbaabb25SPeter Xu */ 996dbaabb25SPeter Xu GHashTableIter iter; 997dbaabb25SPeter Xu 998dbaabb25SPeter Xu g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); 999dbaabb25SPeter Xu while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) { 1000dbaabb25SPeter Xu if (pci_bus_num(vtd_bus->bus) == bus_num) { 1001dbaabb25SPeter Xu s->vtd_as_by_bus_num[bus_num] = vtd_bus; 1002dbaabb25SPeter Xu return vtd_bus; 1003dbaabb25SPeter Xu } 1004dbaabb25SPeter Xu } 1005a2e1cd41SLiu Yi L vtd_bus = NULL; 1006dbaabb25SPeter Xu } 1007dbaabb25SPeter Xu return vtd_bus; 1008dbaabb25SPeter Xu } 1009dbaabb25SPeter Xu 10106e905564SPeter Xu /* Given the @iova, get relevant @slptep. @slpte_level will be the last level 10111da12ec4SLe Tan * of the translation, can be used for deciding the size of large page. 10121da12ec4SLe Tan */ 1013fb43cf73SLiu, Yi L static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce, 1014fb43cf73SLiu, Yi L uint64_t iova, bool is_write, 10151da12ec4SLe Tan uint64_t *slptep, uint32_t *slpte_level, 101637f51384SPrasad Singamsetty bool *reads, bool *writes, uint8_t aw_bits) 10171da12ec4SLe Tan { 1018fb43cf73SLiu, Yi L dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce); 1019fb43cf73SLiu, Yi L uint32_t level = vtd_get_iova_level(s, ce); 10201da12ec4SLe Tan uint32_t offset; 10211da12ec4SLe Tan uint64_t slpte; 10221da12ec4SLe Tan uint64_t access_right_check; 10231da12ec4SLe Tan 1024fb43cf73SLiu, Yi L if (!vtd_iova_range_check(s, iova, ce, aw_bits)) { 10254e4abd11SPeter Xu error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 ")", 10264e4abd11SPeter Xu __func__, iova); 10271da12ec4SLe Tan return -VTD_FR_ADDR_BEYOND_MGAW; 10281da12ec4SLe Tan } 10291da12ec4SLe Tan 10301da12ec4SLe Tan /* FIXME: what is the Atomics request here? */ 10311da12ec4SLe Tan access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 10321da12ec4SLe Tan 10331da12ec4SLe Tan while (true) { 10346e905564SPeter Xu offset = vtd_iova_level_offset(iova, level); 10351da12ec4SLe Tan slpte = vtd_get_slpte(addr, offset); 10361da12ec4SLe Tan 10371da12ec4SLe Tan if (slpte == (uint64_t)-1) { 10384e4abd11SPeter Xu error_report_once("%s: detected read error on DMAR slpte " 10394e4abd11SPeter Xu "(iova=0x%" PRIx64 ")", __func__, iova); 1040fb43cf73SLiu, Yi L if (level == vtd_get_iova_level(s, ce)) { 10411da12ec4SLe Tan /* Invalid programming of context-entry */ 10421da12ec4SLe Tan return -VTD_FR_CONTEXT_ENTRY_INV; 10431da12ec4SLe Tan } else { 10441da12ec4SLe Tan return -VTD_FR_PAGING_ENTRY_INV; 10451da12ec4SLe Tan } 10461da12ec4SLe Tan } 10471da12ec4SLe Tan *reads = (*reads) && (slpte & VTD_SL_R); 10481da12ec4SLe Tan *writes = (*writes) && (slpte & VTD_SL_W); 10491da12ec4SLe Tan if (!(slpte & access_right_check)) { 10504e4abd11SPeter Xu error_report_once("%s: detected slpte permission error " 10514e4abd11SPeter Xu "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " 10524e4abd11SPeter Xu "slpte=0x%" PRIx64 ", write=%d)", __func__, 10534e4abd11SPeter Xu iova, level, slpte, is_write); 10541da12ec4SLe Tan return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 10551da12ec4SLe Tan } 10561da12ec4SLe Tan if (vtd_slpte_nonzero_rsvd(slpte, level)) { 10574e4abd11SPeter Xu error_report_once("%s: detected splte reserve non-zero " 10584e4abd11SPeter Xu "iova=0x%" PRIx64 ", level=0x%" PRIx32 10594e4abd11SPeter Xu "slpte=0x%" PRIx64 ")", __func__, iova, 10604e4abd11SPeter Xu level, slpte); 10611da12ec4SLe Tan return -VTD_FR_PAGING_ENTRY_RSVD; 10621da12ec4SLe Tan } 10631da12ec4SLe Tan 10641da12ec4SLe Tan if (vtd_is_last_slpte(slpte, level)) { 10651da12ec4SLe Tan *slptep = slpte; 10661da12ec4SLe Tan *slpte_level = level; 10671da12ec4SLe Tan return 0; 10681da12ec4SLe Tan } 106937f51384SPrasad Singamsetty addr = vtd_get_slpte_addr(slpte, aw_bits); 10701da12ec4SLe Tan level--; 10711da12ec4SLe Tan } 10721da12ec4SLe Tan } 10731da12ec4SLe Tan 1074f06a696dSPeter Xu typedef int (*vtd_page_walk_hook)(IOMMUTLBEntry *entry, void *private); 1075f06a696dSPeter Xu 1076fe215b0cSPeter Xu /** 1077fe215b0cSPeter Xu * Constant information used during page walking 1078fe215b0cSPeter Xu * 1079fe215b0cSPeter Xu * @hook_fn: hook func to be called when detected page 1080fe215b0cSPeter Xu * @private: private data to be passed into hook func 1081fe215b0cSPeter Xu * @notify_unmap: whether we should notify invalid entries 10822f764fa8SPeter Xu * @as: VT-d address space of the device 1083fe215b0cSPeter Xu * @aw: maximum address width 1084d118c06eSPeter Xu * @domain: domain ID of the page walk 1085fe215b0cSPeter Xu */ 1086fe215b0cSPeter Xu typedef struct { 10872f764fa8SPeter Xu VTDAddressSpace *as; 1088fe215b0cSPeter Xu vtd_page_walk_hook hook_fn; 1089fe215b0cSPeter Xu void *private; 1090fe215b0cSPeter Xu bool notify_unmap; 1091fe215b0cSPeter Xu uint8_t aw; 1092d118c06eSPeter Xu uint16_t domain_id; 1093fe215b0cSPeter Xu } vtd_page_walk_info; 1094fe215b0cSPeter Xu 1095d118c06eSPeter Xu static int vtd_page_walk_one(IOMMUTLBEntry *entry, vtd_page_walk_info *info) 109636d2d52bSPeter Xu { 109763b88968SPeter Xu VTDAddressSpace *as = info->as; 1098fe215b0cSPeter Xu vtd_page_walk_hook hook_fn = info->hook_fn; 1099fe215b0cSPeter Xu void *private = info->private; 110063b88968SPeter Xu DMAMap target = { 110163b88968SPeter Xu .iova = entry->iova, 110263b88968SPeter Xu .size = entry->addr_mask, 110363b88968SPeter Xu .translated_addr = entry->translated_addr, 110463b88968SPeter Xu .perm = entry->perm, 110563b88968SPeter Xu }; 110663b88968SPeter Xu DMAMap *mapped = iova_tree_find(as->iova_tree, &target); 110763b88968SPeter Xu 110863b88968SPeter Xu if (entry->perm == IOMMU_NONE && !info->notify_unmap) { 110963b88968SPeter Xu trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 111063b88968SPeter Xu return 0; 111163b88968SPeter Xu } 1112fe215b0cSPeter Xu 111336d2d52bSPeter Xu assert(hook_fn); 111463b88968SPeter Xu 111563b88968SPeter Xu /* Update local IOVA mapped ranges */ 111663b88968SPeter Xu if (entry->perm) { 111763b88968SPeter Xu if (mapped) { 111863b88968SPeter Xu /* If it's exactly the same translation, skip */ 111963b88968SPeter Xu if (!memcmp(mapped, &target, sizeof(target))) { 112063b88968SPeter Xu trace_vtd_page_walk_one_skip_map(entry->iova, entry->addr_mask, 112163b88968SPeter Xu entry->translated_addr); 112263b88968SPeter Xu return 0; 112363b88968SPeter Xu } else { 112463b88968SPeter Xu /* 112563b88968SPeter Xu * Translation changed. Normally this should not 112663b88968SPeter Xu * happen, but it can happen when with buggy guest 112763b88968SPeter Xu * OSes. Note that there will be a small window that 112863b88968SPeter Xu * we don't have map at all. But that's the best 112963b88968SPeter Xu * effort we can do. The ideal way to emulate this is 113063b88968SPeter Xu * atomically modify the PTE to follow what has 113163b88968SPeter Xu * changed, but we can't. One example is that vfio 113263b88968SPeter Xu * driver only has VFIO_IOMMU_[UN]MAP_DMA but no 113363b88968SPeter Xu * interface to modify a mapping (meanwhile it seems 113463b88968SPeter Xu * meaningless to even provide one). Anyway, let's 113563b88968SPeter Xu * mark this as a TODO in case one day we'll have 113663b88968SPeter Xu * a better solution. 113763b88968SPeter Xu */ 113863b88968SPeter Xu IOMMUAccessFlags cache_perm = entry->perm; 113963b88968SPeter Xu int ret; 114063b88968SPeter Xu 114163b88968SPeter Xu /* Emulate an UNMAP */ 114263b88968SPeter Xu entry->perm = IOMMU_NONE; 114363b88968SPeter Xu trace_vtd_page_walk_one(info->domain_id, 114463b88968SPeter Xu entry->iova, 114563b88968SPeter Xu entry->translated_addr, 114663b88968SPeter Xu entry->addr_mask, 114763b88968SPeter Xu entry->perm); 114863b88968SPeter Xu ret = hook_fn(entry, private); 114963b88968SPeter Xu if (ret) { 115063b88968SPeter Xu return ret; 115163b88968SPeter Xu } 115263b88968SPeter Xu /* Drop any existing mapping */ 115363b88968SPeter Xu iova_tree_remove(as->iova_tree, &target); 115463b88968SPeter Xu /* Recover the correct permission */ 115563b88968SPeter Xu entry->perm = cache_perm; 115663b88968SPeter Xu } 115763b88968SPeter Xu } 115863b88968SPeter Xu iova_tree_insert(as->iova_tree, &target); 115963b88968SPeter Xu } else { 116063b88968SPeter Xu if (!mapped) { 116163b88968SPeter Xu /* Skip since we didn't map this range at all */ 116263b88968SPeter Xu trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 116363b88968SPeter Xu return 0; 116463b88968SPeter Xu } 116563b88968SPeter Xu iova_tree_remove(as->iova_tree, &target); 116663b88968SPeter Xu } 116763b88968SPeter Xu 1168d118c06eSPeter Xu trace_vtd_page_walk_one(info->domain_id, entry->iova, 1169d118c06eSPeter Xu entry->translated_addr, entry->addr_mask, 1170d118c06eSPeter Xu entry->perm); 117136d2d52bSPeter Xu return hook_fn(entry, private); 117236d2d52bSPeter Xu } 117336d2d52bSPeter Xu 1174f06a696dSPeter Xu /** 1175f06a696dSPeter Xu * vtd_page_walk_level - walk over specific level for IOVA range 1176f06a696dSPeter Xu * 1177f06a696dSPeter Xu * @addr: base GPA addr to start the walk 1178f06a696dSPeter Xu * @start: IOVA range start address 1179f06a696dSPeter Xu * @end: IOVA range end address (start <= addr < end) 1180f06a696dSPeter Xu * @read: whether parent level has read permission 1181f06a696dSPeter Xu * @write: whether parent level has write permission 1182fe215b0cSPeter Xu * @info: constant information for the page walk 1183f06a696dSPeter Xu */ 1184f06a696dSPeter Xu static int vtd_page_walk_level(dma_addr_t addr, uint64_t start, 1185fe215b0cSPeter Xu uint64_t end, uint32_t level, bool read, 1186fe215b0cSPeter Xu bool write, vtd_page_walk_info *info) 1187f06a696dSPeter Xu { 1188f06a696dSPeter Xu bool read_cur, write_cur, entry_valid; 1189f06a696dSPeter Xu uint32_t offset; 1190f06a696dSPeter Xu uint64_t slpte; 1191f06a696dSPeter Xu uint64_t subpage_size, subpage_mask; 1192f06a696dSPeter Xu IOMMUTLBEntry entry; 1193f06a696dSPeter Xu uint64_t iova = start; 1194f06a696dSPeter Xu uint64_t iova_next; 1195f06a696dSPeter Xu int ret = 0; 1196f06a696dSPeter Xu 1197f06a696dSPeter Xu trace_vtd_page_walk_level(addr, level, start, end); 1198f06a696dSPeter Xu 1199f06a696dSPeter Xu subpage_size = 1ULL << vtd_slpt_level_shift(level); 1200f06a696dSPeter Xu subpage_mask = vtd_slpt_level_page_mask(level); 1201f06a696dSPeter Xu 1202f06a696dSPeter Xu while (iova < end) { 1203f06a696dSPeter Xu iova_next = (iova & subpage_mask) + subpage_size; 1204f06a696dSPeter Xu 1205f06a696dSPeter Xu offset = vtd_iova_level_offset(iova, level); 1206f06a696dSPeter Xu slpte = vtd_get_slpte(addr, offset); 1207f06a696dSPeter Xu 1208f06a696dSPeter Xu if (slpte == (uint64_t)-1) { 1209f06a696dSPeter Xu trace_vtd_page_walk_skip_read(iova, iova_next); 1210f06a696dSPeter Xu goto next; 1211f06a696dSPeter Xu } 1212f06a696dSPeter Xu 1213f06a696dSPeter Xu if (vtd_slpte_nonzero_rsvd(slpte, level)) { 1214f06a696dSPeter Xu trace_vtd_page_walk_skip_reserve(iova, iova_next); 1215f06a696dSPeter Xu goto next; 1216f06a696dSPeter Xu } 1217f06a696dSPeter Xu 1218f06a696dSPeter Xu /* Permissions are stacked with parents' */ 1219f06a696dSPeter Xu read_cur = read && (slpte & VTD_SL_R); 1220f06a696dSPeter Xu write_cur = write && (slpte & VTD_SL_W); 1221f06a696dSPeter Xu 1222f06a696dSPeter Xu /* 1223f06a696dSPeter Xu * As long as we have either read/write permission, this is a 1224f06a696dSPeter Xu * valid entry. The rule works for both page entries and page 1225f06a696dSPeter Xu * table entries. 1226f06a696dSPeter Xu */ 1227f06a696dSPeter Xu entry_valid = read_cur | write_cur; 1228f06a696dSPeter Xu 122963b88968SPeter Xu if (!vtd_is_last_slpte(slpte, level) && entry_valid) { 123063b88968SPeter Xu /* 123163b88968SPeter Xu * This is a valid PDE (or even bigger than PDE). We need 123263b88968SPeter Xu * to walk one further level. 123363b88968SPeter Xu */ 123463b88968SPeter Xu ret = vtd_page_walk_level(vtd_get_slpte_addr(slpte, info->aw), 123563b88968SPeter Xu iova, MIN(iova_next, end), level - 1, 123663b88968SPeter Xu read_cur, write_cur, info); 123763b88968SPeter Xu } else { 123863b88968SPeter Xu /* 123963b88968SPeter Xu * This means we are either: 124063b88968SPeter Xu * 124163b88968SPeter Xu * (1) the real page entry (either 4K page, or huge page) 124263b88968SPeter Xu * (2) the whole range is invalid 124363b88968SPeter Xu * 124463b88968SPeter Xu * In either case, we send an IOTLB notification down. 124563b88968SPeter Xu */ 1246f06a696dSPeter Xu entry.target_as = &address_space_memory; 1247f06a696dSPeter Xu entry.iova = iova & subpage_mask; 124836d2d52bSPeter Xu entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur); 124936d2d52bSPeter Xu entry.addr_mask = ~subpage_mask; 1250f06a696dSPeter Xu /* NOTE: this is only meaningful if entry_valid == true */ 1251fe215b0cSPeter Xu entry.translated_addr = vtd_get_slpte_addr(slpte, info->aw); 1252d118c06eSPeter Xu ret = vtd_page_walk_one(&entry, info); 125363b88968SPeter Xu } 125463b88968SPeter Xu 1255f06a696dSPeter Xu if (ret < 0) { 1256f06a696dSPeter Xu return ret; 1257f06a696dSPeter Xu } 1258f06a696dSPeter Xu 1259f06a696dSPeter Xu next: 1260f06a696dSPeter Xu iova = iova_next; 1261f06a696dSPeter Xu } 1262f06a696dSPeter Xu 1263f06a696dSPeter Xu return 0; 1264f06a696dSPeter Xu } 1265f06a696dSPeter Xu 1266f06a696dSPeter Xu /** 1267f06a696dSPeter Xu * vtd_page_walk - walk specific IOVA range, and call the hook 1268f06a696dSPeter Xu * 1269fb43cf73SLiu, Yi L * @s: intel iommu state 1270f06a696dSPeter Xu * @ce: context entry to walk upon 1271f06a696dSPeter Xu * @start: IOVA address to start the walk 1272f06a696dSPeter Xu * @end: IOVA range end address (start <= addr < end) 1273fe215b0cSPeter Xu * @info: page walking information struct 1274f06a696dSPeter Xu */ 1275fb43cf73SLiu, Yi L static int vtd_page_walk(IntelIOMMUState *s, VTDContextEntry *ce, 1276fb43cf73SLiu, Yi L uint64_t start, uint64_t end, 1277fe215b0cSPeter Xu vtd_page_walk_info *info) 1278f06a696dSPeter Xu { 1279fb43cf73SLiu, Yi L dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce); 1280fb43cf73SLiu, Yi L uint32_t level = vtd_get_iova_level(s, ce); 1281f06a696dSPeter Xu 1282fb43cf73SLiu, Yi L if (!vtd_iova_range_check(s, start, ce, info->aw)) { 1283f06a696dSPeter Xu return -VTD_FR_ADDR_BEYOND_MGAW; 1284f06a696dSPeter Xu } 1285f06a696dSPeter Xu 1286fb43cf73SLiu, Yi L if (!vtd_iova_range_check(s, end, ce, info->aw)) { 1287f06a696dSPeter Xu /* Fix end so that it reaches the maximum */ 1288fb43cf73SLiu, Yi L end = vtd_iova_limit(s, ce, info->aw); 1289f06a696dSPeter Xu } 1290f06a696dSPeter Xu 1291fe215b0cSPeter Xu return vtd_page_walk_level(addr, start, end, level, true, true, info); 1292f06a696dSPeter Xu } 1293f06a696dSPeter Xu 1294fb43cf73SLiu, Yi L static int vtd_root_entry_rsvd_bits_check(IntelIOMMUState *s, 1295fb43cf73SLiu, Yi L VTDRootEntry *re) 1296fb43cf73SLiu, Yi L { 1297fb43cf73SLiu, Yi L /* Legacy Mode reserved bits check */ 1298fb43cf73SLiu, Yi L if (!s->root_scalable && 1299fb43cf73SLiu, Yi L (re->hi || (re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)))) 1300fb43cf73SLiu, Yi L goto rsvd_err; 1301fb43cf73SLiu, Yi L 1302fb43cf73SLiu, Yi L /* Scalable Mode reserved bits check */ 1303fb43cf73SLiu, Yi L if (s->root_scalable && 1304fb43cf73SLiu, Yi L ((re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)) || 1305fb43cf73SLiu, Yi L (re->hi & VTD_ROOT_ENTRY_RSVD(s->aw_bits)))) 1306fb43cf73SLiu, Yi L goto rsvd_err; 1307fb43cf73SLiu, Yi L 1308fb43cf73SLiu, Yi L return 0; 1309fb43cf73SLiu, Yi L 1310fb43cf73SLiu, Yi L rsvd_err: 1311fb43cf73SLiu, Yi L error_report_once("%s: invalid root entry: hi=0x%"PRIx64 1312fb43cf73SLiu, Yi L ", lo=0x%"PRIx64, 1313fb43cf73SLiu, Yi L __func__, re->hi, re->lo); 1314fb43cf73SLiu, Yi L return -VTD_FR_ROOT_ENTRY_RSVD; 1315fb43cf73SLiu, Yi L } 1316fb43cf73SLiu, Yi L 1317fb43cf73SLiu, Yi L static inline int vtd_context_entry_rsvd_bits_check(IntelIOMMUState *s, 1318fb43cf73SLiu, Yi L VTDContextEntry *ce) 1319fb43cf73SLiu, Yi L { 1320fb43cf73SLiu, Yi L if (!s->root_scalable && 1321fb43cf73SLiu, Yi L (ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI || 1322fb43cf73SLiu, Yi L ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) { 1323fb43cf73SLiu, Yi L error_report_once("%s: invalid context entry: hi=%"PRIx64 1324fb43cf73SLiu, Yi L ", lo=%"PRIx64" (reserved nonzero)", 1325fb43cf73SLiu, Yi L __func__, ce->hi, ce->lo); 1326fb43cf73SLiu, Yi L return -VTD_FR_CONTEXT_ENTRY_RSVD; 1327fb43cf73SLiu, Yi L } 1328fb43cf73SLiu, Yi L 1329fb43cf73SLiu, Yi L if (s->root_scalable && 1330fb43cf73SLiu, Yi L (ce->val[0] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL0(s->aw_bits) || 1331fb43cf73SLiu, Yi L ce->val[1] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL1 || 1332fb43cf73SLiu, Yi L ce->val[2] || 1333fb43cf73SLiu, Yi L ce->val[3])) { 1334fb43cf73SLiu, Yi L error_report_once("%s: invalid context entry: val[3]=%"PRIx64 1335fb43cf73SLiu, Yi L ", val[2]=%"PRIx64 1336fb43cf73SLiu, Yi L ", val[1]=%"PRIx64 1337fb43cf73SLiu, Yi L ", val[0]=%"PRIx64" (reserved nonzero)", 1338fb43cf73SLiu, Yi L __func__, ce->val[3], ce->val[2], 1339fb43cf73SLiu, Yi L ce->val[1], ce->val[0]); 1340fb43cf73SLiu, Yi L return -VTD_FR_CONTEXT_ENTRY_RSVD; 1341fb43cf73SLiu, Yi L } 1342fb43cf73SLiu, Yi L 1343fb43cf73SLiu, Yi L return 0; 1344fb43cf73SLiu, Yi L } 1345fb43cf73SLiu, Yi L 1346fb43cf73SLiu, Yi L static int vtd_ce_rid2pasid_check(IntelIOMMUState *s, 1347fb43cf73SLiu, Yi L VTDContextEntry *ce) 1348fb43cf73SLiu, Yi L { 1349fb43cf73SLiu, Yi L VTDPASIDEntry pe; 1350fb43cf73SLiu, Yi L 1351fb43cf73SLiu, Yi L /* 1352fb43cf73SLiu, Yi L * Make sure in Scalable Mode, a present context entry 1353fb43cf73SLiu, Yi L * has valid rid2pasid setting, which includes valid 1354fb43cf73SLiu, Yi L * rid2pasid field and corresponding pasid entry setting 1355fb43cf73SLiu, Yi L */ 1356fb43cf73SLiu, Yi L return vtd_ce_get_rid2pasid_entry(s, ce, &pe); 1357fb43cf73SLiu, Yi L } 1358fb43cf73SLiu, Yi L 13591da12ec4SLe Tan /* Map a device to its corresponding domain (context-entry) */ 13601da12ec4SLe Tan static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 13611da12ec4SLe Tan uint8_t devfn, VTDContextEntry *ce) 13621da12ec4SLe Tan { 13631da12ec4SLe Tan VTDRootEntry re; 13641da12ec4SLe Tan int ret_fr; 1365f80c9874SPeter Xu X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 13661da12ec4SLe Tan 13671da12ec4SLe Tan ret_fr = vtd_get_root_entry(s, bus_num, &re); 13681da12ec4SLe Tan if (ret_fr) { 13691da12ec4SLe Tan return ret_fr; 13701da12ec4SLe Tan } 13711da12ec4SLe Tan 1372fb43cf73SLiu, Yi L if (!vtd_root_entry_present(s, &re, devfn)) { 13736c441e1dSPeter Xu /* Not error - it's okay we don't have root entry. */ 13746c441e1dSPeter Xu trace_vtd_re_not_present(bus_num); 13751da12ec4SLe Tan return -VTD_FR_ROOT_ENTRY_P; 1376f80c9874SPeter Xu } 1377f80c9874SPeter Xu 1378fb43cf73SLiu, Yi L ret_fr = vtd_root_entry_rsvd_bits_check(s, &re); 1379fb43cf73SLiu, Yi L if (ret_fr) { 1380fb43cf73SLiu, Yi L return ret_fr; 13811da12ec4SLe Tan } 13821da12ec4SLe Tan 1383fb43cf73SLiu, Yi L ret_fr = vtd_get_context_entry_from_root(s, &re, devfn, ce); 13841da12ec4SLe Tan if (ret_fr) { 13851da12ec4SLe Tan return ret_fr; 13861da12ec4SLe Tan } 13871da12ec4SLe Tan 13888f7d7161SPeter Xu if (!vtd_ce_present(ce)) { 13896c441e1dSPeter Xu /* Not error - it's okay we don't have context entry. */ 13906c441e1dSPeter Xu trace_vtd_ce_not_present(bus_num, devfn); 13911da12ec4SLe Tan return -VTD_FR_CONTEXT_ENTRY_P; 1392f80c9874SPeter Xu } 1393f80c9874SPeter Xu 1394fb43cf73SLiu, Yi L ret_fr = vtd_context_entry_rsvd_bits_check(s, ce); 1395fb43cf73SLiu, Yi L if (ret_fr) { 1396fb43cf73SLiu, Yi L return ret_fr; 13971da12ec4SLe Tan } 1398f80c9874SPeter Xu 13991da12ec4SLe Tan /* Check if the programming of context-entry is valid */ 1400fb43cf73SLiu, Yi L if (!s->root_scalable && 1401fb43cf73SLiu, Yi L !vtd_is_level_supported(s, vtd_ce_get_level(ce))) { 1402095955b2SPeter Xu error_report_once("%s: invalid context entry: hi=%"PRIx64 1403095955b2SPeter Xu ", lo=%"PRIx64" (level %d not supported)", 1404fb43cf73SLiu, Yi L __func__, ce->hi, ce->lo, 1405fb43cf73SLiu, Yi L vtd_ce_get_level(ce)); 14061da12ec4SLe Tan return -VTD_FR_CONTEXT_ENTRY_INV; 1407f80c9874SPeter Xu } 1408f80c9874SPeter Xu 1409fb43cf73SLiu, Yi L if (!s->root_scalable) { 1410f80c9874SPeter Xu /* Do translation type check */ 1411f80c9874SPeter Xu if (!vtd_ce_type_check(x86_iommu, ce)) { 1412095955b2SPeter Xu /* Errors dumped in vtd_ce_type_check() */ 14131da12ec4SLe Tan return -VTD_FR_CONTEXT_ENTRY_INV; 14141da12ec4SLe Tan } 1415fb43cf73SLiu, Yi L } else { 1416fb43cf73SLiu, Yi L /* 1417fb43cf73SLiu, Yi L * Check if the programming of context-entry.rid2pasid 1418fb43cf73SLiu, Yi L * and corresponding pasid setting is valid, and thus 1419fb43cf73SLiu, Yi L * avoids to check pasid entry fetching result in future 1420fb43cf73SLiu, Yi L * helper function calling. 1421fb43cf73SLiu, Yi L */ 1422fb43cf73SLiu, Yi L ret_fr = vtd_ce_rid2pasid_check(s, ce); 1423fb43cf73SLiu, Yi L if (ret_fr) { 1424fb43cf73SLiu, Yi L return ret_fr; 1425fb43cf73SLiu, Yi L } 1426fb43cf73SLiu, Yi L } 1427f80c9874SPeter Xu 14281da12ec4SLe Tan return 0; 14291da12ec4SLe Tan } 14301da12ec4SLe Tan 143163b88968SPeter Xu static int vtd_sync_shadow_page_hook(IOMMUTLBEntry *entry, 143263b88968SPeter Xu void *private) 143363b88968SPeter Xu { 1434cb1efcf4SPeter Maydell memory_region_notify_iommu((IOMMUMemoryRegion *)private, 0, *entry); 143563b88968SPeter Xu return 0; 143663b88968SPeter Xu } 143763b88968SPeter Xu 1438fb43cf73SLiu, Yi L static uint16_t vtd_get_domain_id(IntelIOMMUState *s, 1439fb43cf73SLiu, Yi L VTDContextEntry *ce) 1440fb43cf73SLiu, Yi L { 1441fb43cf73SLiu, Yi L VTDPASIDEntry pe; 1442fb43cf73SLiu, Yi L 1443fb43cf73SLiu, Yi L if (s->root_scalable) { 1444fb43cf73SLiu, Yi L vtd_ce_get_rid2pasid_entry(s, ce, &pe); 1445fb43cf73SLiu, Yi L return VTD_SM_PASID_ENTRY_DID(pe.val[1]); 1446fb43cf73SLiu, Yi L } 1447fb43cf73SLiu, Yi L 1448fb43cf73SLiu, Yi L return VTD_CONTEXT_ENTRY_DID(ce->hi); 1449fb43cf73SLiu, Yi L } 1450fb43cf73SLiu, Yi L 145163b88968SPeter Xu static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as, 145263b88968SPeter Xu VTDContextEntry *ce, 145363b88968SPeter Xu hwaddr addr, hwaddr size) 145463b88968SPeter Xu { 145563b88968SPeter Xu IntelIOMMUState *s = vtd_as->iommu_state; 145663b88968SPeter Xu vtd_page_walk_info info = { 145763b88968SPeter Xu .hook_fn = vtd_sync_shadow_page_hook, 145863b88968SPeter Xu .private = (void *)&vtd_as->iommu, 145963b88968SPeter Xu .notify_unmap = true, 146063b88968SPeter Xu .aw = s->aw_bits, 146163b88968SPeter Xu .as = vtd_as, 1462fb43cf73SLiu, Yi L .domain_id = vtd_get_domain_id(s, ce), 146363b88968SPeter Xu }; 146463b88968SPeter Xu 1465fb43cf73SLiu, Yi L return vtd_page_walk(s, ce, addr, addr + size, &info); 146663b88968SPeter Xu } 146763b88968SPeter Xu 146863b88968SPeter Xu static int vtd_sync_shadow_page_table(VTDAddressSpace *vtd_as) 146963b88968SPeter Xu { 147095ecd3dfSPeter Xu int ret; 147195ecd3dfSPeter Xu VTDContextEntry ce; 1472c28b535dSPeter Xu IOMMUNotifier *n; 147395ecd3dfSPeter Xu 147495ecd3dfSPeter Xu ret = vtd_dev_to_context_entry(vtd_as->iommu_state, 147595ecd3dfSPeter Xu pci_bus_num(vtd_as->bus), 147695ecd3dfSPeter Xu vtd_as->devfn, &ce); 147795ecd3dfSPeter Xu if (ret) { 1478c28b535dSPeter Xu if (ret == -VTD_FR_CONTEXT_ENTRY_P) { 1479c28b535dSPeter Xu /* 1480c28b535dSPeter Xu * It's a valid scenario to have a context entry that is 1481c28b535dSPeter Xu * not present. For example, when a device is removed 1482c28b535dSPeter Xu * from an existing domain then the context entry will be 1483c28b535dSPeter Xu * zeroed by the guest before it was put into another 1484c28b535dSPeter Xu * domain. When this happens, instead of synchronizing 1485c28b535dSPeter Xu * the shadow pages we should invalidate all existing 1486c28b535dSPeter Xu * mappings and notify the backends. 1487c28b535dSPeter Xu */ 1488c28b535dSPeter Xu IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 1489c28b535dSPeter Xu vtd_address_space_unmap(vtd_as, n); 1490c28b535dSPeter Xu } 1491c28b535dSPeter Xu ret = 0; 1492c28b535dSPeter Xu } 149395ecd3dfSPeter Xu return ret; 149495ecd3dfSPeter Xu } 149595ecd3dfSPeter Xu 149695ecd3dfSPeter Xu return vtd_sync_shadow_page_table_range(vtd_as, &ce, 0, UINT64_MAX); 149763b88968SPeter Xu } 149863b88968SPeter Xu 1499dbaabb25SPeter Xu /* 1500fb43cf73SLiu, Yi L * Check if specific device is configed to bypass address 1501fb43cf73SLiu, Yi L * translation for DMA requests. In Scalable Mode, bypass 1502fb43cf73SLiu, Yi L * 1st-level translation or 2nd-level translation, it depends 1503fb43cf73SLiu, Yi L * on PGTT setting. 1504dbaabb25SPeter Xu */ 1505fb43cf73SLiu, Yi L static bool vtd_dev_pt_enabled(VTDAddressSpace *as) 1506dbaabb25SPeter Xu { 1507dbaabb25SPeter Xu IntelIOMMUState *s; 1508dbaabb25SPeter Xu VTDContextEntry ce; 1509fb43cf73SLiu, Yi L VTDPASIDEntry pe; 1510dbaabb25SPeter Xu int ret; 1511dbaabb25SPeter Xu 1512dbaabb25SPeter Xu assert(as); 1513dbaabb25SPeter Xu 1514fb43cf73SLiu, Yi L s = as->iommu_state; 1515fb43cf73SLiu, Yi L ret = vtd_dev_to_context_entry(s, pci_bus_num(as->bus), 1516fb43cf73SLiu, Yi L as->devfn, &ce); 1517fb43cf73SLiu, Yi L if (ret) { 1518dbaabb25SPeter Xu /* 1519dbaabb25SPeter Xu * Possibly failed to parse the context entry for some reason 1520dbaabb25SPeter Xu * (e.g., during init, or any guest configuration errors on 1521dbaabb25SPeter Xu * context entries). We should assume PT not enabled for 1522dbaabb25SPeter Xu * safety. 1523dbaabb25SPeter Xu */ 1524dbaabb25SPeter Xu return false; 1525dbaabb25SPeter Xu } 1526dbaabb25SPeter Xu 1527fb43cf73SLiu, Yi L if (s->root_scalable) { 1528fb43cf73SLiu, Yi L ret = vtd_ce_get_rid2pasid_entry(s, &ce, &pe); 1529fb43cf73SLiu, Yi L if (ret) { 1530fb43cf73SLiu, Yi L error_report_once("%s: vtd_ce_get_rid2pasid_entry error: %"PRId32, 1531fb43cf73SLiu, Yi L __func__, ret); 1532fb43cf73SLiu, Yi L return false; 1533fb43cf73SLiu, Yi L } 1534fb43cf73SLiu, Yi L return (VTD_PE_GET_TYPE(&pe) == VTD_SM_PASID_ENTRY_PT); 1535fb43cf73SLiu, Yi L } 1536fb43cf73SLiu, Yi L 1537fb43cf73SLiu, Yi L return (vtd_ce_get_type(&ce) == VTD_CONTEXT_TT_PASS_THROUGH); 1538dbaabb25SPeter Xu } 1539dbaabb25SPeter Xu 1540dbaabb25SPeter Xu /* Return whether the device is using IOMMU translation. */ 1541dbaabb25SPeter Xu static bool vtd_switch_address_space(VTDAddressSpace *as) 1542dbaabb25SPeter Xu { 1543dbaabb25SPeter Xu bool use_iommu; 154466a4a031SPeter Xu /* Whether we need to take the BQL on our own */ 154566a4a031SPeter Xu bool take_bql = !qemu_mutex_iothread_locked(); 1546dbaabb25SPeter Xu 1547dbaabb25SPeter Xu assert(as); 1548dbaabb25SPeter Xu 15492a078b10SPeter Xu use_iommu = as->iommu_state->dmar_enabled && !vtd_dev_pt_enabled(as); 1550dbaabb25SPeter Xu 1551dbaabb25SPeter Xu trace_vtd_switch_address_space(pci_bus_num(as->bus), 1552dbaabb25SPeter Xu VTD_PCI_SLOT(as->devfn), 1553dbaabb25SPeter Xu VTD_PCI_FUNC(as->devfn), 1554dbaabb25SPeter Xu use_iommu); 1555dbaabb25SPeter Xu 155666a4a031SPeter Xu /* 155766a4a031SPeter Xu * It's possible that we reach here without BQL, e.g., when called 155866a4a031SPeter Xu * from vtd_pt_enable_fast_path(). However the memory APIs need 155966a4a031SPeter Xu * it. We'd better make sure we have had it already, or, take it. 156066a4a031SPeter Xu */ 156166a4a031SPeter Xu if (take_bql) { 156266a4a031SPeter Xu qemu_mutex_lock_iothread(); 156366a4a031SPeter Xu } 156466a4a031SPeter Xu 1565dbaabb25SPeter Xu /* Turn off first then on the other */ 1566dbaabb25SPeter Xu if (use_iommu) { 15674b519ef1SPeter Xu memory_region_set_enabled(&as->nodmar, false); 15683df9d748SAlexey Kardashevskiy memory_region_set_enabled(MEMORY_REGION(&as->iommu), true); 1569dbaabb25SPeter Xu } else { 15703df9d748SAlexey Kardashevskiy memory_region_set_enabled(MEMORY_REGION(&as->iommu), false); 15714b519ef1SPeter Xu memory_region_set_enabled(&as->nodmar, true); 1572dbaabb25SPeter Xu } 1573dbaabb25SPeter Xu 157466a4a031SPeter Xu if (take_bql) { 157566a4a031SPeter Xu qemu_mutex_unlock_iothread(); 157666a4a031SPeter Xu } 157766a4a031SPeter Xu 1578dbaabb25SPeter Xu return use_iommu; 1579dbaabb25SPeter Xu } 1580dbaabb25SPeter Xu 1581dbaabb25SPeter Xu static void vtd_switch_address_space_all(IntelIOMMUState *s) 1582dbaabb25SPeter Xu { 1583dbaabb25SPeter Xu GHashTableIter iter; 1584dbaabb25SPeter Xu VTDBus *vtd_bus; 1585dbaabb25SPeter Xu int i; 1586dbaabb25SPeter Xu 1587dbaabb25SPeter Xu g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); 1588dbaabb25SPeter Xu while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) { 1589bf33cc75SPeter Xu for (i = 0; i < PCI_DEVFN_MAX; i++) { 1590dbaabb25SPeter Xu if (!vtd_bus->dev_as[i]) { 1591dbaabb25SPeter Xu continue; 1592dbaabb25SPeter Xu } 1593dbaabb25SPeter Xu vtd_switch_address_space(vtd_bus->dev_as[i]); 1594dbaabb25SPeter Xu } 1595dbaabb25SPeter Xu } 1596dbaabb25SPeter Xu } 1597dbaabb25SPeter Xu 15981da12ec4SLe Tan static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn) 15991da12ec4SLe Tan { 16001da12ec4SLe Tan return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL); 16011da12ec4SLe Tan } 16021da12ec4SLe Tan 16031da12ec4SLe Tan static const bool vtd_qualified_faults[] = { 16041da12ec4SLe Tan [VTD_FR_RESERVED] = false, 16051da12ec4SLe Tan [VTD_FR_ROOT_ENTRY_P] = false, 16061da12ec4SLe Tan [VTD_FR_CONTEXT_ENTRY_P] = true, 16071da12ec4SLe Tan [VTD_FR_CONTEXT_ENTRY_INV] = true, 16081da12ec4SLe Tan [VTD_FR_ADDR_BEYOND_MGAW] = true, 16091da12ec4SLe Tan [VTD_FR_WRITE] = true, 16101da12ec4SLe Tan [VTD_FR_READ] = true, 16111da12ec4SLe Tan [VTD_FR_PAGING_ENTRY_INV] = true, 16121da12ec4SLe Tan [VTD_FR_ROOT_TABLE_INV] = false, 16131da12ec4SLe Tan [VTD_FR_CONTEXT_TABLE_INV] = false, 16141da12ec4SLe Tan [VTD_FR_ROOT_ENTRY_RSVD] = false, 16151da12ec4SLe Tan [VTD_FR_PAGING_ENTRY_RSVD] = true, 16161da12ec4SLe Tan [VTD_FR_CONTEXT_ENTRY_TT] = true, 1617fb43cf73SLiu, Yi L [VTD_FR_PASID_TABLE_INV] = false, 16181da12ec4SLe Tan [VTD_FR_RESERVED_ERR] = false, 16191da12ec4SLe Tan [VTD_FR_MAX] = false, 16201da12ec4SLe Tan }; 16211da12ec4SLe Tan 16221da12ec4SLe Tan /* To see if a fault condition is "qualified", which is reported to software 16231da12ec4SLe Tan * only if the FPD field in the context-entry used to process the faulting 16241da12ec4SLe Tan * request is 0. 16251da12ec4SLe Tan */ 16261da12ec4SLe Tan static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 16271da12ec4SLe Tan { 16281da12ec4SLe Tan return vtd_qualified_faults[fault]; 16291da12ec4SLe Tan } 16301da12ec4SLe Tan 16311da12ec4SLe Tan static inline bool vtd_is_interrupt_addr(hwaddr addr) 16321da12ec4SLe Tan { 16331da12ec4SLe Tan return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 16341da12ec4SLe Tan } 16351da12ec4SLe Tan 1636dbaabb25SPeter Xu static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id) 1637dbaabb25SPeter Xu { 1638dbaabb25SPeter Xu VTDBus *vtd_bus; 1639dbaabb25SPeter Xu VTDAddressSpace *vtd_as; 1640dbaabb25SPeter Xu bool success = false; 1641dbaabb25SPeter Xu 1642dbaabb25SPeter Xu vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id)); 1643dbaabb25SPeter Xu if (!vtd_bus) { 1644dbaabb25SPeter Xu goto out; 1645dbaabb25SPeter Xu } 1646dbaabb25SPeter Xu 1647dbaabb25SPeter Xu vtd_as = vtd_bus->dev_as[VTD_SID_TO_DEVFN(source_id)]; 1648dbaabb25SPeter Xu if (!vtd_as) { 1649dbaabb25SPeter Xu goto out; 1650dbaabb25SPeter Xu } 1651dbaabb25SPeter Xu 1652dbaabb25SPeter Xu if (vtd_switch_address_space(vtd_as) == false) { 1653dbaabb25SPeter Xu /* We switched off IOMMU region successfully. */ 1654dbaabb25SPeter Xu success = true; 1655dbaabb25SPeter Xu } 1656dbaabb25SPeter Xu 1657dbaabb25SPeter Xu out: 1658dbaabb25SPeter Xu trace_vtd_pt_enable_fast_path(source_id, success); 1659dbaabb25SPeter Xu } 1660dbaabb25SPeter Xu 16611da12ec4SLe Tan /* Map dev to context-entry then do a paging-structures walk to do a iommu 16621da12ec4SLe Tan * translation. 166379e2b9aeSPaolo Bonzini * 166479e2b9aeSPaolo Bonzini * Called from RCU critical section. 166579e2b9aeSPaolo Bonzini * 16661da12ec4SLe Tan * @bus_num: The bus number 16671da12ec4SLe Tan * @devfn: The devfn, which is the combined of device and function number 16681da12ec4SLe Tan * @is_write: The access is a write operation 16691da12ec4SLe Tan * @entry: IOMMUTLBEntry that contain the addr to be translated and result 1670b9313021SPeter Xu * 1671b9313021SPeter Xu * Returns true if translation is successful, otherwise false. 16721da12ec4SLe Tan */ 1673b9313021SPeter Xu static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, 16741da12ec4SLe Tan uint8_t devfn, hwaddr addr, bool is_write, 16751da12ec4SLe Tan IOMMUTLBEntry *entry) 16761da12ec4SLe Tan { 1677d92fa2dcSLe Tan IntelIOMMUState *s = vtd_as->iommu_state; 16781da12ec4SLe Tan VTDContextEntry ce; 16797df953bdSKnut Omang uint8_t bus_num = pci_bus_num(bus); 16801d9efa73SPeter Xu VTDContextCacheEntry *cc_entry; 1681d66b969bSJason Wang uint64_t slpte, page_mask; 16821da12ec4SLe Tan uint32_t level; 16831da12ec4SLe Tan uint16_t source_id = vtd_make_source_id(bus_num, devfn); 16841da12ec4SLe Tan int ret_fr; 16851da12ec4SLe Tan bool is_fpd_set = false; 16861da12ec4SLe Tan bool reads = true; 16871da12ec4SLe Tan bool writes = true; 168807f7b733SPeter Xu uint8_t access_flags; 1689b5a280c0SLe Tan VTDIOTLBEntry *iotlb_entry; 16901da12ec4SLe Tan 1691046ab7e9SPeter Xu /* 1692046ab7e9SPeter Xu * We have standalone memory region for interrupt addresses, we 1693046ab7e9SPeter Xu * should never receive translation requests in this region. 16941da12ec4SLe Tan */ 1695046ab7e9SPeter Xu assert(!vtd_is_interrupt_addr(addr)); 1696046ab7e9SPeter Xu 16971d9efa73SPeter Xu vtd_iommu_lock(s); 16981d9efa73SPeter Xu 16991d9efa73SPeter Xu cc_entry = &vtd_as->context_cache_entry; 17001d9efa73SPeter Xu 1701b5a280c0SLe Tan /* Try to fetch slpte form IOTLB */ 1702b5a280c0SLe Tan iotlb_entry = vtd_lookup_iotlb(s, source_id, addr); 1703b5a280c0SLe Tan if (iotlb_entry) { 17046c441e1dSPeter Xu trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, 17056c441e1dSPeter Xu iotlb_entry->domain_id); 1706b5a280c0SLe Tan slpte = iotlb_entry->slpte; 170707f7b733SPeter Xu access_flags = iotlb_entry->access_flags; 1708d66b969bSJason Wang page_mask = iotlb_entry->mask; 1709b5a280c0SLe Tan goto out; 1710b5a280c0SLe Tan } 1711b9313021SPeter Xu 1712d92fa2dcSLe Tan /* Try to fetch context-entry from cache first */ 1713d92fa2dcSLe Tan if (cc_entry->context_cache_gen == s->context_cache_gen) { 17146c441e1dSPeter Xu trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi, 17156c441e1dSPeter Xu cc_entry->context_entry.lo, 17166c441e1dSPeter Xu cc_entry->context_cache_gen); 1717d92fa2dcSLe Tan ce = cc_entry->context_entry; 1718d92fa2dcSLe Tan is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1719fb43cf73SLiu, Yi L if (!is_fpd_set && s->root_scalable) { 1720fb43cf73SLiu, Yi L ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set); 1721fb43cf73SLiu, Yi L VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write); 1722fb43cf73SLiu, Yi L } 1723d92fa2dcSLe Tan } else { 17241da12ec4SLe Tan ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 17251da12ec4SLe Tan is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1726fb43cf73SLiu, Yi L if (!ret_fr && !is_fpd_set && s->root_scalable) { 1727fb43cf73SLiu, Yi L ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set); 17281da12ec4SLe Tan } 1729fb43cf73SLiu, Yi L VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write); 1730d92fa2dcSLe Tan /* Update context-cache */ 17316c441e1dSPeter Xu trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo, 17326c441e1dSPeter Xu cc_entry->context_cache_gen, 17336c441e1dSPeter Xu s->context_cache_gen); 1734d92fa2dcSLe Tan cc_entry->context_entry = ce; 1735d92fa2dcSLe Tan cc_entry->context_cache_gen = s->context_cache_gen; 1736d92fa2dcSLe Tan } 17371da12ec4SLe Tan 1738dbaabb25SPeter Xu /* 1739dbaabb25SPeter Xu * We don't need to translate for pass-through context entries. 1740dbaabb25SPeter Xu * Also, let's ignore IOTLB caching as well for PT devices. 1741dbaabb25SPeter Xu */ 1742dbaabb25SPeter Xu if (vtd_ce_get_type(&ce) == VTD_CONTEXT_TT_PASS_THROUGH) { 1743892721d9SPeter Xu entry->iova = addr & VTD_PAGE_MASK_4K; 1744dbaabb25SPeter Xu entry->translated_addr = entry->iova; 1745892721d9SPeter Xu entry->addr_mask = ~VTD_PAGE_MASK_4K; 1746dbaabb25SPeter Xu entry->perm = IOMMU_RW; 1747dbaabb25SPeter Xu trace_vtd_translate_pt(source_id, entry->iova); 1748dbaabb25SPeter Xu 1749dbaabb25SPeter Xu /* 1750dbaabb25SPeter Xu * When this happens, it means firstly caching-mode is not 1751dbaabb25SPeter Xu * enabled, and this is the first passthrough translation for 1752dbaabb25SPeter Xu * the device. Let's enable the fast path for passthrough. 1753dbaabb25SPeter Xu * 1754dbaabb25SPeter Xu * When passthrough is disabled again for the device, we can 1755dbaabb25SPeter Xu * capture it via the context entry invalidation, then the 1756dbaabb25SPeter Xu * IOMMU region can be swapped back. 1757dbaabb25SPeter Xu */ 1758dbaabb25SPeter Xu vtd_pt_enable_fast_path(s, source_id); 17591d9efa73SPeter Xu vtd_iommu_unlock(s); 1760b9313021SPeter Xu return true; 1761dbaabb25SPeter Xu } 1762dbaabb25SPeter Xu 1763fb43cf73SLiu, Yi L ret_fr = vtd_iova_to_slpte(s, &ce, addr, is_write, &slpte, &level, 176437f51384SPrasad Singamsetty &reads, &writes, s->aw_bits); 1765fb43cf73SLiu, Yi L VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write); 17661da12ec4SLe Tan 1767d66b969bSJason Wang page_mask = vtd_slpt_level_page_mask(level); 176807f7b733SPeter Xu access_flags = IOMMU_ACCESS_FLAG(reads, writes); 1769fb43cf73SLiu, Yi L vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce), addr, slpte, 177007f7b733SPeter Xu access_flags, level); 1771b5a280c0SLe Tan out: 17721d9efa73SPeter Xu vtd_iommu_unlock(s); 1773d66b969bSJason Wang entry->iova = addr & page_mask; 177437f51384SPrasad Singamsetty entry->translated_addr = vtd_get_slpte_addr(slpte, s->aw_bits) & page_mask; 1775d66b969bSJason Wang entry->addr_mask = ~page_mask; 177607f7b733SPeter Xu entry->perm = access_flags; 1777b9313021SPeter Xu return true; 1778b9313021SPeter Xu 1779b9313021SPeter Xu error: 17801d9efa73SPeter Xu vtd_iommu_unlock(s); 1781b9313021SPeter Xu entry->iova = 0; 1782b9313021SPeter Xu entry->translated_addr = 0; 1783b9313021SPeter Xu entry->addr_mask = 0; 1784b9313021SPeter Xu entry->perm = IOMMU_NONE; 1785b9313021SPeter Xu return false; 17861da12ec4SLe Tan } 17871da12ec4SLe Tan 17881da12ec4SLe Tan static void vtd_root_table_setup(IntelIOMMUState *s) 17891da12ec4SLe Tan { 17901da12ec4SLe Tan s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 179137f51384SPrasad Singamsetty s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits); 17921da12ec4SLe Tan 17932811af3bSPeter Xu vtd_update_scalable_state(s); 17942811af3bSPeter Xu 179581fb1e64SPeter Xu trace_vtd_reg_dmar_root(s->root, s->root_scalable); 17961da12ec4SLe Tan } 17971da12ec4SLe Tan 179802a2cbc8SPeter Xu static void vtd_iec_notify_all(IntelIOMMUState *s, bool global, 179902a2cbc8SPeter Xu uint32_t index, uint32_t mask) 180002a2cbc8SPeter Xu { 180102a2cbc8SPeter Xu x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask); 180202a2cbc8SPeter Xu } 180302a2cbc8SPeter Xu 1804a5861439SPeter Xu static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s) 1805a5861439SPeter Xu { 1806a5861439SPeter Xu uint64_t value = 0; 1807a5861439SPeter Xu value = vtd_get_quad_raw(s, DMAR_IRTA_REG); 1808a5861439SPeter Xu s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1); 180937f51384SPrasad Singamsetty s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits); 181028589311SJan Kiszka s->intr_eime = value & VTD_IRTA_EIME; 1811a5861439SPeter Xu 181202a2cbc8SPeter Xu /* Notify global invalidation */ 181302a2cbc8SPeter Xu vtd_iec_notify_all(s, true, 0, 0); 1814a5861439SPeter Xu 18157feb51b7SPeter Xu trace_vtd_reg_ir_root(s->intr_root, s->intr_size); 1816a5861439SPeter Xu } 1817a5861439SPeter Xu 1818dd4d607eSPeter Xu static void vtd_iommu_replay_all(IntelIOMMUState *s) 1819dd4d607eSPeter Xu { 1820b4a4ba0dSPeter Xu VTDAddressSpace *vtd_as; 1821dd4d607eSPeter Xu 1822b4a4ba0dSPeter Xu QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 182363b88968SPeter Xu vtd_sync_shadow_page_table(vtd_as); 1824dd4d607eSPeter Xu } 1825dd4d607eSPeter Xu } 1826dd4d607eSPeter Xu 1827d92fa2dcSLe Tan static void vtd_context_global_invalidate(IntelIOMMUState *s) 1828d92fa2dcSLe Tan { 1829bc535e59SPeter Xu trace_vtd_inv_desc_cc_global(); 18301d9efa73SPeter Xu /* Protects context cache */ 18311d9efa73SPeter Xu vtd_iommu_lock(s); 1832d92fa2dcSLe Tan s->context_cache_gen++; 1833d92fa2dcSLe Tan if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 18341d9efa73SPeter Xu vtd_reset_context_cache_locked(s); 1835d92fa2dcSLe Tan } 18361d9efa73SPeter Xu vtd_iommu_unlock(s); 18372cc9ddccSPeter Xu vtd_address_space_refresh_all(s); 1838dd4d607eSPeter Xu /* 1839dd4d607eSPeter Xu * From VT-d spec 6.5.2.1, a global context entry invalidation 1840dd4d607eSPeter Xu * should be followed by a IOTLB global invalidation, so we should 1841dd4d607eSPeter Xu * be safe even without this. Hoewever, let's replay the region as 1842dd4d607eSPeter Xu * well to be safer, and go back here when we need finer tunes for 1843dd4d607eSPeter Xu * VT-d emulation codes. 1844dd4d607eSPeter Xu */ 1845dd4d607eSPeter Xu vtd_iommu_replay_all(s); 1846d92fa2dcSLe Tan } 1847d92fa2dcSLe Tan 1848d92fa2dcSLe Tan /* Do a context-cache device-selective invalidation. 1849d92fa2dcSLe Tan * @func_mask: FM field after shifting 1850d92fa2dcSLe Tan */ 1851d92fa2dcSLe Tan static void vtd_context_device_invalidate(IntelIOMMUState *s, 1852d92fa2dcSLe Tan uint16_t source_id, 1853d92fa2dcSLe Tan uint16_t func_mask) 1854d92fa2dcSLe Tan { 1855d92fa2dcSLe Tan uint16_t mask; 18567df953bdSKnut Omang VTDBus *vtd_bus; 1857d92fa2dcSLe Tan VTDAddressSpace *vtd_as; 1858bc535e59SPeter Xu uint8_t bus_n, devfn; 1859d92fa2dcSLe Tan uint16_t devfn_it; 1860d92fa2dcSLe Tan 1861bc535e59SPeter Xu trace_vtd_inv_desc_cc_devices(source_id, func_mask); 1862bc535e59SPeter Xu 1863d92fa2dcSLe Tan switch (func_mask & 3) { 1864d92fa2dcSLe Tan case 0: 1865d92fa2dcSLe Tan mask = 0; /* No bits in the SID field masked */ 1866d92fa2dcSLe Tan break; 1867d92fa2dcSLe Tan case 1: 1868d92fa2dcSLe Tan mask = 4; /* Mask bit 2 in the SID field */ 1869d92fa2dcSLe Tan break; 1870d92fa2dcSLe Tan case 2: 1871d92fa2dcSLe Tan mask = 6; /* Mask bit 2:1 in the SID field */ 1872d92fa2dcSLe Tan break; 1873d92fa2dcSLe Tan case 3: 1874d92fa2dcSLe Tan mask = 7; /* Mask bit 2:0 in the SID field */ 1875d92fa2dcSLe Tan break; 1876d92fa2dcSLe Tan } 18776cb99accSPeter Xu mask = ~mask; 1878bc535e59SPeter Xu 1879bc535e59SPeter Xu bus_n = VTD_SID_TO_BUS(source_id); 1880bc535e59SPeter Xu vtd_bus = vtd_find_as_from_bus_num(s, bus_n); 18817df953bdSKnut Omang if (vtd_bus) { 1882d92fa2dcSLe Tan devfn = VTD_SID_TO_DEVFN(source_id); 1883bf33cc75SPeter Xu for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) { 18847df953bdSKnut Omang vtd_as = vtd_bus->dev_as[devfn_it]; 1885d92fa2dcSLe Tan if (vtd_as && ((devfn_it & mask) == (devfn & mask))) { 1886bc535e59SPeter Xu trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(devfn_it), 1887bc535e59SPeter Xu VTD_PCI_FUNC(devfn_it)); 18881d9efa73SPeter Xu vtd_iommu_lock(s); 1889d92fa2dcSLe Tan vtd_as->context_cache_entry.context_cache_gen = 0; 18901d9efa73SPeter Xu vtd_iommu_unlock(s); 1891dd4d607eSPeter Xu /* 1892dbaabb25SPeter Xu * Do switch address space when needed, in case if the 1893dbaabb25SPeter Xu * device passthrough bit is switched. 1894dbaabb25SPeter Xu */ 1895dbaabb25SPeter Xu vtd_switch_address_space(vtd_as); 1896dbaabb25SPeter Xu /* 1897dd4d607eSPeter Xu * So a device is moving out of (or moving into) a 189863b88968SPeter Xu * domain, resync the shadow page table. 1899dd4d607eSPeter Xu * This won't bring bad even if we have no such 1900dd4d607eSPeter Xu * notifier registered - the IOMMU notification 1901dd4d607eSPeter Xu * framework will skip MAP notifications if that 1902dd4d607eSPeter Xu * happened. 1903dd4d607eSPeter Xu */ 190463b88968SPeter Xu vtd_sync_shadow_page_table(vtd_as); 1905d92fa2dcSLe Tan } 1906d92fa2dcSLe Tan } 1907d92fa2dcSLe Tan } 1908d92fa2dcSLe Tan } 1909d92fa2dcSLe Tan 19101da12ec4SLe Tan /* Context-cache invalidation 19111da12ec4SLe Tan * Returns the Context Actual Invalidation Granularity. 19121da12ec4SLe Tan * @val: the content of the CCMD_REG 19131da12ec4SLe Tan */ 19141da12ec4SLe Tan static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 19151da12ec4SLe Tan { 19161da12ec4SLe Tan uint64_t caig; 19171da12ec4SLe Tan uint64_t type = val & VTD_CCMD_CIRG_MASK; 19181da12ec4SLe Tan 19191da12ec4SLe Tan switch (type) { 19201da12ec4SLe Tan case VTD_CCMD_DOMAIN_INVL: 1921d92fa2dcSLe Tan /* Fall through */ 1922d92fa2dcSLe Tan case VTD_CCMD_GLOBAL_INVL: 1923d92fa2dcSLe Tan caig = VTD_CCMD_GLOBAL_INVL_A; 1924d92fa2dcSLe Tan vtd_context_global_invalidate(s); 19251da12ec4SLe Tan break; 19261da12ec4SLe Tan 19271da12ec4SLe Tan case VTD_CCMD_DEVICE_INVL: 19281da12ec4SLe Tan caig = VTD_CCMD_DEVICE_INVL_A; 1929d92fa2dcSLe Tan vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 19301da12ec4SLe Tan break; 19311da12ec4SLe Tan 19321da12ec4SLe Tan default: 19331376211fSPeter Xu error_report_once("%s: invalid context: 0x%" PRIx64, 19341376211fSPeter Xu __func__, val); 19351da12ec4SLe Tan caig = 0; 19361da12ec4SLe Tan } 19371da12ec4SLe Tan return caig; 19381da12ec4SLe Tan } 19391da12ec4SLe Tan 1940b5a280c0SLe Tan static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 1941b5a280c0SLe Tan { 19427feb51b7SPeter Xu trace_vtd_inv_desc_iotlb_global(); 1943b5a280c0SLe Tan vtd_reset_iotlb(s); 1944dd4d607eSPeter Xu vtd_iommu_replay_all(s); 1945b5a280c0SLe Tan } 1946b5a280c0SLe Tan 1947b5a280c0SLe Tan static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 1948b5a280c0SLe Tan { 1949dd4d607eSPeter Xu VTDContextEntry ce; 1950dd4d607eSPeter Xu VTDAddressSpace *vtd_as; 1951dd4d607eSPeter Xu 19527feb51b7SPeter Xu trace_vtd_inv_desc_iotlb_domain(domain_id); 19537feb51b7SPeter Xu 19541d9efa73SPeter Xu vtd_iommu_lock(s); 1955b5a280c0SLe Tan g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 1956b5a280c0SLe Tan &domain_id); 19571d9efa73SPeter Xu vtd_iommu_unlock(s); 1958dd4d607eSPeter Xu 1959b4a4ba0dSPeter Xu QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 1960dd4d607eSPeter Xu if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 1961dd4d607eSPeter Xu vtd_as->devfn, &ce) && 1962fb43cf73SLiu, Yi L domain_id == vtd_get_domain_id(s, &ce)) { 196363b88968SPeter Xu vtd_sync_shadow_page_table(vtd_as); 1964dd4d607eSPeter Xu } 1965dd4d607eSPeter Xu } 1966dd4d607eSPeter Xu } 1967dd4d607eSPeter Xu 1968dd4d607eSPeter Xu static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, 1969dd4d607eSPeter Xu uint16_t domain_id, hwaddr addr, 1970dd4d607eSPeter Xu uint8_t am) 1971dd4d607eSPeter Xu { 1972b4a4ba0dSPeter Xu VTDAddressSpace *vtd_as; 1973dd4d607eSPeter Xu VTDContextEntry ce; 1974dd4d607eSPeter Xu int ret; 19754f8a62a9SPeter Xu hwaddr size = (1 << am) * VTD_PAGE_SIZE; 1976dd4d607eSPeter Xu 1977b4a4ba0dSPeter Xu QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) { 1978dd4d607eSPeter Xu ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 1979dd4d607eSPeter Xu vtd_as->devfn, &ce); 1980fb43cf73SLiu, Yi L if (!ret && domain_id == vtd_get_domain_id(s, &ce)) { 19814f8a62a9SPeter Xu if (vtd_as_has_map_notifier(vtd_as)) { 19824f8a62a9SPeter Xu /* 19834f8a62a9SPeter Xu * As long as we have MAP notifications registered in 19844f8a62a9SPeter Xu * any of our IOMMU notifiers, we need to sync the 19854f8a62a9SPeter Xu * shadow page table. 19864f8a62a9SPeter Xu */ 198763b88968SPeter Xu vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size); 19884f8a62a9SPeter Xu } else { 19894f8a62a9SPeter Xu /* 19904f8a62a9SPeter Xu * For UNMAP-only notifiers, we don't need to walk the 19914f8a62a9SPeter Xu * page tables. We just deliver the PSI down to 19924f8a62a9SPeter Xu * invalidate caches. 19934f8a62a9SPeter Xu */ 19944f8a62a9SPeter Xu IOMMUTLBEntry entry = { 19954f8a62a9SPeter Xu .target_as = &address_space_memory, 19964f8a62a9SPeter Xu .iova = addr, 19974f8a62a9SPeter Xu .translated_addr = 0, 19984f8a62a9SPeter Xu .addr_mask = size - 1, 19994f8a62a9SPeter Xu .perm = IOMMU_NONE, 20004f8a62a9SPeter Xu }; 2001cb1efcf4SPeter Maydell memory_region_notify_iommu(&vtd_as->iommu, 0, entry); 20024f8a62a9SPeter Xu } 2003dd4d607eSPeter Xu } 2004dd4d607eSPeter Xu } 2005b5a280c0SLe Tan } 2006b5a280c0SLe Tan 2007b5a280c0SLe Tan static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 2008b5a280c0SLe Tan hwaddr addr, uint8_t am) 2009b5a280c0SLe Tan { 2010b5a280c0SLe Tan VTDIOTLBPageInvInfo info; 2011b5a280c0SLe Tan 20127feb51b7SPeter Xu trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am); 20137feb51b7SPeter Xu 2014b5a280c0SLe Tan assert(am <= VTD_MAMV); 2015b5a280c0SLe Tan info.domain_id = domain_id; 2016d66b969bSJason Wang info.addr = addr; 2017b5a280c0SLe Tan info.mask = ~((1 << am) - 1); 20181d9efa73SPeter Xu vtd_iommu_lock(s); 2019b5a280c0SLe Tan g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 20201d9efa73SPeter Xu vtd_iommu_unlock(s); 2021dd4d607eSPeter Xu vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am); 2022b5a280c0SLe Tan } 2023b5a280c0SLe Tan 20241da12ec4SLe Tan /* Flush IOTLB 20251da12ec4SLe Tan * Returns the IOTLB Actual Invalidation Granularity. 20261da12ec4SLe Tan * @val: the content of the IOTLB_REG 20271da12ec4SLe Tan */ 20281da12ec4SLe Tan static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 20291da12ec4SLe Tan { 20301da12ec4SLe Tan uint64_t iaig; 20311da12ec4SLe Tan uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 2032b5a280c0SLe Tan uint16_t domain_id; 2033b5a280c0SLe Tan hwaddr addr; 2034b5a280c0SLe Tan uint8_t am; 20351da12ec4SLe Tan 20361da12ec4SLe Tan switch (type) { 20371da12ec4SLe Tan case VTD_TLB_GLOBAL_FLUSH: 20381da12ec4SLe Tan iaig = VTD_TLB_GLOBAL_FLUSH_A; 2039b5a280c0SLe Tan vtd_iotlb_global_invalidate(s); 20401da12ec4SLe Tan break; 20411da12ec4SLe Tan 20421da12ec4SLe Tan case VTD_TLB_DSI_FLUSH: 2043b5a280c0SLe Tan domain_id = VTD_TLB_DID(val); 20441da12ec4SLe Tan iaig = VTD_TLB_DSI_FLUSH_A; 2045b5a280c0SLe Tan vtd_iotlb_domain_invalidate(s, domain_id); 20461da12ec4SLe Tan break; 20471da12ec4SLe Tan 20481da12ec4SLe Tan case VTD_TLB_PSI_FLUSH: 2049b5a280c0SLe Tan domain_id = VTD_TLB_DID(val); 2050b5a280c0SLe Tan addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 2051b5a280c0SLe Tan am = VTD_IVA_AM(addr); 2052b5a280c0SLe Tan addr = VTD_IVA_ADDR(addr); 2053b5a280c0SLe Tan if (am > VTD_MAMV) { 20541376211fSPeter Xu error_report_once("%s: address mask overflow: 0x%" PRIx64, 20551376211fSPeter Xu __func__, vtd_get_quad_raw(s, DMAR_IVA_REG)); 2056b5a280c0SLe Tan iaig = 0; 2057b5a280c0SLe Tan break; 2058b5a280c0SLe Tan } 20591da12ec4SLe Tan iaig = VTD_TLB_PSI_FLUSH_A; 2060b5a280c0SLe Tan vtd_iotlb_page_invalidate(s, domain_id, addr, am); 20611da12ec4SLe Tan break; 20621da12ec4SLe Tan 20631da12ec4SLe Tan default: 20641376211fSPeter Xu error_report_once("%s: invalid granularity: 0x%" PRIx64, 20651376211fSPeter Xu __func__, val); 20661da12ec4SLe Tan iaig = 0; 20671da12ec4SLe Tan } 20681da12ec4SLe Tan return iaig; 20691da12ec4SLe Tan } 20701da12ec4SLe Tan 20718991c460SLadi Prosek static void vtd_fetch_inv_desc(IntelIOMMUState *s); 2072ed7b8fbcSLe Tan 2073ed7b8fbcSLe Tan static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 2074ed7b8fbcSLe Tan { 2075ed7b8fbcSLe Tan return s->qi_enabled && (s->iq_tail == s->iq_head) && 2076ed7b8fbcSLe Tan (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 2077ed7b8fbcSLe Tan } 2078ed7b8fbcSLe Tan 2079ed7b8fbcSLe Tan static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 2080ed7b8fbcSLe Tan { 2081ed7b8fbcSLe Tan uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 2082ed7b8fbcSLe Tan 20837feb51b7SPeter Xu trace_vtd_inv_qi_enable(en); 20847feb51b7SPeter Xu 2085ed7b8fbcSLe Tan if (en) { 208637f51384SPrasad Singamsetty s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits); 2087ed7b8fbcSLe Tan /* 2^(x+8) entries */ 2088c0c1d351SLiu, Yi L s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8 - (s->iq_dw ? 1 : 0)); 2089ed7b8fbcSLe Tan s->qi_enabled = true; 20907feb51b7SPeter Xu trace_vtd_inv_qi_setup(s->iq, s->iq_size); 2091ed7b8fbcSLe Tan /* Ok - report back to driver */ 2092ed7b8fbcSLe Tan vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 20938991c460SLadi Prosek 20948991c460SLadi Prosek if (s->iq_tail != 0) { 20958991c460SLadi Prosek /* 20968991c460SLadi Prosek * This is a spec violation but Windows guests are known to set up 20978991c460SLadi Prosek * Queued Invalidation this way so we allow the write and process 20988991c460SLadi Prosek * Invalidation Descriptors right away. 20998991c460SLadi Prosek */ 21008991c460SLadi Prosek trace_vtd_warn_invalid_qi_tail(s->iq_tail); 21018991c460SLadi Prosek if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 21028991c460SLadi Prosek vtd_fetch_inv_desc(s); 21038991c460SLadi Prosek } 2104ed7b8fbcSLe Tan } 2105ed7b8fbcSLe Tan } else { 2106ed7b8fbcSLe Tan if (vtd_queued_inv_disable_check(s)) { 2107ed7b8fbcSLe Tan /* disable Queued Invalidation */ 2108ed7b8fbcSLe Tan vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 2109ed7b8fbcSLe Tan s->iq_head = 0; 2110ed7b8fbcSLe Tan s->qi_enabled = false; 2111ed7b8fbcSLe Tan /* Ok - report back to driver */ 2112ed7b8fbcSLe Tan vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 2113ed7b8fbcSLe Tan } else { 21144e4abd11SPeter Xu error_report_once("%s: detected improper state when disable QI " 21154e4abd11SPeter Xu "(head=0x%x, tail=0x%x, last_type=%d)", 21164e4abd11SPeter Xu __func__, 21174e4abd11SPeter Xu s->iq_head, s->iq_tail, s->iq_last_desc_type); 2118ed7b8fbcSLe Tan } 2119ed7b8fbcSLe Tan } 2120ed7b8fbcSLe Tan } 2121ed7b8fbcSLe Tan 21221da12ec4SLe Tan /* Set Root Table Pointer */ 21231da12ec4SLe Tan static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 21241da12ec4SLe Tan { 21251da12ec4SLe Tan vtd_root_table_setup(s); 21261da12ec4SLe Tan /* Ok - report back to driver */ 21271da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 21282cc9ddccSPeter Xu vtd_reset_caches(s); 21292cc9ddccSPeter Xu vtd_address_space_refresh_all(s); 21301da12ec4SLe Tan } 21311da12ec4SLe Tan 2132a5861439SPeter Xu /* Set Interrupt Remap Table Pointer */ 2133a5861439SPeter Xu static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s) 2134a5861439SPeter Xu { 2135a5861439SPeter Xu vtd_interrupt_remap_table_setup(s); 2136a5861439SPeter Xu /* Ok - report back to driver */ 2137a5861439SPeter Xu vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS); 2138a5861439SPeter Xu } 2139a5861439SPeter Xu 21401da12ec4SLe Tan /* Handle Translation Enable/Disable */ 21411da12ec4SLe Tan static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 21421da12ec4SLe Tan { 2143558e0024SPeter Xu if (s->dmar_enabled == en) { 2144558e0024SPeter Xu return; 2145558e0024SPeter Xu } 2146558e0024SPeter Xu 21477feb51b7SPeter Xu trace_vtd_dmar_enable(en); 21481da12ec4SLe Tan 21491da12ec4SLe Tan if (en) { 21501da12ec4SLe Tan s->dmar_enabled = true; 21511da12ec4SLe Tan /* Ok - report back to driver */ 21521da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 21531da12ec4SLe Tan } else { 21541da12ec4SLe Tan s->dmar_enabled = false; 21551da12ec4SLe Tan 21561da12ec4SLe Tan /* Clear the index of Fault Recording Register */ 21571da12ec4SLe Tan s->next_frcd_reg = 0; 21581da12ec4SLe Tan /* Ok - report back to driver */ 21591da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 21601da12ec4SLe Tan } 2161558e0024SPeter Xu 21622cc9ddccSPeter Xu vtd_reset_caches(s); 21632cc9ddccSPeter Xu vtd_address_space_refresh_all(s); 21641da12ec4SLe Tan } 21651da12ec4SLe Tan 216680de52baSPeter Xu /* Handle Interrupt Remap Enable/Disable */ 216780de52baSPeter Xu static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en) 216880de52baSPeter Xu { 21697feb51b7SPeter Xu trace_vtd_ir_enable(en); 217080de52baSPeter Xu 217180de52baSPeter Xu if (en) { 217280de52baSPeter Xu s->intr_enabled = true; 217380de52baSPeter Xu /* Ok - report back to driver */ 217480de52baSPeter Xu vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES); 217580de52baSPeter Xu } else { 217680de52baSPeter Xu s->intr_enabled = false; 217780de52baSPeter Xu /* Ok - report back to driver */ 217880de52baSPeter Xu vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0); 217980de52baSPeter Xu } 218080de52baSPeter Xu } 218180de52baSPeter Xu 21821da12ec4SLe Tan /* Handle write to Global Command Register */ 21831da12ec4SLe Tan static void vtd_handle_gcmd_write(IntelIOMMUState *s) 21841da12ec4SLe Tan { 21851da12ec4SLe Tan uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 21861da12ec4SLe Tan uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 21871da12ec4SLe Tan uint32_t changed = status ^ val; 21881da12ec4SLe Tan 21897feb51b7SPeter Xu trace_vtd_reg_write_gcmd(status, val); 21901da12ec4SLe Tan if (changed & VTD_GCMD_TE) { 21911da12ec4SLe Tan /* Translation enable/disable */ 21921da12ec4SLe Tan vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 21931da12ec4SLe Tan } 21941da12ec4SLe Tan if (val & VTD_GCMD_SRTP) { 21951da12ec4SLe Tan /* Set/update the root-table pointer */ 21961da12ec4SLe Tan vtd_handle_gcmd_srtp(s); 21971da12ec4SLe Tan } 2198ed7b8fbcSLe Tan if (changed & VTD_GCMD_QIE) { 2199ed7b8fbcSLe Tan /* Queued Invalidation Enable */ 2200ed7b8fbcSLe Tan vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 2201ed7b8fbcSLe Tan } 2202a5861439SPeter Xu if (val & VTD_GCMD_SIRTP) { 2203a5861439SPeter Xu /* Set/update the interrupt remapping root-table pointer */ 2204a5861439SPeter Xu vtd_handle_gcmd_sirtp(s); 2205a5861439SPeter Xu } 220680de52baSPeter Xu if (changed & VTD_GCMD_IRE) { 220780de52baSPeter Xu /* Interrupt remap enable/disable */ 220880de52baSPeter Xu vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE); 220980de52baSPeter Xu } 22101da12ec4SLe Tan } 22111da12ec4SLe Tan 22121da12ec4SLe Tan /* Handle write to Context Command Register */ 22131da12ec4SLe Tan static void vtd_handle_ccmd_write(IntelIOMMUState *s) 22141da12ec4SLe Tan { 22151da12ec4SLe Tan uint64_t ret; 22161da12ec4SLe Tan uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 22171da12ec4SLe Tan 22181da12ec4SLe Tan /* Context-cache invalidation request */ 22191da12ec4SLe Tan if (val & VTD_CCMD_ICC) { 2220ed7b8fbcSLe Tan if (s->qi_enabled) { 22211376211fSPeter Xu error_report_once("Queued Invalidation enabled, " 2222ed7b8fbcSLe Tan "should not use register-based invalidation"); 2223ed7b8fbcSLe Tan return; 2224ed7b8fbcSLe Tan } 22251da12ec4SLe Tan ret = vtd_context_cache_invalidate(s, val); 22261da12ec4SLe Tan /* Invalidation completed. Change something to show */ 22271da12ec4SLe Tan vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 22281da12ec4SLe Tan ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 22291da12ec4SLe Tan ret); 22301da12ec4SLe Tan } 22311da12ec4SLe Tan } 22321da12ec4SLe Tan 22331da12ec4SLe Tan /* Handle write to IOTLB Invalidation Register */ 22341da12ec4SLe Tan static void vtd_handle_iotlb_write(IntelIOMMUState *s) 22351da12ec4SLe Tan { 22361da12ec4SLe Tan uint64_t ret; 22371da12ec4SLe Tan uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 22381da12ec4SLe Tan 22391da12ec4SLe Tan /* IOTLB invalidation request */ 22401da12ec4SLe Tan if (val & VTD_TLB_IVT) { 2241ed7b8fbcSLe Tan if (s->qi_enabled) { 22421376211fSPeter Xu error_report_once("Queued Invalidation enabled, " 22431376211fSPeter Xu "should not use register-based invalidation"); 2244ed7b8fbcSLe Tan return; 2245ed7b8fbcSLe Tan } 22461da12ec4SLe Tan ret = vtd_iotlb_flush(s, val); 22471da12ec4SLe Tan /* Invalidation completed. Change something to show */ 22481da12ec4SLe Tan vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 22491da12ec4SLe Tan ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 22501da12ec4SLe Tan VTD_TLB_FLUSH_GRANU_MASK_A, ret); 22511da12ec4SLe Tan } 22521da12ec4SLe Tan } 22531da12ec4SLe Tan 2254ed7b8fbcSLe Tan /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 2255c0c1d351SLiu, Yi L static bool vtd_get_inv_desc(IntelIOMMUState *s, 2256ed7b8fbcSLe Tan VTDInvDesc *inv_desc) 2257ed7b8fbcSLe Tan { 2258c0c1d351SLiu, Yi L dma_addr_t base_addr = s->iq; 2259c0c1d351SLiu, Yi L uint32_t offset = s->iq_head; 2260c0c1d351SLiu, Yi L uint32_t dw = s->iq_dw ? 32 : 16; 2261c0c1d351SLiu, Yi L dma_addr_t addr = base_addr + offset * dw; 2262c0c1d351SLiu, Yi L 2263c0c1d351SLiu, Yi L if (dma_memory_read(&address_space_memory, addr, inv_desc, dw)) { 2264c0c1d351SLiu, Yi L error_report_once("Read INV DESC failed."); 2265ed7b8fbcSLe Tan return false; 2266ed7b8fbcSLe Tan } 2267ed7b8fbcSLe Tan inv_desc->lo = le64_to_cpu(inv_desc->lo); 2268ed7b8fbcSLe Tan inv_desc->hi = le64_to_cpu(inv_desc->hi); 2269c0c1d351SLiu, Yi L if (dw == 32) { 2270c0c1d351SLiu, Yi L inv_desc->val[2] = le64_to_cpu(inv_desc->val[2]); 2271c0c1d351SLiu, Yi L inv_desc->val[3] = le64_to_cpu(inv_desc->val[3]); 2272c0c1d351SLiu, Yi L } 2273ed7b8fbcSLe Tan return true; 2274ed7b8fbcSLe Tan } 2275ed7b8fbcSLe Tan 2276ed7b8fbcSLe Tan static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 2277ed7b8fbcSLe Tan { 2278ed7b8fbcSLe Tan if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 2279ed7b8fbcSLe Tan (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 2280095955b2SPeter Xu error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64 2281095955b2SPeter Xu " (reserved nonzero)", __func__, inv_desc->hi, 2282095955b2SPeter Xu inv_desc->lo); 2283ed7b8fbcSLe Tan return false; 2284ed7b8fbcSLe Tan } 2285ed7b8fbcSLe Tan if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 2286ed7b8fbcSLe Tan /* Status Write */ 2287ed7b8fbcSLe Tan uint32_t status_data = (uint32_t)(inv_desc->lo >> 2288ed7b8fbcSLe Tan VTD_INV_DESC_WAIT_DATA_SHIFT); 2289ed7b8fbcSLe Tan 2290ed7b8fbcSLe Tan assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 2291ed7b8fbcSLe Tan 2292ed7b8fbcSLe Tan /* FIXME: need to be masked with HAW? */ 2293ed7b8fbcSLe Tan dma_addr_t status_addr = inv_desc->hi; 2294bc535e59SPeter Xu trace_vtd_inv_desc_wait_sw(status_addr, status_data); 2295ed7b8fbcSLe Tan status_data = cpu_to_le32(status_data); 2296ed7b8fbcSLe Tan if (dma_memory_write(&address_space_memory, status_addr, &status_data, 2297ed7b8fbcSLe Tan sizeof(status_data))) { 2298bc535e59SPeter Xu trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo); 2299ed7b8fbcSLe Tan return false; 2300ed7b8fbcSLe Tan } 2301ed7b8fbcSLe Tan } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 2302ed7b8fbcSLe Tan /* Interrupt flag */ 2303ed7b8fbcSLe Tan vtd_generate_completion_event(s); 2304ed7b8fbcSLe Tan } else { 2305095955b2SPeter Xu error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64 2306095955b2SPeter Xu " (unknown type)", __func__, inv_desc->hi, 2307095955b2SPeter Xu inv_desc->lo); 2308ed7b8fbcSLe Tan return false; 2309ed7b8fbcSLe Tan } 2310ed7b8fbcSLe Tan return true; 2311ed7b8fbcSLe Tan } 2312ed7b8fbcSLe Tan 2313d92fa2dcSLe Tan static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 2314d92fa2dcSLe Tan VTDInvDesc *inv_desc) 2315d92fa2dcSLe Tan { 2316bc535e59SPeter Xu uint16_t sid, fmask; 2317bc535e59SPeter Xu 2318d92fa2dcSLe Tan if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 2319095955b2SPeter Xu error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64 2320095955b2SPeter Xu " (reserved nonzero)", __func__, inv_desc->hi, 2321095955b2SPeter Xu inv_desc->lo); 2322d92fa2dcSLe Tan return false; 2323d92fa2dcSLe Tan } 2324d92fa2dcSLe Tan switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 2325d92fa2dcSLe Tan case VTD_INV_DESC_CC_DOMAIN: 2326bc535e59SPeter Xu trace_vtd_inv_desc_cc_domain( 2327d92fa2dcSLe Tan (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 2328d92fa2dcSLe Tan /* Fall through */ 2329d92fa2dcSLe Tan case VTD_INV_DESC_CC_GLOBAL: 2330d92fa2dcSLe Tan vtd_context_global_invalidate(s); 2331d92fa2dcSLe Tan break; 2332d92fa2dcSLe Tan 2333d92fa2dcSLe Tan case VTD_INV_DESC_CC_DEVICE: 2334bc535e59SPeter Xu sid = VTD_INV_DESC_CC_SID(inv_desc->lo); 2335bc535e59SPeter Xu fmask = VTD_INV_DESC_CC_FM(inv_desc->lo); 2336bc535e59SPeter Xu vtd_context_device_invalidate(s, sid, fmask); 2337d92fa2dcSLe Tan break; 2338d92fa2dcSLe Tan 2339d92fa2dcSLe Tan default: 2340095955b2SPeter Xu error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64 2341095955b2SPeter Xu " (invalid type)", __func__, inv_desc->hi, 2342095955b2SPeter Xu inv_desc->lo); 2343d92fa2dcSLe Tan return false; 2344d92fa2dcSLe Tan } 2345d92fa2dcSLe Tan return true; 2346d92fa2dcSLe Tan } 2347d92fa2dcSLe Tan 2348b5a280c0SLe Tan static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 2349b5a280c0SLe Tan { 2350b5a280c0SLe Tan uint16_t domain_id; 2351b5a280c0SLe Tan uint8_t am; 2352b5a280c0SLe Tan hwaddr addr; 2353b5a280c0SLe Tan 2354b5a280c0SLe Tan if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 2355b5a280c0SLe Tan (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 2356095955b2SPeter Xu error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2357095955b2SPeter Xu ", lo=0x%"PRIx64" (reserved bits unzero)\n", 2358095955b2SPeter Xu __func__, inv_desc->hi, inv_desc->lo); 2359b5a280c0SLe Tan return false; 2360b5a280c0SLe Tan } 2361b5a280c0SLe Tan 2362b5a280c0SLe Tan switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 2363b5a280c0SLe Tan case VTD_INV_DESC_IOTLB_GLOBAL: 2364b5a280c0SLe Tan vtd_iotlb_global_invalidate(s); 2365b5a280c0SLe Tan break; 2366b5a280c0SLe Tan 2367b5a280c0SLe Tan case VTD_INV_DESC_IOTLB_DOMAIN: 2368b5a280c0SLe Tan domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 2369b5a280c0SLe Tan vtd_iotlb_domain_invalidate(s, domain_id); 2370b5a280c0SLe Tan break; 2371b5a280c0SLe Tan 2372b5a280c0SLe Tan case VTD_INV_DESC_IOTLB_PAGE: 2373b5a280c0SLe Tan domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 2374b5a280c0SLe Tan addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 2375b5a280c0SLe Tan am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 2376b5a280c0SLe Tan if (am > VTD_MAMV) { 2377095955b2SPeter Xu error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2378095955b2SPeter Xu ", lo=0x%"PRIx64" (am=%u > VTD_MAMV=%u)\n", 2379095955b2SPeter Xu __func__, inv_desc->hi, inv_desc->lo, 2380095955b2SPeter Xu am, (unsigned)VTD_MAMV); 2381b5a280c0SLe Tan return false; 2382b5a280c0SLe Tan } 2383b5a280c0SLe Tan vtd_iotlb_page_invalidate(s, domain_id, addr, am); 2384b5a280c0SLe Tan break; 2385b5a280c0SLe Tan 2386b5a280c0SLe Tan default: 2387095955b2SPeter Xu error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2388095955b2SPeter Xu ", lo=0x%"PRIx64" (type mismatch: 0x%llx)\n", 2389095955b2SPeter Xu __func__, inv_desc->hi, inv_desc->lo, 2390095955b2SPeter Xu inv_desc->lo & VTD_INV_DESC_IOTLB_G); 2391b5a280c0SLe Tan return false; 2392b5a280c0SLe Tan } 2393b5a280c0SLe Tan return true; 2394b5a280c0SLe Tan } 2395b5a280c0SLe Tan 239602a2cbc8SPeter Xu static bool vtd_process_inv_iec_desc(IntelIOMMUState *s, 239702a2cbc8SPeter Xu VTDInvDesc *inv_desc) 239802a2cbc8SPeter Xu { 23997feb51b7SPeter Xu trace_vtd_inv_desc_iec(inv_desc->iec.granularity, 240002a2cbc8SPeter Xu inv_desc->iec.index, 240102a2cbc8SPeter Xu inv_desc->iec.index_mask); 240202a2cbc8SPeter Xu 240302a2cbc8SPeter Xu vtd_iec_notify_all(s, !inv_desc->iec.granularity, 240402a2cbc8SPeter Xu inv_desc->iec.index, 240502a2cbc8SPeter Xu inv_desc->iec.index_mask); 2406554f5e16SJason Wang return true; 2407554f5e16SJason Wang } 240802a2cbc8SPeter Xu 2409554f5e16SJason Wang static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s, 2410554f5e16SJason Wang VTDInvDesc *inv_desc) 2411554f5e16SJason Wang { 2412554f5e16SJason Wang VTDAddressSpace *vtd_dev_as; 2413554f5e16SJason Wang IOMMUTLBEntry entry; 2414554f5e16SJason Wang struct VTDBus *vtd_bus; 2415554f5e16SJason Wang hwaddr addr; 2416554f5e16SJason Wang uint64_t sz; 2417554f5e16SJason Wang uint16_t sid; 2418554f5e16SJason Wang uint8_t devfn; 2419554f5e16SJason Wang bool size; 2420554f5e16SJason Wang uint8_t bus_num; 2421554f5e16SJason Wang 2422554f5e16SJason Wang addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi); 2423554f5e16SJason Wang sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo); 2424554f5e16SJason Wang devfn = sid & 0xff; 2425554f5e16SJason Wang bus_num = sid >> 8; 2426554f5e16SJason Wang size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi); 2427554f5e16SJason Wang 2428554f5e16SJason Wang if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) || 2429554f5e16SJason Wang (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) { 2430095955b2SPeter Xu error_report_once("%s: invalid dev-iotlb inv desc: hi=%"PRIx64 2431095955b2SPeter Xu ", lo=%"PRIx64" (reserved nonzero)", __func__, 2432095955b2SPeter Xu inv_desc->hi, inv_desc->lo); 2433554f5e16SJason Wang return false; 2434554f5e16SJason Wang } 2435554f5e16SJason Wang 2436554f5e16SJason Wang vtd_bus = vtd_find_as_from_bus_num(s, bus_num); 2437554f5e16SJason Wang if (!vtd_bus) { 2438554f5e16SJason Wang goto done; 2439554f5e16SJason Wang } 2440554f5e16SJason Wang 2441554f5e16SJason Wang vtd_dev_as = vtd_bus->dev_as[devfn]; 2442554f5e16SJason Wang if (!vtd_dev_as) { 2443554f5e16SJason Wang goto done; 2444554f5e16SJason Wang } 2445554f5e16SJason Wang 244604eb6247SJason Wang /* According to ATS spec table 2.4: 244704eb6247SJason Wang * S = 0, bits 15:12 = xxxx range size: 4K 244804eb6247SJason Wang * S = 1, bits 15:12 = xxx0 range size: 8K 244904eb6247SJason Wang * S = 1, bits 15:12 = xx01 range size: 16K 245004eb6247SJason Wang * S = 1, bits 15:12 = x011 range size: 32K 245104eb6247SJason Wang * S = 1, bits 15:12 = 0111 range size: 64K 245204eb6247SJason Wang * ... 245304eb6247SJason Wang */ 2454554f5e16SJason Wang if (size) { 245504eb6247SJason Wang sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT); 2456554f5e16SJason Wang addr &= ~(sz - 1); 2457554f5e16SJason Wang } else { 2458554f5e16SJason Wang sz = VTD_PAGE_SIZE; 2459554f5e16SJason Wang } 2460554f5e16SJason Wang 2461554f5e16SJason Wang entry.target_as = &vtd_dev_as->as; 2462554f5e16SJason Wang entry.addr_mask = sz - 1; 2463554f5e16SJason Wang entry.iova = addr; 2464554f5e16SJason Wang entry.perm = IOMMU_NONE; 2465554f5e16SJason Wang entry.translated_addr = 0; 2466cb1efcf4SPeter Maydell memory_region_notify_iommu(&vtd_dev_as->iommu, 0, entry); 2467554f5e16SJason Wang 2468554f5e16SJason Wang done: 246902a2cbc8SPeter Xu return true; 247002a2cbc8SPeter Xu } 247102a2cbc8SPeter Xu 2472ed7b8fbcSLe Tan static bool vtd_process_inv_desc(IntelIOMMUState *s) 2473ed7b8fbcSLe Tan { 2474ed7b8fbcSLe Tan VTDInvDesc inv_desc; 2475ed7b8fbcSLe Tan uint8_t desc_type; 2476ed7b8fbcSLe Tan 24777feb51b7SPeter Xu trace_vtd_inv_qi_head(s->iq_head); 2478c0c1d351SLiu, Yi L if (!vtd_get_inv_desc(s, &inv_desc)) { 2479ed7b8fbcSLe Tan s->iq_last_desc_type = VTD_INV_DESC_NONE; 2480ed7b8fbcSLe Tan return false; 2481ed7b8fbcSLe Tan } 2482c0c1d351SLiu, Yi L 2483ed7b8fbcSLe Tan desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 2484ed7b8fbcSLe Tan /* FIXME: should update at first or at last? */ 2485ed7b8fbcSLe Tan s->iq_last_desc_type = desc_type; 2486ed7b8fbcSLe Tan 2487ed7b8fbcSLe Tan switch (desc_type) { 2488ed7b8fbcSLe Tan case VTD_INV_DESC_CC: 2489bc535e59SPeter Xu trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo); 2490d92fa2dcSLe Tan if (!vtd_process_context_cache_desc(s, &inv_desc)) { 2491d92fa2dcSLe Tan return false; 2492d92fa2dcSLe Tan } 2493ed7b8fbcSLe Tan break; 2494ed7b8fbcSLe Tan 2495ed7b8fbcSLe Tan case VTD_INV_DESC_IOTLB: 2496bc535e59SPeter Xu trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo); 2497b5a280c0SLe Tan if (!vtd_process_iotlb_desc(s, &inv_desc)) { 2498b5a280c0SLe Tan return false; 2499b5a280c0SLe Tan } 2500ed7b8fbcSLe Tan break; 2501ed7b8fbcSLe Tan 25024a4f219eSYi Sun /* 25034a4f219eSYi Sun * TODO: the entity of below two cases will be implemented in future series. 25044a4f219eSYi Sun * To make guest (which integrates scalable mode support patch set in 25054a4f219eSYi Sun * iommu driver) work, just return true is enough so far. 25064a4f219eSYi Sun */ 25074a4f219eSYi Sun case VTD_INV_DESC_PC: 25084a4f219eSYi Sun break; 25094a4f219eSYi Sun 25104a4f219eSYi Sun case VTD_INV_DESC_PIOTLB: 25114a4f219eSYi Sun break; 25124a4f219eSYi Sun 2513ed7b8fbcSLe Tan case VTD_INV_DESC_WAIT: 2514bc535e59SPeter Xu trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo); 2515ed7b8fbcSLe Tan if (!vtd_process_wait_desc(s, &inv_desc)) { 2516ed7b8fbcSLe Tan return false; 2517ed7b8fbcSLe Tan } 2518ed7b8fbcSLe Tan break; 2519ed7b8fbcSLe Tan 2520b7910472SPeter Xu case VTD_INV_DESC_IEC: 2521bc535e59SPeter Xu trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo); 252202a2cbc8SPeter Xu if (!vtd_process_inv_iec_desc(s, &inv_desc)) { 252302a2cbc8SPeter Xu return false; 252402a2cbc8SPeter Xu } 2525b7910472SPeter Xu break; 2526b7910472SPeter Xu 2527554f5e16SJason Wang case VTD_INV_DESC_DEVICE: 25287feb51b7SPeter Xu trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo); 2529554f5e16SJason Wang if (!vtd_process_device_iotlb_desc(s, &inv_desc)) { 2530554f5e16SJason Wang return false; 2531554f5e16SJason Wang } 2532554f5e16SJason Wang break; 2533554f5e16SJason Wang 2534ed7b8fbcSLe Tan default: 2535095955b2SPeter Xu error_report_once("%s: invalid inv desc: hi=%"PRIx64", lo=%"PRIx64 2536095955b2SPeter Xu " (unknown type)", __func__, inv_desc.hi, 2537095955b2SPeter Xu inv_desc.lo); 2538ed7b8fbcSLe Tan return false; 2539ed7b8fbcSLe Tan } 2540ed7b8fbcSLe Tan s->iq_head++; 2541ed7b8fbcSLe Tan if (s->iq_head == s->iq_size) { 2542ed7b8fbcSLe Tan s->iq_head = 0; 2543ed7b8fbcSLe Tan } 2544ed7b8fbcSLe Tan return true; 2545ed7b8fbcSLe Tan } 2546ed7b8fbcSLe Tan 2547ed7b8fbcSLe Tan /* Try to fetch and process more Invalidation Descriptors */ 2548ed7b8fbcSLe Tan static void vtd_fetch_inv_desc(IntelIOMMUState *s) 2549ed7b8fbcSLe Tan { 25507feb51b7SPeter Xu trace_vtd_inv_qi_fetch(); 25517feb51b7SPeter Xu 2552ed7b8fbcSLe Tan if (s->iq_tail >= s->iq_size) { 2553ed7b8fbcSLe Tan /* Detects an invalid Tail pointer */ 25544e4abd11SPeter Xu error_report_once("%s: detected invalid QI tail " 25554e4abd11SPeter Xu "(tail=0x%x, size=0x%x)", 25564e4abd11SPeter Xu __func__, s->iq_tail, s->iq_size); 2557ed7b8fbcSLe Tan vtd_handle_inv_queue_error(s); 2558ed7b8fbcSLe Tan return; 2559ed7b8fbcSLe Tan } 2560ed7b8fbcSLe Tan while (s->iq_head != s->iq_tail) { 2561ed7b8fbcSLe Tan if (!vtd_process_inv_desc(s)) { 2562ed7b8fbcSLe Tan /* Invalidation Queue Errors */ 2563ed7b8fbcSLe Tan vtd_handle_inv_queue_error(s); 2564ed7b8fbcSLe Tan break; 2565ed7b8fbcSLe Tan } 2566ed7b8fbcSLe Tan /* Must update the IQH_REG in time */ 2567ed7b8fbcSLe Tan vtd_set_quad_raw(s, DMAR_IQH_REG, 2568ed7b8fbcSLe Tan (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) & 2569ed7b8fbcSLe Tan VTD_IQH_QH_MASK); 2570ed7b8fbcSLe Tan } 2571ed7b8fbcSLe Tan } 2572ed7b8fbcSLe Tan 2573ed7b8fbcSLe Tan /* Handle write to Invalidation Queue Tail Register */ 2574ed7b8fbcSLe Tan static void vtd_handle_iqt_write(IntelIOMMUState *s) 2575ed7b8fbcSLe Tan { 2576ed7b8fbcSLe Tan uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 2577ed7b8fbcSLe Tan 2578c0c1d351SLiu, Yi L if (s->iq_dw && (val & VTD_IQT_QT_256_RSV_BIT)) { 2579c0c1d351SLiu, Yi L error_report_once("%s: RSV bit is set: val=0x%"PRIx64, 2580c0c1d351SLiu, Yi L __func__, val); 2581c0c1d351SLiu, Yi L return; 2582c0c1d351SLiu, Yi L } 2583c0c1d351SLiu, Yi L s->iq_tail = VTD_IQT_QT(s->iq_dw, val); 25847feb51b7SPeter Xu trace_vtd_inv_qi_tail(s->iq_tail); 25857feb51b7SPeter Xu 2586ed7b8fbcSLe Tan if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 2587ed7b8fbcSLe Tan /* Process Invalidation Queue here */ 2588ed7b8fbcSLe Tan vtd_fetch_inv_desc(s); 2589ed7b8fbcSLe Tan } 2590ed7b8fbcSLe Tan } 2591ed7b8fbcSLe Tan 25921da12ec4SLe Tan static void vtd_handle_fsts_write(IntelIOMMUState *s) 25931da12ec4SLe Tan { 25941da12ec4SLe Tan uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 25951da12ec4SLe Tan uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 25961da12ec4SLe Tan uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 25971da12ec4SLe Tan 25981da12ec4SLe Tan if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 25991da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 26007feb51b7SPeter Xu trace_vtd_fsts_clear_ip(); 26011da12ec4SLe Tan } 2602ed7b8fbcSLe Tan /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 2603ed7b8fbcSLe Tan * Descriptors if there are any when Queued Invalidation is enabled? 2604ed7b8fbcSLe Tan */ 26051da12ec4SLe Tan } 26061da12ec4SLe Tan 26071da12ec4SLe Tan static void vtd_handle_fectl_write(IntelIOMMUState *s) 26081da12ec4SLe Tan { 26091da12ec4SLe Tan uint32_t fectl_reg; 26101da12ec4SLe Tan /* FIXME: when software clears the IM field, check the IP field. But do we 26111da12ec4SLe Tan * need to compare the old value and the new value to conclude that 26121da12ec4SLe Tan * software clears the IM field? Or just check if the IM field is zero? 26131da12ec4SLe Tan */ 26141da12ec4SLe Tan fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 26157feb51b7SPeter Xu 26167feb51b7SPeter Xu trace_vtd_reg_write_fectl(fectl_reg); 26177feb51b7SPeter Xu 26181da12ec4SLe Tan if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 26191da12ec4SLe Tan vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 26201da12ec4SLe Tan vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 26211da12ec4SLe Tan } 26221da12ec4SLe Tan } 26231da12ec4SLe Tan 2624ed7b8fbcSLe Tan static void vtd_handle_ics_write(IntelIOMMUState *s) 2625ed7b8fbcSLe Tan { 2626ed7b8fbcSLe Tan uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 2627ed7b8fbcSLe Tan uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 2628ed7b8fbcSLe Tan 2629ed7b8fbcSLe Tan if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 26307feb51b7SPeter Xu trace_vtd_reg_ics_clear_ip(); 2631ed7b8fbcSLe Tan vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 2632ed7b8fbcSLe Tan } 2633ed7b8fbcSLe Tan } 2634ed7b8fbcSLe Tan 2635ed7b8fbcSLe Tan static void vtd_handle_iectl_write(IntelIOMMUState *s) 2636ed7b8fbcSLe Tan { 2637ed7b8fbcSLe Tan uint32_t iectl_reg; 2638ed7b8fbcSLe Tan /* FIXME: when software clears the IM field, check the IP field. But do we 2639ed7b8fbcSLe Tan * need to compare the old value and the new value to conclude that 2640ed7b8fbcSLe Tan * software clears the IM field? Or just check if the IM field is zero? 2641ed7b8fbcSLe Tan */ 2642ed7b8fbcSLe Tan iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 26437feb51b7SPeter Xu 26447feb51b7SPeter Xu trace_vtd_reg_write_iectl(iectl_reg); 26457feb51b7SPeter Xu 2646ed7b8fbcSLe Tan if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 2647ed7b8fbcSLe Tan vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 2648ed7b8fbcSLe Tan vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 2649ed7b8fbcSLe Tan } 2650ed7b8fbcSLe Tan } 2651ed7b8fbcSLe Tan 26521da12ec4SLe Tan static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 26531da12ec4SLe Tan { 26541da12ec4SLe Tan IntelIOMMUState *s = opaque; 26551da12ec4SLe Tan uint64_t val; 26561da12ec4SLe Tan 26577feb51b7SPeter Xu trace_vtd_reg_read(addr, size); 26587feb51b7SPeter Xu 26591da12ec4SLe Tan if (addr + size > DMAR_REG_SIZE) { 26601376211fSPeter Xu error_report_once("%s: MMIO over range: addr=0x%" PRIx64 26611376211fSPeter Xu " size=0x%u", __func__, addr, size); 26621da12ec4SLe Tan return (uint64_t)-1; 26631da12ec4SLe Tan } 26641da12ec4SLe Tan 26651da12ec4SLe Tan switch (addr) { 26661da12ec4SLe Tan /* Root Table Address Register, 64-bit */ 26671da12ec4SLe Tan case DMAR_RTADDR_REG: 26688fdee711SYi Sun val = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 26691da12ec4SLe Tan if (size == 4) { 26708fdee711SYi Sun val = val & ((1ULL << 32) - 1); 26711da12ec4SLe Tan } 26721da12ec4SLe Tan break; 26731da12ec4SLe Tan 26741da12ec4SLe Tan case DMAR_RTADDR_REG_HI: 26751da12ec4SLe Tan assert(size == 4); 26768fdee711SYi Sun val = vtd_get_quad_raw(s, DMAR_RTADDR_REG) >> 32; 26771da12ec4SLe Tan break; 26781da12ec4SLe Tan 2679ed7b8fbcSLe Tan /* Invalidation Queue Address Register, 64-bit */ 2680ed7b8fbcSLe Tan case DMAR_IQA_REG: 2681ed7b8fbcSLe Tan val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 2682ed7b8fbcSLe Tan if (size == 4) { 2683ed7b8fbcSLe Tan val = val & ((1ULL << 32) - 1); 2684ed7b8fbcSLe Tan } 2685ed7b8fbcSLe Tan break; 2686ed7b8fbcSLe Tan 2687ed7b8fbcSLe Tan case DMAR_IQA_REG_HI: 2688ed7b8fbcSLe Tan assert(size == 4); 2689ed7b8fbcSLe Tan val = s->iq >> 32; 2690ed7b8fbcSLe Tan break; 2691ed7b8fbcSLe Tan 26921da12ec4SLe Tan default: 26931da12ec4SLe Tan if (size == 4) { 26941da12ec4SLe Tan val = vtd_get_long(s, addr); 26951da12ec4SLe Tan } else { 26961da12ec4SLe Tan val = vtd_get_quad(s, addr); 26971da12ec4SLe Tan } 26981da12ec4SLe Tan } 26997feb51b7SPeter Xu 27001da12ec4SLe Tan return val; 27011da12ec4SLe Tan } 27021da12ec4SLe Tan 27031da12ec4SLe Tan static void vtd_mem_write(void *opaque, hwaddr addr, 27041da12ec4SLe Tan uint64_t val, unsigned size) 27051da12ec4SLe Tan { 27061da12ec4SLe Tan IntelIOMMUState *s = opaque; 27071da12ec4SLe Tan 27087feb51b7SPeter Xu trace_vtd_reg_write(addr, size, val); 27097feb51b7SPeter Xu 27101da12ec4SLe Tan if (addr + size > DMAR_REG_SIZE) { 27111376211fSPeter Xu error_report_once("%s: MMIO over range: addr=0x%" PRIx64 27121376211fSPeter Xu " size=0x%u", __func__, addr, size); 27131da12ec4SLe Tan return; 27141da12ec4SLe Tan } 27151da12ec4SLe Tan 27161da12ec4SLe Tan switch (addr) { 27171da12ec4SLe Tan /* Global Command Register, 32-bit */ 27181da12ec4SLe Tan case DMAR_GCMD_REG: 27191da12ec4SLe Tan vtd_set_long(s, addr, val); 27201da12ec4SLe Tan vtd_handle_gcmd_write(s); 27211da12ec4SLe Tan break; 27221da12ec4SLe Tan 27231da12ec4SLe Tan /* Context Command Register, 64-bit */ 27241da12ec4SLe Tan case DMAR_CCMD_REG: 27251da12ec4SLe Tan if (size == 4) { 27261da12ec4SLe Tan vtd_set_long(s, addr, val); 27271da12ec4SLe Tan } else { 27281da12ec4SLe Tan vtd_set_quad(s, addr, val); 27291da12ec4SLe Tan vtd_handle_ccmd_write(s); 27301da12ec4SLe Tan } 27311da12ec4SLe Tan break; 27321da12ec4SLe Tan 27331da12ec4SLe Tan case DMAR_CCMD_REG_HI: 27341da12ec4SLe Tan assert(size == 4); 27351da12ec4SLe Tan vtd_set_long(s, addr, val); 27361da12ec4SLe Tan vtd_handle_ccmd_write(s); 27371da12ec4SLe Tan break; 27381da12ec4SLe Tan 27391da12ec4SLe Tan /* IOTLB Invalidation Register, 64-bit */ 27401da12ec4SLe Tan case DMAR_IOTLB_REG: 27411da12ec4SLe Tan if (size == 4) { 27421da12ec4SLe Tan vtd_set_long(s, addr, val); 27431da12ec4SLe Tan } else { 27441da12ec4SLe Tan vtd_set_quad(s, addr, val); 27451da12ec4SLe Tan vtd_handle_iotlb_write(s); 27461da12ec4SLe Tan } 27471da12ec4SLe Tan break; 27481da12ec4SLe Tan 27491da12ec4SLe Tan case DMAR_IOTLB_REG_HI: 27501da12ec4SLe Tan assert(size == 4); 27511da12ec4SLe Tan vtd_set_long(s, addr, val); 27521da12ec4SLe Tan vtd_handle_iotlb_write(s); 27531da12ec4SLe Tan break; 27541da12ec4SLe Tan 2755b5a280c0SLe Tan /* Invalidate Address Register, 64-bit */ 2756b5a280c0SLe Tan case DMAR_IVA_REG: 2757b5a280c0SLe Tan if (size == 4) { 2758b5a280c0SLe Tan vtd_set_long(s, addr, val); 2759b5a280c0SLe Tan } else { 2760b5a280c0SLe Tan vtd_set_quad(s, addr, val); 2761b5a280c0SLe Tan } 2762b5a280c0SLe Tan break; 2763b5a280c0SLe Tan 2764b5a280c0SLe Tan case DMAR_IVA_REG_HI: 2765b5a280c0SLe Tan assert(size == 4); 2766b5a280c0SLe Tan vtd_set_long(s, addr, val); 2767b5a280c0SLe Tan break; 2768b5a280c0SLe Tan 27691da12ec4SLe Tan /* Fault Status Register, 32-bit */ 27701da12ec4SLe Tan case DMAR_FSTS_REG: 27711da12ec4SLe Tan assert(size == 4); 27721da12ec4SLe Tan vtd_set_long(s, addr, val); 27731da12ec4SLe Tan vtd_handle_fsts_write(s); 27741da12ec4SLe Tan break; 27751da12ec4SLe Tan 27761da12ec4SLe Tan /* Fault Event Control Register, 32-bit */ 27771da12ec4SLe Tan case DMAR_FECTL_REG: 27781da12ec4SLe Tan assert(size == 4); 27791da12ec4SLe Tan vtd_set_long(s, addr, val); 27801da12ec4SLe Tan vtd_handle_fectl_write(s); 27811da12ec4SLe Tan break; 27821da12ec4SLe Tan 27831da12ec4SLe Tan /* Fault Event Data Register, 32-bit */ 27841da12ec4SLe Tan case DMAR_FEDATA_REG: 27851da12ec4SLe Tan assert(size == 4); 27861da12ec4SLe Tan vtd_set_long(s, addr, val); 27871da12ec4SLe Tan break; 27881da12ec4SLe Tan 27891da12ec4SLe Tan /* Fault Event Address Register, 32-bit */ 27901da12ec4SLe Tan case DMAR_FEADDR_REG: 2791b7a7bb35SJan Kiszka if (size == 4) { 27921da12ec4SLe Tan vtd_set_long(s, addr, val); 2793b7a7bb35SJan Kiszka } else { 2794b7a7bb35SJan Kiszka /* 2795b7a7bb35SJan Kiszka * While the register is 32-bit only, some guests (Xen...) write to 2796b7a7bb35SJan Kiszka * it with 64-bit. 2797b7a7bb35SJan Kiszka */ 2798b7a7bb35SJan Kiszka vtd_set_quad(s, addr, val); 2799b7a7bb35SJan Kiszka } 28001da12ec4SLe Tan break; 28011da12ec4SLe Tan 28021da12ec4SLe Tan /* Fault Event Upper Address Register, 32-bit */ 28031da12ec4SLe Tan case DMAR_FEUADDR_REG: 28041da12ec4SLe Tan assert(size == 4); 28051da12ec4SLe Tan vtd_set_long(s, addr, val); 28061da12ec4SLe Tan break; 28071da12ec4SLe Tan 28081da12ec4SLe Tan /* Protected Memory Enable Register, 32-bit */ 28091da12ec4SLe Tan case DMAR_PMEN_REG: 28101da12ec4SLe Tan assert(size == 4); 28111da12ec4SLe Tan vtd_set_long(s, addr, val); 28121da12ec4SLe Tan break; 28131da12ec4SLe Tan 28141da12ec4SLe Tan /* Root Table Address Register, 64-bit */ 28151da12ec4SLe Tan case DMAR_RTADDR_REG: 28161da12ec4SLe Tan if (size == 4) { 28171da12ec4SLe Tan vtd_set_long(s, addr, val); 28181da12ec4SLe Tan } else { 28191da12ec4SLe Tan vtd_set_quad(s, addr, val); 28201da12ec4SLe Tan } 28211da12ec4SLe Tan break; 28221da12ec4SLe Tan 28231da12ec4SLe Tan case DMAR_RTADDR_REG_HI: 28241da12ec4SLe Tan assert(size == 4); 28251da12ec4SLe Tan vtd_set_long(s, addr, val); 28261da12ec4SLe Tan break; 28271da12ec4SLe Tan 2828ed7b8fbcSLe Tan /* Invalidation Queue Tail Register, 64-bit */ 2829ed7b8fbcSLe Tan case DMAR_IQT_REG: 2830ed7b8fbcSLe Tan if (size == 4) { 2831ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2832ed7b8fbcSLe Tan } else { 2833ed7b8fbcSLe Tan vtd_set_quad(s, addr, val); 2834ed7b8fbcSLe Tan } 2835ed7b8fbcSLe Tan vtd_handle_iqt_write(s); 2836ed7b8fbcSLe Tan break; 2837ed7b8fbcSLe Tan 2838ed7b8fbcSLe Tan case DMAR_IQT_REG_HI: 2839ed7b8fbcSLe Tan assert(size == 4); 2840ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2841ed7b8fbcSLe Tan /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 2842ed7b8fbcSLe Tan break; 2843ed7b8fbcSLe Tan 2844ed7b8fbcSLe Tan /* Invalidation Queue Address Register, 64-bit */ 2845ed7b8fbcSLe Tan case DMAR_IQA_REG: 2846ed7b8fbcSLe Tan if (size == 4) { 2847ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2848ed7b8fbcSLe Tan } else { 2849ed7b8fbcSLe Tan vtd_set_quad(s, addr, val); 2850ed7b8fbcSLe Tan } 2851c0c1d351SLiu, Yi L if (s->ecap & VTD_ECAP_SMTS && 2852c0c1d351SLiu, Yi L val & VTD_IQA_DW_MASK) { 2853c0c1d351SLiu, Yi L s->iq_dw = true; 2854c0c1d351SLiu, Yi L } else { 2855c0c1d351SLiu, Yi L s->iq_dw = false; 2856c0c1d351SLiu, Yi L } 2857ed7b8fbcSLe Tan break; 2858ed7b8fbcSLe Tan 2859ed7b8fbcSLe Tan case DMAR_IQA_REG_HI: 2860ed7b8fbcSLe Tan assert(size == 4); 2861ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2862ed7b8fbcSLe Tan break; 2863ed7b8fbcSLe Tan 2864ed7b8fbcSLe Tan /* Invalidation Completion Status Register, 32-bit */ 2865ed7b8fbcSLe Tan case DMAR_ICS_REG: 2866ed7b8fbcSLe Tan assert(size == 4); 2867ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2868ed7b8fbcSLe Tan vtd_handle_ics_write(s); 2869ed7b8fbcSLe Tan break; 2870ed7b8fbcSLe Tan 2871ed7b8fbcSLe Tan /* Invalidation Event Control Register, 32-bit */ 2872ed7b8fbcSLe Tan case DMAR_IECTL_REG: 2873ed7b8fbcSLe Tan assert(size == 4); 2874ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2875ed7b8fbcSLe Tan vtd_handle_iectl_write(s); 2876ed7b8fbcSLe Tan break; 2877ed7b8fbcSLe Tan 2878ed7b8fbcSLe Tan /* Invalidation Event Data Register, 32-bit */ 2879ed7b8fbcSLe Tan case DMAR_IEDATA_REG: 2880ed7b8fbcSLe Tan assert(size == 4); 2881ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2882ed7b8fbcSLe Tan break; 2883ed7b8fbcSLe Tan 2884ed7b8fbcSLe Tan /* Invalidation Event Address Register, 32-bit */ 2885ed7b8fbcSLe Tan case DMAR_IEADDR_REG: 2886ed7b8fbcSLe Tan assert(size == 4); 2887ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2888ed7b8fbcSLe Tan break; 2889ed7b8fbcSLe Tan 2890ed7b8fbcSLe Tan /* Invalidation Event Upper Address Register, 32-bit */ 2891ed7b8fbcSLe Tan case DMAR_IEUADDR_REG: 2892ed7b8fbcSLe Tan assert(size == 4); 2893ed7b8fbcSLe Tan vtd_set_long(s, addr, val); 2894ed7b8fbcSLe Tan break; 2895ed7b8fbcSLe Tan 28961da12ec4SLe Tan /* Fault Recording Registers, 128-bit */ 28971da12ec4SLe Tan case DMAR_FRCD_REG_0_0: 28981da12ec4SLe Tan if (size == 4) { 28991da12ec4SLe Tan vtd_set_long(s, addr, val); 29001da12ec4SLe Tan } else { 29011da12ec4SLe Tan vtd_set_quad(s, addr, val); 29021da12ec4SLe Tan } 29031da12ec4SLe Tan break; 29041da12ec4SLe Tan 29051da12ec4SLe Tan case DMAR_FRCD_REG_0_1: 29061da12ec4SLe Tan assert(size == 4); 29071da12ec4SLe Tan vtd_set_long(s, addr, val); 29081da12ec4SLe Tan break; 29091da12ec4SLe Tan 29101da12ec4SLe Tan case DMAR_FRCD_REG_0_2: 29111da12ec4SLe Tan if (size == 4) { 29121da12ec4SLe Tan vtd_set_long(s, addr, val); 29131da12ec4SLe Tan } else { 29141da12ec4SLe Tan vtd_set_quad(s, addr, val); 29151da12ec4SLe Tan /* May clear bit 127 (Fault), update PPF */ 29161da12ec4SLe Tan vtd_update_fsts_ppf(s); 29171da12ec4SLe Tan } 29181da12ec4SLe Tan break; 29191da12ec4SLe Tan 29201da12ec4SLe Tan case DMAR_FRCD_REG_0_3: 29211da12ec4SLe Tan assert(size == 4); 29221da12ec4SLe Tan vtd_set_long(s, addr, val); 29231da12ec4SLe Tan /* May clear bit 127 (Fault), update PPF */ 29241da12ec4SLe Tan vtd_update_fsts_ppf(s); 29251da12ec4SLe Tan break; 29261da12ec4SLe Tan 2927a5861439SPeter Xu case DMAR_IRTA_REG: 2928a5861439SPeter Xu if (size == 4) { 2929a5861439SPeter Xu vtd_set_long(s, addr, val); 2930a5861439SPeter Xu } else { 2931a5861439SPeter Xu vtd_set_quad(s, addr, val); 2932a5861439SPeter Xu } 2933a5861439SPeter Xu break; 2934a5861439SPeter Xu 2935a5861439SPeter Xu case DMAR_IRTA_REG_HI: 2936a5861439SPeter Xu assert(size == 4); 2937a5861439SPeter Xu vtd_set_long(s, addr, val); 2938a5861439SPeter Xu break; 2939a5861439SPeter Xu 29401da12ec4SLe Tan default: 29411da12ec4SLe Tan if (size == 4) { 29421da12ec4SLe Tan vtd_set_long(s, addr, val); 29431da12ec4SLe Tan } else { 29441da12ec4SLe Tan vtd_set_quad(s, addr, val); 29451da12ec4SLe Tan } 29461da12ec4SLe Tan } 29471da12ec4SLe Tan } 29481da12ec4SLe Tan 29493df9d748SAlexey Kardashevskiy static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, 29502c91bcf2SPeter Maydell IOMMUAccessFlags flag, int iommu_idx) 29511da12ec4SLe Tan { 29521da12ec4SLe Tan VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 29531da12ec4SLe Tan IntelIOMMUState *s = vtd_as->iommu_state; 2954b9313021SPeter Xu IOMMUTLBEntry iotlb = { 2955b9313021SPeter Xu /* We'll fill in the rest later. */ 29561da12ec4SLe Tan .target_as = &address_space_memory, 29571da12ec4SLe Tan }; 2958b9313021SPeter Xu bool success; 29591da12ec4SLe Tan 2960b9313021SPeter Xu if (likely(s->dmar_enabled)) { 2961b9313021SPeter Xu success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, 2962b9313021SPeter Xu addr, flag & IOMMU_WO, &iotlb); 2963b9313021SPeter Xu } else { 29641da12ec4SLe Tan /* DMAR disabled, passthrough, use 4k-page*/ 2965b9313021SPeter Xu iotlb.iova = addr & VTD_PAGE_MASK_4K; 2966b9313021SPeter Xu iotlb.translated_addr = addr & VTD_PAGE_MASK_4K; 2967b9313021SPeter Xu iotlb.addr_mask = ~VTD_PAGE_MASK_4K; 2968b9313021SPeter Xu iotlb.perm = IOMMU_RW; 2969b9313021SPeter Xu success = true; 29701da12ec4SLe Tan } 29711da12ec4SLe Tan 2972b9313021SPeter Xu if (likely(success)) { 29737feb51b7SPeter Xu trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus), 29747feb51b7SPeter Xu VTD_PCI_SLOT(vtd_as->devfn), 29757feb51b7SPeter Xu VTD_PCI_FUNC(vtd_as->devfn), 2976b9313021SPeter Xu iotlb.iova, iotlb.translated_addr, 2977b9313021SPeter Xu iotlb.addr_mask); 2978b9313021SPeter Xu } else { 29794e4abd11SPeter Xu error_report_once("%s: detected translation failure " 29804e4abd11SPeter Xu "(dev=%02x:%02x:%02x, iova=0x%" PRIx64 ")", 29814e4abd11SPeter Xu __func__, pci_bus_num(vtd_as->bus), 2982b9313021SPeter Xu VTD_PCI_SLOT(vtd_as->devfn), 2983b9313021SPeter Xu VTD_PCI_FUNC(vtd_as->devfn), 2984662b4b69SPeter Xu addr); 2985b9313021SPeter Xu } 29867feb51b7SPeter Xu 2987b9313021SPeter Xu return iotlb; 29881da12ec4SLe Tan } 29891da12ec4SLe Tan 2990549d4005SEric Auger static int vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu, 29915bf3d319SPeter Xu IOMMUNotifierFlag old, 2992549d4005SEric Auger IOMMUNotifierFlag new, 2993549d4005SEric Auger Error **errp) 29943cb3b154SAlex Williamson { 29953cb3b154SAlex Williamson VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 2996dd4d607eSPeter Xu IntelIOMMUState *s = vtd_as->iommu_state; 29973cb3b154SAlex Williamson 29984f8a62a9SPeter Xu /* Update per-address-space notifier flags */ 29994f8a62a9SPeter Xu vtd_as->notifier_flags = new; 30004f8a62a9SPeter Xu 3001dd4d607eSPeter Xu if (old == IOMMU_NOTIFIER_NONE) { 3002b4a4ba0dSPeter Xu QLIST_INSERT_HEAD(&s->vtd_as_with_notifiers, vtd_as, next); 3003b4a4ba0dSPeter Xu } else if (new == IOMMU_NOTIFIER_NONE) { 3004b4a4ba0dSPeter Xu QLIST_REMOVE(vtd_as, next); 3005dd4d607eSPeter Xu } 3006549d4005SEric Auger return 0; 30073cb3b154SAlex Williamson } 30083cb3b154SAlex Williamson 3009552a1e01SPeter Xu static int vtd_post_load(void *opaque, int version_id) 3010552a1e01SPeter Xu { 3011552a1e01SPeter Xu IntelIOMMUState *iommu = opaque; 3012552a1e01SPeter Xu 3013552a1e01SPeter Xu /* 3014552a1e01SPeter Xu * Memory regions are dynamically turned on/off depending on 3015552a1e01SPeter Xu * context entry configurations from the guest. After migration, 3016552a1e01SPeter Xu * we need to make sure the memory regions are still correct. 3017552a1e01SPeter Xu */ 3018552a1e01SPeter Xu vtd_switch_address_space_all(iommu); 3019552a1e01SPeter Xu 30202811af3bSPeter Xu /* 30212811af3bSPeter Xu * We don't need to migrate the root_scalable because we can 30222811af3bSPeter Xu * simply do the calculation after the loading is complete. We 30232811af3bSPeter Xu * can actually do similar things with root, dmar_enabled, etc. 30242811af3bSPeter Xu * however since we've had them already so we'd better keep them 30252811af3bSPeter Xu * for compatibility of migration. 30262811af3bSPeter Xu */ 30272811af3bSPeter Xu vtd_update_scalable_state(iommu); 30282811af3bSPeter Xu 3029552a1e01SPeter Xu return 0; 3030552a1e01SPeter Xu } 3031552a1e01SPeter Xu 30321da12ec4SLe Tan static const VMStateDescription vtd_vmstate = { 30331da12ec4SLe Tan .name = "iommu-intel", 30348cdcf3c1SPeter Xu .version_id = 1, 30358cdcf3c1SPeter Xu .minimum_version_id = 1, 30368cdcf3c1SPeter Xu .priority = MIG_PRI_IOMMU, 3037552a1e01SPeter Xu .post_load = vtd_post_load, 30388cdcf3c1SPeter Xu .fields = (VMStateField[]) { 30398cdcf3c1SPeter Xu VMSTATE_UINT64(root, IntelIOMMUState), 30408cdcf3c1SPeter Xu VMSTATE_UINT64(intr_root, IntelIOMMUState), 30418cdcf3c1SPeter Xu VMSTATE_UINT64(iq, IntelIOMMUState), 30428cdcf3c1SPeter Xu VMSTATE_UINT32(intr_size, IntelIOMMUState), 30438cdcf3c1SPeter Xu VMSTATE_UINT16(iq_head, IntelIOMMUState), 30448cdcf3c1SPeter Xu VMSTATE_UINT16(iq_tail, IntelIOMMUState), 30458cdcf3c1SPeter Xu VMSTATE_UINT16(iq_size, IntelIOMMUState), 30468cdcf3c1SPeter Xu VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState), 30478cdcf3c1SPeter Xu VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE), 30488cdcf3c1SPeter Xu VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState), 304981fb1e64SPeter Xu VMSTATE_UNUSED(1), /* bool root_extended is obsolete by VT-d */ 30508cdcf3c1SPeter Xu VMSTATE_BOOL(dmar_enabled, IntelIOMMUState), 30518cdcf3c1SPeter Xu VMSTATE_BOOL(qi_enabled, IntelIOMMUState), 30528cdcf3c1SPeter Xu VMSTATE_BOOL(intr_enabled, IntelIOMMUState), 30538cdcf3c1SPeter Xu VMSTATE_BOOL(intr_eime, IntelIOMMUState), 30548cdcf3c1SPeter Xu VMSTATE_END_OF_LIST() 30558cdcf3c1SPeter Xu } 30561da12ec4SLe Tan }; 30571da12ec4SLe Tan 30581da12ec4SLe Tan static const MemoryRegionOps vtd_mem_ops = { 30591da12ec4SLe Tan .read = vtd_mem_read, 30601da12ec4SLe Tan .write = vtd_mem_write, 30611da12ec4SLe Tan .endianness = DEVICE_LITTLE_ENDIAN, 30621da12ec4SLe Tan .impl = { 30631da12ec4SLe Tan .min_access_size = 4, 30641da12ec4SLe Tan .max_access_size = 8, 30651da12ec4SLe Tan }, 30661da12ec4SLe Tan .valid = { 30671da12ec4SLe Tan .min_access_size = 4, 30681da12ec4SLe Tan .max_access_size = 8, 30691da12ec4SLe Tan }, 30701da12ec4SLe Tan }; 30711da12ec4SLe Tan 30721da12ec4SLe Tan static Property vtd_properties[] = { 30731da12ec4SLe Tan DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 3074e6b6af05SRadim Krčmář DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim, 3075e6b6af05SRadim Krčmář ON_OFF_AUTO_AUTO), 3076fb506e70SRadim Krčmář DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false), 30774b49b586SPeter Xu DEFINE_PROP_UINT8("aw-bits", IntelIOMMUState, aw_bits, 307837f51384SPrasad Singamsetty VTD_HOST_ADDRESS_WIDTH), 30793b40f0e5SAviv Ben-David DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE), 30804a4f219eSYi Sun DEFINE_PROP_BOOL("x-scalable-mode", IntelIOMMUState, scalable_mode, FALSE), 3081ccc23bb0SPeter Xu DEFINE_PROP_BOOL("dma-drain", IntelIOMMUState, dma_drain, true), 30821da12ec4SLe Tan DEFINE_PROP_END_OF_LIST(), 30831da12ec4SLe Tan }; 30841da12ec4SLe Tan 3085651e4cefSPeter Xu /* Read IRTE entry with specific index */ 3086651e4cefSPeter Xu static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index, 3087bc38ee10SMichael S. Tsirkin VTD_IR_TableEntry *entry, uint16_t sid) 3088651e4cefSPeter Xu { 3089ede9c94aSPeter Xu static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \ 3090ede9c94aSPeter Xu {0xffff, 0xfffb, 0xfff9, 0xfff8}; 3091651e4cefSPeter Xu dma_addr_t addr = 0x00; 3092ede9c94aSPeter Xu uint16_t mask, source_id; 3093ede9c94aSPeter Xu uint8_t bus, bus_max, bus_min; 3094651e4cefSPeter Xu 3095651e4cefSPeter Xu addr = iommu->intr_root + index * sizeof(*entry); 3096651e4cefSPeter Xu if (dma_memory_read(&address_space_memory, addr, entry, 3097651e4cefSPeter Xu sizeof(*entry))) { 30981376211fSPeter Xu error_report_once("%s: read failed: ind=0x%x addr=0x%" PRIx64, 30991376211fSPeter Xu __func__, index, addr); 3100651e4cefSPeter Xu return -VTD_FR_IR_ROOT_INVAL; 3101651e4cefSPeter Xu } 3102651e4cefSPeter Xu 31037feb51b7SPeter Xu trace_vtd_ir_irte_get(index, le64_to_cpu(entry->data[1]), 31047feb51b7SPeter Xu le64_to_cpu(entry->data[0])); 31057feb51b7SPeter Xu 3106bc38ee10SMichael S. Tsirkin if (!entry->irte.present) { 31074e4abd11SPeter Xu error_report_once("%s: detected non-present IRTE " 31084e4abd11SPeter Xu "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")", 31094e4abd11SPeter Xu __func__, index, le64_to_cpu(entry->data[1]), 3110651e4cefSPeter Xu le64_to_cpu(entry->data[0])); 3111651e4cefSPeter Xu return -VTD_FR_IR_ENTRY_P; 3112651e4cefSPeter Xu } 3113651e4cefSPeter Xu 3114bc38ee10SMichael S. Tsirkin if (entry->irte.__reserved_0 || entry->irte.__reserved_1 || 3115bc38ee10SMichael S. Tsirkin entry->irte.__reserved_2) { 31164e4abd11SPeter Xu error_report_once("%s: detected non-zero reserved IRTE " 31174e4abd11SPeter Xu "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")", 31184e4abd11SPeter Xu __func__, index, le64_to_cpu(entry->data[1]), 3119651e4cefSPeter Xu le64_to_cpu(entry->data[0])); 3120651e4cefSPeter Xu return -VTD_FR_IR_IRTE_RSVD; 3121651e4cefSPeter Xu } 3122651e4cefSPeter Xu 3123ede9c94aSPeter Xu if (sid != X86_IOMMU_SID_INVALID) { 3124ede9c94aSPeter Xu /* Validate IRTE SID */ 3125bc38ee10SMichael S. Tsirkin source_id = le32_to_cpu(entry->irte.source_id); 3126bc38ee10SMichael S. Tsirkin switch (entry->irte.sid_vtype) { 3127ede9c94aSPeter Xu case VTD_SVT_NONE: 3128ede9c94aSPeter Xu break; 3129ede9c94aSPeter Xu 3130ede9c94aSPeter Xu case VTD_SVT_ALL: 3131bc38ee10SMichael S. Tsirkin mask = vtd_svt_mask[entry->irte.sid_q]; 3132ede9c94aSPeter Xu if ((source_id & mask) != (sid & mask)) { 31334e4abd11SPeter Xu error_report_once("%s: invalid IRTE SID " 31344e4abd11SPeter Xu "(index=%u, sid=%u, source_id=%u)", 31354e4abd11SPeter Xu __func__, index, sid, source_id); 3136ede9c94aSPeter Xu return -VTD_FR_IR_SID_ERR; 3137ede9c94aSPeter Xu } 3138ede9c94aSPeter Xu break; 3139ede9c94aSPeter Xu 3140ede9c94aSPeter Xu case VTD_SVT_BUS: 3141ede9c94aSPeter Xu bus_max = source_id >> 8; 3142ede9c94aSPeter Xu bus_min = source_id & 0xff; 3143ede9c94aSPeter Xu bus = sid >> 8; 3144ede9c94aSPeter Xu if (bus > bus_max || bus < bus_min) { 31454e4abd11SPeter Xu error_report_once("%s: invalid SVT_BUS " 31464e4abd11SPeter Xu "(index=%u, bus=%u, min=%u, max=%u)", 31474e4abd11SPeter Xu __func__, index, bus, bus_min, bus_max); 3148ede9c94aSPeter Xu return -VTD_FR_IR_SID_ERR; 3149ede9c94aSPeter Xu } 3150ede9c94aSPeter Xu break; 3151ede9c94aSPeter Xu 3152ede9c94aSPeter Xu default: 31534e4abd11SPeter Xu error_report_once("%s: detected invalid IRTE SVT " 31544e4abd11SPeter Xu "(index=%u, type=%d)", __func__, 31554e4abd11SPeter Xu index, entry->irte.sid_vtype); 3156ede9c94aSPeter Xu /* Take this as verification failure. */ 3157ede9c94aSPeter Xu return -VTD_FR_IR_SID_ERR; 3158ede9c94aSPeter Xu break; 3159ede9c94aSPeter Xu } 3160ede9c94aSPeter Xu } 3161651e4cefSPeter Xu 3162651e4cefSPeter Xu return 0; 3163651e4cefSPeter Xu } 3164651e4cefSPeter Xu 3165651e4cefSPeter Xu /* Fetch IRQ information of specific IR index */ 3166ede9c94aSPeter Xu static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index, 316735c24501SSingh, Brijesh X86IOMMUIrq *irq, uint16_t sid) 3168651e4cefSPeter Xu { 3169bc38ee10SMichael S. Tsirkin VTD_IR_TableEntry irte = {}; 3170651e4cefSPeter Xu int ret = 0; 3171651e4cefSPeter Xu 3172ede9c94aSPeter Xu ret = vtd_irte_get(iommu, index, &irte, sid); 3173651e4cefSPeter Xu if (ret) { 3174651e4cefSPeter Xu return ret; 3175651e4cefSPeter Xu } 3176651e4cefSPeter Xu 3177bc38ee10SMichael S. Tsirkin irq->trigger_mode = irte.irte.trigger_mode; 3178bc38ee10SMichael S. Tsirkin irq->vector = irte.irte.vector; 3179bc38ee10SMichael S. Tsirkin irq->delivery_mode = irte.irte.delivery_mode; 3180bc38ee10SMichael S. Tsirkin irq->dest = le32_to_cpu(irte.irte.dest_id); 318128589311SJan Kiszka if (!iommu->intr_eime) { 3182651e4cefSPeter Xu #define VTD_IR_APIC_DEST_MASK (0xff00ULL) 3183651e4cefSPeter Xu #define VTD_IR_APIC_DEST_SHIFT (8) 318428589311SJan Kiszka irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >> 3185651e4cefSPeter Xu VTD_IR_APIC_DEST_SHIFT; 318628589311SJan Kiszka } 3187bc38ee10SMichael S. Tsirkin irq->dest_mode = irte.irte.dest_mode; 3188bc38ee10SMichael S. Tsirkin irq->redir_hint = irte.irte.redir_hint; 3189651e4cefSPeter Xu 31907feb51b7SPeter Xu trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector, 31917feb51b7SPeter Xu irq->delivery_mode, irq->dest, irq->dest_mode); 3192651e4cefSPeter Xu 3193651e4cefSPeter Xu return 0; 3194651e4cefSPeter Xu } 3195651e4cefSPeter Xu 3196651e4cefSPeter Xu /* Interrupt remapping for MSI/MSI-X entry */ 3197651e4cefSPeter Xu static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu, 3198651e4cefSPeter Xu MSIMessage *origin, 3199ede9c94aSPeter Xu MSIMessage *translated, 3200ede9c94aSPeter Xu uint16_t sid) 3201651e4cefSPeter Xu { 3202651e4cefSPeter Xu int ret = 0; 3203651e4cefSPeter Xu VTD_IR_MSIAddress addr; 3204651e4cefSPeter Xu uint16_t index; 320535c24501SSingh, Brijesh X86IOMMUIrq irq = {}; 3206651e4cefSPeter Xu 3207651e4cefSPeter Xu assert(origin && translated); 3208651e4cefSPeter Xu 32097feb51b7SPeter Xu trace_vtd_ir_remap_msi_req(origin->address, origin->data); 32107feb51b7SPeter Xu 3211651e4cefSPeter Xu if (!iommu || !iommu->intr_enabled) { 3212e7a3b91fSPeter Xu memcpy(translated, origin, sizeof(*origin)); 3213e7a3b91fSPeter Xu goto out; 3214651e4cefSPeter Xu } 3215651e4cefSPeter Xu 3216651e4cefSPeter Xu if (origin->address & VTD_MSI_ADDR_HI_MASK) { 32171376211fSPeter Xu error_report_once("%s: MSI address high 32 bits non-zero detected: " 32181376211fSPeter Xu "address=0x%" PRIx64, __func__, origin->address); 3219651e4cefSPeter Xu return -VTD_FR_IR_REQ_RSVD; 3220651e4cefSPeter Xu } 3221651e4cefSPeter Xu 3222651e4cefSPeter Xu addr.data = origin->address & VTD_MSI_ADDR_LO_MASK; 32231a43713bSPeter Xu if (addr.addr.__head != 0xfee) { 32241376211fSPeter Xu error_report_once("%s: MSI address low 32 bit invalid: 0x%" PRIx32, 32251376211fSPeter Xu __func__, addr.data); 3226651e4cefSPeter Xu return -VTD_FR_IR_REQ_RSVD; 3227651e4cefSPeter Xu } 3228651e4cefSPeter Xu 3229651e4cefSPeter Xu /* This is compatible mode. */ 3230bc38ee10SMichael S. Tsirkin if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) { 3231e7a3b91fSPeter Xu memcpy(translated, origin, sizeof(*origin)); 3232e7a3b91fSPeter Xu goto out; 3233651e4cefSPeter Xu } 3234651e4cefSPeter Xu 3235bc38ee10SMichael S. Tsirkin index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l); 3236651e4cefSPeter Xu 3237651e4cefSPeter Xu #define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff) 3238651e4cefSPeter Xu #define VTD_IR_MSI_DATA_RESERVED (0xffff0000) 3239651e4cefSPeter Xu 3240bc38ee10SMichael S. Tsirkin if (addr.addr.sub_valid) { 3241651e4cefSPeter Xu /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */ 3242651e4cefSPeter Xu index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE; 3243651e4cefSPeter Xu } 3244651e4cefSPeter Xu 3245ede9c94aSPeter Xu ret = vtd_remap_irq_get(iommu, index, &irq, sid); 3246651e4cefSPeter Xu if (ret) { 3247651e4cefSPeter Xu return ret; 3248651e4cefSPeter Xu } 3249651e4cefSPeter Xu 3250bc38ee10SMichael S. Tsirkin if (addr.addr.sub_valid) { 32517feb51b7SPeter Xu trace_vtd_ir_remap_type("MSI"); 3252651e4cefSPeter Xu if (origin->data & VTD_IR_MSI_DATA_RESERVED) { 32534e4abd11SPeter Xu error_report_once("%s: invalid IR MSI " 32544e4abd11SPeter Xu "(sid=%u, address=0x%" PRIx64 32554e4abd11SPeter Xu ", data=0x%" PRIx32 ")", 32564e4abd11SPeter Xu __func__, sid, origin->address, origin->data); 3257651e4cefSPeter Xu return -VTD_FR_IR_REQ_RSVD; 3258651e4cefSPeter Xu } 3259651e4cefSPeter Xu } else { 3260651e4cefSPeter Xu uint8_t vector = origin->data & 0xff; 3261dea651a9SFeng Wu uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1; 3262dea651a9SFeng Wu 32637feb51b7SPeter Xu trace_vtd_ir_remap_type("IOAPIC"); 3264651e4cefSPeter Xu /* IOAPIC entry vector should be aligned with IRTE vector 3265651e4cefSPeter Xu * (see vt-d spec 5.1.5.1). */ 3266651e4cefSPeter Xu if (vector != irq.vector) { 32677feb51b7SPeter Xu trace_vtd_warn_ir_vector(sid, index, vector, irq.vector); 3268651e4cefSPeter Xu } 3269dea651a9SFeng Wu 3270dea651a9SFeng Wu /* The Trigger Mode field must match the Trigger Mode in the IRTE. 3271dea651a9SFeng Wu * (see vt-d spec 5.1.5.1). */ 3272dea651a9SFeng Wu if (trigger_mode != irq.trigger_mode) { 32737feb51b7SPeter Xu trace_vtd_warn_ir_trigger(sid, index, trigger_mode, 32747feb51b7SPeter Xu irq.trigger_mode); 3275dea651a9SFeng Wu } 3276651e4cefSPeter Xu } 3277651e4cefSPeter Xu 3278651e4cefSPeter Xu /* 3279651e4cefSPeter Xu * We'd better keep the last two bits, assuming that guest OS 3280651e4cefSPeter Xu * might modify it. Keep it does not hurt after all. 3281651e4cefSPeter Xu */ 3282bc38ee10SMichael S. Tsirkin irq.msi_addr_last_bits = addr.addr.__not_care; 3283651e4cefSPeter Xu 328435c24501SSingh, Brijesh /* Translate X86IOMMUIrq to MSI message */ 328535c24501SSingh, Brijesh x86_iommu_irq_to_msi_message(&irq, translated); 3286651e4cefSPeter Xu 3287e7a3b91fSPeter Xu out: 32887feb51b7SPeter Xu trace_vtd_ir_remap_msi(origin->address, origin->data, 3289651e4cefSPeter Xu translated->address, translated->data); 3290651e4cefSPeter Xu return 0; 3291651e4cefSPeter Xu } 3292651e4cefSPeter Xu 32938b5ed7dfSPeter Xu static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src, 32948b5ed7dfSPeter Xu MSIMessage *dst, uint16_t sid) 32958b5ed7dfSPeter Xu { 3296ede9c94aSPeter Xu return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu), 3297ede9c94aSPeter Xu src, dst, sid); 32988b5ed7dfSPeter Xu } 32998b5ed7dfSPeter Xu 3300651e4cefSPeter Xu static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr, 3301651e4cefSPeter Xu uint64_t *data, unsigned size, 3302651e4cefSPeter Xu MemTxAttrs attrs) 3303651e4cefSPeter Xu { 3304651e4cefSPeter Xu return MEMTX_OK; 3305651e4cefSPeter Xu } 3306651e4cefSPeter Xu 3307651e4cefSPeter Xu static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr, 3308651e4cefSPeter Xu uint64_t value, unsigned size, 3309651e4cefSPeter Xu MemTxAttrs attrs) 3310651e4cefSPeter Xu { 3311651e4cefSPeter Xu int ret = 0; 331209cd058aSMichael S. Tsirkin MSIMessage from = {}, to = {}; 3313ede9c94aSPeter Xu uint16_t sid = X86_IOMMU_SID_INVALID; 3314651e4cefSPeter Xu 3315651e4cefSPeter Xu from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST; 3316651e4cefSPeter Xu from.data = (uint32_t) value; 3317651e4cefSPeter Xu 3318ede9c94aSPeter Xu if (!attrs.unspecified) { 3319ede9c94aSPeter Xu /* We have explicit Source ID */ 3320ede9c94aSPeter Xu sid = attrs.requester_id; 3321ede9c94aSPeter Xu } 3322ede9c94aSPeter Xu 3323ede9c94aSPeter Xu ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid); 3324651e4cefSPeter Xu if (ret) { 3325651e4cefSPeter Xu /* TODO: report error */ 3326651e4cefSPeter Xu /* Drop this interrupt */ 3327651e4cefSPeter Xu return MEMTX_ERROR; 3328651e4cefSPeter Xu } 3329651e4cefSPeter Xu 333032946019SRadim Krčmář apic_get_class()->send_msi(&to); 3331651e4cefSPeter Xu 3332651e4cefSPeter Xu return MEMTX_OK; 3333651e4cefSPeter Xu } 3334651e4cefSPeter Xu 3335651e4cefSPeter Xu static const MemoryRegionOps vtd_mem_ir_ops = { 3336651e4cefSPeter Xu .read_with_attrs = vtd_mem_ir_read, 3337651e4cefSPeter Xu .write_with_attrs = vtd_mem_ir_write, 3338651e4cefSPeter Xu .endianness = DEVICE_LITTLE_ENDIAN, 3339651e4cefSPeter Xu .impl = { 3340651e4cefSPeter Xu .min_access_size = 4, 3341651e4cefSPeter Xu .max_access_size = 4, 3342651e4cefSPeter Xu }, 3343651e4cefSPeter Xu .valid = { 3344651e4cefSPeter Xu .min_access_size = 4, 3345651e4cefSPeter Xu .max_access_size = 4, 3346651e4cefSPeter Xu }, 3347651e4cefSPeter Xu }; 33487df953bdSKnut Omang 33497df953bdSKnut Omang VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) 33507df953bdSKnut Omang { 33517df953bdSKnut Omang uintptr_t key = (uintptr_t)bus; 33527df953bdSKnut Omang VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key); 33537df953bdSKnut Omang VTDAddressSpace *vtd_dev_as; 3354e0a3c8ccSJason Wang char name[128]; 33557df953bdSKnut Omang 33567df953bdSKnut Omang if (!vtd_bus) { 33572d3fc581SJason Wang uintptr_t *new_key = g_malloc(sizeof(*new_key)); 33582d3fc581SJason Wang *new_key = (uintptr_t)bus; 33597df953bdSKnut Omang /* No corresponding free() */ 336004af0e18SPeter Xu vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \ 3361bf33cc75SPeter Xu PCI_DEVFN_MAX); 33627df953bdSKnut Omang vtd_bus->bus = bus; 33632d3fc581SJason Wang g_hash_table_insert(s->vtd_as_by_busptr, new_key, vtd_bus); 33647df953bdSKnut Omang } 33657df953bdSKnut Omang 33667df953bdSKnut Omang vtd_dev_as = vtd_bus->dev_as[devfn]; 33677df953bdSKnut Omang 33687df953bdSKnut Omang if (!vtd_dev_as) { 33694b519ef1SPeter Xu snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn), 33704b519ef1SPeter Xu PCI_FUNC(devfn)); 33717df953bdSKnut Omang vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace)); 33727df953bdSKnut Omang 33737df953bdSKnut Omang vtd_dev_as->bus = bus; 33747df953bdSKnut Omang vtd_dev_as->devfn = (uint8_t)devfn; 33757df953bdSKnut Omang vtd_dev_as->iommu_state = s; 33767df953bdSKnut Omang vtd_dev_as->context_cache_entry.context_cache_gen = 0; 337763b88968SPeter Xu vtd_dev_as->iova_tree = iova_tree_new(); 3378558e0024SPeter Xu 33794b519ef1SPeter Xu memory_region_init(&vtd_dev_as->root, OBJECT(s), name, UINT64_MAX); 33804b519ef1SPeter Xu address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, "vtd-root"); 33814b519ef1SPeter Xu 3382558e0024SPeter Xu /* 33834b519ef1SPeter Xu * Build the DMAR-disabled container with aliases to the 33844b519ef1SPeter Xu * shared MRs. Note that aliasing to a shared memory region 33854b519ef1SPeter Xu * could help the memory API to detect same FlatViews so we 33864b519ef1SPeter Xu * can have devices to share the same FlatView when DMAR is 33874b519ef1SPeter Xu * disabled (either by not providing "intel_iommu=on" or with 33884b519ef1SPeter Xu * "iommu=pt"). It will greatly reduce the total number of 33894b519ef1SPeter Xu * FlatViews of the system hence VM runs faster. 3390558e0024SPeter Xu */ 33914b519ef1SPeter Xu memory_region_init_alias(&vtd_dev_as->nodmar, OBJECT(s), 33924b519ef1SPeter Xu "vtd-nodmar", &s->mr_nodmar, 0, 33934b519ef1SPeter Xu memory_region_size(&s->mr_nodmar)); 33944b519ef1SPeter Xu 33954b519ef1SPeter Xu /* 33964b519ef1SPeter Xu * Build the per-device DMAR-enabled container. 33974b519ef1SPeter Xu * 33984b519ef1SPeter Xu * TODO: currently we have per-device IOMMU memory region only 33994b519ef1SPeter Xu * because we have per-device IOMMU notifiers for devices. If 34004b519ef1SPeter Xu * one day we can abstract the IOMMU notifiers out of the 34014b519ef1SPeter Xu * memory regions then we can also share the same memory 34024b519ef1SPeter Xu * region here just like what we've done above with the nodmar 34034b519ef1SPeter Xu * region. 34044b519ef1SPeter Xu */ 34054b519ef1SPeter Xu strcat(name, "-dmar"); 34061221a474SAlexey Kardashevskiy memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu), 34071221a474SAlexey Kardashevskiy TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s), 34084b519ef1SPeter Xu name, UINT64_MAX); 34094b519ef1SPeter Xu memory_region_init_alias(&vtd_dev_as->iommu_ir, OBJECT(s), "vtd-ir", 34104b519ef1SPeter Xu &s->mr_ir, 0, memory_region_size(&s->mr_ir)); 34114b519ef1SPeter Xu memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->iommu), 3412558e0024SPeter Xu VTD_INTERRUPT_ADDR_FIRST, 34134b519ef1SPeter Xu &vtd_dev_as->iommu_ir, 1); 34144b519ef1SPeter Xu 34154b519ef1SPeter Xu /* 34164b519ef1SPeter Xu * Hook both the containers under the root container, we 34174b519ef1SPeter Xu * switch between DMAR & noDMAR by enable/disable 34184b519ef1SPeter Xu * corresponding sub-containers 34194b519ef1SPeter Xu */ 3420558e0024SPeter Xu memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 34213df9d748SAlexey Kardashevskiy MEMORY_REGION(&vtd_dev_as->iommu), 34224b519ef1SPeter Xu 0); 34234b519ef1SPeter Xu memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 34244b519ef1SPeter Xu &vtd_dev_as->nodmar, 0); 34254b519ef1SPeter Xu 3426558e0024SPeter Xu vtd_switch_address_space(vtd_dev_as); 34277df953bdSKnut Omang } 34287df953bdSKnut Omang return vtd_dev_as; 34297df953bdSKnut Omang } 34307df953bdSKnut Omang 34319a4bb839SPeter Xu static uint64_t get_naturally_aligned_size(uint64_t start, 34329a4bb839SPeter Xu uint64_t size, int gaw) 34339a4bb839SPeter Xu { 34349a4bb839SPeter Xu uint64_t max_mask = 1ULL << gaw; 34359a4bb839SPeter Xu uint64_t alignment = start ? start & -start : max_mask; 34369a4bb839SPeter Xu 34379a4bb839SPeter Xu alignment = MIN(alignment, max_mask); 34389a4bb839SPeter Xu size = MIN(size, max_mask); 34399a4bb839SPeter Xu 34409a4bb839SPeter Xu if (alignment <= size) { 34419a4bb839SPeter Xu /* Increase the alignment of start */ 34429a4bb839SPeter Xu return alignment; 34439a4bb839SPeter Xu } else { 34449a4bb839SPeter Xu /* Find the largest page mask from size */ 34459a4bb839SPeter Xu return 1ULL << (63 - clz64(size)); 34469a4bb839SPeter Xu } 34479a4bb839SPeter Xu } 34489a4bb839SPeter Xu 3449dd4d607eSPeter Xu /* Unmap the whole range in the notifier's scope. */ 3450dd4d607eSPeter Xu static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n) 3451dd4d607eSPeter Xu { 34529a4bb839SPeter Xu hwaddr size, remain; 3453dd4d607eSPeter Xu hwaddr start = n->start; 3454dd4d607eSPeter Xu hwaddr end = n->end; 345537f51384SPrasad Singamsetty IntelIOMMUState *s = as->iommu_state; 345663b88968SPeter Xu DMAMap map; 3457dd4d607eSPeter Xu 3458dd4d607eSPeter Xu /* 3459dd4d607eSPeter Xu * Note: all the codes in this function has a assumption that IOVA 3460dd4d607eSPeter Xu * bits are no more than VTD_MGAW bits (which is restricted by 3461dd4d607eSPeter Xu * VT-d spec), otherwise we need to consider overflow of 64 bits. 3462dd4d607eSPeter Xu */ 3463dd4d607eSPeter Xu 3464d6d10793SYan Zhao if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) { 3465dd4d607eSPeter Xu /* 3466dd4d607eSPeter Xu * Don't need to unmap regions that is bigger than the whole 3467dd4d607eSPeter Xu * VT-d supported address space size 3468dd4d607eSPeter Xu */ 3469d6d10793SYan Zhao end = VTD_ADDRESS_SIZE(s->aw_bits) - 1; 3470dd4d607eSPeter Xu } 3471dd4d607eSPeter Xu 3472dd4d607eSPeter Xu assert(start <= end); 34739a4bb839SPeter Xu size = remain = end - start + 1; 3474dd4d607eSPeter Xu 34759a4bb839SPeter Xu while (remain >= VTD_PAGE_SIZE) { 34769a4bb839SPeter Xu IOMMUTLBEntry entry; 34779a4bb839SPeter Xu uint64_t mask = get_naturally_aligned_size(start, remain, s->aw_bits); 3478dd4d607eSPeter Xu 34799a4bb839SPeter Xu assert(mask); 34809a4bb839SPeter Xu 34819a4bb839SPeter Xu entry.iova = start; 34829a4bb839SPeter Xu entry.addr_mask = mask - 1; 3483dd4d607eSPeter Xu entry.target_as = &address_space_memory; 34849a4bb839SPeter Xu entry.perm = IOMMU_NONE; 3485dd4d607eSPeter Xu /* This field is meaningless for unmap */ 3486dd4d607eSPeter Xu entry.translated_addr = 0; 34879a4bb839SPeter Xu 34889a4bb839SPeter Xu memory_region_notify_one(n, &entry); 34899a4bb839SPeter Xu 34909a4bb839SPeter Xu start += mask; 34919a4bb839SPeter Xu remain -= mask; 34929a4bb839SPeter Xu } 34939a4bb839SPeter Xu 34949a4bb839SPeter Xu assert(!remain); 3495dd4d607eSPeter Xu 3496dd4d607eSPeter Xu trace_vtd_as_unmap_whole(pci_bus_num(as->bus), 3497dd4d607eSPeter Xu VTD_PCI_SLOT(as->devfn), 3498dd4d607eSPeter Xu VTD_PCI_FUNC(as->devfn), 34999a4bb839SPeter Xu n->start, size); 3500dd4d607eSPeter Xu 35019a4bb839SPeter Xu map.iova = n->start; 35029a4bb839SPeter Xu map.size = size; 350363b88968SPeter Xu iova_tree_remove(as->iova_tree, &map); 3504dd4d607eSPeter Xu } 3505dd4d607eSPeter Xu 3506dd4d607eSPeter Xu static void vtd_address_space_unmap_all(IntelIOMMUState *s) 3507dd4d607eSPeter Xu { 3508dd4d607eSPeter Xu VTDAddressSpace *vtd_as; 3509dd4d607eSPeter Xu IOMMUNotifier *n; 3510dd4d607eSPeter Xu 3511b4a4ba0dSPeter Xu QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 3512dd4d607eSPeter Xu IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 3513dd4d607eSPeter Xu vtd_address_space_unmap(vtd_as, n); 3514dd4d607eSPeter Xu } 3515dd4d607eSPeter Xu } 3516dd4d607eSPeter Xu } 3517dd4d607eSPeter Xu 35182cc9ddccSPeter Xu static void vtd_address_space_refresh_all(IntelIOMMUState *s) 35192cc9ddccSPeter Xu { 35202cc9ddccSPeter Xu vtd_address_space_unmap_all(s); 35212cc9ddccSPeter Xu vtd_switch_address_space_all(s); 35222cc9ddccSPeter Xu } 35232cc9ddccSPeter Xu 3524f06a696dSPeter Xu static int vtd_replay_hook(IOMMUTLBEntry *entry, void *private) 3525f06a696dSPeter Xu { 3526f06a696dSPeter Xu memory_region_notify_one((IOMMUNotifier *)private, entry); 3527f06a696dSPeter Xu return 0; 3528f06a696dSPeter Xu } 3529f06a696dSPeter Xu 35303df9d748SAlexey Kardashevskiy static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) 3531f06a696dSPeter Xu { 35323df9d748SAlexey Kardashevskiy VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu); 3533f06a696dSPeter Xu IntelIOMMUState *s = vtd_as->iommu_state; 3534f06a696dSPeter Xu uint8_t bus_n = pci_bus_num(vtd_as->bus); 3535f06a696dSPeter Xu VTDContextEntry ce; 3536f06a696dSPeter Xu 3537f06a696dSPeter Xu /* 3538dd4d607eSPeter Xu * The replay can be triggered by either a invalidation or a newly 3539dd4d607eSPeter Xu * created entry. No matter what, we release existing mappings 3540dd4d607eSPeter Xu * (it means flushing caches for UNMAP-only registers). 3541f06a696dSPeter Xu */ 3542dd4d607eSPeter Xu vtd_address_space_unmap(vtd_as, n); 3543dd4d607eSPeter Xu 3544dd4d607eSPeter Xu if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 3545fb43cf73SLiu, Yi L trace_vtd_replay_ce_valid(s->root_scalable ? "scalable mode" : 3546fb43cf73SLiu, Yi L "legacy mode", 3547fb43cf73SLiu, Yi L bus_n, PCI_SLOT(vtd_as->devfn), 3548f06a696dSPeter Xu PCI_FUNC(vtd_as->devfn), 3549fb43cf73SLiu, Yi L vtd_get_domain_id(s, &ce), 3550f06a696dSPeter Xu ce.hi, ce.lo); 35514f8a62a9SPeter Xu if (vtd_as_has_map_notifier(vtd_as)) { 35524f8a62a9SPeter Xu /* This is required only for MAP typed notifiers */ 3553fe215b0cSPeter Xu vtd_page_walk_info info = { 3554fe215b0cSPeter Xu .hook_fn = vtd_replay_hook, 3555fe215b0cSPeter Xu .private = (void *)n, 3556fe215b0cSPeter Xu .notify_unmap = false, 3557fe215b0cSPeter Xu .aw = s->aw_bits, 35582f764fa8SPeter Xu .as = vtd_as, 3559fb43cf73SLiu, Yi L .domain_id = vtd_get_domain_id(s, &ce), 3560fe215b0cSPeter Xu }; 3561fe215b0cSPeter Xu 3562fb43cf73SLiu, Yi L vtd_page_walk(s, &ce, 0, ~0ULL, &info); 35634f8a62a9SPeter Xu } 3564f06a696dSPeter Xu } else { 3565f06a696dSPeter Xu trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn), 3566f06a696dSPeter Xu PCI_FUNC(vtd_as->devfn)); 3567f06a696dSPeter Xu } 3568f06a696dSPeter Xu 3569f06a696dSPeter Xu return; 3570f06a696dSPeter Xu } 3571f06a696dSPeter Xu 35721da12ec4SLe Tan /* Do the initialization. It will also be called when reset, so pay 35731da12ec4SLe Tan * attention when adding new initialization stuff. 35741da12ec4SLe Tan */ 35751da12ec4SLe Tan static void vtd_init(IntelIOMMUState *s) 35761da12ec4SLe Tan { 3577d54bd7f8SPeter Xu X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3578d54bd7f8SPeter Xu 35791da12ec4SLe Tan memset(s->csr, 0, DMAR_REG_SIZE); 35801da12ec4SLe Tan memset(s->wmask, 0, DMAR_REG_SIZE); 35811da12ec4SLe Tan memset(s->w1cmask, 0, DMAR_REG_SIZE); 35821da12ec4SLe Tan memset(s->womask, 0, DMAR_REG_SIZE); 35831da12ec4SLe Tan 35841da12ec4SLe Tan s->root = 0; 3585fb43cf73SLiu, Yi L s->root_scalable = false; 35861da12ec4SLe Tan s->dmar_enabled = false; 3587d7bb469aSPeter Xu s->intr_enabled = false; 35881da12ec4SLe Tan s->iq_head = 0; 35891da12ec4SLe Tan s->iq_tail = 0; 35901da12ec4SLe Tan s->iq = 0; 35911da12ec4SLe Tan s->iq_size = 0; 35921da12ec4SLe Tan s->qi_enabled = false; 35931da12ec4SLe Tan s->iq_last_desc_type = VTD_INV_DESC_NONE; 3594c0c1d351SLiu, Yi L s->iq_dw = false; 35951da12ec4SLe Tan s->next_frcd_reg = 0; 359692e5d85eSPrasad Singamsetty s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | 359792e5d85eSPrasad Singamsetty VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS | 359837f51384SPrasad Singamsetty VTD_CAP_SAGAW_39bit | VTD_CAP_MGAW(s->aw_bits); 3599ccc23bb0SPeter Xu if (s->dma_drain) { 3600ccc23bb0SPeter Xu s->cap |= VTD_CAP_DRAIN; 3601ccc23bb0SPeter Xu } 360237f51384SPrasad Singamsetty if (s->aw_bits == VTD_HOST_AW_48BIT) { 360337f51384SPrasad Singamsetty s->cap |= VTD_CAP_SAGAW_48bit; 360437f51384SPrasad Singamsetty } 3605ed7b8fbcSLe Tan s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 36061da12ec4SLe Tan 360792e5d85eSPrasad Singamsetty /* 360892e5d85eSPrasad Singamsetty * Rsvd field masks for spte 360992e5d85eSPrasad Singamsetty */ 3610ce586f3bSQi, Yadong vtd_spte_rsvd[0] = ~0ULL; 3611e48929c7SQi, Yadong vtd_spte_rsvd[1] = VTD_SPTE_PAGE_L1_RSVD_MASK(s->aw_bits, 3612e48929c7SQi, Yadong x86_iommu->dt_supported); 3613ce586f3bSQi, Yadong vtd_spte_rsvd[2] = VTD_SPTE_PAGE_L2_RSVD_MASK(s->aw_bits); 3614ce586f3bSQi, Yadong vtd_spte_rsvd[3] = VTD_SPTE_PAGE_L3_RSVD_MASK(s->aw_bits); 3615ce586f3bSQi, Yadong vtd_spte_rsvd[4] = VTD_SPTE_PAGE_L4_RSVD_MASK(s->aw_bits); 3616ce586f3bSQi, Yadong 3617e48929c7SQi, Yadong vtd_spte_rsvd_large[2] = VTD_SPTE_LPAGE_L2_RSVD_MASK(s->aw_bits, 3618e48929c7SQi, Yadong x86_iommu->dt_supported); 3619e48929c7SQi, Yadong vtd_spte_rsvd_large[3] = VTD_SPTE_LPAGE_L3_RSVD_MASK(s->aw_bits, 3620e48929c7SQi, Yadong x86_iommu->dt_supported); 362192e5d85eSPrasad Singamsetty 3622a924b3d8SPeter Xu if (x86_iommu_ir_supported(x86_iommu)) { 3623e6b6af05SRadim Krčmář s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV; 3624e6b6af05SRadim Krčmář if (s->intr_eim == ON_OFF_AUTO_ON) { 3625e6b6af05SRadim Krčmář s->ecap |= VTD_ECAP_EIM; 3626e6b6af05SRadim Krčmář } 3627e6b6af05SRadim Krčmář assert(s->intr_eim != ON_OFF_AUTO_AUTO); 3628d54bd7f8SPeter Xu } 3629d54bd7f8SPeter Xu 3630554f5e16SJason Wang if (x86_iommu->dt_supported) { 3631554f5e16SJason Wang s->ecap |= VTD_ECAP_DT; 3632554f5e16SJason Wang } 3633554f5e16SJason Wang 3634dbaabb25SPeter Xu if (x86_iommu->pt_supported) { 3635dbaabb25SPeter Xu s->ecap |= VTD_ECAP_PT; 3636dbaabb25SPeter Xu } 3637dbaabb25SPeter Xu 36383b40f0e5SAviv Ben-David if (s->caching_mode) { 36393b40f0e5SAviv Ben-David s->cap |= VTD_CAP_CM; 36403b40f0e5SAviv Ben-David } 36413b40f0e5SAviv Ben-David 36424a4f219eSYi Sun /* TODO: read cap/ecap from host to decide which cap to be exposed. */ 36434a4f219eSYi Sun if (s->scalable_mode) { 36444a4f219eSYi Sun s->ecap |= VTD_ECAP_SMTS | VTD_ECAP_SRS | VTD_ECAP_SLTS; 36454a4f219eSYi Sun } 36464a4f219eSYi Sun 364706aba4caSPeter Xu vtd_reset_caches(s); 3648d92fa2dcSLe Tan 36491da12ec4SLe Tan /* Define registers with default values and bit semantics */ 36501da12ec4SLe Tan vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 36511da12ec4SLe Tan vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 36521da12ec4SLe Tan vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 36531da12ec4SLe Tan vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 36541da12ec4SLe Tan vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 36551da12ec4SLe Tan vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 3656fb43cf73SLiu, Yi L vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffffc00ULL, 0); 36571da12ec4SLe Tan vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 36581da12ec4SLe Tan vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 36591da12ec4SLe Tan 36601da12ec4SLe Tan /* Advanced Fault Logging not supported */ 36611da12ec4SLe Tan vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 36621da12ec4SLe Tan vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 36631da12ec4SLe Tan vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 36641da12ec4SLe Tan vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 36651da12ec4SLe Tan 36661da12ec4SLe Tan /* Treated as RsvdZ when EIM in ECAP_REG is not supported 36671da12ec4SLe Tan * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 36681da12ec4SLe Tan */ 36691da12ec4SLe Tan vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 36701da12ec4SLe Tan 36711da12ec4SLe Tan /* Treated as RO for implementations that PLMR and PHMR fields reported 36721da12ec4SLe Tan * as Clear in the CAP_REG. 36731da12ec4SLe Tan * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 36741da12ec4SLe Tan */ 36751da12ec4SLe Tan vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 36761da12ec4SLe Tan 3677ed7b8fbcSLe Tan vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 3678ed7b8fbcSLe Tan vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 3679c0c1d351SLiu, Yi L vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff807ULL, 0); 3680ed7b8fbcSLe Tan vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 3681ed7b8fbcSLe Tan vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 3682ed7b8fbcSLe Tan vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 3683ed7b8fbcSLe Tan vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 3684ed7b8fbcSLe Tan /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 3685ed7b8fbcSLe Tan vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 3686ed7b8fbcSLe Tan 36871da12ec4SLe Tan /* IOTLB registers */ 36881da12ec4SLe Tan vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 36891da12ec4SLe Tan vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 36901da12ec4SLe Tan vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 36911da12ec4SLe Tan 36921da12ec4SLe Tan /* Fault Recording Registers, 128-bit */ 36931da12ec4SLe Tan vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 36941da12ec4SLe Tan vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 3695a5861439SPeter Xu 3696a5861439SPeter Xu /* 369728589311SJan Kiszka * Interrupt remapping registers. 3698a5861439SPeter Xu */ 369928589311SJan Kiszka vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0); 37001da12ec4SLe Tan } 37011da12ec4SLe Tan 37021da12ec4SLe Tan /* Should not reset address_spaces when reset because devices will still use 37031da12ec4SLe Tan * the address space they got at first (won't ask the bus again). 37041da12ec4SLe Tan */ 37051da12ec4SLe Tan static void vtd_reset(DeviceState *dev) 37061da12ec4SLe Tan { 37071da12ec4SLe Tan IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 37081da12ec4SLe Tan 37091da12ec4SLe Tan vtd_init(s); 37102cc9ddccSPeter Xu vtd_address_space_refresh_all(s); 37111da12ec4SLe Tan } 37121da12ec4SLe Tan 3713621d983aSMarcel Apfelbaum static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) 3714621d983aSMarcel Apfelbaum { 3715621d983aSMarcel Apfelbaum IntelIOMMUState *s = opaque; 3716621d983aSMarcel Apfelbaum VTDAddressSpace *vtd_as; 3717621d983aSMarcel Apfelbaum 3718bf33cc75SPeter Xu assert(0 <= devfn && devfn < PCI_DEVFN_MAX); 3719621d983aSMarcel Apfelbaum 3720621d983aSMarcel Apfelbaum vtd_as = vtd_find_add_as(s, bus, devfn); 3721621d983aSMarcel Apfelbaum return &vtd_as->as; 3722621d983aSMarcel Apfelbaum } 3723621d983aSMarcel Apfelbaum 3724e6b6af05SRadim Krčmář static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) 37256333e93cSRadim Krčmář { 3726e6b6af05SRadim Krčmář X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3727e6b6af05SRadim Krčmář 3728a924b3d8SPeter Xu if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu_ir_supported(x86_iommu)) { 3729e6b6af05SRadim Krčmář error_setg(errp, "eim=on cannot be selected without intremap=on"); 3730e6b6af05SRadim Krčmář return false; 3731e6b6af05SRadim Krčmář } 3732e6b6af05SRadim Krčmář 3733e6b6af05SRadim Krčmář if (s->intr_eim == ON_OFF_AUTO_AUTO) { 3734fb506e70SRadim Krčmář s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim) 3735a924b3d8SPeter Xu && x86_iommu_ir_supported(x86_iommu) ? 3736e6b6af05SRadim Krčmář ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 3737e6b6af05SRadim Krčmář } 3738fb506e70SRadim Krčmář if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) { 3739fb506e70SRadim Krčmář if (!kvm_irqchip_in_kernel()) { 3740fb506e70SRadim Krčmář error_setg(errp, "eim=on requires accel=kvm,kernel-irqchip=split"); 3741fb506e70SRadim Krčmář return false; 3742fb506e70SRadim Krčmář } 3743fb506e70SRadim Krčmář if (!kvm_enable_x2apic()) { 3744fb506e70SRadim Krčmář error_setg(errp, "eim=on requires support on the KVM side" 3745fb506e70SRadim Krčmář "(X2APIC_API, first shipped in v4.7)"); 3746fb506e70SRadim Krčmář return false; 3747fb506e70SRadim Krčmář } 3748fb506e70SRadim Krčmář } 3749e6b6af05SRadim Krčmář 375037f51384SPrasad Singamsetty /* Currently only address widths supported are 39 and 48 bits */ 375137f51384SPrasad Singamsetty if ((s->aw_bits != VTD_HOST_AW_39BIT) && 375237f51384SPrasad Singamsetty (s->aw_bits != VTD_HOST_AW_48BIT)) { 375337f51384SPrasad Singamsetty error_setg(errp, "Supported values for x-aw-bits are: %d, %d", 375437f51384SPrasad Singamsetty VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT); 375537f51384SPrasad Singamsetty return false; 375637f51384SPrasad Singamsetty } 375737f51384SPrasad Singamsetty 37584a4f219eSYi Sun if (s->scalable_mode && !s->dma_drain) { 37594a4f219eSYi Sun error_setg(errp, "Need to set dma_drain for scalable mode"); 37604a4f219eSYi Sun return false; 37614a4f219eSYi Sun } 37624a4f219eSYi Sun 37636333e93cSRadim Krčmář return true; 37646333e93cSRadim Krčmář } 37656333e93cSRadim Krčmář 376628cf553aSPeter Xu static int vtd_machine_done_notify_one(Object *child, void *unused) 376728cf553aSPeter Xu { 376828cf553aSPeter Xu IntelIOMMUState *iommu = INTEL_IOMMU_DEVICE(x86_iommu_get_default()); 376928cf553aSPeter Xu 377028cf553aSPeter Xu /* 377128cf553aSPeter Xu * We hard-coded here because vfio-pci is the only special case 377228cf553aSPeter Xu * here. Let's be more elegant in the future when we can, but so 377328cf553aSPeter Xu * far there seems to be no better way. 377428cf553aSPeter Xu */ 377528cf553aSPeter Xu if (object_dynamic_cast(child, "vfio-pci") && !iommu->caching_mode) { 377628cf553aSPeter Xu vtd_panic_require_caching_mode(); 377728cf553aSPeter Xu } 377828cf553aSPeter Xu 377928cf553aSPeter Xu return 0; 378028cf553aSPeter Xu } 378128cf553aSPeter Xu 378228cf553aSPeter Xu static void vtd_machine_done_hook(Notifier *notifier, void *unused) 378328cf553aSPeter Xu { 378428cf553aSPeter Xu object_child_foreach_recursive(object_get_root(), 378528cf553aSPeter Xu vtd_machine_done_notify_one, NULL); 378628cf553aSPeter Xu } 378728cf553aSPeter Xu 378828cf553aSPeter Xu static Notifier vtd_machine_done_notify = { 378928cf553aSPeter Xu .notify = vtd_machine_done_hook, 379028cf553aSPeter Xu }; 379128cf553aSPeter Xu 37921da12ec4SLe Tan static void vtd_realize(DeviceState *dev, Error **errp) 37931da12ec4SLe Tan { 3794ef0e8fc7SEduardo Habkost MachineState *ms = MACHINE(qdev_get_machine()); 379529396ed9SMohammed Gamal PCMachineState *pcms = PC_MACHINE(ms); 3796f0bb276bSPaolo Bonzini X86MachineState *x86ms = X86_MACHINE(ms); 379729396ed9SMohammed Gamal PCIBus *bus = pcms->bus; 37981da12ec4SLe Tan IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 37994684a204SPeter Xu X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev); 38001da12ec4SLe Tan 3801fb9f5926SDavid Kiarie x86_iommu->type = TYPE_INTEL; 38026333e93cSRadim Krčmář 3803e6b6af05SRadim Krčmář if (!vtd_decide_config(s, errp)) { 38046333e93cSRadim Krčmář return; 38056333e93cSRadim Krčmář } 38066333e93cSRadim Krčmář 3807b4a4ba0dSPeter Xu QLIST_INIT(&s->vtd_as_with_notifiers); 38081d9efa73SPeter Xu qemu_mutex_init(&s->iommu_lock); 38097df953bdSKnut Omang memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num)); 38101da12ec4SLe Tan memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 38111da12ec4SLe Tan "intel_iommu", DMAR_REG_SIZE); 38124b519ef1SPeter Xu 38134b519ef1SPeter Xu /* Create the shared memory regions by all devices */ 38144b519ef1SPeter Xu memory_region_init(&s->mr_nodmar, OBJECT(s), "vtd-nodmar", 38154b519ef1SPeter Xu UINT64_MAX); 38164b519ef1SPeter Xu memory_region_init_io(&s->mr_ir, OBJECT(s), &vtd_mem_ir_ops, 38174b519ef1SPeter Xu s, "vtd-ir", VTD_INTERRUPT_ADDR_SIZE); 38184b519ef1SPeter Xu memory_region_init_alias(&s->mr_sys_alias, OBJECT(s), 38194b519ef1SPeter Xu "vtd-sys-alias", get_system_memory(), 0, 38204b519ef1SPeter Xu memory_region_size(get_system_memory())); 38214b519ef1SPeter Xu memory_region_add_subregion_overlap(&s->mr_nodmar, 0, 38224b519ef1SPeter Xu &s->mr_sys_alias, 0); 38234b519ef1SPeter Xu memory_region_add_subregion_overlap(&s->mr_nodmar, 38244b519ef1SPeter Xu VTD_INTERRUPT_ADDR_FIRST, 38254b519ef1SPeter Xu &s->mr_ir, 1); 38264b519ef1SPeter Xu 38271da12ec4SLe Tan sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem); 3828b5a280c0SLe Tan /* No corresponding destroy */ 3829b5a280c0SLe Tan s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 3830b5a280c0SLe Tan g_free, g_free); 38317df953bdSKnut Omang s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 38327df953bdSKnut Omang g_free, g_free); 38331da12ec4SLe Tan vtd_init(s); 3834621d983aSMarcel Apfelbaum sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR); 3835621d983aSMarcel Apfelbaum pci_setup_iommu(bus, vtd_host_dma_iommu, dev); 3836cb135f59SPeter Xu /* Pseudo address space under root PCI bus. */ 3837f0bb276bSPaolo Bonzini x86ms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC); 383828cf553aSPeter Xu qemu_add_machine_init_done_notifier(&vtd_machine_done_notify); 38391da12ec4SLe Tan } 38401da12ec4SLe Tan 38411da12ec4SLe Tan static void vtd_class_init(ObjectClass *klass, void *data) 38421da12ec4SLe Tan { 38431da12ec4SLe Tan DeviceClass *dc = DEVICE_CLASS(klass); 38441c7955c4SPeter Xu X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass); 38451da12ec4SLe Tan 38461da12ec4SLe Tan dc->reset = vtd_reset; 38471da12ec4SLe Tan dc->vmsd = &vtd_vmstate; 38481da12ec4SLe Tan dc->props = vtd_properties; 3849621d983aSMarcel Apfelbaum dc->hotpluggable = false; 38501c7955c4SPeter Xu x86_class->realize = vtd_realize; 38518b5ed7dfSPeter Xu x86_class->int_remap = vtd_int_remap; 38528ab5700cSEduardo Habkost /* Supported by the pc-q35-* machine types */ 3853e4f4fb1eSEduardo Habkost dc->user_creatable = true; 38541ec202c9SErnest Esene set_bit(DEVICE_CATEGORY_MISC, dc->categories); 38551ec202c9SErnest Esene dc->desc = "Intel IOMMU (VT-d) DMA Remapping device"; 38561da12ec4SLe Tan } 38571da12ec4SLe Tan 38581da12ec4SLe Tan static const TypeInfo vtd_info = { 38591da12ec4SLe Tan .name = TYPE_INTEL_IOMMU_DEVICE, 38601c7955c4SPeter Xu .parent = TYPE_X86_IOMMU_DEVICE, 38611da12ec4SLe Tan .instance_size = sizeof(IntelIOMMUState), 38621da12ec4SLe Tan .class_init = vtd_class_init, 38631da12ec4SLe Tan }; 38641da12ec4SLe Tan 38651221a474SAlexey Kardashevskiy static void vtd_iommu_memory_region_class_init(ObjectClass *klass, 38661221a474SAlexey Kardashevskiy void *data) 38671221a474SAlexey Kardashevskiy { 38681221a474SAlexey Kardashevskiy IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 38691221a474SAlexey Kardashevskiy 38701221a474SAlexey Kardashevskiy imrc->translate = vtd_iommu_translate; 38711221a474SAlexey Kardashevskiy imrc->notify_flag_changed = vtd_iommu_notify_flag_changed; 38721221a474SAlexey Kardashevskiy imrc->replay = vtd_iommu_replay; 38731221a474SAlexey Kardashevskiy } 38741221a474SAlexey Kardashevskiy 38751221a474SAlexey Kardashevskiy static const TypeInfo vtd_iommu_memory_region_info = { 38761221a474SAlexey Kardashevskiy .parent = TYPE_IOMMU_MEMORY_REGION, 38771221a474SAlexey Kardashevskiy .name = TYPE_INTEL_IOMMU_MEMORY_REGION, 38781221a474SAlexey Kardashevskiy .class_init = vtd_iommu_memory_region_class_init, 38791221a474SAlexey Kardashevskiy }; 38801221a474SAlexey Kardashevskiy 38811da12ec4SLe Tan static void vtd_register_types(void) 38821da12ec4SLe Tan { 38831da12ec4SLe Tan type_register_static(&vtd_info); 38841221a474SAlexey Kardashevskiy type_register_static(&vtd_iommu_memory_region_info); 38851da12ec4SLe Tan } 38861da12ec4SLe Tan 38871da12ec4SLe Tan type_init(vtd_register_types) 3888