xref: /openbmc/qemu/hw/i386/intel_iommu.c (revision 56fc1e6ac6bde95bc0369d358587f2234d4dddad)
11da12ec4SLe Tan /*
21da12ec4SLe Tan  * QEMU emulation of an Intel IOMMU (VT-d)
31da12ec4SLe Tan  *   (DMA Remapping device)
41da12ec4SLe Tan  *
51da12ec4SLe Tan  * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
61da12ec4SLe Tan  * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
71da12ec4SLe Tan  *
81da12ec4SLe Tan  * This program is free software; you can redistribute it and/or modify
91da12ec4SLe Tan  * it under the terms of the GNU General Public License as published by
101da12ec4SLe Tan  * the Free Software Foundation; either version 2 of the License, or
111da12ec4SLe Tan  * (at your option) any later version.
121da12ec4SLe Tan 
131da12ec4SLe Tan  * This program is distributed in the hope that it will be useful,
141da12ec4SLe Tan  * but WITHOUT ANY WARRANTY; without even the implied warranty of
151da12ec4SLe Tan  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
161da12ec4SLe Tan  * GNU General Public License for more details.
171da12ec4SLe Tan 
181da12ec4SLe Tan  * You should have received a copy of the GNU General Public License along
191da12ec4SLe Tan  * with this program; if not, see <http://www.gnu.org/licenses/>.
201da12ec4SLe Tan  */
211da12ec4SLe Tan 
22b6a0aa05SPeter Maydell #include "qemu/osdep.h"
234684a204SPeter Xu #include "qemu/error-report.h"
24db725815SMarkus Armbruster #include "qemu/main-loop.h"
256333e93cSRadim Krčmář #include "qapi/error.h"
261da12ec4SLe Tan #include "hw/sysbus.h"
271da12ec4SLe Tan #include "exec/address-spaces.h"
281da12ec4SLe Tan #include "intel_iommu_internal.h"
297df953bdSKnut Omang #include "hw/pci/pci.h"
303cb3b154SAlex Williamson #include "hw/pci/pci_bus.h"
31a27bd6c7SMarkus Armbruster #include "hw/qdev-properties.h"
32621d983aSMarcel Apfelbaum #include "hw/i386/pc.h"
33dea651a9SFeng Wu #include "hw/i386/apic-msidef.h"
3404af0e18SPeter Xu #include "hw/boards.h"
3504af0e18SPeter Xu #include "hw/i386/x86-iommu.h"
36cb135f59SPeter Xu #include "hw/pci-host/q35.h"
374684a204SPeter Xu #include "sysemu/kvm.h"
3828cf553aSPeter Xu #include "sysemu/sysemu.h"
3932946019SRadim Krčmář #include "hw/i386/apic_internal.h"
40fb506e70SRadim Krčmář #include "kvm_i386.h"
41d6454270SMarkus Armbruster #include "migration/vmstate.h"
42bc535e59SPeter Xu #include "trace.h"
431da12ec4SLe Tan 
44fb43cf73SLiu, Yi L /* context entry operations */
45fb43cf73SLiu, Yi L #define VTD_CE_GET_RID2PASID(ce) \
46fb43cf73SLiu, Yi L     ((ce)->val[1] & VTD_SM_CONTEXT_ENTRY_RID2PASID_MASK)
47fb43cf73SLiu, Yi L #define VTD_CE_GET_PASID_DIR_TABLE(ce) \
48fb43cf73SLiu, Yi L     ((ce)->val[0] & VTD_PASID_DIR_BASE_ADDR_MASK)
49fb43cf73SLiu, Yi L 
50fb43cf73SLiu, Yi L /* pe operations */
51fb43cf73SLiu, Yi L #define VTD_PE_GET_TYPE(pe) ((pe)->val[0] & VTD_SM_PASID_ENTRY_PGTT)
52fb43cf73SLiu, Yi L #define VTD_PE_GET_LEVEL(pe) (2 + (((pe)->val[0] >> 2) & VTD_SM_PASID_ENTRY_AW))
53fb43cf73SLiu, Yi L #define VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write) {\
54fb43cf73SLiu, Yi L     if (ret_fr) {                                                             \
55fb43cf73SLiu, Yi L         ret_fr = -ret_fr;                                                     \
56fb43cf73SLiu, Yi L         if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {                   \
57fb43cf73SLiu, Yi L             trace_vtd_fault_disabled();                                       \
58fb43cf73SLiu, Yi L         } else {                                                              \
59fb43cf73SLiu, Yi L             vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);      \
60fb43cf73SLiu, Yi L         }                                                                     \
61fb43cf73SLiu, Yi L         goto error;                                                           \
62fb43cf73SLiu, Yi L     }                                                                         \
63fb43cf73SLiu, Yi L }
64fb43cf73SLiu, Yi L 
652cc9ddccSPeter Xu static void vtd_address_space_refresh_all(IntelIOMMUState *s);
66c28b535dSPeter Xu static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n);
672cc9ddccSPeter Xu 
6828cf553aSPeter Xu static void vtd_panic_require_caching_mode(void)
6928cf553aSPeter Xu {
7028cf553aSPeter Xu     error_report("We need to set caching-mode=on for intel-iommu to enable "
7128cf553aSPeter Xu                  "device assignment with IOMMU protection.");
7228cf553aSPeter Xu     exit(1);
7328cf553aSPeter Xu }
7428cf553aSPeter Xu 
751da12ec4SLe Tan static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
761da12ec4SLe Tan                             uint64_t wmask, uint64_t w1cmask)
771da12ec4SLe Tan {
781da12ec4SLe Tan     stq_le_p(&s->csr[addr], val);
791da12ec4SLe Tan     stq_le_p(&s->wmask[addr], wmask);
801da12ec4SLe Tan     stq_le_p(&s->w1cmask[addr], w1cmask);
811da12ec4SLe Tan }
821da12ec4SLe Tan 
831da12ec4SLe Tan static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask)
841da12ec4SLe Tan {
851da12ec4SLe Tan     stq_le_p(&s->womask[addr], mask);
861da12ec4SLe Tan }
871da12ec4SLe Tan 
881da12ec4SLe Tan static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val,
891da12ec4SLe Tan                             uint32_t wmask, uint32_t w1cmask)
901da12ec4SLe Tan {
911da12ec4SLe Tan     stl_le_p(&s->csr[addr], val);
921da12ec4SLe Tan     stl_le_p(&s->wmask[addr], wmask);
931da12ec4SLe Tan     stl_le_p(&s->w1cmask[addr], w1cmask);
941da12ec4SLe Tan }
951da12ec4SLe Tan 
961da12ec4SLe Tan static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask)
971da12ec4SLe Tan {
981da12ec4SLe Tan     stl_le_p(&s->womask[addr], mask);
991da12ec4SLe Tan }
1001da12ec4SLe Tan 
1011da12ec4SLe Tan /* "External" get/set operations */
1021da12ec4SLe Tan static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val)
1031da12ec4SLe Tan {
1041da12ec4SLe Tan     uint64_t oldval = ldq_le_p(&s->csr[addr]);
1051da12ec4SLe Tan     uint64_t wmask = ldq_le_p(&s->wmask[addr]);
1061da12ec4SLe Tan     uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
1071da12ec4SLe Tan     stq_le_p(&s->csr[addr],
1081da12ec4SLe Tan              ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
1091da12ec4SLe Tan }
1101da12ec4SLe Tan 
1111da12ec4SLe Tan static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val)
1121da12ec4SLe Tan {
1131da12ec4SLe Tan     uint32_t oldval = ldl_le_p(&s->csr[addr]);
1141da12ec4SLe Tan     uint32_t wmask = ldl_le_p(&s->wmask[addr]);
1151da12ec4SLe Tan     uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
1161da12ec4SLe Tan     stl_le_p(&s->csr[addr],
1171da12ec4SLe Tan              ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
1181da12ec4SLe Tan }
1191da12ec4SLe Tan 
1201da12ec4SLe Tan static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr)
1211da12ec4SLe Tan {
1221da12ec4SLe Tan     uint64_t val = ldq_le_p(&s->csr[addr]);
1231da12ec4SLe Tan     uint64_t womask = ldq_le_p(&s->womask[addr]);
1241da12ec4SLe Tan     return val & ~womask;
1251da12ec4SLe Tan }
1261da12ec4SLe Tan 
1271da12ec4SLe Tan static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr)
1281da12ec4SLe Tan {
1291da12ec4SLe Tan     uint32_t val = ldl_le_p(&s->csr[addr]);
1301da12ec4SLe Tan     uint32_t womask = ldl_le_p(&s->womask[addr]);
1311da12ec4SLe Tan     return val & ~womask;
1321da12ec4SLe Tan }
1331da12ec4SLe Tan 
1341da12ec4SLe Tan /* "Internal" get/set operations */
1351da12ec4SLe Tan static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr)
1361da12ec4SLe Tan {
1371da12ec4SLe Tan     return ldq_le_p(&s->csr[addr]);
1381da12ec4SLe Tan }
1391da12ec4SLe Tan 
1401da12ec4SLe Tan static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr)
1411da12ec4SLe Tan {
1421da12ec4SLe Tan     return ldl_le_p(&s->csr[addr]);
1431da12ec4SLe Tan }
1441da12ec4SLe Tan 
1451da12ec4SLe Tan static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val)
1461da12ec4SLe Tan {
1471da12ec4SLe Tan     stq_le_p(&s->csr[addr], val);
1481da12ec4SLe Tan }
1491da12ec4SLe Tan 
1501da12ec4SLe Tan static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr,
1511da12ec4SLe Tan                                         uint32_t clear, uint32_t mask)
1521da12ec4SLe Tan {
1531da12ec4SLe Tan     uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask;
1541da12ec4SLe Tan     stl_le_p(&s->csr[addr], new_val);
1551da12ec4SLe Tan     return new_val;
1561da12ec4SLe Tan }
1571da12ec4SLe Tan 
1581da12ec4SLe Tan static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
1591da12ec4SLe Tan                                         uint64_t clear, uint64_t mask)
1601da12ec4SLe Tan {
1611da12ec4SLe Tan     uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask;
1621da12ec4SLe Tan     stq_le_p(&s->csr[addr], new_val);
1631da12ec4SLe Tan     return new_val;
1641da12ec4SLe Tan }
1651da12ec4SLe Tan 
1661d9efa73SPeter Xu static inline void vtd_iommu_lock(IntelIOMMUState *s)
1671d9efa73SPeter Xu {
1681d9efa73SPeter Xu     qemu_mutex_lock(&s->iommu_lock);
1691d9efa73SPeter Xu }
1701d9efa73SPeter Xu 
1711d9efa73SPeter Xu static inline void vtd_iommu_unlock(IntelIOMMUState *s)
1721d9efa73SPeter Xu {
1731d9efa73SPeter Xu     qemu_mutex_unlock(&s->iommu_lock);
1741d9efa73SPeter Xu }
1751d9efa73SPeter Xu 
1762811af3bSPeter Xu static void vtd_update_scalable_state(IntelIOMMUState *s)
1772811af3bSPeter Xu {
1782811af3bSPeter Xu     uint64_t val = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
1792811af3bSPeter Xu 
1802811af3bSPeter Xu     if (s->scalable_mode) {
1812811af3bSPeter Xu         s->root_scalable = val & VTD_RTADDR_SMT;
1822811af3bSPeter Xu     }
1832811af3bSPeter Xu }
1842811af3bSPeter Xu 
1854f8a62a9SPeter Xu /* Whether the address space needs to notify new mappings */
1864f8a62a9SPeter Xu static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as)
1874f8a62a9SPeter Xu {
1884f8a62a9SPeter Xu     return as->notifier_flags & IOMMU_NOTIFIER_MAP;
1894f8a62a9SPeter Xu }
1904f8a62a9SPeter Xu 
191b5a280c0SLe Tan /* GHashTable functions */
192b5a280c0SLe Tan static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2)
193b5a280c0SLe Tan {
194b5a280c0SLe Tan     return *((const uint64_t *)v1) == *((const uint64_t *)v2);
195b5a280c0SLe Tan }
196b5a280c0SLe Tan 
197b5a280c0SLe Tan static guint vtd_uint64_hash(gconstpointer v)
198b5a280c0SLe Tan {
199b5a280c0SLe Tan     return (guint)*(const uint64_t *)v;
200b5a280c0SLe Tan }
201b5a280c0SLe Tan 
202b5a280c0SLe Tan static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
203b5a280c0SLe Tan                                           gpointer user_data)
204b5a280c0SLe Tan {
205b5a280c0SLe Tan     VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
206b5a280c0SLe Tan     uint16_t domain_id = *(uint16_t *)user_data;
207b5a280c0SLe Tan     return entry->domain_id == domain_id;
208b5a280c0SLe Tan }
209b5a280c0SLe Tan 
210d66b969bSJason Wang /* The shift of an addr for a certain level of paging structure */
211d66b969bSJason Wang static inline uint32_t vtd_slpt_level_shift(uint32_t level)
212d66b969bSJason Wang {
2137e58326aSPeter Xu     assert(level != 0);
214d66b969bSJason Wang     return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
215d66b969bSJason Wang }
216d66b969bSJason Wang 
217d66b969bSJason Wang static inline uint64_t vtd_slpt_level_page_mask(uint32_t level)
218d66b969bSJason Wang {
219d66b969bSJason Wang     return ~((1ULL << vtd_slpt_level_shift(level)) - 1);
220d66b969bSJason Wang }
221d66b969bSJason Wang 
222b5a280c0SLe Tan static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
223b5a280c0SLe Tan                                         gpointer user_data)
224b5a280c0SLe Tan {
225b5a280c0SLe Tan     VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
226b5a280c0SLe Tan     VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
227d66b969bSJason Wang     uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
228d66b969bSJason Wang     uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
229b5a280c0SLe Tan     return (entry->domain_id == info->domain_id) &&
230d66b969bSJason Wang             (((entry->gfn & info->mask) == gfn) ||
231d66b969bSJason Wang              (entry->gfn == gfn_tlb));
232b5a280c0SLe Tan }
233b5a280c0SLe Tan 
234d92fa2dcSLe Tan /* Reset all the gen of VTDAddressSpace to zero and set the gen of
2351d9efa73SPeter Xu  * IntelIOMMUState to 1.  Must be called with IOMMU lock held.
236d92fa2dcSLe Tan  */
2371d9efa73SPeter Xu static void vtd_reset_context_cache_locked(IntelIOMMUState *s)
238d92fa2dcSLe Tan {
239d92fa2dcSLe Tan     VTDAddressSpace *vtd_as;
2407df953bdSKnut Omang     VTDBus *vtd_bus;
2417df953bdSKnut Omang     GHashTableIter bus_it;
242d92fa2dcSLe Tan     uint32_t devfn_it;
243d92fa2dcSLe Tan 
2447feb51b7SPeter Xu     trace_vtd_context_cache_reset();
2457feb51b7SPeter Xu 
2467df953bdSKnut Omang     g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
2477df953bdSKnut Omang 
2487df953bdSKnut Omang     while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
249bf33cc75SPeter Xu         for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) {
2507df953bdSKnut Omang             vtd_as = vtd_bus->dev_as[devfn_it];
251d92fa2dcSLe Tan             if (!vtd_as) {
252d92fa2dcSLe Tan                 continue;
253d92fa2dcSLe Tan             }
254d92fa2dcSLe Tan             vtd_as->context_cache_entry.context_cache_gen = 0;
255d92fa2dcSLe Tan         }
256d92fa2dcSLe Tan     }
257d92fa2dcSLe Tan     s->context_cache_gen = 1;
258d92fa2dcSLe Tan }
259d92fa2dcSLe Tan 
2601d9efa73SPeter Xu /* Must be called with IOMMU lock held. */
2611d9efa73SPeter Xu static void vtd_reset_iotlb_locked(IntelIOMMUState *s)
262b5a280c0SLe Tan {
263b5a280c0SLe Tan     assert(s->iotlb);
264b5a280c0SLe Tan     g_hash_table_remove_all(s->iotlb);
265b5a280c0SLe Tan }
266b5a280c0SLe Tan 
2671d9efa73SPeter Xu static void vtd_reset_iotlb(IntelIOMMUState *s)
2681d9efa73SPeter Xu {
2691d9efa73SPeter Xu     vtd_iommu_lock(s);
2701d9efa73SPeter Xu     vtd_reset_iotlb_locked(s);
2711d9efa73SPeter Xu     vtd_iommu_unlock(s);
2721d9efa73SPeter Xu }
2731d9efa73SPeter Xu 
27406aba4caSPeter Xu static void vtd_reset_caches(IntelIOMMUState *s)
27506aba4caSPeter Xu {
27606aba4caSPeter Xu     vtd_iommu_lock(s);
27706aba4caSPeter Xu     vtd_reset_iotlb_locked(s);
27806aba4caSPeter Xu     vtd_reset_context_cache_locked(s);
27906aba4caSPeter Xu     vtd_iommu_unlock(s);
28006aba4caSPeter Xu }
28106aba4caSPeter Xu 
282bacabb0aSJason Wang static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint16_t source_id,
283d66b969bSJason Wang                                   uint32_t level)
284d66b969bSJason Wang {
285d66b969bSJason Wang     return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) |
286d66b969bSJason Wang            ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT);
287d66b969bSJason Wang }
288d66b969bSJason Wang 
289d66b969bSJason Wang static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
290d66b969bSJason Wang {
291d66b969bSJason Wang     return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
292d66b969bSJason Wang }
293d66b969bSJason Wang 
2941d9efa73SPeter Xu /* Must be called with IOMMU lock held */
295b5a280c0SLe Tan static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
296b5a280c0SLe Tan                                        hwaddr addr)
297b5a280c0SLe Tan {
298d66b969bSJason Wang     VTDIOTLBEntry *entry;
299b5a280c0SLe Tan     uint64_t key;
300d66b969bSJason Wang     int level;
301b5a280c0SLe Tan 
302d66b969bSJason Wang     for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) {
303d66b969bSJason Wang         key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level),
304d66b969bSJason Wang                                 source_id, level);
305d66b969bSJason Wang         entry = g_hash_table_lookup(s->iotlb, &key);
306d66b969bSJason Wang         if (entry) {
307d66b969bSJason Wang             goto out;
308d66b969bSJason Wang         }
309d66b969bSJason Wang     }
310b5a280c0SLe Tan 
311d66b969bSJason Wang out:
312d66b969bSJason Wang     return entry;
313b5a280c0SLe Tan }
314b5a280c0SLe Tan 
3151d9efa73SPeter Xu /* Must be with IOMMU lock held */
316b5a280c0SLe Tan static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
317b5a280c0SLe Tan                              uint16_t domain_id, hwaddr addr, uint64_t slpte,
31807f7b733SPeter Xu                              uint8_t access_flags, uint32_t level)
319b5a280c0SLe Tan {
320b5a280c0SLe Tan     VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
321b5a280c0SLe Tan     uint64_t *key = g_malloc(sizeof(*key));
322d66b969bSJason Wang     uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
323b5a280c0SLe Tan 
3246c441e1dSPeter Xu     trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id);
325b5a280c0SLe Tan     if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) {
3266c441e1dSPeter Xu         trace_vtd_iotlb_reset("iotlb exceeds size limit");
3271d9efa73SPeter Xu         vtd_reset_iotlb_locked(s);
328b5a280c0SLe Tan     }
329b5a280c0SLe Tan 
330b5a280c0SLe Tan     entry->gfn = gfn;
331b5a280c0SLe Tan     entry->domain_id = domain_id;
332b5a280c0SLe Tan     entry->slpte = slpte;
33307f7b733SPeter Xu     entry->access_flags = access_flags;
334d66b969bSJason Wang     entry->mask = vtd_slpt_level_page_mask(level);
335d66b969bSJason Wang     *key = vtd_get_iotlb_key(gfn, source_id, level);
336b5a280c0SLe Tan     g_hash_table_replace(s->iotlb, key, entry);
337b5a280c0SLe Tan }
338b5a280c0SLe Tan 
3391da12ec4SLe Tan /* Given the reg addr of both the message data and address, generate an
3401da12ec4SLe Tan  * interrupt via MSI.
3411da12ec4SLe Tan  */
3421da12ec4SLe Tan static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg,
3431da12ec4SLe Tan                                    hwaddr mesg_data_reg)
3441da12ec4SLe Tan {
34532946019SRadim Krčmář     MSIMessage msi;
3461da12ec4SLe Tan 
3471da12ec4SLe Tan     assert(mesg_data_reg < DMAR_REG_SIZE);
3481da12ec4SLe Tan     assert(mesg_addr_reg < DMAR_REG_SIZE);
3491da12ec4SLe Tan 
35032946019SRadim Krčmář     msi.address = vtd_get_long_raw(s, mesg_addr_reg);
35132946019SRadim Krčmář     msi.data = vtd_get_long_raw(s, mesg_data_reg);
3521da12ec4SLe Tan 
3537feb51b7SPeter Xu     trace_vtd_irq_generate(msi.address, msi.data);
3547feb51b7SPeter Xu 
35532946019SRadim Krčmář     apic_get_class()->send_msi(&msi);
3561da12ec4SLe Tan }
3571da12ec4SLe Tan 
3581da12ec4SLe Tan /* Generate a fault event to software via MSI if conditions are met.
3591da12ec4SLe Tan  * Notice that the value of FSTS_REG being passed to it should be the one
3601da12ec4SLe Tan  * before any update.
3611da12ec4SLe Tan  */
3621da12ec4SLe Tan static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts)
3631da12ec4SLe Tan {
3641da12ec4SLe Tan     if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO ||
3651da12ec4SLe Tan         pre_fsts & VTD_FSTS_IQE) {
3661376211fSPeter Xu         error_report_once("There are previous interrupt conditions "
3677feb51b7SPeter Xu                           "to be serviced by software, fault event "
3681376211fSPeter Xu                           "is not generated");
3691da12ec4SLe Tan         return;
3701da12ec4SLe Tan     }
3711da12ec4SLe Tan     vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP);
3721da12ec4SLe Tan     if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) {
3731376211fSPeter Xu         error_report_once("Interrupt Mask set, irq is not generated");
3741da12ec4SLe Tan     } else {
3751da12ec4SLe Tan         vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
3761da12ec4SLe Tan         vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
3771da12ec4SLe Tan     }
3781da12ec4SLe Tan }
3791da12ec4SLe Tan 
3801da12ec4SLe Tan /* Check if the Fault (F) field of the Fault Recording Register referenced by
3811da12ec4SLe Tan  * @index is Set.
3821da12ec4SLe Tan  */
3831da12ec4SLe Tan static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index)
3841da12ec4SLe Tan {
3851da12ec4SLe Tan     /* Each reg is 128-bit */
3861da12ec4SLe Tan     hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
3871da12ec4SLe Tan     addr += 8; /* Access the high 64-bit half */
3881da12ec4SLe Tan 
3891da12ec4SLe Tan     assert(index < DMAR_FRCD_REG_NR);
3901da12ec4SLe Tan 
3911da12ec4SLe Tan     return vtd_get_quad_raw(s, addr) & VTD_FRCD_F;
3921da12ec4SLe Tan }
3931da12ec4SLe Tan 
3941da12ec4SLe Tan /* Update the PPF field of Fault Status Register.
3951da12ec4SLe Tan  * Should be called whenever change the F field of any fault recording
3961da12ec4SLe Tan  * registers.
3971da12ec4SLe Tan  */
3981da12ec4SLe Tan static void vtd_update_fsts_ppf(IntelIOMMUState *s)
3991da12ec4SLe Tan {
4001da12ec4SLe Tan     uint32_t i;
4011da12ec4SLe Tan     uint32_t ppf_mask = 0;
4021da12ec4SLe Tan 
4031da12ec4SLe Tan     for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
4041da12ec4SLe Tan         if (vtd_is_frcd_set(s, i)) {
4051da12ec4SLe Tan             ppf_mask = VTD_FSTS_PPF;
4061da12ec4SLe Tan             break;
4071da12ec4SLe Tan         }
4081da12ec4SLe Tan     }
4091da12ec4SLe Tan     vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask);
4107feb51b7SPeter Xu     trace_vtd_fsts_ppf(!!ppf_mask);
4111da12ec4SLe Tan }
4121da12ec4SLe Tan 
4131da12ec4SLe Tan static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index)
4141da12ec4SLe Tan {
4151da12ec4SLe Tan     /* Each reg is 128-bit */
4161da12ec4SLe Tan     hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
4171da12ec4SLe Tan     addr += 8; /* Access the high 64-bit half */
4181da12ec4SLe Tan 
4191da12ec4SLe Tan     assert(index < DMAR_FRCD_REG_NR);
4201da12ec4SLe Tan 
4211da12ec4SLe Tan     vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F);
4221da12ec4SLe Tan     vtd_update_fsts_ppf(s);
4231da12ec4SLe Tan }
4241da12ec4SLe Tan 
4251da12ec4SLe Tan /* Must not update F field now, should be done later */
4261da12ec4SLe Tan static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index,
4271da12ec4SLe Tan                             uint16_t source_id, hwaddr addr,
4281da12ec4SLe Tan                             VTDFaultReason fault, bool is_write)
4291da12ec4SLe Tan {
4301da12ec4SLe Tan     uint64_t hi = 0, lo;
4311da12ec4SLe Tan     hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
4321da12ec4SLe Tan 
4331da12ec4SLe Tan     assert(index < DMAR_FRCD_REG_NR);
4341da12ec4SLe Tan 
4351da12ec4SLe Tan     lo = VTD_FRCD_FI(addr);
4361da12ec4SLe Tan     hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault);
4371da12ec4SLe Tan     if (!is_write) {
4381da12ec4SLe Tan         hi |= VTD_FRCD_T;
4391da12ec4SLe Tan     }
4401da12ec4SLe Tan     vtd_set_quad_raw(s, frcd_reg_addr, lo);
4411da12ec4SLe Tan     vtd_set_quad_raw(s, frcd_reg_addr + 8, hi);
4427feb51b7SPeter Xu 
4437feb51b7SPeter Xu     trace_vtd_frr_new(index, hi, lo);
4441da12ec4SLe Tan }
4451da12ec4SLe Tan 
4461da12ec4SLe Tan /* Try to collapse multiple pending faults from the same requester */
4471da12ec4SLe Tan static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id)
4481da12ec4SLe Tan {
4491da12ec4SLe Tan     uint32_t i;
4501da12ec4SLe Tan     uint64_t frcd_reg;
4511da12ec4SLe Tan     hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */
4521da12ec4SLe Tan 
4531da12ec4SLe Tan     for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
4541da12ec4SLe Tan         frcd_reg = vtd_get_quad_raw(s, addr);
4551da12ec4SLe Tan         if ((frcd_reg & VTD_FRCD_F) &&
4561da12ec4SLe Tan             ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) {
4571da12ec4SLe Tan             return true;
4581da12ec4SLe Tan         }
4591da12ec4SLe Tan         addr += 16; /* 128-bit for each */
4601da12ec4SLe Tan     }
4611da12ec4SLe Tan     return false;
4621da12ec4SLe Tan }
4631da12ec4SLe Tan 
4641da12ec4SLe Tan /* Log and report an DMAR (address translation) fault to software */
4651da12ec4SLe Tan static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id,
4661da12ec4SLe Tan                                   hwaddr addr, VTDFaultReason fault,
4671da12ec4SLe Tan                                   bool is_write)
4681da12ec4SLe Tan {
4691da12ec4SLe Tan     uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
4701da12ec4SLe Tan 
4711da12ec4SLe Tan     assert(fault < VTD_FR_MAX);
4721da12ec4SLe Tan 
4731da12ec4SLe Tan     if (fault == VTD_FR_RESERVED_ERR) {
4741da12ec4SLe Tan         /* This is not a normal fault reason case. Drop it. */
4751da12ec4SLe Tan         return;
4761da12ec4SLe Tan     }
4777feb51b7SPeter Xu 
4787feb51b7SPeter Xu     trace_vtd_dmar_fault(source_id, fault, addr, is_write);
4797feb51b7SPeter Xu 
4801da12ec4SLe Tan     if (fsts_reg & VTD_FSTS_PFO) {
4811376211fSPeter Xu         error_report_once("New fault is not recorded due to "
4821376211fSPeter Xu                           "Primary Fault Overflow");
4831da12ec4SLe Tan         return;
4841da12ec4SLe Tan     }
4857feb51b7SPeter Xu 
4861da12ec4SLe Tan     if (vtd_try_collapse_fault(s, source_id)) {
4871376211fSPeter Xu         error_report_once("New fault is not recorded due to "
4881376211fSPeter Xu                           "compression of faults");
4891da12ec4SLe Tan         return;
4901da12ec4SLe Tan     }
4917feb51b7SPeter Xu 
4921da12ec4SLe Tan     if (vtd_is_frcd_set(s, s->next_frcd_reg)) {
4931376211fSPeter Xu         error_report_once("Next Fault Recording Reg is used, "
4941376211fSPeter Xu                           "new fault is not recorded, set PFO field");
4951da12ec4SLe Tan         vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO);
4961da12ec4SLe Tan         return;
4971da12ec4SLe Tan     }
4981da12ec4SLe Tan 
4991da12ec4SLe Tan     vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write);
5001da12ec4SLe Tan 
5011da12ec4SLe Tan     if (fsts_reg & VTD_FSTS_PPF) {
5021376211fSPeter Xu         error_report_once("There are pending faults already, "
5031376211fSPeter Xu                           "fault event is not generated");
5041da12ec4SLe Tan         vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg);
5051da12ec4SLe Tan         s->next_frcd_reg++;
5061da12ec4SLe Tan         if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
5071da12ec4SLe Tan             s->next_frcd_reg = 0;
5081da12ec4SLe Tan         }
5091da12ec4SLe Tan     } else {
5101da12ec4SLe Tan         vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK,
5111da12ec4SLe Tan                                 VTD_FSTS_FRI(s->next_frcd_reg));
5121da12ec4SLe Tan         vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */
5131da12ec4SLe Tan         s->next_frcd_reg++;
5141da12ec4SLe Tan         if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
5151da12ec4SLe Tan             s->next_frcd_reg = 0;
5161da12ec4SLe Tan         }
5171da12ec4SLe Tan         /* This case actually cause the PPF to be Set.
5181da12ec4SLe Tan          * So generate fault event (interrupt).
5191da12ec4SLe Tan          */
5201da12ec4SLe Tan          vtd_generate_fault_event(s, fsts_reg);
5211da12ec4SLe Tan     }
5221da12ec4SLe Tan }
5231da12ec4SLe Tan 
524ed7b8fbcSLe Tan /* Handle Invalidation Queue Errors of queued invalidation interface error
525ed7b8fbcSLe Tan  * conditions.
526ed7b8fbcSLe Tan  */
527ed7b8fbcSLe Tan static void vtd_handle_inv_queue_error(IntelIOMMUState *s)
528ed7b8fbcSLe Tan {
529ed7b8fbcSLe Tan     uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
530ed7b8fbcSLe Tan 
531ed7b8fbcSLe Tan     vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE);
532ed7b8fbcSLe Tan     vtd_generate_fault_event(s, fsts_reg);
533ed7b8fbcSLe Tan }
534ed7b8fbcSLe Tan 
535ed7b8fbcSLe Tan /* Set the IWC field and try to generate an invalidation completion interrupt */
536ed7b8fbcSLe Tan static void vtd_generate_completion_event(IntelIOMMUState *s)
537ed7b8fbcSLe Tan {
538ed7b8fbcSLe Tan     if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) {
539bc535e59SPeter Xu         trace_vtd_inv_desc_wait_irq("One pending, skip current");
540ed7b8fbcSLe Tan         return;
541ed7b8fbcSLe Tan     }
542ed7b8fbcSLe Tan     vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC);
543ed7b8fbcSLe Tan     vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP);
544ed7b8fbcSLe Tan     if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) {
545bc535e59SPeter Xu         trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, "
546bc535e59SPeter Xu                                     "new event not generated");
547ed7b8fbcSLe Tan         return;
548ed7b8fbcSLe Tan     } else {
549ed7b8fbcSLe Tan         /* Generate the interrupt event */
550bc535e59SPeter Xu         trace_vtd_inv_desc_wait_irq("Generating complete event");
551ed7b8fbcSLe Tan         vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
552ed7b8fbcSLe Tan         vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
553ed7b8fbcSLe Tan     }
554ed7b8fbcSLe Tan }
555ed7b8fbcSLe Tan 
556fb43cf73SLiu, Yi L static inline bool vtd_root_entry_present(IntelIOMMUState *s,
557fb43cf73SLiu, Yi L                                           VTDRootEntry *re,
558fb43cf73SLiu, Yi L                                           uint8_t devfn)
5591da12ec4SLe Tan {
560fb43cf73SLiu, Yi L     if (s->root_scalable && devfn > UINT8_MAX / 2) {
561fb43cf73SLiu, Yi L         return re->hi & VTD_ROOT_ENTRY_P;
562fb43cf73SLiu, Yi L     }
563fb43cf73SLiu, Yi L 
564fb43cf73SLiu, Yi L     return re->lo & VTD_ROOT_ENTRY_P;
5651da12ec4SLe Tan }
5661da12ec4SLe Tan 
5671da12ec4SLe Tan static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index,
5681da12ec4SLe Tan                               VTDRootEntry *re)
5691da12ec4SLe Tan {
5701da12ec4SLe Tan     dma_addr_t addr;
5711da12ec4SLe Tan 
5721da12ec4SLe Tan     addr = s->root + index * sizeof(*re);
5731da12ec4SLe Tan     if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) {
574fb43cf73SLiu, Yi L         re->lo = 0;
5751da12ec4SLe Tan         return -VTD_FR_ROOT_TABLE_INV;
5761da12ec4SLe Tan     }
577fb43cf73SLiu, Yi L     re->lo = le64_to_cpu(re->lo);
578fb43cf73SLiu, Yi L     re->hi = le64_to_cpu(re->hi);
5791da12ec4SLe Tan     return 0;
5801da12ec4SLe Tan }
5811da12ec4SLe Tan 
5828f7d7161SPeter Xu static inline bool vtd_ce_present(VTDContextEntry *context)
5831da12ec4SLe Tan {
5841da12ec4SLe Tan     return context->lo & VTD_CONTEXT_ENTRY_P;
5851da12ec4SLe Tan }
5861da12ec4SLe Tan 
587fb43cf73SLiu, Yi L static int vtd_get_context_entry_from_root(IntelIOMMUState *s,
588fb43cf73SLiu, Yi L                                            VTDRootEntry *re,
589fb43cf73SLiu, Yi L                                            uint8_t index,
5901da12ec4SLe Tan                                            VTDContextEntry *ce)
5911da12ec4SLe Tan {
592fb43cf73SLiu, Yi L     dma_addr_t addr, ce_size;
5931da12ec4SLe Tan 
5946c441e1dSPeter Xu     /* we have checked that root entry is present */
595fb43cf73SLiu, Yi L     ce_size = s->root_scalable ? VTD_CTX_ENTRY_SCALABLE_SIZE :
596fb43cf73SLiu, Yi L               VTD_CTX_ENTRY_LEGACY_SIZE;
597fb43cf73SLiu, Yi L 
598fb43cf73SLiu, Yi L     if (s->root_scalable && index > UINT8_MAX / 2) {
599fb43cf73SLiu, Yi L         index = index & (~VTD_DEVFN_CHECK_MASK);
600fb43cf73SLiu, Yi L         addr = re->hi & VTD_ROOT_ENTRY_CTP;
601fb43cf73SLiu, Yi L     } else {
602fb43cf73SLiu, Yi L         addr = re->lo & VTD_ROOT_ENTRY_CTP;
603fb43cf73SLiu, Yi L     }
604fb43cf73SLiu, Yi L 
605fb43cf73SLiu, Yi L     addr = addr + index * ce_size;
606fb43cf73SLiu, Yi L     if (dma_memory_read(&address_space_memory, addr, ce, ce_size)) {
6071da12ec4SLe Tan         return -VTD_FR_CONTEXT_TABLE_INV;
6081da12ec4SLe Tan     }
609fb43cf73SLiu, Yi L 
6101da12ec4SLe Tan     ce->lo = le64_to_cpu(ce->lo);
6111da12ec4SLe Tan     ce->hi = le64_to_cpu(ce->hi);
612fb43cf73SLiu, Yi L     if (ce_size == VTD_CTX_ENTRY_SCALABLE_SIZE) {
613fb43cf73SLiu, Yi L         ce->val[2] = le64_to_cpu(ce->val[2]);
614fb43cf73SLiu, Yi L         ce->val[3] = le64_to_cpu(ce->val[3]);
615fb43cf73SLiu, Yi L     }
6161da12ec4SLe Tan     return 0;
6171da12ec4SLe Tan }
6181da12ec4SLe Tan 
6198f7d7161SPeter Xu static inline dma_addr_t vtd_ce_get_slpt_base(VTDContextEntry *ce)
6201da12ec4SLe Tan {
6211da12ec4SLe Tan     return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
6221da12ec4SLe Tan }
6231da12ec4SLe Tan 
62437f51384SPrasad Singamsetty static inline uint64_t vtd_get_slpte_addr(uint64_t slpte, uint8_t aw)
6251da12ec4SLe Tan {
62637f51384SPrasad Singamsetty     return slpte & VTD_SL_PT_BASE_ADDR_MASK(aw);
6271da12ec4SLe Tan }
6281da12ec4SLe Tan 
6291da12ec4SLe Tan /* Whether the pte indicates the address of the page frame */
6301da12ec4SLe Tan static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level)
6311da12ec4SLe Tan {
6321da12ec4SLe Tan     return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK);
6331da12ec4SLe Tan }
6341da12ec4SLe Tan 
6351da12ec4SLe Tan /* Get the content of a spte located in @base_addr[@index] */
6361da12ec4SLe Tan static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index)
6371da12ec4SLe Tan {
6381da12ec4SLe Tan     uint64_t slpte;
6391da12ec4SLe Tan 
6401da12ec4SLe Tan     assert(index < VTD_SL_PT_ENTRY_NR);
6411da12ec4SLe Tan 
6421da12ec4SLe Tan     if (dma_memory_read(&address_space_memory,
6431da12ec4SLe Tan                         base_addr + index * sizeof(slpte), &slpte,
6441da12ec4SLe Tan                         sizeof(slpte))) {
6451da12ec4SLe Tan         slpte = (uint64_t)-1;
6461da12ec4SLe Tan         return slpte;
6471da12ec4SLe Tan     }
6481da12ec4SLe Tan     slpte = le64_to_cpu(slpte);
6491da12ec4SLe Tan     return slpte;
6501da12ec4SLe Tan }
6511da12ec4SLe Tan 
6526e905564SPeter Xu /* Given an iova and the level of paging structure, return the offset
6536e905564SPeter Xu  * of current level.
6541da12ec4SLe Tan  */
6556e905564SPeter Xu static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level)
6561da12ec4SLe Tan {
6576e905564SPeter Xu     return (iova >> vtd_slpt_level_shift(level)) &
6581da12ec4SLe Tan             ((1ULL << VTD_SL_LEVEL_BITS) - 1);
6591da12ec4SLe Tan }
6601da12ec4SLe Tan 
6611da12ec4SLe Tan /* Check Capability Register to see if the @level of page-table is supported */
6621da12ec4SLe Tan static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level)
6631da12ec4SLe Tan {
6641da12ec4SLe Tan     return VTD_CAP_SAGAW_MASK & s->cap &
6651da12ec4SLe Tan            (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT));
6661da12ec4SLe Tan }
6671da12ec4SLe Tan 
668fb43cf73SLiu, Yi L /* Return true if check passed, otherwise false */
669fb43cf73SLiu, Yi L static inline bool vtd_pe_type_check(X86IOMMUState *x86_iommu,
670fb43cf73SLiu, Yi L                                      VTDPASIDEntry *pe)
671fb43cf73SLiu, Yi L {
672fb43cf73SLiu, Yi L     switch (VTD_PE_GET_TYPE(pe)) {
673fb43cf73SLiu, Yi L     case VTD_SM_PASID_ENTRY_FLT:
674fb43cf73SLiu, Yi L     case VTD_SM_PASID_ENTRY_SLT:
675fb43cf73SLiu, Yi L     case VTD_SM_PASID_ENTRY_NESTED:
676fb43cf73SLiu, Yi L         break;
677fb43cf73SLiu, Yi L     case VTD_SM_PASID_ENTRY_PT:
678fb43cf73SLiu, Yi L         if (!x86_iommu->pt_supported) {
679fb43cf73SLiu, Yi L             return false;
680fb43cf73SLiu, Yi L         }
681fb43cf73SLiu, Yi L         break;
682fb43cf73SLiu, Yi L     default:
683fb43cf73SLiu, Yi L         /* Unknwon type */
684fb43cf73SLiu, Yi L         return false;
685fb43cf73SLiu, Yi L     }
686fb43cf73SLiu, Yi L     return true;
687fb43cf73SLiu, Yi L }
688fb43cf73SLiu, Yi L 
689*56fc1e6aSLiu Yi L static inline bool vtd_pdire_present(VTDPASIDDirEntry *pdire)
690*56fc1e6aSLiu Yi L {
691*56fc1e6aSLiu Yi L     return pdire->val & 1;
692*56fc1e6aSLiu Yi L }
693*56fc1e6aSLiu Yi L 
694*56fc1e6aSLiu Yi L /**
695*56fc1e6aSLiu Yi L  * Caller of this function should check present bit if wants
696*56fc1e6aSLiu Yi L  * to use pdir entry for futher usage except for fpd bit check.
697*56fc1e6aSLiu Yi L  */
698*56fc1e6aSLiu Yi L static int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base,
699fb43cf73SLiu, Yi L                                          uint32_t pasid,
700fb43cf73SLiu, Yi L                                          VTDPASIDDirEntry *pdire)
701fb43cf73SLiu, Yi L {
702fb43cf73SLiu, Yi L     uint32_t index;
703fb43cf73SLiu, Yi L     dma_addr_t addr, entry_size;
704fb43cf73SLiu, Yi L 
705fb43cf73SLiu, Yi L     index = VTD_PASID_DIR_INDEX(pasid);
706fb43cf73SLiu, Yi L     entry_size = VTD_PASID_DIR_ENTRY_SIZE;
707fb43cf73SLiu, Yi L     addr = pasid_dir_base + index * entry_size;
708fb43cf73SLiu, Yi L     if (dma_memory_read(&address_space_memory, addr, pdire, entry_size)) {
709fb43cf73SLiu, Yi L         return -VTD_FR_PASID_TABLE_INV;
710fb43cf73SLiu, Yi L     }
711fb43cf73SLiu, Yi L 
712fb43cf73SLiu, Yi L     return 0;
713fb43cf73SLiu, Yi L }
714fb43cf73SLiu, Yi L 
715*56fc1e6aSLiu Yi L static inline bool vtd_pe_present(VTDPASIDEntry *pe)
716*56fc1e6aSLiu Yi L {
717*56fc1e6aSLiu Yi L     return pe->val[0] & VTD_PASID_ENTRY_P;
718*56fc1e6aSLiu Yi L }
719*56fc1e6aSLiu Yi L 
720*56fc1e6aSLiu Yi L static int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s,
721fb43cf73SLiu, Yi L                                           uint32_t pasid,
722*56fc1e6aSLiu Yi L                                           dma_addr_t addr,
723fb43cf73SLiu, Yi L                                           VTDPASIDEntry *pe)
724fb43cf73SLiu, Yi L {
725fb43cf73SLiu, Yi L     uint32_t index;
726*56fc1e6aSLiu Yi L     dma_addr_t entry_size;
727fb43cf73SLiu, Yi L     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
728fb43cf73SLiu, Yi L 
729fb43cf73SLiu, Yi L     index = VTD_PASID_TABLE_INDEX(pasid);
730fb43cf73SLiu, Yi L     entry_size = VTD_PASID_ENTRY_SIZE;
731fb43cf73SLiu, Yi L     addr = addr + index * entry_size;
732fb43cf73SLiu, Yi L     if (dma_memory_read(&address_space_memory, addr, pe, entry_size)) {
733fb43cf73SLiu, Yi L         return -VTD_FR_PASID_TABLE_INV;
734fb43cf73SLiu, Yi L     }
735fb43cf73SLiu, Yi L 
736fb43cf73SLiu, Yi L     /* Do translation type check */
737fb43cf73SLiu, Yi L     if (!vtd_pe_type_check(x86_iommu, pe)) {
738fb43cf73SLiu, Yi L         return -VTD_FR_PASID_TABLE_INV;
739fb43cf73SLiu, Yi L     }
740fb43cf73SLiu, Yi L 
741fb43cf73SLiu, Yi L     if (!vtd_is_level_supported(s, VTD_PE_GET_LEVEL(pe))) {
742fb43cf73SLiu, Yi L         return -VTD_FR_PASID_TABLE_INV;
743fb43cf73SLiu, Yi L     }
744fb43cf73SLiu, Yi L 
745fb43cf73SLiu, Yi L     return 0;
746fb43cf73SLiu, Yi L }
747fb43cf73SLiu, Yi L 
748*56fc1e6aSLiu Yi L /**
749*56fc1e6aSLiu Yi L  * Caller of this function should check present bit if wants
750*56fc1e6aSLiu Yi L  * to use pasid entry for futher usage except for fpd bit check.
751*56fc1e6aSLiu Yi L  */
752*56fc1e6aSLiu Yi L static int vtd_get_pe_from_pdire(IntelIOMMUState *s,
753*56fc1e6aSLiu Yi L                                  uint32_t pasid,
754*56fc1e6aSLiu Yi L                                  VTDPASIDDirEntry *pdire,
755*56fc1e6aSLiu Yi L                                  VTDPASIDEntry *pe)
756*56fc1e6aSLiu Yi L {
757*56fc1e6aSLiu Yi L     dma_addr_t addr = pdire->val & VTD_PASID_TABLE_BASE_ADDR_MASK;
758*56fc1e6aSLiu Yi L 
759*56fc1e6aSLiu Yi L     return vtd_get_pe_in_pasid_leaf_table(s, pasid, addr, pe);
760*56fc1e6aSLiu Yi L }
761*56fc1e6aSLiu Yi L 
762*56fc1e6aSLiu Yi L /**
763*56fc1e6aSLiu Yi L  * This function gets a pasid entry from a specified pasid
764*56fc1e6aSLiu Yi L  * table (includes dir and leaf table) with a specified pasid.
765*56fc1e6aSLiu Yi L  * Sanity check should be done to ensure return a present
766*56fc1e6aSLiu Yi L  * pasid entry to caller.
767*56fc1e6aSLiu Yi L  */
768*56fc1e6aSLiu Yi L static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s,
769fb43cf73SLiu, Yi L                                        dma_addr_t pasid_dir_base,
770fb43cf73SLiu, Yi L                                        uint32_t pasid,
771fb43cf73SLiu, Yi L                                        VTDPASIDEntry *pe)
772fb43cf73SLiu, Yi L {
773fb43cf73SLiu, Yi L     int ret;
774fb43cf73SLiu, Yi L     VTDPASIDDirEntry pdire;
775fb43cf73SLiu, Yi L 
776*56fc1e6aSLiu Yi L     ret = vtd_get_pdire_from_pdir_table(pasid_dir_base,
777*56fc1e6aSLiu Yi L                                         pasid, &pdire);
778fb43cf73SLiu, Yi L     if (ret) {
779fb43cf73SLiu, Yi L         return ret;
780fb43cf73SLiu, Yi L     }
781fb43cf73SLiu, Yi L 
782*56fc1e6aSLiu Yi L     if (!vtd_pdire_present(&pdire)) {
783*56fc1e6aSLiu Yi L         return -VTD_FR_PASID_TABLE_INV;
784*56fc1e6aSLiu Yi L     }
785*56fc1e6aSLiu Yi L 
786*56fc1e6aSLiu Yi L     ret = vtd_get_pe_from_pdire(s, pasid, &pdire, pe);
787fb43cf73SLiu, Yi L     if (ret) {
788fb43cf73SLiu, Yi L         return ret;
789fb43cf73SLiu, Yi L     }
790fb43cf73SLiu, Yi L 
791*56fc1e6aSLiu Yi L     if (!vtd_pe_present(pe)) {
792*56fc1e6aSLiu Yi L         return -VTD_FR_PASID_TABLE_INV;
793*56fc1e6aSLiu Yi L     }
794*56fc1e6aSLiu Yi L 
795*56fc1e6aSLiu Yi L     return 0;
796fb43cf73SLiu, Yi L }
797fb43cf73SLiu, Yi L 
798fb43cf73SLiu, Yi L static int vtd_ce_get_rid2pasid_entry(IntelIOMMUState *s,
799fb43cf73SLiu, Yi L                                       VTDContextEntry *ce,
800fb43cf73SLiu, Yi L                                       VTDPASIDEntry *pe)
801fb43cf73SLiu, Yi L {
802fb43cf73SLiu, Yi L     uint32_t pasid;
803fb43cf73SLiu, Yi L     dma_addr_t pasid_dir_base;
804fb43cf73SLiu, Yi L     int ret = 0;
805fb43cf73SLiu, Yi L 
806fb43cf73SLiu, Yi L     pasid = VTD_CE_GET_RID2PASID(ce);
807fb43cf73SLiu, Yi L     pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce);
808*56fc1e6aSLiu Yi L     ret = vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe);
809fb43cf73SLiu, Yi L 
810fb43cf73SLiu, Yi L     return ret;
811fb43cf73SLiu, Yi L }
812fb43cf73SLiu, Yi L 
813fb43cf73SLiu, Yi L static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s,
814fb43cf73SLiu, Yi L                                 VTDContextEntry *ce,
815fb43cf73SLiu, Yi L                                 bool *pe_fpd_set)
816fb43cf73SLiu, Yi L {
817fb43cf73SLiu, Yi L     int ret;
818fb43cf73SLiu, Yi L     uint32_t pasid;
819fb43cf73SLiu, Yi L     dma_addr_t pasid_dir_base;
820fb43cf73SLiu, Yi L     VTDPASIDDirEntry pdire;
821fb43cf73SLiu, Yi L     VTDPASIDEntry pe;
822fb43cf73SLiu, Yi L 
823fb43cf73SLiu, Yi L     pasid = VTD_CE_GET_RID2PASID(ce);
824fb43cf73SLiu, Yi L     pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce);
825fb43cf73SLiu, Yi L 
826*56fc1e6aSLiu Yi L     /*
827*56fc1e6aSLiu Yi L      * No present bit check since fpd is meaningful even
828*56fc1e6aSLiu Yi L      * if the present bit is clear.
829*56fc1e6aSLiu Yi L      */
830*56fc1e6aSLiu Yi L     ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, pasid, &pdire);
831fb43cf73SLiu, Yi L     if (ret) {
832fb43cf73SLiu, Yi L         return ret;
833fb43cf73SLiu, Yi L     }
834fb43cf73SLiu, Yi L 
835fb43cf73SLiu, Yi L     if (pdire.val & VTD_PASID_DIR_FPD) {
836fb43cf73SLiu, Yi L         *pe_fpd_set = true;
837fb43cf73SLiu, Yi L         return 0;
838fb43cf73SLiu, Yi L     }
839fb43cf73SLiu, Yi L 
840*56fc1e6aSLiu Yi L     if (!vtd_pdire_present(&pdire)) {
841*56fc1e6aSLiu Yi L         return -VTD_FR_PASID_TABLE_INV;
842*56fc1e6aSLiu Yi L     }
843*56fc1e6aSLiu Yi L 
844*56fc1e6aSLiu Yi L     /*
845*56fc1e6aSLiu Yi L      * No present bit check since fpd is meaningful even
846*56fc1e6aSLiu Yi L      * if the present bit is clear.
847*56fc1e6aSLiu Yi L      */
848*56fc1e6aSLiu Yi L     ret = vtd_get_pe_from_pdire(s, pasid, &pdire, &pe);
849fb43cf73SLiu, Yi L     if (ret) {
850fb43cf73SLiu, Yi L         return ret;
851fb43cf73SLiu, Yi L     }
852fb43cf73SLiu, Yi L 
853fb43cf73SLiu, Yi L     if (pe.val[0] & VTD_PASID_ENTRY_FPD) {
854fb43cf73SLiu, Yi L         *pe_fpd_set = true;
855fb43cf73SLiu, Yi L     }
856fb43cf73SLiu, Yi L 
857fb43cf73SLiu, Yi L     return 0;
858fb43cf73SLiu, Yi L }
859fb43cf73SLiu, Yi L 
8601da12ec4SLe Tan /* Get the page-table level that hardware should use for the second-level
8611da12ec4SLe Tan  * page-table walk from the Address Width field of context-entry.
8621da12ec4SLe Tan  */
8638f7d7161SPeter Xu static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce)
8641da12ec4SLe Tan {
8651da12ec4SLe Tan     return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW);
8661da12ec4SLe Tan }
8671da12ec4SLe Tan 
868fb43cf73SLiu, Yi L static uint32_t vtd_get_iova_level(IntelIOMMUState *s,
869fb43cf73SLiu, Yi L                                    VTDContextEntry *ce)
870fb43cf73SLiu, Yi L {
871fb43cf73SLiu, Yi L     VTDPASIDEntry pe;
872fb43cf73SLiu, Yi L 
873fb43cf73SLiu, Yi L     if (s->root_scalable) {
874fb43cf73SLiu, Yi L         vtd_ce_get_rid2pasid_entry(s, ce, &pe);
875fb43cf73SLiu, Yi L         return VTD_PE_GET_LEVEL(&pe);
876fb43cf73SLiu, Yi L     }
877fb43cf73SLiu, Yi L 
878fb43cf73SLiu, Yi L     return vtd_ce_get_level(ce);
879fb43cf73SLiu, Yi L }
880fb43cf73SLiu, Yi L 
8818f7d7161SPeter Xu static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce)
8821da12ec4SLe Tan {
8831da12ec4SLe Tan     return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9;
8841da12ec4SLe Tan }
8851da12ec4SLe Tan 
886fb43cf73SLiu, Yi L static uint32_t vtd_get_iova_agaw(IntelIOMMUState *s,
887fb43cf73SLiu, Yi L                                   VTDContextEntry *ce)
888fb43cf73SLiu, Yi L {
889fb43cf73SLiu, Yi L     VTDPASIDEntry pe;
890fb43cf73SLiu, Yi L 
891fb43cf73SLiu, Yi L     if (s->root_scalable) {
892fb43cf73SLiu, Yi L         vtd_ce_get_rid2pasid_entry(s, ce, &pe);
893fb43cf73SLiu, Yi L         return 30 + ((pe.val[0] >> 2) & VTD_SM_PASID_ENTRY_AW) * 9;
894fb43cf73SLiu, Yi L     }
895fb43cf73SLiu, Yi L 
896fb43cf73SLiu, Yi L     return vtd_ce_get_agaw(ce);
897fb43cf73SLiu, Yi L }
898fb43cf73SLiu, Yi L 
899127ff5c3SPeter Xu static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce)
900127ff5c3SPeter Xu {
901127ff5c3SPeter Xu     return ce->lo & VTD_CONTEXT_ENTRY_TT;
902127ff5c3SPeter Xu }
903127ff5c3SPeter Xu 
904fb43cf73SLiu, Yi L /* Only for Legacy Mode. Return true if check passed, otherwise false */
905f80c9874SPeter Xu static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu,
906f80c9874SPeter Xu                                      VTDContextEntry *ce)
907f80c9874SPeter Xu {
908f80c9874SPeter Xu     switch (vtd_ce_get_type(ce)) {
909f80c9874SPeter Xu     case VTD_CONTEXT_TT_MULTI_LEVEL:
910f80c9874SPeter Xu         /* Always supported */
911f80c9874SPeter Xu         break;
912f80c9874SPeter Xu     case VTD_CONTEXT_TT_DEV_IOTLB:
913f80c9874SPeter Xu         if (!x86_iommu->dt_supported) {
914095955b2SPeter Xu             error_report_once("%s: DT specified but not supported", __func__);
915f80c9874SPeter Xu             return false;
916f80c9874SPeter Xu         }
917f80c9874SPeter Xu         break;
918dbaabb25SPeter Xu     case VTD_CONTEXT_TT_PASS_THROUGH:
919dbaabb25SPeter Xu         if (!x86_iommu->pt_supported) {
920095955b2SPeter Xu             error_report_once("%s: PT specified but not supported", __func__);
921dbaabb25SPeter Xu             return false;
922dbaabb25SPeter Xu         }
923dbaabb25SPeter Xu         break;
924f80c9874SPeter Xu     default:
925fb43cf73SLiu, Yi L         /* Unknown type */
926095955b2SPeter Xu         error_report_once("%s: unknown ce type: %"PRIu32, __func__,
927095955b2SPeter Xu                           vtd_ce_get_type(ce));
928f80c9874SPeter Xu         return false;
929f80c9874SPeter Xu     }
930f80c9874SPeter Xu     return true;
931f80c9874SPeter Xu }
932f80c9874SPeter Xu 
933fb43cf73SLiu, Yi L static inline uint64_t vtd_iova_limit(IntelIOMMUState *s,
934fb43cf73SLiu, Yi L                                       VTDContextEntry *ce, uint8_t aw)
935f06a696dSPeter Xu {
936fb43cf73SLiu, Yi L     uint32_t ce_agaw = vtd_get_iova_agaw(s, ce);
93737f51384SPrasad Singamsetty     return 1ULL << MIN(ce_agaw, aw);
938f06a696dSPeter Xu }
939f06a696dSPeter Xu 
940f06a696dSPeter Xu /* Return true if IOVA passes range check, otherwise false. */
941fb43cf73SLiu, Yi L static inline bool vtd_iova_range_check(IntelIOMMUState *s,
942fb43cf73SLiu, Yi L                                         uint64_t iova, VTDContextEntry *ce,
94337f51384SPrasad Singamsetty                                         uint8_t aw)
944f06a696dSPeter Xu {
945f06a696dSPeter Xu     /*
946f06a696dSPeter Xu      * Check if @iova is above 2^X-1, where X is the minimum of MGAW
947f06a696dSPeter Xu      * in CAP_REG and AW in context-entry.
948f06a696dSPeter Xu      */
949fb43cf73SLiu, Yi L     return !(iova & ~(vtd_iova_limit(s, ce, aw) - 1));
950fb43cf73SLiu, Yi L }
951fb43cf73SLiu, Yi L 
952fb43cf73SLiu, Yi L static dma_addr_t vtd_get_iova_pgtbl_base(IntelIOMMUState *s,
953fb43cf73SLiu, Yi L                                           VTDContextEntry *ce)
954fb43cf73SLiu, Yi L {
955fb43cf73SLiu, Yi L     VTDPASIDEntry pe;
956fb43cf73SLiu, Yi L 
957fb43cf73SLiu, Yi L     if (s->root_scalable) {
958fb43cf73SLiu, Yi L         vtd_ce_get_rid2pasid_entry(s, ce, &pe);
959fb43cf73SLiu, Yi L         return pe.val[0] & VTD_SM_PASID_ENTRY_SLPTPTR;
960fb43cf73SLiu, Yi L     }
961fb43cf73SLiu, Yi L 
962fb43cf73SLiu, Yi L     return vtd_ce_get_slpt_base(ce);
963f06a696dSPeter Xu }
964f06a696dSPeter Xu 
96592e5d85eSPrasad Singamsetty /*
96692e5d85eSPrasad Singamsetty  * Rsvd field masks for spte:
967ce586f3bSQi, Yadong  *     vtd_spte_rsvd 4k pages
968ce586f3bSQi, Yadong  *     vtd_spte_rsvd_large large pages
96992e5d85eSPrasad Singamsetty  */
970ce586f3bSQi, Yadong static uint64_t vtd_spte_rsvd[5];
971ce586f3bSQi, Yadong static uint64_t vtd_spte_rsvd_large[5];
9721da12ec4SLe Tan 
9731da12ec4SLe Tan static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level)
9741da12ec4SLe Tan {
975ce586f3bSQi, Yadong     uint64_t rsvd_mask = vtd_spte_rsvd[level];
976ce586f3bSQi, Yadong 
977ce586f3bSQi, Yadong     if ((level == VTD_SL_PD_LEVEL || level == VTD_SL_PDP_LEVEL) &&
978ce586f3bSQi, Yadong         (slpte & VTD_SL_PT_PAGE_SIZE_MASK)) {
979ce586f3bSQi, Yadong         /* large page */
980ce586f3bSQi, Yadong         rsvd_mask = vtd_spte_rsvd_large[level];
9811da12ec4SLe Tan     }
982ce586f3bSQi, Yadong 
983ce586f3bSQi, Yadong     return slpte & rsvd_mask;
9841da12ec4SLe Tan }
9851da12ec4SLe Tan 
986dbaabb25SPeter Xu /* Find the VTD address space associated with a given bus number */
987dbaabb25SPeter Xu static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
988dbaabb25SPeter Xu {
989dbaabb25SPeter Xu     VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
990dbaabb25SPeter Xu     if (!vtd_bus) {
991dbaabb25SPeter Xu         /*
992dbaabb25SPeter Xu          * Iterate over the registered buses to find the one which
993dbaabb25SPeter Xu          * currently hold this bus number, and update the bus_num
994dbaabb25SPeter Xu          * lookup table:
995dbaabb25SPeter Xu          */
996dbaabb25SPeter Xu         GHashTableIter iter;
997dbaabb25SPeter Xu 
998dbaabb25SPeter Xu         g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
999dbaabb25SPeter Xu         while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) {
1000dbaabb25SPeter Xu             if (pci_bus_num(vtd_bus->bus) == bus_num) {
1001dbaabb25SPeter Xu                 s->vtd_as_by_bus_num[bus_num] = vtd_bus;
1002dbaabb25SPeter Xu                 return vtd_bus;
1003dbaabb25SPeter Xu             }
1004dbaabb25SPeter Xu         }
1005a2e1cd41SLiu Yi L         vtd_bus = NULL;
1006dbaabb25SPeter Xu     }
1007dbaabb25SPeter Xu     return vtd_bus;
1008dbaabb25SPeter Xu }
1009dbaabb25SPeter Xu 
10106e905564SPeter Xu /* Given the @iova, get relevant @slptep. @slpte_level will be the last level
10111da12ec4SLe Tan  * of the translation, can be used for deciding the size of large page.
10121da12ec4SLe Tan  */
1013fb43cf73SLiu, Yi L static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce,
1014fb43cf73SLiu, Yi L                              uint64_t iova, bool is_write,
10151da12ec4SLe Tan                              uint64_t *slptep, uint32_t *slpte_level,
101637f51384SPrasad Singamsetty                              bool *reads, bool *writes, uint8_t aw_bits)
10171da12ec4SLe Tan {
1018fb43cf73SLiu, Yi L     dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce);
1019fb43cf73SLiu, Yi L     uint32_t level = vtd_get_iova_level(s, ce);
10201da12ec4SLe Tan     uint32_t offset;
10211da12ec4SLe Tan     uint64_t slpte;
10221da12ec4SLe Tan     uint64_t access_right_check;
10231da12ec4SLe Tan 
1024fb43cf73SLiu, Yi L     if (!vtd_iova_range_check(s, iova, ce, aw_bits)) {
10254e4abd11SPeter Xu         error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 ")",
10264e4abd11SPeter Xu                           __func__, iova);
10271da12ec4SLe Tan         return -VTD_FR_ADDR_BEYOND_MGAW;
10281da12ec4SLe Tan     }
10291da12ec4SLe Tan 
10301da12ec4SLe Tan     /* FIXME: what is the Atomics request here? */
10311da12ec4SLe Tan     access_right_check = is_write ? VTD_SL_W : VTD_SL_R;
10321da12ec4SLe Tan 
10331da12ec4SLe Tan     while (true) {
10346e905564SPeter Xu         offset = vtd_iova_level_offset(iova, level);
10351da12ec4SLe Tan         slpte = vtd_get_slpte(addr, offset);
10361da12ec4SLe Tan 
10371da12ec4SLe Tan         if (slpte == (uint64_t)-1) {
10384e4abd11SPeter Xu             error_report_once("%s: detected read error on DMAR slpte "
10394e4abd11SPeter Xu                               "(iova=0x%" PRIx64 ")", __func__, iova);
1040fb43cf73SLiu, Yi L             if (level == vtd_get_iova_level(s, ce)) {
10411da12ec4SLe Tan                 /* Invalid programming of context-entry */
10421da12ec4SLe Tan                 return -VTD_FR_CONTEXT_ENTRY_INV;
10431da12ec4SLe Tan             } else {
10441da12ec4SLe Tan                 return -VTD_FR_PAGING_ENTRY_INV;
10451da12ec4SLe Tan             }
10461da12ec4SLe Tan         }
10471da12ec4SLe Tan         *reads = (*reads) && (slpte & VTD_SL_R);
10481da12ec4SLe Tan         *writes = (*writes) && (slpte & VTD_SL_W);
10491da12ec4SLe Tan         if (!(slpte & access_right_check)) {
10504e4abd11SPeter Xu             error_report_once("%s: detected slpte permission error "
10514e4abd11SPeter Xu                               "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", "
10524e4abd11SPeter Xu                               "slpte=0x%" PRIx64 ", write=%d)", __func__,
10534e4abd11SPeter Xu                               iova, level, slpte, is_write);
10541da12ec4SLe Tan             return is_write ? -VTD_FR_WRITE : -VTD_FR_READ;
10551da12ec4SLe Tan         }
10561da12ec4SLe Tan         if (vtd_slpte_nonzero_rsvd(slpte, level)) {
10574e4abd11SPeter Xu             error_report_once("%s: detected splte reserve non-zero "
10584e4abd11SPeter Xu                               "iova=0x%" PRIx64 ", level=0x%" PRIx32
10594e4abd11SPeter Xu                               "slpte=0x%" PRIx64 ")", __func__, iova,
10604e4abd11SPeter Xu                               level, slpte);
10611da12ec4SLe Tan             return -VTD_FR_PAGING_ENTRY_RSVD;
10621da12ec4SLe Tan         }
10631da12ec4SLe Tan 
10641da12ec4SLe Tan         if (vtd_is_last_slpte(slpte, level)) {
10651da12ec4SLe Tan             *slptep = slpte;
10661da12ec4SLe Tan             *slpte_level = level;
10671da12ec4SLe Tan             return 0;
10681da12ec4SLe Tan         }
106937f51384SPrasad Singamsetty         addr = vtd_get_slpte_addr(slpte, aw_bits);
10701da12ec4SLe Tan         level--;
10711da12ec4SLe Tan     }
10721da12ec4SLe Tan }
10731da12ec4SLe Tan 
1074f06a696dSPeter Xu typedef int (*vtd_page_walk_hook)(IOMMUTLBEntry *entry, void *private);
1075f06a696dSPeter Xu 
1076fe215b0cSPeter Xu /**
1077fe215b0cSPeter Xu  * Constant information used during page walking
1078fe215b0cSPeter Xu  *
1079fe215b0cSPeter Xu  * @hook_fn: hook func to be called when detected page
1080fe215b0cSPeter Xu  * @private: private data to be passed into hook func
1081fe215b0cSPeter Xu  * @notify_unmap: whether we should notify invalid entries
10822f764fa8SPeter Xu  * @as: VT-d address space of the device
1083fe215b0cSPeter Xu  * @aw: maximum address width
1084d118c06eSPeter Xu  * @domain: domain ID of the page walk
1085fe215b0cSPeter Xu  */
1086fe215b0cSPeter Xu typedef struct {
10872f764fa8SPeter Xu     VTDAddressSpace *as;
1088fe215b0cSPeter Xu     vtd_page_walk_hook hook_fn;
1089fe215b0cSPeter Xu     void *private;
1090fe215b0cSPeter Xu     bool notify_unmap;
1091fe215b0cSPeter Xu     uint8_t aw;
1092d118c06eSPeter Xu     uint16_t domain_id;
1093fe215b0cSPeter Xu } vtd_page_walk_info;
1094fe215b0cSPeter Xu 
1095d118c06eSPeter Xu static int vtd_page_walk_one(IOMMUTLBEntry *entry, vtd_page_walk_info *info)
109636d2d52bSPeter Xu {
109763b88968SPeter Xu     VTDAddressSpace *as = info->as;
1098fe215b0cSPeter Xu     vtd_page_walk_hook hook_fn = info->hook_fn;
1099fe215b0cSPeter Xu     void *private = info->private;
110063b88968SPeter Xu     DMAMap target = {
110163b88968SPeter Xu         .iova = entry->iova,
110263b88968SPeter Xu         .size = entry->addr_mask,
110363b88968SPeter Xu         .translated_addr = entry->translated_addr,
110463b88968SPeter Xu         .perm = entry->perm,
110563b88968SPeter Xu     };
110663b88968SPeter Xu     DMAMap *mapped = iova_tree_find(as->iova_tree, &target);
110763b88968SPeter Xu 
110863b88968SPeter Xu     if (entry->perm == IOMMU_NONE && !info->notify_unmap) {
110963b88968SPeter Xu         trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask);
111063b88968SPeter Xu         return 0;
111163b88968SPeter Xu     }
1112fe215b0cSPeter Xu 
111336d2d52bSPeter Xu     assert(hook_fn);
111463b88968SPeter Xu 
111563b88968SPeter Xu     /* Update local IOVA mapped ranges */
111663b88968SPeter Xu     if (entry->perm) {
111763b88968SPeter Xu         if (mapped) {
111863b88968SPeter Xu             /* If it's exactly the same translation, skip */
111963b88968SPeter Xu             if (!memcmp(mapped, &target, sizeof(target))) {
112063b88968SPeter Xu                 trace_vtd_page_walk_one_skip_map(entry->iova, entry->addr_mask,
112163b88968SPeter Xu                                                  entry->translated_addr);
112263b88968SPeter Xu                 return 0;
112363b88968SPeter Xu             } else {
112463b88968SPeter Xu                 /*
112563b88968SPeter Xu                  * Translation changed.  Normally this should not
112663b88968SPeter Xu                  * happen, but it can happen when with buggy guest
112763b88968SPeter Xu                  * OSes.  Note that there will be a small window that
112863b88968SPeter Xu                  * we don't have map at all.  But that's the best
112963b88968SPeter Xu                  * effort we can do.  The ideal way to emulate this is
113063b88968SPeter Xu                  * atomically modify the PTE to follow what has
113163b88968SPeter Xu                  * changed, but we can't.  One example is that vfio
113263b88968SPeter Xu                  * driver only has VFIO_IOMMU_[UN]MAP_DMA but no
113363b88968SPeter Xu                  * interface to modify a mapping (meanwhile it seems
113463b88968SPeter Xu                  * meaningless to even provide one).  Anyway, let's
113563b88968SPeter Xu                  * mark this as a TODO in case one day we'll have
113663b88968SPeter Xu                  * a better solution.
113763b88968SPeter Xu                  */
113863b88968SPeter Xu                 IOMMUAccessFlags cache_perm = entry->perm;
113963b88968SPeter Xu                 int ret;
114063b88968SPeter Xu 
114163b88968SPeter Xu                 /* Emulate an UNMAP */
114263b88968SPeter Xu                 entry->perm = IOMMU_NONE;
114363b88968SPeter Xu                 trace_vtd_page_walk_one(info->domain_id,
114463b88968SPeter Xu                                         entry->iova,
114563b88968SPeter Xu                                         entry->translated_addr,
114663b88968SPeter Xu                                         entry->addr_mask,
114763b88968SPeter Xu                                         entry->perm);
114863b88968SPeter Xu                 ret = hook_fn(entry, private);
114963b88968SPeter Xu                 if (ret) {
115063b88968SPeter Xu                     return ret;
115163b88968SPeter Xu                 }
115263b88968SPeter Xu                 /* Drop any existing mapping */
115363b88968SPeter Xu                 iova_tree_remove(as->iova_tree, &target);
115463b88968SPeter Xu                 /* Recover the correct permission */
115563b88968SPeter Xu                 entry->perm = cache_perm;
115663b88968SPeter Xu             }
115763b88968SPeter Xu         }
115863b88968SPeter Xu         iova_tree_insert(as->iova_tree, &target);
115963b88968SPeter Xu     } else {
116063b88968SPeter Xu         if (!mapped) {
116163b88968SPeter Xu             /* Skip since we didn't map this range at all */
116263b88968SPeter Xu             trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask);
116363b88968SPeter Xu             return 0;
116463b88968SPeter Xu         }
116563b88968SPeter Xu         iova_tree_remove(as->iova_tree, &target);
116663b88968SPeter Xu     }
116763b88968SPeter Xu 
1168d118c06eSPeter Xu     trace_vtd_page_walk_one(info->domain_id, entry->iova,
1169d118c06eSPeter Xu                             entry->translated_addr, entry->addr_mask,
1170d118c06eSPeter Xu                             entry->perm);
117136d2d52bSPeter Xu     return hook_fn(entry, private);
117236d2d52bSPeter Xu }
117336d2d52bSPeter Xu 
1174f06a696dSPeter Xu /**
1175f06a696dSPeter Xu  * vtd_page_walk_level - walk over specific level for IOVA range
1176f06a696dSPeter Xu  *
1177f06a696dSPeter Xu  * @addr: base GPA addr to start the walk
1178f06a696dSPeter Xu  * @start: IOVA range start address
1179f06a696dSPeter Xu  * @end: IOVA range end address (start <= addr < end)
1180f06a696dSPeter Xu  * @read: whether parent level has read permission
1181f06a696dSPeter Xu  * @write: whether parent level has write permission
1182fe215b0cSPeter Xu  * @info: constant information for the page walk
1183f06a696dSPeter Xu  */
1184f06a696dSPeter Xu static int vtd_page_walk_level(dma_addr_t addr, uint64_t start,
1185fe215b0cSPeter Xu                                uint64_t end, uint32_t level, bool read,
1186fe215b0cSPeter Xu                                bool write, vtd_page_walk_info *info)
1187f06a696dSPeter Xu {
1188f06a696dSPeter Xu     bool read_cur, write_cur, entry_valid;
1189f06a696dSPeter Xu     uint32_t offset;
1190f06a696dSPeter Xu     uint64_t slpte;
1191f06a696dSPeter Xu     uint64_t subpage_size, subpage_mask;
1192f06a696dSPeter Xu     IOMMUTLBEntry entry;
1193f06a696dSPeter Xu     uint64_t iova = start;
1194f06a696dSPeter Xu     uint64_t iova_next;
1195f06a696dSPeter Xu     int ret = 0;
1196f06a696dSPeter Xu 
1197f06a696dSPeter Xu     trace_vtd_page_walk_level(addr, level, start, end);
1198f06a696dSPeter Xu 
1199f06a696dSPeter Xu     subpage_size = 1ULL << vtd_slpt_level_shift(level);
1200f06a696dSPeter Xu     subpage_mask = vtd_slpt_level_page_mask(level);
1201f06a696dSPeter Xu 
1202f06a696dSPeter Xu     while (iova < end) {
1203f06a696dSPeter Xu         iova_next = (iova & subpage_mask) + subpage_size;
1204f06a696dSPeter Xu 
1205f06a696dSPeter Xu         offset = vtd_iova_level_offset(iova, level);
1206f06a696dSPeter Xu         slpte = vtd_get_slpte(addr, offset);
1207f06a696dSPeter Xu 
1208f06a696dSPeter Xu         if (slpte == (uint64_t)-1) {
1209f06a696dSPeter Xu             trace_vtd_page_walk_skip_read(iova, iova_next);
1210f06a696dSPeter Xu             goto next;
1211f06a696dSPeter Xu         }
1212f06a696dSPeter Xu 
1213f06a696dSPeter Xu         if (vtd_slpte_nonzero_rsvd(slpte, level)) {
1214f06a696dSPeter Xu             trace_vtd_page_walk_skip_reserve(iova, iova_next);
1215f06a696dSPeter Xu             goto next;
1216f06a696dSPeter Xu         }
1217f06a696dSPeter Xu 
1218f06a696dSPeter Xu         /* Permissions are stacked with parents' */
1219f06a696dSPeter Xu         read_cur = read && (slpte & VTD_SL_R);
1220f06a696dSPeter Xu         write_cur = write && (slpte & VTD_SL_W);
1221f06a696dSPeter Xu 
1222f06a696dSPeter Xu         /*
1223f06a696dSPeter Xu          * As long as we have either read/write permission, this is a
1224f06a696dSPeter Xu          * valid entry. The rule works for both page entries and page
1225f06a696dSPeter Xu          * table entries.
1226f06a696dSPeter Xu          */
1227f06a696dSPeter Xu         entry_valid = read_cur | write_cur;
1228f06a696dSPeter Xu 
122963b88968SPeter Xu         if (!vtd_is_last_slpte(slpte, level) && entry_valid) {
123063b88968SPeter Xu             /*
123163b88968SPeter Xu              * This is a valid PDE (or even bigger than PDE).  We need
123263b88968SPeter Xu              * to walk one further level.
123363b88968SPeter Xu              */
123463b88968SPeter Xu             ret = vtd_page_walk_level(vtd_get_slpte_addr(slpte, info->aw),
123563b88968SPeter Xu                                       iova, MIN(iova_next, end), level - 1,
123663b88968SPeter Xu                                       read_cur, write_cur, info);
123763b88968SPeter Xu         } else {
123863b88968SPeter Xu             /*
123963b88968SPeter Xu              * This means we are either:
124063b88968SPeter Xu              *
124163b88968SPeter Xu              * (1) the real page entry (either 4K page, or huge page)
124263b88968SPeter Xu              * (2) the whole range is invalid
124363b88968SPeter Xu              *
124463b88968SPeter Xu              * In either case, we send an IOTLB notification down.
124563b88968SPeter Xu              */
1246f06a696dSPeter Xu             entry.target_as = &address_space_memory;
1247f06a696dSPeter Xu             entry.iova = iova & subpage_mask;
124836d2d52bSPeter Xu             entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur);
124936d2d52bSPeter Xu             entry.addr_mask = ~subpage_mask;
1250f06a696dSPeter Xu             /* NOTE: this is only meaningful if entry_valid == true */
1251fe215b0cSPeter Xu             entry.translated_addr = vtd_get_slpte_addr(slpte, info->aw);
1252d118c06eSPeter Xu             ret = vtd_page_walk_one(&entry, info);
125363b88968SPeter Xu         }
125463b88968SPeter Xu 
1255f06a696dSPeter Xu         if (ret < 0) {
1256f06a696dSPeter Xu             return ret;
1257f06a696dSPeter Xu         }
1258f06a696dSPeter Xu 
1259f06a696dSPeter Xu next:
1260f06a696dSPeter Xu         iova = iova_next;
1261f06a696dSPeter Xu     }
1262f06a696dSPeter Xu 
1263f06a696dSPeter Xu     return 0;
1264f06a696dSPeter Xu }
1265f06a696dSPeter Xu 
1266f06a696dSPeter Xu /**
1267f06a696dSPeter Xu  * vtd_page_walk - walk specific IOVA range, and call the hook
1268f06a696dSPeter Xu  *
1269fb43cf73SLiu, Yi L  * @s: intel iommu state
1270f06a696dSPeter Xu  * @ce: context entry to walk upon
1271f06a696dSPeter Xu  * @start: IOVA address to start the walk
1272f06a696dSPeter Xu  * @end: IOVA range end address (start <= addr < end)
1273fe215b0cSPeter Xu  * @info: page walking information struct
1274f06a696dSPeter Xu  */
1275fb43cf73SLiu, Yi L static int vtd_page_walk(IntelIOMMUState *s, VTDContextEntry *ce,
1276fb43cf73SLiu, Yi L                          uint64_t start, uint64_t end,
1277fe215b0cSPeter Xu                          vtd_page_walk_info *info)
1278f06a696dSPeter Xu {
1279fb43cf73SLiu, Yi L     dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce);
1280fb43cf73SLiu, Yi L     uint32_t level = vtd_get_iova_level(s, ce);
1281f06a696dSPeter Xu 
1282fb43cf73SLiu, Yi L     if (!vtd_iova_range_check(s, start, ce, info->aw)) {
1283f06a696dSPeter Xu         return -VTD_FR_ADDR_BEYOND_MGAW;
1284f06a696dSPeter Xu     }
1285f06a696dSPeter Xu 
1286fb43cf73SLiu, Yi L     if (!vtd_iova_range_check(s, end, ce, info->aw)) {
1287f06a696dSPeter Xu         /* Fix end so that it reaches the maximum */
1288fb43cf73SLiu, Yi L         end = vtd_iova_limit(s, ce, info->aw);
1289f06a696dSPeter Xu     }
1290f06a696dSPeter Xu 
1291fe215b0cSPeter Xu     return vtd_page_walk_level(addr, start, end, level, true, true, info);
1292f06a696dSPeter Xu }
1293f06a696dSPeter Xu 
1294fb43cf73SLiu, Yi L static int vtd_root_entry_rsvd_bits_check(IntelIOMMUState *s,
1295fb43cf73SLiu, Yi L                                           VTDRootEntry *re)
1296fb43cf73SLiu, Yi L {
1297fb43cf73SLiu, Yi L     /* Legacy Mode reserved bits check */
1298fb43cf73SLiu, Yi L     if (!s->root_scalable &&
1299fb43cf73SLiu, Yi L         (re->hi || (re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits))))
1300fb43cf73SLiu, Yi L         goto rsvd_err;
1301fb43cf73SLiu, Yi L 
1302fb43cf73SLiu, Yi L     /* Scalable Mode reserved bits check */
1303fb43cf73SLiu, Yi L     if (s->root_scalable &&
1304fb43cf73SLiu, Yi L         ((re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)) ||
1305fb43cf73SLiu, Yi L          (re->hi & VTD_ROOT_ENTRY_RSVD(s->aw_bits))))
1306fb43cf73SLiu, Yi L         goto rsvd_err;
1307fb43cf73SLiu, Yi L 
1308fb43cf73SLiu, Yi L     return 0;
1309fb43cf73SLiu, Yi L 
1310fb43cf73SLiu, Yi L rsvd_err:
1311fb43cf73SLiu, Yi L     error_report_once("%s: invalid root entry: hi=0x%"PRIx64
1312fb43cf73SLiu, Yi L                       ", lo=0x%"PRIx64,
1313fb43cf73SLiu, Yi L                       __func__, re->hi, re->lo);
1314fb43cf73SLiu, Yi L     return -VTD_FR_ROOT_ENTRY_RSVD;
1315fb43cf73SLiu, Yi L }
1316fb43cf73SLiu, Yi L 
1317fb43cf73SLiu, Yi L static inline int vtd_context_entry_rsvd_bits_check(IntelIOMMUState *s,
1318fb43cf73SLiu, Yi L                                                     VTDContextEntry *ce)
1319fb43cf73SLiu, Yi L {
1320fb43cf73SLiu, Yi L     if (!s->root_scalable &&
1321fb43cf73SLiu, Yi L         (ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI ||
1322fb43cf73SLiu, Yi L          ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) {
1323fb43cf73SLiu, Yi L         error_report_once("%s: invalid context entry: hi=%"PRIx64
1324fb43cf73SLiu, Yi L                           ", lo=%"PRIx64" (reserved nonzero)",
1325fb43cf73SLiu, Yi L                           __func__, ce->hi, ce->lo);
1326fb43cf73SLiu, Yi L         return -VTD_FR_CONTEXT_ENTRY_RSVD;
1327fb43cf73SLiu, Yi L     }
1328fb43cf73SLiu, Yi L 
1329fb43cf73SLiu, Yi L     if (s->root_scalable &&
1330fb43cf73SLiu, Yi L         (ce->val[0] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL0(s->aw_bits) ||
1331fb43cf73SLiu, Yi L          ce->val[1] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL1 ||
1332fb43cf73SLiu, Yi L          ce->val[2] ||
1333fb43cf73SLiu, Yi L          ce->val[3])) {
1334fb43cf73SLiu, Yi L         error_report_once("%s: invalid context entry: val[3]=%"PRIx64
1335fb43cf73SLiu, Yi L                           ", val[2]=%"PRIx64
1336fb43cf73SLiu, Yi L                           ", val[1]=%"PRIx64
1337fb43cf73SLiu, Yi L                           ", val[0]=%"PRIx64" (reserved nonzero)",
1338fb43cf73SLiu, Yi L                           __func__, ce->val[3], ce->val[2],
1339fb43cf73SLiu, Yi L                           ce->val[1], ce->val[0]);
1340fb43cf73SLiu, Yi L         return -VTD_FR_CONTEXT_ENTRY_RSVD;
1341fb43cf73SLiu, Yi L     }
1342fb43cf73SLiu, Yi L 
1343fb43cf73SLiu, Yi L     return 0;
1344fb43cf73SLiu, Yi L }
1345fb43cf73SLiu, Yi L 
1346fb43cf73SLiu, Yi L static int vtd_ce_rid2pasid_check(IntelIOMMUState *s,
1347fb43cf73SLiu, Yi L                                   VTDContextEntry *ce)
1348fb43cf73SLiu, Yi L {
1349fb43cf73SLiu, Yi L     VTDPASIDEntry pe;
1350fb43cf73SLiu, Yi L 
1351fb43cf73SLiu, Yi L     /*
1352fb43cf73SLiu, Yi L      * Make sure in Scalable Mode, a present context entry
1353fb43cf73SLiu, Yi L      * has valid rid2pasid setting, which includes valid
1354fb43cf73SLiu, Yi L      * rid2pasid field and corresponding pasid entry setting
1355fb43cf73SLiu, Yi L      */
1356fb43cf73SLiu, Yi L     return vtd_ce_get_rid2pasid_entry(s, ce, &pe);
1357fb43cf73SLiu, Yi L }
1358fb43cf73SLiu, Yi L 
13591da12ec4SLe Tan /* Map a device to its corresponding domain (context-entry) */
13601da12ec4SLe Tan static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
13611da12ec4SLe Tan                                     uint8_t devfn, VTDContextEntry *ce)
13621da12ec4SLe Tan {
13631da12ec4SLe Tan     VTDRootEntry re;
13641da12ec4SLe Tan     int ret_fr;
1365f80c9874SPeter Xu     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
13661da12ec4SLe Tan 
13671da12ec4SLe Tan     ret_fr = vtd_get_root_entry(s, bus_num, &re);
13681da12ec4SLe Tan     if (ret_fr) {
13691da12ec4SLe Tan         return ret_fr;
13701da12ec4SLe Tan     }
13711da12ec4SLe Tan 
1372fb43cf73SLiu, Yi L     if (!vtd_root_entry_present(s, &re, devfn)) {
13736c441e1dSPeter Xu         /* Not error - it's okay we don't have root entry. */
13746c441e1dSPeter Xu         trace_vtd_re_not_present(bus_num);
13751da12ec4SLe Tan         return -VTD_FR_ROOT_ENTRY_P;
1376f80c9874SPeter Xu     }
1377f80c9874SPeter Xu 
1378fb43cf73SLiu, Yi L     ret_fr = vtd_root_entry_rsvd_bits_check(s, &re);
1379fb43cf73SLiu, Yi L     if (ret_fr) {
1380fb43cf73SLiu, Yi L         return ret_fr;
13811da12ec4SLe Tan     }
13821da12ec4SLe Tan 
1383fb43cf73SLiu, Yi L     ret_fr = vtd_get_context_entry_from_root(s, &re, devfn, ce);
13841da12ec4SLe Tan     if (ret_fr) {
13851da12ec4SLe Tan         return ret_fr;
13861da12ec4SLe Tan     }
13871da12ec4SLe Tan 
13888f7d7161SPeter Xu     if (!vtd_ce_present(ce)) {
13896c441e1dSPeter Xu         /* Not error - it's okay we don't have context entry. */
13906c441e1dSPeter Xu         trace_vtd_ce_not_present(bus_num, devfn);
13911da12ec4SLe Tan         return -VTD_FR_CONTEXT_ENTRY_P;
1392f80c9874SPeter Xu     }
1393f80c9874SPeter Xu 
1394fb43cf73SLiu, Yi L     ret_fr = vtd_context_entry_rsvd_bits_check(s, ce);
1395fb43cf73SLiu, Yi L     if (ret_fr) {
1396fb43cf73SLiu, Yi L         return ret_fr;
13971da12ec4SLe Tan     }
1398f80c9874SPeter Xu 
13991da12ec4SLe Tan     /* Check if the programming of context-entry is valid */
1400fb43cf73SLiu, Yi L     if (!s->root_scalable &&
1401fb43cf73SLiu, Yi L         !vtd_is_level_supported(s, vtd_ce_get_level(ce))) {
1402095955b2SPeter Xu         error_report_once("%s: invalid context entry: hi=%"PRIx64
1403095955b2SPeter Xu                           ", lo=%"PRIx64" (level %d not supported)",
1404fb43cf73SLiu, Yi L                           __func__, ce->hi, ce->lo,
1405fb43cf73SLiu, Yi L                           vtd_ce_get_level(ce));
14061da12ec4SLe Tan         return -VTD_FR_CONTEXT_ENTRY_INV;
1407f80c9874SPeter Xu     }
1408f80c9874SPeter Xu 
1409fb43cf73SLiu, Yi L     if (!s->root_scalable) {
1410f80c9874SPeter Xu         /* Do translation type check */
1411f80c9874SPeter Xu         if (!vtd_ce_type_check(x86_iommu, ce)) {
1412095955b2SPeter Xu             /* Errors dumped in vtd_ce_type_check() */
14131da12ec4SLe Tan             return -VTD_FR_CONTEXT_ENTRY_INV;
14141da12ec4SLe Tan         }
1415fb43cf73SLiu, Yi L     } else {
1416fb43cf73SLiu, Yi L         /*
1417fb43cf73SLiu, Yi L          * Check if the programming of context-entry.rid2pasid
1418fb43cf73SLiu, Yi L          * and corresponding pasid setting is valid, and thus
1419fb43cf73SLiu, Yi L          * avoids to check pasid entry fetching result in future
1420fb43cf73SLiu, Yi L          * helper function calling.
1421fb43cf73SLiu, Yi L          */
1422fb43cf73SLiu, Yi L         ret_fr = vtd_ce_rid2pasid_check(s, ce);
1423fb43cf73SLiu, Yi L         if (ret_fr) {
1424fb43cf73SLiu, Yi L             return ret_fr;
1425fb43cf73SLiu, Yi L         }
1426fb43cf73SLiu, Yi L     }
1427f80c9874SPeter Xu 
14281da12ec4SLe Tan     return 0;
14291da12ec4SLe Tan }
14301da12ec4SLe Tan 
143163b88968SPeter Xu static int vtd_sync_shadow_page_hook(IOMMUTLBEntry *entry,
143263b88968SPeter Xu                                      void *private)
143363b88968SPeter Xu {
1434cb1efcf4SPeter Maydell     memory_region_notify_iommu((IOMMUMemoryRegion *)private, 0, *entry);
143563b88968SPeter Xu     return 0;
143663b88968SPeter Xu }
143763b88968SPeter Xu 
1438fb43cf73SLiu, Yi L static uint16_t vtd_get_domain_id(IntelIOMMUState *s,
1439fb43cf73SLiu, Yi L                                   VTDContextEntry *ce)
1440fb43cf73SLiu, Yi L {
1441fb43cf73SLiu, Yi L     VTDPASIDEntry pe;
1442fb43cf73SLiu, Yi L 
1443fb43cf73SLiu, Yi L     if (s->root_scalable) {
1444fb43cf73SLiu, Yi L         vtd_ce_get_rid2pasid_entry(s, ce, &pe);
1445fb43cf73SLiu, Yi L         return VTD_SM_PASID_ENTRY_DID(pe.val[1]);
1446fb43cf73SLiu, Yi L     }
1447fb43cf73SLiu, Yi L 
1448fb43cf73SLiu, Yi L     return VTD_CONTEXT_ENTRY_DID(ce->hi);
1449fb43cf73SLiu, Yi L }
1450fb43cf73SLiu, Yi L 
145163b88968SPeter Xu static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as,
145263b88968SPeter Xu                                             VTDContextEntry *ce,
145363b88968SPeter Xu                                             hwaddr addr, hwaddr size)
145463b88968SPeter Xu {
145563b88968SPeter Xu     IntelIOMMUState *s = vtd_as->iommu_state;
145663b88968SPeter Xu     vtd_page_walk_info info = {
145763b88968SPeter Xu         .hook_fn = vtd_sync_shadow_page_hook,
145863b88968SPeter Xu         .private = (void *)&vtd_as->iommu,
145963b88968SPeter Xu         .notify_unmap = true,
146063b88968SPeter Xu         .aw = s->aw_bits,
146163b88968SPeter Xu         .as = vtd_as,
1462fb43cf73SLiu, Yi L         .domain_id = vtd_get_domain_id(s, ce),
146363b88968SPeter Xu     };
146463b88968SPeter Xu 
1465fb43cf73SLiu, Yi L     return vtd_page_walk(s, ce, addr, addr + size, &info);
146663b88968SPeter Xu }
146763b88968SPeter Xu 
146863b88968SPeter Xu static int vtd_sync_shadow_page_table(VTDAddressSpace *vtd_as)
146963b88968SPeter Xu {
147095ecd3dfSPeter Xu     int ret;
147195ecd3dfSPeter Xu     VTDContextEntry ce;
1472c28b535dSPeter Xu     IOMMUNotifier *n;
147395ecd3dfSPeter Xu 
147495ecd3dfSPeter Xu     ret = vtd_dev_to_context_entry(vtd_as->iommu_state,
147595ecd3dfSPeter Xu                                    pci_bus_num(vtd_as->bus),
147695ecd3dfSPeter Xu                                    vtd_as->devfn, &ce);
147795ecd3dfSPeter Xu     if (ret) {
1478c28b535dSPeter Xu         if (ret == -VTD_FR_CONTEXT_ENTRY_P) {
1479c28b535dSPeter Xu             /*
1480c28b535dSPeter Xu              * It's a valid scenario to have a context entry that is
1481c28b535dSPeter Xu              * not present.  For example, when a device is removed
1482c28b535dSPeter Xu              * from an existing domain then the context entry will be
1483c28b535dSPeter Xu              * zeroed by the guest before it was put into another
1484c28b535dSPeter Xu              * domain.  When this happens, instead of synchronizing
1485c28b535dSPeter Xu              * the shadow pages we should invalidate all existing
1486c28b535dSPeter Xu              * mappings and notify the backends.
1487c28b535dSPeter Xu              */
1488c28b535dSPeter Xu             IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) {
1489c28b535dSPeter Xu                 vtd_address_space_unmap(vtd_as, n);
1490c28b535dSPeter Xu             }
1491c28b535dSPeter Xu             ret = 0;
1492c28b535dSPeter Xu         }
149395ecd3dfSPeter Xu         return ret;
149495ecd3dfSPeter Xu     }
149595ecd3dfSPeter Xu 
149695ecd3dfSPeter Xu     return vtd_sync_shadow_page_table_range(vtd_as, &ce, 0, UINT64_MAX);
149763b88968SPeter Xu }
149863b88968SPeter Xu 
1499dbaabb25SPeter Xu /*
1500fb43cf73SLiu, Yi L  * Check if specific device is configed to bypass address
1501fb43cf73SLiu, Yi L  * translation for DMA requests. In Scalable Mode, bypass
1502fb43cf73SLiu, Yi L  * 1st-level translation or 2nd-level translation, it depends
1503fb43cf73SLiu, Yi L  * on PGTT setting.
1504dbaabb25SPeter Xu  */
1505fb43cf73SLiu, Yi L static bool vtd_dev_pt_enabled(VTDAddressSpace *as)
1506dbaabb25SPeter Xu {
1507dbaabb25SPeter Xu     IntelIOMMUState *s;
1508dbaabb25SPeter Xu     VTDContextEntry ce;
1509fb43cf73SLiu, Yi L     VTDPASIDEntry pe;
1510dbaabb25SPeter Xu     int ret;
1511dbaabb25SPeter Xu 
1512dbaabb25SPeter Xu     assert(as);
1513dbaabb25SPeter Xu 
1514fb43cf73SLiu, Yi L     s = as->iommu_state;
1515fb43cf73SLiu, Yi L     ret = vtd_dev_to_context_entry(s, pci_bus_num(as->bus),
1516fb43cf73SLiu, Yi L                                    as->devfn, &ce);
1517fb43cf73SLiu, Yi L     if (ret) {
1518dbaabb25SPeter Xu         /*
1519dbaabb25SPeter Xu          * Possibly failed to parse the context entry for some reason
1520dbaabb25SPeter Xu          * (e.g., during init, or any guest configuration errors on
1521dbaabb25SPeter Xu          * context entries). We should assume PT not enabled for
1522dbaabb25SPeter Xu          * safety.
1523dbaabb25SPeter Xu          */
1524dbaabb25SPeter Xu         return false;
1525dbaabb25SPeter Xu     }
1526dbaabb25SPeter Xu 
1527fb43cf73SLiu, Yi L     if (s->root_scalable) {
1528fb43cf73SLiu, Yi L         ret = vtd_ce_get_rid2pasid_entry(s, &ce, &pe);
1529fb43cf73SLiu, Yi L         if (ret) {
1530fb43cf73SLiu, Yi L             error_report_once("%s: vtd_ce_get_rid2pasid_entry error: %"PRId32,
1531fb43cf73SLiu, Yi L                               __func__, ret);
1532fb43cf73SLiu, Yi L             return false;
1533fb43cf73SLiu, Yi L         }
1534fb43cf73SLiu, Yi L         return (VTD_PE_GET_TYPE(&pe) == VTD_SM_PASID_ENTRY_PT);
1535fb43cf73SLiu, Yi L     }
1536fb43cf73SLiu, Yi L 
1537fb43cf73SLiu, Yi L     return (vtd_ce_get_type(&ce) == VTD_CONTEXT_TT_PASS_THROUGH);
1538dbaabb25SPeter Xu }
1539dbaabb25SPeter Xu 
1540dbaabb25SPeter Xu /* Return whether the device is using IOMMU translation. */
1541dbaabb25SPeter Xu static bool vtd_switch_address_space(VTDAddressSpace *as)
1542dbaabb25SPeter Xu {
1543dbaabb25SPeter Xu     bool use_iommu;
154466a4a031SPeter Xu     /* Whether we need to take the BQL on our own */
154566a4a031SPeter Xu     bool take_bql = !qemu_mutex_iothread_locked();
1546dbaabb25SPeter Xu 
1547dbaabb25SPeter Xu     assert(as);
1548dbaabb25SPeter Xu 
15492a078b10SPeter Xu     use_iommu = as->iommu_state->dmar_enabled && !vtd_dev_pt_enabled(as);
1550dbaabb25SPeter Xu 
1551dbaabb25SPeter Xu     trace_vtd_switch_address_space(pci_bus_num(as->bus),
1552dbaabb25SPeter Xu                                    VTD_PCI_SLOT(as->devfn),
1553dbaabb25SPeter Xu                                    VTD_PCI_FUNC(as->devfn),
1554dbaabb25SPeter Xu                                    use_iommu);
1555dbaabb25SPeter Xu 
155666a4a031SPeter Xu     /*
155766a4a031SPeter Xu      * It's possible that we reach here without BQL, e.g., when called
155866a4a031SPeter Xu      * from vtd_pt_enable_fast_path(). However the memory APIs need
155966a4a031SPeter Xu      * it. We'd better make sure we have had it already, or, take it.
156066a4a031SPeter Xu      */
156166a4a031SPeter Xu     if (take_bql) {
156266a4a031SPeter Xu         qemu_mutex_lock_iothread();
156366a4a031SPeter Xu     }
156466a4a031SPeter Xu 
1565dbaabb25SPeter Xu     /* Turn off first then on the other */
1566dbaabb25SPeter Xu     if (use_iommu) {
15674b519ef1SPeter Xu         memory_region_set_enabled(&as->nodmar, false);
15683df9d748SAlexey Kardashevskiy         memory_region_set_enabled(MEMORY_REGION(&as->iommu), true);
1569dbaabb25SPeter Xu     } else {
15703df9d748SAlexey Kardashevskiy         memory_region_set_enabled(MEMORY_REGION(&as->iommu), false);
15714b519ef1SPeter Xu         memory_region_set_enabled(&as->nodmar, true);
1572dbaabb25SPeter Xu     }
1573dbaabb25SPeter Xu 
157466a4a031SPeter Xu     if (take_bql) {
157566a4a031SPeter Xu         qemu_mutex_unlock_iothread();
157666a4a031SPeter Xu     }
157766a4a031SPeter Xu 
1578dbaabb25SPeter Xu     return use_iommu;
1579dbaabb25SPeter Xu }
1580dbaabb25SPeter Xu 
1581dbaabb25SPeter Xu static void vtd_switch_address_space_all(IntelIOMMUState *s)
1582dbaabb25SPeter Xu {
1583dbaabb25SPeter Xu     GHashTableIter iter;
1584dbaabb25SPeter Xu     VTDBus *vtd_bus;
1585dbaabb25SPeter Xu     int i;
1586dbaabb25SPeter Xu 
1587dbaabb25SPeter Xu     g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
1588dbaabb25SPeter Xu     while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) {
1589bf33cc75SPeter Xu         for (i = 0; i < PCI_DEVFN_MAX; i++) {
1590dbaabb25SPeter Xu             if (!vtd_bus->dev_as[i]) {
1591dbaabb25SPeter Xu                 continue;
1592dbaabb25SPeter Xu             }
1593dbaabb25SPeter Xu             vtd_switch_address_space(vtd_bus->dev_as[i]);
1594dbaabb25SPeter Xu         }
1595dbaabb25SPeter Xu     }
1596dbaabb25SPeter Xu }
1597dbaabb25SPeter Xu 
15981da12ec4SLe Tan static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn)
15991da12ec4SLe Tan {
16001da12ec4SLe Tan     return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL);
16011da12ec4SLe Tan }
16021da12ec4SLe Tan 
16031da12ec4SLe Tan static const bool vtd_qualified_faults[] = {
16041da12ec4SLe Tan     [VTD_FR_RESERVED] = false,
16051da12ec4SLe Tan     [VTD_FR_ROOT_ENTRY_P] = false,
16061da12ec4SLe Tan     [VTD_FR_CONTEXT_ENTRY_P] = true,
16071da12ec4SLe Tan     [VTD_FR_CONTEXT_ENTRY_INV] = true,
16081da12ec4SLe Tan     [VTD_FR_ADDR_BEYOND_MGAW] = true,
16091da12ec4SLe Tan     [VTD_FR_WRITE] = true,
16101da12ec4SLe Tan     [VTD_FR_READ] = true,
16111da12ec4SLe Tan     [VTD_FR_PAGING_ENTRY_INV] = true,
16121da12ec4SLe Tan     [VTD_FR_ROOT_TABLE_INV] = false,
16131da12ec4SLe Tan     [VTD_FR_CONTEXT_TABLE_INV] = false,
16141da12ec4SLe Tan     [VTD_FR_ROOT_ENTRY_RSVD] = false,
16151da12ec4SLe Tan     [VTD_FR_PAGING_ENTRY_RSVD] = true,
16161da12ec4SLe Tan     [VTD_FR_CONTEXT_ENTRY_TT] = true,
1617fb43cf73SLiu, Yi L     [VTD_FR_PASID_TABLE_INV] = false,
16181da12ec4SLe Tan     [VTD_FR_RESERVED_ERR] = false,
16191da12ec4SLe Tan     [VTD_FR_MAX] = false,
16201da12ec4SLe Tan };
16211da12ec4SLe Tan 
16221da12ec4SLe Tan /* To see if a fault condition is "qualified", which is reported to software
16231da12ec4SLe Tan  * only if the FPD field in the context-entry used to process the faulting
16241da12ec4SLe Tan  * request is 0.
16251da12ec4SLe Tan  */
16261da12ec4SLe Tan static inline bool vtd_is_qualified_fault(VTDFaultReason fault)
16271da12ec4SLe Tan {
16281da12ec4SLe Tan     return vtd_qualified_faults[fault];
16291da12ec4SLe Tan }
16301da12ec4SLe Tan 
16311da12ec4SLe Tan static inline bool vtd_is_interrupt_addr(hwaddr addr)
16321da12ec4SLe Tan {
16331da12ec4SLe Tan     return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST;
16341da12ec4SLe Tan }
16351da12ec4SLe Tan 
1636dbaabb25SPeter Xu static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id)
1637dbaabb25SPeter Xu {
1638dbaabb25SPeter Xu     VTDBus *vtd_bus;
1639dbaabb25SPeter Xu     VTDAddressSpace *vtd_as;
1640dbaabb25SPeter Xu     bool success = false;
1641dbaabb25SPeter Xu 
1642dbaabb25SPeter Xu     vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id));
1643dbaabb25SPeter Xu     if (!vtd_bus) {
1644dbaabb25SPeter Xu         goto out;
1645dbaabb25SPeter Xu     }
1646dbaabb25SPeter Xu 
1647dbaabb25SPeter Xu     vtd_as = vtd_bus->dev_as[VTD_SID_TO_DEVFN(source_id)];
1648dbaabb25SPeter Xu     if (!vtd_as) {
1649dbaabb25SPeter Xu         goto out;
1650dbaabb25SPeter Xu     }
1651dbaabb25SPeter Xu 
1652dbaabb25SPeter Xu     if (vtd_switch_address_space(vtd_as) == false) {
1653dbaabb25SPeter Xu         /* We switched off IOMMU region successfully. */
1654dbaabb25SPeter Xu         success = true;
1655dbaabb25SPeter Xu     }
1656dbaabb25SPeter Xu 
1657dbaabb25SPeter Xu out:
1658dbaabb25SPeter Xu     trace_vtd_pt_enable_fast_path(source_id, success);
1659dbaabb25SPeter Xu }
1660dbaabb25SPeter Xu 
16611da12ec4SLe Tan /* Map dev to context-entry then do a paging-structures walk to do a iommu
16621da12ec4SLe Tan  * translation.
166379e2b9aeSPaolo Bonzini  *
166479e2b9aeSPaolo Bonzini  * Called from RCU critical section.
166579e2b9aeSPaolo Bonzini  *
16661da12ec4SLe Tan  * @bus_num: The bus number
16671da12ec4SLe Tan  * @devfn: The devfn, which is the  combined of device and function number
16681da12ec4SLe Tan  * @is_write: The access is a write operation
16691da12ec4SLe Tan  * @entry: IOMMUTLBEntry that contain the addr to be translated and result
1670b9313021SPeter Xu  *
1671b9313021SPeter Xu  * Returns true if translation is successful, otherwise false.
16721da12ec4SLe Tan  */
1673b9313021SPeter Xu static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
16741da12ec4SLe Tan                                    uint8_t devfn, hwaddr addr, bool is_write,
16751da12ec4SLe Tan                                    IOMMUTLBEntry *entry)
16761da12ec4SLe Tan {
1677d92fa2dcSLe Tan     IntelIOMMUState *s = vtd_as->iommu_state;
16781da12ec4SLe Tan     VTDContextEntry ce;
16797df953bdSKnut Omang     uint8_t bus_num = pci_bus_num(bus);
16801d9efa73SPeter Xu     VTDContextCacheEntry *cc_entry;
1681d66b969bSJason Wang     uint64_t slpte, page_mask;
16821da12ec4SLe Tan     uint32_t level;
16831da12ec4SLe Tan     uint16_t source_id = vtd_make_source_id(bus_num, devfn);
16841da12ec4SLe Tan     int ret_fr;
16851da12ec4SLe Tan     bool is_fpd_set = false;
16861da12ec4SLe Tan     bool reads = true;
16871da12ec4SLe Tan     bool writes = true;
168807f7b733SPeter Xu     uint8_t access_flags;
1689b5a280c0SLe Tan     VTDIOTLBEntry *iotlb_entry;
16901da12ec4SLe Tan 
1691046ab7e9SPeter Xu     /*
1692046ab7e9SPeter Xu      * We have standalone memory region for interrupt addresses, we
1693046ab7e9SPeter Xu      * should never receive translation requests in this region.
16941da12ec4SLe Tan      */
1695046ab7e9SPeter Xu     assert(!vtd_is_interrupt_addr(addr));
1696046ab7e9SPeter Xu 
16971d9efa73SPeter Xu     vtd_iommu_lock(s);
16981d9efa73SPeter Xu 
16991d9efa73SPeter Xu     cc_entry = &vtd_as->context_cache_entry;
17001d9efa73SPeter Xu 
1701b5a280c0SLe Tan     /* Try to fetch slpte form IOTLB */
1702b5a280c0SLe Tan     iotlb_entry = vtd_lookup_iotlb(s, source_id, addr);
1703b5a280c0SLe Tan     if (iotlb_entry) {
17046c441e1dSPeter Xu         trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte,
17056c441e1dSPeter Xu                                  iotlb_entry->domain_id);
1706b5a280c0SLe Tan         slpte = iotlb_entry->slpte;
170707f7b733SPeter Xu         access_flags = iotlb_entry->access_flags;
1708d66b969bSJason Wang         page_mask = iotlb_entry->mask;
1709b5a280c0SLe Tan         goto out;
1710b5a280c0SLe Tan     }
1711b9313021SPeter Xu 
1712d92fa2dcSLe Tan     /* Try to fetch context-entry from cache first */
1713d92fa2dcSLe Tan     if (cc_entry->context_cache_gen == s->context_cache_gen) {
17146c441e1dSPeter Xu         trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi,
17156c441e1dSPeter Xu                                cc_entry->context_entry.lo,
17166c441e1dSPeter Xu                                cc_entry->context_cache_gen);
1717d92fa2dcSLe Tan         ce = cc_entry->context_entry;
1718d92fa2dcSLe Tan         is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
1719fb43cf73SLiu, Yi L         if (!is_fpd_set && s->root_scalable) {
1720fb43cf73SLiu, Yi L             ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set);
1721fb43cf73SLiu, Yi L             VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write);
1722fb43cf73SLiu, Yi L         }
1723d92fa2dcSLe Tan     } else {
17241da12ec4SLe Tan         ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
17251da12ec4SLe Tan         is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
1726fb43cf73SLiu, Yi L         if (!ret_fr && !is_fpd_set && s->root_scalable) {
1727fb43cf73SLiu, Yi L             ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set);
17281da12ec4SLe Tan         }
1729fb43cf73SLiu, Yi L         VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write);
1730d92fa2dcSLe Tan         /* Update context-cache */
17316c441e1dSPeter Xu         trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo,
17326c441e1dSPeter Xu                                   cc_entry->context_cache_gen,
17336c441e1dSPeter Xu                                   s->context_cache_gen);
1734d92fa2dcSLe Tan         cc_entry->context_entry = ce;
1735d92fa2dcSLe Tan         cc_entry->context_cache_gen = s->context_cache_gen;
1736d92fa2dcSLe Tan     }
17371da12ec4SLe Tan 
1738dbaabb25SPeter Xu     /*
1739dbaabb25SPeter Xu      * We don't need to translate for pass-through context entries.
1740dbaabb25SPeter Xu      * Also, let's ignore IOTLB caching as well for PT devices.
1741dbaabb25SPeter Xu      */
1742dbaabb25SPeter Xu     if (vtd_ce_get_type(&ce) == VTD_CONTEXT_TT_PASS_THROUGH) {
1743892721d9SPeter Xu         entry->iova = addr & VTD_PAGE_MASK_4K;
1744dbaabb25SPeter Xu         entry->translated_addr = entry->iova;
1745892721d9SPeter Xu         entry->addr_mask = ~VTD_PAGE_MASK_4K;
1746dbaabb25SPeter Xu         entry->perm = IOMMU_RW;
1747dbaabb25SPeter Xu         trace_vtd_translate_pt(source_id, entry->iova);
1748dbaabb25SPeter Xu 
1749dbaabb25SPeter Xu         /*
1750dbaabb25SPeter Xu          * When this happens, it means firstly caching-mode is not
1751dbaabb25SPeter Xu          * enabled, and this is the first passthrough translation for
1752dbaabb25SPeter Xu          * the device. Let's enable the fast path for passthrough.
1753dbaabb25SPeter Xu          *
1754dbaabb25SPeter Xu          * When passthrough is disabled again for the device, we can
1755dbaabb25SPeter Xu          * capture it via the context entry invalidation, then the
1756dbaabb25SPeter Xu          * IOMMU region can be swapped back.
1757dbaabb25SPeter Xu          */
1758dbaabb25SPeter Xu         vtd_pt_enable_fast_path(s, source_id);
17591d9efa73SPeter Xu         vtd_iommu_unlock(s);
1760b9313021SPeter Xu         return true;
1761dbaabb25SPeter Xu     }
1762dbaabb25SPeter Xu 
1763fb43cf73SLiu, Yi L     ret_fr = vtd_iova_to_slpte(s, &ce, addr, is_write, &slpte, &level,
176437f51384SPrasad Singamsetty                                &reads, &writes, s->aw_bits);
1765fb43cf73SLiu, Yi L     VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write);
17661da12ec4SLe Tan 
1767d66b969bSJason Wang     page_mask = vtd_slpt_level_page_mask(level);
176807f7b733SPeter Xu     access_flags = IOMMU_ACCESS_FLAG(reads, writes);
1769fb43cf73SLiu, Yi L     vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce), addr, slpte,
177007f7b733SPeter Xu                      access_flags, level);
1771b5a280c0SLe Tan out:
17721d9efa73SPeter Xu     vtd_iommu_unlock(s);
1773d66b969bSJason Wang     entry->iova = addr & page_mask;
177437f51384SPrasad Singamsetty     entry->translated_addr = vtd_get_slpte_addr(slpte, s->aw_bits) & page_mask;
1775d66b969bSJason Wang     entry->addr_mask = ~page_mask;
177607f7b733SPeter Xu     entry->perm = access_flags;
1777b9313021SPeter Xu     return true;
1778b9313021SPeter Xu 
1779b9313021SPeter Xu error:
17801d9efa73SPeter Xu     vtd_iommu_unlock(s);
1781b9313021SPeter Xu     entry->iova = 0;
1782b9313021SPeter Xu     entry->translated_addr = 0;
1783b9313021SPeter Xu     entry->addr_mask = 0;
1784b9313021SPeter Xu     entry->perm = IOMMU_NONE;
1785b9313021SPeter Xu     return false;
17861da12ec4SLe Tan }
17871da12ec4SLe Tan 
17881da12ec4SLe Tan static void vtd_root_table_setup(IntelIOMMUState *s)
17891da12ec4SLe Tan {
17901da12ec4SLe Tan     s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
179137f51384SPrasad Singamsetty     s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits);
17921da12ec4SLe Tan 
17932811af3bSPeter Xu     vtd_update_scalable_state(s);
17942811af3bSPeter Xu 
179581fb1e64SPeter Xu     trace_vtd_reg_dmar_root(s->root, s->root_scalable);
17961da12ec4SLe Tan }
17971da12ec4SLe Tan 
179802a2cbc8SPeter Xu static void vtd_iec_notify_all(IntelIOMMUState *s, bool global,
179902a2cbc8SPeter Xu                                uint32_t index, uint32_t mask)
180002a2cbc8SPeter Xu {
180102a2cbc8SPeter Xu     x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
180202a2cbc8SPeter Xu }
180302a2cbc8SPeter Xu 
1804a5861439SPeter Xu static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s)
1805a5861439SPeter Xu {
1806a5861439SPeter Xu     uint64_t value = 0;
1807a5861439SPeter Xu     value = vtd_get_quad_raw(s, DMAR_IRTA_REG);
1808a5861439SPeter Xu     s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1);
180937f51384SPrasad Singamsetty     s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits);
181028589311SJan Kiszka     s->intr_eime = value & VTD_IRTA_EIME;
1811a5861439SPeter Xu 
181202a2cbc8SPeter Xu     /* Notify global invalidation */
181302a2cbc8SPeter Xu     vtd_iec_notify_all(s, true, 0, 0);
1814a5861439SPeter Xu 
18157feb51b7SPeter Xu     trace_vtd_reg_ir_root(s->intr_root, s->intr_size);
1816a5861439SPeter Xu }
1817a5861439SPeter Xu 
1818dd4d607eSPeter Xu static void vtd_iommu_replay_all(IntelIOMMUState *s)
1819dd4d607eSPeter Xu {
1820b4a4ba0dSPeter Xu     VTDAddressSpace *vtd_as;
1821dd4d607eSPeter Xu 
1822b4a4ba0dSPeter Xu     QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
182363b88968SPeter Xu         vtd_sync_shadow_page_table(vtd_as);
1824dd4d607eSPeter Xu     }
1825dd4d607eSPeter Xu }
1826dd4d607eSPeter Xu 
1827d92fa2dcSLe Tan static void vtd_context_global_invalidate(IntelIOMMUState *s)
1828d92fa2dcSLe Tan {
1829bc535e59SPeter Xu     trace_vtd_inv_desc_cc_global();
18301d9efa73SPeter Xu     /* Protects context cache */
18311d9efa73SPeter Xu     vtd_iommu_lock(s);
1832d92fa2dcSLe Tan     s->context_cache_gen++;
1833d92fa2dcSLe Tan     if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
18341d9efa73SPeter Xu         vtd_reset_context_cache_locked(s);
1835d92fa2dcSLe Tan     }
18361d9efa73SPeter Xu     vtd_iommu_unlock(s);
18372cc9ddccSPeter Xu     vtd_address_space_refresh_all(s);
1838dd4d607eSPeter Xu     /*
1839dd4d607eSPeter Xu      * From VT-d spec 6.5.2.1, a global context entry invalidation
1840dd4d607eSPeter Xu      * should be followed by a IOTLB global invalidation, so we should
1841dd4d607eSPeter Xu      * be safe even without this. Hoewever, let's replay the region as
1842dd4d607eSPeter Xu      * well to be safer, and go back here when we need finer tunes for
1843dd4d607eSPeter Xu      * VT-d emulation codes.
1844dd4d607eSPeter Xu      */
1845dd4d607eSPeter Xu     vtd_iommu_replay_all(s);
1846d92fa2dcSLe Tan }
1847d92fa2dcSLe Tan 
1848d92fa2dcSLe Tan /* Do a context-cache device-selective invalidation.
1849d92fa2dcSLe Tan  * @func_mask: FM field after shifting
1850d92fa2dcSLe Tan  */
1851d92fa2dcSLe Tan static void vtd_context_device_invalidate(IntelIOMMUState *s,
1852d92fa2dcSLe Tan                                           uint16_t source_id,
1853d92fa2dcSLe Tan                                           uint16_t func_mask)
1854d92fa2dcSLe Tan {
1855d92fa2dcSLe Tan     uint16_t mask;
18567df953bdSKnut Omang     VTDBus *vtd_bus;
1857d92fa2dcSLe Tan     VTDAddressSpace *vtd_as;
1858bc535e59SPeter Xu     uint8_t bus_n, devfn;
1859d92fa2dcSLe Tan     uint16_t devfn_it;
1860d92fa2dcSLe Tan 
1861bc535e59SPeter Xu     trace_vtd_inv_desc_cc_devices(source_id, func_mask);
1862bc535e59SPeter Xu 
1863d92fa2dcSLe Tan     switch (func_mask & 3) {
1864d92fa2dcSLe Tan     case 0:
1865d92fa2dcSLe Tan         mask = 0;   /* No bits in the SID field masked */
1866d92fa2dcSLe Tan         break;
1867d92fa2dcSLe Tan     case 1:
1868d92fa2dcSLe Tan         mask = 4;   /* Mask bit 2 in the SID field */
1869d92fa2dcSLe Tan         break;
1870d92fa2dcSLe Tan     case 2:
1871d92fa2dcSLe Tan         mask = 6;   /* Mask bit 2:1 in the SID field */
1872d92fa2dcSLe Tan         break;
1873d92fa2dcSLe Tan     case 3:
1874d92fa2dcSLe Tan         mask = 7;   /* Mask bit 2:0 in the SID field */
1875d92fa2dcSLe Tan         break;
1876d92fa2dcSLe Tan     }
18776cb99accSPeter Xu     mask = ~mask;
1878bc535e59SPeter Xu 
1879bc535e59SPeter Xu     bus_n = VTD_SID_TO_BUS(source_id);
1880bc535e59SPeter Xu     vtd_bus = vtd_find_as_from_bus_num(s, bus_n);
18817df953bdSKnut Omang     if (vtd_bus) {
1882d92fa2dcSLe Tan         devfn = VTD_SID_TO_DEVFN(source_id);
1883bf33cc75SPeter Xu         for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) {
18847df953bdSKnut Omang             vtd_as = vtd_bus->dev_as[devfn_it];
1885d92fa2dcSLe Tan             if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
1886bc535e59SPeter Xu                 trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(devfn_it),
1887bc535e59SPeter Xu                                              VTD_PCI_FUNC(devfn_it));
18881d9efa73SPeter Xu                 vtd_iommu_lock(s);
1889d92fa2dcSLe Tan                 vtd_as->context_cache_entry.context_cache_gen = 0;
18901d9efa73SPeter Xu                 vtd_iommu_unlock(s);
1891dd4d607eSPeter Xu                 /*
1892dbaabb25SPeter Xu                  * Do switch address space when needed, in case if the
1893dbaabb25SPeter Xu                  * device passthrough bit is switched.
1894dbaabb25SPeter Xu                  */
1895dbaabb25SPeter Xu                 vtd_switch_address_space(vtd_as);
1896dbaabb25SPeter Xu                 /*
1897dd4d607eSPeter Xu                  * So a device is moving out of (or moving into) a
189863b88968SPeter Xu                  * domain, resync the shadow page table.
1899dd4d607eSPeter Xu                  * This won't bring bad even if we have no such
1900dd4d607eSPeter Xu                  * notifier registered - the IOMMU notification
1901dd4d607eSPeter Xu                  * framework will skip MAP notifications if that
1902dd4d607eSPeter Xu                  * happened.
1903dd4d607eSPeter Xu                  */
190463b88968SPeter Xu                 vtd_sync_shadow_page_table(vtd_as);
1905d92fa2dcSLe Tan             }
1906d92fa2dcSLe Tan         }
1907d92fa2dcSLe Tan     }
1908d92fa2dcSLe Tan }
1909d92fa2dcSLe Tan 
19101da12ec4SLe Tan /* Context-cache invalidation
19111da12ec4SLe Tan  * Returns the Context Actual Invalidation Granularity.
19121da12ec4SLe Tan  * @val: the content of the CCMD_REG
19131da12ec4SLe Tan  */
19141da12ec4SLe Tan static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
19151da12ec4SLe Tan {
19161da12ec4SLe Tan     uint64_t caig;
19171da12ec4SLe Tan     uint64_t type = val & VTD_CCMD_CIRG_MASK;
19181da12ec4SLe Tan 
19191da12ec4SLe Tan     switch (type) {
19201da12ec4SLe Tan     case VTD_CCMD_DOMAIN_INVL:
1921d92fa2dcSLe Tan         /* Fall through */
1922d92fa2dcSLe Tan     case VTD_CCMD_GLOBAL_INVL:
1923d92fa2dcSLe Tan         caig = VTD_CCMD_GLOBAL_INVL_A;
1924d92fa2dcSLe Tan         vtd_context_global_invalidate(s);
19251da12ec4SLe Tan         break;
19261da12ec4SLe Tan 
19271da12ec4SLe Tan     case VTD_CCMD_DEVICE_INVL:
19281da12ec4SLe Tan         caig = VTD_CCMD_DEVICE_INVL_A;
1929d92fa2dcSLe Tan         vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
19301da12ec4SLe Tan         break;
19311da12ec4SLe Tan 
19321da12ec4SLe Tan     default:
19331376211fSPeter Xu         error_report_once("%s: invalid context: 0x%" PRIx64,
19341376211fSPeter Xu                           __func__, val);
19351da12ec4SLe Tan         caig = 0;
19361da12ec4SLe Tan     }
19371da12ec4SLe Tan     return caig;
19381da12ec4SLe Tan }
19391da12ec4SLe Tan 
1940b5a280c0SLe Tan static void vtd_iotlb_global_invalidate(IntelIOMMUState *s)
1941b5a280c0SLe Tan {
19427feb51b7SPeter Xu     trace_vtd_inv_desc_iotlb_global();
1943b5a280c0SLe Tan     vtd_reset_iotlb(s);
1944dd4d607eSPeter Xu     vtd_iommu_replay_all(s);
1945b5a280c0SLe Tan }
1946b5a280c0SLe Tan 
1947b5a280c0SLe Tan static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
1948b5a280c0SLe Tan {
1949dd4d607eSPeter Xu     VTDContextEntry ce;
1950dd4d607eSPeter Xu     VTDAddressSpace *vtd_as;
1951dd4d607eSPeter Xu 
19527feb51b7SPeter Xu     trace_vtd_inv_desc_iotlb_domain(domain_id);
19537feb51b7SPeter Xu 
19541d9efa73SPeter Xu     vtd_iommu_lock(s);
1955b5a280c0SLe Tan     g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain,
1956b5a280c0SLe Tan                                 &domain_id);
19571d9efa73SPeter Xu     vtd_iommu_unlock(s);
1958dd4d607eSPeter Xu 
1959b4a4ba0dSPeter Xu     QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
1960dd4d607eSPeter Xu         if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus),
1961dd4d607eSPeter Xu                                       vtd_as->devfn, &ce) &&
1962fb43cf73SLiu, Yi L             domain_id == vtd_get_domain_id(s, &ce)) {
196363b88968SPeter Xu             vtd_sync_shadow_page_table(vtd_as);
1964dd4d607eSPeter Xu         }
1965dd4d607eSPeter Xu     }
1966dd4d607eSPeter Xu }
1967dd4d607eSPeter Xu 
1968dd4d607eSPeter Xu static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s,
1969dd4d607eSPeter Xu                                            uint16_t domain_id, hwaddr addr,
1970dd4d607eSPeter Xu                                            uint8_t am)
1971dd4d607eSPeter Xu {
1972b4a4ba0dSPeter Xu     VTDAddressSpace *vtd_as;
1973dd4d607eSPeter Xu     VTDContextEntry ce;
1974dd4d607eSPeter Xu     int ret;
19754f8a62a9SPeter Xu     hwaddr size = (1 << am) * VTD_PAGE_SIZE;
1976dd4d607eSPeter Xu 
1977b4a4ba0dSPeter Xu     QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) {
1978dd4d607eSPeter Xu         ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus),
1979dd4d607eSPeter Xu                                        vtd_as->devfn, &ce);
1980fb43cf73SLiu, Yi L         if (!ret && domain_id == vtd_get_domain_id(s, &ce)) {
19814f8a62a9SPeter Xu             if (vtd_as_has_map_notifier(vtd_as)) {
19824f8a62a9SPeter Xu                 /*
19834f8a62a9SPeter Xu                  * As long as we have MAP notifications registered in
19844f8a62a9SPeter Xu                  * any of our IOMMU notifiers, we need to sync the
19854f8a62a9SPeter Xu                  * shadow page table.
19864f8a62a9SPeter Xu                  */
198763b88968SPeter Xu                 vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size);
19884f8a62a9SPeter Xu             } else {
19894f8a62a9SPeter Xu                 /*
19904f8a62a9SPeter Xu                  * For UNMAP-only notifiers, we don't need to walk the
19914f8a62a9SPeter Xu                  * page tables.  We just deliver the PSI down to
19924f8a62a9SPeter Xu                  * invalidate caches.
19934f8a62a9SPeter Xu                  */
19944f8a62a9SPeter Xu                 IOMMUTLBEntry entry = {
19954f8a62a9SPeter Xu                     .target_as = &address_space_memory,
19964f8a62a9SPeter Xu                     .iova = addr,
19974f8a62a9SPeter Xu                     .translated_addr = 0,
19984f8a62a9SPeter Xu                     .addr_mask = size - 1,
19994f8a62a9SPeter Xu                     .perm = IOMMU_NONE,
20004f8a62a9SPeter Xu                 };
2001cb1efcf4SPeter Maydell                 memory_region_notify_iommu(&vtd_as->iommu, 0, entry);
20024f8a62a9SPeter Xu             }
2003dd4d607eSPeter Xu         }
2004dd4d607eSPeter Xu     }
2005b5a280c0SLe Tan }
2006b5a280c0SLe Tan 
2007b5a280c0SLe Tan static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
2008b5a280c0SLe Tan                                       hwaddr addr, uint8_t am)
2009b5a280c0SLe Tan {
2010b5a280c0SLe Tan     VTDIOTLBPageInvInfo info;
2011b5a280c0SLe Tan 
20127feb51b7SPeter Xu     trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am);
20137feb51b7SPeter Xu 
2014b5a280c0SLe Tan     assert(am <= VTD_MAMV);
2015b5a280c0SLe Tan     info.domain_id = domain_id;
2016d66b969bSJason Wang     info.addr = addr;
2017b5a280c0SLe Tan     info.mask = ~((1 << am) - 1);
20181d9efa73SPeter Xu     vtd_iommu_lock(s);
2019b5a280c0SLe Tan     g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
20201d9efa73SPeter Xu     vtd_iommu_unlock(s);
2021dd4d607eSPeter Xu     vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am);
2022b5a280c0SLe Tan }
2023b5a280c0SLe Tan 
20241da12ec4SLe Tan /* Flush IOTLB
20251da12ec4SLe Tan  * Returns the IOTLB Actual Invalidation Granularity.
20261da12ec4SLe Tan  * @val: the content of the IOTLB_REG
20271da12ec4SLe Tan  */
20281da12ec4SLe Tan static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val)
20291da12ec4SLe Tan {
20301da12ec4SLe Tan     uint64_t iaig;
20311da12ec4SLe Tan     uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK;
2032b5a280c0SLe Tan     uint16_t domain_id;
2033b5a280c0SLe Tan     hwaddr addr;
2034b5a280c0SLe Tan     uint8_t am;
20351da12ec4SLe Tan 
20361da12ec4SLe Tan     switch (type) {
20371da12ec4SLe Tan     case VTD_TLB_GLOBAL_FLUSH:
20381da12ec4SLe Tan         iaig = VTD_TLB_GLOBAL_FLUSH_A;
2039b5a280c0SLe Tan         vtd_iotlb_global_invalidate(s);
20401da12ec4SLe Tan         break;
20411da12ec4SLe Tan 
20421da12ec4SLe Tan     case VTD_TLB_DSI_FLUSH:
2043b5a280c0SLe Tan         domain_id = VTD_TLB_DID(val);
20441da12ec4SLe Tan         iaig = VTD_TLB_DSI_FLUSH_A;
2045b5a280c0SLe Tan         vtd_iotlb_domain_invalidate(s, domain_id);
20461da12ec4SLe Tan         break;
20471da12ec4SLe Tan 
20481da12ec4SLe Tan     case VTD_TLB_PSI_FLUSH:
2049b5a280c0SLe Tan         domain_id = VTD_TLB_DID(val);
2050b5a280c0SLe Tan         addr = vtd_get_quad_raw(s, DMAR_IVA_REG);
2051b5a280c0SLe Tan         am = VTD_IVA_AM(addr);
2052b5a280c0SLe Tan         addr = VTD_IVA_ADDR(addr);
2053b5a280c0SLe Tan         if (am > VTD_MAMV) {
20541376211fSPeter Xu             error_report_once("%s: address mask overflow: 0x%" PRIx64,
20551376211fSPeter Xu                               __func__, vtd_get_quad_raw(s, DMAR_IVA_REG));
2056b5a280c0SLe Tan             iaig = 0;
2057b5a280c0SLe Tan             break;
2058b5a280c0SLe Tan         }
20591da12ec4SLe Tan         iaig = VTD_TLB_PSI_FLUSH_A;
2060b5a280c0SLe Tan         vtd_iotlb_page_invalidate(s, domain_id, addr, am);
20611da12ec4SLe Tan         break;
20621da12ec4SLe Tan 
20631da12ec4SLe Tan     default:
20641376211fSPeter Xu         error_report_once("%s: invalid granularity: 0x%" PRIx64,
20651376211fSPeter Xu                           __func__, val);
20661da12ec4SLe Tan         iaig = 0;
20671da12ec4SLe Tan     }
20681da12ec4SLe Tan     return iaig;
20691da12ec4SLe Tan }
20701da12ec4SLe Tan 
20718991c460SLadi Prosek static void vtd_fetch_inv_desc(IntelIOMMUState *s);
2072ed7b8fbcSLe Tan 
2073ed7b8fbcSLe Tan static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s)
2074ed7b8fbcSLe Tan {
2075ed7b8fbcSLe Tan     return s->qi_enabled && (s->iq_tail == s->iq_head) &&
2076ed7b8fbcSLe Tan            (s->iq_last_desc_type == VTD_INV_DESC_WAIT);
2077ed7b8fbcSLe Tan }
2078ed7b8fbcSLe Tan 
2079ed7b8fbcSLe Tan static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en)
2080ed7b8fbcSLe Tan {
2081ed7b8fbcSLe Tan     uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG);
2082ed7b8fbcSLe Tan 
20837feb51b7SPeter Xu     trace_vtd_inv_qi_enable(en);
20847feb51b7SPeter Xu 
2085ed7b8fbcSLe Tan     if (en) {
208637f51384SPrasad Singamsetty         s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits);
2087ed7b8fbcSLe Tan         /* 2^(x+8) entries */
2088c0c1d351SLiu, Yi L         s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8 - (s->iq_dw ? 1 : 0));
2089ed7b8fbcSLe Tan         s->qi_enabled = true;
20907feb51b7SPeter Xu         trace_vtd_inv_qi_setup(s->iq, s->iq_size);
2091ed7b8fbcSLe Tan         /* Ok - report back to driver */
2092ed7b8fbcSLe Tan         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES);
20938991c460SLadi Prosek 
20948991c460SLadi Prosek         if (s->iq_tail != 0) {
20958991c460SLadi Prosek             /*
20968991c460SLadi Prosek              * This is a spec violation but Windows guests are known to set up
20978991c460SLadi Prosek              * Queued Invalidation this way so we allow the write and process
20988991c460SLadi Prosek              * Invalidation Descriptors right away.
20998991c460SLadi Prosek              */
21008991c460SLadi Prosek             trace_vtd_warn_invalid_qi_tail(s->iq_tail);
21018991c460SLadi Prosek             if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
21028991c460SLadi Prosek                 vtd_fetch_inv_desc(s);
21038991c460SLadi Prosek             }
2104ed7b8fbcSLe Tan         }
2105ed7b8fbcSLe Tan     } else {
2106ed7b8fbcSLe Tan         if (vtd_queued_inv_disable_check(s)) {
2107ed7b8fbcSLe Tan             /* disable Queued Invalidation */
2108ed7b8fbcSLe Tan             vtd_set_quad_raw(s, DMAR_IQH_REG, 0);
2109ed7b8fbcSLe Tan             s->iq_head = 0;
2110ed7b8fbcSLe Tan             s->qi_enabled = false;
2111ed7b8fbcSLe Tan             /* Ok - report back to driver */
2112ed7b8fbcSLe Tan             vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0);
2113ed7b8fbcSLe Tan         } else {
21144e4abd11SPeter Xu             error_report_once("%s: detected improper state when disable QI "
21154e4abd11SPeter Xu                               "(head=0x%x, tail=0x%x, last_type=%d)",
21164e4abd11SPeter Xu                               __func__,
21174e4abd11SPeter Xu                               s->iq_head, s->iq_tail, s->iq_last_desc_type);
2118ed7b8fbcSLe Tan         }
2119ed7b8fbcSLe Tan     }
2120ed7b8fbcSLe Tan }
2121ed7b8fbcSLe Tan 
21221da12ec4SLe Tan /* Set Root Table Pointer */
21231da12ec4SLe Tan static void vtd_handle_gcmd_srtp(IntelIOMMUState *s)
21241da12ec4SLe Tan {
21251da12ec4SLe Tan     vtd_root_table_setup(s);
21261da12ec4SLe Tan     /* Ok - report back to driver */
21271da12ec4SLe Tan     vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
21282cc9ddccSPeter Xu     vtd_reset_caches(s);
21292cc9ddccSPeter Xu     vtd_address_space_refresh_all(s);
21301da12ec4SLe Tan }
21311da12ec4SLe Tan 
2132a5861439SPeter Xu /* Set Interrupt Remap Table Pointer */
2133a5861439SPeter Xu static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s)
2134a5861439SPeter Xu {
2135a5861439SPeter Xu     vtd_interrupt_remap_table_setup(s);
2136a5861439SPeter Xu     /* Ok - report back to driver */
2137a5861439SPeter Xu     vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS);
2138a5861439SPeter Xu }
2139a5861439SPeter Xu 
21401da12ec4SLe Tan /* Handle Translation Enable/Disable */
21411da12ec4SLe Tan static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
21421da12ec4SLe Tan {
2143558e0024SPeter Xu     if (s->dmar_enabled == en) {
2144558e0024SPeter Xu         return;
2145558e0024SPeter Xu     }
2146558e0024SPeter Xu 
21477feb51b7SPeter Xu     trace_vtd_dmar_enable(en);
21481da12ec4SLe Tan 
21491da12ec4SLe Tan     if (en) {
21501da12ec4SLe Tan         s->dmar_enabled = true;
21511da12ec4SLe Tan         /* Ok - report back to driver */
21521da12ec4SLe Tan         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES);
21531da12ec4SLe Tan     } else {
21541da12ec4SLe Tan         s->dmar_enabled = false;
21551da12ec4SLe Tan 
21561da12ec4SLe Tan         /* Clear the index of Fault Recording Register */
21571da12ec4SLe Tan         s->next_frcd_reg = 0;
21581da12ec4SLe Tan         /* Ok - report back to driver */
21591da12ec4SLe Tan         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0);
21601da12ec4SLe Tan     }
2161558e0024SPeter Xu 
21622cc9ddccSPeter Xu     vtd_reset_caches(s);
21632cc9ddccSPeter Xu     vtd_address_space_refresh_all(s);
21641da12ec4SLe Tan }
21651da12ec4SLe Tan 
216680de52baSPeter Xu /* Handle Interrupt Remap Enable/Disable */
216780de52baSPeter Xu static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en)
216880de52baSPeter Xu {
21697feb51b7SPeter Xu     trace_vtd_ir_enable(en);
217080de52baSPeter Xu 
217180de52baSPeter Xu     if (en) {
217280de52baSPeter Xu         s->intr_enabled = true;
217380de52baSPeter Xu         /* Ok - report back to driver */
217480de52baSPeter Xu         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES);
217580de52baSPeter Xu     } else {
217680de52baSPeter Xu         s->intr_enabled = false;
217780de52baSPeter Xu         /* Ok - report back to driver */
217880de52baSPeter Xu         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0);
217980de52baSPeter Xu     }
218080de52baSPeter Xu }
218180de52baSPeter Xu 
21821da12ec4SLe Tan /* Handle write to Global Command Register */
21831da12ec4SLe Tan static void vtd_handle_gcmd_write(IntelIOMMUState *s)
21841da12ec4SLe Tan {
21851da12ec4SLe Tan     uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG);
21861da12ec4SLe Tan     uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG);
21871da12ec4SLe Tan     uint32_t changed = status ^ val;
21881da12ec4SLe Tan 
21897feb51b7SPeter Xu     trace_vtd_reg_write_gcmd(status, val);
21901da12ec4SLe Tan     if (changed & VTD_GCMD_TE) {
21911da12ec4SLe Tan         /* Translation enable/disable */
21921da12ec4SLe Tan         vtd_handle_gcmd_te(s, val & VTD_GCMD_TE);
21931da12ec4SLe Tan     }
21941da12ec4SLe Tan     if (val & VTD_GCMD_SRTP) {
21951da12ec4SLe Tan         /* Set/update the root-table pointer */
21961da12ec4SLe Tan         vtd_handle_gcmd_srtp(s);
21971da12ec4SLe Tan     }
2198ed7b8fbcSLe Tan     if (changed & VTD_GCMD_QIE) {
2199ed7b8fbcSLe Tan         /* Queued Invalidation Enable */
2200ed7b8fbcSLe Tan         vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
2201ed7b8fbcSLe Tan     }
2202a5861439SPeter Xu     if (val & VTD_GCMD_SIRTP) {
2203a5861439SPeter Xu         /* Set/update the interrupt remapping root-table pointer */
2204a5861439SPeter Xu         vtd_handle_gcmd_sirtp(s);
2205a5861439SPeter Xu     }
220680de52baSPeter Xu     if (changed & VTD_GCMD_IRE) {
220780de52baSPeter Xu         /* Interrupt remap enable/disable */
220880de52baSPeter Xu         vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE);
220980de52baSPeter Xu     }
22101da12ec4SLe Tan }
22111da12ec4SLe Tan 
22121da12ec4SLe Tan /* Handle write to Context Command Register */
22131da12ec4SLe Tan static void vtd_handle_ccmd_write(IntelIOMMUState *s)
22141da12ec4SLe Tan {
22151da12ec4SLe Tan     uint64_t ret;
22161da12ec4SLe Tan     uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG);
22171da12ec4SLe Tan 
22181da12ec4SLe Tan     /* Context-cache invalidation request */
22191da12ec4SLe Tan     if (val & VTD_CCMD_ICC) {
2220ed7b8fbcSLe Tan         if (s->qi_enabled) {
22211376211fSPeter Xu             error_report_once("Queued Invalidation enabled, "
2222ed7b8fbcSLe Tan                               "should not use register-based invalidation");
2223ed7b8fbcSLe Tan             return;
2224ed7b8fbcSLe Tan         }
22251da12ec4SLe Tan         ret = vtd_context_cache_invalidate(s, val);
22261da12ec4SLe Tan         /* Invalidation completed. Change something to show */
22271da12ec4SLe Tan         vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL);
22281da12ec4SLe Tan         ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK,
22291da12ec4SLe Tan                                       ret);
22301da12ec4SLe Tan     }
22311da12ec4SLe Tan }
22321da12ec4SLe Tan 
22331da12ec4SLe Tan /* Handle write to IOTLB Invalidation Register */
22341da12ec4SLe Tan static void vtd_handle_iotlb_write(IntelIOMMUState *s)
22351da12ec4SLe Tan {
22361da12ec4SLe Tan     uint64_t ret;
22371da12ec4SLe Tan     uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG);
22381da12ec4SLe Tan 
22391da12ec4SLe Tan     /* IOTLB invalidation request */
22401da12ec4SLe Tan     if (val & VTD_TLB_IVT) {
2241ed7b8fbcSLe Tan         if (s->qi_enabled) {
22421376211fSPeter Xu             error_report_once("Queued Invalidation enabled, "
22431376211fSPeter Xu                               "should not use register-based invalidation");
2244ed7b8fbcSLe Tan             return;
2245ed7b8fbcSLe Tan         }
22461da12ec4SLe Tan         ret = vtd_iotlb_flush(s, val);
22471da12ec4SLe Tan         /* Invalidation completed. Change something to show */
22481da12ec4SLe Tan         vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL);
22491da12ec4SLe Tan         ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG,
22501da12ec4SLe Tan                                       VTD_TLB_FLUSH_GRANU_MASK_A, ret);
22511da12ec4SLe Tan     }
22521da12ec4SLe Tan }
22531da12ec4SLe Tan 
2254ed7b8fbcSLe Tan /* Fetch an Invalidation Descriptor from the Invalidation Queue */
2255c0c1d351SLiu, Yi L static bool vtd_get_inv_desc(IntelIOMMUState *s,
2256ed7b8fbcSLe Tan                              VTDInvDesc *inv_desc)
2257ed7b8fbcSLe Tan {
2258c0c1d351SLiu, Yi L     dma_addr_t base_addr = s->iq;
2259c0c1d351SLiu, Yi L     uint32_t offset = s->iq_head;
2260c0c1d351SLiu, Yi L     uint32_t dw = s->iq_dw ? 32 : 16;
2261c0c1d351SLiu, Yi L     dma_addr_t addr = base_addr + offset * dw;
2262c0c1d351SLiu, Yi L 
2263c0c1d351SLiu, Yi L     if (dma_memory_read(&address_space_memory, addr, inv_desc, dw)) {
2264c0c1d351SLiu, Yi L         error_report_once("Read INV DESC failed.");
2265ed7b8fbcSLe Tan         return false;
2266ed7b8fbcSLe Tan     }
2267ed7b8fbcSLe Tan     inv_desc->lo = le64_to_cpu(inv_desc->lo);
2268ed7b8fbcSLe Tan     inv_desc->hi = le64_to_cpu(inv_desc->hi);
2269c0c1d351SLiu, Yi L     if (dw == 32) {
2270c0c1d351SLiu, Yi L         inv_desc->val[2] = le64_to_cpu(inv_desc->val[2]);
2271c0c1d351SLiu, Yi L         inv_desc->val[3] = le64_to_cpu(inv_desc->val[3]);
2272c0c1d351SLiu, Yi L     }
2273ed7b8fbcSLe Tan     return true;
2274ed7b8fbcSLe Tan }
2275ed7b8fbcSLe Tan 
2276ed7b8fbcSLe Tan static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
2277ed7b8fbcSLe Tan {
2278ed7b8fbcSLe Tan     if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) ||
2279ed7b8fbcSLe Tan         (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) {
2280095955b2SPeter Xu         error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64
2281095955b2SPeter Xu                           " (reserved nonzero)", __func__, inv_desc->hi,
2282095955b2SPeter Xu                           inv_desc->lo);
2283ed7b8fbcSLe Tan         return false;
2284ed7b8fbcSLe Tan     }
2285ed7b8fbcSLe Tan     if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) {
2286ed7b8fbcSLe Tan         /* Status Write */
2287ed7b8fbcSLe Tan         uint32_t status_data = (uint32_t)(inv_desc->lo >>
2288ed7b8fbcSLe Tan                                VTD_INV_DESC_WAIT_DATA_SHIFT);
2289ed7b8fbcSLe Tan 
2290ed7b8fbcSLe Tan         assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF));
2291ed7b8fbcSLe Tan 
2292ed7b8fbcSLe Tan         /* FIXME: need to be masked with HAW? */
2293ed7b8fbcSLe Tan         dma_addr_t status_addr = inv_desc->hi;
2294bc535e59SPeter Xu         trace_vtd_inv_desc_wait_sw(status_addr, status_data);
2295ed7b8fbcSLe Tan         status_data = cpu_to_le32(status_data);
2296ed7b8fbcSLe Tan         if (dma_memory_write(&address_space_memory, status_addr, &status_data,
2297ed7b8fbcSLe Tan                              sizeof(status_data))) {
2298bc535e59SPeter Xu             trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo);
2299ed7b8fbcSLe Tan             return false;
2300ed7b8fbcSLe Tan         }
2301ed7b8fbcSLe Tan     } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) {
2302ed7b8fbcSLe Tan         /* Interrupt flag */
2303ed7b8fbcSLe Tan         vtd_generate_completion_event(s);
2304ed7b8fbcSLe Tan     } else {
2305095955b2SPeter Xu         error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64
2306095955b2SPeter Xu                           " (unknown type)", __func__, inv_desc->hi,
2307095955b2SPeter Xu                           inv_desc->lo);
2308ed7b8fbcSLe Tan         return false;
2309ed7b8fbcSLe Tan     }
2310ed7b8fbcSLe Tan     return true;
2311ed7b8fbcSLe Tan }
2312ed7b8fbcSLe Tan 
2313d92fa2dcSLe Tan static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
2314d92fa2dcSLe Tan                                            VTDInvDesc *inv_desc)
2315d92fa2dcSLe Tan {
2316bc535e59SPeter Xu     uint16_t sid, fmask;
2317bc535e59SPeter Xu 
2318d92fa2dcSLe Tan     if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {
2319095955b2SPeter Xu         error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64
2320095955b2SPeter Xu                           " (reserved nonzero)", __func__, inv_desc->hi,
2321095955b2SPeter Xu                           inv_desc->lo);
2322d92fa2dcSLe Tan         return false;
2323d92fa2dcSLe Tan     }
2324d92fa2dcSLe Tan     switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
2325d92fa2dcSLe Tan     case VTD_INV_DESC_CC_DOMAIN:
2326bc535e59SPeter Xu         trace_vtd_inv_desc_cc_domain(
2327d92fa2dcSLe Tan             (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
2328d92fa2dcSLe Tan         /* Fall through */
2329d92fa2dcSLe Tan     case VTD_INV_DESC_CC_GLOBAL:
2330d92fa2dcSLe Tan         vtd_context_global_invalidate(s);
2331d92fa2dcSLe Tan         break;
2332d92fa2dcSLe Tan 
2333d92fa2dcSLe Tan     case VTD_INV_DESC_CC_DEVICE:
2334bc535e59SPeter Xu         sid = VTD_INV_DESC_CC_SID(inv_desc->lo);
2335bc535e59SPeter Xu         fmask = VTD_INV_DESC_CC_FM(inv_desc->lo);
2336bc535e59SPeter Xu         vtd_context_device_invalidate(s, sid, fmask);
2337d92fa2dcSLe Tan         break;
2338d92fa2dcSLe Tan 
2339d92fa2dcSLe Tan     default:
2340095955b2SPeter Xu         error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64
2341095955b2SPeter Xu                           " (invalid type)", __func__, inv_desc->hi,
2342095955b2SPeter Xu                           inv_desc->lo);
2343d92fa2dcSLe Tan         return false;
2344d92fa2dcSLe Tan     }
2345d92fa2dcSLe Tan     return true;
2346d92fa2dcSLe Tan }
2347d92fa2dcSLe Tan 
2348b5a280c0SLe Tan static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
2349b5a280c0SLe Tan {
2350b5a280c0SLe Tan     uint16_t domain_id;
2351b5a280c0SLe Tan     uint8_t am;
2352b5a280c0SLe Tan     hwaddr addr;
2353b5a280c0SLe Tan 
2354b5a280c0SLe Tan     if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) ||
2355b5a280c0SLe Tan         (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) {
2356095955b2SPeter Xu         error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64
2357095955b2SPeter Xu                           ", lo=0x%"PRIx64" (reserved bits unzero)\n",
2358095955b2SPeter Xu                           __func__, inv_desc->hi, inv_desc->lo);
2359b5a280c0SLe Tan         return false;
2360b5a280c0SLe Tan     }
2361b5a280c0SLe Tan 
2362b5a280c0SLe Tan     switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) {
2363b5a280c0SLe Tan     case VTD_INV_DESC_IOTLB_GLOBAL:
2364b5a280c0SLe Tan         vtd_iotlb_global_invalidate(s);
2365b5a280c0SLe Tan         break;
2366b5a280c0SLe Tan 
2367b5a280c0SLe Tan     case VTD_INV_DESC_IOTLB_DOMAIN:
2368b5a280c0SLe Tan         domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
2369b5a280c0SLe Tan         vtd_iotlb_domain_invalidate(s, domain_id);
2370b5a280c0SLe Tan         break;
2371b5a280c0SLe Tan 
2372b5a280c0SLe Tan     case VTD_INV_DESC_IOTLB_PAGE:
2373b5a280c0SLe Tan         domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
2374b5a280c0SLe Tan         addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi);
2375b5a280c0SLe Tan         am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi);
2376b5a280c0SLe Tan         if (am > VTD_MAMV) {
2377095955b2SPeter Xu             error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64
2378095955b2SPeter Xu                               ", lo=0x%"PRIx64" (am=%u > VTD_MAMV=%u)\n",
2379095955b2SPeter Xu                               __func__, inv_desc->hi, inv_desc->lo,
2380095955b2SPeter Xu                               am, (unsigned)VTD_MAMV);
2381b5a280c0SLe Tan             return false;
2382b5a280c0SLe Tan         }
2383b5a280c0SLe Tan         vtd_iotlb_page_invalidate(s, domain_id, addr, am);
2384b5a280c0SLe Tan         break;
2385b5a280c0SLe Tan 
2386b5a280c0SLe Tan     default:
2387095955b2SPeter Xu         error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64
2388095955b2SPeter Xu                           ", lo=0x%"PRIx64" (type mismatch: 0x%llx)\n",
2389095955b2SPeter Xu                           __func__, inv_desc->hi, inv_desc->lo,
2390095955b2SPeter Xu                           inv_desc->lo & VTD_INV_DESC_IOTLB_G);
2391b5a280c0SLe Tan         return false;
2392b5a280c0SLe Tan     }
2393b5a280c0SLe Tan     return true;
2394b5a280c0SLe Tan }
2395b5a280c0SLe Tan 
239602a2cbc8SPeter Xu static bool vtd_process_inv_iec_desc(IntelIOMMUState *s,
239702a2cbc8SPeter Xu                                      VTDInvDesc *inv_desc)
239802a2cbc8SPeter Xu {
23997feb51b7SPeter Xu     trace_vtd_inv_desc_iec(inv_desc->iec.granularity,
240002a2cbc8SPeter Xu                            inv_desc->iec.index,
240102a2cbc8SPeter Xu                            inv_desc->iec.index_mask);
240202a2cbc8SPeter Xu 
240302a2cbc8SPeter Xu     vtd_iec_notify_all(s, !inv_desc->iec.granularity,
240402a2cbc8SPeter Xu                        inv_desc->iec.index,
240502a2cbc8SPeter Xu                        inv_desc->iec.index_mask);
2406554f5e16SJason Wang     return true;
2407554f5e16SJason Wang }
240802a2cbc8SPeter Xu 
2409554f5e16SJason Wang static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s,
2410554f5e16SJason Wang                                           VTDInvDesc *inv_desc)
2411554f5e16SJason Wang {
2412554f5e16SJason Wang     VTDAddressSpace *vtd_dev_as;
2413554f5e16SJason Wang     IOMMUTLBEntry entry;
2414554f5e16SJason Wang     struct VTDBus *vtd_bus;
2415554f5e16SJason Wang     hwaddr addr;
2416554f5e16SJason Wang     uint64_t sz;
2417554f5e16SJason Wang     uint16_t sid;
2418554f5e16SJason Wang     uint8_t devfn;
2419554f5e16SJason Wang     bool size;
2420554f5e16SJason Wang     uint8_t bus_num;
2421554f5e16SJason Wang 
2422554f5e16SJason Wang     addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi);
2423554f5e16SJason Wang     sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo);
2424554f5e16SJason Wang     devfn = sid & 0xff;
2425554f5e16SJason Wang     bus_num = sid >> 8;
2426554f5e16SJason Wang     size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi);
2427554f5e16SJason Wang 
2428554f5e16SJason Wang     if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) ||
2429554f5e16SJason Wang         (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) {
2430095955b2SPeter Xu         error_report_once("%s: invalid dev-iotlb inv desc: hi=%"PRIx64
2431095955b2SPeter Xu                           ", lo=%"PRIx64" (reserved nonzero)", __func__,
2432095955b2SPeter Xu                           inv_desc->hi, inv_desc->lo);
2433554f5e16SJason Wang         return false;
2434554f5e16SJason Wang     }
2435554f5e16SJason Wang 
2436554f5e16SJason Wang     vtd_bus = vtd_find_as_from_bus_num(s, bus_num);
2437554f5e16SJason Wang     if (!vtd_bus) {
2438554f5e16SJason Wang         goto done;
2439554f5e16SJason Wang     }
2440554f5e16SJason Wang 
2441554f5e16SJason Wang     vtd_dev_as = vtd_bus->dev_as[devfn];
2442554f5e16SJason Wang     if (!vtd_dev_as) {
2443554f5e16SJason Wang         goto done;
2444554f5e16SJason Wang     }
2445554f5e16SJason Wang 
244604eb6247SJason Wang     /* According to ATS spec table 2.4:
244704eb6247SJason Wang      * S = 0, bits 15:12 = xxxx     range size: 4K
244804eb6247SJason Wang      * S = 1, bits 15:12 = xxx0     range size: 8K
244904eb6247SJason Wang      * S = 1, bits 15:12 = xx01     range size: 16K
245004eb6247SJason Wang      * S = 1, bits 15:12 = x011     range size: 32K
245104eb6247SJason Wang      * S = 1, bits 15:12 = 0111     range size: 64K
245204eb6247SJason Wang      * ...
245304eb6247SJason Wang      */
2454554f5e16SJason Wang     if (size) {
245504eb6247SJason Wang         sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT);
2456554f5e16SJason Wang         addr &= ~(sz - 1);
2457554f5e16SJason Wang     } else {
2458554f5e16SJason Wang         sz = VTD_PAGE_SIZE;
2459554f5e16SJason Wang     }
2460554f5e16SJason Wang 
2461554f5e16SJason Wang     entry.target_as = &vtd_dev_as->as;
2462554f5e16SJason Wang     entry.addr_mask = sz - 1;
2463554f5e16SJason Wang     entry.iova = addr;
2464554f5e16SJason Wang     entry.perm = IOMMU_NONE;
2465554f5e16SJason Wang     entry.translated_addr = 0;
2466cb1efcf4SPeter Maydell     memory_region_notify_iommu(&vtd_dev_as->iommu, 0, entry);
2467554f5e16SJason Wang 
2468554f5e16SJason Wang done:
246902a2cbc8SPeter Xu     return true;
247002a2cbc8SPeter Xu }
247102a2cbc8SPeter Xu 
2472ed7b8fbcSLe Tan static bool vtd_process_inv_desc(IntelIOMMUState *s)
2473ed7b8fbcSLe Tan {
2474ed7b8fbcSLe Tan     VTDInvDesc inv_desc;
2475ed7b8fbcSLe Tan     uint8_t desc_type;
2476ed7b8fbcSLe Tan 
24777feb51b7SPeter Xu     trace_vtd_inv_qi_head(s->iq_head);
2478c0c1d351SLiu, Yi L     if (!vtd_get_inv_desc(s, &inv_desc)) {
2479ed7b8fbcSLe Tan         s->iq_last_desc_type = VTD_INV_DESC_NONE;
2480ed7b8fbcSLe Tan         return false;
2481ed7b8fbcSLe Tan     }
2482c0c1d351SLiu, Yi L 
2483ed7b8fbcSLe Tan     desc_type = inv_desc.lo & VTD_INV_DESC_TYPE;
2484ed7b8fbcSLe Tan     /* FIXME: should update at first or at last? */
2485ed7b8fbcSLe Tan     s->iq_last_desc_type = desc_type;
2486ed7b8fbcSLe Tan 
2487ed7b8fbcSLe Tan     switch (desc_type) {
2488ed7b8fbcSLe Tan     case VTD_INV_DESC_CC:
2489bc535e59SPeter Xu         trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo);
2490d92fa2dcSLe Tan         if (!vtd_process_context_cache_desc(s, &inv_desc)) {
2491d92fa2dcSLe Tan             return false;
2492d92fa2dcSLe Tan         }
2493ed7b8fbcSLe Tan         break;
2494ed7b8fbcSLe Tan 
2495ed7b8fbcSLe Tan     case VTD_INV_DESC_IOTLB:
2496bc535e59SPeter Xu         trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo);
2497b5a280c0SLe Tan         if (!vtd_process_iotlb_desc(s, &inv_desc)) {
2498b5a280c0SLe Tan             return false;
2499b5a280c0SLe Tan         }
2500ed7b8fbcSLe Tan         break;
2501ed7b8fbcSLe Tan 
25024a4f219eSYi Sun     /*
25034a4f219eSYi Sun      * TODO: the entity of below two cases will be implemented in future series.
25044a4f219eSYi Sun      * To make guest (which integrates scalable mode support patch set in
25054a4f219eSYi Sun      * iommu driver) work, just return true is enough so far.
25064a4f219eSYi Sun      */
25074a4f219eSYi Sun     case VTD_INV_DESC_PC:
25084a4f219eSYi Sun         break;
25094a4f219eSYi Sun 
25104a4f219eSYi Sun     case VTD_INV_DESC_PIOTLB:
25114a4f219eSYi Sun         break;
25124a4f219eSYi Sun 
2513ed7b8fbcSLe Tan     case VTD_INV_DESC_WAIT:
2514bc535e59SPeter Xu         trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo);
2515ed7b8fbcSLe Tan         if (!vtd_process_wait_desc(s, &inv_desc)) {
2516ed7b8fbcSLe Tan             return false;
2517ed7b8fbcSLe Tan         }
2518ed7b8fbcSLe Tan         break;
2519ed7b8fbcSLe Tan 
2520b7910472SPeter Xu     case VTD_INV_DESC_IEC:
2521bc535e59SPeter Xu         trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo);
252202a2cbc8SPeter Xu         if (!vtd_process_inv_iec_desc(s, &inv_desc)) {
252302a2cbc8SPeter Xu             return false;
252402a2cbc8SPeter Xu         }
2525b7910472SPeter Xu         break;
2526b7910472SPeter Xu 
2527554f5e16SJason Wang     case VTD_INV_DESC_DEVICE:
25287feb51b7SPeter Xu         trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo);
2529554f5e16SJason Wang         if (!vtd_process_device_iotlb_desc(s, &inv_desc)) {
2530554f5e16SJason Wang             return false;
2531554f5e16SJason Wang         }
2532554f5e16SJason Wang         break;
2533554f5e16SJason Wang 
2534ed7b8fbcSLe Tan     default:
2535095955b2SPeter Xu         error_report_once("%s: invalid inv desc: hi=%"PRIx64", lo=%"PRIx64
2536095955b2SPeter Xu                           " (unknown type)", __func__, inv_desc.hi,
2537095955b2SPeter Xu                           inv_desc.lo);
2538ed7b8fbcSLe Tan         return false;
2539ed7b8fbcSLe Tan     }
2540ed7b8fbcSLe Tan     s->iq_head++;
2541ed7b8fbcSLe Tan     if (s->iq_head == s->iq_size) {
2542ed7b8fbcSLe Tan         s->iq_head = 0;
2543ed7b8fbcSLe Tan     }
2544ed7b8fbcSLe Tan     return true;
2545ed7b8fbcSLe Tan }
2546ed7b8fbcSLe Tan 
2547ed7b8fbcSLe Tan /* Try to fetch and process more Invalidation Descriptors */
2548ed7b8fbcSLe Tan static void vtd_fetch_inv_desc(IntelIOMMUState *s)
2549ed7b8fbcSLe Tan {
25507feb51b7SPeter Xu     trace_vtd_inv_qi_fetch();
25517feb51b7SPeter Xu 
2552ed7b8fbcSLe Tan     if (s->iq_tail >= s->iq_size) {
2553ed7b8fbcSLe Tan         /* Detects an invalid Tail pointer */
25544e4abd11SPeter Xu         error_report_once("%s: detected invalid QI tail "
25554e4abd11SPeter Xu                           "(tail=0x%x, size=0x%x)",
25564e4abd11SPeter Xu                           __func__, s->iq_tail, s->iq_size);
2557ed7b8fbcSLe Tan         vtd_handle_inv_queue_error(s);
2558ed7b8fbcSLe Tan         return;
2559ed7b8fbcSLe Tan     }
2560ed7b8fbcSLe Tan     while (s->iq_head != s->iq_tail) {
2561ed7b8fbcSLe Tan         if (!vtd_process_inv_desc(s)) {
2562ed7b8fbcSLe Tan             /* Invalidation Queue Errors */
2563ed7b8fbcSLe Tan             vtd_handle_inv_queue_error(s);
2564ed7b8fbcSLe Tan             break;
2565ed7b8fbcSLe Tan         }
2566ed7b8fbcSLe Tan         /* Must update the IQH_REG in time */
2567ed7b8fbcSLe Tan         vtd_set_quad_raw(s, DMAR_IQH_REG,
2568ed7b8fbcSLe Tan                          (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) &
2569ed7b8fbcSLe Tan                          VTD_IQH_QH_MASK);
2570ed7b8fbcSLe Tan     }
2571ed7b8fbcSLe Tan }
2572ed7b8fbcSLe Tan 
2573ed7b8fbcSLe Tan /* Handle write to Invalidation Queue Tail Register */
2574ed7b8fbcSLe Tan static void vtd_handle_iqt_write(IntelIOMMUState *s)
2575ed7b8fbcSLe Tan {
2576ed7b8fbcSLe Tan     uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG);
2577ed7b8fbcSLe Tan 
2578c0c1d351SLiu, Yi L     if (s->iq_dw && (val & VTD_IQT_QT_256_RSV_BIT)) {
2579c0c1d351SLiu, Yi L         error_report_once("%s: RSV bit is set: val=0x%"PRIx64,
2580c0c1d351SLiu, Yi L                           __func__, val);
2581c0c1d351SLiu, Yi L         return;
2582c0c1d351SLiu, Yi L     }
2583c0c1d351SLiu, Yi L     s->iq_tail = VTD_IQT_QT(s->iq_dw, val);
25847feb51b7SPeter Xu     trace_vtd_inv_qi_tail(s->iq_tail);
25857feb51b7SPeter Xu 
2586ed7b8fbcSLe Tan     if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
2587ed7b8fbcSLe Tan         /* Process Invalidation Queue here */
2588ed7b8fbcSLe Tan         vtd_fetch_inv_desc(s);
2589ed7b8fbcSLe Tan     }
2590ed7b8fbcSLe Tan }
2591ed7b8fbcSLe Tan 
25921da12ec4SLe Tan static void vtd_handle_fsts_write(IntelIOMMUState *s)
25931da12ec4SLe Tan {
25941da12ec4SLe Tan     uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
25951da12ec4SLe Tan     uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
25961da12ec4SLe Tan     uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE;
25971da12ec4SLe Tan 
25981da12ec4SLe Tan     if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) {
25991da12ec4SLe Tan         vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
26007feb51b7SPeter Xu         trace_vtd_fsts_clear_ip();
26011da12ec4SLe Tan     }
2602ed7b8fbcSLe Tan     /* FIXME: when IQE is Clear, should we try to fetch some Invalidation
2603ed7b8fbcSLe Tan      * Descriptors if there are any when Queued Invalidation is enabled?
2604ed7b8fbcSLe Tan      */
26051da12ec4SLe Tan }
26061da12ec4SLe Tan 
26071da12ec4SLe Tan static void vtd_handle_fectl_write(IntelIOMMUState *s)
26081da12ec4SLe Tan {
26091da12ec4SLe Tan     uint32_t fectl_reg;
26101da12ec4SLe Tan     /* FIXME: when software clears the IM field, check the IP field. But do we
26111da12ec4SLe Tan      * need to compare the old value and the new value to conclude that
26121da12ec4SLe Tan      * software clears the IM field? Or just check if the IM field is zero?
26131da12ec4SLe Tan      */
26141da12ec4SLe Tan     fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
26157feb51b7SPeter Xu 
26167feb51b7SPeter Xu     trace_vtd_reg_write_fectl(fectl_reg);
26177feb51b7SPeter Xu 
26181da12ec4SLe Tan     if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) {
26191da12ec4SLe Tan         vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
26201da12ec4SLe Tan         vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
26211da12ec4SLe Tan     }
26221da12ec4SLe Tan }
26231da12ec4SLe Tan 
2624ed7b8fbcSLe Tan static void vtd_handle_ics_write(IntelIOMMUState *s)
2625ed7b8fbcSLe Tan {
2626ed7b8fbcSLe Tan     uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG);
2627ed7b8fbcSLe Tan     uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
2628ed7b8fbcSLe Tan 
2629ed7b8fbcSLe Tan     if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) {
26307feb51b7SPeter Xu         trace_vtd_reg_ics_clear_ip();
2631ed7b8fbcSLe Tan         vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
2632ed7b8fbcSLe Tan     }
2633ed7b8fbcSLe Tan }
2634ed7b8fbcSLe Tan 
2635ed7b8fbcSLe Tan static void vtd_handle_iectl_write(IntelIOMMUState *s)
2636ed7b8fbcSLe Tan {
2637ed7b8fbcSLe Tan     uint32_t iectl_reg;
2638ed7b8fbcSLe Tan     /* FIXME: when software clears the IM field, check the IP field. But do we
2639ed7b8fbcSLe Tan      * need to compare the old value and the new value to conclude that
2640ed7b8fbcSLe Tan      * software clears the IM field? Or just check if the IM field is zero?
2641ed7b8fbcSLe Tan      */
2642ed7b8fbcSLe Tan     iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
26437feb51b7SPeter Xu 
26447feb51b7SPeter Xu     trace_vtd_reg_write_iectl(iectl_reg);
26457feb51b7SPeter Xu 
2646ed7b8fbcSLe Tan     if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) {
2647ed7b8fbcSLe Tan         vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
2648ed7b8fbcSLe Tan         vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
2649ed7b8fbcSLe Tan     }
2650ed7b8fbcSLe Tan }
2651ed7b8fbcSLe Tan 
26521da12ec4SLe Tan static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size)
26531da12ec4SLe Tan {
26541da12ec4SLe Tan     IntelIOMMUState *s = opaque;
26551da12ec4SLe Tan     uint64_t val;
26561da12ec4SLe Tan 
26577feb51b7SPeter Xu     trace_vtd_reg_read(addr, size);
26587feb51b7SPeter Xu 
26591da12ec4SLe Tan     if (addr + size > DMAR_REG_SIZE) {
26601376211fSPeter Xu         error_report_once("%s: MMIO over range: addr=0x%" PRIx64
26611376211fSPeter Xu                           " size=0x%u", __func__, addr, size);
26621da12ec4SLe Tan         return (uint64_t)-1;
26631da12ec4SLe Tan     }
26641da12ec4SLe Tan 
26651da12ec4SLe Tan     switch (addr) {
26661da12ec4SLe Tan     /* Root Table Address Register, 64-bit */
26671da12ec4SLe Tan     case DMAR_RTADDR_REG:
26688fdee711SYi Sun         val = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
26691da12ec4SLe Tan         if (size == 4) {
26708fdee711SYi Sun             val = val & ((1ULL << 32) - 1);
26711da12ec4SLe Tan         }
26721da12ec4SLe Tan         break;
26731da12ec4SLe Tan 
26741da12ec4SLe Tan     case DMAR_RTADDR_REG_HI:
26751da12ec4SLe Tan         assert(size == 4);
26768fdee711SYi Sun         val = vtd_get_quad_raw(s, DMAR_RTADDR_REG) >> 32;
26771da12ec4SLe Tan         break;
26781da12ec4SLe Tan 
2679ed7b8fbcSLe Tan     /* Invalidation Queue Address Register, 64-bit */
2680ed7b8fbcSLe Tan     case DMAR_IQA_REG:
2681ed7b8fbcSLe Tan         val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS);
2682ed7b8fbcSLe Tan         if (size == 4) {
2683ed7b8fbcSLe Tan             val = val & ((1ULL << 32) - 1);
2684ed7b8fbcSLe Tan         }
2685ed7b8fbcSLe Tan         break;
2686ed7b8fbcSLe Tan 
2687ed7b8fbcSLe Tan     case DMAR_IQA_REG_HI:
2688ed7b8fbcSLe Tan         assert(size == 4);
2689ed7b8fbcSLe Tan         val = s->iq >> 32;
2690ed7b8fbcSLe Tan         break;
2691ed7b8fbcSLe Tan 
26921da12ec4SLe Tan     default:
26931da12ec4SLe Tan         if (size == 4) {
26941da12ec4SLe Tan             val = vtd_get_long(s, addr);
26951da12ec4SLe Tan         } else {
26961da12ec4SLe Tan             val = vtd_get_quad(s, addr);
26971da12ec4SLe Tan         }
26981da12ec4SLe Tan     }
26997feb51b7SPeter Xu 
27001da12ec4SLe Tan     return val;
27011da12ec4SLe Tan }
27021da12ec4SLe Tan 
27031da12ec4SLe Tan static void vtd_mem_write(void *opaque, hwaddr addr,
27041da12ec4SLe Tan                           uint64_t val, unsigned size)
27051da12ec4SLe Tan {
27061da12ec4SLe Tan     IntelIOMMUState *s = opaque;
27071da12ec4SLe Tan 
27087feb51b7SPeter Xu     trace_vtd_reg_write(addr, size, val);
27097feb51b7SPeter Xu 
27101da12ec4SLe Tan     if (addr + size > DMAR_REG_SIZE) {
27111376211fSPeter Xu         error_report_once("%s: MMIO over range: addr=0x%" PRIx64
27121376211fSPeter Xu                           " size=0x%u", __func__, addr, size);
27131da12ec4SLe Tan         return;
27141da12ec4SLe Tan     }
27151da12ec4SLe Tan 
27161da12ec4SLe Tan     switch (addr) {
27171da12ec4SLe Tan     /* Global Command Register, 32-bit */
27181da12ec4SLe Tan     case DMAR_GCMD_REG:
27191da12ec4SLe Tan         vtd_set_long(s, addr, val);
27201da12ec4SLe Tan         vtd_handle_gcmd_write(s);
27211da12ec4SLe Tan         break;
27221da12ec4SLe Tan 
27231da12ec4SLe Tan     /* Context Command Register, 64-bit */
27241da12ec4SLe Tan     case DMAR_CCMD_REG:
27251da12ec4SLe Tan         if (size == 4) {
27261da12ec4SLe Tan             vtd_set_long(s, addr, val);
27271da12ec4SLe Tan         } else {
27281da12ec4SLe Tan             vtd_set_quad(s, addr, val);
27291da12ec4SLe Tan             vtd_handle_ccmd_write(s);
27301da12ec4SLe Tan         }
27311da12ec4SLe Tan         break;
27321da12ec4SLe Tan 
27331da12ec4SLe Tan     case DMAR_CCMD_REG_HI:
27341da12ec4SLe Tan         assert(size == 4);
27351da12ec4SLe Tan         vtd_set_long(s, addr, val);
27361da12ec4SLe Tan         vtd_handle_ccmd_write(s);
27371da12ec4SLe Tan         break;
27381da12ec4SLe Tan 
27391da12ec4SLe Tan     /* IOTLB Invalidation Register, 64-bit */
27401da12ec4SLe Tan     case DMAR_IOTLB_REG:
27411da12ec4SLe Tan         if (size == 4) {
27421da12ec4SLe Tan             vtd_set_long(s, addr, val);
27431da12ec4SLe Tan         } else {
27441da12ec4SLe Tan             vtd_set_quad(s, addr, val);
27451da12ec4SLe Tan             vtd_handle_iotlb_write(s);
27461da12ec4SLe Tan         }
27471da12ec4SLe Tan         break;
27481da12ec4SLe Tan 
27491da12ec4SLe Tan     case DMAR_IOTLB_REG_HI:
27501da12ec4SLe Tan         assert(size == 4);
27511da12ec4SLe Tan         vtd_set_long(s, addr, val);
27521da12ec4SLe Tan         vtd_handle_iotlb_write(s);
27531da12ec4SLe Tan         break;
27541da12ec4SLe Tan 
2755b5a280c0SLe Tan     /* Invalidate Address Register, 64-bit */
2756b5a280c0SLe Tan     case DMAR_IVA_REG:
2757b5a280c0SLe Tan         if (size == 4) {
2758b5a280c0SLe Tan             vtd_set_long(s, addr, val);
2759b5a280c0SLe Tan         } else {
2760b5a280c0SLe Tan             vtd_set_quad(s, addr, val);
2761b5a280c0SLe Tan         }
2762b5a280c0SLe Tan         break;
2763b5a280c0SLe Tan 
2764b5a280c0SLe Tan     case DMAR_IVA_REG_HI:
2765b5a280c0SLe Tan         assert(size == 4);
2766b5a280c0SLe Tan         vtd_set_long(s, addr, val);
2767b5a280c0SLe Tan         break;
2768b5a280c0SLe Tan 
27691da12ec4SLe Tan     /* Fault Status Register, 32-bit */
27701da12ec4SLe Tan     case DMAR_FSTS_REG:
27711da12ec4SLe Tan         assert(size == 4);
27721da12ec4SLe Tan         vtd_set_long(s, addr, val);
27731da12ec4SLe Tan         vtd_handle_fsts_write(s);
27741da12ec4SLe Tan         break;
27751da12ec4SLe Tan 
27761da12ec4SLe Tan     /* Fault Event Control Register, 32-bit */
27771da12ec4SLe Tan     case DMAR_FECTL_REG:
27781da12ec4SLe Tan         assert(size == 4);
27791da12ec4SLe Tan         vtd_set_long(s, addr, val);
27801da12ec4SLe Tan         vtd_handle_fectl_write(s);
27811da12ec4SLe Tan         break;
27821da12ec4SLe Tan 
27831da12ec4SLe Tan     /* Fault Event Data Register, 32-bit */
27841da12ec4SLe Tan     case DMAR_FEDATA_REG:
27851da12ec4SLe Tan         assert(size == 4);
27861da12ec4SLe Tan         vtd_set_long(s, addr, val);
27871da12ec4SLe Tan         break;
27881da12ec4SLe Tan 
27891da12ec4SLe Tan     /* Fault Event Address Register, 32-bit */
27901da12ec4SLe Tan     case DMAR_FEADDR_REG:
2791b7a7bb35SJan Kiszka         if (size == 4) {
27921da12ec4SLe Tan             vtd_set_long(s, addr, val);
2793b7a7bb35SJan Kiszka         } else {
2794b7a7bb35SJan Kiszka             /*
2795b7a7bb35SJan Kiszka              * While the register is 32-bit only, some guests (Xen...) write to
2796b7a7bb35SJan Kiszka              * it with 64-bit.
2797b7a7bb35SJan Kiszka              */
2798b7a7bb35SJan Kiszka             vtd_set_quad(s, addr, val);
2799b7a7bb35SJan Kiszka         }
28001da12ec4SLe Tan         break;
28011da12ec4SLe Tan 
28021da12ec4SLe Tan     /* Fault Event Upper Address Register, 32-bit */
28031da12ec4SLe Tan     case DMAR_FEUADDR_REG:
28041da12ec4SLe Tan         assert(size == 4);
28051da12ec4SLe Tan         vtd_set_long(s, addr, val);
28061da12ec4SLe Tan         break;
28071da12ec4SLe Tan 
28081da12ec4SLe Tan     /* Protected Memory Enable Register, 32-bit */
28091da12ec4SLe Tan     case DMAR_PMEN_REG:
28101da12ec4SLe Tan         assert(size == 4);
28111da12ec4SLe Tan         vtd_set_long(s, addr, val);
28121da12ec4SLe Tan         break;
28131da12ec4SLe Tan 
28141da12ec4SLe Tan     /* Root Table Address Register, 64-bit */
28151da12ec4SLe Tan     case DMAR_RTADDR_REG:
28161da12ec4SLe Tan         if (size == 4) {
28171da12ec4SLe Tan             vtd_set_long(s, addr, val);
28181da12ec4SLe Tan         } else {
28191da12ec4SLe Tan             vtd_set_quad(s, addr, val);
28201da12ec4SLe Tan         }
28211da12ec4SLe Tan         break;
28221da12ec4SLe Tan 
28231da12ec4SLe Tan     case DMAR_RTADDR_REG_HI:
28241da12ec4SLe Tan         assert(size == 4);
28251da12ec4SLe Tan         vtd_set_long(s, addr, val);
28261da12ec4SLe Tan         break;
28271da12ec4SLe Tan 
2828ed7b8fbcSLe Tan     /* Invalidation Queue Tail Register, 64-bit */
2829ed7b8fbcSLe Tan     case DMAR_IQT_REG:
2830ed7b8fbcSLe Tan         if (size == 4) {
2831ed7b8fbcSLe Tan             vtd_set_long(s, addr, val);
2832ed7b8fbcSLe Tan         } else {
2833ed7b8fbcSLe Tan             vtd_set_quad(s, addr, val);
2834ed7b8fbcSLe Tan         }
2835ed7b8fbcSLe Tan         vtd_handle_iqt_write(s);
2836ed7b8fbcSLe Tan         break;
2837ed7b8fbcSLe Tan 
2838ed7b8fbcSLe Tan     case DMAR_IQT_REG_HI:
2839ed7b8fbcSLe Tan         assert(size == 4);
2840ed7b8fbcSLe Tan         vtd_set_long(s, addr, val);
2841ed7b8fbcSLe Tan         /* 19:63 of IQT_REG is RsvdZ, do nothing here */
2842ed7b8fbcSLe Tan         break;
2843ed7b8fbcSLe Tan 
2844ed7b8fbcSLe Tan     /* Invalidation Queue Address Register, 64-bit */
2845ed7b8fbcSLe Tan     case DMAR_IQA_REG:
2846ed7b8fbcSLe Tan         if (size == 4) {
2847ed7b8fbcSLe Tan             vtd_set_long(s, addr, val);
2848ed7b8fbcSLe Tan         } else {
2849ed7b8fbcSLe Tan             vtd_set_quad(s, addr, val);
2850ed7b8fbcSLe Tan         }
2851c0c1d351SLiu, Yi L         if (s->ecap & VTD_ECAP_SMTS &&
2852c0c1d351SLiu, Yi L             val & VTD_IQA_DW_MASK) {
2853c0c1d351SLiu, Yi L             s->iq_dw = true;
2854c0c1d351SLiu, Yi L         } else {
2855c0c1d351SLiu, Yi L             s->iq_dw = false;
2856c0c1d351SLiu, Yi L         }
2857ed7b8fbcSLe Tan         break;
2858ed7b8fbcSLe Tan 
2859ed7b8fbcSLe Tan     case DMAR_IQA_REG_HI:
2860ed7b8fbcSLe Tan         assert(size == 4);
2861ed7b8fbcSLe Tan         vtd_set_long(s, addr, val);
2862ed7b8fbcSLe Tan         break;
2863ed7b8fbcSLe Tan 
2864ed7b8fbcSLe Tan     /* Invalidation Completion Status Register, 32-bit */
2865ed7b8fbcSLe Tan     case DMAR_ICS_REG:
2866ed7b8fbcSLe Tan         assert(size == 4);
2867ed7b8fbcSLe Tan         vtd_set_long(s, addr, val);
2868ed7b8fbcSLe Tan         vtd_handle_ics_write(s);
2869ed7b8fbcSLe Tan         break;
2870ed7b8fbcSLe Tan 
2871ed7b8fbcSLe Tan     /* Invalidation Event Control Register, 32-bit */
2872ed7b8fbcSLe Tan     case DMAR_IECTL_REG:
2873ed7b8fbcSLe Tan         assert(size == 4);
2874ed7b8fbcSLe Tan         vtd_set_long(s, addr, val);
2875ed7b8fbcSLe Tan         vtd_handle_iectl_write(s);
2876ed7b8fbcSLe Tan         break;
2877ed7b8fbcSLe Tan 
2878ed7b8fbcSLe Tan     /* Invalidation Event Data Register, 32-bit */
2879ed7b8fbcSLe Tan     case DMAR_IEDATA_REG:
2880ed7b8fbcSLe Tan         assert(size == 4);
2881ed7b8fbcSLe Tan         vtd_set_long(s, addr, val);
2882ed7b8fbcSLe Tan         break;
2883ed7b8fbcSLe Tan 
2884ed7b8fbcSLe Tan     /* Invalidation Event Address Register, 32-bit */
2885ed7b8fbcSLe Tan     case DMAR_IEADDR_REG:
2886ed7b8fbcSLe Tan         assert(size == 4);
2887ed7b8fbcSLe Tan         vtd_set_long(s, addr, val);
2888ed7b8fbcSLe Tan         break;
2889ed7b8fbcSLe Tan 
2890ed7b8fbcSLe Tan     /* Invalidation Event Upper Address Register, 32-bit */
2891ed7b8fbcSLe Tan     case DMAR_IEUADDR_REG:
2892ed7b8fbcSLe Tan         assert(size == 4);
2893ed7b8fbcSLe Tan         vtd_set_long(s, addr, val);
2894ed7b8fbcSLe Tan         break;
2895ed7b8fbcSLe Tan 
28961da12ec4SLe Tan     /* Fault Recording Registers, 128-bit */
28971da12ec4SLe Tan     case DMAR_FRCD_REG_0_0:
28981da12ec4SLe Tan         if (size == 4) {
28991da12ec4SLe Tan             vtd_set_long(s, addr, val);
29001da12ec4SLe Tan         } else {
29011da12ec4SLe Tan             vtd_set_quad(s, addr, val);
29021da12ec4SLe Tan         }
29031da12ec4SLe Tan         break;
29041da12ec4SLe Tan 
29051da12ec4SLe Tan     case DMAR_FRCD_REG_0_1:
29061da12ec4SLe Tan         assert(size == 4);
29071da12ec4SLe Tan         vtd_set_long(s, addr, val);
29081da12ec4SLe Tan         break;
29091da12ec4SLe Tan 
29101da12ec4SLe Tan     case DMAR_FRCD_REG_0_2:
29111da12ec4SLe Tan         if (size == 4) {
29121da12ec4SLe Tan             vtd_set_long(s, addr, val);
29131da12ec4SLe Tan         } else {
29141da12ec4SLe Tan             vtd_set_quad(s, addr, val);
29151da12ec4SLe Tan             /* May clear bit 127 (Fault), update PPF */
29161da12ec4SLe Tan             vtd_update_fsts_ppf(s);
29171da12ec4SLe Tan         }
29181da12ec4SLe Tan         break;
29191da12ec4SLe Tan 
29201da12ec4SLe Tan     case DMAR_FRCD_REG_0_3:
29211da12ec4SLe Tan         assert(size == 4);
29221da12ec4SLe Tan         vtd_set_long(s, addr, val);
29231da12ec4SLe Tan         /* May clear bit 127 (Fault), update PPF */
29241da12ec4SLe Tan         vtd_update_fsts_ppf(s);
29251da12ec4SLe Tan         break;
29261da12ec4SLe Tan 
2927a5861439SPeter Xu     case DMAR_IRTA_REG:
2928a5861439SPeter Xu         if (size == 4) {
2929a5861439SPeter Xu             vtd_set_long(s, addr, val);
2930a5861439SPeter Xu         } else {
2931a5861439SPeter Xu             vtd_set_quad(s, addr, val);
2932a5861439SPeter Xu         }
2933a5861439SPeter Xu         break;
2934a5861439SPeter Xu 
2935a5861439SPeter Xu     case DMAR_IRTA_REG_HI:
2936a5861439SPeter Xu         assert(size == 4);
2937a5861439SPeter Xu         vtd_set_long(s, addr, val);
2938a5861439SPeter Xu         break;
2939a5861439SPeter Xu 
29401da12ec4SLe Tan     default:
29411da12ec4SLe Tan         if (size == 4) {
29421da12ec4SLe Tan             vtd_set_long(s, addr, val);
29431da12ec4SLe Tan         } else {
29441da12ec4SLe Tan             vtd_set_quad(s, addr, val);
29451da12ec4SLe Tan         }
29461da12ec4SLe Tan     }
29471da12ec4SLe Tan }
29481da12ec4SLe Tan 
29493df9d748SAlexey Kardashevskiy static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
29502c91bcf2SPeter Maydell                                          IOMMUAccessFlags flag, int iommu_idx)
29511da12ec4SLe Tan {
29521da12ec4SLe Tan     VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
29531da12ec4SLe Tan     IntelIOMMUState *s = vtd_as->iommu_state;
2954b9313021SPeter Xu     IOMMUTLBEntry iotlb = {
2955b9313021SPeter Xu         /* We'll fill in the rest later. */
29561da12ec4SLe Tan         .target_as = &address_space_memory,
29571da12ec4SLe Tan     };
2958b9313021SPeter Xu     bool success;
29591da12ec4SLe Tan 
2960b9313021SPeter Xu     if (likely(s->dmar_enabled)) {
2961b9313021SPeter Xu         success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn,
2962b9313021SPeter Xu                                          addr, flag & IOMMU_WO, &iotlb);
2963b9313021SPeter Xu     } else {
29641da12ec4SLe Tan         /* DMAR disabled, passthrough, use 4k-page*/
2965b9313021SPeter Xu         iotlb.iova = addr & VTD_PAGE_MASK_4K;
2966b9313021SPeter Xu         iotlb.translated_addr = addr & VTD_PAGE_MASK_4K;
2967b9313021SPeter Xu         iotlb.addr_mask = ~VTD_PAGE_MASK_4K;
2968b9313021SPeter Xu         iotlb.perm = IOMMU_RW;
2969b9313021SPeter Xu         success = true;
29701da12ec4SLe Tan     }
29711da12ec4SLe Tan 
2972b9313021SPeter Xu     if (likely(success)) {
29737feb51b7SPeter Xu         trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus),
29747feb51b7SPeter Xu                                  VTD_PCI_SLOT(vtd_as->devfn),
29757feb51b7SPeter Xu                                  VTD_PCI_FUNC(vtd_as->devfn),
2976b9313021SPeter Xu                                  iotlb.iova, iotlb.translated_addr,
2977b9313021SPeter Xu                                  iotlb.addr_mask);
2978b9313021SPeter Xu     } else {
29794e4abd11SPeter Xu         error_report_once("%s: detected translation failure "
29804e4abd11SPeter Xu                           "(dev=%02x:%02x:%02x, iova=0x%" PRIx64 ")",
29814e4abd11SPeter Xu                           __func__, pci_bus_num(vtd_as->bus),
2982b9313021SPeter Xu                           VTD_PCI_SLOT(vtd_as->devfn),
2983b9313021SPeter Xu                           VTD_PCI_FUNC(vtd_as->devfn),
2984662b4b69SPeter Xu                           addr);
2985b9313021SPeter Xu     }
29867feb51b7SPeter Xu 
2987b9313021SPeter Xu     return iotlb;
29881da12ec4SLe Tan }
29891da12ec4SLe Tan 
2990549d4005SEric Auger static int vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu,
29915bf3d319SPeter Xu                                          IOMMUNotifierFlag old,
2992549d4005SEric Auger                                          IOMMUNotifierFlag new,
2993549d4005SEric Auger                                          Error **errp)
29943cb3b154SAlex Williamson {
29953cb3b154SAlex Williamson     VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
2996dd4d607eSPeter Xu     IntelIOMMUState *s = vtd_as->iommu_state;
29973cb3b154SAlex Williamson 
29984f8a62a9SPeter Xu     /* Update per-address-space notifier flags */
29994f8a62a9SPeter Xu     vtd_as->notifier_flags = new;
30004f8a62a9SPeter Xu 
3001dd4d607eSPeter Xu     if (old == IOMMU_NOTIFIER_NONE) {
3002b4a4ba0dSPeter Xu         QLIST_INSERT_HEAD(&s->vtd_as_with_notifiers, vtd_as, next);
3003b4a4ba0dSPeter Xu     } else if (new == IOMMU_NOTIFIER_NONE) {
3004b4a4ba0dSPeter Xu         QLIST_REMOVE(vtd_as, next);
3005dd4d607eSPeter Xu     }
3006549d4005SEric Auger     return 0;
30073cb3b154SAlex Williamson }
30083cb3b154SAlex Williamson 
3009552a1e01SPeter Xu static int vtd_post_load(void *opaque, int version_id)
3010552a1e01SPeter Xu {
3011552a1e01SPeter Xu     IntelIOMMUState *iommu = opaque;
3012552a1e01SPeter Xu 
3013552a1e01SPeter Xu     /*
3014552a1e01SPeter Xu      * Memory regions are dynamically turned on/off depending on
3015552a1e01SPeter Xu      * context entry configurations from the guest. After migration,
3016552a1e01SPeter Xu      * we need to make sure the memory regions are still correct.
3017552a1e01SPeter Xu      */
3018552a1e01SPeter Xu     vtd_switch_address_space_all(iommu);
3019552a1e01SPeter Xu 
30202811af3bSPeter Xu     /*
30212811af3bSPeter Xu      * We don't need to migrate the root_scalable because we can
30222811af3bSPeter Xu      * simply do the calculation after the loading is complete.  We
30232811af3bSPeter Xu      * can actually do similar things with root, dmar_enabled, etc.
30242811af3bSPeter Xu      * however since we've had them already so we'd better keep them
30252811af3bSPeter Xu      * for compatibility of migration.
30262811af3bSPeter Xu      */
30272811af3bSPeter Xu     vtd_update_scalable_state(iommu);
30282811af3bSPeter Xu 
3029552a1e01SPeter Xu     return 0;
3030552a1e01SPeter Xu }
3031552a1e01SPeter Xu 
30321da12ec4SLe Tan static const VMStateDescription vtd_vmstate = {
30331da12ec4SLe Tan     .name = "iommu-intel",
30348cdcf3c1SPeter Xu     .version_id = 1,
30358cdcf3c1SPeter Xu     .minimum_version_id = 1,
30368cdcf3c1SPeter Xu     .priority = MIG_PRI_IOMMU,
3037552a1e01SPeter Xu     .post_load = vtd_post_load,
30388cdcf3c1SPeter Xu     .fields = (VMStateField[]) {
30398cdcf3c1SPeter Xu         VMSTATE_UINT64(root, IntelIOMMUState),
30408cdcf3c1SPeter Xu         VMSTATE_UINT64(intr_root, IntelIOMMUState),
30418cdcf3c1SPeter Xu         VMSTATE_UINT64(iq, IntelIOMMUState),
30428cdcf3c1SPeter Xu         VMSTATE_UINT32(intr_size, IntelIOMMUState),
30438cdcf3c1SPeter Xu         VMSTATE_UINT16(iq_head, IntelIOMMUState),
30448cdcf3c1SPeter Xu         VMSTATE_UINT16(iq_tail, IntelIOMMUState),
30458cdcf3c1SPeter Xu         VMSTATE_UINT16(iq_size, IntelIOMMUState),
30468cdcf3c1SPeter Xu         VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState),
30478cdcf3c1SPeter Xu         VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE),
30488cdcf3c1SPeter Xu         VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState),
304981fb1e64SPeter Xu         VMSTATE_UNUSED(1),      /* bool root_extended is obsolete by VT-d */
30508cdcf3c1SPeter Xu         VMSTATE_BOOL(dmar_enabled, IntelIOMMUState),
30518cdcf3c1SPeter Xu         VMSTATE_BOOL(qi_enabled, IntelIOMMUState),
30528cdcf3c1SPeter Xu         VMSTATE_BOOL(intr_enabled, IntelIOMMUState),
30538cdcf3c1SPeter Xu         VMSTATE_BOOL(intr_eime, IntelIOMMUState),
30548cdcf3c1SPeter Xu         VMSTATE_END_OF_LIST()
30558cdcf3c1SPeter Xu     }
30561da12ec4SLe Tan };
30571da12ec4SLe Tan 
30581da12ec4SLe Tan static const MemoryRegionOps vtd_mem_ops = {
30591da12ec4SLe Tan     .read = vtd_mem_read,
30601da12ec4SLe Tan     .write = vtd_mem_write,
30611da12ec4SLe Tan     .endianness = DEVICE_LITTLE_ENDIAN,
30621da12ec4SLe Tan     .impl = {
30631da12ec4SLe Tan         .min_access_size = 4,
30641da12ec4SLe Tan         .max_access_size = 8,
30651da12ec4SLe Tan     },
30661da12ec4SLe Tan     .valid = {
30671da12ec4SLe Tan         .min_access_size = 4,
30681da12ec4SLe Tan         .max_access_size = 8,
30691da12ec4SLe Tan     },
30701da12ec4SLe Tan };
30711da12ec4SLe Tan 
30721da12ec4SLe Tan static Property vtd_properties[] = {
30731da12ec4SLe Tan     DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0),
3074e6b6af05SRadim Krčmář     DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim,
3075e6b6af05SRadim Krčmář                             ON_OFF_AUTO_AUTO),
3076fb506e70SRadim Krčmář     DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false),
30774b49b586SPeter Xu     DEFINE_PROP_UINT8("aw-bits", IntelIOMMUState, aw_bits,
307837f51384SPrasad Singamsetty                       VTD_HOST_ADDRESS_WIDTH),
30793b40f0e5SAviv Ben-David     DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE),
30804a4f219eSYi Sun     DEFINE_PROP_BOOL("x-scalable-mode", IntelIOMMUState, scalable_mode, FALSE),
3081ccc23bb0SPeter Xu     DEFINE_PROP_BOOL("dma-drain", IntelIOMMUState, dma_drain, true),
30821da12ec4SLe Tan     DEFINE_PROP_END_OF_LIST(),
30831da12ec4SLe Tan };
30841da12ec4SLe Tan 
3085651e4cefSPeter Xu /* Read IRTE entry with specific index */
3086651e4cefSPeter Xu static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index,
3087bc38ee10SMichael S. Tsirkin                         VTD_IR_TableEntry *entry, uint16_t sid)
3088651e4cefSPeter Xu {
3089ede9c94aSPeter Xu     static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \
3090ede9c94aSPeter Xu         {0xffff, 0xfffb, 0xfff9, 0xfff8};
3091651e4cefSPeter Xu     dma_addr_t addr = 0x00;
3092ede9c94aSPeter Xu     uint16_t mask, source_id;
3093ede9c94aSPeter Xu     uint8_t bus, bus_max, bus_min;
3094651e4cefSPeter Xu 
3095651e4cefSPeter Xu     addr = iommu->intr_root + index * sizeof(*entry);
3096651e4cefSPeter Xu     if (dma_memory_read(&address_space_memory, addr, entry,
3097651e4cefSPeter Xu                         sizeof(*entry))) {
30981376211fSPeter Xu         error_report_once("%s: read failed: ind=0x%x addr=0x%" PRIx64,
30991376211fSPeter Xu                           __func__, index, addr);
3100651e4cefSPeter Xu         return -VTD_FR_IR_ROOT_INVAL;
3101651e4cefSPeter Xu     }
3102651e4cefSPeter Xu 
31037feb51b7SPeter Xu     trace_vtd_ir_irte_get(index, le64_to_cpu(entry->data[1]),
31047feb51b7SPeter Xu                           le64_to_cpu(entry->data[0]));
31057feb51b7SPeter Xu 
3106bc38ee10SMichael S. Tsirkin     if (!entry->irte.present) {
31074e4abd11SPeter Xu         error_report_once("%s: detected non-present IRTE "
31084e4abd11SPeter Xu                           "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")",
31094e4abd11SPeter Xu                           __func__, index, le64_to_cpu(entry->data[1]),
3110651e4cefSPeter Xu                           le64_to_cpu(entry->data[0]));
3111651e4cefSPeter Xu         return -VTD_FR_IR_ENTRY_P;
3112651e4cefSPeter Xu     }
3113651e4cefSPeter Xu 
3114bc38ee10SMichael S. Tsirkin     if (entry->irte.__reserved_0 || entry->irte.__reserved_1 ||
3115bc38ee10SMichael S. Tsirkin         entry->irte.__reserved_2) {
31164e4abd11SPeter Xu         error_report_once("%s: detected non-zero reserved IRTE "
31174e4abd11SPeter Xu                           "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")",
31184e4abd11SPeter Xu                           __func__, index, le64_to_cpu(entry->data[1]),
3119651e4cefSPeter Xu                           le64_to_cpu(entry->data[0]));
3120651e4cefSPeter Xu         return -VTD_FR_IR_IRTE_RSVD;
3121651e4cefSPeter Xu     }
3122651e4cefSPeter Xu 
3123ede9c94aSPeter Xu     if (sid != X86_IOMMU_SID_INVALID) {
3124ede9c94aSPeter Xu         /* Validate IRTE SID */
3125bc38ee10SMichael S. Tsirkin         source_id = le32_to_cpu(entry->irte.source_id);
3126bc38ee10SMichael S. Tsirkin         switch (entry->irte.sid_vtype) {
3127ede9c94aSPeter Xu         case VTD_SVT_NONE:
3128ede9c94aSPeter Xu             break;
3129ede9c94aSPeter Xu 
3130ede9c94aSPeter Xu         case VTD_SVT_ALL:
3131bc38ee10SMichael S. Tsirkin             mask = vtd_svt_mask[entry->irte.sid_q];
3132ede9c94aSPeter Xu             if ((source_id & mask) != (sid & mask)) {
31334e4abd11SPeter Xu                 error_report_once("%s: invalid IRTE SID "
31344e4abd11SPeter Xu                                   "(index=%u, sid=%u, source_id=%u)",
31354e4abd11SPeter Xu                                   __func__, index, sid, source_id);
3136ede9c94aSPeter Xu                 return -VTD_FR_IR_SID_ERR;
3137ede9c94aSPeter Xu             }
3138ede9c94aSPeter Xu             break;
3139ede9c94aSPeter Xu 
3140ede9c94aSPeter Xu         case VTD_SVT_BUS:
3141ede9c94aSPeter Xu             bus_max = source_id >> 8;
3142ede9c94aSPeter Xu             bus_min = source_id & 0xff;
3143ede9c94aSPeter Xu             bus = sid >> 8;
3144ede9c94aSPeter Xu             if (bus > bus_max || bus < bus_min) {
31454e4abd11SPeter Xu                 error_report_once("%s: invalid SVT_BUS "
31464e4abd11SPeter Xu                                   "(index=%u, bus=%u, min=%u, max=%u)",
31474e4abd11SPeter Xu                                   __func__, index, bus, bus_min, bus_max);
3148ede9c94aSPeter Xu                 return -VTD_FR_IR_SID_ERR;
3149ede9c94aSPeter Xu             }
3150ede9c94aSPeter Xu             break;
3151ede9c94aSPeter Xu 
3152ede9c94aSPeter Xu         default:
31534e4abd11SPeter Xu             error_report_once("%s: detected invalid IRTE SVT "
31544e4abd11SPeter Xu                               "(index=%u, type=%d)", __func__,
31554e4abd11SPeter Xu                               index, entry->irte.sid_vtype);
3156ede9c94aSPeter Xu             /* Take this as verification failure. */
3157ede9c94aSPeter Xu             return -VTD_FR_IR_SID_ERR;
3158ede9c94aSPeter Xu             break;
3159ede9c94aSPeter Xu         }
3160ede9c94aSPeter Xu     }
3161651e4cefSPeter Xu 
3162651e4cefSPeter Xu     return 0;
3163651e4cefSPeter Xu }
3164651e4cefSPeter Xu 
3165651e4cefSPeter Xu /* Fetch IRQ information of specific IR index */
3166ede9c94aSPeter Xu static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index,
316735c24501SSingh, Brijesh                              X86IOMMUIrq *irq, uint16_t sid)
3168651e4cefSPeter Xu {
3169bc38ee10SMichael S. Tsirkin     VTD_IR_TableEntry irte = {};
3170651e4cefSPeter Xu     int ret = 0;
3171651e4cefSPeter Xu 
3172ede9c94aSPeter Xu     ret = vtd_irte_get(iommu, index, &irte, sid);
3173651e4cefSPeter Xu     if (ret) {
3174651e4cefSPeter Xu         return ret;
3175651e4cefSPeter Xu     }
3176651e4cefSPeter Xu 
3177bc38ee10SMichael S. Tsirkin     irq->trigger_mode = irte.irte.trigger_mode;
3178bc38ee10SMichael S. Tsirkin     irq->vector = irte.irte.vector;
3179bc38ee10SMichael S. Tsirkin     irq->delivery_mode = irte.irte.delivery_mode;
3180bc38ee10SMichael S. Tsirkin     irq->dest = le32_to_cpu(irte.irte.dest_id);
318128589311SJan Kiszka     if (!iommu->intr_eime) {
3182651e4cefSPeter Xu #define  VTD_IR_APIC_DEST_MASK         (0xff00ULL)
3183651e4cefSPeter Xu #define  VTD_IR_APIC_DEST_SHIFT        (8)
318428589311SJan Kiszka         irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >>
3185651e4cefSPeter Xu             VTD_IR_APIC_DEST_SHIFT;
318628589311SJan Kiszka     }
3187bc38ee10SMichael S. Tsirkin     irq->dest_mode = irte.irte.dest_mode;
3188bc38ee10SMichael S. Tsirkin     irq->redir_hint = irte.irte.redir_hint;
3189651e4cefSPeter Xu 
31907feb51b7SPeter Xu     trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector,
31917feb51b7SPeter Xu                        irq->delivery_mode, irq->dest, irq->dest_mode);
3192651e4cefSPeter Xu 
3193651e4cefSPeter Xu     return 0;
3194651e4cefSPeter Xu }
3195651e4cefSPeter Xu 
3196651e4cefSPeter Xu /* Interrupt remapping for MSI/MSI-X entry */
3197651e4cefSPeter Xu static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu,
3198651e4cefSPeter Xu                                    MSIMessage *origin,
3199ede9c94aSPeter Xu                                    MSIMessage *translated,
3200ede9c94aSPeter Xu                                    uint16_t sid)
3201651e4cefSPeter Xu {
3202651e4cefSPeter Xu     int ret = 0;
3203651e4cefSPeter Xu     VTD_IR_MSIAddress addr;
3204651e4cefSPeter Xu     uint16_t index;
320535c24501SSingh, Brijesh     X86IOMMUIrq irq = {};
3206651e4cefSPeter Xu 
3207651e4cefSPeter Xu     assert(origin && translated);
3208651e4cefSPeter Xu 
32097feb51b7SPeter Xu     trace_vtd_ir_remap_msi_req(origin->address, origin->data);
32107feb51b7SPeter Xu 
3211651e4cefSPeter Xu     if (!iommu || !iommu->intr_enabled) {
3212e7a3b91fSPeter Xu         memcpy(translated, origin, sizeof(*origin));
3213e7a3b91fSPeter Xu         goto out;
3214651e4cefSPeter Xu     }
3215651e4cefSPeter Xu 
3216651e4cefSPeter Xu     if (origin->address & VTD_MSI_ADDR_HI_MASK) {
32171376211fSPeter Xu         error_report_once("%s: MSI address high 32 bits non-zero detected: "
32181376211fSPeter Xu                           "address=0x%" PRIx64, __func__, origin->address);
3219651e4cefSPeter Xu         return -VTD_FR_IR_REQ_RSVD;
3220651e4cefSPeter Xu     }
3221651e4cefSPeter Xu 
3222651e4cefSPeter Xu     addr.data = origin->address & VTD_MSI_ADDR_LO_MASK;
32231a43713bSPeter Xu     if (addr.addr.__head != 0xfee) {
32241376211fSPeter Xu         error_report_once("%s: MSI address low 32 bit invalid: 0x%" PRIx32,
32251376211fSPeter Xu                           __func__, addr.data);
3226651e4cefSPeter Xu         return -VTD_FR_IR_REQ_RSVD;
3227651e4cefSPeter Xu     }
3228651e4cefSPeter Xu 
3229651e4cefSPeter Xu     /* This is compatible mode. */
3230bc38ee10SMichael S. Tsirkin     if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) {
3231e7a3b91fSPeter Xu         memcpy(translated, origin, sizeof(*origin));
3232e7a3b91fSPeter Xu         goto out;
3233651e4cefSPeter Xu     }
3234651e4cefSPeter Xu 
3235bc38ee10SMichael S. Tsirkin     index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l);
3236651e4cefSPeter Xu 
3237651e4cefSPeter Xu #define  VTD_IR_MSI_DATA_SUBHANDLE       (0x0000ffff)
3238651e4cefSPeter Xu #define  VTD_IR_MSI_DATA_RESERVED        (0xffff0000)
3239651e4cefSPeter Xu 
3240bc38ee10SMichael S. Tsirkin     if (addr.addr.sub_valid) {
3241651e4cefSPeter Xu         /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */
3242651e4cefSPeter Xu         index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE;
3243651e4cefSPeter Xu     }
3244651e4cefSPeter Xu 
3245ede9c94aSPeter Xu     ret = vtd_remap_irq_get(iommu, index, &irq, sid);
3246651e4cefSPeter Xu     if (ret) {
3247651e4cefSPeter Xu         return ret;
3248651e4cefSPeter Xu     }
3249651e4cefSPeter Xu 
3250bc38ee10SMichael S. Tsirkin     if (addr.addr.sub_valid) {
32517feb51b7SPeter Xu         trace_vtd_ir_remap_type("MSI");
3252651e4cefSPeter Xu         if (origin->data & VTD_IR_MSI_DATA_RESERVED) {
32534e4abd11SPeter Xu             error_report_once("%s: invalid IR MSI "
32544e4abd11SPeter Xu                               "(sid=%u, address=0x%" PRIx64
32554e4abd11SPeter Xu                               ", data=0x%" PRIx32 ")",
32564e4abd11SPeter Xu                               __func__, sid, origin->address, origin->data);
3257651e4cefSPeter Xu             return -VTD_FR_IR_REQ_RSVD;
3258651e4cefSPeter Xu         }
3259651e4cefSPeter Xu     } else {
3260651e4cefSPeter Xu         uint8_t vector = origin->data & 0xff;
3261dea651a9SFeng Wu         uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1;
3262dea651a9SFeng Wu 
32637feb51b7SPeter Xu         trace_vtd_ir_remap_type("IOAPIC");
3264651e4cefSPeter Xu         /* IOAPIC entry vector should be aligned with IRTE vector
3265651e4cefSPeter Xu          * (see vt-d spec 5.1.5.1). */
3266651e4cefSPeter Xu         if (vector != irq.vector) {
32677feb51b7SPeter Xu             trace_vtd_warn_ir_vector(sid, index, vector, irq.vector);
3268651e4cefSPeter Xu         }
3269dea651a9SFeng Wu 
3270dea651a9SFeng Wu         /* The Trigger Mode field must match the Trigger Mode in the IRTE.
3271dea651a9SFeng Wu          * (see vt-d spec 5.1.5.1). */
3272dea651a9SFeng Wu         if (trigger_mode != irq.trigger_mode) {
32737feb51b7SPeter Xu             trace_vtd_warn_ir_trigger(sid, index, trigger_mode,
32747feb51b7SPeter Xu                                       irq.trigger_mode);
3275dea651a9SFeng Wu         }
3276651e4cefSPeter Xu     }
3277651e4cefSPeter Xu 
3278651e4cefSPeter Xu     /*
3279651e4cefSPeter Xu      * We'd better keep the last two bits, assuming that guest OS
3280651e4cefSPeter Xu      * might modify it. Keep it does not hurt after all.
3281651e4cefSPeter Xu      */
3282bc38ee10SMichael S. Tsirkin     irq.msi_addr_last_bits = addr.addr.__not_care;
3283651e4cefSPeter Xu 
328435c24501SSingh, Brijesh     /* Translate X86IOMMUIrq to MSI message */
328535c24501SSingh, Brijesh     x86_iommu_irq_to_msi_message(&irq, translated);
3286651e4cefSPeter Xu 
3287e7a3b91fSPeter Xu out:
32887feb51b7SPeter Xu     trace_vtd_ir_remap_msi(origin->address, origin->data,
3289651e4cefSPeter Xu                            translated->address, translated->data);
3290651e4cefSPeter Xu     return 0;
3291651e4cefSPeter Xu }
3292651e4cefSPeter Xu 
32938b5ed7dfSPeter Xu static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src,
32948b5ed7dfSPeter Xu                          MSIMessage *dst, uint16_t sid)
32958b5ed7dfSPeter Xu {
3296ede9c94aSPeter Xu     return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu),
3297ede9c94aSPeter Xu                                    src, dst, sid);
32988b5ed7dfSPeter Xu }
32998b5ed7dfSPeter Xu 
3300651e4cefSPeter Xu static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr,
3301651e4cefSPeter Xu                                    uint64_t *data, unsigned size,
3302651e4cefSPeter Xu                                    MemTxAttrs attrs)
3303651e4cefSPeter Xu {
3304651e4cefSPeter Xu     return MEMTX_OK;
3305651e4cefSPeter Xu }
3306651e4cefSPeter Xu 
3307651e4cefSPeter Xu static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr,
3308651e4cefSPeter Xu                                     uint64_t value, unsigned size,
3309651e4cefSPeter Xu                                     MemTxAttrs attrs)
3310651e4cefSPeter Xu {
3311651e4cefSPeter Xu     int ret = 0;
331209cd058aSMichael S. Tsirkin     MSIMessage from = {}, to = {};
3313ede9c94aSPeter Xu     uint16_t sid = X86_IOMMU_SID_INVALID;
3314651e4cefSPeter Xu 
3315651e4cefSPeter Xu     from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST;
3316651e4cefSPeter Xu     from.data = (uint32_t) value;
3317651e4cefSPeter Xu 
3318ede9c94aSPeter Xu     if (!attrs.unspecified) {
3319ede9c94aSPeter Xu         /* We have explicit Source ID */
3320ede9c94aSPeter Xu         sid = attrs.requester_id;
3321ede9c94aSPeter Xu     }
3322ede9c94aSPeter Xu 
3323ede9c94aSPeter Xu     ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid);
3324651e4cefSPeter Xu     if (ret) {
3325651e4cefSPeter Xu         /* TODO: report error */
3326651e4cefSPeter Xu         /* Drop this interrupt */
3327651e4cefSPeter Xu         return MEMTX_ERROR;
3328651e4cefSPeter Xu     }
3329651e4cefSPeter Xu 
333032946019SRadim Krčmář     apic_get_class()->send_msi(&to);
3331651e4cefSPeter Xu 
3332651e4cefSPeter Xu     return MEMTX_OK;
3333651e4cefSPeter Xu }
3334651e4cefSPeter Xu 
3335651e4cefSPeter Xu static const MemoryRegionOps vtd_mem_ir_ops = {
3336651e4cefSPeter Xu     .read_with_attrs = vtd_mem_ir_read,
3337651e4cefSPeter Xu     .write_with_attrs = vtd_mem_ir_write,
3338651e4cefSPeter Xu     .endianness = DEVICE_LITTLE_ENDIAN,
3339651e4cefSPeter Xu     .impl = {
3340651e4cefSPeter Xu         .min_access_size = 4,
3341651e4cefSPeter Xu         .max_access_size = 4,
3342651e4cefSPeter Xu     },
3343651e4cefSPeter Xu     .valid = {
3344651e4cefSPeter Xu         .min_access_size = 4,
3345651e4cefSPeter Xu         .max_access_size = 4,
3346651e4cefSPeter Xu     },
3347651e4cefSPeter Xu };
33487df953bdSKnut Omang 
33497df953bdSKnut Omang VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
33507df953bdSKnut Omang {
33517df953bdSKnut Omang     uintptr_t key = (uintptr_t)bus;
33527df953bdSKnut Omang     VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
33537df953bdSKnut Omang     VTDAddressSpace *vtd_dev_as;
3354e0a3c8ccSJason Wang     char name[128];
33557df953bdSKnut Omang 
33567df953bdSKnut Omang     if (!vtd_bus) {
33572d3fc581SJason Wang         uintptr_t *new_key = g_malloc(sizeof(*new_key));
33582d3fc581SJason Wang         *new_key = (uintptr_t)bus;
33597df953bdSKnut Omang         /* No corresponding free() */
336004af0e18SPeter Xu         vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \
3361bf33cc75SPeter Xu                             PCI_DEVFN_MAX);
33627df953bdSKnut Omang         vtd_bus->bus = bus;
33632d3fc581SJason Wang         g_hash_table_insert(s->vtd_as_by_busptr, new_key, vtd_bus);
33647df953bdSKnut Omang     }
33657df953bdSKnut Omang 
33667df953bdSKnut Omang     vtd_dev_as = vtd_bus->dev_as[devfn];
33677df953bdSKnut Omang 
33687df953bdSKnut Omang     if (!vtd_dev_as) {
33694b519ef1SPeter Xu         snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn),
33704b519ef1SPeter Xu                  PCI_FUNC(devfn));
33717df953bdSKnut Omang         vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
33727df953bdSKnut Omang 
33737df953bdSKnut Omang         vtd_dev_as->bus = bus;
33747df953bdSKnut Omang         vtd_dev_as->devfn = (uint8_t)devfn;
33757df953bdSKnut Omang         vtd_dev_as->iommu_state = s;
33767df953bdSKnut Omang         vtd_dev_as->context_cache_entry.context_cache_gen = 0;
337763b88968SPeter Xu         vtd_dev_as->iova_tree = iova_tree_new();
3378558e0024SPeter Xu 
33794b519ef1SPeter Xu         memory_region_init(&vtd_dev_as->root, OBJECT(s), name, UINT64_MAX);
33804b519ef1SPeter Xu         address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, "vtd-root");
33814b519ef1SPeter Xu 
3382558e0024SPeter Xu         /*
33834b519ef1SPeter Xu          * Build the DMAR-disabled container with aliases to the
33844b519ef1SPeter Xu          * shared MRs.  Note that aliasing to a shared memory region
33854b519ef1SPeter Xu          * could help the memory API to detect same FlatViews so we
33864b519ef1SPeter Xu          * can have devices to share the same FlatView when DMAR is
33874b519ef1SPeter Xu          * disabled (either by not providing "intel_iommu=on" or with
33884b519ef1SPeter Xu          * "iommu=pt").  It will greatly reduce the total number of
33894b519ef1SPeter Xu          * FlatViews of the system hence VM runs faster.
3390558e0024SPeter Xu          */
33914b519ef1SPeter Xu         memory_region_init_alias(&vtd_dev_as->nodmar, OBJECT(s),
33924b519ef1SPeter Xu                                  "vtd-nodmar", &s->mr_nodmar, 0,
33934b519ef1SPeter Xu                                  memory_region_size(&s->mr_nodmar));
33944b519ef1SPeter Xu 
33954b519ef1SPeter Xu         /*
33964b519ef1SPeter Xu          * Build the per-device DMAR-enabled container.
33974b519ef1SPeter Xu          *
33984b519ef1SPeter Xu          * TODO: currently we have per-device IOMMU memory region only
33994b519ef1SPeter Xu          * because we have per-device IOMMU notifiers for devices.  If
34004b519ef1SPeter Xu          * one day we can abstract the IOMMU notifiers out of the
34014b519ef1SPeter Xu          * memory regions then we can also share the same memory
34024b519ef1SPeter Xu          * region here just like what we've done above with the nodmar
34034b519ef1SPeter Xu          * region.
34044b519ef1SPeter Xu          */
34054b519ef1SPeter Xu         strcat(name, "-dmar");
34061221a474SAlexey Kardashevskiy         memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu),
34071221a474SAlexey Kardashevskiy                                  TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s),
34084b519ef1SPeter Xu                                  name, UINT64_MAX);
34094b519ef1SPeter Xu         memory_region_init_alias(&vtd_dev_as->iommu_ir, OBJECT(s), "vtd-ir",
34104b519ef1SPeter Xu                                  &s->mr_ir, 0, memory_region_size(&s->mr_ir));
34114b519ef1SPeter Xu         memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->iommu),
3412558e0024SPeter Xu                                             VTD_INTERRUPT_ADDR_FIRST,
34134b519ef1SPeter Xu                                             &vtd_dev_as->iommu_ir, 1);
34144b519ef1SPeter Xu 
34154b519ef1SPeter Xu         /*
34164b519ef1SPeter Xu          * Hook both the containers under the root container, we
34174b519ef1SPeter Xu          * switch between DMAR & noDMAR by enable/disable
34184b519ef1SPeter Xu          * corresponding sub-containers
34194b519ef1SPeter Xu          */
3420558e0024SPeter Xu         memory_region_add_subregion_overlap(&vtd_dev_as->root, 0,
34213df9d748SAlexey Kardashevskiy                                             MEMORY_REGION(&vtd_dev_as->iommu),
34224b519ef1SPeter Xu                                             0);
34234b519ef1SPeter Xu         memory_region_add_subregion_overlap(&vtd_dev_as->root, 0,
34244b519ef1SPeter Xu                                             &vtd_dev_as->nodmar, 0);
34254b519ef1SPeter Xu 
3426558e0024SPeter Xu         vtd_switch_address_space(vtd_dev_as);
34277df953bdSKnut Omang     }
34287df953bdSKnut Omang     return vtd_dev_as;
34297df953bdSKnut Omang }
34307df953bdSKnut Omang 
34319a4bb839SPeter Xu static uint64_t get_naturally_aligned_size(uint64_t start,
34329a4bb839SPeter Xu                                            uint64_t size, int gaw)
34339a4bb839SPeter Xu {
34349a4bb839SPeter Xu     uint64_t max_mask = 1ULL << gaw;
34359a4bb839SPeter Xu     uint64_t alignment = start ? start & -start : max_mask;
34369a4bb839SPeter Xu 
34379a4bb839SPeter Xu     alignment = MIN(alignment, max_mask);
34389a4bb839SPeter Xu     size = MIN(size, max_mask);
34399a4bb839SPeter Xu 
34409a4bb839SPeter Xu     if (alignment <= size) {
34419a4bb839SPeter Xu         /* Increase the alignment of start */
34429a4bb839SPeter Xu         return alignment;
34439a4bb839SPeter Xu     } else {
34449a4bb839SPeter Xu         /* Find the largest page mask from size */
34459a4bb839SPeter Xu         return 1ULL << (63 - clz64(size));
34469a4bb839SPeter Xu     }
34479a4bb839SPeter Xu }
34489a4bb839SPeter Xu 
3449dd4d607eSPeter Xu /* Unmap the whole range in the notifier's scope. */
3450dd4d607eSPeter Xu static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n)
3451dd4d607eSPeter Xu {
34529a4bb839SPeter Xu     hwaddr size, remain;
3453dd4d607eSPeter Xu     hwaddr start = n->start;
3454dd4d607eSPeter Xu     hwaddr end = n->end;
345537f51384SPrasad Singamsetty     IntelIOMMUState *s = as->iommu_state;
345663b88968SPeter Xu     DMAMap map;
3457dd4d607eSPeter Xu 
3458dd4d607eSPeter Xu     /*
3459dd4d607eSPeter Xu      * Note: all the codes in this function has a assumption that IOVA
3460dd4d607eSPeter Xu      * bits are no more than VTD_MGAW bits (which is restricted by
3461dd4d607eSPeter Xu      * VT-d spec), otherwise we need to consider overflow of 64 bits.
3462dd4d607eSPeter Xu      */
3463dd4d607eSPeter Xu 
3464d6d10793SYan Zhao     if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) {
3465dd4d607eSPeter Xu         /*
3466dd4d607eSPeter Xu          * Don't need to unmap regions that is bigger than the whole
3467dd4d607eSPeter Xu          * VT-d supported address space size
3468dd4d607eSPeter Xu          */
3469d6d10793SYan Zhao         end = VTD_ADDRESS_SIZE(s->aw_bits) - 1;
3470dd4d607eSPeter Xu     }
3471dd4d607eSPeter Xu 
3472dd4d607eSPeter Xu     assert(start <= end);
34739a4bb839SPeter Xu     size = remain = end - start + 1;
3474dd4d607eSPeter Xu 
34759a4bb839SPeter Xu     while (remain >= VTD_PAGE_SIZE) {
34769a4bb839SPeter Xu         IOMMUTLBEntry entry;
34779a4bb839SPeter Xu         uint64_t mask = get_naturally_aligned_size(start, remain, s->aw_bits);
3478dd4d607eSPeter Xu 
34799a4bb839SPeter Xu         assert(mask);
34809a4bb839SPeter Xu 
34819a4bb839SPeter Xu         entry.iova = start;
34829a4bb839SPeter Xu         entry.addr_mask = mask - 1;
3483dd4d607eSPeter Xu         entry.target_as = &address_space_memory;
34849a4bb839SPeter Xu         entry.perm = IOMMU_NONE;
3485dd4d607eSPeter Xu         /* This field is meaningless for unmap */
3486dd4d607eSPeter Xu         entry.translated_addr = 0;
34879a4bb839SPeter Xu 
34889a4bb839SPeter Xu         memory_region_notify_one(n, &entry);
34899a4bb839SPeter Xu 
34909a4bb839SPeter Xu         start += mask;
34919a4bb839SPeter Xu         remain -= mask;
34929a4bb839SPeter Xu     }
34939a4bb839SPeter Xu 
34949a4bb839SPeter Xu     assert(!remain);
3495dd4d607eSPeter Xu 
3496dd4d607eSPeter Xu     trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
3497dd4d607eSPeter Xu                              VTD_PCI_SLOT(as->devfn),
3498dd4d607eSPeter Xu                              VTD_PCI_FUNC(as->devfn),
34999a4bb839SPeter Xu                              n->start, size);
3500dd4d607eSPeter Xu 
35019a4bb839SPeter Xu     map.iova = n->start;
35029a4bb839SPeter Xu     map.size = size;
350363b88968SPeter Xu     iova_tree_remove(as->iova_tree, &map);
3504dd4d607eSPeter Xu }
3505dd4d607eSPeter Xu 
3506dd4d607eSPeter Xu static void vtd_address_space_unmap_all(IntelIOMMUState *s)
3507dd4d607eSPeter Xu {
3508dd4d607eSPeter Xu     VTDAddressSpace *vtd_as;
3509dd4d607eSPeter Xu     IOMMUNotifier *n;
3510dd4d607eSPeter Xu 
3511b4a4ba0dSPeter Xu     QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
3512dd4d607eSPeter Xu         IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) {
3513dd4d607eSPeter Xu             vtd_address_space_unmap(vtd_as, n);
3514dd4d607eSPeter Xu         }
3515dd4d607eSPeter Xu     }
3516dd4d607eSPeter Xu }
3517dd4d607eSPeter Xu 
35182cc9ddccSPeter Xu static void vtd_address_space_refresh_all(IntelIOMMUState *s)
35192cc9ddccSPeter Xu {
35202cc9ddccSPeter Xu     vtd_address_space_unmap_all(s);
35212cc9ddccSPeter Xu     vtd_switch_address_space_all(s);
35222cc9ddccSPeter Xu }
35232cc9ddccSPeter Xu 
3524f06a696dSPeter Xu static int vtd_replay_hook(IOMMUTLBEntry *entry, void *private)
3525f06a696dSPeter Xu {
3526f06a696dSPeter Xu     memory_region_notify_one((IOMMUNotifier *)private, entry);
3527f06a696dSPeter Xu     return 0;
3528f06a696dSPeter Xu }
3529f06a696dSPeter Xu 
35303df9d748SAlexey Kardashevskiy static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n)
3531f06a696dSPeter Xu {
35323df9d748SAlexey Kardashevskiy     VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu);
3533f06a696dSPeter Xu     IntelIOMMUState *s = vtd_as->iommu_state;
3534f06a696dSPeter Xu     uint8_t bus_n = pci_bus_num(vtd_as->bus);
3535f06a696dSPeter Xu     VTDContextEntry ce;
3536f06a696dSPeter Xu 
3537f06a696dSPeter Xu     /*
3538dd4d607eSPeter Xu      * The replay can be triggered by either a invalidation or a newly
3539dd4d607eSPeter Xu      * created entry. No matter what, we release existing mappings
3540dd4d607eSPeter Xu      * (it means flushing caches for UNMAP-only registers).
3541f06a696dSPeter Xu      */
3542dd4d607eSPeter Xu     vtd_address_space_unmap(vtd_as, n);
3543dd4d607eSPeter Xu 
3544dd4d607eSPeter Xu     if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) {
3545fb43cf73SLiu, Yi L         trace_vtd_replay_ce_valid(s->root_scalable ? "scalable mode" :
3546fb43cf73SLiu, Yi L                                   "legacy mode",
3547fb43cf73SLiu, Yi L                                   bus_n, PCI_SLOT(vtd_as->devfn),
3548f06a696dSPeter Xu                                   PCI_FUNC(vtd_as->devfn),
3549fb43cf73SLiu, Yi L                                   vtd_get_domain_id(s, &ce),
3550f06a696dSPeter Xu                                   ce.hi, ce.lo);
35514f8a62a9SPeter Xu         if (vtd_as_has_map_notifier(vtd_as)) {
35524f8a62a9SPeter Xu             /* This is required only for MAP typed notifiers */
3553fe215b0cSPeter Xu             vtd_page_walk_info info = {
3554fe215b0cSPeter Xu                 .hook_fn = vtd_replay_hook,
3555fe215b0cSPeter Xu                 .private = (void *)n,
3556fe215b0cSPeter Xu                 .notify_unmap = false,
3557fe215b0cSPeter Xu                 .aw = s->aw_bits,
35582f764fa8SPeter Xu                 .as = vtd_as,
3559fb43cf73SLiu, Yi L                 .domain_id = vtd_get_domain_id(s, &ce),
3560fe215b0cSPeter Xu             };
3561fe215b0cSPeter Xu 
3562fb43cf73SLiu, Yi L             vtd_page_walk(s, &ce, 0, ~0ULL, &info);
35634f8a62a9SPeter Xu         }
3564f06a696dSPeter Xu     } else {
3565f06a696dSPeter Xu         trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn),
3566f06a696dSPeter Xu                                     PCI_FUNC(vtd_as->devfn));
3567f06a696dSPeter Xu     }
3568f06a696dSPeter Xu 
3569f06a696dSPeter Xu     return;
3570f06a696dSPeter Xu }
3571f06a696dSPeter Xu 
35721da12ec4SLe Tan /* Do the initialization. It will also be called when reset, so pay
35731da12ec4SLe Tan  * attention when adding new initialization stuff.
35741da12ec4SLe Tan  */
35751da12ec4SLe Tan static void vtd_init(IntelIOMMUState *s)
35761da12ec4SLe Tan {
3577d54bd7f8SPeter Xu     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
3578d54bd7f8SPeter Xu 
35791da12ec4SLe Tan     memset(s->csr, 0, DMAR_REG_SIZE);
35801da12ec4SLe Tan     memset(s->wmask, 0, DMAR_REG_SIZE);
35811da12ec4SLe Tan     memset(s->w1cmask, 0, DMAR_REG_SIZE);
35821da12ec4SLe Tan     memset(s->womask, 0, DMAR_REG_SIZE);
35831da12ec4SLe Tan 
35841da12ec4SLe Tan     s->root = 0;
3585fb43cf73SLiu, Yi L     s->root_scalable = false;
35861da12ec4SLe Tan     s->dmar_enabled = false;
3587d7bb469aSPeter Xu     s->intr_enabled = false;
35881da12ec4SLe Tan     s->iq_head = 0;
35891da12ec4SLe Tan     s->iq_tail = 0;
35901da12ec4SLe Tan     s->iq = 0;
35911da12ec4SLe Tan     s->iq_size = 0;
35921da12ec4SLe Tan     s->qi_enabled = false;
35931da12ec4SLe Tan     s->iq_last_desc_type = VTD_INV_DESC_NONE;
3594c0c1d351SLiu, Yi L     s->iq_dw = false;
35951da12ec4SLe Tan     s->next_frcd_reg = 0;
359692e5d85eSPrasad Singamsetty     s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND |
359792e5d85eSPrasad Singamsetty              VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS |
359837f51384SPrasad Singamsetty              VTD_CAP_SAGAW_39bit | VTD_CAP_MGAW(s->aw_bits);
3599ccc23bb0SPeter Xu     if (s->dma_drain) {
3600ccc23bb0SPeter Xu         s->cap |= VTD_CAP_DRAIN;
3601ccc23bb0SPeter Xu     }
360237f51384SPrasad Singamsetty     if (s->aw_bits == VTD_HOST_AW_48BIT) {
360337f51384SPrasad Singamsetty         s->cap |= VTD_CAP_SAGAW_48bit;
360437f51384SPrasad Singamsetty     }
3605ed7b8fbcSLe Tan     s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
36061da12ec4SLe Tan 
360792e5d85eSPrasad Singamsetty     /*
360892e5d85eSPrasad Singamsetty      * Rsvd field masks for spte
360992e5d85eSPrasad Singamsetty      */
3610ce586f3bSQi, Yadong     vtd_spte_rsvd[0] = ~0ULL;
3611e48929c7SQi, Yadong     vtd_spte_rsvd[1] = VTD_SPTE_PAGE_L1_RSVD_MASK(s->aw_bits,
3612e48929c7SQi, Yadong                                                   x86_iommu->dt_supported);
3613ce586f3bSQi, Yadong     vtd_spte_rsvd[2] = VTD_SPTE_PAGE_L2_RSVD_MASK(s->aw_bits);
3614ce586f3bSQi, Yadong     vtd_spte_rsvd[3] = VTD_SPTE_PAGE_L3_RSVD_MASK(s->aw_bits);
3615ce586f3bSQi, Yadong     vtd_spte_rsvd[4] = VTD_SPTE_PAGE_L4_RSVD_MASK(s->aw_bits);
3616ce586f3bSQi, Yadong 
3617e48929c7SQi, Yadong     vtd_spte_rsvd_large[2] = VTD_SPTE_LPAGE_L2_RSVD_MASK(s->aw_bits,
3618e48929c7SQi, Yadong                                                          x86_iommu->dt_supported);
3619e48929c7SQi, Yadong     vtd_spte_rsvd_large[3] = VTD_SPTE_LPAGE_L3_RSVD_MASK(s->aw_bits,
3620e48929c7SQi, Yadong                                                          x86_iommu->dt_supported);
362192e5d85eSPrasad Singamsetty 
3622a924b3d8SPeter Xu     if (x86_iommu_ir_supported(x86_iommu)) {
3623e6b6af05SRadim Krčmář         s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV;
3624e6b6af05SRadim Krčmář         if (s->intr_eim == ON_OFF_AUTO_ON) {
3625e6b6af05SRadim Krčmář             s->ecap |= VTD_ECAP_EIM;
3626e6b6af05SRadim Krčmář         }
3627e6b6af05SRadim Krčmář         assert(s->intr_eim != ON_OFF_AUTO_AUTO);
3628d54bd7f8SPeter Xu     }
3629d54bd7f8SPeter Xu 
3630554f5e16SJason Wang     if (x86_iommu->dt_supported) {
3631554f5e16SJason Wang         s->ecap |= VTD_ECAP_DT;
3632554f5e16SJason Wang     }
3633554f5e16SJason Wang 
3634dbaabb25SPeter Xu     if (x86_iommu->pt_supported) {
3635dbaabb25SPeter Xu         s->ecap |= VTD_ECAP_PT;
3636dbaabb25SPeter Xu     }
3637dbaabb25SPeter Xu 
36383b40f0e5SAviv Ben-David     if (s->caching_mode) {
36393b40f0e5SAviv Ben-David         s->cap |= VTD_CAP_CM;
36403b40f0e5SAviv Ben-David     }
36413b40f0e5SAviv Ben-David 
36424a4f219eSYi Sun     /* TODO: read cap/ecap from host to decide which cap to be exposed. */
36434a4f219eSYi Sun     if (s->scalable_mode) {
36444a4f219eSYi Sun         s->ecap |= VTD_ECAP_SMTS | VTD_ECAP_SRS | VTD_ECAP_SLTS;
36454a4f219eSYi Sun     }
36464a4f219eSYi Sun 
364706aba4caSPeter Xu     vtd_reset_caches(s);
3648d92fa2dcSLe Tan 
36491da12ec4SLe Tan     /* Define registers with default values and bit semantics */
36501da12ec4SLe Tan     vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);
36511da12ec4SLe Tan     vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);
36521da12ec4SLe Tan     vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0);
36531da12ec4SLe Tan     vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0);
36541da12ec4SLe Tan     vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL);
36551da12ec4SLe Tan     vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0);
3656fb43cf73SLiu, Yi L     vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffffc00ULL, 0);
36571da12ec4SLe Tan     vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0);
36581da12ec4SLe Tan     vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL);
36591da12ec4SLe Tan 
36601da12ec4SLe Tan     /* Advanced Fault Logging not supported */
36611da12ec4SLe Tan     vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL);
36621da12ec4SLe Tan     vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0);
36631da12ec4SLe Tan     vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0);
36641da12ec4SLe Tan     vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0);
36651da12ec4SLe Tan 
36661da12ec4SLe Tan     /* Treated as RsvdZ when EIM in ECAP_REG is not supported
36671da12ec4SLe Tan      * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0);
36681da12ec4SLe Tan      */
36691da12ec4SLe Tan     vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0);
36701da12ec4SLe Tan 
36711da12ec4SLe Tan     /* Treated as RO for implementations that PLMR and PHMR fields reported
36721da12ec4SLe Tan      * as Clear in the CAP_REG.
36731da12ec4SLe Tan      * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0);
36741da12ec4SLe Tan      */
36751da12ec4SLe Tan     vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0);
36761da12ec4SLe Tan 
3677ed7b8fbcSLe Tan     vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0);
3678ed7b8fbcSLe Tan     vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0);
3679c0c1d351SLiu, Yi L     vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff807ULL, 0);
3680ed7b8fbcSLe Tan     vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL);
3681ed7b8fbcSLe Tan     vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0);
3682ed7b8fbcSLe Tan     vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0);
3683ed7b8fbcSLe Tan     vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0);
3684ed7b8fbcSLe Tan     /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */
3685ed7b8fbcSLe Tan     vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0);
3686ed7b8fbcSLe Tan 
36871da12ec4SLe Tan     /* IOTLB registers */
36881da12ec4SLe Tan     vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0);
36891da12ec4SLe Tan     vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0);
36901da12ec4SLe Tan     vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL);
36911da12ec4SLe Tan 
36921da12ec4SLe Tan     /* Fault Recording Registers, 128-bit */
36931da12ec4SLe Tan     vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0);
36941da12ec4SLe Tan     vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL);
3695a5861439SPeter Xu 
3696a5861439SPeter Xu     /*
369728589311SJan Kiszka      * Interrupt remapping registers.
3698a5861439SPeter Xu      */
369928589311SJan Kiszka     vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0);
37001da12ec4SLe Tan }
37011da12ec4SLe Tan 
37021da12ec4SLe Tan /* Should not reset address_spaces when reset because devices will still use
37031da12ec4SLe Tan  * the address space they got at first (won't ask the bus again).
37041da12ec4SLe Tan  */
37051da12ec4SLe Tan static void vtd_reset(DeviceState *dev)
37061da12ec4SLe Tan {
37071da12ec4SLe Tan     IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
37081da12ec4SLe Tan 
37091da12ec4SLe Tan     vtd_init(s);
37102cc9ddccSPeter Xu     vtd_address_space_refresh_all(s);
37111da12ec4SLe Tan }
37121da12ec4SLe Tan 
3713621d983aSMarcel Apfelbaum static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
3714621d983aSMarcel Apfelbaum {
3715621d983aSMarcel Apfelbaum     IntelIOMMUState *s = opaque;
3716621d983aSMarcel Apfelbaum     VTDAddressSpace *vtd_as;
3717621d983aSMarcel Apfelbaum 
3718bf33cc75SPeter Xu     assert(0 <= devfn && devfn < PCI_DEVFN_MAX);
3719621d983aSMarcel Apfelbaum 
3720621d983aSMarcel Apfelbaum     vtd_as = vtd_find_add_as(s, bus, devfn);
3721621d983aSMarcel Apfelbaum     return &vtd_as->as;
3722621d983aSMarcel Apfelbaum }
3723621d983aSMarcel Apfelbaum 
3724e6b6af05SRadim Krčmář static bool vtd_decide_config(IntelIOMMUState *s, Error **errp)
37256333e93cSRadim Krčmář {
3726e6b6af05SRadim Krčmář     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
3727e6b6af05SRadim Krčmář 
3728a924b3d8SPeter Xu     if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu_ir_supported(x86_iommu)) {
3729e6b6af05SRadim Krčmář         error_setg(errp, "eim=on cannot be selected without intremap=on");
3730e6b6af05SRadim Krčmář         return false;
3731e6b6af05SRadim Krčmář     }
3732e6b6af05SRadim Krčmář 
3733e6b6af05SRadim Krčmář     if (s->intr_eim == ON_OFF_AUTO_AUTO) {
3734fb506e70SRadim Krčmář         s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim)
3735a924b3d8SPeter Xu                       && x86_iommu_ir_supported(x86_iommu) ?
3736e6b6af05SRadim Krčmář                                               ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
3737e6b6af05SRadim Krčmář     }
3738fb506e70SRadim Krčmář     if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) {
3739fb506e70SRadim Krčmář         if (!kvm_irqchip_in_kernel()) {
3740fb506e70SRadim Krčmář             error_setg(errp, "eim=on requires accel=kvm,kernel-irqchip=split");
3741fb506e70SRadim Krčmář             return false;
3742fb506e70SRadim Krčmář         }
3743fb506e70SRadim Krčmář         if (!kvm_enable_x2apic()) {
3744fb506e70SRadim Krčmář             error_setg(errp, "eim=on requires support on the KVM side"
3745fb506e70SRadim Krčmář                              "(X2APIC_API, first shipped in v4.7)");
3746fb506e70SRadim Krčmář             return false;
3747fb506e70SRadim Krčmář         }
3748fb506e70SRadim Krčmář     }
3749e6b6af05SRadim Krčmář 
375037f51384SPrasad Singamsetty     /* Currently only address widths supported are 39 and 48 bits */
375137f51384SPrasad Singamsetty     if ((s->aw_bits != VTD_HOST_AW_39BIT) &&
375237f51384SPrasad Singamsetty         (s->aw_bits != VTD_HOST_AW_48BIT)) {
375337f51384SPrasad Singamsetty         error_setg(errp, "Supported values for x-aw-bits are: %d, %d",
375437f51384SPrasad Singamsetty                    VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT);
375537f51384SPrasad Singamsetty         return false;
375637f51384SPrasad Singamsetty     }
375737f51384SPrasad Singamsetty 
37584a4f219eSYi Sun     if (s->scalable_mode && !s->dma_drain) {
37594a4f219eSYi Sun         error_setg(errp, "Need to set dma_drain for scalable mode");
37604a4f219eSYi Sun         return false;
37614a4f219eSYi Sun     }
37624a4f219eSYi Sun 
37636333e93cSRadim Krčmář     return true;
37646333e93cSRadim Krčmář }
37656333e93cSRadim Krčmář 
376628cf553aSPeter Xu static int vtd_machine_done_notify_one(Object *child, void *unused)
376728cf553aSPeter Xu {
376828cf553aSPeter Xu     IntelIOMMUState *iommu = INTEL_IOMMU_DEVICE(x86_iommu_get_default());
376928cf553aSPeter Xu 
377028cf553aSPeter Xu     /*
377128cf553aSPeter Xu      * We hard-coded here because vfio-pci is the only special case
377228cf553aSPeter Xu      * here.  Let's be more elegant in the future when we can, but so
377328cf553aSPeter Xu      * far there seems to be no better way.
377428cf553aSPeter Xu      */
377528cf553aSPeter Xu     if (object_dynamic_cast(child, "vfio-pci") && !iommu->caching_mode) {
377628cf553aSPeter Xu         vtd_panic_require_caching_mode();
377728cf553aSPeter Xu     }
377828cf553aSPeter Xu 
377928cf553aSPeter Xu     return 0;
378028cf553aSPeter Xu }
378128cf553aSPeter Xu 
378228cf553aSPeter Xu static void vtd_machine_done_hook(Notifier *notifier, void *unused)
378328cf553aSPeter Xu {
378428cf553aSPeter Xu     object_child_foreach_recursive(object_get_root(),
378528cf553aSPeter Xu                                    vtd_machine_done_notify_one, NULL);
378628cf553aSPeter Xu }
378728cf553aSPeter Xu 
378828cf553aSPeter Xu static Notifier vtd_machine_done_notify = {
378928cf553aSPeter Xu     .notify = vtd_machine_done_hook,
379028cf553aSPeter Xu };
379128cf553aSPeter Xu 
37921da12ec4SLe Tan static void vtd_realize(DeviceState *dev, Error **errp)
37931da12ec4SLe Tan {
3794ef0e8fc7SEduardo Habkost     MachineState *ms = MACHINE(qdev_get_machine());
379529396ed9SMohammed Gamal     PCMachineState *pcms = PC_MACHINE(ms);
3796f0bb276bSPaolo Bonzini     X86MachineState *x86ms = X86_MACHINE(ms);
379729396ed9SMohammed Gamal     PCIBus *bus = pcms->bus;
37981da12ec4SLe Tan     IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
37994684a204SPeter Xu     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev);
38001da12ec4SLe Tan 
3801fb9f5926SDavid Kiarie     x86_iommu->type = TYPE_INTEL;
38026333e93cSRadim Krčmář 
3803e6b6af05SRadim Krčmář     if (!vtd_decide_config(s, errp)) {
38046333e93cSRadim Krčmář         return;
38056333e93cSRadim Krčmář     }
38066333e93cSRadim Krčmář 
3807b4a4ba0dSPeter Xu     QLIST_INIT(&s->vtd_as_with_notifiers);
38081d9efa73SPeter Xu     qemu_mutex_init(&s->iommu_lock);
38097df953bdSKnut Omang     memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
38101da12ec4SLe Tan     memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
38111da12ec4SLe Tan                           "intel_iommu", DMAR_REG_SIZE);
38124b519ef1SPeter Xu 
38134b519ef1SPeter Xu     /* Create the shared memory regions by all devices */
38144b519ef1SPeter Xu     memory_region_init(&s->mr_nodmar, OBJECT(s), "vtd-nodmar",
38154b519ef1SPeter Xu                        UINT64_MAX);
38164b519ef1SPeter Xu     memory_region_init_io(&s->mr_ir, OBJECT(s), &vtd_mem_ir_ops,
38174b519ef1SPeter Xu                           s, "vtd-ir", VTD_INTERRUPT_ADDR_SIZE);
38184b519ef1SPeter Xu     memory_region_init_alias(&s->mr_sys_alias, OBJECT(s),
38194b519ef1SPeter Xu                              "vtd-sys-alias", get_system_memory(), 0,
38204b519ef1SPeter Xu                              memory_region_size(get_system_memory()));
38214b519ef1SPeter Xu     memory_region_add_subregion_overlap(&s->mr_nodmar, 0,
38224b519ef1SPeter Xu                                         &s->mr_sys_alias, 0);
38234b519ef1SPeter Xu     memory_region_add_subregion_overlap(&s->mr_nodmar,
38244b519ef1SPeter Xu                                         VTD_INTERRUPT_ADDR_FIRST,
38254b519ef1SPeter Xu                                         &s->mr_ir, 1);
38264b519ef1SPeter Xu 
38271da12ec4SLe Tan     sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
3828b5a280c0SLe Tan     /* No corresponding destroy */
3829b5a280c0SLe Tan     s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
3830b5a280c0SLe Tan                                      g_free, g_free);
38317df953bdSKnut Omang     s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
38327df953bdSKnut Omang                                               g_free, g_free);
38331da12ec4SLe Tan     vtd_init(s);
3834621d983aSMarcel Apfelbaum     sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
3835621d983aSMarcel Apfelbaum     pci_setup_iommu(bus, vtd_host_dma_iommu, dev);
3836cb135f59SPeter Xu     /* Pseudo address space under root PCI bus. */
3837f0bb276bSPaolo Bonzini     x86ms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC);
383828cf553aSPeter Xu     qemu_add_machine_init_done_notifier(&vtd_machine_done_notify);
38391da12ec4SLe Tan }
38401da12ec4SLe Tan 
38411da12ec4SLe Tan static void vtd_class_init(ObjectClass *klass, void *data)
38421da12ec4SLe Tan {
38431da12ec4SLe Tan     DeviceClass *dc = DEVICE_CLASS(klass);
38441c7955c4SPeter Xu     X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass);
38451da12ec4SLe Tan 
38461da12ec4SLe Tan     dc->reset = vtd_reset;
38471da12ec4SLe Tan     dc->vmsd = &vtd_vmstate;
38481da12ec4SLe Tan     dc->props = vtd_properties;
3849621d983aSMarcel Apfelbaum     dc->hotpluggable = false;
38501c7955c4SPeter Xu     x86_class->realize = vtd_realize;
38518b5ed7dfSPeter Xu     x86_class->int_remap = vtd_int_remap;
38528ab5700cSEduardo Habkost     /* Supported by the pc-q35-* machine types */
3853e4f4fb1eSEduardo Habkost     dc->user_creatable = true;
38541ec202c9SErnest Esene     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
38551ec202c9SErnest Esene     dc->desc = "Intel IOMMU (VT-d) DMA Remapping device";
38561da12ec4SLe Tan }
38571da12ec4SLe Tan 
38581da12ec4SLe Tan static const TypeInfo vtd_info = {
38591da12ec4SLe Tan     .name          = TYPE_INTEL_IOMMU_DEVICE,
38601c7955c4SPeter Xu     .parent        = TYPE_X86_IOMMU_DEVICE,
38611da12ec4SLe Tan     .instance_size = sizeof(IntelIOMMUState),
38621da12ec4SLe Tan     .class_init    = vtd_class_init,
38631da12ec4SLe Tan };
38641da12ec4SLe Tan 
38651221a474SAlexey Kardashevskiy static void vtd_iommu_memory_region_class_init(ObjectClass *klass,
38661221a474SAlexey Kardashevskiy                                                      void *data)
38671221a474SAlexey Kardashevskiy {
38681221a474SAlexey Kardashevskiy     IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
38691221a474SAlexey Kardashevskiy 
38701221a474SAlexey Kardashevskiy     imrc->translate = vtd_iommu_translate;
38711221a474SAlexey Kardashevskiy     imrc->notify_flag_changed = vtd_iommu_notify_flag_changed;
38721221a474SAlexey Kardashevskiy     imrc->replay = vtd_iommu_replay;
38731221a474SAlexey Kardashevskiy }
38741221a474SAlexey Kardashevskiy 
38751221a474SAlexey Kardashevskiy static const TypeInfo vtd_iommu_memory_region_info = {
38761221a474SAlexey Kardashevskiy     .parent = TYPE_IOMMU_MEMORY_REGION,
38771221a474SAlexey Kardashevskiy     .name = TYPE_INTEL_IOMMU_MEMORY_REGION,
38781221a474SAlexey Kardashevskiy     .class_init = vtd_iommu_memory_region_class_init,
38791221a474SAlexey Kardashevskiy };
38801221a474SAlexey Kardashevskiy 
38811da12ec4SLe Tan static void vtd_register_types(void)
38821da12ec4SLe Tan {
38831da12ec4SLe Tan     type_register_static(&vtd_info);
38841221a474SAlexey Kardashevskiy     type_register_static(&vtd_iommu_memory_region_info);
38851da12ec4SLe Tan }
38861da12ec4SLe Tan 
38871da12ec4SLe Tan type_init(vtd_register_types)
3888