xref: /openbmc/qemu/hw/i386/intel_iommu.c (revision 22ec0c69)
1 /*
2  * QEMU emulation of an Intel IOMMU (VT-d)
3  *   (DMA Remapping device)
4  *
5  * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
6  * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12 
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  * GNU General Public License for more details.
17 
18  * You should have received a copy of the GNU General Public License along
19  * with this program; if not, see <http://www.gnu.org/licenses/>.
20  */
21 
22 #include "qemu/osdep.h"
23 #include "qemu/error-report.h"
24 #include "qemu/main-loop.h"
25 #include "qapi/error.h"
26 #include "hw/sysbus.h"
27 #include "exec/address-spaces.h"
28 #include "intel_iommu_internal.h"
29 #include "hw/pci/pci.h"
30 #include "hw/pci/pci_bus.h"
31 #include "hw/qdev-properties.h"
32 #include "hw/i386/pc.h"
33 #include "hw/i386/apic-msidef.h"
34 #include "hw/boards.h"
35 #include "hw/i386/x86-iommu.h"
36 #include "hw/pci-host/q35.h"
37 #include "sysemu/kvm.h"
38 #include "sysemu/sysemu.h"
39 #include "hw/i386/apic_internal.h"
40 #include "kvm/kvm_i386.h"
41 #include "migration/vmstate.h"
42 #include "trace.h"
43 
44 /* context entry operations */
45 #define VTD_CE_GET_RID2PASID(ce) \
46     ((ce)->val[1] & VTD_SM_CONTEXT_ENTRY_RID2PASID_MASK)
47 #define VTD_CE_GET_PASID_DIR_TABLE(ce) \
48     ((ce)->val[0] & VTD_PASID_DIR_BASE_ADDR_MASK)
49 
50 /* pe operations */
51 #define VTD_PE_GET_TYPE(pe) ((pe)->val[0] & VTD_SM_PASID_ENTRY_PGTT)
52 #define VTD_PE_GET_LEVEL(pe) (2 + (((pe)->val[0] >> 2) & VTD_SM_PASID_ENTRY_AW))
53 #define VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write) {\
54     if (ret_fr) {                                                             \
55         ret_fr = -ret_fr;                                                     \
56         if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {                   \
57             trace_vtd_fault_disabled();                                       \
58         } else {                                                              \
59             vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);      \
60         }                                                                     \
61         goto error;                                                           \
62     }                                                                         \
63 }
64 
65 static void vtd_address_space_refresh_all(IntelIOMMUState *s);
66 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n);
67 
68 static void vtd_panic_require_caching_mode(void)
69 {
70     error_report("We need to set caching-mode=on for intel-iommu to enable "
71                  "device assignment with IOMMU protection.");
72     exit(1);
73 }
74 
75 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
76                             uint64_t wmask, uint64_t w1cmask)
77 {
78     stq_le_p(&s->csr[addr], val);
79     stq_le_p(&s->wmask[addr], wmask);
80     stq_le_p(&s->w1cmask[addr], w1cmask);
81 }
82 
83 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask)
84 {
85     stq_le_p(&s->womask[addr], mask);
86 }
87 
88 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val,
89                             uint32_t wmask, uint32_t w1cmask)
90 {
91     stl_le_p(&s->csr[addr], val);
92     stl_le_p(&s->wmask[addr], wmask);
93     stl_le_p(&s->w1cmask[addr], w1cmask);
94 }
95 
96 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask)
97 {
98     stl_le_p(&s->womask[addr], mask);
99 }
100 
101 /* "External" get/set operations */
102 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val)
103 {
104     uint64_t oldval = ldq_le_p(&s->csr[addr]);
105     uint64_t wmask = ldq_le_p(&s->wmask[addr]);
106     uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
107     stq_le_p(&s->csr[addr],
108              ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
109 }
110 
111 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val)
112 {
113     uint32_t oldval = ldl_le_p(&s->csr[addr]);
114     uint32_t wmask = ldl_le_p(&s->wmask[addr]);
115     uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
116     stl_le_p(&s->csr[addr],
117              ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
118 }
119 
120 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr)
121 {
122     uint64_t val = ldq_le_p(&s->csr[addr]);
123     uint64_t womask = ldq_le_p(&s->womask[addr]);
124     return val & ~womask;
125 }
126 
127 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr)
128 {
129     uint32_t val = ldl_le_p(&s->csr[addr]);
130     uint32_t womask = ldl_le_p(&s->womask[addr]);
131     return val & ~womask;
132 }
133 
134 /* "Internal" get/set operations */
135 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr)
136 {
137     return ldq_le_p(&s->csr[addr]);
138 }
139 
140 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr)
141 {
142     return ldl_le_p(&s->csr[addr]);
143 }
144 
145 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val)
146 {
147     stq_le_p(&s->csr[addr], val);
148 }
149 
150 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr,
151                                         uint32_t clear, uint32_t mask)
152 {
153     uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask;
154     stl_le_p(&s->csr[addr], new_val);
155     return new_val;
156 }
157 
158 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
159                                         uint64_t clear, uint64_t mask)
160 {
161     uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask;
162     stq_le_p(&s->csr[addr], new_val);
163     return new_val;
164 }
165 
166 static inline void vtd_iommu_lock(IntelIOMMUState *s)
167 {
168     qemu_mutex_lock(&s->iommu_lock);
169 }
170 
171 static inline void vtd_iommu_unlock(IntelIOMMUState *s)
172 {
173     qemu_mutex_unlock(&s->iommu_lock);
174 }
175 
176 static void vtd_update_scalable_state(IntelIOMMUState *s)
177 {
178     uint64_t val = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
179 
180     if (s->scalable_mode) {
181         s->root_scalable = val & VTD_RTADDR_SMT;
182     }
183 }
184 
185 /* Whether the address space needs to notify new mappings */
186 static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as)
187 {
188     return as->notifier_flags & IOMMU_NOTIFIER_MAP;
189 }
190 
191 /* GHashTable functions */
192 static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2)
193 {
194     return *((const uint64_t *)v1) == *((const uint64_t *)v2);
195 }
196 
197 static guint vtd_uint64_hash(gconstpointer v)
198 {
199     return (guint)*(const uint64_t *)v;
200 }
201 
202 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
203                                           gpointer user_data)
204 {
205     VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
206     uint16_t domain_id = *(uint16_t *)user_data;
207     return entry->domain_id == domain_id;
208 }
209 
210 /* The shift of an addr for a certain level of paging structure */
211 static inline uint32_t vtd_slpt_level_shift(uint32_t level)
212 {
213     assert(level != 0);
214     return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
215 }
216 
217 static inline uint64_t vtd_slpt_level_page_mask(uint32_t level)
218 {
219     return ~((1ULL << vtd_slpt_level_shift(level)) - 1);
220 }
221 
222 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
223                                         gpointer user_data)
224 {
225     VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
226     VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
227     uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
228     uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
229     return (entry->domain_id == info->domain_id) &&
230             (((entry->gfn & info->mask) == gfn) ||
231              (entry->gfn == gfn_tlb));
232 }
233 
234 /* Reset all the gen of VTDAddressSpace to zero and set the gen of
235  * IntelIOMMUState to 1.  Must be called with IOMMU lock held.
236  */
237 static void vtd_reset_context_cache_locked(IntelIOMMUState *s)
238 {
239     VTDAddressSpace *vtd_as;
240     VTDBus *vtd_bus;
241     GHashTableIter bus_it;
242     uint32_t devfn_it;
243 
244     trace_vtd_context_cache_reset();
245 
246     g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
247 
248     while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
249         for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) {
250             vtd_as = vtd_bus->dev_as[devfn_it];
251             if (!vtd_as) {
252                 continue;
253             }
254             vtd_as->context_cache_entry.context_cache_gen = 0;
255         }
256     }
257     s->context_cache_gen = 1;
258 }
259 
260 /* Must be called with IOMMU lock held. */
261 static void vtd_reset_iotlb_locked(IntelIOMMUState *s)
262 {
263     assert(s->iotlb);
264     g_hash_table_remove_all(s->iotlb);
265 }
266 
267 static void vtd_reset_iotlb(IntelIOMMUState *s)
268 {
269     vtd_iommu_lock(s);
270     vtd_reset_iotlb_locked(s);
271     vtd_iommu_unlock(s);
272 }
273 
274 static void vtd_reset_caches(IntelIOMMUState *s)
275 {
276     vtd_iommu_lock(s);
277     vtd_reset_iotlb_locked(s);
278     vtd_reset_context_cache_locked(s);
279     vtd_iommu_unlock(s);
280 }
281 
282 static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint16_t source_id,
283                                   uint32_t level)
284 {
285     return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) |
286            ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT);
287 }
288 
289 static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
290 {
291     return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
292 }
293 
294 /* Must be called with IOMMU lock held */
295 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
296                                        hwaddr addr)
297 {
298     VTDIOTLBEntry *entry;
299     uint64_t key;
300     int level;
301 
302     for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) {
303         key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level),
304                                 source_id, level);
305         entry = g_hash_table_lookup(s->iotlb, &key);
306         if (entry) {
307             goto out;
308         }
309     }
310 
311 out:
312     return entry;
313 }
314 
315 /* Must be with IOMMU lock held */
316 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
317                              uint16_t domain_id, hwaddr addr, uint64_t slpte,
318                              uint8_t access_flags, uint32_t level)
319 {
320     VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
321     uint64_t *key = g_malloc(sizeof(*key));
322     uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
323 
324     trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id);
325     if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) {
326         trace_vtd_iotlb_reset("iotlb exceeds size limit");
327         vtd_reset_iotlb_locked(s);
328     }
329 
330     entry->gfn = gfn;
331     entry->domain_id = domain_id;
332     entry->slpte = slpte;
333     entry->access_flags = access_flags;
334     entry->mask = vtd_slpt_level_page_mask(level);
335     *key = vtd_get_iotlb_key(gfn, source_id, level);
336     g_hash_table_replace(s->iotlb, key, entry);
337 }
338 
339 /* Given the reg addr of both the message data and address, generate an
340  * interrupt via MSI.
341  */
342 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg,
343                                    hwaddr mesg_data_reg)
344 {
345     MSIMessage msi;
346 
347     assert(mesg_data_reg < DMAR_REG_SIZE);
348     assert(mesg_addr_reg < DMAR_REG_SIZE);
349 
350     msi.address = vtd_get_long_raw(s, mesg_addr_reg);
351     msi.data = vtd_get_long_raw(s, mesg_data_reg);
352 
353     trace_vtd_irq_generate(msi.address, msi.data);
354 
355     apic_get_class()->send_msi(&msi);
356 }
357 
358 /* Generate a fault event to software via MSI if conditions are met.
359  * Notice that the value of FSTS_REG being passed to it should be the one
360  * before any update.
361  */
362 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts)
363 {
364     if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO ||
365         pre_fsts & VTD_FSTS_IQE) {
366         error_report_once("There are previous interrupt conditions "
367                           "to be serviced by software, fault event "
368                           "is not generated");
369         return;
370     }
371     vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP);
372     if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) {
373         error_report_once("Interrupt Mask set, irq is not generated");
374     } else {
375         vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
376         vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
377     }
378 }
379 
380 /* Check if the Fault (F) field of the Fault Recording Register referenced by
381  * @index is Set.
382  */
383 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index)
384 {
385     /* Each reg is 128-bit */
386     hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
387     addr += 8; /* Access the high 64-bit half */
388 
389     assert(index < DMAR_FRCD_REG_NR);
390 
391     return vtd_get_quad_raw(s, addr) & VTD_FRCD_F;
392 }
393 
394 /* Update the PPF field of Fault Status Register.
395  * Should be called whenever change the F field of any fault recording
396  * registers.
397  */
398 static void vtd_update_fsts_ppf(IntelIOMMUState *s)
399 {
400     uint32_t i;
401     uint32_t ppf_mask = 0;
402 
403     for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
404         if (vtd_is_frcd_set(s, i)) {
405             ppf_mask = VTD_FSTS_PPF;
406             break;
407         }
408     }
409     vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask);
410     trace_vtd_fsts_ppf(!!ppf_mask);
411 }
412 
413 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index)
414 {
415     /* Each reg is 128-bit */
416     hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
417     addr += 8; /* Access the high 64-bit half */
418 
419     assert(index < DMAR_FRCD_REG_NR);
420 
421     vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F);
422     vtd_update_fsts_ppf(s);
423 }
424 
425 /* Must not update F field now, should be done later */
426 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index,
427                             uint16_t source_id, hwaddr addr,
428                             VTDFaultReason fault, bool is_write)
429 {
430     uint64_t hi = 0, lo;
431     hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
432 
433     assert(index < DMAR_FRCD_REG_NR);
434 
435     lo = VTD_FRCD_FI(addr);
436     hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault);
437     if (!is_write) {
438         hi |= VTD_FRCD_T;
439     }
440     vtd_set_quad_raw(s, frcd_reg_addr, lo);
441     vtd_set_quad_raw(s, frcd_reg_addr + 8, hi);
442 
443     trace_vtd_frr_new(index, hi, lo);
444 }
445 
446 /* Try to collapse multiple pending faults from the same requester */
447 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id)
448 {
449     uint32_t i;
450     uint64_t frcd_reg;
451     hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */
452 
453     for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
454         frcd_reg = vtd_get_quad_raw(s, addr);
455         if ((frcd_reg & VTD_FRCD_F) &&
456             ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) {
457             return true;
458         }
459         addr += 16; /* 128-bit for each */
460     }
461     return false;
462 }
463 
464 /* Log and report an DMAR (address translation) fault to software */
465 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id,
466                                   hwaddr addr, VTDFaultReason fault,
467                                   bool is_write)
468 {
469     uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
470 
471     assert(fault < VTD_FR_MAX);
472 
473     if (fault == VTD_FR_RESERVED_ERR) {
474         /* This is not a normal fault reason case. Drop it. */
475         return;
476     }
477 
478     trace_vtd_dmar_fault(source_id, fault, addr, is_write);
479 
480     if (fsts_reg & VTD_FSTS_PFO) {
481         error_report_once("New fault is not recorded due to "
482                           "Primary Fault Overflow");
483         return;
484     }
485 
486     if (vtd_try_collapse_fault(s, source_id)) {
487         error_report_once("New fault is not recorded due to "
488                           "compression of faults");
489         return;
490     }
491 
492     if (vtd_is_frcd_set(s, s->next_frcd_reg)) {
493         error_report_once("Next Fault Recording Reg is used, "
494                           "new fault is not recorded, set PFO field");
495         vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO);
496         return;
497     }
498 
499     vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write);
500 
501     if (fsts_reg & VTD_FSTS_PPF) {
502         error_report_once("There are pending faults already, "
503                           "fault event is not generated");
504         vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg);
505         s->next_frcd_reg++;
506         if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
507             s->next_frcd_reg = 0;
508         }
509     } else {
510         vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK,
511                                 VTD_FSTS_FRI(s->next_frcd_reg));
512         vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */
513         s->next_frcd_reg++;
514         if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
515             s->next_frcd_reg = 0;
516         }
517         /* This case actually cause the PPF to be Set.
518          * So generate fault event (interrupt).
519          */
520          vtd_generate_fault_event(s, fsts_reg);
521     }
522 }
523 
524 /* Handle Invalidation Queue Errors of queued invalidation interface error
525  * conditions.
526  */
527 static void vtd_handle_inv_queue_error(IntelIOMMUState *s)
528 {
529     uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
530 
531     vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE);
532     vtd_generate_fault_event(s, fsts_reg);
533 }
534 
535 /* Set the IWC field and try to generate an invalidation completion interrupt */
536 static void vtd_generate_completion_event(IntelIOMMUState *s)
537 {
538     if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) {
539         trace_vtd_inv_desc_wait_irq("One pending, skip current");
540         return;
541     }
542     vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC);
543     vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP);
544     if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) {
545         trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, "
546                                     "new event not generated");
547         return;
548     } else {
549         /* Generate the interrupt event */
550         trace_vtd_inv_desc_wait_irq("Generating complete event");
551         vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
552         vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
553     }
554 }
555 
556 static inline bool vtd_root_entry_present(IntelIOMMUState *s,
557                                           VTDRootEntry *re,
558                                           uint8_t devfn)
559 {
560     if (s->root_scalable && devfn > UINT8_MAX / 2) {
561         return re->hi & VTD_ROOT_ENTRY_P;
562     }
563 
564     return re->lo & VTD_ROOT_ENTRY_P;
565 }
566 
567 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index,
568                               VTDRootEntry *re)
569 {
570     dma_addr_t addr;
571 
572     addr = s->root + index * sizeof(*re);
573     if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) {
574         re->lo = 0;
575         return -VTD_FR_ROOT_TABLE_INV;
576     }
577     re->lo = le64_to_cpu(re->lo);
578     re->hi = le64_to_cpu(re->hi);
579     return 0;
580 }
581 
582 static inline bool vtd_ce_present(VTDContextEntry *context)
583 {
584     return context->lo & VTD_CONTEXT_ENTRY_P;
585 }
586 
587 static int vtd_get_context_entry_from_root(IntelIOMMUState *s,
588                                            VTDRootEntry *re,
589                                            uint8_t index,
590                                            VTDContextEntry *ce)
591 {
592     dma_addr_t addr, ce_size;
593 
594     /* we have checked that root entry is present */
595     ce_size = s->root_scalable ? VTD_CTX_ENTRY_SCALABLE_SIZE :
596               VTD_CTX_ENTRY_LEGACY_SIZE;
597 
598     if (s->root_scalable && index > UINT8_MAX / 2) {
599         index = index & (~VTD_DEVFN_CHECK_MASK);
600         addr = re->hi & VTD_ROOT_ENTRY_CTP;
601     } else {
602         addr = re->lo & VTD_ROOT_ENTRY_CTP;
603     }
604 
605     addr = addr + index * ce_size;
606     if (dma_memory_read(&address_space_memory, addr, ce, ce_size)) {
607         return -VTD_FR_CONTEXT_TABLE_INV;
608     }
609 
610     ce->lo = le64_to_cpu(ce->lo);
611     ce->hi = le64_to_cpu(ce->hi);
612     if (ce_size == VTD_CTX_ENTRY_SCALABLE_SIZE) {
613         ce->val[2] = le64_to_cpu(ce->val[2]);
614         ce->val[3] = le64_to_cpu(ce->val[3]);
615     }
616     return 0;
617 }
618 
619 static inline dma_addr_t vtd_ce_get_slpt_base(VTDContextEntry *ce)
620 {
621     return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
622 }
623 
624 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte, uint8_t aw)
625 {
626     return slpte & VTD_SL_PT_BASE_ADDR_MASK(aw);
627 }
628 
629 /* Whether the pte indicates the address of the page frame */
630 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level)
631 {
632     return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK);
633 }
634 
635 /* Get the content of a spte located in @base_addr[@index] */
636 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index)
637 {
638     uint64_t slpte;
639 
640     assert(index < VTD_SL_PT_ENTRY_NR);
641 
642     if (dma_memory_read(&address_space_memory,
643                         base_addr + index * sizeof(slpte), &slpte,
644                         sizeof(slpte))) {
645         slpte = (uint64_t)-1;
646         return slpte;
647     }
648     slpte = le64_to_cpu(slpte);
649     return slpte;
650 }
651 
652 /* Given an iova and the level of paging structure, return the offset
653  * of current level.
654  */
655 static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level)
656 {
657     return (iova >> vtd_slpt_level_shift(level)) &
658             ((1ULL << VTD_SL_LEVEL_BITS) - 1);
659 }
660 
661 /* Check Capability Register to see if the @level of page-table is supported */
662 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level)
663 {
664     return VTD_CAP_SAGAW_MASK & s->cap &
665            (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT));
666 }
667 
668 /* Return true if check passed, otherwise false */
669 static inline bool vtd_pe_type_check(X86IOMMUState *x86_iommu,
670                                      VTDPASIDEntry *pe)
671 {
672     switch (VTD_PE_GET_TYPE(pe)) {
673     case VTD_SM_PASID_ENTRY_FLT:
674     case VTD_SM_PASID_ENTRY_SLT:
675     case VTD_SM_PASID_ENTRY_NESTED:
676         break;
677     case VTD_SM_PASID_ENTRY_PT:
678         if (!x86_iommu->pt_supported) {
679             return false;
680         }
681         break;
682     default:
683         /* Unknwon type */
684         return false;
685     }
686     return true;
687 }
688 
689 static inline bool vtd_pdire_present(VTDPASIDDirEntry *pdire)
690 {
691     return pdire->val & 1;
692 }
693 
694 /**
695  * Caller of this function should check present bit if wants
696  * to use pdir entry for futher usage except for fpd bit check.
697  */
698 static int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base,
699                                          uint32_t pasid,
700                                          VTDPASIDDirEntry *pdire)
701 {
702     uint32_t index;
703     dma_addr_t addr, entry_size;
704 
705     index = VTD_PASID_DIR_INDEX(pasid);
706     entry_size = VTD_PASID_DIR_ENTRY_SIZE;
707     addr = pasid_dir_base + index * entry_size;
708     if (dma_memory_read(&address_space_memory, addr, pdire, entry_size)) {
709         return -VTD_FR_PASID_TABLE_INV;
710     }
711 
712     return 0;
713 }
714 
715 static inline bool vtd_pe_present(VTDPASIDEntry *pe)
716 {
717     return pe->val[0] & VTD_PASID_ENTRY_P;
718 }
719 
720 static int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s,
721                                           uint32_t pasid,
722                                           dma_addr_t addr,
723                                           VTDPASIDEntry *pe)
724 {
725     uint32_t index;
726     dma_addr_t entry_size;
727     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
728 
729     index = VTD_PASID_TABLE_INDEX(pasid);
730     entry_size = VTD_PASID_ENTRY_SIZE;
731     addr = addr + index * entry_size;
732     if (dma_memory_read(&address_space_memory, addr, pe, entry_size)) {
733         return -VTD_FR_PASID_TABLE_INV;
734     }
735 
736     /* Do translation type check */
737     if (!vtd_pe_type_check(x86_iommu, pe)) {
738         return -VTD_FR_PASID_TABLE_INV;
739     }
740 
741     if (!vtd_is_level_supported(s, VTD_PE_GET_LEVEL(pe))) {
742         return -VTD_FR_PASID_TABLE_INV;
743     }
744 
745     return 0;
746 }
747 
748 /**
749  * Caller of this function should check present bit if wants
750  * to use pasid entry for futher usage except for fpd bit check.
751  */
752 static int vtd_get_pe_from_pdire(IntelIOMMUState *s,
753                                  uint32_t pasid,
754                                  VTDPASIDDirEntry *pdire,
755                                  VTDPASIDEntry *pe)
756 {
757     dma_addr_t addr = pdire->val & VTD_PASID_TABLE_BASE_ADDR_MASK;
758 
759     return vtd_get_pe_in_pasid_leaf_table(s, pasid, addr, pe);
760 }
761 
762 /**
763  * This function gets a pasid entry from a specified pasid
764  * table (includes dir and leaf table) with a specified pasid.
765  * Sanity check should be done to ensure return a present
766  * pasid entry to caller.
767  */
768 static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s,
769                                        dma_addr_t pasid_dir_base,
770                                        uint32_t pasid,
771                                        VTDPASIDEntry *pe)
772 {
773     int ret;
774     VTDPASIDDirEntry pdire;
775 
776     ret = vtd_get_pdire_from_pdir_table(pasid_dir_base,
777                                         pasid, &pdire);
778     if (ret) {
779         return ret;
780     }
781 
782     if (!vtd_pdire_present(&pdire)) {
783         return -VTD_FR_PASID_TABLE_INV;
784     }
785 
786     ret = vtd_get_pe_from_pdire(s, pasid, &pdire, pe);
787     if (ret) {
788         return ret;
789     }
790 
791     if (!vtd_pe_present(pe)) {
792         return -VTD_FR_PASID_TABLE_INV;
793     }
794 
795     return 0;
796 }
797 
798 static int vtd_ce_get_rid2pasid_entry(IntelIOMMUState *s,
799                                       VTDContextEntry *ce,
800                                       VTDPASIDEntry *pe)
801 {
802     uint32_t pasid;
803     dma_addr_t pasid_dir_base;
804     int ret = 0;
805 
806     pasid = VTD_CE_GET_RID2PASID(ce);
807     pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce);
808     ret = vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe);
809 
810     return ret;
811 }
812 
813 static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s,
814                                 VTDContextEntry *ce,
815                                 bool *pe_fpd_set)
816 {
817     int ret;
818     uint32_t pasid;
819     dma_addr_t pasid_dir_base;
820     VTDPASIDDirEntry pdire;
821     VTDPASIDEntry pe;
822 
823     pasid = VTD_CE_GET_RID2PASID(ce);
824     pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce);
825 
826     /*
827      * No present bit check since fpd is meaningful even
828      * if the present bit is clear.
829      */
830     ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, pasid, &pdire);
831     if (ret) {
832         return ret;
833     }
834 
835     if (pdire.val & VTD_PASID_DIR_FPD) {
836         *pe_fpd_set = true;
837         return 0;
838     }
839 
840     if (!vtd_pdire_present(&pdire)) {
841         return -VTD_FR_PASID_TABLE_INV;
842     }
843 
844     /*
845      * No present bit check since fpd is meaningful even
846      * if the present bit is clear.
847      */
848     ret = vtd_get_pe_from_pdire(s, pasid, &pdire, &pe);
849     if (ret) {
850         return ret;
851     }
852 
853     if (pe.val[0] & VTD_PASID_ENTRY_FPD) {
854         *pe_fpd_set = true;
855     }
856 
857     return 0;
858 }
859 
860 /* Get the page-table level that hardware should use for the second-level
861  * page-table walk from the Address Width field of context-entry.
862  */
863 static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce)
864 {
865     return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW);
866 }
867 
868 static uint32_t vtd_get_iova_level(IntelIOMMUState *s,
869                                    VTDContextEntry *ce)
870 {
871     VTDPASIDEntry pe;
872 
873     if (s->root_scalable) {
874         vtd_ce_get_rid2pasid_entry(s, ce, &pe);
875         return VTD_PE_GET_LEVEL(&pe);
876     }
877 
878     return vtd_ce_get_level(ce);
879 }
880 
881 static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce)
882 {
883     return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9;
884 }
885 
886 static uint32_t vtd_get_iova_agaw(IntelIOMMUState *s,
887                                   VTDContextEntry *ce)
888 {
889     VTDPASIDEntry pe;
890 
891     if (s->root_scalable) {
892         vtd_ce_get_rid2pasid_entry(s, ce, &pe);
893         return 30 + ((pe.val[0] >> 2) & VTD_SM_PASID_ENTRY_AW) * 9;
894     }
895 
896     return vtd_ce_get_agaw(ce);
897 }
898 
899 static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce)
900 {
901     return ce->lo & VTD_CONTEXT_ENTRY_TT;
902 }
903 
904 /* Only for Legacy Mode. Return true if check passed, otherwise false */
905 static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu,
906                                      VTDContextEntry *ce)
907 {
908     switch (vtd_ce_get_type(ce)) {
909     case VTD_CONTEXT_TT_MULTI_LEVEL:
910         /* Always supported */
911         break;
912     case VTD_CONTEXT_TT_DEV_IOTLB:
913         if (!x86_iommu->dt_supported) {
914             error_report_once("%s: DT specified but not supported", __func__);
915             return false;
916         }
917         break;
918     case VTD_CONTEXT_TT_PASS_THROUGH:
919         if (!x86_iommu->pt_supported) {
920             error_report_once("%s: PT specified but not supported", __func__);
921             return false;
922         }
923         break;
924     default:
925         /* Unknown type */
926         error_report_once("%s: unknown ce type: %"PRIu32, __func__,
927                           vtd_ce_get_type(ce));
928         return false;
929     }
930     return true;
931 }
932 
933 static inline uint64_t vtd_iova_limit(IntelIOMMUState *s,
934                                       VTDContextEntry *ce, uint8_t aw)
935 {
936     uint32_t ce_agaw = vtd_get_iova_agaw(s, ce);
937     return 1ULL << MIN(ce_agaw, aw);
938 }
939 
940 /* Return true if IOVA passes range check, otherwise false. */
941 static inline bool vtd_iova_range_check(IntelIOMMUState *s,
942                                         uint64_t iova, VTDContextEntry *ce,
943                                         uint8_t aw)
944 {
945     /*
946      * Check if @iova is above 2^X-1, where X is the minimum of MGAW
947      * in CAP_REG and AW in context-entry.
948      */
949     return !(iova & ~(vtd_iova_limit(s, ce, aw) - 1));
950 }
951 
952 static dma_addr_t vtd_get_iova_pgtbl_base(IntelIOMMUState *s,
953                                           VTDContextEntry *ce)
954 {
955     VTDPASIDEntry pe;
956 
957     if (s->root_scalable) {
958         vtd_ce_get_rid2pasid_entry(s, ce, &pe);
959         return pe.val[0] & VTD_SM_PASID_ENTRY_SLPTPTR;
960     }
961 
962     return vtd_ce_get_slpt_base(ce);
963 }
964 
965 /*
966  * Rsvd field masks for spte:
967  *     vtd_spte_rsvd 4k pages
968  *     vtd_spte_rsvd_large large pages
969  */
970 static uint64_t vtd_spte_rsvd[5];
971 static uint64_t vtd_spte_rsvd_large[5];
972 
973 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level)
974 {
975     uint64_t rsvd_mask = vtd_spte_rsvd[level];
976 
977     if ((level == VTD_SL_PD_LEVEL || level == VTD_SL_PDP_LEVEL) &&
978         (slpte & VTD_SL_PT_PAGE_SIZE_MASK)) {
979         /* large page */
980         rsvd_mask = vtd_spte_rsvd_large[level];
981     }
982 
983     return slpte & rsvd_mask;
984 }
985 
986 /* Find the VTD address space associated with a given bus number */
987 static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
988 {
989     VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
990     GHashTableIter iter;
991 
992     if (vtd_bus) {
993         return vtd_bus;
994     }
995 
996     /*
997      * Iterate over the registered buses to find the one which
998      * currently holds this bus number and update the bus_num
999      * lookup table.
1000      */
1001     g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
1002     while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) {
1003         if (pci_bus_num(vtd_bus->bus) == bus_num) {
1004             s->vtd_as_by_bus_num[bus_num] = vtd_bus;
1005             return vtd_bus;
1006         }
1007     }
1008 
1009     return NULL;
1010 }
1011 
1012 /* Given the @iova, get relevant @slptep. @slpte_level will be the last level
1013  * of the translation, can be used for deciding the size of large page.
1014  */
1015 static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce,
1016                              uint64_t iova, bool is_write,
1017                              uint64_t *slptep, uint32_t *slpte_level,
1018                              bool *reads, bool *writes, uint8_t aw_bits)
1019 {
1020     dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce);
1021     uint32_t level = vtd_get_iova_level(s, ce);
1022     uint32_t offset;
1023     uint64_t slpte;
1024     uint64_t access_right_check;
1025 
1026     if (!vtd_iova_range_check(s, iova, ce, aw_bits)) {
1027         error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 ")",
1028                           __func__, iova);
1029         return -VTD_FR_ADDR_BEYOND_MGAW;
1030     }
1031 
1032     /* FIXME: what is the Atomics request here? */
1033     access_right_check = is_write ? VTD_SL_W : VTD_SL_R;
1034 
1035     while (true) {
1036         offset = vtd_iova_level_offset(iova, level);
1037         slpte = vtd_get_slpte(addr, offset);
1038 
1039         if (slpte == (uint64_t)-1) {
1040             error_report_once("%s: detected read error on DMAR slpte "
1041                               "(iova=0x%" PRIx64 ")", __func__, iova);
1042             if (level == vtd_get_iova_level(s, ce)) {
1043                 /* Invalid programming of context-entry */
1044                 return -VTD_FR_CONTEXT_ENTRY_INV;
1045             } else {
1046                 return -VTD_FR_PAGING_ENTRY_INV;
1047             }
1048         }
1049         *reads = (*reads) && (slpte & VTD_SL_R);
1050         *writes = (*writes) && (slpte & VTD_SL_W);
1051         if (!(slpte & access_right_check)) {
1052             error_report_once("%s: detected slpte permission error "
1053                               "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", "
1054                               "slpte=0x%" PRIx64 ", write=%d)", __func__,
1055                               iova, level, slpte, is_write);
1056             return is_write ? -VTD_FR_WRITE : -VTD_FR_READ;
1057         }
1058         if (vtd_slpte_nonzero_rsvd(slpte, level)) {
1059             error_report_once("%s: detected splte reserve non-zero "
1060                               "iova=0x%" PRIx64 ", level=0x%" PRIx32
1061                               "slpte=0x%" PRIx64 ")", __func__, iova,
1062                               level, slpte);
1063             return -VTD_FR_PAGING_ENTRY_RSVD;
1064         }
1065 
1066         if (vtd_is_last_slpte(slpte, level)) {
1067             *slptep = slpte;
1068             *slpte_level = level;
1069             return 0;
1070         }
1071         addr = vtd_get_slpte_addr(slpte, aw_bits);
1072         level--;
1073     }
1074 }
1075 
1076 typedef int (*vtd_page_walk_hook)(IOMMUTLBEvent *event, void *private);
1077 
1078 /**
1079  * Constant information used during page walking
1080  *
1081  * @hook_fn: hook func to be called when detected page
1082  * @private: private data to be passed into hook func
1083  * @notify_unmap: whether we should notify invalid entries
1084  * @as: VT-d address space of the device
1085  * @aw: maximum address width
1086  * @domain: domain ID of the page walk
1087  */
1088 typedef struct {
1089     VTDAddressSpace *as;
1090     vtd_page_walk_hook hook_fn;
1091     void *private;
1092     bool notify_unmap;
1093     uint8_t aw;
1094     uint16_t domain_id;
1095 } vtd_page_walk_info;
1096 
1097 static int vtd_page_walk_one(IOMMUTLBEvent *event, vtd_page_walk_info *info)
1098 {
1099     VTDAddressSpace *as = info->as;
1100     vtd_page_walk_hook hook_fn = info->hook_fn;
1101     void *private = info->private;
1102     IOMMUTLBEntry *entry = &event->entry;
1103     DMAMap target = {
1104         .iova = entry->iova,
1105         .size = entry->addr_mask,
1106         .translated_addr = entry->translated_addr,
1107         .perm = entry->perm,
1108     };
1109     DMAMap *mapped = iova_tree_find(as->iova_tree, &target);
1110 
1111     if (event->type == IOMMU_NOTIFIER_UNMAP && !info->notify_unmap) {
1112         trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask);
1113         return 0;
1114     }
1115 
1116     assert(hook_fn);
1117 
1118     /* Update local IOVA mapped ranges */
1119     if (event->type == IOMMU_NOTIFIER_MAP) {
1120         if (mapped) {
1121             /* If it's exactly the same translation, skip */
1122             if (!memcmp(mapped, &target, sizeof(target))) {
1123                 trace_vtd_page_walk_one_skip_map(entry->iova, entry->addr_mask,
1124                                                  entry->translated_addr);
1125                 return 0;
1126             } else {
1127                 /*
1128                  * Translation changed.  Normally this should not
1129                  * happen, but it can happen when with buggy guest
1130                  * OSes.  Note that there will be a small window that
1131                  * we don't have map at all.  But that's the best
1132                  * effort we can do.  The ideal way to emulate this is
1133                  * atomically modify the PTE to follow what has
1134                  * changed, but we can't.  One example is that vfio
1135                  * driver only has VFIO_IOMMU_[UN]MAP_DMA but no
1136                  * interface to modify a mapping (meanwhile it seems
1137                  * meaningless to even provide one).  Anyway, let's
1138                  * mark this as a TODO in case one day we'll have
1139                  * a better solution.
1140                  */
1141                 IOMMUAccessFlags cache_perm = entry->perm;
1142                 int ret;
1143 
1144                 /* Emulate an UNMAP */
1145                 event->type = IOMMU_NOTIFIER_UNMAP;
1146                 entry->perm = IOMMU_NONE;
1147                 trace_vtd_page_walk_one(info->domain_id,
1148                                         entry->iova,
1149                                         entry->translated_addr,
1150                                         entry->addr_mask,
1151                                         entry->perm);
1152                 ret = hook_fn(event, private);
1153                 if (ret) {
1154                     return ret;
1155                 }
1156                 /* Drop any existing mapping */
1157                 iova_tree_remove(as->iova_tree, &target);
1158                 /* Recover the correct type */
1159                 event->type = IOMMU_NOTIFIER_MAP;
1160                 entry->perm = cache_perm;
1161             }
1162         }
1163         iova_tree_insert(as->iova_tree, &target);
1164     } else {
1165         if (!mapped) {
1166             /* Skip since we didn't map this range at all */
1167             trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask);
1168             return 0;
1169         }
1170         iova_tree_remove(as->iova_tree, &target);
1171     }
1172 
1173     trace_vtd_page_walk_one(info->domain_id, entry->iova,
1174                             entry->translated_addr, entry->addr_mask,
1175                             entry->perm);
1176     return hook_fn(event, private);
1177 }
1178 
1179 /**
1180  * vtd_page_walk_level - walk over specific level for IOVA range
1181  *
1182  * @addr: base GPA addr to start the walk
1183  * @start: IOVA range start address
1184  * @end: IOVA range end address (start <= addr < end)
1185  * @read: whether parent level has read permission
1186  * @write: whether parent level has write permission
1187  * @info: constant information for the page walk
1188  */
1189 static int vtd_page_walk_level(dma_addr_t addr, uint64_t start,
1190                                uint64_t end, uint32_t level, bool read,
1191                                bool write, vtd_page_walk_info *info)
1192 {
1193     bool read_cur, write_cur, entry_valid;
1194     uint32_t offset;
1195     uint64_t slpte;
1196     uint64_t subpage_size, subpage_mask;
1197     IOMMUTLBEvent event;
1198     uint64_t iova = start;
1199     uint64_t iova_next;
1200     int ret = 0;
1201 
1202     trace_vtd_page_walk_level(addr, level, start, end);
1203 
1204     subpage_size = 1ULL << vtd_slpt_level_shift(level);
1205     subpage_mask = vtd_slpt_level_page_mask(level);
1206 
1207     while (iova < end) {
1208         iova_next = (iova & subpage_mask) + subpage_size;
1209 
1210         offset = vtd_iova_level_offset(iova, level);
1211         slpte = vtd_get_slpte(addr, offset);
1212 
1213         if (slpte == (uint64_t)-1) {
1214             trace_vtd_page_walk_skip_read(iova, iova_next);
1215             goto next;
1216         }
1217 
1218         if (vtd_slpte_nonzero_rsvd(slpte, level)) {
1219             trace_vtd_page_walk_skip_reserve(iova, iova_next);
1220             goto next;
1221         }
1222 
1223         /* Permissions are stacked with parents' */
1224         read_cur = read && (slpte & VTD_SL_R);
1225         write_cur = write && (slpte & VTD_SL_W);
1226 
1227         /*
1228          * As long as we have either read/write permission, this is a
1229          * valid entry. The rule works for both page entries and page
1230          * table entries.
1231          */
1232         entry_valid = read_cur | write_cur;
1233 
1234         if (!vtd_is_last_slpte(slpte, level) && entry_valid) {
1235             /*
1236              * This is a valid PDE (or even bigger than PDE).  We need
1237              * to walk one further level.
1238              */
1239             ret = vtd_page_walk_level(vtd_get_slpte_addr(slpte, info->aw),
1240                                       iova, MIN(iova_next, end), level - 1,
1241                                       read_cur, write_cur, info);
1242         } else {
1243             /*
1244              * This means we are either:
1245              *
1246              * (1) the real page entry (either 4K page, or huge page)
1247              * (2) the whole range is invalid
1248              *
1249              * In either case, we send an IOTLB notification down.
1250              */
1251             event.entry.target_as = &address_space_memory;
1252             event.entry.iova = iova & subpage_mask;
1253             event.entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur);
1254             event.entry.addr_mask = ~subpage_mask;
1255             /* NOTE: this is only meaningful if entry_valid == true */
1256             event.entry.translated_addr = vtd_get_slpte_addr(slpte, info->aw);
1257             event.type = event.entry.perm ? IOMMU_NOTIFIER_MAP :
1258                                             IOMMU_NOTIFIER_UNMAP;
1259             ret = vtd_page_walk_one(&event, info);
1260         }
1261 
1262         if (ret < 0) {
1263             return ret;
1264         }
1265 
1266 next:
1267         iova = iova_next;
1268     }
1269 
1270     return 0;
1271 }
1272 
1273 /**
1274  * vtd_page_walk - walk specific IOVA range, and call the hook
1275  *
1276  * @s: intel iommu state
1277  * @ce: context entry to walk upon
1278  * @start: IOVA address to start the walk
1279  * @end: IOVA range end address (start <= addr < end)
1280  * @info: page walking information struct
1281  */
1282 static int vtd_page_walk(IntelIOMMUState *s, VTDContextEntry *ce,
1283                          uint64_t start, uint64_t end,
1284                          vtd_page_walk_info *info)
1285 {
1286     dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce);
1287     uint32_t level = vtd_get_iova_level(s, ce);
1288 
1289     if (!vtd_iova_range_check(s, start, ce, info->aw)) {
1290         return -VTD_FR_ADDR_BEYOND_MGAW;
1291     }
1292 
1293     if (!vtd_iova_range_check(s, end, ce, info->aw)) {
1294         /* Fix end so that it reaches the maximum */
1295         end = vtd_iova_limit(s, ce, info->aw);
1296     }
1297 
1298     return vtd_page_walk_level(addr, start, end, level, true, true, info);
1299 }
1300 
1301 static int vtd_root_entry_rsvd_bits_check(IntelIOMMUState *s,
1302                                           VTDRootEntry *re)
1303 {
1304     /* Legacy Mode reserved bits check */
1305     if (!s->root_scalable &&
1306         (re->hi || (re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits))))
1307         goto rsvd_err;
1308 
1309     /* Scalable Mode reserved bits check */
1310     if (s->root_scalable &&
1311         ((re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)) ||
1312          (re->hi & VTD_ROOT_ENTRY_RSVD(s->aw_bits))))
1313         goto rsvd_err;
1314 
1315     return 0;
1316 
1317 rsvd_err:
1318     error_report_once("%s: invalid root entry: hi=0x%"PRIx64
1319                       ", lo=0x%"PRIx64,
1320                       __func__, re->hi, re->lo);
1321     return -VTD_FR_ROOT_ENTRY_RSVD;
1322 }
1323 
1324 static inline int vtd_context_entry_rsvd_bits_check(IntelIOMMUState *s,
1325                                                     VTDContextEntry *ce)
1326 {
1327     if (!s->root_scalable &&
1328         (ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI ||
1329          ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) {
1330         error_report_once("%s: invalid context entry: hi=%"PRIx64
1331                           ", lo=%"PRIx64" (reserved nonzero)",
1332                           __func__, ce->hi, ce->lo);
1333         return -VTD_FR_CONTEXT_ENTRY_RSVD;
1334     }
1335 
1336     if (s->root_scalable &&
1337         (ce->val[0] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL0(s->aw_bits) ||
1338          ce->val[1] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL1 ||
1339          ce->val[2] ||
1340          ce->val[3])) {
1341         error_report_once("%s: invalid context entry: val[3]=%"PRIx64
1342                           ", val[2]=%"PRIx64
1343                           ", val[1]=%"PRIx64
1344                           ", val[0]=%"PRIx64" (reserved nonzero)",
1345                           __func__, ce->val[3], ce->val[2],
1346                           ce->val[1], ce->val[0]);
1347         return -VTD_FR_CONTEXT_ENTRY_RSVD;
1348     }
1349 
1350     return 0;
1351 }
1352 
1353 static int vtd_ce_rid2pasid_check(IntelIOMMUState *s,
1354                                   VTDContextEntry *ce)
1355 {
1356     VTDPASIDEntry pe;
1357 
1358     /*
1359      * Make sure in Scalable Mode, a present context entry
1360      * has valid rid2pasid setting, which includes valid
1361      * rid2pasid field and corresponding pasid entry setting
1362      */
1363     return vtd_ce_get_rid2pasid_entry(s, ce, &pe);
1364 }
1365 
1366 /* Map a device to its corresponding domain (context-entry) */
1367 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
1368                                     uint8_t devfn, VTDContextEntry *ce)
1369 {
1370     VTDRootEntry re;
1371     int ret_fr;
1372     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
1373 
1374     ret_fr = vtd_get_root_entry(s, bus_num, &re);
1375     if (ret_fr) {
1376         return ret_fr;
1377     }
1378 
1379     if (!vtd_root_entry_present(s, &re, devfn)) {
1380         /* Not error - it's okay we don't have root entry. */
1381         trace_vtd_re_not_present(bus_num);
1382         return -VTD_FR_ROOT_ENTRY_P;
1383     }
1384 
1385     ret_fr = vtd_root_entry_rsvd_bits_check(s, &re);
1386     if (ret_fr) {
1387         return ret_fr;
1388     }
1389 
1390     ret_fr = vtd_get_context_entry_from_root(s, &re, devfn, ce);
1391     if (ret_fr) {
1392         return ret_fr;
1393     }
1394 
1395     if (!vtd_ce_present(ce)) {
1396         /* Not error - it's okay we don't have context entry. */
1397         trace_vtd_ce_not_present(bus_num, devfn);
1398         return -VTD_FR_CONTEXT_ENTRY_P;
1399     }
1400 
1401     ret_fr = vtd_context_entry_rsvd_bits_check(s, ce);
1402     if (ret_fr) {
1403         return ret_fr;
1404     }
1405 
1406     /* Check if the programming of context-entry is valid */
1407     if (!s->root_scalable &&
1408         !vtd_is_level_supported(s, vtd_ce_get_level(ce))) {
1409         error_report_once("%s: invalid context entry: hi=%"PRIx64
1410                           ", lo=%"PRIx64" (level %d not supported)",
1411                           __func__, ce->hi, ce->lo,
1412                           vtd_ce_get_level(ce));
1413         return -VTD_FR_CONTEXT_ENTRY_INV;
1414     }
1415 
1416     if (!s->root_scalable) {
1417         /* Do translation type check */
1418         if (!vtd_ce_type_check(x86_iommu, ce)) {
1419             /* Errors dumped in vtd_ce_type_check() */
1420             return -VTD_FR_CONTEXT_ENTRY_INV;
1421         }
1422     } else {
1423         /*
1424          * Check if the programming of context-entry.rid2pasid
1425          * and corresponding pasid setting is valid, and thus
1426          * avoids to check pasid entry fetching result in future
1427          * helper function calling.
1428          */
1429         ret_fr = vtd_ce_rid2pasid_check(s, ce);
1430         if (ret_fr) {
1431             return ret_fr;
1432         }
1433     }
1434 
1435     return 0;
1436 }
1437 
1438 static int vtd_sync_shadow_page_hook(IOMMUTLBEvent *event,
1439                                      void *private)
1440 {
1441     memory_region_notify_iommu(private, 0, *event);
1442     return 0;
1443 }
1444 
1445 static uint16_t vtd_get_domain_id(IntelIOMMUState *s,
1446                                   VTDContextEntry *ce)
1447 {
1448     VTDPASIDEntry pe;
1449 
1450     if (s->root_scalable) {
1451         vtd_ce_get_rid2pasid_entry(s, ce, &pe);
1452         return VTD_SM_PASID_ENTRY_DID(pe.val[1]);
1453     }
1454 
1455     return VTD_CONTEXT_ENTRY_DID(ce->hi);
1456 }
1457 
1458 static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as,
1459                                             VTDContextEntry *ce,
1460                                             hwaddr addr, hwaddr size)
1461 {
1462     IntelIOMMUState *s = vtd_as->iommu_state;
1463     vtd_page_walk_info info = {
1464         .hook_fn = vtd_sync_shadow_page_hook,
1465         .private = (void *)&vtd_as->iommu,
1466         .notify_unmap = true,
1467         .aw = s->aw_bits,
1468         .as = vtd_as,
1469         .domain_id = vtd_get_domain_id(s, ce),
1470     };
1471 
1472     return vtd_page_walk(s, ce, addr, addr + size, &info);
1473 }
1474 
1475 static int vtd_sync_shadow_page_table(VTDAddressSpace *vtd_as)
1476 {
1477     int ret;
1478     VTDContextEntry ce;
1479     IOMMUNotifier *n;
1480 
1481     if (!(vtd_as->iommu.iommu_notify_flags & IOMMU_NOTIFIER_IOTLB_EVENTS)) {
1482         return 0;
1483     }
1484 
1485     ret = vtd_dev_to_context_entry(vtd_as->iommu_state,
1486                                    pci_bus_num(vtd_as->bus),
1487                                    vtd_as->devfn, &ce);
1488     if (ret) {
1489         if (ret == -VTD_FR_CONTEXT_ENTRY_P) {
1490             /*
1491              * It's a valid scenario to have a context entry that is
1492              * not present.  For example, when a device is removed
1493              * from an existing domain then the context entry will be
1494              * zeroed by the guest before it was put into another
1495              * domain.  When this happens, instead of synchronizing
1496              * the shadow pages we should invalidate all existing
1497              * mappings and notify the backends.
1498              */
1499             IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) {
1500                 vtd_address_space_unmap(vtd_as, n);
1501             }
1502             ret = 0;
1503         }
1504         return ret;
1505     }
1506 
1507     return vtd_sync_shadow_page_table_range(vtd_as, &ce, 0, UINT64_MAX);
1508 }
1509 
1510 /*
1511  * Check if specific device is configed to bypass address
1512  * translation for DMA requests. In Scalable Mode, bypass
1513  * 1st-level translation or 2nd-level translation, it depends
1514  * on PGTT setting.
1515  */
1516 static bool vtd_dev_pt_enabled(VTDAddressSpace *as)
1517 {
1518     IntelIOMMUState *s;
1519     VTDContextEntry ce;
1520     VTDPASIDEntry pe;
1521     int ret;
1522 
1523     assert(as);
1524 
1525     s = as->iommu_state;
1526     ret = vtd_dev_to_context_entry(s, pci_bus_num(as->bus),
1527                                    as->devfn, &ce);
1528     if (ret) {
1529         /*
1530          * Possibly failed to parse the context entry for some reason
1531          * (e.g., during init, or any guest configuration errors on
1532          * context entries). We should assume PT not enabled for
1533          * safety.
1534          */
1535         return false;
1536     }
1537 
1538     if (s->root_scalable) {
1539         ret = vtd_ce_get_rid2pasid_entry(s, &ce, &pe);
1540         if (ret) {
1541             error_report_once("%s: vtd_ce_get_rid2pasid_entry error: %"PRId32,
1542                               __func__, ret);
1543             return false;
1544         }
1545         return (VTD_PE_GET_TYPE(&pe) == VTD_SM_PASID_ENTRY_PT);
1546     }
1547 
1548     return (vtd_ce_get_type(&ce) == VTD_CONTEXT_TT_PASS_THROUGH);
1549 }
1550 
1551 /* Return whether the device is using IOMMU translation. */
1552 static bool vtd_switch_address_space(VTDAddressSpace *as)
1553 {
1554     bool use_iommu;
1555     /* Whether we need to take the BQL on our own */
1556     bool take_bql = !qemu_mutex_iothread_locked();
1557 
1558     assert(as);
1559 
1560     use_iommu = as->iommu_state->dmar_enabled && !vtd_dev_pt_enabled(as);
1561 
1562     trace_vtd_switch_address_space(pci_bus_num(as->bus),
1563                                    VTD_PCI_SLOT(as->devfn),
1564                                    VTD_PCI_FUNC(as->devfn),
1565                                    use_iommu);
1566 
1567     /*
1568      * It's possible that we reach here without BQL, e.g., when called
1569      * from vtd_pt_enable_fast_path(). However the memory APIs need
1570      * it. We'd better make sure we have had it already, or, take it.
1571      */
1572     if (take_bql) {
1573         qemu_mutex_lock_iothread();
1574     }
1575 
1576     /* Turn off first then on the other */
1577     if (use_iommu) {
1578         memory_region_set_enabled(&as->nodmar, false);
1579         memory_region_set_enabled(MEMORY_REGION(&as->iommu), true);
1580     } else {
1581         memory_region_set_enabled(MEMORY_REGION(&as->iommu), false);
1582         memory_region_set_enabled(&as->nodmar, true);
1583     }
1584 
1585     if (take_bql) {
1586         qemu_mutex_unlock_iothread();
1587     }
1588 
1589     return use_iommu;
1590 }
1591 
1592 static void vtd_switch_address_space_all(IntelIOMMUState *s)
1593 {
1594     GHashTableIter iter;
1595     VTDBus *vtd_bus;
1596     int i;
1597 
1598     g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
1599     while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) {
1600         for (i = 0; i < PCI_DEVFN_MAX; i++) {
1601             if (!vtd_bus->dev_as[i]) {
1602                 continue;
1603             }
1604             vtd_switch_address_space(vtd_bus->dev_as[i]);
1605         }
1606     }
1607 }
1608 
1609 static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn)
1610 {
1611     return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL);
1612 }
1613 
1614 static const bool vtd_qualified_faults[] = {
1615     [VTD_FR_RESERVED] = false,
1616     [VTD_FR_ROOT_ENTRY_P] = false,
1617     [VTD_FR_CONTEXT_ENTRY_P] = true,
1618     [VTD_FR_CONTEXT_ENTRY_INV] = true,
1619     [VTD_FR_ADDR_BEYOND_MGAW] = true,
1620     [VTD_FR_WRITE] = true,
1621     [VTD_FR_READ] = true,
1622     [VTD_FR_PAGING_ENTRY_INV] = true,
1623     [VTD_FR_ROOT_TABLE_INV] = false,
1624     [VTD_FR_CONTEXT_TABLE_INV] = false,
1625     [VTD_FR_ROOT_ENTRY_RSVD] = false,
1626     [VTD_FR_PAGING_ENTRY_RSVD] = true,
1627     [VTD_FR_CONTEXT_ENTRY_TT] = true,
1628     [VTD_FR_PASID_TABLE_INV] = false,
1629     [VTD_FR_RESERVED_ERR] = false,
1630     [VTD_FR_MAX] = false,
1631 };
1632 
1633 /* To see if a fault condition is "qualified", which is reported to software
1634  * only if the FPD field in the context-entry used to process the faulting
1635  * request is 0.
1636  */
1637 static inline bool vtd_is_qualified_fault(VTDFaultReason fault)
1638 {
1639     return vtd_qualified_faults[fault];
1640 }
1641 
1642 static inline bool vtd_is_interrupt_addr(hwaddr addr)
1643 {
1644     return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST;
1645 }
1646 
1647 static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id)
1648 {
1649     VTDBus *vtd_bus;
1650     VTDAddressSpace *vtd_as;
1651     bool success = false;
1652 
1653     vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id));
1654     if (!vtd_bus) {
1655         goto out;
1656     }
1657 
1658     vtd_as = vtd_bus->dev_as[VTD_SID_TO_DEVFN(source_id)];
1659     if (!vtd_as) {
1660         goto out;
1661     }
1662 
1663     if (vtd_switch_address_space(vtd_as) == false) {
1664         /* We switched off IOMMU region successfully. */
1665         success = true;
1666     }
1667 
1668 out:
1669     trace_vtd_pt_enable_fast_path(source_id, success);
1670 }
1671 
1672 /* Map dev to context-entry then do a paging-structures walk to do a iommu
1673  * translation.
1674  *
1675  * Called from RCU critical section.
1676  *
1677  * @bus_num: The bus number
1678  * @devfn: The devfn, which is the  combined of device and function number
1679  * @is_write: The access is a write operation
1680  * @entry: IOMMUTLBEntry that contain the addr to be translated and result
1681  *
1682  * Returns true if translation is successful, otherwise false.
1683  */
1684 static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
1685                                    uint8_t devfn, hwaddr addr, bool is_write,
1686                                    IOMMUTLBEntry *entry)
1687 {
1688     IntelIOMMUState *s = vtd_as->iommu_state;
1689     VTDContextEntry ce;
1690     uint8_t bus_num = pci_bus_num(bus);
1691     VTDContextCacheEntry *cc_entry;
1692     uint64_t slpte, page_mask;
1693     uint32_t level;
1694     uint16_t source_id = vtd_make_source_id(bus_num, devfn);
1695     int ret_fr;
1696     bool is_fpd_set = false;
1697     bool reads = true;
1698     bool writes = true;
1699     uint8_t access_flags;
1700     VTDIOTLBEntry *iotlb_entry;
1701 
1702     /*
1703      * We have standalone memory region for interrupt addresses, we
1704      * should never receive translation requests in this region.
1705      */
1706     assert(!vtd_is_interrupt_addr(addr));
1707 
1708     vtd_iommu_lock(s);
1709 
1710     cc_entry = &vtd_as->context_cache_entry;
1711 
1712     /* Try to fetch slpte form IOTLB */
1713     iotlb_entry = vtd_lookup_iotlb(s, source_id, addr);
1714     if (iotlb_entry) {
1715         trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte,
1716                                  iotlb_entry->domain_id);
1717         slpte = iotlb_entry->slpte;
1718         access_flags = iotlb_entry->access_flags;
1719         page_mask = iotlb_entry->mask;
1720         goto out;
1721     }
1722 
1723     /* Try to fetch context-entry from cache first */
1724     if (cc_entry->context_cache_gen == s->context_cache_gen) {
1725         trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi,
1726                                cc_entry->context_entry.lo,
1727                                cc_entry->context_cache_gen);
1728         ce = cc_entry->context_entry;
1729         is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
1730         if (!is_fpd_set && s->root_scalable) {
1731             ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set);
1732             VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write);
1733         }
1734     } else {
1735         ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
1736         is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
1737         if (!ret_fr && !is_fpd_set && s->root_scalable) {
1738             ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set);
1739         }
1740         VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write);
1741         /* Update context-cache */
1742         trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo,
1743                                   cc_entry->context_cache_gen,
1744                                   s->context_cache_gen);
1745         cc_entry->context_entry = ce;
1746         cc_entry->context_cache_gen = s->context_cache_gen;
1747     }
1748 
1749     /*
1750      * We don't need to translate for pass-through context entries.
1751      * Also, let's ignore IOTLB caching as well for PT devices.
1752      */
1753     if (vtd_ce_get_type(&ce) == VTD_CONTEXT_TT_PASS_THROUGH) {
1754         entry->iova = addr & VTD_PAGE_MASK_4K;
1755         entry->translated_addr = entry->iova;
1756         entry->addr_mask = ~VTD_PAGE_MASK_4K;
1757         entry->perm = IOMMU_RW;
1758         trace_vtd_translate_pt(source_id, entry->iova);
1759 
1760         /*
1761          * When this happens, it means firstly caching-mode is not
1762          * enabled, and this is the first passthrough translation for
1763          * the device. Let's enable the fast path for passthrough.
1764          *
1765          * When passthrough is disabled again for the device, we can
1766          * capture it via the context entry invalidation, then the
1767          * IOMMU region can be swapped back.
1768          */
1769         vtd_pt_enable_fast_path(s, source_id);
1770         vtd_iommu_unlock(s);
1771         return true;
1772     }
1773 
1774     ret_fr = vtd_iova_to_slpte(s, &ce, addr, is_write, &slpte, &level,
1775                                &reads, &writes, s->aw_bits);
1776     VTD_PE_GET_FPD_ERR(ret_fr, is_fpd_set, s, source_id, addr, is_write);
1777 
1778     page_mask = vtd_slpt_level_page_mask(level);
1779     access_flags = IOMMU_ACCESS_FLAG(reads, writes);
1780     vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce), addr, slpte,
1781                      access_flags, level);
1782 out:
1783     vtd_iommu_unlock(s);
1784     entry->iova = addr & page_mask;
1785     entry->translated_addr = vtd_get_slpte_addr(slpte, s->aw_bits) & page_mask;
1786     entry->addr_mask = ~page_mask;
1787     entry->perm = access_flags;
1788     return true;
1789 
1790 error:
1791     vtd_iommu_unlock(s);
1792     entry->iova = 0;
1793     entry->translated_addr = 0;
1794     entry->addr_mask = 0;
1795     entry->perm = IOMMU_NONE;
1796     return false;
1797 }
1798 
1799 static void vtd_root_table_setup(IntelIOMMUState *s)
1800 {
1801     s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
1802     s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits);
1803 
1804     vtd_update_scalable_state(s);
1805 
1806     trace_vtd_reg_dmar_root(s->root, s->root_scalable);
1807 }
1808 
1809 static void vtd_iec_notify_all(IntelIOMMUState *s, bool global,
1810                                uint32_t index, uint32_t mask)
1811 {
1812     x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
1813 }
1814 
1815 static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s)
1816 {
1817     uint64_t value = 0;
1818     value = vtd_get_quad_raw(s, DMAR_IRTA_REG);
1819     s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1);
1820     s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits);
1821     s->intr_eime = value & VTD_IRTA_EIME;
1822 
1823     /* Notify global invalidation */
1824     vtd_iec_notify_all(s, true, 0, 0);
1825 
1826     trace_vtd_reg_ir_root(s->intr_root, s->intr_size);
1827 }
1828 
1829 static void vtd_iommu_replay_all(IntelIOMMUState *s)
1830 {
1831     VTDAddressSpace *vtd_as;
1832 
1833     QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
1834         vtd_sync_shadow_page_table(vtd_as);
1835     }
1836 }
1837 
1838 static void vtd_context_global_invalidate(IntelIOMMUState *s)
1839 {
1840     trace_vtd_inv_desc_cc_global();
1841     /* Protects context cache */
1842     vtd_iommu_lock(s);
1843     s->context_cache_gen++;
1844     if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
1845         vtd_reset_context_cache_locked(s);
1846     }
1847     vtd_iommu_unlock(s);
1848     vtd_address_space_refresh_all(s);
1849     /*
1850      * From VT-d spec 6.5.2.1, a global context entry invalidation
1851      * should be followed by a IOTLB global invalidation, so we should
1852      * be safe even without this. Hoewever, let's replay the region as
1853      * well to be safer, and go back here when we need finer tunes for
1854      * VT-d emulation codes.
1855      */
1856     vtd_iommu_replay_all(s);
1857 }
1858 
1859 /* Do a context-cache device-selective invalidation.
1860  * @func_mask: FM field after shifting
1861  */
1862 static void vtd_context_device_invalidate(IntelIOMMUState *s,
1863                                           uint16_t source_id,
1864                                           uint16_t func_mask)
1865 {
1866     uint16_t mask;
1867     VTDBus *vtd_bus;
1868     VTDAddressSpace *vtd_as;
1869     uint8_t bus_n, devfn;
1870     uint16_t devfn_it;
1871 
1872     trace_vtd_inv_desc_cc_devices(source_id, func_mask);
1873 
1874     switch (func_mask & 3) {
1875     case 0:
1876         mask = 0;   /* No bits in the SID field masked */
1877         break;
1878     case 1:
1879         mask = 4;   /* Mask bit 2 in the SID field */
1880         break;
1881     case 2:
1882         mask = 6;   /* Mask bit 2:1 in the SID field */
1883         break;
1884     case 3:
1885         mask = 7;   /* Mask bit 2:0 in the SID field */
1886         break;
1887     }
1888     mask = ~mask;
1889 
1890     bus_n = VTD_SID_TO_BUS(source_id);
1891     vtd_bus = vtd_find_as_from_bus_num(s, bus_n);
1892     if (vtd_bus) {
1893         devfn = VTD_SID_TO_DEVFN(source_id);
1894         for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) {
1895             vtd_as = vtd_bus->dev_as[devfn_it];
1896             if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
1897                 trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(devfn_it),
1898                                              VTD_PCI_FUNC(devfn_it));
1899                 vtd_iommu_lock(s);
1900                 vtd_as->context_cache_entry.context_cache_gen = 0;
1901                 vtd_iommu_unlock(s);
1902                 /*
1903                  * Do switch address space when needed, in case if the
1904                  * device passthrough bit is switched.
1905                  */
1906                 vtd_switch_address_space(vtd_as);
1907                 /*
1908                  * So a device is moving out of (or moving into) a
1909                  * domain, resync the shadow page table.
1910                  * This won't bring bad even if we have no such
1911                  * notifier registered - the IOMMU notification
1912                  * framework will skip MAP notifications if that
1913                  * happened.
1914                  */
1915                 vtd_sync_shadow_page_table(vtd_as);
1916             }
1917         }
1918     }
1919 }
1920 
1921 /* Context-cache invalidation
1922  * Returns the Context Actual Invalidation Granularity.
1923  * @val: the content of the CCMD_REG
1924  */
1925 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
1926 {
1927     uint64_t caig;
1928     uint64_t type = val & VTD_CCMD_CIRG_MASK;
1929 
1930     switch (type) {
1931     case VTD_CCMD_DOMAIN_INVL:
1932         /* Fall through */
1933     case VTD_CCMD_GLOBAL_INVL:
1934         caig = VTD_CCMD_GLOBAL_INVL_A;
1935         vtd_context_global_invalidate(s);
1936         break;
1937 
1938     case VTD_CCMD_DEVICE_INVL:
1939         caig = VTD_CCMD_DEVICE_INVL_A;
1940         vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
1941         break;
1942 
1943     default:
1944         error_report_once("%s: invalid context: 0x%" PRIx64,
1945                           __func__, val);
1946         caig = 0;
1947     }
1948     return caig;
1949 }
1950 
1951 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s)
1952 {
1953     trace_vtd_inv_desc_iotlb_global();
1954     vtd_reset_iotlb(s);
1955     vtd_iommu_replay_all(s);
1956 }
1957 
1958 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
1959 {
1960     VTDContextEntry ce;
1961     VTDAddressSpace *vtd_as;
1962 
1963     trace_vtd_inv_desc_iotlb_domain(domain_id);
1964 
1965     vtd_iommu_lock(s);
1966     g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain,
1967                                 &domain_id);
1968     vtd_iommu_unlock(s);
1969 
1970     QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
1971         if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus),
1972                                       vtd_as->devfn, &ce) &&
1973             domain_id == vtd_get_domain_id(s, &ce)) {
1974             vtd_sync_shadow_page_table(vtd_as);
1975         }
1976     }
1977 }
1978 
1979 static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s,
1980                                            uint16_t domain_id, hwaddr addr,
1981                                            uint8_t am)
1982 {
1983     VTDAddressSpace *vtd_as;
1984     VTDContextEntry ce;
1985     int ret;
1986     hwaddr size = (1 << am) * VTD_PAGE_SIZE;
1987 
1988     QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) {
1989         ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus),
1990                                        vtd_as->devfn, &ce);
1991         if (!ret && domain_id == vtd_get_domain_id(s, &ce)) {
1992             if (vtd_as_has_map_notifier(vtd_as)) {
1993                 /*
1994                  * As long as we have MAP notifications registered in
1995                  * any of our IOMMU notifiers, we need to sync the
1996                  * shadow page table.
1997                  */
1998                 vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size);
1999             } else {
2000                 /*
2001                  * For UNMAP-only notifiers, we don't need to walk the
2002                  * page tables.  We just deliver the PSI down to
2003                  * invalidate caches.
2004                  */
2005                 IOMMUTLBEvent event = {
2006                     .type = IOMMU_NOTIFIER_UNMAP,
2007                     .entry = {
2008                         .target_as = &address_space_memory,
2009                         .iova = addr,
2010                         .translated_addr = 0,
2011                         .addr_mask = size - 1,
2012                         .perm = IOMMU_NONE,
2013                     },
2014                 };
2015                 memory_region_notify_iommu(&vtd_as->iommu, 0, event);
2016             }
2017         }
2018     }
2019 }
2020 
2021 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
2022                                       hwaddr addr, uint8_t am)
2023 {
2024     VTDIOTLBPageInvInfo info;
2025 
2026     trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am);
2027 
2028     assert(am <= VTD_MAMV);
2029     info.domain_id = domain_id;
2030     info.addr = addr;
2031     info.mask = ~((1 << am) - 1);
2032     vtd_iommu_lock(s);
2033     g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
2034     vtd_iommu_unlock(s);
2035     vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am);
2036 }
2037 
2038 /* Flush IOTLB
2039  * Returns the IOTLB Actual Invalidation Granularity.
2040  * @val: the content of the IOTLB_REG
2041  */
2042 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val)
2043 {
2044     uint64_t iaig;
2045     uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK;
2046     uint16_t domain_id;
2047     hwaddr addr;
2048     uint8_t am;
2049 
2050     switch (type) {
2051     case VTD_TLB_GLOBAL_FLUSH:
2052         iaig = VTD_TLB_GLOBAL_FLUSH_A;
2053         vtd_iotlb_global_invalidate(s);
2054         break;
2055 
2056     case VTD_TLB_DSI_FLUSH:
2057         domain_id = VTD_TLB_DID(val);
2058         iaig = VTD_TLB_DSI_FLUSH_A;
2059         vtd_iotlb_domain_invalidate(s, domain_id);
2060         break;
2061 
2062     case VTD_TLB_PSI_FLUSH:
2063         domain_id = VTD_TLB_DID(val);
2064         addr = vtd_get_quad_raw(s, DMAR_IVA_REG);
2065         am = VTD_IVA_AM(addr);
2066         addr = VTD_IVA_ADDR(addr);
2067         if (am > VTD_MAMV) {
2068             error_report_once("%s: address mask overflow: 0x%" PRIx64,
2069                               __func__, vtd_get_quad_raw(s, DMAR_IVA_REG));
2070             iaig = 0;
2071             break;
2072         }
2073         iaig = VTD_TLB_PSI_FLUSH_A;
2074         vtd_iotlb_page_invalidate(s, domain_id, addr, am);
2075         break;
2076 
2077     default:
2078         error_report_once("%s: invalid granularity: 0x%" PRIx64,
2079                           __func__, val);
2080         iaig = 0;
2081     }
2082     return iaig;
2083 }
2084 
2085 static void vtd_fetch_inv_desc(IntelIOMMUState *s);
2086 
2087 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s)
2088 {
2089     return s->qi_enabled && (s->iq_tail == s->iq_head) &&
2090            (s->iq_last_desc_type == VTD_INV_DESC_WAIT);
2091 }
2092 
2093 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en)
2094 {
2095     uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG);
2096 
2097     trace_vtd_inv_qi_enable(en);
2098 
2099     if (en) {
2100         s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits);
2101         /* 2^(x+8) entries */
2102         s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8 - (s->iq_dw ? 1 : 0));
2103         s->qi_enabled = true;
2104         trace_vtd_inv_qi_setup(s->iq, s->iq_size);
2105         /* Ok - report back to driver */
2106         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES);
2107 
2108         if (s->iq_tail != 0) {
2109             /*
2110              * This is a spec violation but Windows guests are known to set up
2111              * Queued Invalidation this way so we allow the write and process
2112              * Invalidation Descriptors right away.
2113              */
2114             trace_vtd_warn_invalid_qi_tail(s->iq_tail);
2115             if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
2116                 vtd_fetch_inv_desc(s);
2117             }
2118         }
2119     } else {
2120         if (vtd_queued_inv_disable_check(s)) {
2121             /* disable Queued Invalidation */
2122             vtd_set_quad_raw(s, DMAR_IQH_REG, 0);
2123             s->iq_head = 0;
2124             s->qi_enabled = false;
2125             /* Ok - report back to driver */
2126             vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0);
2127         } else {
2128             error_report_once("%s: detected improper state when disable QI "
2129                               "(head=0x%x, tail=0x%x, last_type=%d)",
2130                               __func__,
2131                               s->iq_head, s->iq_tail, s->iq_last_desc_type);
2132         }
2133     }
2134 }
2135 
2136 /* Set Root Table Pointer */
2137 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s)
2138 {
2139     vtd_root_table_setup(s);
2140     /* Ok - report back to driver */
2141     vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
2142     vtd_reset_caches(s);
2143     vtd_address_space_refresh_all(s);
2144 }
2145 
2146 /* Set Interrupt Remap Table Pointer */
2147 static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s)
2148 {
2149     vtd_interrupt_remap_table_setup(s);
2150     /* Ok - report back to driver */
2151     vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS);
2152 }
2153 
2154 /* Handle Translation Enable/Disable */
2155 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
2156 {
2157     if (s->dmar_enabled == en) {
2158         return;
2159     }
2160 
2161     trace_vtd_dmar_enable(en);
2162 
2163     if (en) {
2164         s->dmar_enabled = true;
2165         /* Ok - report back to driver */
2166         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES);
2167     } else {
2168         s->dmar_enabled = false;
2169 
2170         /* Clear the index of Fault Recording Register */
2171         s->next_frcd_reg = 0;
2172         /* Ok - report back to driver */
2173         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0);
2174     }
2175 
2176     vtd_reset_caches(s);
2177     vtd_address_space_refresh_all(s);
2178 }
2179 
2180 /* Handle Interrupt Remap Enable/Disable */
2181 static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en)
2182 {
2183     trace_vtd_ir_enable(en);
2184 
2185     if (en) {
2186         s->intr_enabled = true;
2187         /* Ok - report back to driver */
2188         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES);
2189     } else {
2190         s->intr_enabled = false;
2191         /* Ok - report back to driver */
2192         vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0);
2193     }
2194 }
2195 
2196 /* Handle write to Global Command Register */
2197 static void vtd_handle_gcmd_write(IntelIOMMUState *s)
2198 {
2199     uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG);
2200     uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG);
2201     uint32_t changed = status ^ val;
2202 
2203     trace_vtd_reg_write_gcmd(status, val);
2204     if (changed & VTD_GCMD_TE) {
2205         /* Translation enable/disable */
2206         vtd_handle_gcmd_te(s, val & VTD_GCMD_TE);
2207     }
2208     if (val & VTD_GCMD_SRTP) {
2209         /* Set/update the root-table pointer */
2210         vtd_handle_gcmd_srtp(s);
2211     }
2212     if (changed & VTD_GCMD_QIE) {
2213         /* Queued Invalidation Enable */
2214         vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
2215     }
2216     if (val & VTD_GCMD_SIRTP) {
2217         /* Set/update the interrupt remapping root-table pointer */
2218         vtd_handle_gcmd_sirtp(s);
2219     }
2220     if (changed & VTD_GCMD_IRE) {
2221         /* Interrupt remap enable/disable */
2222         vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE);
2223     }
2224 }
2225 
2226 /* Handle write to Context Command Register */
2227 static void vtd_handle_ccmd_write(IntelIOMMUState *s)
2228 {
2229     uint64_t ret;
2230     uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG);
2231 
2232     /* Context-cache invalidation request */
2233     if (val & VTD_CCMD_ICC) {
2234         if (s->qi_enabled) {
2235             error_report_once("Queued Invalidation enabled, "
2236                               "should not use register-based invalidation");
2237             return;
2238         }
2239         ret = vtd_context_cache_invalidate(s, val);
2240         /* Invalidation completed. Change something to show */
2241         vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL);
2242         ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK,
2243                                       ret);
2244     }
2245 }
2246 
2247 /* Handle write to IOTLB Invalidation Register */
2248 static void vtd_handle_iotlb_write(IntelIOMMUState *s)
2249 {
2250     uint64_t ret;
2251     uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG);
2252 
2253     /* IOTLB invalidation request */
2254     if (val & VTD_TLB_IVT) {
2255         if (s->qi_enabled) {
2256             error_report_once("Queued Invalidation enabled, "
2257                               "should not use register-based invalidation");
2258             return;
2259         }
2260         ret = vtd_iotlb_flush(s, val);
2261         /* Invalidation completed. Change something to show */
2262         vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL);
2263         ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG,
2264                                       VTD_TLB_FLUSH_GRANU_MASK_A, ret);
2265     }
2266 }
2267 
2268 /* Fetch an Invalidation Descriptor from the Invalidation Queue */
2269 static bool vtd_get_inv_desc(IntelIOMMUState *s,
2270                              VTDInvDesc *inv_desc)
2271 {
2272     dma_addr_t base_addr = s->iq;
2273     uint32_t offset = s->iq_head;
2274     uint32_t dw = s->iq_dw ? 32 : 16;
2275     dma_addr_t addr = base_addr + offset * dw;
2276 
2277     if (dma_memory_read(&address_space_memory, addr, inv_desc, dw)) {
2278         error_report_once("Read INV DESC failed.");
2279         return false;
2280     }
2281     inv_desc->lo = le64_to_cpu(inv_desc->lo);
2282     inv_desc->hi = le64_to_cpu(inv_desc->hi);
2283     if (dw == 32) {
2284         inv_desc->val[2] = le64_to_cpu(inv_desc->val[2]);
2285         inv_desc->val[3] = le64_to_cpu(inv_desc->val[3]);
2286     }
2287     return true;
2288 }
2289 
2290 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
2291 {
2292     if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) ||
2293         (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) {
2294         error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64
2295                           " (reserved nonzero)", __func__, inv_desc->hi,
2296                           inv_desc->lo);
2297         return false;
2298     }
2299     if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) {
2300         /* Status Write */
2301         uint32_t status_data = (uint32_t)(inv_desc->lo >>
2302                                VTD_INV_DESC_WAIT_DATA_SHIFT);
2303 
2304         assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF));
2305 
2306         /* FIXME: need to be masked with HAW? */
2307         dma_addr_t status_addr = inv_desc->hi;
2308         trace_vtd_inv_desc_wait_sw(status_addr, status_data);
2309         status_data = cpu_to_le32(status_data);
2310         if (dma_memory_write(&address_space_memory, status_addr, &status_data,
2311                              sizeof(status_data))) {
2312             trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo);
2313             return false;
2314         }
2315     } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) {
2316         /* Interrupt flag */
2317         vtd_generate_completion_event(s);
2318     } else {
2319         error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64
2320                           " (unknown type)", __func__, inv_desc->hi,
2321                           inv_desc->lo);
2322         return false;
2323     }
2324     return true;
2325 }
2326 
2327 static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
2328                                            VTDInvDesc *inv_desc)
2329 {
2330     uint16_t sid, fmask;
2331 
2332     if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {
2333         error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64
2334                           " (reserved nonzero)", __func__, inv_desc->hi,
2335                           inv_desc->lo);
2336         return false;
2337     }
2338     switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
2339     case VTD_INV_DESC_CC_DOMAIN:
2340         trace_vtd_inv_desc_cc_domain(
2341             (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
2342         /* Fall through */
2343     case VTD_INV_DESC_CC_GLOBAL:
2344         vtd_context_global_invalidate(s);
2345         break;
2346 
2347     case VTD_INV_DESC_CC_DEVICE:
2348         sid = VTD_INV_DESC_CC_SID(inv_desc->lo);
2349         fmask = VTD_INV_DESC_CC_FM(inv_desc->lo);
2350         vtd_context_device_invalidate(s, sid, fmask);
2351         break;
2352 
2353     default:
2354         error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64
2355                           " (invalid type)", __func__, inv_desc->hi,
2356                           inv_desc->lo);
2357         return false;
2358     }
2359     return true;
2360 }
2361 
2362 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
2363 {
2364     uint16_t domain_id;
2365     uint8_t am;
2366     hwaddr addr;
2367 
2368     if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) ||
2369         (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) {
2370         error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64
2371                           ", lo=0x%"PRIx64" (reserved bits unzero)",
2372                           __func__, inv_desc->hi, inv_desc->lo);
2373         return false;
2374     }
2375 
2376     switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) {
2377     case VTD_INV_DESC_IOTLB_GLOBAL:
2378         vtd_iotlb_global_invalidate(s);
2379         break;
2380 
2381     case VTD_INV_DESC_IOTLB_DOMAIN:
2382         domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
2383         vtd_iotlb_domain_invalidate(s, domain_id);
2384         break;
2385 
2386     case VTD_INV_DESC_IOTLB_PAGE:
2387         domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
2388         addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi);
2389         am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi);
2390         if (am > VTD_MAMV) {
2391             error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64
2392                               ", lo=0x%"PRIx64" (am=%u > VTD_MAMV=%u)",
2393                               __func__, inv_desc->hi, inv_desc->lo,
2394                               am, (unsigned)VTD_MAMV);
2395             return false;
2396         }
2397         vtd_iotlb_page_invalidate(s, domain_id, addr, am);
2398         break;
2399 
2400     default:
2401         error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64
2402                           ", lo=0x%"PRIx64" (type mismatch: 0x%llx)",
2403                           __func__, inv_desc->hi, inv_desc->lo,
2404                           inv_desc->lo & VTD_INV_DESC_IOTLB_G);
2405         return false;
2406     }
2407     return true;
2408 }
2409 
2410 static bool vtd_process_inv_iec_desc(IntelIOMMUState *s,
2411                                      VTDInvDesc *inv_desc)
2412 {
2413     trace_vtd_inv_desc_iec(inv_desc->iec.granularity,
2414                            inv_desc->iec.index,
2415                            inv_desc->iec.index_mask);
2416 
2417     vtd_iec_notify_all(s, !inv_desc->iec.granularity,
2418                        inv_desc->iec.index,
2419                        inv_desc->iec.index_mask);
2420     return true;
2421 }
2422 
2423 static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s,
2424                                           VTDInvDesc *inv_desc)
2425 {
2426     VTDAddressSpace *vtd_dev_as;
2427     IOMMUTLBEvent event;
2428     struct VTDBus *vtd_bus;
2429     hwaddr addr;
2430     uint64_t sz;
2431     uint16_t sid;
2432     uint8_t devfn;
2433     bool size;
2434     uint8_t bus_num;
2435 
2436     addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi);
2437     sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo);
2438     devfn = sid & 0xff;
2439     bus_num = sid >> 8;
2440     size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi);
2441 
2442     if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) ||
2443         (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) {
2444         error_report_once("%s: invalid dev-iotlb inv desc: hi=%"PRIx64
2445                           ", lo=%"PRIx64" (reserved nonzero)", __func__,
2446                           inv_desc->hi, inv_desc->lo);
2447         return false;
2448     }
2449 
2450     vtd_bus = vtd_find_as_from_bus_num(s, bus_num);
2451     if (!vtd_bus) {
2452         goto done;
2453     }
2454 
2455     vtd_dev_as = vtd_bus->dev_as[devfn];
2456     if (!vtd_dev_as) {
2457         goto done;
2458     }
2459 
2460     /* According to ATS spec table 2.4:
2461      * S = 0, bits 15:12 = xxxx     range size: 4K
2462      * S = 1, bits 15:12 = xxx0     range size: 8K
2463      * S = 1, bits 15:12 = xx01     range size: 16K
2464      * S = 1, bits 15:12 = x011     range size: 32K
2465      * S = 1, bits 15:12 = 0111     range size: 64K
2466      * ...
2467      */
2468     if (size) {
2469         sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT);
2470         addr &= ~(sz - 1);
2471     } else {
2472         sz = VTD_PAGE_SIZE;
2473     }
2474 
2475     event.type = IOMMU_NOTIFIER_DEVIOTLB_UNMAP;
2476     event.entry.target_as = &vtd_dev_as->as;
2477     event.entry.addr_mask = sz - 1;
2478     event.entry.iova = addr;
2479     event.entry.perm = IOMMU_NONE;
2480     event.entry.translated_addr = 0;
2481     memory_region_notify_iommu(&vtd_dev_as->iommu, 0, event);
2482 
2483 done:
2484     return true;
2485 }
2486 
2487 static bool vtd_process_inv_desc(IntelIOMMUState *s)
2488 {
2489     VTDInvDesc inv_desc;
2490     uint8_t desc_type;
2491 
2492     trace_vtd_inv_qi_head(s->iq_head);
2493     if (!vtd_get_inv_desc(s, &inv_desc)) {
2494         s->iq_last_desc_type = VTD_INV_DESC_NONE;
2495         return false;
2496     }
2497 
2498     desc_type = inv_desc.lo & VTD_INV_DESC_TYPE;
2499     /* FIXME: should update at first or at last? */
2500     s->iq_last_desc_type = desc_type;
2501 
2502     switch (desc_type) {
2503     case VTD_INV_DESC_CC:
2504         trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo);
2505         if (!vtd_process_context_cache_desc(s, &inv_desc)) {
2506             return false;
2507         }
2508         break;
2509 
2510     case VTD_INV_DESC_IOTLB:
2511         trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo);
2512         if (!vtd_process_iotlb_desc(s, &inv_desc)) {
2513             return false;
2514         }
2515         break;
2516 
2517     /*
2518      * TODO: the entity of below two cases will be implemented in future series.
2519      * To make guest (which integrates scalable mode support patch set in
2520      * iommu driver) work, just return true is enough so far.
2521      */
2522     case VTD_INV_DESC_PC:
2523         break;
2524 
2525     case VTD_INV_DESC_PIOTLB:
2526         break;
2527 
2528     case VTD_INV_DESC_WAIT:
2529         trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo);
2530         if (!vtd_process_wait_desc(s, &inv_desc)) {
2531             return false;
2532         }
2533         break;
2534 
2535     case VTD_INV_DESC_IEC:
2536         trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo);
2537         if (!vtd_process_inv_iec_desc(s, &inv_desc)) {
2538             return false;
2539         }
2540         break;
2541 
2542     case VTD_INV_DESC_DEVICE:
2543         trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo);
2544         if (!vtd_process_device_iotlb_desc(s, &inv_desc)) {
2545             return false;
2546         }
2547         break;
2548 
2549     default:
2550         error_report_once("%s: invalid inv desc: hi=%"PRIx64", lo=%"PRIx64
2551                           " (unknown type)", __func__, inv_desc.hi,
2552                           inv_desc.lo);
2553         return false;
2554     }
2555     s->iq_head++;
2556     if (s->iq_head == s->iq_size) {
2557         s->iq_head = 0;
2558     }
2559     return true;
2560 }
2561 
2562 /* Try to fetch and process more Invalidation Descriptors */
2563 static void vtd_fetch_inv_desc(IntelIOMMUState *s)
2564 {
2565     int qi_shift;
2566 
2567     /* Refer to 10.4.23 of VT-d spec 3.0 */
2568     qi_shift = s->iq_dw ? VTD_IQH_QH_SHIFT_5 : VTD_IQH_QH_SHIFT_4;
2569 
2570     trace_vtd_inv_qi_fetch();
2571 
2572     if (s->iq_tail >= s->iq_size) {
2573         /* Detects an invalid Tail pointer */
2574         error_report_once("%s: detected invalid QI tail "
2575                           "(tail=0x%x, size=0x%x)",
2576                           __func__, s->iq_tail, s->iq_size);
2577         vtd_handle_inv_queue_error(s);
2578         return;
2579     }
2580     while (s->iq_head != s->iq_tail) {
2581         if (!vtd_process_inv_desc(s)) {
2582             /* Invalidation Queue Errors */
2583             vtd_handle_inv_queue_error(s);
2584             break;
2585         }
2586         /* Must update the IQH_REG in time */
2587         vtd_set_quad_raw(s, DMAR_IQH_REG,
2588                          (((uint64_t)(s->iq_head)) << qi_shift) &
2589                          VTD_IQH_QH_MASK);
2590     }
2591 }
2592 
2593 /* Handle write to Invalidation Queue Tail Register */
2594 static void vtd_handle_iqt_write(IntelIOMMUState *s)
2595 {
2596     uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG);
2597 
2598     if (s->iq_dw && (val & VTD_IQT_QT_256_RSV_BIT)) {
2599         error_report_once("%s: RSV bit is set: val=0x%"PRIx64,
2600                           __func__, val);
2601         return;
2602     }
2603     s->iq_tail = VTD_IQT_QT(s->iq_dw, val);
2604     trace_vtd_inv_qi_tail(s->iq_tail);
2605 
2606     if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
2607         /* Process Invalidation Queue here */
2608         vtd_fetch_inv_desc(s);
2609     }
2610 }
2611 
2612 static void vtd_handle_fsts_write(IntelIOMMUState *s)
2613 {
2614     uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
2615     uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
2616     uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE;
2617 
2618     if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) {
2619         vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
2620         trace_vtd_fsts_clear_ip();
2621     }
2622     /* FIXME: when IQE is Clear, should we try to fetch some Invalidation
2623      * Descriptors if there are any when Queued Invalidation is enabled?
2624      */
2625 }
2626 
2627 static void vtd_handle_fectl_write(IntelIOMMUState *s)
2628 {
2629     uint32_t fectl_reg;
2630     /* FIXME: when software clears the IM field, check the IP field. But do we
2631      * need to compare the old value and the new value to conclude that
2632      * software clears the IM field? Or just check if the IM field is zero?
2633      */
2634     fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
2635 
2636     trace_vtd_reg_write_fectl(fectl_reg);
2637 
2638     if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) {
2639         vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
2640         vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
2641     }
2642 }
2643 
2644 static void vtd_handle_ics_write(IntelIOMMUState *s)
2645 {
2646     uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG);
2647     uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
2648 
2649     if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) {
2650         trace_vtd_reg_ics_clear_ip();
2651         vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
2652     }
2653 }
2654 
2655 static void vtd_handle_iectl_write(IntelIOMMUState *s)
2656 {
2657     uint32_t iectl_reg;
2658     /* FIXME: when software clears the IM field, check the IP field. But do we
2659      * need to compare the old value and the new value to conclude that
2660      * software clears the IM field? Or just check if the IM field is zero?
2661      */
2662     iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
2663 
2664     trace_vtd_reg_write_iectl(iectl_reg);
2665 
2666     if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) {
2667         vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
2668         vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
2669     }
2670 }
2671 
2672 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size)
2673 {
2674     IntelIOMMUState *s = opaque;
2675     uint64_t val;
2676 
2677     trace_vtd_reg_read(addr, size);
2678 
2679     if (addr + size > DMAR_REG_SIZE) {
2680         error_report_once("%s: MMIO over range: addr=0x%" PRIx64
2681                           " size=0x%x", __func__, addr, size);
2682         return (uint64_t)-1;
2683     }
2684 
2685     switch (addr) {
2686     /* Root Table Address Register, 64-bit */
2687     case DMAR_RTADDR_REG:
2688         val = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
2689         if (size == 4) {
2690             val = val & ((1ULL << 32) - 1);
2691         }
2692         break;
2693 
2694     case DMAR_RTADDR_REG_HI:
2695         assert(size == 4);
2696         val = vtd_get_quad_raw(s, DMAR_RTADDR_REG) >> 32;
2697         break;
2698 
2699     /* Invalidation Queue Address Register, 64-bit */
2700     case DMAR_IQA_REG:
2701         val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS);
2702         if (size == 4) {
2703             val = val & ((1ULL << 32) - 1);
2704         }
2705         break;
2706 
2707     case DMAR_IQA_REG_HI:
2708         assert(size == 4);
2709         val = s->iq >> 32;
2710         break;
2711 
2712     default:
2713         if (size == 4) {
2714             val = vtd_get_long(s, addr);
2715         } else {
2716             val = vtd_get_quad(s, addr);
2717         }
2718     }
2719 
2720     return val;
2721 }
2722 
2723 static void vtd_mem_write(void *opaque, hwaddr addr,
2724                           uint64_t val, unsigned size)
2725 {
2726     IntelIOMMUState *s = opaque;
2727 
2728     trace_vtd_reg_write(addr, size, val);
2729 
2730     if (addr + size > DMAR_REG_SIZE) {
2731         error_report_once("%s: MMIO over range: addr=0x%" PRIx64
2732                           " size=0x%x", __func__, addr, size);
2733         return;
2734     }
2735 
2736     switch (addr) {
2737     /* Global Command Register, 32-bit */
2738     case DMAR_GCMD_REG:
2739         vtd_set_long(s, addr, val);
2740         vtd_handle_gcmd_write(s);
2741         break;
2742 
2743     /* Context Command Register, 64-bit */
2744     case DMAR_CCMD_REG:
2745         if (size == 4) {
2746             vtd_set_long(s, addr, val);
2747         } else {
2748             vtd_set_quad(s, addr, val);
2749             vtd_handle_ccmd_write(s);
2750         }
2751         break;
2752 
2753     case DMAR_CCMD_REG_HI:
2754         assert(size == 4);
2755         vtd_set_long(s, addr, val);
2756         vtd_handle_ccmd_write(s);
2757         break;
2758 
2759     /* IOTLB Invalidation Register, 64-bit */
2760     case DMAR_IOTLB_REG:
2761         if (size == 4) {
2762             vtd_set_long(s, addr, val);
2763         } else {
2764             vtd_set_quad(s, addr, val);
2765             vtd_handle_iotlb_write(s);
2766         }
2767         break;
2768 
2769     case DMAR_IOTLB_REG_HI:
2770         assert(size == 4);
2771         vtd_set_long(s, addr, val);
2772         vtd_handle_iotlb_write(s);
2773         break;
2774 
2775     /* Invalidate Address Register, 64-bit */
2776     case DMAR_IVA_REG:
2777         if (size == 4) {
2778             vtd_set_long(s, addr, val);
2779         } else {
2780             vtd_set_quad(s, addr, val);
2781         }
2782         break;
2783 
2784     case DMAR_IVA_REG_HI:
2785         assert(size == 4);
2786         vtd_set_long(s, addr, val);
2787         break;
2788 
2789     /* Fault Status Register, 32-bit */
2790     case DMAR_FSTS_REG:
2791         assert(size == 4);
2792         vtd_set_long(s, addr, val);
2793         vtd_handle_fsts_write(s);
2794         break;
2795 
2796     /* Fault Event Control Register, 32-bit */
2797     case DMAR_FECTL_REG:
2798         assert(size == 4);
2799         vtd_set_long(s, addr, val);
2800         vtd_handle_fectl_write(s);
2801         break;
2802 
2803     /* Fault Event Data Register, 32-bit */
2804     case DMAR_FEDATA_REG:
2805         assert(size == 4);
2806         vtd_set_long(s, addr, val);
2807         break;
2808 
2809     /* Fault Event Address Register, 32-bit */
2810     case DMAR_FEADDR_REG:
2811         if (size == 4) {
2812             vtd_set_long(s, addr, val);
2813         } else {
2814             /*
2815              * While the register is 32-bit only, some guests (Xen...) write to
2816              * it with 64-bit.
2817              */
2818             vtd_set_quad(s, addr, val);
2819         }
2820         break;
2821 
2822     /* Fault Event Upper Address Register, 32-bit */
2823     case DMAR_FEUADDR_REG:
2824         assert(size == 4);
2825         vtd_set_long(s, addr, val);
2826         break;
2827 
2828     /* Protected Memory Enable Register, 32-bit */
2829     case DMAR_PMEN_REG:
2830         assert(size == 4);
2831         vtd_set_long(s, addr, val);
2832         break;
2833 
2834     /* Root Table Address Register, 64-bit */
2835     case DMAR_RTADDR_REG:
2836         if (size == 4) {
2837             vtd_set_long(s, addr, val);
2838         } else {
2839             vtd_set_quad(s, addr, val);
2840         }
2841         break;
2842 
2843     case DMAR_RTADDR_REG_HI:
2844         assert(size == 4);
2845         vtd_set_long(s, addr, val);
2846         break;
2847 
2848     /* Invalidation Queue Tail Register, 64-bit */
2849     case DMAR_IQT_REG:
2850         if (size == 4) {
2851             vtd_set_long(s, addr, val);
2852         } else {
2853             vtd_set_quad(s, addr, val);
2854         }
2855         vtd_handle_iqt_write(s);
2856         break;
2857 
2858     case DMAR_IQT_REG_HI:
2859         assert(size == 4);
2860         vtd_set_long(s, addr, val);
2861         /* 19:63 of IQT_REG is RsvdZ, do nothing here */
2862         break;
2863 
2864     /* Invalidation Queue Address Register, 64-bit */
2865     case DMAR_IQA_REG:
2866         if (size == 4) {
2867             vtd_set_long(s, addr, val);
2868         } else {
2869             vtd_set_quad(s, addr, val);
2870         }
2871         if (s->ecap & VTD_ECAP_SMTS &&
2872             val & VTD_IQA_DW_MASK) {
2873             s->iq_dw = true;
2874         } else {
2875             s->iq_dw = false;
2876         }
2877         break;
2878 
2879     case DMAR_IQA_REG_HI:
2880         assert(size == 4);
2881         vtd_set_long(s, addr, val);
2882         break;
2883 
2884     /* Invalidation Completion Status Register, 32-bit */
2885     case DMAR_ICS_REG:
2886         assert(size == 4);
2887         vtd_set_long(s, addr, val);
2888         vtd_handle_ics_write(s);
2889         break;
2890 
2891     /* Invalidation Event Control Register, 32-bit */
2892     case DMAR_IECTL_REG:
2893         assert(size == 4);
2894         vtd_set_long(s, addr, val);
2895         vtd_handle_iectl_write(s);
2896         break;
2897 
2898     /* Invalidation Event Data Register, 32-bit */
2899     case DMAR_IEDATA_REG:
2900         assert(size == 4);
2901         vtd_set_long(s, addr, val);
2902         break;
2903 
2904     /* Invalidation Event Address Register, 32-bit */
2905     case DMAR_IEADDR_REG:
2906         assert(size == 4);
2907         vtd_set_long(s, addr, val);
2908         break;
2909 
2910     /* Invalidation Event Upper Address Register, 32-bit */
2911     case DMAR_IEUADDR_REG:
2912         assert(size == 4);
2913         vtd_set_long(s, addr, val);
2914         break;
2915 
2916     /* Fault Recording Registers, 128-bit */
2917     case DMAR_FRCD_REG_0_0:
2918         if (size == 4) {
2919             vtd_set_long(s, addr, val);
2920         } else {
2921             vtd_set_quad(s, addr, val);
2922         }
2923         break;
2924 
2925     case DMAR_FRCD_REG_0_1:
2926         assert(size == 4);
2927         vtd_set_long(s, addr, val);
2928         break;
2929 
2930     case DMAR_FRCD_REG_0_2:
2931         if (size == 4) {
2932             vtd_set_long(s, addr, val);
2933         } else {
2934             vtd_set_quad(s, addr, val);
2935             /* May clear bit 127 (Fault), update PPF */
2936             vtd_update_fsts_ppf(s);
2937         }
2938         break;
2939 
2940     case DMAR_FRCD_REG_0_3:
2941         assert(size == 4);
2942         vtd_set_long(s, addr, val);
2943         /* May clear bit 127 (Fault), update PPF */
2944         vtd_update_fsts_ppf(s);
2945         break;
2946 
2947     case DMAR_IRTA_REG:
2948         if (size == 4) {
2949             vtd_set_long(s, addr, val);
2950         } else {
2951             vtd_set_quad(s, addr, val);
2952         }
2953         break;
2954 
2955     case DMAR_IRTA_REG_HI:
2956         assert(size == 4);
2957         vtd_set_long(s, addr, val);
2958         break;
2959 
2960     default:
2961         if (size == 4) {
2962             vtd_set_long(s, addr, val);
2963         } else {
2964             vtd_set_quad(s, addr, val);
2965         }
2966     }
2967 }
2968 
2969 static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr,
2970                                          IOMMUAccessFlags flag, int iommu_idx)
2971 {
2972     VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
2973     IntelIOMMUState *s = vtd_as->iommu_state;
2974     IOMMUTLBEntry iotlb = {
2975         /* We'll fill in the rest later. */
2976         .target_as = &address_space_memory,
2977     };
2978     bool success;
2979 
2980     if (likely(s->dmar_enabled)) {
2981         success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn,
2982                                          addr, flag & IOMMU_WO, &iotlb);
2983     } else {
2984         /* DMAR disabled, passthrough, use 4k-page*/
2985         iotlb.iova = addr & VTD_PAGE_MASK_4K;
2986         iotlb.translated_addr = addr & VTD_PAGE_MASK_4K;
2987         iotlb.addr_mask = ~VTD_PAGE_MASK_4K;
2988         iotlb.perm = IOMMU_RW;
2989         success = true;
2990     }
2991 
2992     if (likely(success)) {
2993         trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus),
2994                                  VTD_PCI_SLOT(vtd_as->devfn),
2995                                  VTD_PCI_FUNC(vtd_as->devfn),
2996                                  iotlb.iova, iotlb.translated_addr,
2997                                  iotlb.addr_mask);
2998     } else {
2999         error_report_once("%s: detected translation failure "
3000                           "(dev=%02x:%02x:%02x, iova=0x%" PRIx64 ")",
3001                           __func__, pci_bus_num(vtd_as->bus),
3002                           VTD_PCI_SLOT(vtd_as->devfn),
3003                           VTD_PCI_FUNC(vtd_as->devfn),
3004                           addr);
3005     }
3006 
3007     return iotlb;
3008 }
3009 
3010 static int vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu,
3011                                          IOMMUNotifierFlag old,
3012                                          IOMMUNotifierFlag new,
3013                                          Error **errp)
3014 {
3015     VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
3016     IntelIOMMUState *s = vtd_as->iommu_state;
3017 
3018     /* Update per-address-space notifier flags */
3019     vtd_as->notifier_flags = new;
3020 
3021     if (old == IOMMU_NOTIFIER_NONE) {
3022         QLIST_INSERT_HEAD(&s->vtd_as_with_notifiers, vtd_as, next);
3023     } else if (new == IOMMU_NOTIFIER_NONE) {
3024         QLIST_REMOVE(vtd_as, next);
3025     }
3026     return 0;
3027 }
3028 
3029 static int vtd_post_load(void *opaque, int version_id)
3030 {
3031     IntelIOMMUState *iommu = opaque;
3032 
3033     /*
3034      * Memory regions are dynamically turned on/off depending on
3035      * context entry configurations from the guest. After migration,
3036      * we need to make sure the memory regions are still correct.
3037      */
3038     vtd_switch_address_space_all(iommu);
3039 
3040     /*
3041      * We don't need to migrate the root_scalable because we can
3042      * simply do the calculation after the loading is complete.  We
3043      * can actually do similar things with root, dmar_enabled, etc.
3044      * however since we've had them already so we'd better keep them
3045      * for compatibility of migration.
3046      */
3047     vtd_update_scalable_state(iommu);
3048 
3049     return 0;
3050 }
3051 
3052 static const VMStateDescription vtd_vmstate = {
3053     .name = "iommu-intel",
3054     .version_id = 1,
3055     .minimum_version_id = 1,
3056     .priority = MIG_PRI_IOMMU,
3057     .post_load = vtd_post_load,
3058     .fields = (VMStateField[]) {
3059         VMSTATE_UINT64(root, IntelIOMMUState),
3060         VMSTATE_UINT64(intr_root, IntelIOMMUState),
3061         VMSTATE_UINT64(iq, IntelIOMMUState),
3062         VMSTATE_UINT32(intr_size, IntelIOMMUState),
3063         VMSTATE_UINT16(iq_head, IntelIOMMUState),
3064         VMSTATE_UINT16(iq_tail, IntelIOMMUState),
3065         VMSTATE_UINT16(iq_size, IntelIOMMUState),
3066         VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState),
3067         VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE),
3068         VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState),
3069         VMSTATE_UNUSED(1),      /* bool root_extended is obsolete by VT-d */
3070         VMSTATE_BOOL(dmar_enabled, IntelIOMMUState),
3071         VMSTATE_BOOL(qi_enabled, IntelIOMMUState),
3072         VMSTATE_BOOL(intr_enabled, IntelIOMMUState),
3073         VMSTATE_BOOL(intr_eime, IntelIOMMUState),
3074         VMSTATE_END_OF_LIST()
3075     }
3076 };
3077 
3078 static const MemoryRegionOps vtd_mem_ops = {
3079     .read = vtd_mem_read,
3080     .write = vtd_mem_write,
3081     .endianness = DEVICE_LITTLE_ENDIAN,
3082     .impl = {
3083         .min_access_size = 4,
3084         .max_access_size = 8,
3085     },
3086     .valid = {
3087         .min_access_size = 4,
3088         .max_access_size = 8,
3089     },
3090 };
3091 
3092 static Property vtd_properties[] = {
3093     DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0),
3094     DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim,
3095                             ON_OFF_AUTO_AUTO),
3096     DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false),
3097     DEFINE_PROP_UINT8("aw-bits", IntelIOMMUState, aw_bits,
3098                       VTD_HOST_ADDRESS_WIDTH),
3099     DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE),
3100     DEFINE_PROP_BOOL("x-scalable-mode", IntelIOMMUState, scalable_mode, FALSE),
3101     DEFINE_PROP_BOOL("dma-drain", IntelIOMMUState, dma_drain, true),
3102     DEFINE_PROP_END_OF_LIST(),
3103 };
3104 
3105 /* Read IRTE entry with specific index */
3106 static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index,
3107                         VTD_IR_TableEntry *entry, uint16_t sid)
3108 {
3109     static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \
3110         {0xffff, 0xfffb, 0xfff9, 0xfff8};
3111     dma_addr_t addr = 0x00;
3112     uint16_t mask, source_id;
3113     uint8_t bus, bus_max, bus_min;
3114 
3115     if (index >= iommu->intr_size) {
3116         error_report_once("%s: index too large: ind=0x%x",
3117                           __func__, index);
3118         return -VTD_FR_IR_INDEX_OVER;
3119     }
3120 
3121     addr = iommu->intr_root + index * sizeof(*entry);
3122     if (dma_memory_read(&address_space_memory, addr, entry,
3123                         sizeof(*entry))) {
3124         error_report_once("%s: read failed: ind=0x%x addr=0x%" PRIx64,
3125                           __func__, index, addr);
3126         return -VTD_FR_IR_ROOT_INVAL;
3127     }
3128 
3129     trace_vtd_ir_irte_get(index, le64_to_cpu(entry->data[1]),
3130                           le64_to_cpu(entry->data[0]));
3131 
3132     if (!entry->irte.present) {
3133         error_report_once("%s: detected non-present IRTE "
3134                           "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")",
3135                           __func__, index, le64_to_cpu(entry->data[1]),
3136                           le64_to_cpu(entry->data[0]));
3137         return -VTD_FR_IR_ENTRY_P;
3138     }
3139 
3140     if (entry->irte.__reserved_0 || entry->irte.__reserved_1 ||
3141         entry->irte.__reserved_2) {
3142         error_report_once("%s: detected non-zero reserved IRTE "
3143                           "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")",
3144                           __func__, index, le64_to_cpu(entry->data[1]),
3145                           le64_to_cpu(entry->data[0]));
3146         return -VTD_FR_IR_IRTE_RSVD;
3147     }
3148 
3149     if (sid != X86_IOMMU_SID_INVALID) {
3150         /* Validate IRTE SID */
3151         source_id = le32_to_cpu(entry->irte.source_id);
3152         switch (entry->irte.sid_vtype) {
3153         case VTD_SVT_NONE:
3154             break;
3155 
3156         case VTD_SVT_ALL:
3157             mask = vtd_svt_mask[entry->irte.sid_q];
3158             if ((source_id & mask) != (sid & mask)) {
3159                 error_report_once("%s: invalid IRTE SID "
3160                                   "(index=%u, sid=%u, source_id=%u)",
3161                                   __func__, index, sid, source_id);
3162                 return -VTD_FR_IR_SID_ERR;
3163             }
3164             break;
3165 
3166         case VTD_SVT_BUS:
3167             bus_max = source_id >> 8;
3168             bus_min = source_id & 0xff;
3169             bus = sid >> 8;
3170             if (bus > bus_max || bus < bus_min) {
3171                 error_report_once("%s: invalid SVT_BUS "
3172                                   "(index=%u, bus=%u, min=%u, max=%u)",
3173                                   __func__, index, bus, bus_min, bus_max);
3174                 return -VTD_FR_IR_SID_ERR;
3175             }
3176             break;
3177 
3178         default:
3179             error_report_once("%s: detected invalid IRTE SVT "
3180                               "(index=%u, type=%d)", __func__,
3181                               index, entry->irte.sid_vtype);
3182             /* Take this as verification failure. */
3183             return -VTD_FR_IR_SID_ERR;
3184         }
3185     }
3186 
3187     return 0;
3188 }
3189 
3190 /* Fetch IRQ information of specific IR index */
3191 static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index,
3192                              X86IOMMUIrq *irq, uint16_t sid)
3193 {
3194     VTD_IR_TableEntry irte = {};
3195     int ret = 0;
3196 
3197     ret = vtd_irte_get(iommu, index, &irte, sid);
3198     if (ret) {
3199         return ret;
3200     }
3201 
3202     irq->trigger_mode = irte.irte.trigger_mode;
3203     irq->vector = irte.irte.vector;
3204     irq->delivery_mode = irte.irte.delivery_mode;
3205     irq->dest = le32_to_cpu(irte.irte.dest_id);
3206     if (!iommu->intr_eime) {
3207 #define  VTD_IR_APIC_DEST_MASK         (0xff00ULL)
3208 #define  VTD_IR_APIC_DEST_SHIFT        (8)
3209         irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >>
3210             VTD_IR_APIC_DEST_SHIFT;
3211     }
3212     irq->dest_mode = irte.irte.dest_mode;
3213     irq->redir_hint = irte.irte.redir_hint;
3214 
3215     trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector,
3216                        irq->delivery_mode, irq->dest, irq->dest_mode);
3217 
3218     return 0;
3219 }
3220 
3221 /* Interrupt remapping for MSI/MSI-X entry */
3222 static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu,
3223                                    MSIMessage *origin,
3224                                    MSIMessage *translated,
3225                                    uint16_t sid)
3226 {
3227     int ret = 0;
3228     VTD_IR_MSIAddress addr;
3229     uint16_t index;
3230     X86IOMMUIrq irq = {};
3231 
3232     assert(origin && translated);
3233 
3234     trace_vtd_ir_remap_msi_req(origin->address, origin->data);
3235 
3236     if (!iommu || !iommu->intr_enabled) {
3237         memcpy(translated, origin, sizeof(*origin));
3238         goto out;
3239     }
3240 
3241     if (origin->address & VTD_MSI_ADDR_HI_MASK) {
3242         error_report_once("%s: MSI address high 32 bits non-zero detected: "
3243                           "address=0x%" PRIx64, __func__, origin->address);
3244         return -VTD_FR_IR_REQ_RSVD;
3245     }
3246 
3247     addr.data = origin->address & VTD_MSI_ADDR_LO_MASK;
3248     if (addr.addr.__head != 0xfee) {
3249         error_report_once("%s: MSI address low 32 bit invalid: 0x%" PRIx32,
3250                           __func__, addr.data);
3251         return -VTD_FR_IR_REQ_RSVD;
3252     }
3253 
3254     /* This is compatible mode. */
3255     if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) {
3256         memcpy(translated, origin, sizeof(*origin));
3257         goto out;
3258     }
3259 
3260     index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l);
3261 
3262 #define  VTD_IR_MSI_DATA_SUBHANDLE       (0x0000ffff)
3263 #define  VTD_IR_MSI_DATA_RESERVED        (0xffff0000)
3264 
3265     if (addr.addr.sub_valid) {
3266         /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */
3267         index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE;
3268     }
3269 
3270     ret = vtd_remap_irq_get(iommu, index, &irq, sid);
3271     if (ret) {
3272         return ret;
3273     }
3274 
3275     if (addr.addr.sub_valid) {
3276         trace_vtd_ir_remap_type("MSI");
3277         if (origin->data & VTD_IR_MSI_DATA_RESERVED) {
3278             error_report_once("%s: invalid IR MSI "
3279                               "(sid=%u, address=0x%" PRIx64
3280                               ", data=0x%" PRIx32 ")",
3281                               __func__, sid, origin->address, origin->data);
3282             return -VTD_FR_IR_REQ_RSVD;
3283         }
3284     } else {
3285         uint8_t vector = origin->data & 0xff;
3286         uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1;
3287 
3288         trace_vtd_ir_remap_type("IOAPIC");
3289         /* IOAPIC entry vector should be aligned with IRTE vector
3290          * (see vt-d spec 5.1.5.1). */
3291         if (vector != irq.vector) {
3292             trace_vtd_warn_ir_vector(sid, index, vector, irq.vector);
3293         }
3294 
3295         /* The Trigger Mode field must match the Trigger Mode in the IRTE.
3296          * (see vt-d spec 5.1.5.1). */
3297         if (trigger_mode != irq.trigger_mode) {
3298             trace_vtd_warn_ir_trigger(sid, index, trigger_mode,
3299                                       irq.trigger_mode);
3300         }
3301     }
3302 
3303     /*
3304      * We'd better keep the last two bits, assuming that guest OS
3305      * might modify it. Keep it does not hurt after all.
3306      */
3307     irq.msi_addr_last_bits = addr.addr.__not_care;
3308 
3309     /* Translate X86IOMMUIrq to MSI message */
3310     x86_iommu_irq_to_msi_message(&irq, translated);
3311 
3312 out:
3313     trace_vtd_ir_remap_msi(origin->address, origin->data,
3314                            translated->address, translated->data);
3315     return 0;
3316 }
3317 
3318 static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src,
3319                          MSIMessage *dst, uint16_t sid)
3320 {
3321     return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu),
3322                                    src, dst, sid);
3323 }
3324 
3325 static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr,
3326                                    uint64_t *data, unsigned size,
3327                                    MemTxAttrs attrs)
3328 {
3329     return MEMTX_OK;
3330 }
3331 
3332 static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr,
3333                                     uint64_t value, unsigned size,
3334                                     MemTxAttrs attrs)
3335 {
3336     int ret = 0;
3337     MSIMessage from = {}, to = {};
3338     uint16_t sid = X86_IOMMU_SID_INVALID;
3339 
3340     from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST;
3341     from.data = (uint32_t) value;
3342 
3343     if (!attrs.unspecified) {
3344         /* We have explicit Source ID */
3345         sid = attrs.requester_id;
3346     }
3347 
3348     ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid);
3349     if (ret) {
3350         /* TODO: report error */
3351         /* Drop this interrupt */
3352         return MEMTX_ERROR;
3353     }
3354 
3355     apic_get_class()->send_msi(&to);
3356 
3357     return MEMTX_OK;
3358 }
3359 
3360 static const MemoryRegionOps vtd_mem_ir_ops = {
3361     .read_with_attrs = vtd_mem_ir_read,
3362     .write_with_attrs = vtd_mem_ir_write,
3363     .endianness = DEVICE_LITTLE_ENDIAN,
3364     .impl = {
3365         .min_access_size = 4,
3366         .max_access_size = 4,
3367     },
3368     .valid = {
3369         .min_access_size = 4,
3370         .max_access_size = 4,
3371     },
3372 };
3373 
3374 VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
3375 {
3376     uintptr_t key = (uintptr_t)bus;
3377     VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
3378     VTDAddressSpace *vtd_dev_as;
3379     char name[128];
3380 
3381     if (!vtd_bus) {
3382         uintptr_t *new_key = g_malloc(sizeof(*new_key));
3383         *new_key = (uintptr_t)bus;
3384         /* No corresponding free() */
3385         vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \
3386                             PCI_DEVFN_MAX);
3387         vtd_bus->bus = bus;
3388         g_hash_table_insert(s->vtd_as_by_busptr, new_key, vtd_bus);
3389     }
3390 
3391     vtd_dev_as = vtd_bus->dev_as[devfn];
3392 
3393     if (!vtd_dev_as) {
3394         snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn),
3395                  PCI_FUNC(devfn));
3396         vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
3397 
3398         vtd_dev_as->bus = bus;
3399         vtd_dev_as->devfn = (uint8_t)devfn;
3400         vtd_dev_as->iommu_state = s;
3401         vtd_dev_as->context_cache_entry.context_cache_gen = 0;
3402         vtd_dev_as->iova_tree = iova_tree_new();
3403 
3404         memory_region_init(&vtd_dev_as->root, OBJECT(s), name, UINT64_MAX);
3405         address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, "vtd-root");
3406 
3407         /*
3408          * Build the DMAR-disabled container with aliases to the
3409          * shared MRs.  Note that aliasing to a shared memory region
3410          * could help the memory API to detect same FlatViews so we
3411          * can have devices to share the same FlatView when DMAR is
3412          * disabled (either by not providing "intel_iommu=on" or with
3413          * "iommu=pt").  It will greatly reduce the total number of
3414          * FlatViews of the system hence VM runs faster.
3415          */
3416         memory_region_init_alias(&vtd_dev_as->nodmar, OBJECT(s),
3417                                  "vtd-nodmar", &s->mr_nodmar, 0,
3418                                  memory_region_size(&s->mr_nodmar));
3419 
3420         /*
3421          * Build the per-device DMAR-enabled container.
3422          *
3423          * TODO: currently we have per-device IOMMU memory region only
3424          * because we have per-device IOMMU notifiers for devices.  If
3425          * one day we can abstract the IOMMU notifiers out of the
3426          * memory regions then we can also share the same memory
3427          * region here just like what we've done above with the nodmar
3428          * region.
3429          */
3430         strcat(name, "-dmar");
3431         memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu),
3432                                  TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s),
3433                                  name, UINT64_MAX);
3434         memory_region_init_alias(&vtd_dev_as->iommu_ir, OBJECT(s), "vtd-ir",
3435                                  &s->mr_ir, 0, memory_region_size(&s->mr_ir));
3436         memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->iommu),
3437                                             VTD_INTERRUPT_ADDR_FIRST,
3438                                             &vtd_dev_as->iommu_ir, 1);
3439 
3440         /*
3441          * Hook both the containers under the root container, we
3442          * switch between DMAR & noDMAR by enable/disable
3443          * corresponding sub-containers
3444          */
3445         memory_region_add_subregion_overlap(&vtd_dev_as->root, 0,
3446                                             MEMORY_REGION(&vtd_dev_as->iommu),
3447                                             0);
3448         memory_region_add_subregion_overlap(&vtd_dev_as->root, 0,
3449                                             &vtd_dev_as->nodmar, 0);
3450 
3451         vtd_switch_address_space(vtd_dev_as);
3452     }
3453     return vtd_dev_as;
3454 }
3455 
3456 static uint64_t get_naturally_aligned_size(uint64_t start,
3457                                            uint64_t size, int gaw)
3458 {
3459     uint64_t max_mask = 1ULL << gaw;
3460     uint64_t alignment = start ? start & -start : max_mask;
3461 
3462     alignment = MIN(alignment, max_mask);
3463     size = MIN(size, max_mask);
3464 
3465     if (alignment <= size) {
3466         /* Increase the alignment of start */
3467         return alignment;
3468     } else {
3469         /* Find the largest page mask from size */
3470         return 1ULL << (63 - clz64(size));
3471     }
3472 }
3473 
3474 /* Unmap the whole range in the notifier's scope. */
3475 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n)
3476 {
3477     hwaddr size, remain;
3478     hwaddr start = n->start;
3479     hwaddr end = n->end;
3480     IntelIOMMUState *s = as->iommu_state;
3481     DMAMap map;
3482 
3483     /*
3484      * Note: all the codes in this function has a assumption that IOVA
3485      * bits are no more than VTD_MGAW bits (which is restricted by
3486      * VT-d spec), otherwise we need to consider overflow of 64 bits.
3487      */
3488 
3489     if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) {
3490         /*
3491          * Don't need to unmap regions that is bigger than the whole
3492          * VT-d supported address space size
3493          */
3494         end = VTD_ADDRESS_SIZE(s->aw_bits) - 1;
3495     }
3496 
3497     assert(start <= end);
3498     size = remain = end - start + 1;
3499 
3500     while (remain >= VTD_PAGE_SIZE) {
3501         IOMMUTLBEvent event;
3502         uint64_t mask = get_naturally_aligned_size(start, remain, s->aw_bits);
3503 
3504         assert(mask);
3505 
3506         event.type = IOMMU_NOTIFIER_UNMAP;
3507         event.entry.iova = start;
3508         event.entry.addr_mask = mask - 1;
3509         event.entry.target_as = &address_space_memory;
3510         event.entry.perm = IOMMU_NONE;
3511         /* This field is meaningless for unmap */
3512         event.entry.translated_addr = 0;
3513 
3514         memory_region_notify_iommu_one(n, &event);
3515 
3516         start += mask;
3517         remain -= mask;
3518     }
3519 
3520     assert(!remain);
3521 
3522     trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
3523                              VTD_PCI_SLOT(as->devfn),
3524                              VTD_PCI_FUNC(as->devfn),
3525                              n->start, size);
3526 
3527     map.iova = n->start;
3528     map.size = size;
3529     iova_tree_remove(as->iova_tree, &map);
3530 }
3531 
3532 static void vtd_address_space_unmap_all(IntelIOMMUState *s)
3533 {
3534     VTDAddressSpace *vtd_as;
3535     IOMMUNotifier *n;
3536 
3537     QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) {
3538         IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) {
3539             vtd_address_space_unmap(vtd_as, n);
3540         }
3541     }
3542 }
3543 
3544 static void vtd_address_space_refresh_all(IntelIOMMUState *s)
3545 {
3546     vtd_address_space_unmap_all(s);
3547     vtd_switch_address_space_all(s);
3548 }
3549 
3550 static int vtd_replay_hook(IOMMUTLBEvent *event, void *private)
3551 {
3552     memory_region_notify_iommu_one(private, event);
3553     return 0;
3554 }
3555 
3556 static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n)
3557 {
3558     VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu);
3559     IntelIOMMUState *s = vtd_as->iommu_state;
3560     uint8_t bus_n = pci_bus_num(vtd_as->bus);
3561     VTDContextEntry ce;
3562 
3563     /*
3564      * The replay can be triggered by either a invalidation or a newly
3565      * created entry. No matter what, we release existing mappings
3566      * (it means flushing caches for UNMAP-only registers).
3567      */
3568     vtd_address_space_unmap(vtd_as, n);
3569 
3570     if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) {
3571         trace_vtd_replay_ce_valid(s->root_scalable ? "scalable mode" :
3572                                   "legacy mode",
3573                                   bus_n, PCI_SLOT(vtd_as->devfn),
3574                                   PCI_FUNC(vtd_as->devfn),
3575                                   vtd_get_domain_id(s, &ce),
3576                                   ce.hi, ce.lo);
3577         if (vtd_as_has_map_notifier(vtd_as)) {
3578             /* This is required only for MAP typed notifiers */
3579             vtd_page_walk_info info = {
3580                 .hook_fn = vtd_replay_hook,
3581                 .private = (void *)n,
3582                 .notify_unmap = false,
3583                 .aw = s->aw_bits,
3584                 .as = vtd_as,
3585                 .domain_id = vtd_get_domain_id(s, &ce),
3586             };
3587 
3588             vtd_page_walk(s, &ce, 0, ~0ULL, &info);
3589         }
3590     } else {
3591         trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn),
3592                                     PCI_FUNC(vtd_as->devfn));
3593     }
3594 
3595     return;
3596 }
3597 
3598 /* Do the initialization. It will also be called when reset, so pay
3599  * attention when adding new initialization stuff.
3600  */
3601 static void vtd_init(IntelIOMMUState *s)
3602 {
3603     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
3604 
3605     memset(s->csr, 0, DMAR_REG_SIZE);
3606     memset(s->wmask, 0, DMAR_REG_SIZE);
3607     memset(s->w1cmask, 0, DMAR_REG_SIZE);
3608     memset(s->womask, 0, DMAR_REG_SIZE);
3609 
3610     s->root = 0;
3611     s->root_scalable = false;
3612     s->dmar_enabled = false;
3613     s->intr_enabled = false;
3614     s->iq_head = 0;
3615     s->iq_tail = 0;
3616     s->iq = 0;
3617     s->iq_size = 0;
3618     s->qi_enabled = false;
3619     s->iq_last_desc_type = VTD_INV_DESC_NONE;
3620     s->iq_dw = false;
3621     s->next_frcd_reg = 0;
3622     s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND |
3623              VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS |
3624              VTD_CAP_SAGAW_39bit | VTD_CAP_MGAW(s->aw_bits);
3625     if (s->dma_drain) {
3626         s->cap |= VTD_CAP_DRAIN;
3627     }
3628     if (s->aw_bits == VTD_HOST_AW_48BIT) {
3629         s->cap |= VTD_CAP_SAGAW_48bit;
3630     }
3631     s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
3632 
3633     /*
3634      * Rsvd field masks for spte
3635      */
3636     vtd_spte_rsvd[0] = ~0ULL;
3637     vtd_spte_rsvd[1] = VTD_SPTE_PAGE_L1_RSVD_MASK(s->aw_bits,
3638                                                   x86_iommu->dt_supported);
3639     vtd_spte_rsvd[2] = VTD_SPTE_PAGE_L2_RSVD_MASK(s->aw_bits);
3640     vtd_spte_rsvd[3] = VTD_SPTE_PAGE_L3_RSVD_MASK(s->aw_bits);
3641     vtd_spte_rsvd[4] = VTD_SPTE_PAGE_L4_RSVD_MASK(s->aw_bits);
3642 
3643     vtd_spte_rsvd_large[2] = VTD_SPTE_LPAGE_L2_RSVD_MASK(s->aw_bits,
3644                                                          x86_iommu->dt_supported);
3645     vtd_spte_rsvd_large[3] = VTD_SPTE_LPAGE_L3_RSVD_MASK(s->aw_bits,
3646                                                          x86_iommu->dt_supported);
3647 
3648     if (x86_iommu_ir_supported(x86_iommu)) {
3649         s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV;
3650         if (s->intr_eim == ON_OFF_AUTO_ON) {
3651             s->ecap |= VTD_ECAP_EIM;
3652         }
3653         assert(s->intr_eim != ON_OFF_AUTO_AUTO);
3654     }
3655 
3656     if (x86_iommu->dt_supported) {
3657         s->ecap |= VTD_ECAP_DT;
3658     }
3659 
3660     if (x86_iommu->pt_supported) {
3661         s->ecap |= VTD_ECAP_PT;
3662     }
3663 
3664     if (s->caching_mode) {
3665         s->cap |= VTD_CAP_CM;
3666     }
3667 
3668     /* TODO: read cap/ecap from host to decide which cap to be exposed. */
3669     if (s->scalable_mode) {
3670         s->ecap |= VTD_ECAP_SMTS | VTD_ECAP_SRS | VTD_ECAP_SLTS;
3671     }
3672 
3673     vtd_reset_caches(s);
3674 
3675     /* Define registers with default values and bit semantics */
3676     vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);
3677     vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);
3678     vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0);
3679     vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0);
3680     vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL);
3681     vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0);
3682     vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffffc00ULL, 0);
3683     vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0);
3684     vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL);
3685 
3686     /* Advanced Fault Logging not supported */
3687     vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL);
3688     vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0);
3689     vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0);
3690     vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0);
3691 
3692     /* Treated as RsvdZ when EIM in ECAP_REG is not supported
3693      * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0);
3694      */
3695     vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0);
3696 
3697     /* Treated as RO for implementations that PLMR and PHMR fields reported
3698      * as Clear in the CAP_REG.
3699      * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0);
3700      */
3701     vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0);
3702 
3703     vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0);
3704     vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0);
3705     vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff807ULL, 0);
3706     vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL);
3707     vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0);
3708     vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0);
3709     vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0);
3710     /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */
3711     vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0);
3712 
3713     /* IOTLB registers */
3714     vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0);
3715     vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0);
3716     vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL);
3717 
3718     /* Fault Recording Registers, 128-bit */
3719     vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0);
3720     vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL);
3721 
3722     /*
3723      * Interrupt remapping registers.
3724      */
3725     vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0);
3726 }
3727 
3728 /* Should not reset address_spaces when reset because devices will still use
3729  * the address space they got at first (won't ask the bus again).
3730  */
3731 static void vtd_reset(DeviceState *dev)
3732 {
3733     IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
3734 
3735     vtd_init(s);
3736     vtd_address_space_refresh_all(s);
3737 }
3738 
3739 static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
3740 {
3741     IntelIOMMUState *s = opaque;
3742     VTDAddressSpace *vtd_as;
3743 
3744     assert(0 <= devfn && devfn < PCI_DEVFN_MAX);
3745 
3746     vtd_as = vtd_find_add_as(s, bus, devfn);
3747     return &vtd_as->as;
3748 }
3749 
3750 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp)
3751 {
3752     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
3753 
3754     if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu_ir_supported(x86_iommu)) {
3755         error_setg(errp, "eim=on cannot be selected without intremap=on");
3756         return false;
3757     }
3758 
3759     if (s->intr_eim == ON_OFF_AUTO_AUTO) {
3760         s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim)
3761                       && x86_iommu_ir_supported(x86_iommu) ?
3762                                               ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
3763     }
3764     if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) {
3765         if (!kvm_irqchip_in_kernel()) {
3766             error_setg(errp, "eim=on requires accel=kvm,kernel-irqchip=split");
3767             return false;
3768         }
3769         if (!kvm_enable_x2apic()) {
3770             error_setg(errp, "eim=on requires support on the KVM side"
3771                              "(X2APIC_API, first shipped in v4.7)");
3772             return false;
3773         }
3774     }
3775 
3776     /* Currently only address widths supported are 39 and 48 bits */
3777     if ((s->aw_bits != VTD_HOST_AW_39BIT) &&
3778         (s->aw_bits != VTD_HOST_AW_48BIT)) {
3779         error_setg(errp, "Supported values for aw-bits are: %d, %d",
3780                    VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT);
3781         return false;
3782     }
3783 
3784     if (s->scalable_mode && !s->dma_drain) {
3785         error_setg(errp, "Need to set dma_drain for scalable mode");
3786         return false;
3787     }
3788 
3789     return true;
3790 }
3791 
3792 static int vtd_machine_done_notify_one(Object *child, void *unused)
3793 {
3794     IntelIOMMUState *iommu = INTEL_IOMMU_DEVICE(x86_iommu_get_default());
3795 
3796     /*
3797      * We hard-coded here because vfio-pci is the only special case
3798      * here.  Let's be more elegant in the future when we can, but so
3799      * far there seems to be no better way.
3800      */
3801     if (object_dynamic_cast(child, "vfio-pci") && !iommu->caching_mode) {
3802         vtd_panic_require_caching_mode();
3803     }
3804 
3805     return 0;
3806 }
3807 
3808 static void vtd_machine_done_hook(Notifier *notifier, void *unused)
3809 {
3810     object_child_foreach_recursive(object_get_root(),
3811                                    vtd_machine_done_notify_one, NULL);
3812 }
3813 
3814 static Notifier vtd_machine_done_notify = {
3815     .notify = vtd_machine_done_hook,
3816 };
3817 
3818 static void vtd_realize(DeviceState *dev, Error **errp)
3819 {
3820     MachineState *ms = MACHINE(qdev_get_machine());
3821     PCMachineState *pcms = PC_MACHINE(ms);
3822     X86MachineState *x86ms = X86_MACHINE(ms);
3823     PCIBus *bus = pcms->bus;
3824     IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
3825     X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev);
3826 
3827     x86_iommu->type = TYPE_INTEL;
3828 
3829     if (!vtd_decide_config(s, errp)) {
3830         return;
3831     }
3832 
3833     QLIST_INIT(&s->vtd_as_with_notifiers);
3834     qemu_mutex_init(&s->iommu_lock);
3835     memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
3836     memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
3837                           "intel_iommu", DMAR_REG_SIZE);
3838 
3839     /* Create the shared memory regions by all devices */
3840     memory_region_init(&s->mr_nodmar, OBJECT(s), "vtd-nodmar",
3841                        UINT64_MAX);
3842     memory_region_init_io(&s->mr_ir, OBJECT(s), &vtd_mem_ir_ops,
3843                           s, "vtd-ir", VTD_INTERRUPT_ADDR_SIZE);
3844     memory_region_init_alias(&s->mr_sys_alias, OBJECT(s),
3845                              "vtd-sys-alias", get_system_memory(), 0,
3846                              memory_region_size(get_system_memory()));
3847     memory_region_add_subregion_overlap(&s->mr_nodmar, 0,
3848                                         &s->mr_sys_alias, 0);
3849     memory_region_add_subregion_overlap(&s->mr_nodmar,
3850                                         VTD_INTERRUPT_ADDR_FIRST,
3851                                         &s->mr_ir, 1);
3852 
3853     sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
3854     /* No corresponding destroy */
3855     s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
3856                                      g_free, g_free);
3857     s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
3858                                               g_free, g_free);
3859     vtd_init(s);
3860     sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
3861     pci_setup_iommu(bus, vtd_host_dma_iommu, dev);
3862     /* Pseudo address space under root PCI bus. */
3863     x86ms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC);
3864     qemu_add_machine_init_done_notifier(&vtd_machine_done_notify);
3865 }
3866 
3867 static void vtd_class_init(ObjectClass *klass, void *data)
3868 {
3869     DeviceClass *dc = DEVICE_CLASS(klass);
3870     X86IOMMUClass *x86_class = X86_IOMMU_DEVICE_CLASS(klass);
3871 
3872     dc->reset = vtd_reset;
3873     dc->vmsd = &vtd_vmstate;
3874     device_class_set_props(dc, vtd_properties);
3875     dc->hotpluggable = false;
3876     x86_class->realize = vtd_realize;
3877     x86_class->int_remap = vtd_int_remap;
3878     /* Supported by the pc-q35-* machine types */
3879     dc->user_creatable = true;
3880     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
3881     dc->desc = "Intel IOMMU (VT-d) DMA Remapping device";
3882 }
3883 
3884 static const TypeInfo vtd_info = {
3885     .name          = TYPE_INTEL_IOMMU_DEVICE,
3886     .parent        = TYPE_X86_IOMMU_DEVICE,
3887     .instance_size = sizeof(IntelIOMMUState),
3888     .class_init    = vtd_class_init,
3889 };
3890 
3891 static void vtd_iommu_memory_region_class_init(ObjectClass *klass,
3892                                                      void *data)
3893 {
3894     IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
3895 
3896     imrc->translate = vtd_iommu_translate;
3897     imrc->notify_flag_changed = vtd_iommu_notify_flag_changed;
3898     imrc->replay = vtd_iommu_replay;
3899 }
3900 
3901 static const TypeInfo vtd_iommu_memory_region_info = {
3902     .parent = TYPE_IOMMU_MEMORY_REGION,
3903     .name = TYPE_INTEL_IOMMU_MEMORY_REGION,
3904     .class_init = vtd_iommu_memory_region_class_init,
3905 };
3906 
3907 static void vtd_register_types(void)
3908 {
3909     type_register_static(&vtd_info);
3910     type_register_static(&vtd_iommu_memory_region_info);
3911 }
3912 
3913 type_init(vtd_register_types)
3914