1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "qemu/osdep.h" 23 #include "qemu/error-report.h" 24 #include "qemu/main-loop.h" 25 #include "qapi/error.h" 26 #include "hw/sysbus.h" 27 #include "intel_iommu_internal.h" 28 #include "hw/pci/pci.h" 29 #include "hw/pci/pci_bus.h" 30 #include "hw/qdev-properties.h" 31 #include "hw/i386/pc.h" 32 #include "hw/i386/apic-msidef.h" 33 #include "hw/i386/x86-iommu.h" 34 #include "hw/pci-host/q35.h" 35 #include "sysemu/kvm.h" 36 #include "sysemu/dma.h" 37 #include "sysemu/sysemu.h" 38 #include "hw/i386/apic_internal.h" 39 #include "kvm/kvm_i386.h" 40 #include "migration/vmstate.h" 41 #include "trace.h" 42 43 /* context entry operations */ 44 #define VTD_CE_GET_RID2PASID(ce) \ 45 ((ce)->val[1] & VTD_SM_CONTEXT_ENTRY_RID2PASID_MASK) 46 #define VTD_CE_GET_PASID_DIR_TABLE(ce) \ 47 ((ce)->val[0] & VTD_PASID_DIR_BASE_ADDR_MASK) 48 49 /* pe operations */ 50 #define VTD_PE_GET_TYPE(pe) ((pe)->val[0] & VTD_SM_PASID_ENTRY_PGTT) 51 #define VTD_PE_GET_LEVEL(pe) (2 + (((pe)->val[0] >> 2) & VTD_SM_PASID_ENTRY_AW)) 52 53 /* 54 * PCI bus number (or SID) is not reliable since the device is usaully 55 * initialized before guest can configure the PCI bridge 56 * (SECONDARY_BUS_NUMBER). 57 */ 58 struct vtd_as_key { 59 PCIBus *bus; 60 uint8_t devfn; 61 uint32_t pasid; 62 }; 63 64 struct vtd_iotlb_key { 65 uint64_t gfn; 66 uint32_t pasid; 67 uint16_t sid; 68 uint8_t level; 69 }; 70 71 static void vtd_address_space_refresh_all(IntelIOMMUState *s); 72 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n); 73 74 static void vtd_panic_require_caching_mode(void) 75 { 76 error_report("We need to set caching-mode=on for intel-iommu to enable " 77 "device assignment with IOMMU protection."); 78 exit(1); 79 } 80 81 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 82 uint64_t wmask, uint64_t w1cmask) 83 { 84 stq_le_p(&s->csr[addr], val); 85 stq_le_p(&s->wmask[addr], wmask); 86 stq_le_p(&s->w1cmask[addr], w1cmask); 87 } 88 89 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 90 { 91 stq_le_p(&s->womask[addr], mask); 92 } 93 94 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 95 uint32_t wmask, uint32_t w1cmask) 96 { 97 stl_le_p(&s->csr[addr], val); 98 stl_le_p(&s->wmask[addr], wmask); 99 stl_le_p(&s->w1cmask[addr], w1cmask); 100 } 101 102 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 103 { 104 stl_le_p(&s->womask[addr], mask); 105 } 106 107 /* "External" get/set operations */ 108 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 109 { 110 uint64_t oldval = ldq_le_p(&s->csr[addr]); 111 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 112 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 113 stq_le_p(&s->csr[addr], 114 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 115 } 116 117 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 118 { 119 uint32_t oldval = ldl_le_p(&s->csr[addr]); 120 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 121 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 122 stl_le_p(&s->csr[addr], 123 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 124 } 125 126 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 127 { 128 uint64_t val = ldq_le_p(&s->csr[addr]); 129 uint64_t womask = ldq_le_p(&s->womask[addr]); 130 return val & ~womask; 131 } 132 133 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 134 { 135 uint32_t val = ldl_le_p(&s->csr[addr]); 136 uint32_t womask = ldl_le_p(&s->womask[addr]); 137 return val & ~womask; 138 } 139 140 /* "Internal" get/set operations */ 141 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 142 { 143 return ldq_le_p(&s->csr[addr]); 144 } 145 146 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 147 { 148 return ldl_le_p(&s->csr[addr]); 149 } 150 151 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 152 { 153 stq_le_p(&s->csr[addr], val); 154 } 155 156 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 157 uint32_t clear, uint32_t mask) 158 { 159 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 160 stl_le_p(&s->csr[addr], new_val); 161 return new_val; 162 } 163 164 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 165 uint64_t clear, uint64_t mask) 166 { 167 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 168 stq_le_p(&s->csr[addr], new_val); 169 return new_val; 170 } 171 172 static inline void vtd_iommu_lock(IntelIOMMUState *s) 173 { 174 qemu_mutex_lock(&s->iommu_lock); 175 } 176 177 static inline void vtd_iommu_unlock(IntelIOMMUState *s) 178 { 179 qemu_mutex_unlock(&s->iommu_lock); 180 } 181 182 static void vtd_update_scalable_state(IntelIOMMUState *s) 183 { 184 uint64_t val = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 185 186 if (s->scalable_mode) { 187 s->root_scalable = val & VTD_RTADDR_SMT; 188 } 189 } 190 191 static void vtd_update_iq_dw(IntelIOMMUState *s) 192 { 193 uint64_t val = vtd_get_quad_raw(s, DMAR_IQA_REG); 194 195 if (s->ecap & VTD_ECAP_SMTS && 196 val & VTD_IQA_DW_MASK) { 197 s->iq_dw = true; 198 } else { 199 s->iq_dw = false; 200 } 201 } 202 203 /* Whether the address space needs to notify new mappings */ 204 static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as) 205 { 206 return as->notifier_flags & IOMMU_NOTIFIER_MAP; 207 } 208 209 /* GHashTable functions */ 210 static gboolean vtd_iotlb_equal(gconstpointer v1, gconstpointer v2) 211 { 212 const struct vtd_iotlb_key *key1 = v1; 213 const struct vtd_iotlb_key *key2 = v2; 214 215 return key1->sid == key2->sid && 216 key1->pasid == key2->pasid && 217 key1->level == key2->level && 218 key1->gfn == key2->gfn; 219 } 220 221 static guint vtd_iotlb_hash(gconstpointer v) 222 { 223 const struct vtd_iotlb_key *key = v; 224 uint64_t hash64 = key->gfn | ((uint64_t)(key->sid) << VTD_IOTLB_SID_SHIFT) | 225 (uint64_t)(key->level - 1) << VTD_IOTLB_LVL_SHIFT | 226 (uint64_t)(key->pasid) << VTD_IOTLB_PASID_SHIFT; 227 228 return (guint)((hash64 >> 32) ^ (hash64 & 0xffffffffU)); 229 } 230 231 static gboolean vtd_as_equal(gconstpointer v1, gconstpointer v2) 232 { 233 const struct vtd_as_key *key1 = v1; 234 const struct vtd_as_key *key2 = v2; 235 236 return (key1->bus == key2->bus) && (key1->devfn == key2->devfn) && 237 (key1->pasid == key2->pasid); 238 } 239 240 /* 241 * Note that we use pointer to PCIBus as the key, so hashing/shifting 242 * based on the pointer value is intended. Note that we deal with 243 * collisions through vtd_as_equal(). 244 */ 245 static guint vtd_as_hash(gconstpointer v) 246 { 247 const struct vtd_as_key *key = v; 248 guint value = (guint)(uintptr_t)key->bus; 249 250 return (guint)(value << 8 | key->devfn); 251 } 252 253 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 254 gpointer user_data) 255 { 256 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 257 uint16_t domain_id = *(uint16_t *)user_data; 258 return entry->domain_id == domain_id; 259 } 260 261 /* The shift of an addr for a certain level of paging structure */ 262 static inline uint32_t vtd_slpt_level_shift(uint32_t level) 263 { 264 assert(level != 0); 265 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 266 } 267 268 static inline uint64_t vtd_slpt_level_page_mask(uint32_t level) 269 { 270 return ~((1ULL << vtd_slpt_level_shift(level)) - 1); 271 } 272 273 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 274 gpointer user_data) 275 { 276 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 277 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 278 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask; 279 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K; 280 return (entry->domain_id == info->domain_id) && 281 (((entry->gfn & info->mask) == gfn) || 282 (entry->gfn == gfn_tlb)); 283 } 284 285 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 286 * IntelIOMMUState to 1. Must be called with IOMMU lock held. 287 */ 288 static void vtd_reset_context_cache_locked(IntelIOMMUState *s) 289 { 290 VTDAddressSpace *vtd_as; 291 GHashTableIter as_it; 292 293 trace_vtd_context_cache_reset(); 294 295 g_hash_table_iter_init(&as_it, s->vtd_address_spaces); 296 297 while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) { 298 vtd_as->context_cache_entry.context_cache_gen = 0; 299 } 300 s->context_cache_gen = 1; 301 } 302 303 /* Must be called with IOMMU lock held. */ 304 static void vtd_reset_iotlb_locked(IntelIOMMUState *s) 305 { 306 assert(s->iotlb); 307 g_hash_table_remove_all(s->iotlb); 308 } 309 310 static void vtd_reset_iotlb(IntelIOMMUState *s) 311 { 312 vtd_iommu_lock(s); 313 vtd_reset_iotlb_locked(s); 314 vtd_iommu_unlock(s); 315 } 316 317 static void vtd_reset_caches(IntelIOMMUState *s) 318 { 319 vtd_iommu_lock(s); 320 vtd_reset_iotlb_locked(s); 321 vtd_reset_context_cache_locked(s); 322 vtd_iommu_unlock(s); 323 } 324 325 static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) 326 { 327 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K; 328 } 329 330 /* Must be called with IOMMU lock held */ 331 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 332 uint32_t pasid, hwaddr addr) 333 { 334 struct vtd_iotlb_key key; 335 VTDIOTLBEntry *entry; 336 int level; 337 338 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) { 339 key.gfn = vtd_get_iotlb_gfn(addr, level); 340 key.level = level; 341 key.sid = source_id; 342 key.pasid = pasid; 343 entry = g_hash_table_lookup(s->iotlb, &key); 344 if (entry) { 345 goto out; 346 } 347 } 348 349 out: 350 return entry; 351 } 352 353 /* Must be with IOMMU lock held */ 354 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 355 uint16_t domain_id, hwaddr addr, uint64_t slpte, 356 uint8_t access_flags, uint32_t level, 357 uint32_t pasid) 358 { 359 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 360 struct vtd_iotlb_key *key = g_malloc(sizeof(*key)); 361 uint64_t gfn = vtd_get_iotlb_gfn(addr, level); 362 363 trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id); 364 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 365 trace_vtd_iotlb_reset("iotlb exceeds size limit"); 366 vtd_reset_iotlb_locked(s); 367 } 368 369 entry->gfn = gfn; 370 entry->domain_id = domain_id; 371 entry->slpte = slpte; 372 entry->access_flags = access_flags; 373 entry->mask = vtd_slpt_level_page_mask(level); 374 entry->pasid = pasid; 375 376 key->gfn = gfn; 377 key->sid = source_id; 378 key->level = level; 379 key->pasid = pasid; 380 381 g_hash_table_replace(s->iotlb, key, entry); 382 } 383 384 /* Given the reg addr of both the message data and address, generate an 385 * interrupt via MSI. 386 */ 387 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 388 hwaddr mesg_data_reg) 389 { 390 MSIMessage msi; 391 392 assert(mesg_data_reg < DMAR_REG_SIZE); 393 assert(mesg_addr_reg < DMAR_REG_SIZE); 394 395 msi.address = vtd_get_long_raw(s, mesg_addr_reg); 396 msi.data = vtd_get_long_raw(s, mesg_data_reg); 397 398 trace_vtd_irq_generate(msi.address, msi.data); 399 400 apic_get_class(NULL)->send_msi(&msi); 401 } 402 403 /* Generate a fault event to software via MSI if conditions are met. 404 * Notice that the value of FSTS_REG being passed to it should be the one 405 * before any update. 406 */ 407 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 408 { 409 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 410 pre_fsts & VTD_FSTS_IQE) { 411 error_report_once("There are previous interrupt conditions " 412 "to be serviced by software, fault event " 413 "is not generated"); 414 return; 415 } 416 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 417 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 418 error_report_once("Interrupt Mask set, irq is not generated"); 419 } else { 420 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 421 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 422 } 423 } 424 425 /* Check if the Fault (F) field of the Fault Recording Register referenced by 426 * @index is Set. 427 */ 428 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 429 { 430 /* Each reg is 128-bit */ 431 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 432 addr += 8; /* Access the high 64-bit half */ 433 434 assert(index < DMAR_FRCD_REG_NR); 435 436 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 437 } 438 439 /* Update the PPF field of Fault Status Register. 440 * Should be called whenever change the F field of any fault recording 441 * registers. 442 */ 443 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 444 { 445 uint32_t i; 446 uint32_t ppf_mask = 0; 447 448 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 449 if (vtd_is_frcd_set(s, i)) { 450 ppf_mask = VTD_FSTS_PPF; 451 break; 452 } 453 } 454 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 455 trace_vtd_fsts_ppf(!!ppf_mask); 456 } 457 458 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 459 { 460 /* Each reg is 128-bit */ 461 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 462 addr += 8; /* Access the high 64-bit half */ 463 464 assert(index < DMAR_FRCD_REG_NR); 465 466 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 467 vtd_update_fsts_ppf(s); 468 } 469 470 /* Must not update F field now, should be done later */ 471 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 472 uint64_t hi, uint64_t lo) 473 { 474 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 475 476 assert(index < DMAR_FRCD_REG_NR); 477 478 vtd_set_quad_raw(s, frcd_reg_addr, lo); 479 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 480 481 trace_vtd_frr_new(index, hi, lo); 482 } 483 484 /* Try to collapse multiple pending faults from the same requester */ 485 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 486 { 487 uint32_t i; 488 uint64_t frcd_reg; 489 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 490 491 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 492 frcd_reg = vtd_get_quad_raw(s, addr); 493 if ((frcd_reg & VTD_FRCD_F) && 494 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 495 return true; 496 } 497 addr += 16; /* 128-bit for each */ 498 } 499 return false; 500 } 501 502 /* Log and report an DMAR (address translation) fault to software */ 503 static void vtd_report_frcd_fault(IntelIOMMUState *s, uint64_t source_id, 504 uint64_t hi, uint64_t lo) 505 { 506 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 507 508 if (fsts_reg & VTD_FSTS_PFO) { 509 error_report_once("New fault is not recorded due to " 510 "Primary Fault Overflow"); 511 return; 512 } 513 514 if (vtd_try_collapse_fault(s, source_id)) { 515 error_report_once("New fault is not recorded due to " 516 "compression of faults"); 517 return; 518 } 519 520 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 521 error_report_once("Next Fault Recording Reg is used, " 522 "new fault is not recorded, set PFO field"); 523 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 524 return; 525 } 526 527 vtd_record_frcd(s, s->next_frcd_reg, hi, lo); 528 529 if (fsts_reg & VTD_FSTS_PPF) { 530 error_report_once("There are pending faults already, " 531 "fault event is not generated"); 532 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 533 s->next_frcd_reg++; 534 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 535 s->next_frcd_reg = 0; 536 } 537 } else { 538 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 539 VTD_FSTS_FRI(s->next_frcd_reg)); 540 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 541 s->next_frcd_reg++; 542 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 543 s->next_frcd_reg = 0; 544 } 545 /* This case actually cause the PPF to be Set. 546 * So generate fault event (interrupt). 547 */ 548 vtd_generate_fault_event(s, fsts_reg); 549 } 550 } 551 552 /* Log and report an DMAR (address translation) fault to software */ 553 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 554 hwaddr addr, VTDFaultReason fault, 555 bool is_write, bool is_pasid, 556 uint32_t pasid) 557 { 558 uint64_t hi, lo; 559 560 assert(fault < VTD_FR_MAX); 561 562 trace_vtd_dmar_fault(source_id, fault, addr, is_write); 563 564 lo = VTD_FRCD_FI(addr); 565 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault) | 566 VTD_FRCD_PV(pasid) | VTD_FRCD_PP(is_pasid); 567 if (!is_write) { 568 hi |= VTD_FRCD_T; 569 } 570 571 vtd_report_frcd_fault(s, source_id, hi, lo); 572 } 573 574 575 static void vtd_report_ir_fault(IntelIOMMUState *s, uint64_t source_id, 576 VTDFaultReason fault, uint16_t index) 577 { 578 uint64_t hi, lo; 579 580 lo = VTD_FRCD_IR_IDX(index); 581 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 582 583 vtd_report_frcd_fault(s, source_id, hi, lo); 584 } 585 586 /* Handle Invalidation Queue Errors of queued invalidation interface error 587 * conditions. 588 */ 589 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 590 { 591 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 592 593 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 594 vtd_generate_fault_event(s, fsts_reg); 595 } 596 597 /* Set the IWC field and try to generate an invalidation completion interrupt */ 598 static void vtd_generate_completion_event(IntelIOMMUState *s) 599 { 600 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 601 trace_vtd_inv_desc_wait_irq("One pending, skip current"); 602 return; 603 } 604 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 605 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 606 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 607 trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, " 608 "new event not generated"); 609 return; 610 } else { 611 /* Generate the interrupt event */ 612 trace_vtd_inv_desc_wait_irq("Generating complete event"); 613 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 614 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 615 } 616 } 617 618 static inline bool vtd_root_entry_present(IntelIOMMUState *s, 619 VTDRootEntry *re, 620 uint8_t devfn) 621 { 622 if (s->root_scalable && devfn > UINT8_MAX / 2) { 623 return re->hi & VTD_ROOT_ENTRY_P; 624 } 625 626 return re->lo & VTD_ROOT_ENTRY_P; 627 } 628 629 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 630 VTDRootEntry *re) 631 { 632 dma_addr_t addr; 633 634 addr = s->root + index * sizeof(*re); 635 if (dma_memory_read(&address_space_memory, addr, 636 re, sizeof(*re), MEMTXATTRS_UNSPECIFIED)) { 637 re->lo = 0; 638 return -VTD_FR_ROOT_TABLE_INV; 639 } 640 re->lo = le64_to_cpu(re->lo); 641 re->hi = le64_to_cpu(re->hi); 642 return 0; 643 } 644 645 static inline bool vtd_ce_present(VTDContextEntry *context) 646 { 647 return context->lo & VTD_CONTEXT_ENTRY_P; 648 } 649 650 static int vtd_get_context_entry_from_root(IntelIOMMUState *s, 651 VTDRootEntry *re, 652 uint8_t index, 653 VTDContextEntry *ce) 654 { 655 dma_addr_t addr, ce_size; 656 657 /* we have checked that root entry is present */ 658 ce_size = s->root_scalable ? VTD_CTX_ENTRY_SCALABLE_SIZE : 659 VTD_CTX_ENTRY_LEGACY_SIZE; 660 661 if (s->root_scalable && index > UINT8_MAX / 2) { 662 index = index & (~VTD_DEVFN_CHECK_MASK); 663 addr = re->hi & VTD_ROOT_ENTRY_CTP; 664 } else { 665 addr = re->lo & VTD_ROOT_ENTRY_CTP; 666 } 667 668 addr = addr + index * ce_size; 669 if (dma_memory_read(&address_space_memory, addr, 670 ce, ce_size, MEMTXATTRS_UNSPECIFIED)) { 671 return -VTD_FR_CONTEXT_TABLE_INV; 672 } 673 674 ce->lo = le64_to_cpu(ce->lo); 675 ce->hi = le64_to_cpu(ce->hi); 676 if (ce_size == VTD_CTX_ENTRY_SCALABLE_SIZE) { 677 ce->val[2] = le64_to_cpu(ce->val[2]); 678 ce->val[3] = le64_to_cpu(ce->val[3]); 679 } 680 return 0; 681 } 682 683 static inline dma_addr_t vtd_ce_get_slpt_base(VTDContextEntry *ce) 684 { 685 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 686 } 687 688 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte, uint8_t aw) 689 { 690 return slpte & VTD_SL_PT_BASE_ADDR_MASK(aw); 691 } 692 693 /* Whether the pte indicates the address of the page frame */ 694 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 695 { 696 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 697 } 698 699 /* Get the content of a spte located in @base_addr[@index] */ 700 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 701 { 702 uint64_t slpte; 703 704 assert(index < VTD_SL_PT_ENTRY_NR); 705 706 if (dma_memory_read(&address_space_memory, 707 base_addr + index * sizeof(slpte), 708 &slpte, sizeof(slpte), MEMTXATTRS_UNSPECIFIED)) { 709 slpte = (uint64_t)-1; 710 return slpte; 711 } 712 slpte = le64_to_cpu(slpte); 713 return slpte; 714 } 715 716 /* Given an iova and the level of paging structure, return the offset 717 * of current level. 718 */ 719 static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level) 720 { 721 return (iova >> vtd_slpt_level_shift(level)) & 722 ((1ULL << VTD_SL_LEVEL_BITS) - 1); 723 } 724 725 /* Check Capability Register to see if the @level of page-table is supported */ 726 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 727 { 728 return VTD_CAP_SAGAW_MASK & s->cap & 729 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 730 } 731 732 /* Return true if check passed, otherwise false */ 733 static inline bool vtd_pe_type_check(X86IOMMUState *x86_iommu, 734 VTDPASIDEntry *pe) 735 { 736 switch (VTD_PE_GET_TYPE(pe)) { 737 case VTD_SM_PASID_ENTRY_FLT: 738 case VTD_SM_PASID_ENTRY_SLT: 739 case VTD_SM_PASID_ENTRY_NESTED: 740 break; 741 case VTD_SM_PASID_ENTRY_PT: 742 if (!x86_iommu->pt_supported) { 743 return false; 744 } 745 break; 746 default: 747 /* Unknown type */ 748 return false; 749 } 750 return true; 751 } 752 753 static inline bool vtd_pdire_present(VTDPASIDDirEntry *pdire) 754 { 755 return pdire->val & 1; 756 } 757 758 /** 759 * Caller of this function should check present bit if wants 760 * to use pdir entry for further usage except for fpd bit check. 761 */ 762 static int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, 763 uint32_t pasid, 764 VTDPASIDDirEntry *pdire) 765 { 766 uint32_t index; 767 dma_addr_t addr, entry_size; 768 769 index = VTD_PASID_DIR_INDEX(pasid); 770 entry_size = VTD_PASID_DIR_ENTRY_SIZE; 771 addr = pasid_dir_base + index * entry_size; 772 if (dma_memory_read(&address_space_memory, addr, 773 pdire, entry_size, MEMTXATTRS_UNSPECIFIED)) { 774 return -VTD_FR_PASID_TABLE_INV; 775 } 776 777 pdire->val = le64_to_cpu(pdire->val); 778 779 return 0; 780 } 781 782 static inline bool vtd_pe_present(VTDPASIDEntry *pe) 783 { 784 return pe->val[0] & VTD_PASID_ENTRY_P; 785 } 786 787 static int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s, 788 uint32_t pasid, 789 dma_addr_t addr, 790 VTDPASIDEntry *pe) 791 { 792 uint32_t index; 793 dma_addr_t entry_size; 794 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 795 796 index = VTD_PASID_TABLE_INDEX(pasid); 797 entry_size = VTD_PASID_ENTRY_SIZE; 798 addr = addr + index * entry_size; 799 if (dma_memory_read(&address_space_memory, addr, 800 pe, entry_size, MEMTXATTRS_UNSPECIFIED)) { 801 return -VTD_FR_PASID_TABLE_INV; 802 } 803 for (size_t i = 0; i < ARRAY_SIZE(pe->val); i++) { 804 pe->val[i] = le64_to_cpu(pe->val[i]); 805 } 806 807 /* Do translation type check */ 808 if (!vtd_pe_type_check(x86_iommu, pe)) { 809 return -VTD_FR_PASID_TABLE_INV; 810 } 811 812 if (!vtd_is_level_supported(s, VTD_PE_GET_LEVEL(pe))) { 813 return -VTD_FR_PASID_TABLE_INV; 814 } 815 816 return 0; 817 } 818 819 /** 820 * Caller of this function should check present bit if wants 821 * to use pasid entry for further usage except for fpd bit check. 822 */ 823 static int vtd_get_pe_from_pdire(IntelIOMMUState *s, 824 uint32_t pasid, 825 VTDPASIDDirEntry *pdire, 826 VTDPASIDEntry *pe) 827 { 828 dma_addr_t addr = pdire->val & VTD_PASID_TABLE_BASE_ADDR_MASK; 829 830 return vtd_get_pe_in_pasid_leaf_table(s, pasid, addr, pe); 831 } 832 833 /** 834 * This function gets a pasid entry from a specified pasid 835 * table (includes dir and leaf table) with a specified pasid. 836 * Sanity check should be done to ensure return a present 837 * pasid entry to caller. 838 */ 839 static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s, 840 dma_addr_t pasid_dir_base, 841 uint32_t pasid, 842 VTDPASIDEntry *pe) 843 { 844 int ret; 845 VTDPASIDDirEntry pdire; 846 847 ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, 848 pasid, &pdire); 849 if (ret) { 850 return ret; 851 } 852 853 if (!vtd_pdire_present(&pdire)) { 854 return -VTD_FR_PASID_TABLE_INV; 855 } 856 857 ret = vtd_get_pe_from_pdire(s, pasid, &pdire, pe); 858 if (ret) { 859 return ret; 860 } 861 862 if (!vtd_pe_present(pe)) { 863 return -VTD_FR_PASID_TABLE_INV; 864 } 865 866 return 0; 867 } 868 869 static int vtd_ce_get_rid2pasid_entry(IntelIOMMUState *s, 870 VTDContextEntry *ce, 871 VTDPASIDEntry *pe, 872 uint32_t pasid) 873 { 874 dma_addr_t pasid_dir_base; 875 int ret = 0; 876 877 if (pasid == PCI_NO_PASID) { 878 pasid = VTD_CE_GET_RID2PASID(ce); 879 } 880 pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); 881 ret = vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe); 882 883 return ret; 884 } 885 886 static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s, 887 VTDContextEntry *ce, 888 bool *pe_fpd_set, 889 uint32_t pasid) 890 { 891 int ret; 892 dma_addr_t pasid_dir_base; 893 VTDPASIDDirEntry pdire; 894 VTDPASIDEntry pe; 895 896 if (pasid == PCI_NO_PASID) { 897 pasid = VTD_CE_GET_RID2PASID(ce); 898 } 899 pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); 900 901 /* 902 * No present bit check since fpd is meaningful even 903 * if the present bit is clear. 904 */ 905 ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, pasid, &pdire); 906 if (ret) { 907 return ret; 908 } 909 910 if (pdire.val & VTD_PASID_DIR_FPD) { 911 *pe_fpd_set = true; 912 return 0; 913 } 914 915 if (!vtd_pdire_present(&pdire)) { 916 return -VTD_FR_PASID_TABLE_INV; 917 } 918 919 /* 920 * No present bit check since fpd is meaningful even 921 * if the present bit is clear. 922 */ 923 ret = vtd_get_pe_from_pdire(s, pasid, &pdire, &pe); 924 if (ret) { 925 return ret; 926 } 927 928 if (pe.val[0] & VTD_PASID_ENTRY_FPD) { 929 *pe_fpd_set = true; 930 } 931 932 return 0; 933 } 934 935 /* Get the page-table level that hardware should use for the second-level 936 * page-table walk from the Address Width field of context-entry. 937 */ 938 static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce) 939 { 940 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 941 } 942 943 static uint32_t vtd_get_iova_level(IntelIOMMUState *s, 944 VTDContextEntry *ce, 945 uint32_t pasid) 946 { 947 VTDPASIDEntry pe; 948 949 if (s->root_scalable) { 950 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 951 return VTD_PE_GET_LEVEL(&pe); 952 } 953 954 return vtd_ce_get_level(ce); 955 } 956 957 static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce) 958 { 959 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 960 } 961 962 static uint32_t vtd_get_iova_agaw(IntelIOMMUState *s, 963 VTDContextEntry *ce, 964 uint32_t pasid) 965 { 966 VTDPASIDEntry pe; 967 968 if (s->root_scalable) { 969 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 970 return 30 + ((pe.val[0] >> 2) & VTD_SM_PASID_ENTRY_AW) * 9; 971 } 972 973 return vtd_ce_get_agaw(ce); 974 } 975 976 static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce) 977 { 978 return ce->lo & VTD_CONTEXT_ENTRY_TT; 979 } 980 981 /* Only for Legacy Mode. Return true if check passed, otherwise false */ 982 static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu, 983 VTDContextEntry *ce) 984 { 985 switch (vtd_ce_get_type(ce)) { 986 case VTD_CONTEXT_TT_MULTI_LEVEL: 987 /* Always supported */ 988 break; 989 case VTD_CONTEXT_TT_DEV_IOTLB: 990 if (!x86_iommu->dt_supported) { 991 error_report_once("%s: DT specified but not supported", __func__); 992 return false; 993 } 994 break; 995 case VTD_CONTEXT_TT_PASS_THROUGH: 996 if (!x86_iommu->pt_supported) { 997 error_report_once("%s: PT specified but not supported", __func__); 998 return false; 999 } 1000 break; 1001 default: 1002 /* Unknown type */ 1003 error_report_once("%s: unknown ce type: %"PRIu32, __func__, 1004 vtd_ce_get_type(ce)); 1005 return false; 1006 } 1007 return true; 1008 } 1009 1010 static inline uint64_t vtd_iova_limit(IntelIOMMUState *s, 1011 VTDContextEntry *ce, uint8_t aw, 1012 uint32_t pasid) 1013 { 1014 uint32_t ce_agaw = vtd_get_iova_agaw(s, ce, pasid); 1015 return 1ULL << MIN(ce_agaw, aw); 1016 } 1017 1018 /* Return true if IOVA passes range check, otherwise false. */ 1019 static inline bool vtd_iova_range_check(IntelIOMMUState *s, 1020 uint64_t iova, VTDContextEntry *ce, 1021 uint8_t aw, uint32_t pasid) 1022 { 1023 /* 1024 * Check if @iova is above 2^X-1, where X is the minimum of MGAW 1025 * in CAP_REG and AW in context-entry. 1026 */ 1027 return !(iova & ~(vtd_iova_limit(s, ce, aw, pasid) - 1)); 1028 } 1029 1030 static dma_addr_t vtd_get_iova_pgtbl_base(IntelIOMMUState *s, 1031 VTDContextEntry *ce, 1032 uint32_t pasid) 1033 { 1034 VTDPASIDEntry pe; 1035 1036 if (s->root_scalable) { 1037 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1038 return pe.val[0] & VTD_SM_PASID_ENTRY_SLPTPTR; 1039 } 1040 1041 return vtd_ce_get_slpt_base(ce); 1042 } 1043 1044 /* 1045 * Rsvd field masks for spte: 1046 * vtd_spte_rsvd 4k pages 1047 * vtd_spte_rsvd_large large pages 1048 */ 1049 static uint64_t vtd_spte_rsvd[5]; 1050 static uint64_t vtd_spte_rsvd_large[5]; 1051 1052 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 1053 { 1054 uint64_t rsvd_mask = vtd_spte_rsvd[level]; 1055 1056 if ((level == VTD_SL_PD_LEVEL || level == VTD_SL_PDP_LEVEL) && 1057 (slpte & VTD_SL_PT_PAGE_SIZE_MASK)) { 1058 /* large page */ 1059 rsvd_mask = vtd_spte_rsvd_large[level]; 1060 } 1061 1062 return slpte & rsvd_mask; 1063 } 1064 1065 /* Given the @iova, get relevant @slptep. @slpte_level will be the last level 1066 * of the translation, can be used for deciding the size of large page. 1067 */ 1068 static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce, 1069 uint64_t iova, bool is_write, 1070 uint64_t *slptep, uint32_t *slpte_level, 1071 bool *reads, bool *writes, uint8_t aw_bits, 1072 uint32_t pasid) 1073 { 1074 dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); 1075 uint32_t level = vtd_get_iova_level(s, ce, pasid); 1076 uint32_t offset; 1077 uint64_t slpte; 1078 uint64_t access_right_check; 1079 uint64_t xlat, size; 1080 1081 if (!vtd_iova_range_check(s, iova, ce, aw_bits, pasid)) { 1082 error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 "," 1083 "pasid=0x%" PRIx32 ")", __func__, iova, pasid); 1084 return -VTD_FR_ADDR_BEYOND_MGAW; 1085 } 1086 1087 /* FIXME: what is the Atomics request here? */ 1088 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 1089 1090 while (true) { 1091 offset = vtd_iova_level_offset(iova, level); 1092 slpte = vtd_get_slpte(addr, offset); 1093 1094 if (slpte == (uint64_t)-1) { 1095 error_report_once("%s: detected read error on DMAR slpte " 1096 "(iova=0x%" PRIx64 ", pasid=0x%" PRIx32 ")", 1097 __func__, iova, pasid); 1098 if (level == vtd_get_iova_level(s, ce, pasid)) { 1099 /* Invalid programming of context-entry */ 1100 return -VTD_FR_CONTEXT_ENTRY_INV; 1101 } else { 1102 return -VTD_FR_PAGING_ENTRY_INV; 1103 } 1104 } 1105 *reads = (*reads) && (slpte & VTD_SL_R); 1106 *writes = (*writes) && (slpte & VTD_SL_W); 1107 if (!(slpte & access_right_check)) { 1108 error_report_once("%s: detected slpte permission error " 1109 "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " 1110 "slpte=0x%" PRIx64 ", write=%d, pasid=0x%" 1111 PRIx32 ")", __func__, iova, level, 1112 slpte, is_write, pasid); 1113 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 1114 } 1115 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 1116 error_report_once("%s: detected splte reserve non-zero " 1117 "iova=0x%" PRIx64 ", level=0x%" PRIx32 1118 "slpte=0x%" PRIx64 ", pasid=0x%" PRIX32 ")", 1119 __func__, iova, level, slpte, pasid); 1120 return -VTD_FR_PAGING_ENTRY_RSVD; 1121 } 1122 1123 if (vtd_is_last_slpte(slpte, level)) { 1124 *slptep = slpte; 1125 *slpte_level = level; 1126 break; 1127 } 1128 addr = vtd_get_slpte_addr(slpte, aw_bits); 1129 level--; 1130 } 1131 1132 xlat = vtd_get_slpte_addr(*slptep, aw_bits); 1133 size = ~vtd_slpt_level_page_mask(level) + 1; 1134 1135 /* 1136 * From VT-d spec 3.14: Untranslated requests and translation 1137 * requests that result in an address in the interrupt range will be 1138 * blocked with condition code LGN.4 or SGN.8. 1139 */ 1140 if ((xlat > VTD_INTERRUPT_ADDR_LAST || 1141 xlat + size - 1 < VTD_INTERRUPT_ADDR_FIRST)) { 1142 return 0; 1143 } else { 1144 error_report_once("%s: xlat address is in interrupt range " 1145 "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " 1146 "slpte=0x%" PRIx64 ", write=%d, " 1147 "xlat=0x%" PRIx64 ", size=0x%" PRIx64 ", " 1148 "pasid=0x%" PRIx32 ")", 1149 __func__, iova, level, slpte, is_write, 1150 xlat, size, pasid); 1151 return s->scalable_mode ? -VTD_FR_SM_INTERRUPT_ADDR : 1152 -VTD_FR_INTERRUPT_ADDR; 1153 } 1154 } 1155 1156 typedef int (*vtd_page_walk_hook)(IOMMUTLBEvent *event, void *private); 1157 1158 /** 1159 * Constant information used during page walking 1160 * 1161 * @hook_fn: hook func to be called when detected page 1162 * @private: private data to be passed into hook func 1163 * @notify_unmap: whether we should notify invalid entries 1164 * @as: VT-d address space of the device 1165 * @aw: maximum address width 1166 * @domain: domain ID of the page walk 1167 */ 1168 typedef struct { 1169 VTDAddressSpace *as; 1170 vtd_page_walk_hook hook_fn; 1171 void *private; 1172 bool notify_unmap; 1173 uint8_t aw; 1174 uint16_t domain_id; 1175 } vtd_page_walk_info; 1176 1177 static int vtd_page_walk_one(IOMMUTLBEvent *event, vtd_page_walk_info *info) 1178 { 1179 VTDAddressSpace *as = info->as; 1180 vtd_page_walk_hook hook_fn = info->hook_fn; 1181 void *private = info->private; 1182 IOMMUTLBEntry *entry = &event->entry; 1183 DMAMap target = { 1184 .iova = entry->iova, 1185 .size = entry->addr_mask, 1186 .translated_addr = entry->translated_addr, 1187 .perm = entry->perm, 1188 }; 1189 const DMAMap *mapped = iova_tree_find(as->iova_tree, &target); 1190 1191 if (event->type == IOMMU_NOTIFIER_UNMAP && !info->notify_unmap) { 1192 trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 1193 return 0; 1194 } 1195 1196 assert(hook_fn); 1197 1198 /* Update local IOVA mapped ranges */ 1199 if (event->type == IOMMU_NOTIFIER_MAP) { 1200 if (mapped) { 1201 /* If it's exactly the same translation, skip */ 1202 if (!memcmp(mapped, &target, sizeof(target))) { 1203 trace_vtd_page_walk_one_skip_map(entry->iova, entry->addr_mask, 1204 entry->translated_addr); 1205 return 0; 1206 } else { 1207 /* 1208 * Translation changed. Normally this should not 1209 * happen, but it can happen when with buggy guest 1210 * OSes. Note that there will be a small window that 1211 * we don't have map at all. But that's the best 1212 * effort we can do. The ideal way to emulate this is 1213 * atomically modify the PTE to follow what has 1214 * changed, but we can't. One example is that vfio 1215 * driver only has VFIO_IOMMU_[UN]MAP_DMA but no 1216 * interface to modify a mapping (meanwhile it seems 1217 * meaningless to even provide one). Anyway, let's 1218 * mark this as a TODO in case one day we'll have 1219 * a better solution. 1220 */ 1221 IOMMUAccessFlags cache_perm = entry->perm; 1222 int ret; 1223 1224 /* Emulate an UNMAP */ 1225 event->type = IOMMU_NOTIFIER_UNMAP; 1226 entry->perm = IOMMU_NONE; 1227 trace_vtd_page_walk_one(info->domain_id, 1228 entry->iova, 1229 entry->translated_addr, 1230 entry->addr_mask, 1231 entry->perm); 1232 ret = hook_fn(event, private); 1233 if (ret) { 1234 return ret; 1235 } 1236 /* Drop any existing mapping */ 1237 iova_tree_remove(as->iova_tree, target); 1238 /* Recover the correct type */ 1239 event->type = IOMMU_NOTIFIER_MAP; 1240 entry->perm = cache_perm; 1241 } 1242 } 1243 iova_tree_insert(as->iova_tree, &target); 1244 } else { 1245 if (!mapped) { 1246 /* Skip since we didn't map this range at all */ 1247 trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 1248 return 0; 1249 } 1250 iova_tree_remove(as->iova_tree, target); 1251 } 1252 1253 trace_vtd_page_walk_one(info->domain_id, entry->iova, 1254 entry->translated_addr, entry->addr_mask, 1255 entry->perm); 1256 return hook_fn(event, private); 1257 } 1258 1259 /** 1260 * vtd_page_walk_level - walk over specific level for IOVA range 1261 * 1262 * @addr: base GPA addr to start the walk 1263 * @start: IOVA range start address 1264 * @end: IOVA range end address (start <= addr < end) 1265 * @read: whether parent level has read permission 1266 * @write: whether parent level has write permission 1267 * @info: constant information for the page walk 1268 */ 1269 static int vtd_page_walk_level(dma_addr_t addr, uint64_t start, 1270 uint64_t end, uint32_t level, bool read, 1271 bool write, vtd_page_walk_info *info) 1272 { 1273 bool read_cur, write_cur, entry_valid; 1274 uint32_t offset; 1275 uint64_t slpte; 1276 uint64_t subpage_size, subpage_mask; 1277 IOMMUTLBEvent event; 1278 uint64_t iova = start; 1279 uint64_t iova_next; 1280 int ret = 0; 1281 1282 trace_vtd_page_walk_level(addr, level, start, end); 1283 1284 subpage_size = 1ULL << vtd_slpt_level_shift(level); 1285 subpage_mask = vtd_slpt_level_page_mask(level); 1286 1287 while (iova < end) { 1288 iova_next = (iova & subpage_mask) + subpage_size; 1289 1290 offset = vtd_iova_level_offset(iova, level); 1291 slpte = vtd_get_slpte(addr, offset); 1292 1293 if (slpte == (uint64_t)-1) { 1294 trace_vtd_page_walk_skip_read(iova, iova_next); 1295 goto next; 1296 } 1297 1298 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 1299 trace_vtd_page_walk_skip_reserve(iova, iova_next); 1300 goto next; 1301 } 1302 1303 /* Permissions are stacked with parents' */ 1304 read_cur = read && (slpte & VTD_SL_R); 1305 write_cur = write && (slpte & VTD_SL_W); 1306 1307 /* 1308 * As long as we have either read/write permission, this is a 1309 * valid entry. The rule works for both page entries and page 1310 * table entries. 1311 */ 1312 entry_valid = read_cur | write_cur; 1313 1314 if (!vtd_is_last_slpte(slpte, level) && entry_valid) { 1315 /* 1316 * This is a valid PDE (or even bigger than PDE). We need 1317 * to walk one further level. 1318 */ 1319 ret = vtd_page_walk_level(vtd_get_slpte_addr(slpte, info->aw), 1320 iova, MIN(iova_next, end), level - 1, 1321 read_cur, write_cur, info); 1322 } else { 1323 /* 1324 * This means we are either: 1325 * 1326 * (1) the real page entry (either 4K page, or huge page) 1327 * (2) the whole range is invalid 1328 * 1329 * In either case, we send an IOTLB notification down. 1330 */ 1331 event.entry.target_as = &address_space_memory; 1332 event.entry.iova = iova & subpage_mask; 1333 event.entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur); 1334 event.entry.addr_mask = ~subpage_mask; 1335 /* NOTE: this is only meaningful if entry_valid == true */ 1336 event.entry.translated_addr = vtd_get_slpte_addr(slpte, info->aw); 1337 event.type = event.entry.perm ? IOMMU_NOTIFIER_MAP : 1338 IOMMU_NOTIFIER_UNMAP; 1339 ret = vtd_page_walk_one(&event, info); 1340 } 1341 1342 if (ret < 0) { 1343 return ret; 1344 } 1345 1346 next: 1347 iova = iova_next; 1348 } 1349 1350 return 0; 1351 } 1352 1353 /** 1354 * vtd_page_walk - walk specific IOVA range, and call the hook 1355 * 1356 * @s: intel iommu state 1357 * @ce: context entry to walk upon 1358 * @start: IOVA address to start the walk 1359 * @end: IOVA range end address (start <= addr < end) 1360 * @info: page walking information struct 1361 */ 1362 static int vtd_page_walk(IntelIOMMUState *s, VTDContextEntry *ce, 1363 uint64_t start, uint64_t end, 1364 vtd_page_walk_info *info, 1365 uint32_t pasid) 1366 { 1367 dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); 1368 uint32_t level = vtd_get_iova_level(s, ce, pasid); 1369 1370 if (!vtd_iova_range_check(s, start, ce, info->aw, pasid)) { 1371 return -VTD_FR_ADDR_BEYOND_MGAW; 1372 } 1373 1374 if (!vtd_iova_range_check(s, end, ce, info->aw, pasid)) { 1375 /* Fix end so that it reaches the maximum */ 1376 end = vtd_iova_limit(s, ce, info->aw, pasid); 1377 } 1378 1379 return vtd_page_walk_level(addr, start, end, level, true, true, info); 1380 } 1381 1382 static int vtd_root_entry_rsvd_bits_check(IntelIOMMUState *s, 1383 VTDRootEntry *re) 1384 { 1385 /* Legacy Mode reserved bits check */ 1386 if (!s->root_scalable && 1387 (re->hi || (re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)))) 1388 goto rsvd_err; 1389 1390 /* Scalable Mode reserved bits check */ 1391 if (s->root_scalable && 1392 ((re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)) || 1393 (re->hi & VTD_ROOT_ENTRY_RSVD(s->aw_bits)))) 1394 goto rsvd_err; 1395 1396 return 0; 1397 1398 rsvd_err: 1399 error_report_once("%s: invalid root entry: hi=0x%"PRIx64 1400 ", lo=0x%"PRIx64, 1401 __func__, re->hi, re->lo); 1402 return -VTD_FR_ROOT_ENTRY_RSVD; 1403 } 1404 1405 static inline int vtd_context_entry_rsvd_bits_check(IntelIOMMUState *s, 1406 VTDContextEntry *ce) 1407 { 1408 if (!s->root_scalable && 1409 (ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI || 1410 ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) { 1411 error_report_once("%s: invalid context entry: hi=%"PRIx64 1412 ", lo=%"PRIx64" (reserved nonzero)", 1413 __func__, ce->hi, ce->lo); 1414 return -VTD_FR_CONTEXT_ENTRY_RSVD; 1415 } 1416 1417 if (s->root_scalable && 1418 (ce->val[0] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL0(s->aw_bits) || 1419 ce->val[1] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL1 || 1420 ce->val[2] || 1421 ce->val[3])) { 1422 error_report_once("%s: invalid context entry: val[3]=%"PRIx64 1423 ", val[2]=%"PRIx64 1424 ", val[1]=%"PRIx64 1425 ", val[0]=%"PRIx64" (reserved nonzero)", 1426 __func__, ce->val[3], ce->val[2], 1427 ce->val[1], ce->val[0]); 1428 return -VTD_FR_CONTEXT_ENTRY_RSVD; 1429 } 1430 1431 return 0; 1432 } 1433 1434 static int vtd_ce_rid2pasid_check(IntelIOMMUState *s, 1435 VTDContextEntry *ce) 1436 { 1437 VTDPASIDEntry pe; 1438 1439 /* 1440 * Make sure in Scalable Mode, a present context entry 1441 * has valid rid2pasid setting, which includes valid 1442 * rid2pasid field and corresponding pasid entry setting 1443 */ 1444 return vtd_ce_get_rid2pasid_entry(s, ce, &pe, PCI_NO_PASID); 1445 } 1446 1447 /* Map a device to its corresponding domain (context-entry) */ 1448 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 1449 uint8_t devfn, VTDContextEntry *ce) 1450 { 1451 VTDRootEntry re; 1452 int ret_fr; 1453 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 1454 1455 ret_fr = vtd_get_root_entry(s, bus_num, &re); 1456 if (ret_fr) { 1457 return ret_fr; 1458 } 1459 1460 if (!vtd_root_entry_present(s, &re, devfn)) { 1461 /* Not error - it's okay we don't have root entry. */ 1462 trace_vtd_re_not_present(bus_num); 1463 return -VTD_FR_ROOT_ENTRY_P; 1464 } 1465 1466 ret_fr = vtd_root_entry_rsvd_bits_check(s, &re); 1467 if (ret_fr) { 1468 return ret_fr; 1469 } 1470 1471 ret_fr = vtd_get_context_entry_from_root(s, &re, devfn, ce); 1472 if (ret_fr) { 1473 return ret_fr; 1474 } 1475 1476 if (!vtd_ce_present(ce)) { 1477 /* Not error - it's okay we don't have context entry. */ 1478 trace_vtd_ce_not_present(bus_num, devfn); 1479 return -VTD_FR_CONTEXT_ENTRY_P; 1480 } 1481 1482 ret_fr = vtd_context_entry_rsvd_bits_check(s, ce); 1483 if (ret_fr) { 1484 return ret_fr; 1485 } 1486 1487 /* Check if the programming of context-entry is valid */ 1488 if (!s->root_scalable && 1489 !vtd_is_level_supported(s, vtd_ce_get_level(ce))) { 1490 error_report_once("%s: invalid context entry: hi=%"PRIx64 1491 ", lo=%"PRIx64" (level %d not supported)", 1492 __func__, ce->hi, ce->lo, 1493 vtd_ce_get_level(ce)); 1494 return -VTD_FR_CONTEXT_ENTRY_INV; 1495 } 1496 1497 if (!s->root_scalable) { 1498 /* Do translation type check */ 1499 if (!vtd_ce_type_check(x86_iommu, ce)) { 1500 /* Errors dumped in vtd_ce_type_check() */ 1501 return -VTD_FR_CONTEXT_ENTRY_INV; 1502 } 1503 } else { 1504 /* 1505 * Check if the programming of context-entry.rid2pasid 1506 * and corresponding pasid setting is valid, and thus 1507 * avoids to check pasid entry fetching result in future 1508 * helper function calling. 1509 */ 1510 ret_fr = vtd_ce_rid2pasid_check(s, ce); 1511 if (ret_fr) { 1512 return ret_fr; 1513 } 1514 } 1515 1516 return 0; 1517 } 1518 1519 static int vtd_sync_shadow_page_hook(IOMMUTLBEvent *event, 1520 void *private) 1521 { 1522 memory_region_notify_iommu(private, 0, *event); 1523 return 0; 1524 } 1525 1526 static uint16_t vtd_get_domain_id(IntelIOMMUState *s, 1527 VTDContextEntry *ce, 1528 uint32_t pasid) 1529 { 1530 VTDPASIDEntry pe; 1531 1532 if (s->root_scalable) { 1533 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1534 return VTD_SM_PASID_ENTRY_DID(pe.val[1]); 1535 } 1536 1537 return VTD_CONTEXT_ENTRY_DID(ce->hi); 1538 } 1539 1540 static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as, 1541 VTDContextEntry *ce, 1542 hwaddr addr, hwaddr size) 1543 { 1544 IntelIOMMUState *s = vtd_as->iommu_state; 1545 vtd_page_walk_info info = { 1546 .hook_fn = vtd_sync_shadow_page_hook, 1547 .private = (void *)&vtd_as->iommu, 1548 .notify_unmap = true, 1549 .aw = s->aw_bits, 1550 .as = vtd_as, 1551 .domain_id = vtd_get_domain_id(s, ce, vtd_as->pasid), 1552 }; 1553 1554 return vtd_page_walk(s, ce, addr, addr + size, &info, vtd_as->pasid); 1555 } 1556 1557 static int vtd_address_space_sync(VTDAddressSpace *vtd_as) 1558 { 1559 int ret; 1560 VTDContextEntry ce; 1561 IOMMUNotifier *n; 1562 1563 /* If no MAP notifier registered, we simply invalidate all the cache */ 1564 if (!vtd_as_has_map_notifier(vtd_as)) { 1565 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 1566 memory_region_unmap_iommu_notifier_range(n); 1567 } 1568 return 0; 1569 } 1570 1571 ret = vtd_dev_to_context_entry(vtd_as->iommu_state, 1572 pci_bus_num(vtd_as->bus), 1573 vtd_as->devfn, &ce); 1574 if (ret) { 1575 if (ret == -VTD_FR_CONTEXT_ENTRY_P) { 1576 /* 1577 * It's a valid scenario to have a context entry that is 1578 * not present. For example, when a device is removed 1579 * from an existing domain then the context entry will be 1580 * zeroed by the guest before it was put into another 1581 * domain. When this happens, instead of synchronizing 1582 * the shadow pages we should invalidate all existing 1583 * mappings and notify the backends. 1584 */ 1585 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 1586 vtd_address_space_unmap(vtd_as, n); 1587 } 1588 ret = 0; 1589 } 1590 return ret; 1591 } 1592 1593 return vtd_sync_shadow_page_table_range(vtd_as, &ce, 0, UINT64_MAX); 1594 } 1595 1596 /* 1597 * Check if specific device is configured to bypass address 1598 * translation for DMA requests. In Scalable Mode, bypass 1599 * 1st-level translation or 2nd-level translation, it depends 1600 * on PGTT setting. 1601 */ 1602 static bool vtd_dev_pt_enabled(IntelIOMMUState *s, VTDContextEntry *ce, 1603 uint32_t pasid) 1604 { 1605 VTDPASIDEntry pe; 1606 int ret; 1607 1608 if (s->root_scalable) { 1609 ret = vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1610 if (ret) { 1611 /* 1612 * This error is guest triggerable. We should assumt PT 1613 * not enabled for safety. 1614 */ 1615 return false; 1616 } 1617 return (VTD_PE_GET_TYPE(&pe) == VTD_SM_PASID_ENTRY_PT); 1618 } 1619 1620 return (vtd_ce_get_type(ce) == VTD_CONTEXT_TT_PASS_THROUGH); 1621 1622 } 1623 1624 static bool vtd_as_pt_enabled(VTDAddressSpace *as) 1625 { 1626 IntelIOMMUState *s; 1627 VTDContextEntry ce; 1628 1629 assert(as); 1630 1631 s = as->iommu_state; 1632 if (vtd_dev_to_context_entry(s, pci_bus_num(as->bus), as->devfn, 1633 &ce)) { 1634 /* 1635 * Possibly failed to parse the context entry for some reason 1636 * (e.g., during init, or any guest configuration errors on 1637 * context entries). We should assume PT not enabled for 1638 * safety. 1639 */ 1640 return false; 1641 } 1642 1643 return vtd_dev_pt_enabled(s, &ce, as->pasid); 1644 } 1645 1646 /* Return whether the device is using IOMMU translation. */ 1647 static bool vtd_switch_address_space(VTDAddressSpace *as) 1648 { 1649 bool use_iommu, pt; 1650 /* Whether we need to take the BQL on our own */ 1651 bool take_bql = !qemu_mutex_iothread_locked(); 1652 1653 assert(as); 1654 1655 use_iommu = as->iommu_state->dmar_enabled && !vtd_as_pt_enabled(as); 1656 pt = as->iommu_state->dmar_enabled && vtd_as_pt_enabled(as); 1657 1658 trace_vtd_switch_address_space(pci_bus_num(as->bus), 1659 VTD_PCI_SLOT(as->devfn), 1660 VTD_PCI_FUNC(as->devfn), 1661 use_iommu); 1662 1663 /* 1664 * It's possible that we reach here without BQL, e.g., when called 1665 * from vtd_pt_enable_fast_path(). However the memory APIs need 1666 * it. We'd better make sure we have had it already, or, take it. 1667 */ 1668 if (take_bql) { 1669 qemu_mutex_lock_iothread(); 1670 } 1671 1672 /* Turn off first then on the other */ 1673 if (use_iommu) { 1674 memory_region_set_enabled(&as->nodmar, false); 1675 memory_region_set_enabled(MEMORY_REGION(&as->iommu), true); 1676 /* 1677 * vt-d spec v3.4 3.14: 1678 * 1679 * """ 1680 * Requests-with-PASID with input address in range 0xFEEx_xxxx 1681 * are translated normally like any other request-with-PASID 1682 * through DMA-remapping hardware. 1683 * """ 1684 * 1685 * Need to disable ir for as with PASID. 1686 */ 1687 if (as->pasid != PCI_NO_PASID) { 1688 memory_region_set_enabled(&as->iommu_ir, false); 1689 } else { 1690 memory_region_set_enabled(&as->iommu_ir, true); 1691 } 1692 } else { 1693 memory_region_set_enabled(MEMORY_REGION(&as->iommu), false); 1694 memory_region_set_enabled(&as->nodmar, true); 1695 } 1696 1697 /* 1698 * vtd-spec v3.4 3.14: 1699 * 1700 * """ 1701 * Requests-with-PASID with input address in range 0xFEEx_xxxx are 1702 * translated normally like any other request-with-PASID through 1703 * DMA-remapping hardware. However, if such a request is processed 1704 * using pass-through translation, it will be blocked as described 1705 * in the paragraph below. 1706 * 1707 * Software must not program paging-structure entries to remap any 1708 * address to the interrupt address range. Untranslated requests 1709 * and translation requests that result in an address in the 1710 * interrupt range will be blocked with condition code LGN.4 or 1711 * SGN.8. 1712 * """ 1713 * 1714 * We enable per as memory region (iommu_ir_fault) for catching 1715 * the translation for interrupt range through PASID + PT. 1716 */ 1717 if (pt && as->pasid != PCI_NO_PASID) { 1718 memory_region_set_enabled(&as->iommu_ir_fault, true); 1719 } else { 1720 memory_region_set_enabled(&as->iommu_ir_fault, false); 1721 } 1722 1723 if (take_bql) { 1724 qemu_mutex_unlock_iothread(); 1725 } 1726 1727 return use_iommu; 1728 } 1729 1730 static void vtd_switch_address_space_all(IntelIOMMUState *s) 1731 { 1732 VTDAddressSpace *vtd_as; 1733 GHashTableIter iter; 1734 1735 g_hash_table_iter_init(&iter, s->vtd_address_spaces); 1736 while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_as)) { 1737 vtd_switch_address_space(vtd_as); 1738 } 1739 } 1740 1741 static const bool vtd_qualified_faults[] = { 1742 [VTD_FR_RESERVED] = false, 1743 [VTD_FR_ROOT_ENTRY_P] = false, 1744 [VTD_FR_CONTEXT_ENTRY_P] = true, 1745 [VTD_FR_CONTEXT_ENTRY_INV] = true, 1746 [VTD_FR_ADDR_BEYOND_MGAW] = true, 1747 [VTD_FR_WRITE] = true, 1748 [VTD_FR_READ] = true, 1749 [VTD_FR_PAGING_ENTRY_INV] = true, 1750 [VTD_FR_ROOT_TABLE_INV] = false, 1751 [VTD_FR_CONTEXT_TABLE_INV] = false, 1752 [VTD_FR_INTERRUPT_ADDR] = true, 1753 [VTD_FR_ROOT_ENTRY_RSVD] = false, 1754 [VTD_FR_PAGING_ENTRY_RSVD] = true, 1755 [VTD_FR_CONTEXT_ENTRY_TT] = true, 1756 [VTD_FR_PASID_TABLE_INV] = false, 1757 [VTD_FR_SM_INTERRUPT_ADDR] = true, 1758 [VTD_FR_MAX] = false, 1759 }; 1760 1761 /* To see if a fault condition is "qualified", which is reported to software 1762 * only if the FPD field in the context-entry used to process the faulting 1763 * request is 0. 1764 */ 1765 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 1766 { 1767 return vtd_qualified_faults[fault]; 1768 } 1769 1770 static inline bool vtd_is_interrupt_addr(hwaddr addr) 1771 { 1772 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 1773 } 1774 1775 static gboolean vtd_find_as_by_sid(gpointer key, gpointer value, 1776 gpointer user_data) 1777 { 1778 struct vtd_as_key *as_key = (struct vtd_as_key *)key; 1779 uint16_t target_sid = *(uint16_t *)user_data; 1780 uint16_t sid = PCI_BUILD_BDF(pci_bus_num(as_key->bus), as_key->devfn); 1781 return sid == target_sid; 1782 } 1783 1784 static VTDAddressSpace *vtd_get_as_by_sid(IntelIOMMUState *s, uint16_t sid) 1785 { 1786 uint8_t bus_num = PCI_BUS_NUM(sid); 1787 VTDAddressSpace *vtd_as = s->vtd_as_cache[bus_num]; 1788 1789 if (vtd_as && 1790 (sid == PCI_BUILD_BDF(pci_bus_num(vtd_as->bus), vtd_as->devfn))) { 1791 return vtd_as; 1792 } 1793 1794 vtd_as = g_hash_table_find(s->vtd_address_spaces, vtd_find_as_by_sid, &sid); 1795 s->vtd_as_cache[bus_num] = vtd_as; 1796 1797 return vtd_as; 1798 } 1799 1800 static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id) 1801 { 1802 VTDAddressSpace *vtd_as; 1803 bool success = false; 1804 1805 vtd_as = vtd_get_as_by_sid(s, source_id); 1806 if (!vtd_as) { 1807 goto out; 1808 } 1809 1810 if (vtd_switch_address_space(vtd_as) == false) { 1811 /* We switched off IOMMU region successfully. */ 1812 success = true; 1813 } 1814 1815 out: 1816 trace_vtd_pt_enable_fast_path(source_id, success); 1817 } 1818 1819 static void vtd_report_fault(IntelIOMMUState *s, 1820 int err, bool is_fpd_set, 1821 uint16_t source_id, 1822 hwaddr addr, 1823 bool is_write, 1824 bool is_pasid, 1825 uint32_t pasid) 1826 { 1827 if (is_fpd_set && vtd_is_qualified_fault(err)) { 1828 trace_vtd_fault_disabled(); 1829 } else { 1830 vtd_report_dmar_fault(s, source_id, addr, err, is_write, 1831 is_pasid, pasid); 1832 } 1833 } 1834 1835 /* Map dev to context-entry then do a paging-structures walk to do a iommu 1836 * translation. 1837 * 1838 * Called from RCU critical section. 1839 * 1840 * @bus_num: The bus number 1841 * @devfn: The devfn, which is the combined of device and function number 1842 * @is_write: The access is a write operation 1843 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 1844 * 1845 * Returns true if translation is successful, otherwise false. 1846 */ 1847 static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, 1848 uint8_t devfn, hwaddr addr, bool is_write, 1849 IOMMUTLBEntry *entry) 1850 { 1851 IntelIOMMUState *s = vtd_as->iommu_state; 1852 VTDContextEntry ce; 1853 uint8_t bus_num = pci_bus_num(bus); 1854 VTDContextCacheEntry *cc_entry; 1855 uint64_t slpte, page_mask; 1856 uint32_t level, pasid = vtd_as->pasid; 1857 uint16_t source_id = PCI_BUILD_BDF(bus_num, devfn); 1858 int ret_fr; 1859 bool is_fpd_set = false; 1860 bool reads = true; 1861 bool writes = true; 1862 uint8_t access_flags; 1863 bool rid2pasid = (pasid == PCI_NO_PASID) && s->root_scalable; 1864 VTDIOTLBEntry *iotlb_entry; 1865 1866 /* 1867 * We have standalone memory region for interrupt addresses, we 1868 * should never receive translation requests in this region. 1869 */ 1870 assert(!vtd_is_interrupt_addr(addr)); 1871 1872 vtd_iommu_lock(s); 1873 1874 cc_entry = &vtd_as->context_cache_entry; 1875 1876 /* Try to fetch slpte form IOTLB, we don't need RID2PASID logic */ 1877 if (!rid2pasid) { 1878 iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); 1879 if (iotlb_entry) { 1880 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, 1881 iotlb_entry->domain_id); 1882 slpte = iotlb_entry->slpte; 1883 access_flags = iotlb_entry->access_flags; 1884 page_mask = iotlb_entry->mask; 1885 goto out; 1886 } 1887 } 1888 1889 /* Try to fetch context-entry from cache first */ 1890 if (cc_entry->context_cache_gen == s->context_cache_gen) { 1891 trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi, 1892 cc_entry->context_entry.lo, 1893 cc_entry->context_cache_gen); 1894 ce = cc_entry->context_entry; 1895 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1896 if (!is_fpd_set && s->root_scalable) { 1897 ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid); 1898 if (ret_fr) { 1899 vtd_report_fault(s, -ret_fr, is_fpd_set, 1900 source_id, addr, is_write, 1901 false, 0); 1902 goto error; 1903 } 1904 } 1905 } else { 1906 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 1907 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1908 if (!ret_fr && !is_fpd_set && s->root_scalable) { 1909 ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid); 1910 } 1911 if (ret_fr) { 1912 vtd_report_fault(s, -ret_fr, is_fpd_set, 1913 source_id, addr, is_write, 1914 false, 0); 1915 goto error; 1916 } 1917 /* Update context-cache */ 1918 trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo, 1919 cc_entry->context_cache_gen, 1920 s->context_cache_gen); 1921 cc_entry->context_entry = ce; 1922 cc_entry->context_cache_gen = s->context_cache_gen; 1923 } 1924 1925 if (rid2pasid) { 1926 pasid = VTD_CE_GET_RID2PASID(&ce); 1927 } 1928 1929 /* 1930 * We don't need to translate for pass-through context entries. 1931 * Also, let's ignore IOTLB caching as well for PT devices. 1932 */ 1933 if (vtd_dev_pt_enabled(s, &ce, pasid)) { 1934 entry->iova = addr & VTD_PAGE_MASK_4K; 1935 entry->translated_addr = entry->iova; 1936 entry->addr_mask = ~VTD_PAGE_MASK_4K; 1937 entry->perm = IOMMU_RW; 1938 trace_vtd_translate_pt(source_id, entry->iova); 1939 1940 /* 1941 * When this happens, it means firstly caching-mode is not 1942 * enabled, and this is the first passthrough translation for 1943 * the device. Let's enable the fast path for passthrough. 1944 * 1945 * When passthrough is disabled again for the device, we can 1946 * capture it via the context entry invalidation, then the 1947 * IOMMU region can be swapped back. 1948 */ 1949 vtd_pt_enable_fast_path(s, source_id); 1950 vtd_iommu_unlock(s); 1951 return true; 1952 } 1953 1954 /* Try to fetch slpte form IOTLB for RID2PASID slow path */ 1955 if (rid2pasid) { 1956 iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); 1957 if (iotlb_entry) { 1958 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, 1959 iotlb_entry->domain_id); 1960 slpte = iotlb_entry->slpte; 1961 access_flags = iotlb_entry->access_flags; 1962 page_mask = iotlb_entry->mask; 1963 goto out; 1964 } 1965 } 1966 1967 ret_fr = vtd_iova_to_slpte(s, &ce, addr, is_write, &slpte, &level, 1968 &reads, &writes, s->aw_bits, pasid); 1969 if (ret_fr) { 1970 vtd_report_fault(s, -ret_fr, is_fpd_set, source_id, 1971 addr, is_write, pasid != PCI_NO_PASID, pasid); 1972 goto error; 1973 } 1974 1975 page_mask = vtd_slpt_level_page_mask(level); 1976 access_flags = IOMMU_ACCESS_FLAG(reads, writes); 1977 vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce, pasid), 1978 addr, slpte, access_flags, level, pasid); 1979 out: 1980 vtd_iommu_unlock(s); 1981 entry->iova = addr & page_mask; 1982 entry->translated_addr = vtd_get_slpte_addr(slpte, s->aw_bits) & page_mask; 1983 entry->addr_mask = ~page_mask; 1984 entry->perm = access_flags; 1985 return true; 1986 1987 error: 1988 vtd_iommu_unlock(s); 1989 entry->iova = 0; 1990 entry->translated_addr = 0; 1991 entry->addr_mask = 0; 1992 entry->perm = IOMMU_NONE; 1993 return false; 1994 } 1995 1996 static void vtd_root_table_setup(IntelIOMMUState *s) 1997 { 1998 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 1999 s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits); 2000 2001 vtd_update_scalable_state(s); 2002 2003 trace_vtd_reg_dmar_root(s->root, s->root_scalable); 2004 } 2005 2006 static void vtd_iec_notify_all(IntelIOMMUState *s, bool global, 2007 uint32_t index, uint32_t mask) 2008 { 2009 x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask); 2010 } 2011 2012 static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s) 2013 { 2014 uint64_t value = 0; 2015 value = vtd_get_quad_raw(s, DMAR_IRTA_REG); 2016 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1); 2017 s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits); 2018 s->intr_eime = value & VTD_IRTA_EIME; 2019 2020 /* Notify global invalidation */ 2021 vtd_iec_notify_all(s, true, 0, 0); 2022 2023 trace_vtd_reg_ir_root(s->intr_root, s->intr_size); 2024 } 2025 2026 static void vtd_iommu_replay_all(IntelIOMMUState *s) 2027 { 2028 VTDAddressSpace *vtd_as; 2029 2030 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 2031 vtd_address_space_sync(vtd_as); 2032 } 2033 } 2034 2035 static void vtd_context_global_invalidate(IntelIOMMUState *s) 2036 { 2037 trace_vtd_inv_desc_cc_global(); 2038 /* Protects context cache */ 2039 vtd_iommu_lock(s); 2040 s->context_cache_gen++; 2041 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 2042 vtd_reset_context_cache_locked(s); 2043 } 2044 vtd_iommu_unlock(s); 2045 vtd_address_space_refresh_all(s); 2046 /* 2047 * From VT-d spec 6.5.2.1, a global context entry invalidation 2048 * should be followed by a IOTLB global invalidation, so we should 2049 * be safe even without this. Hoewever, let's replay the region as 2050 * well to be safer, and go back here when we need finer tunes for 2051 * VT-d emulation codes. 2052 */ 2053 vtd_iommu_replay_all(s); 2054 } 2055 2056 /* Do a context-cache device-selective invalidation. 2057 * @func_mask: FM field after shifting 2058 */ 2059 static void vtd_context_device_invalidate(IntelIOMMUState *s, 2060 uint16_t source_id, 2061 uint16_t func_mask) 2062 { 2063 GHashTableIter as_it; 2064 uint16_t mask; 2065 VTDAddressSpace *vtd_as; 2066 uint8_t bus_n, devfn; 2067 2068 trace_vtd_inv_desc_cc_devices(source_id, func_mask); 2069 2070 switch (func_mask & 3) { 2071 case 0: 2072 mask = 0; /* No bits in the SID field masked */ 2073 break; 2074 case 1: 2075 mask = 4; /* Mask bit 2 in the SID field */ 2076 break; 2077 case 2: 2078 mask = 6; /* Mask bit 2:1 in the SID field */ 2079 break; 2080 case 3: 2081 mask = 7; /* Mask bit 2:0 in the SID field */ 2082 break; 2083 default: 2084 g_assert_not_reached(); 2085 } 2086 mask = ~mask; 2087 2088 bus_n = VTD_SID_TO_BUS(source_id); 2089 devfn = VTD_SID_TO_DEVFN(source_id); 2090 2091 g_hash_table_iter_init(&as_it, s->vtd_address_spaces); 2092 while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) { 2093 if ((pci_bus_num(vtd_as->bus) == bus_n) && 2094 (vtd_as->devfn & mask) == (devfn & mask)) { 2095 trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(vtd_as->devfn), 2096 VTD_PCI_FUNC(vtd_as->devfn)); 2097 vtd_iommu_lock(s); 2098 vtd_as->context_cache_entry.context_cache_gen = 0; 2099 vtd_iommu_unlock(s); 2100 /* 2101 * Do switch address space when needed, in case if the 2102 * device passthrough bit is switched. 2103 */ 2104 vtd_switch_address_space(vtd_as); 2105 /* 2106 * So a device is moving out of (or moving into) a 2107 * domain, resync the shadow page table. 2108 * This won't bring bad even if we have no such 2109 * notifier registered - the IOMMU notification 2110 * framework will skip MAP notifications if that 2111 * happened. 2112 */ 2113 vtd_address_space_sync(vtd_as); 2114 } 2115 } 2116 } 2117 2118 /* Context-cache invalidation 2119 * Returns the Context Actual Invalidation Granularity. 2120 * @val: the content of the CCMD_REG 2121 */ 2122 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 2123 { 2124 uint64_t caig; 2125 uint64_t type = val & VTD_CCMD_CIRG_MASK; 2126 2127 switch (type) { 2128 case VTD_CCMD_DOMAIN_INVL: 2129 /* Fall through */ 2130 case VTD_CCMD_GLOBAL_INVL: 2131 caig = VTD_CCMD_GLOBAL_INVL_A; 2132 vtd_context_global_invalidate(s); 2133 break; 2134 2135 case VTD_CCMD_DEVICE_INVL: 2136 caig = VTD_CCMD_DEVICE_INVL_A; 2137 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 2138 break; 2139 2140 default: 2141 error_report_once("%s: invalid context: 0x%" PRIx64, 2142 __func__, val); 2143 caig = 0; 2144 } 2145 return caig; 2146 } 2147 2148 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 2149 { 2150 trace_vtd_inv_desc_iotlb_global(); 2151 vtd_reset_iotlb(s); 2152 vtd_iommu_replay_all(s); 2153 } 2154 2155 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 2156 { 2157 VTDContextEntry ce; 2158 VTDAddressSpace *vtd_as; 2159 2160 trace_vtd_inv_desc_iotlb_domain(domain_id); 2161 2162 vtd_iommu_lock(s); 2163 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 2164 &domain_id); 2165 vtd_iommu_unlock(s); 2166 2167 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 2168 if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 2169 vtd_as->devfn, &ce) && 2170 domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { 2171 vtd_address_space_sync(vtd_as); 2172 } 2173 } 2174 } 2175 2176 static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, 2177 uint16_t domain_id, hwaddr addr, 2178 uint8_t am, uint32_t pasid) 2179 { 2180 VTDAddressSpace *vtd_as; 2181 VTDContextEntry ce; 2182 int ret; 2183 hwaddr size = (1 << am) * VTD_PAGE_SIZE; 2184 2185 QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) { 2186 if (pasid != PCI_NO_PASID && pasid != vtd_as->pasid) { 2187 continue; 2188 } 2189 ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 2190 vtd_as->devfn, &ce); 2191 if (!ret && domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { 2192 if (vtd_as_has_map_notifier(vtd_as)) { 2193 /* 2194 * As long as we have MAP notifications registered in 2195 * any of our IOMMU notifiers, we need to sync the 2196 * shadow page table. 2197 */ 2198 vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size); 2199 } else { 2200 /* 2201 * For UNMAP-only notifiers, we don't need to walk the 2202 * page tables. We just deliver the PSI down to 2203 * invalidate caches. 2204 */ 2205 IOMMUTLBEvent event = { 2206 .type = IOMMU_NOTIFIER_UNMAP, 2207 .entry = { 2208 .target_as = &address_space_memory, 2209 .iova = addr, 2210 .translated_addr = 0, 2211 .addr_mask = size - 1, 2212 .perm = IOMMU_NONE, 2213 }, 2214 }; 2215 memory_region_notify_iommu(&vtd_as->iommu, 0, event); 2216 } 2217 } 2218 } 2219 } 2220 2221 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 2222 hwaddr addr, uint8_t am) 2223 { 2224 VTDIOTLBPageInvInfo info; 2225 2226 trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am); 2227 2228 assert(am <= VTD_MAMV); 2229 info.domain_id = domain_id; 2230 info.addr = addr; 2231 info.mask = ~((1 << am) - 1); 2232 vtd_iommu_lock(s); 2233 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 2234 vtd_iommu_unlock(s); 2235 vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, PCI_NO_PASID); 2236 } 2237 2238 /* Flush IOTLB 2239 * Returns the IOTLB Actual Invalidation Granularity. 2240 * @val: the content of the IOTLB_REG 2241 */ 2242 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 2243 { 2244 uint64_t iaig; 2245 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 2246 uint16_t domain_id; 2247 hwaddr addr; 2248 uint8_t am; 2249 2250 switch (type) { 2251 case VTD_TLB_GLOBAL_FLUSH: 2252 iaig = VTD_TLB_GLOBAL_FLUSH_A; 2253 vtd_iotlb_global_invalidate(s); 2254 break; 2255 2256 case VTD_TLB_DSI_FLUSH: 2257 domain_id = VTD_TLB_DID(val); 2258 iaig = VTD_TLB_DSI_FLUSH_A; 2259 vtd_iotlb_domain_invalidate(s, domain_id); 2260 break; 2261 2262 case VTD_TLB_PSI_FLUSH: 2263 domain_id = VTD_TLB_DID(val); 2264 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 2265 am = VTD_IVA_AM(addr); 2266 addr = VTD_IVA_ADDR(addr); 2267 if (am > VTD_MAMV) { 2268 error_report_once("%s: address mask overflow: 0x%" PRIx64, 2269 __func__, vtd_get_quad_raw(s, DMAR_IVA_REG)); 2270 iaig = 0; 2271 break; 2272 } 2273 iaig = VTD_TLB_PSI_FLUSH_A; 2274 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 2275 break; 2276 2277 default: 2278 error_report_once("%s: invalid granularity: 0x%" PRIx64, 2279 __func__, val); 2280 iaig = 0; 2281 } 2282 return iaig; 2283 } 2284 2285 static void vtd_fetch_inv_desc(IntelIOMMUState *s); 2286 2287 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 2288 { 2289 return s->qi_enabled && (s->iq_tail == s->iq_head) && 2290 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 2291 } 2292 2293 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 2294 { 2295 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 2296 2297 trace_vtd_inv_qi_enable(en); 2298 2299 if (en) { 2300 s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits); 2301 /* 2^(x+8) entries */ 2302 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8 - (s->iq_dw ? 1 : 0)); 2303 s->qi_enabled = true; 2304 trace_vtd_inv_qi_setup(s->iq, s->iq_size); 2305 /* Ok - report back to driver */ 2306 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 2307 2308 if (s->iq_tail != 0) { 2309 /* 2310 * This is a spec violation but Windows guests are known to set up 2311 * Queued Invalidation this way so we allow the write and process 2312 * Invalidation Descriptors right away. 2313 */ 2314 trace_vtd_warn_invalid_qi_tail(s->iq_tail); 2315 if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 2316 vtd_fetch_inv_desc(s); 2317 } 2318 } 2319 } else { 2320 if (vtd_queued_inv_disable_check(s)) { 2321 /* disable Queued Invalidation */ 2322 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 2323 s->iq_head = 0; 2324 s->qi_enabled = false; 2325 /* Ok - report back to driver */ 2326 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 2327 } else { 2328 error_report_once("%s: detected improper state when disable QI " 2329 "(head=0x%x, tail=0x%x, last_type=%d)", 2330 __func__, 2331 s->iq_head, s->iq_tail, s->iq_last_desc_type); 2332 } 2333 } 2334 } 2335 2336 /* Set Root Table Pointer */ 2337 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 2338 { 2339 vtd_root_table_setup(s); 2340 /* Ok - report back to driver */ 2341 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 2342 vtd_reset_caches(s); 2343 vtd_address_space_refresh_all(s); 2344 } 2345 2346 /* Set Interrupt Remap Table Pointer */ 2347 static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s) 2348 { 2349 vtd_interrupt_remap_table_setup(s); 2350 /* Ok - report back to driver */ 2351 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS); 2352 } 2353 2354 /* Handle Translation Enable/Disable */ 2355 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 2356 { 2357 if (s->dmar_enabled == en) { 2358 return; 2359 } 2360 2361 trace_vtd_dmar_enable(en); 2362 2363 if (en) { 2364 s->dmar_enabled = true; 2365 /* Ok - report back to driver */ 2366 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 2367 } else { 2368 s->dmar_enabled = false; 2369 2370 /* Clear the index of Fault Recording Register */ 2371 s->next_frcd_reg = 0; 2372 /* Ok - report back to driver */ 2373 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 2374 } 2375 2376 vtd_reset_caches(s); 2377 vtd_address_space_refresh_all(s); 2378 } 2379 2380 /* Handle Interrupt Remap Enable/Disable */ 2381 static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en) 2382 { 2383 trace_vtd_ir_enable(en); 2384 2385 if (en) { 2386 s->intr_enabled = true; 2387 /* Ok - report back to driver */ 2388 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES); 2389 } else { 2390 s->intr_enabled = false; 2391 /* Ok - report back to driver */ 2392 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0); 2393 } 2394 } 2395 2396 /* Handle write to Global Command Register */ 2397 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 2398 { 2399 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 2400 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 2401 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 2402 uint32_t changed = status ^ val; 2403 2404 trace_vtd_reg_write_gcmd(status, val); 2405 if ((changed & VTD_GCMD_TE) && s->dma_translation) { 2406 /* Translation enable/disable */ 2407 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 2408 } 2409 if (val & VTD_GCMD_SRTP) { 2410 /* Set/update the root-table pointer */ 2411 vtd_handle_gcmd_srtp(s); 2412 } 2413 if (changed & VTD_GCMD_QIE) { 2414 /* Queued Invalidation Enable */ 2415 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 2416 } 2417 if (val & VTD_GCMD_SIRTP) { 2418 /* Set/update the interrupt remapping root-table pointer */ 2419 vtd_handle_gcmd_sirtp(s); 2420 } 2421 if ((changed & VTD_GCMD_IRE) && 2422 x86_iommu_ir_supported(x86_iommu)) { 2423 /* Interrupt remap enable/disable */ 2424 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE); 2425 } 2426 } 2427 2428 /* Handle write to Context Command Register */ 2429 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 2430 { 2431 uint64_t ret; 2432 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 2433 2434 /* Context-cache invalidation request */ 2435 if (val & VTD_CCMD_ICC) { 2436 if (s->qi_enabled) { 2437 error_report_once("Queued Invalidation enabled, " 2438 "should not use register-based invalidation"); 2439 return; 2440 } 2441 ret = vtd_context_cache_invalidate(s, val); 2442 /* Invalidation completed. Change something to show */ 2443 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 2444 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 2445 ret); 2446 } 2447 } 2448 2449 /* Handle write to IOTLB Invalidation Register */ 2450 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 2451 { 2452 uint64_t ret; 2453 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 2454 2455 /* IOTLB invalidation request */ 2456 if (val & VTD_TLB_IVT) { 2457 if (s->qi_enabled) { 2458 error_report_once("Queued Invalidation enabled, " 2459 "should not use register-based invalidation"); 2460 return; 2461 } 2462 ret = vtd_iotlb_flush(s, val); 2463 /* Invalidation completed. Change something to show */ 2464 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 2465 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 2466 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 2467 } 2468 } 2469 2470 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 2471 static bool vtd_get_inv_desc(IntelIOMMUState *s, 2472 VTDInvDesc *inv_desc) 2473 { 2474 dma_addr_t base_addr = s->iq; 2475 uint32_t offset = s->iq_head; 2476 uint32_t dw = s->iq_dw ? 32 : 16; 2477 dma_addr_t addr = base_addr + offset * dw; 2478 2479 if (dma_memory_read(&address_space_memory, addr, 2480 inv_desc, dw, MEMTXATTRS_UNSPECIFIED)) { 2481 error_report_once("Read INV DESC failed."); 2482 return false; 2483 } 2484 inv_desc->lo = le64_to_cpu(inv_desc->lo); 2485 inv_desc->hi = le64_to_cpu(inv_desc->hi); 2486 if (dw == 32) { 2487 inv_desc->val[2] = le64_to_cpu(inv_desc->val[2]); 2488 inv_desc->val[3] = le64_to_cpu(inv_desc->val[3]); 2489 } 2490 return true; 2491 } 2492 2493 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 2494 { 2495 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 2496 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 2497 error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64 2498 " (reserved nonzero)", __func__, inv_desc->hi, 2499 inv_desc->lo); 2500 return false; 2501 } 2502 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 2503 /* Status Write */ 2504 uint32_t status_data = (uint32_t)(inv_desc->lo >> 2505 VTD_INV_DESC_WAIT_DATA_SHIFT); 2506 2507 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 2508 2509 /* FIXME: need to be masked with HAW? */ 2510 dma_addr_t status_addr = inv_desc->hi; 2511 trace_vtd_inv_desc_wait_sw(status_addr, status_data); 2512 status_data = cpu_to_le32(status_data); 2513 if (dma_memory_write(&address_space_memory, status_addr, 2514 &status_data, sizeof(status_data), 2515 MEMTXATTRS_UNSPECIFIED)) { 2516 trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo); 2517 return false; 2518 } 2519 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 2520 /* Interrupt flag */ 2521 vtd_generate_completion_event(s); 2522 } else { 2523 error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64 2524 " (unknown type)", __func__, inv_desc->hi, 2525 inv_desc->lo); 2526 return false; 2527 } 2528 return true; 2529 } 2530 2531 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 2532 VTDInvDesc *inv_desc) 2533 { 2534 uint16_t sid, fmask; 2535 2536 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 2537 error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64 2538 " (reserved nonzero)", __func__, inv_desc->hi, 2539 inv_desc->lo); 2540 return false; 2541 } 2542 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 2543 case VTD_INV_DESC_CC_DOMAIN: 2544 trace_vtd_inv_desc_cc_domain( 2545 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 2546 /* Fall through */ 2547 case VTD_INV_DESC_CC_GLOBAL: 2548 vtd_context_global_invalidate(s); 2549 break; 2550 2551 case VTD_INV_DESC_CC_DEVICE: 2552 sid = VTD_INV_DESC_CC_SID(inv_desc->lo); 2553 fmask = VTD_INV_DESC_CC_FM(inv_desc->lo); 2554 vtd_context_device_invalidate(s, sid, fmask); 2555 break; 2556 2557 default: 2558 error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64 2559 " (invalid type)", __func__, inv_desc->hi, 2560 inv_desc->lo); 2561 return false; 2562 } 2563 return true; 2564 } 2565 2566 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 2567 { 2568 uint16_t domain_id; 2569 uint8_t am; 2570 hwaddr addr; 2571 2572 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 2573 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 2574 error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2575 ", lo=0x%"PRIx64" (reserved bits unzero)", 2576 __func__, inv_desc->hi, inv_desc->lo); 2577 return false; 2578 } 2579 2580 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 2581 case VTD_INV_DESC_IOTLB_GLOBAL: 2582 vtd_iotlb_global_invalidate(s); 2583 break; 2584 2585 case VTD_INV_DESC_IOTLB_DOMAIN: 2586 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 2587 vtd_iotlb_domain_invalidate(s, domain_id); 2588 break; 2589 2590 case VTD_INV_DESC_IOTLB_PAGE: 2591 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 2592 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 2593 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 2594 if (am > VTD_MAMV) { 2595 error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2596 ", lo=0x%"PRIx64" (am=%u > VTD_MAMV=%u)", 2597 __func__, inv_desc->hi, inv_desc->lo, 2598 am, (unsigned)VTD_MAMV); 2599 return false; 2600 } 2601 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 2602 break; 2603 2604 default: 2605 error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2606 ", lo=0x%"PRIx64" (type mismatch: 0x%llx)", 2607 __func__, inv_desc->hi, inv_desc->lo, 2608 inv_desc->lo & VTD_INV_DESC_IOTLB_G); 2609 return false; 2610 } 2611 return true; 2612 } 2613 2614 static bool vtd_process_inv_iec_desc(IntelIOMMUState *s, 2615 VTDInvDesc *inv_desc) 2616 { 2617 trace_vtd_inv_desc_iec(inv_desc->iec.granularity, 2618 inv_desc->iec.index, 2619 inv_desc->iec.index_mask); 2620 2621 vtd_iec_notify_all(s, !inv_desc->iec.granularity, 2622 inv_desc->iec.index, 2623 inv_desc->iec.index_mask); 2624 return true; 2625 } 2626 2627 static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s, 2628 VTDInvDesc *inv_desc) 2629 { 2630 VTDAddressSpace *vtd_dev_as; 2631 IOMMUTLBEvent event; 2632 hwaddr addr; 2633 uint64_t sz; 2634 uint16_t sid; 2635 bool size; 2636 2637 addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi); 2638 sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo); 2639 size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi); 2640 2641 if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) || 2642 (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) { 2643 error_report_once("%s: invalid dev-iotlb inv desc: hi=%"PRIx64 2644 ", lo=%"PRIx64" (reserved nonzero)", __func__, 2645 inv_desc->hi, inv_desc->lo); 2646 return false; 2647 } 2648 2649 /* 2650 * Using sid is OK since the guest should have finished the 2651 * initialization of both the bus and device. 2652 */ 2653 vtd_dev_as = vtd_get_as_by_sid(s, sid); 2654 if (!vtd_dev_as) { 2655 goto done; 2656 } 2657 2658 /* According to ATS spec table 2.4: 2659 * S = 0, bits 15:12 = xxxx range size: 4K 2660 * S = 1, bits 15:12 = xxx0 range size: 8K 2661 * S = 1, bits 15:12 = xx01 range size: 16K 2662 * S = 1, bits 15:12 = x011 range size: 32K 2663 * S = 1, bits 15:12 = 0111 range size: 64K 2664 * ... 2665 */ 2666 if (size) { 2667 sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT); 2668 addr &= ~(sz - 1); 2669 } else { 2670 sz = VTD_PAGE_SIZE; 2671 } 2672 2673 event.type = IOMMU_NOTIFIER_DEVIOTLB_UNMAP; 2674 event.entry.target_as = &vtd_dev_as->as; 2675 event.entry.addr_mask = sz - 1; 2676 event.entry.iova = addr; 2677 event.entry.perm = IOMMU_NONE; 2678 event.entry.translated_addr = 0; 2679 memory_region_notify_iommu(&vtd_dev_as->iommu, 0, event); 2680 2681 done: 2682 return true; 2683 } 2684 2685 static bool vtd_process_inv_desc(IntelIOMMUState *s) 2686 { 2687 VTDInvDesc inv_desc; 2688 uint8_t desc_type; 2689 2690 trace_vtd_inv_qi_head(s->iq_head); 2691 if (!vtd_get_inv_desc(s, &inv_desc)) { 2692 s->iq_last_desc_type = VTD_INV_DESC_NONE; 2693 return false; 2694 } 2695 2696 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 2697 /* FIXME: should update at first or at last? */ 2698 s->iq_last_desc_type = desc_type; 2699 2700 switch (desc_type) { 2701 case VTD_INV_DESC_CC: 2702 trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo); 2703 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 2704 return false; 2705 } 2706 break; 2707 2708 case VTD_INV_DESC_IOTLB: 2709 trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo); 2710 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 2711 return false; 2712 } 2713 break; 2714 2715 /* 2716 * TODO: the entity of below two cases will be implemented in future series. 2717 * To make guest (which integrates scalable mode support patch set in 2718 * iommu driver) work, just return true is enough so far. 2719 */ 2720 case VTD_INV_DESC_PC: 2721 break; 2722 2723 case VTD_INV_DESC_PIOTLB: 2724 break; 2725 2726 case VTD_INV_DESC_WAIT: 2727 trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo); 2728 if (!vtd_process_wait_desc(s, &inv_desc)) { 2729 return false; 2730 } 2731 break; 2732 2733 case VTD_INV_DESC_IEC: 2734 trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo); 2735 if (!vtd_process_inv_iec_desc(s, &inv_desc)) { 2736 return false; 2737 } 2738 break; 2739 2740 case VTD_INV_DESC_DEVICE: 2741 trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo); 2742 if (!vtd_process_device_iotlb_desc(s, &inv_desc)) { 2743 return false; 2744 } 2745 break; 2746 2747 default: 2748 error_report_once("%s: invalid inv desc: hi=%"PRIx64", lo=%"PRIx64 2749 " (unknown type)", __func__, inv_desc.hi, 2750 inv_desc.lo); 2751 return false; 2752 } 2753 s->iq_head++; 2754 if (s->iq_head == s->iq_size) { 2755 s->iq_head = 0; 2756 } 2757 return true; 2758 } 2759 2760 /* Try to fetch and process more Invalidation Descriptors */ 2761 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 2762 { 2763 int qi_shift; 2764 2765 /* Refer to 10.4.23 of VT-d spec 3.0 */ 2766 qi_shift = s->iq_dw ? VTD_IQH_QH_SHIFT_5 : VTD_IQH_QH_SHIFT_4; 2767 2768 trace_vtd_inv_qi_fetch(); 2769 2770 if (s->iq_tail >= s->iq_size) { 2771 /* Detects an invalid Tail pointer */ 2772 error_report_once("%s: detected invalid QI tail " 2773 "(tail=0x%x, size=0x%x)", 2774 __func__, s->iq_tail, s->iq_size); 2775 vtd_handle_inv_queue_error(s); 2776 return; 2777 } 2778 while (s->iq_head != s->iq_tail) { 2779 if (!vtd_process_inv_desc(s)) { 2780 /* Invalidation Queue Errors */ 2781 vtd_handle_inv_queue_error(s); 2782 break; 2783 } 2784 /* Must update the IQH_REG in time */ 2785 vtd_set_quad_raw(s, DMAR_IQH_REG, 2786 (((uint64_t)(s->iq_head)) << qi_shift) & 2787 VTD_IQH_QH_MASK); 2788 } 2789 } 2790 2791 /* Handle write to Invalidation Queue Tail Register */ 2792 static void vtd_handle_iqt_write(IntelIOMMUState *s) 2793 { 2794 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 2795 2796 if (s->iq_dw && (val & VTD_IQT_QT_256_RSV_BIT)) { 2797 error_report_once("%s: RSV bit is set: val=0x%"PRIx64, 2798 __func__, val); 2799 return; 2800 } 2801 s->iq_tail = VTD_IQT_QT(s->iq_dw, val); 2802 trace_vtd_inv_qi_tail(s->iq_tail); 2803 2804 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 2805 /* Process Invalidation Queue here */ 2806 vtd_fetch_inv_desc(s); 2807 } 2808 } 2809 2810 static void vtd_handle_fsts_write(IntelIOMMUState *s) 2811 { 2812 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 2813 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 2814 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 2815 2816 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 2817 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 2818 trace_vtd_fsts_clear_ip(); 2819 } 2820 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 2821 * Descriptors if there are any when Queued Invalidation is enabled? 2822 */ 2823 } 2824 2825 static void vtd_handle_fectl_write(IntelIOMMUState *s) 2826 { 2827 uint32_t fectl_reg; 2828 /* FIXME: when software clears the IM field, check the IP field. But do we 2829 * need to compare the old value and the new value to conclude that 2830 * software clears the IM field? Or just check if the IM field is zero? 2831 */ 2832 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 2833 2834 trace_vtd_reg_write_fectl(fectl_reg); 2835 2836 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 2837 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 2838 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 2839 } 2840 } 2841 2842 static void vtd_handle_ics_write(IntelIOMMUState *s) 2843 { 2844 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 2845 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 2846 2847 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 2848 trace_vtd_reg_ics_clear_ip(); 2849 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 2850 } 2851 } 2852 2853 static void vtd_handle_iectl_write(IntelIOMMUState *s) 2854 { 2855 uint32_t iectl_reg; 2856 /* FIXME: when software clears the IM field, check the IP field. But do we 2857 * need to compare the old value and the new value to conclude that 2858 * software clears the IM field? Or just check if the IM field is zero? 2859 */ 2860 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 2861 2862 trace_vtd_reg_write_iectl(iectl_reg); 2863 2864 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 2865 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 2866 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 2867 } 2868 } 2869 2870 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 2871 { 2872 IntelIOMMUState *s = opaque; 2873 uint64_t val; 2874 2875 trace_vtd_reg_read(addr, size); 2876 2877 if (addr + size > DMAR_REG_SIZE) { 2878 error_report_once("%s: MMIO over range: addr=0x%" PRIx64 2879 " size=0x%x", __func__, addr, size); 2880 return (uint64_t)-1; 2881 } 2882 2883 switch (addr) { 2884 /* Root Table Address Register, 64-bit */ 2885 case DMAR_RTADDR_REG: 2886 val = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 2887 if (size == 4) { 2888 val = val & ((1ULL << 32) - 1); 2889 } 2890 break; 2891 2892 case DMAR_RTADDR_REG_HI: 2893 assert(size == 4); 2894 val = vtd_get_quad_raw(s, DMAR_RTADDR_REG) >> 32; 2895 break; 2896 2897 /* Invalidation Queue Address Register, 64-bit */ 2898 case DMAR_IQA_REG: 2899 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 2900 if (size == 4) { 2901 val = val & ((1ULL << 32) - 1); 2902 } 2903 break; 2904 2905 case DMAR_IQA_REG_HI: 2906 assert(size == 4); 2907 val = s->iq >> 32; 2908 break; 2909 2910 default: 2911 if (size == 4) { 2912 val = vtd_get_long(s, addr); 2913 } else { 2914 val = vtd_get_quad(s, addr); 2915 } 2916 } 2917 2918 return val; 2919 } 2920 2921 static void vtd_mem_write(void *opaque, hwaddr addr, 2922 uint64_t val, unsigned size) 2923 { 2924 IntelIOMMUState *s = opaque; 2925 2926 trace_vtd_reg_write(addr, size, val); 2927 2928 if (addr + size > DMAR_REG_SIZE) { 2929 error_report_once("%s: MMIO over range: addr=0x%" PRIx64 2930 " size=0x%x", __func__, addr, size); 2931 return; 2932 } 2933 2934 switch (addr) { 2935 /* Global Command Register, 32-bit */ 2936 case DMAR_GCMD_REG: 2937 vtd_set_long(s, addr, val); 2938 vtd_handle_gcmd_write(s); 2939 break; 2940 2941 /* Context Command Register, 64-bit */ 2942 case DMAR_CCMD_REG: 2943 if (size == 4) { 2944 vtd_set_long(s, addr, val); 2945 } else { 2946 vtd_set_quad(s, addr, val); 2947 vtd_handle_ccmd_write(s); 2948 } 2949 break; 2950 2951 case DMAR_CCMD_REG_HI: 2952 assert(size == 4); 2953 vtd_set_long(s, addr, val); 2954 vtd_handle_ccmd_write(s); 2955 break; 2956 2957 /* IOTLB Invalidation Register, 64-bit */ 2958 case DMAR_IOTLB_REG: 2959 if (size == 4) { 2960 vtd_set_long(s, addr, val); 2961 } else { 2962 vtd_set_quad(s, addr, val); 2963 vtd_handle_iotlb_write(s); 2964 } 2965 break; 2966 2967 case DMAR_IOTLB_REG_HI: 2968 assert(size == 4); 2969 vtd_set_long(s, addr, val); 2970 vtd_handle_iotlb_write(s); 2971 break; 2972 2973 /* Invalidate Address Register, 64-bit */ 2974 case DMAR_IVA_REG: 2975 if (size == 4) { 2976 vtd_set_long(s, addr, val); 2977 } else { 2978 vtd_set_quad(s, addr, val); 2979 } 2980 break; 2981 2982 case DMAR_IVA_REG_HI: 2983 assert(size == 4); 2984 vtd_set_long(s, addr, val); 2985 break; 2986 2987 /* Fault Status Register, 32-bit */ 2988 case DMAR_FSTS_REG: 2989 assert(size == 4); 2990 vtd_set_long(s, addr, val); 2991 vtd_handle_fsts_write(s); 2992 break; 2993 2994 /* Fault Event Control Register, 32-bit */ 2995 case DMAR_FECTL_REG: 2996 assert(size == 4); 2997 vtd_set_long(s, addr, val); 2998 vtd_handle_fectl_write(s); 2999 break; 3000 3001 /* Fault Event Data Register, 32-bit */ 3002 case DMAR_FEDATA_REG: 3003 assert(size == 4); 3004 vtd_set_long(s, addr, val); 3005 break; 3006 3007 /* Fault Event Address Register, 32-bit */ 3008 case DMAR_FEADDR_REG: 3009 if (size == 4) { 3010 vtd_set_long(s, addr, val); 3011 } else { 3012 /* 3013 * While the register is 32-bit only, some guests (Xen...) write to 3014 * it with 64-bit. 3015 */ 3016 vtd_set_quad(s, addr, val); 3017 } 3018 break; 3019 3020 /* Fault Event Upper Address Register, 32-bit */ 3021 case DMAR_FEUADDR_REG: 3022 assert(size == 4); 3023 vtd_set_long(s, addr, val); 3024 break; 3025 3026 /* Protected Memory Enable Register, 32-bit */ 3027 case DMAR_PMEN_REG: 3028 assert(size == 4); 3029 vtd_set_long(s, addr, val); 3030 break; 3031 3032 /* Root Table Address Register, 64-bit */ 3033 case DMAR_RTADDR_REG: 3034 if (size == 4) { 3035 vtd_set_long(s, addr, val); 3036 } else { 3037 vtd_set_quad(s, addr, val); 3038 } 3039 break; 3040 3041 case DMAR_RTADDR_REG_HI: 3042 assert(size == 4); 3043 vtd_set_long(s, addr, val); 3044 break; 3045 3046 /* Invalidation Queue Tail Register, 64-bit */ 3047 case DMAR_IQT_REG: 3048 if (size == 4) { 3049 vtd_set_long(s, addr, val); 3050 } else { 3051 vtd_set_quad(s, addr, val); 3052 } 3053 vtd_handle_iqt_write(s); 3054 break; 3055 3056 case DMAR_IQT_REG_HI: 3057 assert(size == 4); 3058 vtd_set_long(s, addr, val); 3059 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 3060 break; 3061 3062 /* Invalidation Queue Address Register, 64-bit */ 3063 case DMAR_IQA_REG: 3064 if (size == 4) { 3065 vtd_set_long(s, addr, val); 3066 } else { 3067 vtd_set_quad(s, addr, val); 3068 } 3069 vtd_update_iq_dw(s); 3070 break; 3071 3072 case DMAR_IQA_REG_HI: 3073 assert(size == 4); 3074 vtd_set_long(s, addr, val); 3075 break; 3076 3077 /* Invalidation Completion Status Register, 32-bit */ 3078 case DMAR_ICS_REG: 3079 assert(size == 4); 3080 vtd_set_long(s, addr, val); 3081 vtd_handle_ics_write(s); 3082 break; 3083 3084 /* Invalidation Event Control Register, 32-bit */ 3085 case DMAR_IECTL_REG: 3086 assert(size == 4); 3087 vtd_set_long(s, addr, val); 3088 vtd_handle_iectl_write(s); 3089 break; 3090 3091 /* Invalidation Event Data Register, 32-bit */ 3092 case DMAR_IEDATA_REG: 3093 assert(size == 4); 3094 vtd_set_long(s, addr, val); 3095 break; 3096 3097 /* Invalidation Event Address Register, 32-bit */ 3098 case DMAR_IEADDR_REG: 3099 assert(size == 4); 3100 vtd_set_long(s, addr, val); 3101 break; 3102 3103 /* Invalidation Event Upper Address Register, 32-bit */ 3104 case DMAR_IEUADDR_REG: 3105 assert(size == 4); 3106 vtd_set_long(s, addr, val); 3107 break; 3108 3109 /* Fault Recording Registers, 128-bit */ 3110 case DMAR_FRCD_REG_0_0: 3111 if (size == 4) { 3112 vtd_set_long(s, addr, val); 3113 } else { 3114 vtd_set_quad(s, addr, val); 3115 } 3116 break; 3117 3118 case DMAR_FRCD_REG_0_1: 3119 assert(size == 4); 3120 vtd_set_long(s, addr, val); 3121 break; 3122 3123 case DMAR_FRCD_REG_0_2: 3124 if (size == 4) { 3125 vtd_set_long(s, addr, val); 3126 } else { 3127 vtd_set_quad(s, addr, val); 3128 /* May clear bit 127 (Fault), update PPF */ 3129 vtd_update_fsts_ppf(s); 3130 } 3131 break; 3132 3133 case DMAR_FRCD_REG_0_3: 3134 assert(size == 4); 3135 vtd_set_long(s, addr, val); 3136 /* May clear bit 127 (Fault), update PPF */ 3137 vtd_update_fsts_ppf(s); 3138 break; 3139 3140 case DMAR_IRTA_REG: 3141 if (size == 4) { 3142 vtd_set_long(s, addr, val); 3143 } else { 3144 vtd_set_quad(s, addr, val); 3145 } 3146 break; 3147 3148 case DMAR_IRTA_REG_HI: 3149 assert(size == 4); 3150 vtd_set_long(s, addr, val); 3151 break; 3152 3153 default: 3154 if (size == 4) { 3155 vtd_set_long(s, addr, val); 3156 } else { 3157 vtd_set_quad(s, addr, val); 3158 } 3159 } 3160 } 3161 3162 static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, 3163 IOMMUAccessFlags flag, int iommu_idx) 3164 { 3165 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 3166 IntelIOMMUState *s = vtd_as->iommu_state; 3167 IOMMUTLBEntry iotlb = { 3168 /* We'll fill in the rest later. */ 3169 .target_as = &address_space_memory, 3170 }; 3171 bool success; 3172 3173 if (likely(s->dmar_enabled)) { 3174 success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, 3175 addr, flag & IOMMU_WO, &iotlb); 3176 } else { 3177 /* DMAR disabled, passthrough, use 4k-page*/ 3178 iotlb.iova = addr & VTD_PAGE_MASK_4K; 3179 iotlb.translated_addr = addr & VTD_PAGE_MASK_4K; 3180 iotlb.addr_mask = ~VTD_PAGE_MASK_4K; 3181 iotlb.perm = IOMMU_RW; 3182 success = true; 3183 } 3184 3185 if (likely(success)) { 3186 trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus), 3187 VTD_PCI_SLOT(vtd_as->devfn), 3188 VTD_PCI_FUNC(vtd_as->devfn), 3189 iotlb.iova, iotlb.translated_addr, 3190 iotlb.addr_mask); 3191 } else { 3192 error_report_once("%s: detected translation failure " 3193 "(dev=%02x:%02x:%02x, iova=0x%" PRIx64 ")", 3194 __func__, pci_bus_num(vtd_as->bus), 3195 VTD_PCI_SLOT(vtd_as->devfn), 3196 VTD_PCI_FUNC(vtd_as->devfn), 3197 addr); 3198 } 3199 3200 return iotlb; 3201 } 3202 3203 static int vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu, 3204 IOMMUNotifierFlag old, 3205 IOMMUNotifierFlag new, 3206 Error **errp) 3207 { 3208 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 3209 IntelIOMMUState *s = vtd_as->iommu_state; 3210 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3211 3212 /* TODO: add support for VFIO and vhost users */ 3213 if (s->snoop_control) { 3214 error_setg_errno(errp, ENOTSUP, 3215 "Snoop Control with vhost or VFIO is not supported"); 3216 return -ENOTSUP; 3217 } 3218 if (!s->caching_mode && (new & IOMMU_NOTIFIER_MAP)) { 3219 error_setg_errno(errp, ENOTSUP, 3220 "device %02x.%02x.%x requires caching mode", 3221 pci_bus_num(vtd_as->bus), PCI_SLOT(vtd_as->devfn), 3222 PCI_FUNC(vtd_as->devfn)); 3223 return -ENOTSUP; 3224 } 3225 if (!x86_iommu->dt_supported && (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP)) { 3226 error_setg_errno(errp, ENOTSUP, 3227 "device %02x.%02x.%x requires device IOTLB mode", 3228 pci_bus_num(vtd_as->bus), PCI_SLOT(vtd_as->devfn), 3229 PCI_FUNC(vtd_as->devfn)); 3230 return -ENOTSUP; 3231 } 3232 3233 /* Update per-address-space notifier flags */ 3234 vtd_as->notifier_flags = new; 3235 3236 if (old == IOMMU_NOTIFIER_NONE) { 3237 QLIST_INSERT_HEAD(&s->vtd_as_with_notifiers, vtd_as, next); 3238 } else if (new == IOMMU_NOTIFIER_NONE) { 3239 QLIST_REMOVE(vtd_as, next); 3240 } 3241 return 0; 3242 } 3243 3244 static int vtd_post_load(void *opaque, int version_id) 3245 { 3246 IntelIOMMUState *iommu = opaque; 3247 3248 /* 3249 * We don't need to migrate the root_scalable because we can 3250 * simply do the calculation after the loading is complete. We 3251 * can actually do similar things with root, dmar_enabled, etc. 3252 * however since we've had them already so we'd better keep them 3253 * for compatibility of migration. 3254 */ 3255 vtd_update_scalable_state(iommu); 3256 3257 vtd_update_iq_dw(iommu); 3258 3259 /* 3260 * Memory regions are dynamically turned on/off depending on 3261 * context entry configurations from the guest. After migration, 3262 * we need to make sure the memory regions are still correct. 3263 */ 3264 vtd_switch_address_space_all(iommu); 3265 3266 return 0; 3267 } 3268 3269 static const VMStateDescription vtd_vmstate = { 3270 .name = "iommu-intel", 3271 .version_id = 1, 3272 .minimum_version_id = 1, 3273 .priority = MIG_PRI_IOMMU, 3274 .post_load = vtd_post_load, 3275 .fields = (VMStateField[]) { 3276 VMSTATE_UINT64(root, IntelIOMMUState), 3277 VMSTATE_UINT64(intr_root, IntelIOMMUState), 3278 VMSTATE_UINT64(iq, IntelIOMMUState), 3279 VMSTATE_UINT32(intr_size, IntelIOMMUState), 3280 VMSTATE_UINT16(iq_head, IntelIOMMUState), 3281 VMSTATE_UINT16(iq_tail, IntelIOMMUState), 3282 VMSTATE_UINT16(iq_size, IntelIOMMUState), 3283 VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState), 3284 VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE), 3285 VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState), 3286 VMSTATE_UNUSED(1), /* bool root_extended is obsolete by VT-d */ 3287 VMSTATE_BOOL(dmar_enabled, IntelIOMMUState), 3288 VMSTATE_BOOL(qi_enabled, IntelIOMMUState), 3289 VMSTATE_BOOL(intr_enabled, IntelIOMMUState), 3290 VMSTATE_BOOL(intr_eime, IntelIOMMUState), 3291 VMSTATE_END_OF_LIST() 3292 } 3293 }; 3294 3295 static const MemoryRegionOps vtd_mem_ops = { 3296 .read = vtd_mem_read, 3297 .write = vtd_mem_write, 3298 .endianness = DEVICE_LITTLE_ENDIAN, 3299 .impl = { 3300 .min_access_size = 4, 3301 .max_access_size = 8, 3302 }, 3303 .valid = { 3304 .min_access_size = 4, 3305 .max_access_size = 8, 3306 }, 3307 }; 3308 3309 static Property vtd_properties[] = { 3310 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 3311 DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim, 3312 ON_OFF_AUTO_AUTO), 3313 DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false), 3314 DEFINE_PROP_UINT8("aw-bits", IntelIOMMUState, aw_bits, 3315 VTD_HOST_ADDRESS_WIDTH), 3316 DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE), 3317 DEFINE_PROP_BOOL("x-scalable-mode", IntelIOMMUState, scalable_mode, FALSE), 3318 DEFINE_PROP_BOOL("snoop-control", IntelIOMMUState, snoop_control, false), 3319 DEFINE_PROP_BOOL("x-pasid-mode", IntelIOMMUState, pasid, false), 3320 DEFINE_PROP_BOOL("dma-drain", IntelIOMMUState, dma_drain, true), 3321 DEFINE_PROP_BOOL("dma-translation", IntelIOMMUState, dma_translation, true), 3322 DEFINE_PROP_END_OF_LIST(), 3323 }; 3324 3325 /* Read IRTE entry with specific index */ 3326 static bool vtd_irte_get(IntelIOMMUState *iommu, uint16_t index, 3327 VTD_IR_TableEntry *entry, uint16_t sid, 3328 bool do_fault) 3329 { 3330 static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \ 3331 {0xffff, 0xfffb, 0xfff9, 0xfff8}; 3332 dma_addr_t addr = 0x00; 3333 uint16_t mask, source_id; 3334 uint8_t bus, bus_max, bus_min; 3335 3336 if (index >= iommu->intr_size) { 3337 error_report_once("%s: index too large: ind=0x%x", 3338 __func__, index); 3339 if (do_fault) { 3340 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_INDEX_OVER, index); 3341 } 3342 return false; 3343 } 3344 3345 addr = iommu->intr_root + index * sizeof(*entry); 3346 if (dma_memory_read(&address_space_memory, addr, 3347 entry, sizeof(*entry), MEMTXATTRS_UNSPECIFIED)) { 3348 error_report_once("%s: read failed: ind=0x%x addr=0x%" PRIx64, 3349 __func__, index, addr); 3350 if (do_fault) { 3351 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_ROOT_INVAL, index); 3352 } 3353 return false; 3354 } 3355 3356 entry->data[0] = le64_to_cpu(entry->data[0]); 3357 entry->data[1] = le64_to_cpu(entry->data[1]); 3358 3359 trace_vtd_ir_irte_get(index, entry->data[1], entry->data[0]); 3360 3361 /* 3362 * The remaining potential fault conditions are "qualified" by the 3363 * Fault Processing Disable bit in the IRTE. Even "not present". 3364 * So just clear the do_fault flag if PFD is set, which will 3365 * prevent faults being raised. 3366 */ 3367 if (entry->irte.fault_disable) { 3368 do_fault = false; 3369 } 3370 3371 if (!entry->irte.present) { 3372 error_report_once("%s: detected non-present IRTE " 3373 "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")", 3374 __func__, index, entry->data[1], entry->data[0]); 3375 if (do_fault) { 3376 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_ENTRY_P, index); 3377 } 3378 return false; 3379 } 3380 3381 if (entry->irte.__reserved_0 || entry->irte.__reserved_1 || 3382 entry->irte.__reserved_2) { 3383 error_report_once("%s: detected non-zero reserved IRTE " 3384 "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")", 3385 __func__, index, entry->data[1], entry->data[0]); 3386 if (do_fault) { 3387 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_IRTE_RSVD, index); 3388 } 3389 return false; 3390 } 3391 3392 if (sid != X86_IOMMU_SID_INVALID) { 3393 /* Validate IRTE SID */ 3394 source_id = entry->irte.source_id; 3395 switch (entry->irte.sid_vtype) { 3396 case VTD_SVT_NONE: 3397 break; 3398 3399 case VTD_SVT_ALL: 3400 mask = vtd_svt_mask[entry->irte.sid_q]; 3401 if ((source_id & mask) != (sid & mask)) { 3402 error_report_once("%s: invalid IRTE SID " 3403 "(index=%u, sid=%u, source_id=%u)", 3404 __func__, index, sid, source_id); 3405 if (do_fault) { 3406 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3407 } 3408 return false; 3409 } 3410 break; 3411 3412 case VTD_SVT_BUS: 3413 bus_max = source_id >> 8; 3414 bus_min = source_id & 0xff; 3415 bus = sid >> 8; 3416 if (bus > bus_max || bus < bus_min) { 3417 error_report_once("%s: invalid SVT_BUS " 3418 "(index=%u, bus=%u, min=%u, max=%u)", 3419 __func__, index, bus, bus_min, bus_max); 3420 if (do_fault) { 3421 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3422 } 3423 return false; 3424 } 3425 break; 3426 3427 default: 3428 error_report_once("%s: detected invalid IRTE SVT " 3429 "(index=%u, type=%d)", __func__, 3430 index, entry->irte.sid_vtype); 3431 /* Take this as verification failure. */ 3432 if (do_fault) { 3433 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3434 } 3435 return false; 3436 } 3437 } 3438 3439 return true; 3440 } 3441 3442 /* Fetch IRQ information of specific IR index */ 3443 static bool vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index, 3444 X86IOMMUIrq *irq, uint16_t sid, bool do_fault) 3445 { 3446 VTD_IR_TableEntry irte = {}; 3447 3448 if (!vtd_irte_get(iommu, index, &irte, sid, do_fault)) { 3449 return false; 3450 } 3451 3452 irq->trigger_mode = irte.irte.trigger_mode; 3453 irq->vector = irte.irte.vector; 3454 irq->delivery_mode = irte.irte.delivery_mode; 3455 irq->dest = irte.irte.dest_id; 3456 if (!iommu->intr_eime) { 3457 #define VTD_IR_APIC_DEST_MASK (0xff00ULL) 3458 #define VTD_IR_APIC_DEST_SHIFT (8) 3459 irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >> 3460 VTD_IR_APIC_DEST_SHIFT; 3461 } 3462 irq->dest_mode = irte.irte.dest_mode; 3463 irq->redir_hint = irte.irte.redir_hint; 3464 3465 trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector, 3466 irq->delivery_mode, irq->dest, irq->dest_mode); 3467 3468 return true; 3469 } 3470 3471 /* Interrupt remapping for MSI/MSI-X entry */ 3472 static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu, 3473 MSIMessage *origin, 3474 MSIMessage *translated, 3475 uint16_t sid, bool do_fault) 3476 { 3477 VTD_IR_MSIAddress addr; 3478 uint16_t index; 3479 X86IOMMUIrq irq = {}; 3480 3481 assert(origin && translated); 3482 3483 trace_vtd_ir_remap_msi_req(origin->address, origin->data); 3484 3485 if (!iommu || !iommu->intr_enabled) { 3486 memcpy(translated, origin, sizeof(*origin)); 3487 goto out; 3488 } 3489 3490 if (origin->address & VTD_MSI_ADDR_HI_MASK) { 3491 error_report_once("%s: MSI address high 32 bits non-zero detected: " 3492 "address=0x%" PRIx64, __func__, origin->address); 3493 if (do_fault) { 3494 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 3495 } 3496 return -EINVAL; 3497 } 3498 3499 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK; 3500 if (addr.addr.__head != 0xfee) { 3501 error_report_once("%s: MSI address low 32 bit invalid: 0x%" PRIx32, 3502 __func__, addr.data); 3503 if (do_fault) { 3504 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 3505 } 3506 return -EINVAL; 3507 } 3508 3509 /* This is compatible mode. */ 3510 if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) { 3511 memcpy(translated, origin, sizeof(*origin)); 3512 goto out; 3513 } 3514 3515 index = addr.addr.index_h << 15 | addr.addr.index_l; 3516 3517 #define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff) 3518 #define VTD_IR_MSI_DATA_RESERVED (0xffff0000) 3519 3520 if (addr.addr.sub_valid) { 3521 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */ 3522 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE; 3523 } 3524 3525 if (!vtd_remap_irq_get(iommu, index, &irq, sid, do_fault)) { 3526 return -EINVAL; 3527 } 3528 3529 if (addr.addr.sub_valid) { 3530 trace_vtd_ir_remap_type("MSI"); 3531 if (origin->data & VTD_IR_MSI_DATA_RESERVED) { 3532 error_report_once("%s: invalid IR MSI " 3533 "(sid=%u, address=0x%" PRIx64 3534 ", data=0x%" PRIx32 ")", 3535 __func__, sid, origin->address, origin->data); 3536 if (do_fault) { 3537 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 3538 } 3539 return -EINVAL; 3540 } 3541 } else { 3542 uint8_t vector = origin->data & 0xff; 3543 uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1; 3544 3545 trace_vtd_ir_remap_type("IOAPIC"); 3546 /* IOAPIC entry vector should be aligned with IRTE vector 3547 * (see vt-d spec 5.1.5.1). */ 3548 if (vector != irq.vector) { 3549 trace_vtd_warn_ir_vector(sid, index, vector, irq.vector); 3550 } 3551 3552 /* The Trigger Mode field must match the Trigger Mode in the IRTE. 3553 * (see vt-d spec 5.1.5.1). */ 3554 if (trigger_mode != irq.trigger_mode) { 3555 trace_vtd_warn_ir_trigger(sid, index, trigger_mode, 3556 irq.trigger_mode); 3557 } 3558 } 3559 3560 /* 3561 * We'd better keep the last two bits, assuming that guest OS 3562 * might modify it. Keep it does not hurt after all. 3563 */ 3564 irq.msi_addr_last_bits = addr.addr.__not_care; 3565 3566 /* Translate X86IOMMUIrq to MSI message */ 3567 x86_iommu_irq_to_msi_message(&irq, translated); 3568 3569 out: 3570 trace_vtd_ir_remap_msi(origin->address, origin->data, 3571 translated->address, translated->data); 3572 return 0; 3573 } 3574 3575 static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src, 3576 MSIMessage *dst, uint16_t sid) 3577 { 3578 return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu), 3579 src, dst, sid, false); 3580 } 3581 3582 static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr, 3583 uint64_t *data, unsigned size, 3584 MemTxAttrs attrs) 3585 { 3586 return MEMTX_OK; 3587 } 3588 3589 static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr, 3590 uint64_t value, unsigned size, 3591 MemTxAttrs attrs) 3592 { 3593 int ret = 0; 3594 MSIMessage from = {}, to = {}; 3595 uint16_t sid = X86_IOMMU_SID_INVALID; 3596 3597 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST; 3598 from.data = (uint32_t) value; 3599 3600 if (!attrs.unspecified) { 3601 /* We have explicit Source ID */ 3602 sid = attrs.requester_id; 3603 } 3604 3605 ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid, true); 3606 if (ret) { 3607 /* Drop this interrupt */ 3608 return MEMTX_ERROR; 3609 } 3610 3611 apic_get_class(NULL)->send_msi(&to); 3612 3613 return MEMTX_OK; 3614 } 3615 3616 static const MemoryRegionOps vtd_mem_ir_ops = { 3617 .read_with_attrs = vtd_mem_ir_read, 3618 .write_with_attrs = vtd_mem_ir_write, 3619 .endianness = DEVICE_LITTLE_ENDIAN, 3620 .impl = { 3621 .min_access_size = 4, 3622 .max_access_size = 4, 3623 }, 3624 .valid = { 3625 .min_access_size = 4, 3626 .max_access_size = 4, 3627 }, 3628 }; 3629 3630 static void vtd_report_ir_illegal_access(VTDAddressSpace *vtd_as, 3631 hwaddr addr, bool is_write) 3632 { 3633 IntelIOMMUState *s = vtd_as->iommu_state; 3634 uint8_t bus_n = pci_bus_num(vtd_as->bus); 3635 uint16_t sid = PCI_BUILD_BDF(bus_n, vtd_as->devfn); 3636 bool is_fpd_set = false; 3637 VTDContextEntry ce; 3638 3639 assert(vtd_as->pasid != PCI_NO_PASID); 3640 3641 /* Try out best to fetch FPD, we can't do anything more */ 3642 if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 3643 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 3644 if (!is_fpd_set && s->root_scalable) { 3645 vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, vtd_as->pasid); 3646 } 3647 } 3648 3649 vtd_report_fault(s, VTD_FR_SM_INTERRUPT_ADDR, 3650 is_fpd_set, sid, addr, is_write, 3651 true, vtd_as->pasid); 3652 } 3653 3654 static MemTxResult vtd_mem_ir_fault_read(void *opaque, hwaddr addr, 3655 uint64_t *data, unsigned size, 3656 MemTxAttrs attrs) 3657 { 3658 vtd_report_ir_illegal_access(opaque, addr, false); 3659 3660 return MEMTX_ERROR; 3661 } 3662 3663 static MemTxResult vtd_mem_ir_fault_write(void *opaque, hwaddr addr, 3664 uint64_t value, unsigned size, 3665 MemTxAttrs attrs) 3666 { 3667 vtd_report_ir_illegal_access(opaque, addr, true); 3668 3669 return MEMTX_ERROR; 3670 } 3671 3672 static const MemoryRegionOps vtd_mem_ir_fault_ops = { 3673 .read_with_attrs = vtd_mem_ir_fault_read, 3674 .write_with_attrs = vtd_mem_ir_fault_write, 3675 .endianness = DEVICE_LITTLE_ENDIAN, 3676 .impl = { 3677 .min_access_size = 1, 3678 .max_access_size = 8, 3679 }, 3680 .valid = { 3681 .min_access_size = 1, 3682 .max_access_size = 8, 3683 }, 3684 }; 3685 3686 VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, 3687 int devfn, unsigned int pasid) 3688 { 3689 /* 3690 * We can't simply use sid here since the bus number might not be 3691 * initialized by the guest. 3692 */ 3693 struct vtd_as_key key = { 3694 .bus = bus, 3695 .devfn = devfn, 3696 .pasid = pasid, 3697 }; 3698 VTDAddressSpace *vtd_dev_as; 3699 char name[128]; 3700 3701 vtd_dev_as = g_hash_table_lookup(s->vtd_address_spaces, &key); 3702 if (!vtd_dev_as) { 3703 struct vtd_as_key *new_key = g_malloc(sizeof(*new_key)); 3704 3705 new_key->bus = bus; 3706 new_key->devfn = devfn; 3707 new_key->pasid = pasid; 3708 3709 if (pasid == PCI_NO_PASID) { 3710 snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn), 3711 PCI_FUNC(devfn)); 3712 } else { 3713 snprintf(name, sizeof(name), "vtd-%02x.%x-pasid-%x", PCI_SLOT(devfn), 3714 PCI_FUNC(devfn), pasid); 3715 } 3716 3717 vtd_dev_as = g_new0(VTDAddressSpace, 1); 3718 3719 vtd_dev_as->bus = bus; 3720 vtd_dev_as->devfn = (uint8_t)devfn; 3721 vtd_dev_as->pasid = pasid; 3722 vtd_dev_as->iommu_state = s; 3723 vtd_dev_as->context_cache_entry.context_cache_gen = 0; 3724 vtd_dev_as->iova_tree = iova_tree_new(); 3725 3726 memory_region_init(&vtd_dev_as->root, OBJECT(s), name, UINT64_MAX); 3727 address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, "vtd-root"); 3728 3729 /* 3730 * Build the DMAR-disabled container with aliases to the 3731 * shared MRs. Note that aliasing to a shared memory region 3732 * could help the memory API to detect same FlatViews so we 3733 * can have devices to share the same FlatView when DMAR is 3734 * disabled (either by not providing "intel_iommu=on" or with 3735 * "iommu=pt"). It will greatly reduce the total number of 3736 * FlatViews of the system hence VM runs faster. 3737 */ 3738 memory_region_init_alias(&vtd_dev_as->nodmar, OBJECT(s), 3739 "vtd-nodmar", &s->mr_nodmar, 0, 3740 memory_region_size(&s->mr_nodmar)); 3741 3742 /* 3743 * Build the per-device DMAR-enabled container. 3744 * 3745 * TODO: currently we have per-device IOMMU memory region only 3746 * because we have per-device IOMMU notifiers for devices. If 3747 * one day we can abstract the IOMMU notifiers out of the 3748 * memory regions then we can also share the same memory 3749 * region here just like what we've done above with the nodmar 3750 * region. 3751 */ 3752 strcat(name, "-dmar"); 3753 memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu), 3754 TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s), 3755 name, UINT64_MAX); 3756 memory_region_init_alias(&vtd_dev_as->iommu_ir, OBJECT(s), "vtd-ir", 3757 &s->mr_ir, 0, memory_region_size(&s->mr_ir)); 3758 memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->iommu), 3759 VTD_INTERRUPT_ADDR_FIRST, 3760 &vtd_dev_as->iommu_ir, 1); 3761 3762 /* 3763 * This region is used for catching fault to access interrupt 3764 * range via passthrough + PASID. See also 3765 * vtd_switch_address_space(). We can't use alias since we 3766 * need to know the sid which is valid for MSI who uses 3767 * bus_master_as (see msi_send_message()). 3768 */ 3769 memory_region_init_io(&vtd_dev_as->iommu_ir_fault, OBJECT(s), 3770 &vtd_mem_ir_fault_ops, vtd_dev_as, "vtd-no-ir", 3771 VTD_INTERRUPT_ADDR_SIZE); 3772 /* 3773 * Hook to root since when PT is enabled vtd_dev_as->iommu 3774 * will be disabled. 3775 */ 3776 memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->root), 3777 VTD_INTERRUPT_ADDR_FIRST, 3778 &vtd_dev_as->iommu_ir_fault, 2); 3779 3780 /* 3781 * Hook both the containers under the root container, we 3782 * switch between DMAR & noDMAR by enable/disable 3783 * corresponding sub-containers 3784 */ 3785 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 3786 MEMORY_REGION(&vtd_dev_as->iommu), 3787 0); 3788 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 3789 &vtd_dev_as->nodmar, 0); 3790 3791 vtd_switch_address_space(vtd_dev_as); 3792 3793 g_hash_table_insert(s->vtd_address_spaces, new_key, vtd_dev_as); 3794 } 3795 return vtd_dev_as; 3796 } 3797 3798 /* Unmap the whole range in the notifier's scope. */ 3799 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n) 3800 { 3801 hwaddr total, remain; 3802 hwaddr start = n->start; 3803 hwaddr end = n->end; 3804 IntelIOMMUState *s = as->iommu_state; 3805 DMAMap map; 3806 3807 /* 3808 * Note: all the codes in this function has a assumption that IOVA 3809 * bits are no more than VTD_MGAW bits (which is restricted by 3810 * VT-d spec), otherwise we need to consider overflow of 64 bits. 3811 */ 3812 3813 if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) { 3814 /* 3815 * Don't need to unmap regions that is bigger than the whole 3816 * VT-d supported address space size 3817 */ 3818 end = VTD_ADDRESS_SIZE(s->aw_bits) - 1; 3819 } 3820 3821 assert(start <= end); 3822 total = remain = end - start + 1; 3823 3824 while (remain >= VTD_PAGE_SIZE) { 3825 IOMMUTLBEvent event; 3826 uint64_t mask = dma_aligned_pow2_mask(start, end, s->aw_bits); 3827 uint64_t size = mask + 1; 3828 3829 assert(size); 3830 3831 event.type = IOMMU_NOTIFIER_UNMAP; 3832 event.entry.iova = start; 3833 event.entry.addr_mask = mask; 3834 event.entry.target_as = &address_space_memory; 3835 event.entry.perm = IOMMU_NONE; 3836 /* This field is meaningless for unmap */ 3837 event.entry.translated_addr = 0; 3838 3839 memory_region_notify_iommu_one(n, &event); 3840 3841 start += size; 3842 remain -= size; 3843 } 3844 3845 assert(!remain); 3846 3847 trace_vtd_as_unmap_whole(pci_bus_num(as->bus), 3848 VTD_PCI_SLOT(as->devfn), 3849 VTD_PCI_FUNC(as->devfn), 3850 n->start, total); 3851 3852 map.iova = n->start; 3853 map.size = total - 1; /* Inclusive */ 3854 iova_tree_remove(as->iova_tree, map); 3855 } 3856 3857 static void vtd_address_space_unmap_all(IntelIOMMUState *s) 3858 { 3859 VTDAddressSpace *vtd_as; 3860 IOMMUNotifier *n; 3861 3862 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 3863 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 3864 vtd_address_space_unmap(vtd_as, n); 3865 } 3866 } 3867 } 3868 3869 static void vtd_address_space_refresh_all(IntelIOMMUState *s) 3870 { 3871 vtd_address_space_unmap_all(s); 3872 vtd_switch_address_space_all(s); 3873 } 3874 3875 static int vtd_replay_hook(IOMMUTLBEvent *event, void *private) 3876 { 3877 memory_region_notify_iommu_one(private, event); 3878 return 0; 3879 } 3880 3881 static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) 3882 { 3883 VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu); 3884 IntelIOMMUState *s = vtd_as->iommu_state; 3885 uint8_t bus_n = pci_bus_num(vtd_as->bus); 3886 VTDContextEntry ce; 3887 DMAMap map = { .iova = 0, .size = HWADDR_MAX }; 3888 3889 /* replay is protected by BQL, page walk will re-setup it safely */ 3890 iova_tree_remove(vtd_as->iova_tree, map); 3891 3892 if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 3893 trace_vtd_replay_ce_valid(s->root_scalable ? "scalable mode" : 3894 "legacy mode", 3895 bus_n, PCI_SLOT(vtd_as->devfn), 3896 PCI_FUNC(vtd_as->devfn), 3897 vtd_get_domain_id(s, &ce, vtd_as->pasid), 3898 ce.hi, ce.lo); 3899 if (n->notifier_flags & IOMMU_NOTIFIER_MAP) { 3900 /* This is required only for MAP typed notifiers */ 3901 vtd_page_walk_info info = { 3902 .hook_fn = vtd_replay_hook, 3903 .private = (void *)n, 3904 .notify_unmap = false, 3905 .aw = s->aw_bits, 3906 .as = vtd_as, 3907 .domain_id = vtd_get_domain_id(s, &ce, vtd_as->pasid), 3908 }; 3909 3910 vtd_page_walk(s, &ce, 0, ~0ULL, &info, vtd_as->pasid); 3911 } 3912 } else { 3913 trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn), 3914 PCI_FUNC(vtd_as->devfn)); 3915 } 3916 3917 return; 3918 } 3919 3920 /* Do the initialization. It will also be called when reset, so pay 3921 * attention when adding new initialization stuff. 3922 */ 3923 static void vtd_init(IntelIOMMUState *s) 3924 { 3925 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3926 3927 memset(s->csr, 0, DMAR_REG_SIZE); 3928 memset(s->wmask, 0, DMAR_REG_SIZE); 3929 memset(s->w1cmask, 0, DMAR_REG_SIZE); 3930 memset(s->womask, 0, DMAR_REG_SIZE); 3931 3932 s->root = 0; 3933 s->root_scalable = false; 3934 s->dmar_enabled = false; 3935 s->intr_enabled = false; 3936 s->iq_head = 0; 3937 s->iq_tail = 0; 3938 s->iq = 0; 3939 s->iq_size = 0; 3940 s->qi_enabled = false; 3941 s->iq_last_desc_type = VTD_INV_DESC_NONE; 3942 s->iq_dw = false; 3943 s->next_frcd_reg = 0; 3944 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | 3945 VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS | 3946 VTD_CAP_MGAW(s->aw_bits); 3947 if (s->dma_drain) { 3948 s->cap |= VTD_CAP_DRAIN; 3949 } 3950 if (s->dma_translation) { 3951 if (s->aw_bits >= VTD_HOST_AW_39BIT) { 3952 s->cap |= VTD_CAP_SAGAW_39bit; 3953 } 3954 if (s->aw_bits >= VTD_HOST_AW_48BIT) { 3955 s->cap |= VTD_CAP_SAGAW_48bit; 3956 } 3957 } 3958 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 3959 3960 /* 3961 * Rsvd field masks for spte 3962 */ 3963 vtd_spte_rsvd[0] = ~0ULL; 3964 vtd_spte_rsvd[1] = VTD_SPTE_PAGE_L1_RSVD_MASK(s->aw_bits, 3965 x86_iommu->dt_supported); 3966 vtd_spte_rsvd[2] = VTD_SPTE_PAGE_L2_RSVD_MASK(s->aw_bits); 3967 vtd_spte_rsvd[3] = VTD_SPTE_PAGE_L3_RSVD_MASK(s->aw_bits); 3968 vtd_spte_rsvd[4] = VTD_SPTE_PAGE_L4_RSVD_MASK(s->aw_bits); 3969 3970 vtd_spte_rsvd_large[2] = VTD_SPTE_LPAGE_L2_RSVD_MASK(s->aw_bits, 3971 x86_iommu->dt_supported); 3972 vtd_spte_rsvd_large[3] = VTD_SPTE_LPAGE_L3_RSVD_MASK(s->aw_bits, 3973 x86_iommu->dt_supported); 3974 3975 if (s->scalable_mode || s->snoop_control) { 3976 vtd_spte_rsvd[1] &= ~VTD_SPTE_SNP; 3977 vtd_spte_rsvd_large[2] &= ~VTD_SPTE_SNP; 3978 vtd_spte_rsvd_large[3] &= ~VTD_SPTE_SNP; 3979 } 3980 3981 if (x86_iommu_ir_supported(x86_iommu)) { 3982 s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV; 3983 if (s->intr_eim == ON_OFF_AUTO_ON) { 3984 s->ecap |= VTD_ECAP_EIM; 3985 } 3986 assert(s->intr_eim != ON_OFF_AUTO_AUTO); 3987 } 3988 3989 if (x86_iommu->dt_supported) { 3990 s->ecap |= VTD_ECAP_DT; 3991 } 3992 3993 if (x86_iommu->pt_supported) { 3994 s->ecap |= VTD_ECAP_PT; 3995 } 3996 3997 if (s->caching_mode) { 3998 s->cap |= VTD_CAP_CM; 3999 } 4000 4001 /* TODO: read cap/ecap from host to decide which cap to be exposed. */ 4002 if (s->scalable_mode) { 4003 s->ecap |= VTD_ECAP_SMTS | VTD_ECAP_SRS | VTD_ECAP_SLTS; 4004 } 4005 4006 if (s->snoop_control) { 4007 s->ecap |= VTD_ECAP_SC; 4008 } 4009 4010 if (s->pasid) { 4011 s->ecap |= VTD_ECAP_PASID; 4012 } 4013 4014 vtd_reset_caches(s); 4015 4016 /* Define registers with default values and bit semantics */ 4017 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 4018 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 4019 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 4020 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 4021 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 4022 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 4023 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffffc00ULL, 0); 4024 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 4025 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 4026 4027 /* Advanced Fault Logging not supported */ 4028 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 4029 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 4030 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 4031 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 4032 4033 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 4034 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 4035 */ 4036 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 4037 4038 /* Treated as RO for implementations that PLMR and PHMR fields reported 4039 * as Clear in the CAP_REG. 4040 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 4041 */ 4042 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 4043 4044 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 4045 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 4046 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff807ULL, 0); 4047 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 4048 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 4049 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 4050 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 4051 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 4052 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 4053 4054 /* IOTLB registers */ 4055 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 4056 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 4057 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 4058 4059 /* Fault Recording Registers, 128-bit */ 4060 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 4061 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 4062 4063 /* 4064 * Interrupt remapping registers. 4065 */ 4066 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0); 4067 } 4068 4069 /* Should not reset address_spaces when reset because devices will still use 4070 * the address space they got at first (won't ask the bus again). 4071 */ 4072 static void vtd_reset(DeviceState *dev) 4073 { 4074 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 4075 4076 vtd_init(s); 4077 vtd_address_space_refresh_all(s); 4078 } 4079 4080 static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) 4081 { 4082 IntelIOMMUState *s = opaque; 4083 VTDAddressSpace *vtd_as; 4084 4085 assert(0 <= devfn && devfn < PCI_DEVFN_MAX); 4086 4087 vtd_as = vtd_find_add_as(s, bus, devfn, PCI_NO_PASID); 4088 return &vtd_as->as; 4089 } 4090 4091 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) 4092 { 4093 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 4094 4095 if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu_ir_supported(x86_iommu)) { 4096 error_setg(errp, "eim=on cannot be selected without intremap=on"); 4097 return false; 4098 } 4099 4100 if (s->intr_eim == ON_OFF_AUTO_AUTO) { 4101 s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim) 4102 && x86_iommu_ir_supported(x86_iommu) ? 4103 ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 4104 } 4105 if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) { 4106 if (!kvm_irqchip_is_split()) { 4107 error_setg(errp, "eim=on requires accel=kvm,kernel-irqchip=split"); 4108 return false; 4109 } 4110 if (kvm_enabled() && !kvm_enable_x2apic()) { 4111 error_setg(errp, "eim=on requires support on the KVM side" 4112 "(X2APIC_API, first shipped in v4.7)"); 4113 return false; 4114 } 4115 } 4116 4117 /* Currently only address widths supported are 39 and 48 bits */ 4118 if ((s->aw_bits != VTD_HOST_AW_39BIT) && 4119 (s->aw_bits != VTD_HOST_AW_48BIT)) { 4120 error_setg(errp, "Supported values for aw-bits are: %d, %d", 4121 VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT); 4122 return false; 4123 } 4124 4125 if (s->scalable_mode && !s->dma_drain) { 4126 error_setg(errp, "Need to set dma_drain for scalable mode"); 4127 return false; 4128 } 4129 4130 if (s->pasid && !s->scalable_mode) { 4131 error_setg(errp, "Need to set scalable mode for PASID"); 4132 return false; 4133 } 4134 4135 return true; 4136 } 4137 4138 static int vtd_machine_done_notify_one(Object *child, void *unused) 4139 { 4140 IntelIOMMUState *iommu = INTEL_IOMMU_DEVICE(x86_iommu_get_default()); 4141 4142 /* 4143 * We hard-coded here because vfio-pci is the only special case 4144 * here. Let's be more elegant in the future when we can, but so 4145 * far there seems to be no better way. 4146 */ 4147 if (object_dynamic_cast(child, "vfio-pci") && !iommu->caching_mode) { 4148 vtd_panic_require_caching_mode(); 4149 } 4150 4151 return 0; 4152 } 4153 4154 static void vtd_machine_done_hook(Notifier *notifier, void *unused) 4155 { 4156 object_child_foreach_recursive(object_get_root(), 4157 vtd_machine_done_notify_one, NULL); 4158 } 4159 4160 static Notifier vtd_machine_done_notify = { 4161 .notify = vtd_machine_done_hook, 4162 }; 4163 4164 static void vtd_realize(DeviceState *dev, Error **errp) 4165 { 4166 MachineState *ms = MACHINE(qdev_get_machine()); 4167 PCMachineState *pcms = PC_MACHINE(ms); 4168 X86MachineState *x86ms = X86_MACHINE(ms); 4169 PCIBus *bus = pcms->bus; 4170 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 4171 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 4172 4173 if (s->pasid && x86_iommu->dt_supported) { 4174 /* 4175 * PASID-based-Device-TLB Invalidate Descriptor is not 4176 * implemented and it requires support from vhost layer which 4177 * needs to be implemented in the future. 4178 */ 4179 error_setg(errp, "PASID based device IOTLB is not supported"); 4180 return; 4181 } 4182 4183 if (!vtd_decide_config(s, errp)) { 4184 return; 4185 } 4186 4187 QLIST_INIT(&s->vtd_as_with_notifiers); 4188 qemu_mutex_init(&s->iommu_lock); 4189 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 4190 "intel_iommu", DMAR_REG_SIZE); 4191 memory_region_add_subregion(get_system_memory(), 4192 Q35_HOST_BRIDGE_IOMMU_ADDR, &s->csrmem); 4193 4194 /* Create the shared memory regions by all devices */ 4195 memory_region_init(&s->mr_nodmar, OBJECT(s), "vtd-nodmar", 4196 UINT64_MAX); 4197 memory_region_init_io(&s->mr_ir, OBJECT(s), &vtd_mem_ir_ops, 4198 s, "vtd-ir", VTD_INTERRUPT_ADDR_SIZE); 4199 memory_region_init_alias(&s->mr_sys_alias, OBJECT(s), 4200 "vtd-sys-alias", get_system_memory(), 0, 4201 memory_region_size(get_system_memory())); 4202 memory_region_add_subregion_overlap(&s->mr_nodmar, 0, 4203 &s->mr_sys_alias, 0); 4204 memory_region_add_subregion_overlap(&s->mr_nodmar, 4205 VTD_INTERRUPT_ADDR_FIRST, 4206 &s->mr_ir, 1); 4207 /* No corresponding destroy */ 4208 s->iotlb = g_hash_table_new_full(vtd_iotlb_hash, vtd_iotlb_equal, 4209 g_free, g_free); 4210 s->vtd_address_spaces = g_hash_table_new_full(vtd_as_hash, vtd_as_equal, 4211 g_free, g_free); 4212 vtd_init(s); 4213 pci_setup_iommu(bus, vtd_host_dma_iommu, dev); 4214 /* Pseudo address space under root PCI bus. */ 4215 x86ms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC); 4216 qemu_add_machine_init_done_notifier(&vtd_machine_done_notify); 4217 } 4218 4219 static void vtd_class_init(ObjectClass *klass, void *data) 4220 { 4221 DeviceClass *dc = DEVICE_CLASS(klass); 4222 X86IOMMUClass *x86_class = X86_IOMMU_DEVICE_CLASS(klass); 4223 4224 dc->reset = vtd_reset; 4225 dc->vmsd = &vtd_vmstate; 4226 device_class_set_props(dc, vtd_properties); 4227 dc->hotpluggable = false; 4228 x86_class->realize = vtd_realize; 4229 x86_class->int_remap = vtd_int_remap; 4230 /* Supported by the pc-q35-* machine types */ 4231 dc->user_creatable = true; 4232 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 4233 dc->desc = "Intel IOMMU (VT-d) DMA Remapping device"; 4234 } 4235 4236 static const TypeInfo vtd_info = { 4237 .name = TYPE_INTEL_IOMMU_DEVICE, 4238 .parent = TYPE_X86_IOMMU_DEVICE, 4239 .instance_size = sizeof(IntelIOMMUState), 4240 .class_init = vtd_class_init, 4241 }; 4242 4243 static void vtd_iommu_memory_region_class_init(ObjectClass *klass, 4244 void *data) 4245 { 4246 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 4247 4248 imrc->translate = vtd_iommu_translate; 4249 imrc->notify_flag_changed = vtd_iommu_notify_flag_changed; 4250 imrc->replay = vtd_iommu_replay; 4251 } 4252 4253 static const TypeInfo vtd_iommu_memory_region_info = { 4254 .parent = TYPE_IOMMU_MEMORY_REGION, 4255 .name = TYPE_INTEL_IOMMU_MEMORY_REGION, 4256 .class_init = vtd_iommu_memory_region_class_init, 4257 }; 4258 4259 static void vtd_register_types(void) 4260 { 4261 type_register_static(&vtd_info); 4262 type_register_static(&vtd_iommu_memory_region_info); 4263 } 4264 4265 type_init(vtd_register_types) 4266