1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "qemu/osdep.h" 23 #include "qemu/error-report.h" 24 #include "qemu/main-loop.h" 25 #include "qapi/error.h" 26 #include "hw/sysbus.h" 27 #include "intel_iommu_internal.h" 28 #include "hw/pci/pci.h" 29 #include "hw/pci/pci_bus.h" 30 #include "hw/qdev-properties.h" 31 #include "hw/i386/pc.h" 32 #include "hw/i386/apic-msidef.h" 33 #include "hw/i386/x86-iommu.h" 34 #include "hw/pci-host/q35.h" 35 #include "sysemu/kvm.h" 36 #include "sysemu/dma.h" 37 #include "sysemu/sysemu.h" 38 #include "hw/i386/apic_internal.h" 39 #include "kvm/kvm_i386.h" 40 #include "migration/vmstate.h" 41 #include "trace.h" 42 43 /* context entry operations */ 44 #define VTD_CE_GET_RID2PASID(ce) \ 45 ((ce)->val[1] & VTD_SM_CONTEXT_ENTRY_RID2PASID_MASK) 46 #define VTD_CE_GET_PASID_DIR_TABLE(ce) \ 47 ((ce)->val[0] & VTD_PASID_DIR_BASE_ADDR_MASK) 48 49 /* pe operations */ 50 #define VTD_PE_GET_TYPE(pe) ((pe)->val[0] & VTD_SM_PASID_ENTRY_PGTT) 51 #define VTD_PE_GET_LEVEL(pe) (2 + (((pe)->val[0] >> 2) & VTD_SM_PASID_ENTRY_AW)) 52 53 /* 54 * PCI bus number (or SID) is not reliable since the device is usaully 55 * initialized before guest can configure the PCI bridge 56 * (SECONDARY_BUS_NUMBER). 57 */ 58 struct vtd_as_key { 59 PCIBus *bus; 60 uint8_t devfn; 61 uint32_t pasid; 62 }; 63 64 struct vtd_iotlb_key { 65 uint64_t gfn; 66 uint32_t pasid; 67 uint16_t sid; 68 uint8_t level; 69 }; 70 71 static void vtd_address_space_refresh_all(IntelIOMMUState *s); 72 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n); 73 74 static void vtd_panic_require_caching_mode(void) 75 { 76 error_report("We need to set caching-mode=on for intel-iommu to enable " 77 "device assignment with IOMMU protection."); 78 exit(1); 79 } 80 81 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 82 uint64_t wmask, uint64_t w1cmask) 83 { 84 stq_le_p(&s->csr[addr], val); 85 stq_le_p(&s->wmask[addr], wmask); 86 stq_le_p(&s->w1cmask[addr], w1cmask); 87 } 88 89 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 90 { 91 stq_le_p(&s->womask[addr], mask); 92 } 93 94 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 95 uint32_t wmask, uint32_t w1cmask) 96 { 97 stl_le_p(&s->csr[addr], val); 98 stl_le_p(&s->wmask[addr], wmask); 99 stl_le_p(&s->w1cmask[addr], w1cmask); 100 } 101 102 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 103 { 104 stl_le_p(&s->womask[addr], mask); 105 } 106 107 /* "External" get/set operations */ 108 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 109 { 110 uint64_t oldval = ldq_le_p(&s->csr[addr]); 111 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 112 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 113 stq_le_p(&s->csr[addr], 114 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 115 } 116 117 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 118 { 119 uint32_t oldval = ldl_le_p(&s->csr[addr]); 120 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 121 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 122 stl_le_p(&s->csr[addr], 123 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 124 } 125 126 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 127 { 128 uint64_t val = ldq_le_p(&s->csr[addr]); 129 uint64_t womask = ldq_le_p(&s->womask[addr]); 130 return val & ~womask; 131 } 132 133 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 134 { 135 uint32_t val = ldl_le_p(&s->csr[addr]); 136 uint32_t womask = ldl_le_p(&s->womask[addr]); 137 return val & ~womask; 138 } 139 140 /* "Internal" get/set operations */ 141 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 142 { 143 return ldq_le_p(&s->csr[addr]); 144 } 145 146 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 147 { 148 return ldl_le_p(&s->csr[addr]); 149 } 150 151 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 152 { 153 stq_le_p(&s->csr[addr], val); 154 } 155 156 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 157 uint32_t clear, uint32_t mask) 158 { 159 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 160 stl_le_p(&s->csr[addr], new_val); 161 return new_val; 162 } 163 164 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 165 uint64_t clear, uint64_t mask) 166 { 167 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 168 stq_le_p(&s->csr[addr], new_val); 169 return new_val; 170 } 171 172 static inline void vtd_iommu_lock(IntelIOMMUState *s) 173 { 174 qemu_mutex_lock(&s->iommu_lock); 175 } 176 177 static inline void vtd_iommu_unlock(IntelIOMMUState *s) 178 { 179 qemu_mutex_unlock(&s->iommu_lock); 180 } 181 182 static void vtd_update_scalable_state(IntelIOMMUState *s) 183 { 184 uint64_t val = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 185 186 if (s->scalable_mode) { 187 s->root_scalable = val & VTD_RTADDR_SMT; 188 } 189 } 190 191 static void vtd_update_iq_dw(IntelIOMMUState *s) 192 { 193 uint64_t val = vtd_get_quad_raw(s, DMAR_IQA_REG); 194 195 if (s->ecap & VTD_ECAP_SMTS && 196 val & VTD_IQA_DW_MASK) { 197 s->iq_dw = true; 198 } else { 199 s->iq_dw = false; 200 } 201 } 202 203 /* Whether the address space needs to notify new mappings */ 204 static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as) 205 { 206 return as->notifier_flags & IOMMU_NOTIFIER_MAP; 207 } 208 209 /* GHashTable functions */ 210 static gboolean vtd_iotlb_equal(gconstpointer v1, gconstpointer v2) 211 { 212 const struct vtd_iotlb_key *key1 = v1; 213 const struct vtd_iotlb_key *key2 = v2; 214 215 return key1->sid == key2->sid && 216 key1->pasid == key2->pasid && 217 key1->level == key2->level && 218 key1->gfn == key2->gfn; 219 } 220 221 static guint vtd_iotlb_hash(gconstpointer v) 222 { 223 const struct vtd_iotlb_key *key = v; 224 uint64_t hash64 = key->gfn | ((uint64_t)(key->sid) << VTD_IOTLB_SID_SHIFT) | 225 (uint64_t)(key->level - 1) << VTD_IOTLB_LVL_SHIFT | 226 (uint64_t)(key->pasid) << VTD_IOTLB_PASID_SHIFT; 227 228 return (guint)((hash64 >> 32) ^ (hash64 & 0xffffffffU)); 229 } 230 231 static gboolean vtd_as_equal(gconstpointer v1, gconstpointer v2) 232 { 233 const struct vtd_as_key *key1 = v1; 234 const struct vtd_as_key *key2 = v2; 235 236 return (key1->bus == key2->bus) && (key1->devfn == key2->devfn) && 237 (key1->pasid == key2->pasid); 238 } 239 240 /* 241 * Note that we use pointer to PCIBus as the key, so hashing/shifting 242 * based on the pointer value is intended. Note that we deal with 243 * collisions through vtd_as_equal(). 244 */ 245 static guint vtd_as_hash(gconstpointer v) 246 { 247 const struct vtd_as_key *key = v; 248 guint value = (guint)(uintptr_t)key->bus; 249 250 return (guint)(value << 8 | key->devfn); 251 } 252 253 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 254 gpointer user_data) 255 { 256 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 257 uint16_t domain_id = *(uint16_t *)user_data; 258 return entry->domain_id == domain_id; 259 } 260 261 /* The shift of an addr for a certain level of paging structure */ 262 static inline uint32_t vtd_slpt_level_shift(uint32_t level) 263 { 264 assert(level != 0); 265 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 266 } 267 268 static inline uint64_t vtd_slpt_level_page_mask(uint32_t level) 269 { 270 return ~((1ULL << vtd_slpt_level_shift(level)) - 1); 271 } 272 273 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 274 gpointer user_data) 275 { 276 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 277 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 278 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask; 279 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K; 280 return (entry->domain_id == info->domain_id) && 281 (((entry->gfn & info->mask) == gfn) || 282 (entry->gfn == gfn_tlb)); 283 } 284 285 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 286 * IntelIOMMUState to 1. Must be called with IOMMU lock held. 287 */ 288 static void vtd_reset_context_cache_locked(IntelIOMMUState *s) 289 { 290 VTDAddressSpace *vtd_as; 291 GHashTableIter as_it; 292 293 trace_vtd_context_cache_reset(); 294 295 g_hash_table_iter_init(&as_it, s->vtd_address_spaces); 296 297 while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) { 298 vtd_as->context_cache_entry.context_cache_gen = 0; 299 } 300 s->context_cache_gen = 1; 301 } 302 303 /* Must be called with IOMMU lock held. */ 304 static void vtd_reset_iotlb_locked(IntelIOMMUState *s) 305 { 306 assert(s->iotlb); 307 g_hash_table_remove_all(s->iotlb); 308 } 309 310 static void vtd_reset_iotlb(IntelIOMMUState *s) 311 { 312 vtd_iommu_lock(s); 313 vtd_reset_iotlb_locked(s); 314 vtd_iommu_unlock(s); 315 } 316 317 static void vtd_reset_caches(IntelIOMMUState *s) 318 { 319 vtd_iommu_lock(s); 320 vtd_reset_iotlb_locked(s); 321 vtd_reset_context_cache_locked(s); 322 vtd_iommu_unlock(s); 323 } 324 325 static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) 326 { 327 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K; 328 } 329 330 /* Must be called with IOMMU lock held */ 331 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 332 uint32_t pasid, hwaddr addr) 333 { 334 struct vtd_iotlb_key key; 335 VTDIOTLBEntry *entry; 336 int level; 337 338 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) { 339 key.gfn = vtd_get_iotlb_gfn(addr, level); 340 key.level = level; 341 key.sid = source_id; 342 key.pasid = pasid; 343 entry = g_hash_table_lookup(s->iotlb, &key); 344 if (entry) { 345 goto out; 346 } 347 } 348 349 out: 350 return entry; 351 } 352 353 /* Must be with IOMMU lock held */ 354 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 355 uint16_t domain_id, hwaddr addr, uint64_t slpte, 356 uint8_t access_flags, uint32_t level, 357 uint32_t pasid) 358 { 359 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 360 struct vtd_iotlb_key *key = g_malloc(sizeof(*key)); 361 uint64_t gfn = vtd_get_iotlb_gfn(addr, level); 362 363 trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id); 364 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 365 trace_vtd_iotlb_reset("iotlb exceeds size limit"); 366 vtd_reset_iotlb_locked(s); 367 } 368 369 entry->gfn = gfn; 370 entry->domain_id = domain_id; 371 entry->slpte = slpte; 372 entry->access_flags = access_flags; 373 entry->mask = vtd_slpt_level_page_mask(level); 374 entry->pasid = pasid; 375 376 key->gfn = gfn; 377 key->sid = source_id; 378 key->level = level; 379 key->pasid = pasid; 380 381 g_hash_table_replace(s->iotlb, key, entry); 382 } 383 384 /* Given the reg addr of both the message data and address, generate an 385 * interrupt via MSI. 386 */ 387 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 388 hwaddr mesg_data_reg) 389 { 390 MSIMessage msi; 391 392 assert(mesg_data_reg < DMAR_REG_SIZE); 393 assert(mesg_addr_reg < DMAR_REG_SIZE); 394 395 msi.address = vtd_get_long_raw(s, mesg_addr_reg); 396 msi.data = vtd_get_long_raw(s, mesg_data_reg); 397 398 trace_vtd_irq_generate(msi.address, msi.data); 399 400 apic_get_class(NULL)->send_msi(&msi); 401 } 402 403 /* Generate a fault event to software via MSI if conditions are met. 404 * Notice that the value of FSTS_REG being passed to it should be the one 405 * before any update. 406 */ 407 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 408 { 409 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 410 pre_fsts & VTD_FSTS_IQE) { 411 error_report_once("There are previous interrupt conditions " 412 "to be serviced by software, fault event " 413 "is not generated"); 414 return; 415 } 416 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 417 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 418 error_report_once("Interrupt Mask set, irq is not generated"); 419 } else { 420 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 421 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 422 } 423 } 424 425 /* Check if the Fault (F) field of the Fault Recording Register referenced by 426 * @index is Set. 427 */ 428 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 429 { 430 /* Each reg is 128-bit */ 431 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 432 addr += 8; /* Access the high 64-bit half */ 433 434 assert(index < DMAR_FRCD_REG_NR); 435 436 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 437 } 438 439 /* Update the PPF field of Fault Status Register. 440 * Should be called whenever change the F field of any fault recording 441 * registers. 442 */ 443 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 444 { 445 uint32_t i; 446 uint32_t ppf_mask = 0; 447 448 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 449 if (vtd_is_frcd_set(s, i)) { 450 ppf_mask = VTD_FSTS_PPF; 451 break; 452 } 453 } 454 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 455 trace_vtd_fsts_ppf(!!ppf_mask); 456 } 457 458 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 459 { 460 /* Each reg is 128-bit */ 461 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 462 addr += 8; /* Access the high 64-bit half */ 463 464 assert(index < DMAR_FRCD_REG_NR); 465 466 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 467 vtd_update_fsts_ppf(s); 468 } 469 470 /* Must not update F field now, should be done later */ 471 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 472 uint64_t hi, uint64_t lo) 473 { 474 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 475 476 assert(index < DMAR_FRCD_REG_NR); 477 478 vtd_set_quad_raw(s, frcd_reg_addr, lo); 479 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 480 481 trace_vtd_frr_new(index, hi, lo); 482 } 483 484 /* Try to collapse multiple pending faults from the same requester */ 485 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 486 { 487 uint32_t i; 488 uint64_t frcd_reg; 489 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 490 491 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 492 frcd_reg = vtd_get_quad_raw(s, addr); 493 if ((frcd_reg & VTD_FRCD_F) && 494 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 495 return true; 496 } 497 addr += 16; /* 128-bit for each */ 498 } 499 return false; 500 } 501 502 /* Log and report an DMAR (address translation) fault to software */ 503 static void vtd_report_frcd_fault(IntelIOMMUState *s, uint64_t source_id, 504 uint64_t hi, uint64_t lo) 505 { 506 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 507 508 if (fsts_reg & VTD_FSTS_PFO) { 509 error_report_once("New fault is not recorded due to " 510 "Primary Fault Overflow"); 511 return; 512 } 513 514 if (vtd_try_collapse_fault(s, source_id)) { 515 error_report_once("New fault is not recorded due to " 516 "compression of faults"); 517 return; 518 } 519 520 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 521 error_report_once("Next Fault Recording Reg is used, " 522 "new fault is not recorded, set PFO field"); 523 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 524 return; 525 } 526 527 vtd_record_frcd(s, s->next_frcd_reg, hi, lo); 528 529 if (fsts_reg & VTD_FSTS_PPF) { 530 error_report_once("There are pending faults already, " 531 "fault event is not generated"); 532 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 533 s->next_frcd_reg++; 534 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 535 s->next_frcd_reg = 0; 536 } 537 } else { 538 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 539 VTD_FSTS_FRI(s->next_frcd_reg)); 540 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 541 s->next_frcd_reg++; 542 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 543 s->next_frcd_reg = 0; 544 } 545 /* This case actually cause the PPF to be Set. 546 * So generate fault event (interrupt). 547 */ 548 vtd_generate_fault_event(s, fsts_reg); 549 } 550 } 551 552 /* Log and report an DMAR (address translation) fault to software */ 553 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 554 hwaddr addr, VTDFaultReason fault, 555 bool is_write, bool is_pasid, 556 uint32_t pasid) 557 { 558 uint64_t hi, lo; 559 560 assert(fault < VTD_FR_MAX); 561 562 trace_vtd_dmar_fault(source_id, fault, addr, is_write); 563 564 lo = VTD_FRCD_FI(addr); 565 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault) | 566 VTD_FRCD_PV(pasid) | VTD_FRCD_PP(is_pasid); 567 if (!is_write) { 568 hi |= VTD_FRCD_T; 569 } 570 571 vtd_report_frcd_fault(s, source_id, hi, lo); 572 } 573 574 575 static void vtd_report_ir_fault(IntelIOMMUState *s, uint64_t source_id, 576 VTDFaultReason fault, uint16_t index) 577 { 578 uint64_t hi, lo; 579 580 lo = VTD_FRCD_IR_IDX(index); 581 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 582 583 vtd_report_frcd_fault(s, source_id, hi, lo); 584 } 585 586 /* Handle Invalidation Queue Errors of queued invalidation interface error 587 * conditions. 588 */ 589 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 590 { 591 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 592 593 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 594 vtd_generate_fault_event(s, fsts_reg); 595 } 596 597 /* Set the IWC field and try to generate an invalidation completion interrupt */ 598 static void vtd_generate_completion_event(IntelIOMMUState *s) 599 { 600 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 601 trace_vtd_inv_desc_wait_irq("One pending, skip current"); 602 return; 603 } 604 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 605 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 606 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 607 trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, " 608 "new event not generated"); 609 return; 610 } else { 611 /* Generate the interrupt event */ 612 trace_vtd_inv_desc_wait_irq("Generating complete event"); 613 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 614 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 615 } 616 } 617 618 static inline bool vtd_root_entry_present(IntelIOMMUState *s, 619 VTDRootEntry *re, 620 uint8_t devfn) 621 { 622 if (s->root_scalable && devfn > UINT8_MAX / 2) { 623 return re->hi & VTD_ROOT_ENTRY_P; 624 } 625 626 return re->lo & VTD_ROOT_ENTRY_P; 627 } 628 629 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 630 VTDRootEntry *re) 631 { 632 dma_addr_t addr; 633 634 addr = s->root + index * sizeof(*re); 635 if (dma_memory_read(&address_space_memory, addr, 636 re, sizeof(*re), MEMTXATTRS_UNSPECIFIED)) { 637 re->lo = 0; 638 return -VTD_FR_ROOT_TABLE_INV; 639 } 640 re->lo = le64_to_cpu(re->lo); 641 re->hi = le64_to_cpu(re->hi); 642 return 0; 643 } 644 645 static inline bool vtd_ce_present(VTDContextEntry *context) 646 { 647 return context->lo & VTD_CONTEXT_ENTRY_P; 648 } 649 650 static int vtd_get_context_entry_from_root(IntelIOMMUState *s, 651 VTDRootEntry *re, 652 uint8_t index, 653 VTDContextEntry *ce) 654 { 655 dma_addr_t addr, ce_size; 656 657 /* we have checked that root entry is present */ 658 ce_size = s->root_scalable ? VTD_CTX_ENTRY_SCALABLE_SIZE : 659 VTD_CTX_ENTRY_LEGACY_SIZE; 660 661 if (s->root_scalable && index > UINT8_MAX / 2) { 662 index = index & (~VTD_DEVFN_CHECK_MASK); 663 addr = re->hi & VTD_ROOT_ENTRY_CTP; 664 } else { 665 addr = re->lo & VTD_ROOT_ENTRY_CTP; 666 } 667 668 addr = addr + index * ce_size; 669 if (dma_memory_read(&address_space_memory, addr, 670 ce, ce_size, MEMTXATTRS_UNSPECIFIED)) { 671 return -VTD_FR_CONTEXT_TABLE_INV; 672 } 673 674 ce->lo = le64_to_cpu(ce->lo); 675 ce->hi = le64_to_cpu(ce->hi); 676 if (ce_size == VTD_CTX_ENTRY_SCALABLE_SIZE) { 677 ce->val[2] = le64_to_cpu(ce->val[2]); 678 ce->val[3] = le64_to_cpu(ce->val[3]); 679 } 680 return 0; 681 } 682 683 static inline dma_addr_t vtd_ce_get_slpt_base(VTDContextEntry *ce) 684 { 685 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 686 } 687 688 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte, uint8_t aw) 689 { 690 return slpte & VTD_SL_PT_BASE_ADDR_MASK(aw); 691 } 692 693 /* Whether the pte indicates the address of the page frame */ 694 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 695 { 696 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 697 } 698 699 /* Get the content of a spte located in @base_addr[@index] */ 700 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 701 { 702 uint64_t slpte; 703 704 assert(index < VTD_SL_PT_ENTRY_NR); 705 706 if (dma_memory_read(&address_space_memory, 707 base_addr + index * sizeof(slpte), 708 &slpte, sizeof(slpte), MEMTXATTRS_UNSPECIFIED)) { 709 slpte = (uint64_t)-1; 710 return slpte; 711 } 712 slpte = le64_to_cpu(slpte); 713 return slpte; 714 } 715 716 /* Given an iova and the level of paging structure, return the offset 717 * of current level. 718 */ 719 static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level) 720 { 721 return (iova >> vtd_slpt_level_shift(level)) & 722 ((1ULL << VTD_SL_LEVEL_BITS) - 1); 723 } 724 725 /* Check Capability Register to see if the @level of page-table is supported */ 726 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 727 { 728 return VTD_CAP_SAGAW_MASK & s->cap & 729 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 730 } 731 732 /* Return true if check passed, otherwise false */ 733 static inline bool vtd_pe_type_check(X86IOMMUState *x86_iommu, 734 VTDPASIDEntry *pe) 735 { 736 switch (VTD_PE_GET_TYPE(pe)) { 737 case VTD_SM_PASID_ENTRY_FLT: 738 case VTD_SM_PASID_ENTRY_SLT: 739 case VTD_SM_PASID_ENTRY_NESTED: 740 break; 741 case VTD_SM_PASID_ENTRY_PT: 742 if (!x86_iommu->pt_supported) { 743 return false; 744 } 745 break; 746 default: 747 /* Unknown type */ 748 return false; 749 } 750 return true; 751 } 752 753 static inline bool vtd_pdire_present(VTDPASIDDirEntry *pdire) 754 { 755 return pdire->val & 1; 756 } 757 758 /** 759 * Caller of this function should check present bit if wants 760 * to use pdir entry for further usage except for fpd bit check. 761 */ 762 static int vtd_get_pdire_from_pdir_table(dma_addr_t pasid_dir_base, 763 uint32_t pasid, 764 VTDPASIDDirEntry *pdire) 765 { 766 uint32_t index; 767 dma_addr_t addr, entry_size; 768 769 index = VTD_PASID_DIR_INDEX(pasid); 770 entry_size = VTD_PASID_DIR_ENTRY_SIZE; 771 addr = pasid_dir_base + index * entry_size; 772 if (dma_memory_read(&address_space_memory, addr, 773 pdire, entry_size, MEMTXATTRS_UNSPECIFIED)) { 774 return -VTD_FR_PASID_TABLE_INV; 775 } 776 777 pdire->val = le64_to_cpu(pdire->val); 778 779 return 0; 780 } 781 782 static inline bool vtd_pe_present(VTDPASIDEntry *pe) 783 { 784 return pe->val[0] & VTD_PASID_ENTRY_P; 785 } 786 787 static int vtd_get_pe_in_pasid_leaf_table(IntelIOMMUState *s, 788 uint32_t pasid, 789 dma_addr_t addr, 790 VTDPASIDEntry *pe) 791 { 792 uint32_t index; 793 dma_addr_t entry_size; 794 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 795 796 index = VTD_PASID_TABLE_INDEX(pasid); 797 entry_size = VTD_PASID_ENTRY_SIZE; 798 addr = addr + index * entry_size; 799 if (dma_memory_read(&address_space_memory, addr, 800 pe, entry_size, MEMTXATTRS_UNSPECIFIED)) { 801 return -VTD_FR_PASID_TABLE_INV; 802 } 803 for (size_t i = 0; i < ARRAY_SIZE(pe->val); i++) { 804 pe->val[i] = le64_to_cpu(pe->val[i]); 805 } 806 807 /* Do translation type check */ 808 if (!vtd_pe_type_check(x86_iommu, pe)) { 809 return -VTD_FR_PASID_TABLE_INV; 810 } 811 812 if (!vtd_is_level_supported(s, VTD_PE_GET_LEVEL(pe))) { 813 return -VTD_FR_PASID_TABLE_INV; 814 } 815 816 return 0; 817 } 818 819 /** 820 * Caller of this function should check present bit if wants 821 * to use pasid entry for further usage except for fpd bit check. 822 */ 823 static int vtd_get_pe_from_pdire(IntelIOMMUState *s, 824 uint32_t pasid, 825 VTDPASIDDirEntry *pdire, 826 VTDPASIDEntry *pe) 827 { 828 dma_addr_t addr = pdire->val & VTD_PASID_TABLE_BASE_ADDR_MASK; 829 830 return vtd_get_pe_in_pasid_leaf_table(s, pasid, addr, pe); 831 } 832 833 /** 834 * This function gets a pasid entry from a specified pasid 835 * table (includes dir and leaf table) with a specified pasid. 836 * Sanity check should be done to ensure return a present 837 * pasid entry to caller. 838 */ 839 static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s, 840 dma_addr_t pasid_dir_base, 841 uint32_t pasid, 842 VTDPASIDEntry *pe) 843 { 844 int ret; 845 VTDPASIDDirEntry pdire; 846 847 ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, 848 pasid, &pdire); 849 if (ret) { 850 return ret; 851 } 852 853 if (!vtd_pdire_present(&pdire)) { 854 return -VTD_FR_PASID_TABLE_INV; 855 } 856 857 ret = vtd_get_pe_from_pdire(s, pasid, &pdire, pe); 858 if (ret) { 859 return ret; 860 } 861 862 if (!vtd_pe_present(pe)) { 863 return -VTD_FR_PASID_TABLE_INV; 864 } 865 866 return 0; 867 } 868 869 static int vtd_ce_get_rid2pasid_entry(IntelIOMMUState *s, 870 VTDContextEntry *ce, 871 VTDPASIDEntry *pe, 872 uint32_t pasid) 873 { 874 dma_addr_t pasid_dir_base; 875 int ret = 0; 876 877 if (pasid == PCI_NO_PASID) { 878 pasid = VTD_CE_GET_RID2PASID(ce); 879 } 880 pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); 881 ret = vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe); 882 883 return ret; 884 } 885 886 static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s, 887 VTDContextEntry *ce, 888 bool *pe_fpd_set, 889 uint32_t pasid) 890 { 891 int ret; 892 dma_addr_t pasid_dir_base; 893 VTDPASIDDirEntry pdire; 894 VTDPASIDEntry pe; 895 896 if (pasid == PCI_NO_PASID) { 897 pasid = VTD_CE_GET_RID2PASID(ce); 898 } 899 pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); 900 901 /* 902 * No present bit check since fpd is meaningful even 903 * if the present bit is clear. 904 */ 905 ret = vtd_get_pdire_from_pdir_table(pasid_dir_base, pasid, &pdire); 906 if (ret) { 907 return ret; 908 } 909 910 if (pdire.val & VTD_PASID_DIR_FPD) { 911 *pe_fpd_set = true; 912 return 0; 913 } 914 915 if (!vtd_pdire_present(&pdire)) { 916 return -VTD_FR_PASID_TABLE_INV; 917 } 918 919 /* 920 * No present bit check since fpd is meaningful even 921 * if the present bit is clear. 922 */ 923 ret = vtd_get_pe_from_pdire(s, pasid, &pdire, &pe); 924 if (ret) { 925 return ret; 926 } 927 928 if (pe.val[0] & VTD_PASID_ENTRY_FPD) { 929 *pe_fpd_set = true; 930 } 931 932 return 0; 933 } 934 935 /* Get the page-table level that hardware should use for the second-level 936 * page-table walk from the Address Width field of context-entry. 937 */ 938 static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce) 939 { 940 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 941 } 942 943 static uint32_t vtd_get_iova_level(IntelIOMMUState *s, 944 VTDContextEntry *ce, 945 uint32_t pasid) 946 { 947 VTDPASIDEntry pe; 948 949 if (s->root_scalable) { 950 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 951 return VTD_PE_GET_LEVEL(&pe); 952 } 953 954 return vtd_ce_get_level(ce); 955 } 956 957 static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce) 958 { 959 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 960 } 961 962 static uint32_t vtd_get_iova_agaw(IntelIOMMUState *s, 963 VTDContextEntry *ce, 964 uint32_t pasid) 965 { 966 VTDPASIDEntry pe; 967 968 if (s->root_scalable) { 969 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 970 return 30 + ((pe.val[0] >> 2) & VTD_SM_PASID_ENTRY_AW) * 9; 971 } 972 973 return vtd_ce_get_agaw(ce); 974 } 975 976 static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce) 977 { 978 return ce->lo & VTD_CONTEXT_ENTRY_TT; 979 } 980 981 /* Only for Legacy Mode. Return true if check passed, otherwise false */ 982 static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu, 983 VTDContextEntry *ce) 984 { 985 switch (vtd_ce_get_type(ce)) { 986 case VTD_CONTEXT_TT_MULTI_LEVEL: 987 /* Always supported */ 988 break; 989 case VTD_CONTEXT_TT_DEV_IOTLB: 990 if (!x86_iommu->dt_supported) { 991 error_report_once("%s: DT specified but not supported", __func__); 992 return false; 993 } 994 break; 995 case VTD_CONTEXT_TT_PASS_THROUGH: 996 if (!x86_iommu->pt_supported) { 997 error_report_once("%s: PT specified but not supported", __func__); 998 return false; 999 } 1000 break; 1001 default: 1002 /* Unknown type */ 1003 error_report_once("%s: unknown ce type: %"PRIu32, __func__, 1004 vtd_ce_get_type(ce)); 1005 return false; 1006 } 1007 return true; 1008 } 1009 1010 static inline uint64_t vtd_iova_limit(IntelIOMMUState *s, 1011 VTDContextEntry *ce, uint8_t aw, 1012 uint32_t pasid) 1013 { 1014 uint32_t ce_agaw = vtd_get_iova_agaw(s, ce, pasid); 1015 return 1ULL << MIN(ce_agaw, aw); 1016 } 1017 1018 /* Return true if IOVA passes range check, otherwise false. */ 1019 static inline bool vtd_iova_range_check(IntelIOMMUState *s, 1020 uint64_t iova, VTDContextEntry *ce, 1021 uint8_t aw, uint32_t pasid) 1022 { 1023 /* 1024 * Check if @iova is above 2^X-1, where X is the minimum of MGAW 1025 * in CAP_REG and AW in context-entry. 1026 */ 1027 return !(iova & ~(vtd_iova_limit(s, ce, aw, pasid) - 1)); 1028 } 1029 1030 static dma_addr_t vtd_get_iova_pgtbl_base(IntelIOMMUState *s, 1031 VTDContextEntry *ce, 1032 uint32_t pasid) 1033 { 1034 VTDPASIDEntry pe; 1035 1036 if (s->root_scalable) { 1037 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1038 return pe.val[0] & VTD_SM_PASID_ENTRY_SLPTPTR; 1039 } 1040 1041 return vtd_ce_get_slpt_base(ce); 1042 } 1043 1044 /* 1045 * Rsvd field masks for spte: 1046 * vtd_spte_rsvd 4k pages 1047 * vtd_spte_rsvd_large large pages 1048 * 1049 * We support only 3-level and 4-level page tables (see vtd_init() which 1050 * sets only VTD_CAP_SAGAW_39bit and maybe VTD_CAP_SAGAW_48bit bits in s->cap). 1051 */ 1052 #define VTD_SPTE_RSVD_LEN 5 1053 static uint64_t vtd_spte_rsvd[VTD_SPTE_RSVD_LEN]; 1054 static uint64_t vtd_spte_rsvd_large[VTD_SPTE_RSVD_LEN]; 1055 1056 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 1057 { 1058 uint64_t rsvd_mask; 1059 1060 /* 1061 * We should have caught a guest-mis-programmed level earlier, 1062 * via vtd_is_level_supported. 1063 */ 1064 assert(level < VTD_SPTE_RSVD_LEN); 1065 /* 1066 * Zero level doesn't exist. The smallest level is VTD_SL_PT_LEVEL=1 and 1067 * checked by vtd_is_last_slpte(). 1068 */ 1069 assert(level); 1070 1071 if ((level == VTD_SL_PD_LEVEL || level == VTD_SL_PDP_LEVEL) && 1072 (slpte & VTD_SL_PT_PAGE_SIZE_MASK)) { 1073 /* large page */ 1074 rsvd_mask = vtd_spte_rsvd_large[level]; 1075 } else { 1076 rsvd_mask = vtd_spte_rsvd[level]; 1077 } 1078 1079 return slpte & rsvd_mask; 1080 } 1081 1082 /* Given the @iova, get relevant @slptep. @slpte_level will be the last level 1083 * of the translation, can be used for deciding the size of large page. 1084 */ 1085 static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce, 1086 uint64_t iova, bool is_write, 1087 uint64_t *slptep, uint32_t *slpte_level, 1088 bool *reads, bool *writes, uint8_t aw_bits, 1089 uint32_t pasid) 1090 { 1091 dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); 1092 uint32_t level = vtd_get_iova_level(s, ce, pasid); 1093 uint32_t offset; 1094 uint64_t slpte; 1095 uint64_t access_right_check; 1096 uint64_t xlat, size; 1097 1098 if (!vtd_iova_range_check(s, iova, ce, aw_bits, pasid)) { 1099 error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 "," 1100 "pasid=0x%" PRIx32 ")", __func__, iova, pasid); 1101 return -VTD_FR_ADDR_BEYOND_MGAW; 1102 } 1103 1104 /* FIXME: what is the Atomics request here? */ 1105 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 1106 1107 while (true) { 1108 offset = vtd_iova_level_offset(iova, level); 1109 slpte = vtd_get_slpte(addr, offset); 1110 1111 if (slpte == (uint64_t)-1) { 1112 error_report_once("%s: detected read error on DMAR slpte " 1113 "(iova=0x%" PRIx64 ", pasid=0x%" PRIx32 ")", 1114 __func__, iova, pasid); 1115 if (level == vtd_get_iova_level(s, ce, pasid)) { 1116 /* Invalid programming of context-entry */ 1117 return -VTD_FR_CONTEXT_ENTRY_INV; 1118 } else { 1119 return -VTD_FR_PAGING_ENTRY_INV; 1120 } 1121 } 1122 *reads = (*reads) && (slpte & VTD_SL_R); 1123 *writes = (*writes) && (slpte & VTD_SL_W); 1124 if (!(slpte & access_right_check)) { 1125 error_report_once("%s: detected slpte permission error " 1126 "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " 1127 "slpte=0x%" PRIx64 ", write=%d, pasid=0x%" 1128 PRIx32 ")", __func__, iova, level, 1129 slpte, is_write, pasid); 1130 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 1131 } 1132 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 1133 error_report_once("%s: detected splte reserve non-zero " 1134 "iova=0x%" PRIx64 ", level=0x%" PRIx32 1135 "slpte=0x%" PRIx64 ", pasid=0x%" PRIX32 ")", 1136 __func__, iova, level, slpte, pasid); 1137 return -VTD_FR_PAGING_ENTRY_RSVD; 1138 } 1139 1140 if (vtd_is_last_slpte(slpte, level)) { 1141 *slptep = slpte; 1142 *slpte_level = level; 1143 break; 1144 } 1145 addr = vtd_get_slpte_addr(slpte, aw_bits); 1146 level--; 1147 } 1148 1149 xlat = vtd_get_slpte_addr(*slptep, aw_bits); 1150 size = ~vtd_slpt_level_page_mask(level) + 1; 1151 1152 /* 1153 * From VT-d spec 3.14: Untranslated requests and translation 1154 * requests that result in an address in the interrupt range will be 1155 * blocked with condition code LGN.4 or SGN.8. 1156 */ 1157 if ((xlat > VTD_INTERRUPT_ADDR_LAST || 1158 xlat + size - 1 < VTD_INTERRUPT_ADDR_FIRST)) { 1159 return 0; 1160 } else { 1161 error_report_once("%s: xlat address is in interrupt range " 1162 "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " 1163 "slpte=0x%" PRIx64 ", write=%d, " 1164 "xlat=0x%" PRIx64 ", size=0x%" PRIx64 ", " 1165 "pasid=0x%" PRIx32 ")", 1166 __func__, iova, level, slpte, is_write, 1167 xlat, size, pasid); 1168 return s->scalable_mode ? -VTD_FR_SM_INTERRUPT_ADDR : 1169 -VTD_FR_INTERRUPT_ADDR; 1170 } 1171 } 1172 1173 typedef int (*vtd_page_walk_hook)(IOMMUTLBEvent *event, void *private); 1174 1175 /** 1176 * Constant information used during page walking 1177 * 1178 * @hook_fn: hook func to be called when detected page 1179 * @private: private data to be passed into hook func 1180 * @notify_unmap: whether we should notify invalid entries 1181 * @as: VT-d address space of the device 1182 * @aw: maximum address width 1183 * @domain: domain ID of the page walk 1184 */ 1185 typedef struct { 1186 VTDAddressSpace *as; 1187 vtd_page_walk_hook hook_fn; 1188 void *private; 1189 bool notify_unmap; 1190 uint8_t aw; 1191 uint16_t domain_id; 1192 } vtd_page_walk_info; 1193 1194 static int vtd_page_walk_one(IOMMUTLBEvent *event, vtd_page_walk_info *info) 1195 { 1196 VTDAddressSpace *as = info->as; 1197 vtd_page_walk_hook hook_fn = info->hook_fn; 1198 void *private = info->private; 1199 IOMMUTLBEntry *entry = &event->entry; 1200 DMAMap target = { 1201 .iova = entry->iova, 1202 .size = entry->addr_mask, 1203 .translated_addr = entry->translated_addr, 1204 .perm = entry->perm, 1205 }; 1206 const DMAMap *mapped = iova_tree_find(as->iova_tree, &target); 1207 1208 if (event->type == IOMMU_NOTIFIER_UNMAP && !info->notify_unmap) { 1209 trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 1210 return 0; 1211 } 1212 1213 assert(hook_fn); 1214 1215 /* Update local IOVA mapped ranges */ 1216 if (event->type == IOMMU_NOTIFIER_MAP) { 1217 if (mapped) { 1218 /* If it's exactly the same translation, skip */ 1219 if (!memcmp(mapped, &target, sizeof(target))) { 1220 trace_vtd_page_walk_one_skip_map(entry->iova, entry->addr_mask, 1221 entry->translated_addr); 1222 return 0; 1223 } else { 1224 /* 1225 * Translation changed. Normally this should not 1226 * happen, but it can happen when with buggy guest 1227 * OSes. Note that there will be a small window that 1228 * we don't have map at all. But that's the best 1229 * effort we can do. The ideal way to emulate this is 1230 * atomically modify the PTE to follow what has 1231 * changed, but we can't. One example is that vfio 1232 * driver only has VFIO_IOMMU_[UN]MAP_DMA but no 1233 * interface to modify a mapping (meanwhile it seems 1234 * meaningless to even provide one). Anyway, let's 1235 * mark this as a TODO in case one day we'll have 1236 * a better solution. 1237 */ 1238 IOMMUAccessFlags cache_perm = entry->perm; 1239 int ret; 1240 1241 /* Emulate an UNMAP */ 1242 event->type = IOMMU_NOTIFIER_UNMAP; 1243 entry->perm = IOMMU_NONE; 1244 trace_vtd_page_walk_one(info->domain_id, 1245 entry->iova, 1246 entry->translated_addr, 1247 entry->addr_mask, 1248 entry->perm); 1249 ret = hook_fn(event, private); 1250 if (ret) { 1251 return ret; 1252 } 1253 /* Drop any existing mapping */ 1254 iova_tree_remove(as->iova_tree, target); 1255 /* Recover the correct type */ 1256 event->type = IOMMU_NOTIFIER_MAP; 1257 entry->perm = cache_perm; 1258 } 1259 } 1260 iova_tree_insert(as->iova_tree, &target); 1261 } else { 1262 if (!mapped) { 1263 /* Skip since we didn't map this range at all */ 1264 trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 1265 return 0; 1266 } 1267 iova_tree_remove(as->iova_tree, target); 1268 } 1269 1270 trace_vtd_page_walk_one(info->domain_id, entry->iova, 1271 entry->translated_addr, entry->addr_mask, 1272 entry->perm); 1273 return hook_fn(event, private); 1274 } 1275 1276 /** 1277 * vtd_page_walk_level - walk over specific level for IOVA range 1278 * 1279 * @addr: base GPA addr to start the walk 1280 * @start: IOVA range start address 1281 * @end: IOVA range end address (start <= addr < end) 1282 * @read: whether parent level has read permission 1283 * @write: whether parent level has write permission 1284 * @info: constant information for the page walk 1285 */ 1286 static int vtd_page_walk_level(dma_addr_t addr, uint64_t start, 1287 uint64_t end, uint32_t level, bool read, 1288 bool write, vtd_page_walk_info *info) 1289 { 1290 bool read_cur, write_cur, entry_valid; 1291 uint32_t offset; 1292 uint64_t slpte; 1293 uint64_t subpage_size, subpage_mask; 1294 IOMMUTLBEvent event; 1295 uint64_t iova = start; 1296 uint64_t iova_next; 1297 int ret = 0; 1298 1299 trace_vtd_page_walk_level(addr, level, start, end); 1300 1301 subpage_size = 1ULL << vtd_slpt_level_shift(level); 1302 subpage_mask = vtd_slpt_level_page_mask(level); 1303 1304 while (iova < end) { 1305 iova_next = (iova & subpage_mask) + subpage_size; 1306 1307 offset = vtd_iova_level_offset(iova, level); 1308 slpte = vtd_get_slpte(addr, offset); 1309 1310 if (slpte == (uint64_t)-1) { 1311 trace_vtd_page_walk_skip_read(iova, iova_next); 1312 goto next; 1313 } 1314 1315 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 1316 trace_vtd_page_walk_skip_reserve(iova, iova_next); 1317 goto next; 1318 } 1319 1320 /* Permissions are stacked with parents' */ 1321 read_cur = read && (slpte & VTD_SL_R); 1322 write_cur = write && (slpte & VTD_SL_W); 1323 1324 /* 1325 * As long as we have either read/write permission, this is a 1326 * valid entry. The rule works for both page entries and page 1327 * table entries. 1328 */ 1329 entry_valid = read_cur | write_cur; 1330 1331 if (!vtd_is_last_slpte(slpte, level) && entry_valid) { 1332 /* 1333 * This is a valid PDE (or even bigger than PDE). We need 1334 * to walk one further level. 1335 */ 1336 ret = vtd_page_walk_level(vtd_get_slpte_addr(slpte, info->aw), 1337 iova, MIN(iova_next, end), level - 1, 1338 read_cur, write_cur, info); 1339 } else { 1340 /* 1341 * This means we are either: 1342 * 1343 * (1) the real page entry (either 4K page, or huge page) 1344 * (2) the whole range is invalid 1345 * 1346 * In either case, we send an IOTLB notification down. 1347 */ 1348 event.entry.target_as = &address_space_memory; 1349 event.entry.iova = iova & subpage_mask; 1350 event.entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur); 1351 event.entry.addr_mask = ~subpage_mask; 1352 /* NOTE: this is only meaningful if entry_valid == true */ 1353 event.entry.translated_addr = vtd_get_slpte_addr(slpte, info->aw); 1354 event.type = event.entry.perm ? IOMMU_NOTIFIER_MAP : 1355 IOMMU_NOTIFIER_UNMAP; 1356 ret = vtd_page_walk_one(&event, info); 1357 } 1358 1359 if (ret < 0) { 1360 return ret; 1361 } 1362 1363 next: 1364 iova = iova_next; 1365 } 1366 1367 return 0; 1368 } 1369 1370 /** 1371 * vtd_page_walk - walk specific IOVA range, and call the hook 1372 * 1373 * @s: intel iommu state 1374 * @ce: context entry to walk upon 1375 * @start: IOVA address to start the walk 1376 * @end: IOVA range end address (start <= addr < end) 1377 * @info: page walking information struct 1378 */ 1379 static int vtd_page_walk(IntelIOMMUState *s, VTDContextEntry *ce, 1380 uint64_t start, uint64_t end, 1381 vtd_page_walk_info *info, 1382 uint32_t pasid) 1383 { 1384 dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); 1385 uint32_t level = vtd_get_iova_level(s, ce, pasid); 1386 1387 if (!vtd_iova_range_check(s, start, ce, info->aw, pasid)) { 1388 return -VTD_FR_ADDR_BEYOND_MGAW; 1389 } 1390 1391 if (!vtd_iova_range_check(s, end, ce, info->aw, pasid)) { 1392 /* Fix end so that it reaches the maximum */ 1393 end = vtd_iova_limit(s, ce, info->aw, pasid); 1394 } 1395 1396 return vtd_page_walk_level(addr, start, end, level, true, true, info); 1397 } 1398 1399 static int vtd_root_entry_rsvd_bits_check(IntelIOMMUState *s, 1400 VTDRootEntry *re) 1401 { 1402 /* Legacy Mode reserved bits check */ 1403 if (!s->root_scalable && 1404 (re->hi || (re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)))) 1405 goto rsvd_err; 1406 1407 /* Scalable Mode reserved bits check */ 1408 if (s->root_scalable && 1409 ((re->lo & VTD_ROOT_ENTRY_RSVD(s->aw_bits)) || 1410 (re->hi & VTD_ROOT_ENTRY_RSVD(s->aw_bits)))) 1411 goto rsvd_err; 1412 1413 return 0; 1414 1415 rsvd_err: 1416 error_report_once("%s: invalid root entry: hi=0x%"PRIx64 1417 ", lo=0x%"PRIx64, 1418 __func__, re->hi, re->lo); 1419 return -VTD_FR_ROOT_ENTRY_RSVD; 1420 } 1421 1422 static inline int vtd_context_entry_rsvd_bits_check(IntelIOMMUState *s, 1423 VTDContextEntry *ce) 1424 { 1425 if (!s->root_scalable && 1426 (ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI || 1427 ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) { 1428 error_report_once("%s: invalid context entry: hi=%"PRIx64 1429 ", lo=%"PRIx64" (reserved nonzero)", 1430 __func__, ce->hi, ce->lo); 1431 return -VTD_FR_CONTEXT_ENTRY_RSVD; 1432 } 1433 1434 if (s->root_scalable && 1435 (ce->val[0] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL0(s->aw_bits) || 1436 ce->val[1] & VTD_SM_CONTEXT_ENTRY_RSVD_VAL1 || 1437 ce->val[2] || 1438 ce->val[3])) { 1439 error_report_once("%s: invalid context entry: val[3]=%"PRIx64 1440 ", val[2]=%"PRIx64 1441 ", val[1]=%"PRIx64 1442 ", val[0]=%"PRIx64" (reserved nonzero)", 1443 __func__, ce->val[3], ce->val[2], 1444 ce->val[1], ce->val[0]); 1445 return -VTD_FR_CONTEXT_ENTRY_RSVD; 1446 } 1447 1448 return 0; 1449 } 1450 1451 static int vtd_ce_rid2pasid_check(IntelIOMMUState *s, 1452 VTDContextEntry *ce) 1453 { 1454 VTDPASIDEntry pe; 1455 1456 /* 1457 * Make sure in Scalable Mode, a present context entry 1458 * has valid rid2pasid setting, which includes valid 1459 * rid2pasid field and corresponding pasid entry setting 1460 */ 1461 return vtd_ce_get_rid2pasid_entry(s, ce, &pe, PCI_NO_PASID); 1462 } 1463 1464 /* Map a device to its corresponding domain (context-entry) */ 1465 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 1466 uint8_t devfn, VTDContextEntry *ce) 1467 { 1468 VTDRootEntry re; 1469 int ret_fr; 1470 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 1471 1472 ret_fr = vtd_get_root_entry(s, bus_num, &re); 1473 if (ret_fr) { 1474 return ret_fr; 1475 } 1476 1477 if (!vtd_root_entry_present(s, &re, devfn)) { 1478 /* Not error - it's okay we don't have root entry. */ 1479 trace_vtd_re_not_present(bus_num); 1480 return -VTD_FR_ROOT_ENTRY_P; 1481 } 1482 1483 ret_fr = vtd_root_entry_rsvd_bits_check(s, &re); 1484 if (ret_fr) { 1485 return ret_fr; 1486 } 1487 1488 ret_fr = vtd_get_context_entry_from_root(s, &re, devfn, ce); 1489 if (ret_fr) { 1490 return ret_fr; 1491 } 1492 1493 if (!vtd_ce_present(ce)) { 1494 /* Not error - it's okay we don't have context entry. */ 1495 trace_vtd_ce_not_present(bus_num, devfn); 1496 return -VTD_FR_CONTEXT_ENTRY_P; 1497 } 1498 1499 ret_fr = vtd_context_entry_rsvd_bits_check(s, ce); 1500 if (ret_fr) { 1501 return ret_fr; 1502 } 1503 1504 /* Check if the programming of context-entry is valid */ 1505 if (!s->root_scalable && 1506 !vtd_is_level_supported(s, vtd_ce_get_level(ce))) { 1507 error_report_once("%s: invalid context entry: hi=%"PRIx64 1508 ", lo=%"PRIx64" (level %d not supported)", 1509 __func__, ce->hi, ce->lo, 1510 vtd_ce_get_level(ce)); 1511 return -VTD_FR_CONTEXT_ENTRY_INV; 1512 } 1513 1514 if (!s->root_scalable) { 1515 /* Do translation type check */ 1516 if (!vtd_ce_type_check(x86_iommu, ce)) { 1517 /* Errors dumped in vtd_ce_type_check() */ 1518 return -VTD_FR_CONTEXT_ENTRY_INV; 1519 } 1520 } else { 1521 /* 1522 * Check if the programming of context-entry.rid2pasid 1523 * and corresponding pasid setting is valid, and thus 1524 * avoids to check pasid entry fetching result in future 1525 * helper function calling. 1526 */ 1527 ret_fr = vtd_ce_rid2pasid_check(s, ce); 1528 if (ret_fr) { 1529 return ret_fr; 1530 } 1531 } 1532 1533 return 0; 1534 } 1535 1536 static int vtd_sync_shadow_page_hook(IOMMUTLBEvent *event, 1537 void *private) 1538 { 1539 memory_region_notify_iommu(private, 0, *event); 1540 return 0; 1541 } 1542 1543 static uint16_t vtd_get_domain_id(IntelIOMMUState *s, 1544 VTDContextEntry *ce, 1545 uint32_t pasid) 1546 { 1547 VTDPASIDEntry pe; 1548 1549 if (s->root_scalable) { 1550 vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1551 return VTD_SM_PASID_ENTRY_DID(pe.val[1]); 1552 } 1553 1554 return VTD_CONTEXT_ENTRY_DID(ce->hi); 1555 } 1556 1557 static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as, 1558 VTDContextEntry *ce, 1559 hwaddr addr, hwaddr size) 1560 { 1561 IntelIOMMUState *s = vtd_as->iommu_state; 1562 vtd_page_walk_info info = { 1563 .hook_fn = vtd_sync_shadow_page_hook, 1564 .private = (void *)&vtd_as->iommu, 1565 .notify_unmap = true, 1566 .aw = s->aw_bits, 1567 .as = vtd_as, 1568 .domain_id = vtd_get_domain_id(s, ce, vtd_as->pasid), 1569 }; 1570 1571 return vtd_page_walk(s, ce, addr, addr + size, &info, vtd_as->pasid); 1572 } 1573 1574 static int vtd_address_space_sync(VTDAddressSpace *vtd_as) 1575 { 1576 int ret; 1577 VTDContextEntry ce; 1578 IOMMUNotifier *n; 1579 1580 /* If no MAP notifier registered, we simply invalidate all the cache */ 1581 if (!vtd_as_has_map_notifier(vtd_as)) { 1582 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 1583 memory_region_unmap_iommu_notifier_range(n); 1584 } 1585 return 0; 1586 } 1587 1588 ret = vtd_dev_to_context_entry(vtd_as->iommu_state, 1589 pci_bus_num(vtd_as->bus), 1590 vtd_as->devfn, &ce); 1591 if (ret) { 1592 if (ret == -VTD_FR_CONTEXT_ENTRY_P) { 1593 /* 1594 * It's a valid scenario to have a context entry that is 1595 * not present. For example, when a device is removed 1596 * from an existing domain then the context entry will be 1597 * zeroed by the guest before it was put into another 1598 * domain. When this happens, instead of synchronizing 1599 * the shadow pages we should invalidate all existing 1600 * mappings and notify the backends. 1601 */ 1602 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 1603 vtd_address_space_unmap(vtd_as, n); 1604 } 1605 ret = 0; 1606 } 1607 return ret; 1608 } 1609 1610 return vtd_sync_shadow_page_table_range(vtd_as, &ce, 0, UINT64_MAX); 1611 } 1612 1613 /* 1614 * Check if specific device is configured to bypass address 1615 * translation for DMA requests. In Scalable Mode, bypass 1616 * 1st-level translation or 2nd-level translation, it depends 1617 * on PGTT setting. 1618 */ 1619 static bool vtd_dev_pt_enabled(IntelIOMMUState *s, VTDContextEntry *ce, 1620 uint32_t pasid) 1621 { 1622 VTDPASIDEntry pe; 1623 int ret; 1624 1625 if (s->root_scalable) { 1626 ret = vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); 1627 if (ret) { 1628 /* 1629 * This error is guest triggerable. We should assumt PT 1630 * not enabled for safety. 1631 */ 1632 return false; 1633 } 1634 return (VTD_PE_GET_TYPE(&pe) == VTD_SM_PASID_ENTRY_PT); 1635 } 1636 1637 return (vtd_ce_get_type(ce) == VTD_CONTEXT_TT_PASS_THROUGH); 1638 1639 } 1640 1641 static bool vtd_as_pt_enabled(VTDAddressSpace *as) 1642 { 1643 IntelIOMMUState *s; 1644 VTDContextEntry ce; 1645 1646 assert(as); 1647 1648 s = as->iommu_state; 1649 if (vtd_dev_to_context_entry(s, pci_bus_num(as->bus), as->devfn, 1650 &ce)) { 1651 /* 1652 * Possibly failed to parse the context entry for some reason 1653 * (e.g., during init, or any guest configuration errors on 1654 * context entries). We should assume PT not enabled for 1655 * safety. 1656 */ 1657 return false; 1658 } 1659 1660 return vtd_dev_pt_enabled(s, &ce, as->pasid); 1661 } 1662 1663 /* Return whether the device is using IOMMU translation. */ 1664 static bool vtd_switch_address_space(VTDAddressSpace *as) 1665 { 1666 bool use_iommu, pt; 1667 /* Whether we need to take the BQL on our own */ 1668 bool take_bql = !bql_locked(); 1669 1670 assert(as); 1671 1672 use_iommu = as->iommu_state->dmar_enabled && !vtd_as_pt_enabled(as); 1673 pt = as->iommu_state->dmar_enabled && vtd_as_pt_enabled(as); 1674 1675 trace_vtd_switch_address_space(pci_bus_num(as->bus), 1676 VTD_PCI_SLOT(as->devfn), 1677 VTD_PCI_FUNC(as->devfn), 1678 use_iommu); 1679 1680 /* 1681 * It's possible that we reach here without BQL, e.g., when called 1682 * from vtd_pt_enable_fast_path(). However the memory APIs need 1683 * it. We'd better make sure we have had it already, or, take it. 1684 */ 1685 if (take_bql) { 1686 bql_lock(); 1687 } 1688 1689 /* Turn off first then on the other */ 1690 if (use_iommu) { 1691 memory_region_set_enabled(&as->nodmar, false); 1692 memory_region_set_enabled(MEMORY_REGION(&as->iommu), true); 1693 /* 1694 * vt-d spec v3.4 3.14: 1695 * 1696 * """ 1697 * Requests-with-PASID with input address in range 0xFEEx_xxxx 1698 * are translated normally like any other request-with-PASID 1699 * through DMA-remapping hardware. 1700 * """ 1701 * 1702 * Need to disable ir for as with PASID. 1703 */ 1704 if (as->pasid != PCI_NO_PASID) { 1705 memory_region_set_enabled(&as->iommu_ir, false); 1706 } else { 1707 memory_region_set_enabled(&as->iommu_ir, true); 1708 } 1709 } else { 1710 memory_region_set_enabled(MEMORY_REGION(&as->iommu), false); 1711 memory_region_set_enabled(&as->nodmar, true); 1712 } 1713 1714 /* 1715 * vtd-spec v3.4 3.14: 1716 * 1717 * """ 1718 * Requests-with-PASID with input address in range 0xFEEx_xxxx are 1719 * translated normally like any other request-with-PASID through 1720 * DMA-remapping hardware. However, if such a request is processed 1721 * using pass-through translation, it will be blocked as described 1722 * in the paragraph below. 1723 * 1724 * Software must not program paging-structure entries to remap any 1725 * address to the interrupt address range. Untranslated requests 1726 * and translation requests that result in an address in the 1727 * interrupt range will be blocked with condition code LGN.4 or 1728 * SGN.8. 1729 * """ 1730 * 1731 * We enable per as memory region (iommu_ir_fault) for catching 1732 * the translation for interrupt range through PASID + PT. 1733 */ 1734 if (pt && as->pasid != PCI_NO_PASID) { 1735 memory_region_set_enabled(&as->iommu_ir_fault, true); 1736 } else { 1737 memory_region_set_enabled(&as->iommu_ir_fault, false); 1738 } 1739 1740 if (take_bql) { 1741 bql_unlock(); 1742 } 1743 1744 return use_iommu; 1745 } 1746 1747 static void vtd_switch_address_space_all(IntelIOMMUState *s) 1748 { 1749 VTDAddressSpace *vtd_as; 1750 GHashTableIter iter; 1751 1752 g_hash_table_iter_init(&iter, s->vtd_address_spaces); 1753 while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_as)) { 1754 vtd_switch_address_space(vtd_as); 1755 } 1756 } 1757 1758 static const bool vtd_qualified_faults[] = { 1759 [VTD_FR_RESERVED] = false, 1760 [VTD_FR_ROOT_ENTRY_P] = false, 1761 [VTD_FR_CONTEXT_ENTRY_P] = true, 1762 [VTD_FR_CONTEXT_ENTRY_INV] = true, 1763 [VTD_FR_ADDR_BEYOND_MGAW] = true, 1764 [VTD_FR_WRITE] = true, 1765 [VTD_FR_READ] = true, 1766 [VTD_FR_PAGING_ENTRY_INV] = true, 1767 [VTD_FR_ROOT_TABLE_INV] = false, 1768 [VTD_FR_CONTEXT_TABLE_INV] = false, 1769 [VTD_FR_INTERRUPT_ADDR] = true, 1770 [VTD_FR_ROOT_ENTRY_RSVD] = false, 1771 [VTD_FR_PAGING_ENTRY_RSVD] = true, 1772 [VTD_FR_CONTEXT_ENTRY_TT] = true, 1773 [VTD_FR_PASID_TABLE_INV] = false, 1774 [VTD_FR_SM_INTERRUPT_ADDR] = true, 1775 [VTD_FR_MAX] = false, 1776 }; 1777 1778 /* To see if a fault condition is "qualified", which is reported to software 1779 * only if the FPD field in the context-entry used to process the faulting 1780 * request is 0. 1781 */ 1782 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 1783 { 1784 return vtd_qualified_faults[fault]; 1785 } 1786 1787 static inline bool vtd_is_interrupt_addr(hwaddr addr) 1788 { 1789 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 1790 } 1791 1792 static gboolean vtd_find_as_by_sid(gpointer key, gpointer value, 1793 gpointer user_data) 1794 { 1795 struct vtd_as_key *as_key = (struct vtd_as_key *)key; 1796 uint16_t target_sid = *(uint16_t *)user_data; 1797 uint16_t sid = PCI_BUILD_BDF(pci_bus_num(as_key->bus), as_key->devfn); 1798 return sid == target_sid; 1799 } 1800 1801 static VTDAddressSpace *vtd_get_as_by_sid(IntelIOMMUState *s, uint16_t sid) 1802 { 1803 uint8_t bus_num = PCI_BUS_NUM(sid); 1804 VTDAddressSpace *vtd_as = s->vtd_as_cache[bus_num]; 1805 1806 if (vtd_as && 1807 (sid == PCI_BUILD_BDF(pci_bus_num(vtd_as->bus), vtd_as->devfn))) { 1808 return vtd_as; 1809 } 1810 1811 vtd_as = g_hash_table_find(s->vtd_address_spaces, vtd_find_as_by_sid, &sid); 1812 s->vtd_as_cache[bus_num] = vtd_as; 1813 1814 return vtd_as; 1815 } 1816 1817 static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id) 1818 { 1819 VTDAddressSpace *vtd_as; 1820 bool success = false; 1821 1822 vtd_as = vtd_get_as_by_sid(s, source_id); 1823 if (!vtd_as) { 1824 goto out; 1825 } 1826 1827 if (vtd_switch_address_space(vtd_as) == false) { 1828 /* We switched off IOMMU region successfully. */ 1829 success = true; 1830 } 1831 1832 out: 1833 trace_vtd_pt_enable_fast_path(source_id, success); 1834 } 1835 1836 static void vtd_report_fault(IntelIOMMUState *s, 1837 int err, bool is_fpd_set, 1838 uint16_t source_id, 1839 hwaddr addr, 1840 bool is_write, 1841 bool is_pasid, 1842 uint32_t pasid) 1843 { 1844 if (is_fpd_set && vtd_is_qualified_fault(err)) { 1845 trace_vtd_fault_disabled(); 1846 } else { 1847 vtd_report_dmar_fault(s, source_id, addr, err, is_write, 1848 is_pasid, pasid); 1849 } 1850 } 1851 1852 /* Map dev to context-entry then do a paging-structures walk to do a iommu 1853 * translation. 1854 * 1855 * Called from RCU critical section. 1856 * 1857 * @bus_num: The bus number 1858 * @devfn: The devfn, which is the combined of device and function number 1859 * @is_write: The access is a write operation 1860 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 1861 * 1862 * Returns true if translation is successful, otherwise false. 1863 */ 1864 static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, 1865 uint8_t devfn, hwaddr addr, bool is_write, 1866 IOMMUTLBEntry *entry) 1867 { 1868 IntelIOMMUState *s = vtd_as->iommu_state; 1869 VTDContextEntry ce; 1870 uint8_t bus_num = pci_bus_num(bus); 1871 VTDContextCacheEntry *cc_entry; 1872 uint64_t slpte, page_mask; 1873 uint32_t level, pasid = vtd_as->pasid; 1874 uint16_t source_id = PCI_BUILD_BDF(bus_num, devfn); 1875 int ret_fr; 1876 bool is_fpd_set = false; 1877 bool reads = true; 1878 bool writes = true; 1879 uint8_t access_flags; 1880 bool rid2pasid = (pasid == PCI_NO_PASID) && s->root_scalable; 1881 VTDIOTLBEntry *iotlb_entry; 1882 1883 /* 1884 * We have standalone memory region for interrupt addresses, we 1885 * should never receive translation requests in this region. 1886 */ 1887 assert(!vtd_is_interrupt_addr(addr)); 1888 1889 vtd_iommu_lock(s); 1890 1891 cc_entry = &vtd_as->context_cache_entry; 1892 1893 /* Try to fetch slpte form IOTLB, we don't need RID2PASID logic */ 1894 if (!rid2pasid) { 1895 iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); 1896 if (iotlb_entry) { 1897 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, 1898 iotlb_entry->domain_id); 1899 slpte = iotlb_entry->slpte; 1900 access_flags = iotlb_entry->access_flags; 1901 page_mask = iotlb_entry->mask; 1902 goto out; 1903 } 1904 } 1905 1906 /* Try to fetch context-entry from cache first */ 1907 if (cc_entry->context_cache_gen == s->context_cache_gen) { 1908 trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi, 1909 cc_entry->context_entry.lo, 1910 cc_entry->context_cache_gen); 1911 ce = cc_entry->context_entry; 1912 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1913 if (!is_fpd_set && s->root_scalable) { 1914 ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid); 1915 if (ret_fr) { 1916 vtd_report_fault(s, -ret_fr, is_fpd_set, 1917 source_id, addr, is_write, 1918 false, 0); 1919 goto error; 1920 } 1921 } 1922 } else { 1923 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 1924 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1925 if (!ret_fr && !is_fpd_set && s->root_scalable) { 1926 ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid); 1927 } 1928 if (ret_fr) { 1929 vtd_report_fault(s, -ret_fr, is_fpd_set, 1930 source_id, addr, is_write, 1931 false, 0); 1932 goto error; 1933 } 1934 /* Update context-cache */ 1935 trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo, 1936 cc_entry->context_cache_gen, 1937 s->context_cache_gen); 1938 cc_entry->context_entry = ce; 1939 cc_entry->context_cache_gen = s->context_cache_gen; 1940 } 1941 1942 if (rid2pasid) { 1943 pasid = VTD_CE_GET_RID2PASID(&ce); 1944 } 1945 1946 /* 1947 * We don't need to translate for pass-through context entries. 1948 * Also, let's ignore IOTLB caching as well for PT devices. 1949 */ 1950 if (vtd_dev_pt_enabled(s, &ce, pasid)) { 1951 entry->iova = addr & VTD_PAGE_MASK_4K; 1952 entry->translated_addr = entry->iova; 1953 entry->addr_mask = ~VTD_PAGE_MASK_4K; 1954 entry->perm = IOMMU_RW; 1955 trace_vtd_translate_pt(source_id, entry->iova); 1956 1957 /* 1958 * When this happens, it means firstly caching-mode is not 1959 * enabled, and this is the first passthrough translation for 1960 * the device. Let's enable the fast path for passthrough. 1961 * 1962 * When passthrough is disabled again for the device, we can 1963 * capture it via the context entry invalidation, then the 1964 * IOMMU region can be swapped back. 1965 */ 1966 vtd_pt_enable_fast_path(s, source_id); 1967 vtd_iommu_unlock(s); 1968 return true; 1969 } 1970 1971 /* Try to fetch slpte form IOTLB for RID2PASID slow path */ 1972 if (rid2pasid) { 1973 iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); 1974 if (iotlb_entry) { 1975 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, 1976 iotlb_entry->domain_id); 1977 slpte = iotlb_entry->slpte; 1978 access_flags = iotlb_entry->access_flags; 1979 page_mask = iotlb_entry->mask; 1980 goto out; 1981 } 1982 } 1983 1984 ret_fr = vtd_iova_to_slpte(s, &ce, addr, is_write, &slpte, &level, 1985 &reads, &writes, s->aw_bits, pasid); 1986 if (ret_fr) { 1987 vtd_report_fault(s, -ret_fr, is_fpd_set, source_id, 1988 addr, is_write, pasid != PCI_NO_PASID, pasid); 1989 goto error; 1990 } 1991 1992 page_mask = vtd_slpt_level_page_mask(level); 1993 access_flags = IOMMU_ACCESS_FLAG(reads, writes); 1994 vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce, pasid), 1995 addr, slpte, access_flags, level, pasid); 1996 out: 1997 vtd_iommu_unlock(s); 1998 entry->iova = addr & page_mask; 1999 entry->translated_addr = vtd_get_slpte_addr(slpte, s->aw_bits) & page_mask; 2000 entry->addr_mask = ~page_mask; 2001 entry->perm = access_flags; 2002 return true; 2003 2004 error: 2005 vtd_iommu_unlock(s); 2006 entry->iova = 0; 2007 entry->translated_addr = 0; 2008 entry->addr_mask = 0; 2009 entry->perm = IOMMU_NONE; 2010 return false; 2011 } 2012 2013 static void vtd_root_table_setup(IntelIOMMUState *s) 2014 { 2015 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 2016 s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits); 2017 2018 vtd_update_scalable_state(s); 2019 2020 trace_vtd_reg_dmar_root(s->root, s->root_scalable); 2021 } 2022 2023 static void vtd_iec_notify_all(IntelIOMMUState *s, bool global, 2024 uint32_t index, uint32_t mask) 2025 { 2026 x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask); 2027 } 2028 2029 static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s) 2030 { 2031 uint64_t value = 0; 2032 value = vtd_get_quad_raw(s, DMAR_IRTA_REG); 2033 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1); 2034 s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits); 2035 s->intr_eime = value & VTD_IRTA_EIME; 2036 2037 /* Notify global invalidation */ 2038 vtd_iec_notify_all(s, true, 0, 0); 2039 2040 trace_vtd_reg_ir_root(s->intr_root, s->intr_size); 2041 } 2042 2043 static void vtd_iommu_replay_all(IntelIOMMUState *s) 2044 { 2045 VTDAddressSpace *vtd_as; 2046 2047 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 2048 vtd_address_space_sync(vtd_as); 2049 } 2050 } 2051 2052 static void vtd_context_global_invalidate(IntelIOMMUState *s) 2053 { 2054 trace_vtd_inv_desc_cc_global(); 2055 /* Protects context cache */ 2056 vtd_iommu_lock(s); 2057 s->context_cache_gen++; 2058 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 2059 vtd_reset_context_cache_locked(s); 2060 } 2061 vtd_iommu_unlock(s); 2062 vtd_address_space_refresh_all(s); 2063 /* 2064 * From VT-d spec 6.5.2.1, a global context entry invalidation 2065 * should be followed by a IOTLB global invalidation, so we should 2066 * be safe even without this. Hoewever, let's replay the region as 2067 * well to be safer, and go back here when we need finer tunes for 2068 * VT-d emulation codes. 2069 */ 2070 vtd_iommu_replay_all(s); 2071 } 2072 2073 /* Do a context-cache device-selective invalidation. 2074 * @func_mask: FM field after shifting 2075 */ 2076 static void vtd_context_device_invalidate(IntelIOMMUState *s, 2077 uint16_t source_id, 2078 uint16_t func_mask) 2079 { 2080 GHashTableIter as_it; 2081 uint16_t mask; 2082 VTDAddressSpace *vtd_as; 2083 uint8_t bus_n, devfn; 2084 2085 trace_vtd_inv_desc_cc_devices(source_id, func_mask); 2086 2087 switch (func_mask & 3) { 2088 case 0: 2089 mask = 0; /* No bits in the SID field masked */ 2090 break; 2091 case 1: 2092 mask = 4; /* Mask bit 2 in the SID field */ 2093 break; 2094 case 2: 2095 mask = 6; /* Mask bit 2:1 in the SID field */ 2096 break; 2097 case 3: 2098 mask = 7; /* Mask bit 2:0 in the SID field */ 2099 break; 2100 default: 2101 g_assert_not_reached(); 2102 } 2103 mask = ~mask; 2104 2105 bus_n = VTD_SID_TO_BUS(source_id); 2106 devfn = VTD_SID_TO_DEVFN(source_id); 2107 2108 g_hash_table_iter_init(&as_it, s->vtd_address_spaces); 2109 while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) { 2110 if ((pci_bus_num(vtd_as->bus) == bus_n) && 2111 (vtd_as->devfn & mask) == (devfn & mask)) { 2112 trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(vtd_as->devfn), 2113 VTD_PCI_FUNC(vtd_as->devfn)); 2114 vtd_iommu_lock(s); 2115 vtd_as->context_cache_entry.context_cache_gen = 0; 2116 vtd_iommu_unlock(s); 2117 /* 2118 * Do switch address space when needed, in case if the 2119 * device passthrough bit is switched. 2120 */ 2121 vtd_switch_address_space(vtd_as); 2122 /* 2123 * So a device is moving out of (or moving into) a 2124 * domain, resync the shadow page table. 2125 * This won't bring bad even if we have no such 2126 * notifier registered - the IOMMU notification 2127 * framework will skip MAP notifications if that 2128 * happened. 2129 */ 2130 vtd_address_space_sync(vtd_as); 2131 } 2132 } 2133 } 2134 2135 /* Context-cache invalidation 2136 * Returns the Context Actual Invalidation Granularity. 2137 * @val: the content of the CCMD_REG 2138 */ 2139 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 2140 { 2141 uint64_t caig; 2142 uint64_t type = val & VTD_CCMD_CIRG_MASK; 2143 2144 switch (type) { 2145 case VTD_CCMD_DOMAIN_INVL: 2146 /* Fall through */ 2147 case VTD_CCMD_GLOBAL_INVL: 2148 caig = VTD_CCMD_GLOBAL_INVL_A; 2149 vtd_context_global_invalidate(s); 2150 break; 2151 2152 case VTD_CCMD_DEVICE_INVL: 2153 caig = VTD_CCMD_DEVICE_INVL_A; 2154 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 2155 break; 2156 2157 default: 2158 error_report_once("%s: invalid context: 0x%" PRIx64, 2159 __func__, val); 2160 caig = 0; 2161 } 2162 return caig; 2163 } 2164 2165 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 2166 { 2167 trace_vtd_inv_desc_iotlb_global(); 2168 vtd_reset_iotlb(s); 2169 vtd_iommu_replay_all(s); 2170 } 2171 2172 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 2173 { 2174 VTDContextEntry ce; 2175 VTDAddressSpace *vtd_as; 2176 2177 trace_vtd_inv_desc_iotlb_domain(domain_id); 2178 2179 vtd_iommu_lock(s); 2180 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 2181 &domain_id); 2182 vtd_iommu_unlock(s); 2183 2184 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 2185 if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 2186 vtd_as->devfn, &ce) && 2187 domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { 2188 vtd_address_space_sync(vtd_as); 2189 } 2190 } 2191 } 2192 2193 static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, 2194 uint16_t domain_id, hwaddr addr, 2195 uint8_t am, uint32_t pasid) 2196 { 2197 VTDAddressSpace *vtd_as; 2198 VTDContextEntry ce; 2199 int ret; 2200 hwaddr size = (1 << am) * VTD_PAGE_SIZE; 2201 2202 QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) { 2203 if (pasid != PCI_NO_PASID && pasid != vtd_as->pasid) { 2204 continue; 2205 } 2206 ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 2207 vtd_as->devfn, &ce); 2208 if (!ret && domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { 2209 if (vtd_as_has_map_notifier(vtd_as)) { 2210 /* 2211 * As long as we have MAP notifications registered in 2212 * any of our IOMMU notifiers, we need to sync the 2213 * shadow page table. 2214 */ 2215 vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size); 2216 } else { 2217 /* 2218 * For UNMAP-only notifiers, we don't need to walk the 2219 * page tables. We just deliver the PSI down to 2220 * invalidate caches. 2221 */ 2222 IOMMUTLBEvent event = { 2223 .type = IOMMU_NOTIFIER_UNMAP, 2224 .entry = { 2225 .target_as = &address_space_memory, 2226 .iova = addr, 2227 .translated_addr = 0, 2228 .addr_mask = size - 1, 2229 .perm = IOMMU_NONE, 2230 }, 2231 }; 2232 memory_region_notify_iommu(&vtd_as->iommu, 0, event); 2233 } 2234 } 2235 } 2236 } 2237 2238 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 2239 hwaddr addr, uint8_t am) 2240 { 2241 VTDIOTLBPageInvInfo info; 2242 2243 trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am); 2244 2245 assert(am <= VTD_MAMV); 2246 info.domain_id = domain_id; 2247 info.addr = addr; 2248 info.mask = ~((1 << am) - 1); 2249 vtd_iommu_lock(s); 2250 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 2251 vtd_iommu_unlock(s); 2252 vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, PCI_NO_PASID); 2253 } 2254 2255 /* Flush IOTLB 2256 * Returns the IOTLB Actual Invalidation Granularity. 2257 * @val: the content of the IOTLB_REG 2258 */ 2259 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 2260 { 2261 uint64_t iaig; 2262 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 2263 uint16_t domain_id; 2264 hwaddr addr; 2265 uint8_t am; 2266 2267 switch (type) { 2268 case VTD_TLB_GLOBAL_FLUSH: 2269 iaig = VTD_TLB_GLOBAL_FLUSH_A; 2270 vtd_iotlb_global_invalidate(s); 2271 break; 2272 2273 case VTD_TLB_DSI_FLUSH: 2274 domain_id = VTD_TLB_DID(val); 2275 iaig = VTD_TLB_DSI_FLUSH_A; 2276 vtd_iotlb_domain_invalidate(s, domain_id); 2277 break; 2278 2279 case VTD_TLB_PSI_FLUSH: 2280 domain_id = VTD_TLB_DID(val); 2281 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 2282 am = VTD_IVA_AM(addr); 2283 addr = VTD_IVA_ADDR(addr); 2284 if (am > VTD_MAMV) { 2285 error_report_once("%s: address mask overflow: 0x%" PRIx64, 2286 __func__, vtd_get_quad_raw(s, DMAR_IVA_REG)); 2287 iaig = 0; 2288 break; 2289 } 2290 iaig = VTD_TLB_PSI_FLUSH_A; 2291 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 2292 break; 2293 2294 default: 2295 error_report_once("%s: invalid granularity: 0x%" PRIx64, 2296 __func__, val); 2297 iaig = 0; 2298 } 2299 return iaig; 2300 } 2301 2302 static void vtd_fetch_inv_desc(IntelIOMMUState *s); 2303 2304 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 2305 { 2306 return s->qi_enabled && (s->iq_tail == s->iq_head) && 2307 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 2308 } 2309 2310 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 2311 { 2312 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 2313 2314 trace_vtd_inv_qi_enable(en); 2315 2316 if (en) { 2317 s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits); 2318 /* 2^(x+8) entries */ 2319 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8 - (s->iq_dw ? 1 : 0)); 2320 s->qi_enabled = true; 2321 trace_vtd_inv_qi_setup(s->iq, s->iq_size); 2322 /* Ok - report back to driver */ 2323 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 2324 2325 if (s->iq_tail != 0) { 2326 /* 2327 * This is a spec violation but Windows guests are known to set up 2328 * Queued Invalidation this way so we allow the write and process 2329 * Invalidation Descriptors right away. 2330 */ 2331 trace_vtd_warn_invalid_qi_tail(s->iq_tail); 2332 if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 2333 vtd_fetch_inv_desc(s); 2334 } 2335 } 2336 } else { 2337 if (vtd_queued_inv_disable_check(s)) { 2338 /* disable Queued Invalidation */ 2339 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 2340 s->iq_head = 0; 2341 s->qi_enabled = false; 2342 /* Ok - report back to driver */ 2343 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 2344 } else { 2345 error_report_once("%s: detected improper state when disable QI " 2346 "(head=0x%x, tail=0x%x, last_type=%d)", 2347 __func__, 2348 s->iq_head, s->iq_tail, s->iq_last_desc_type); 2349 } 2350 } 2351 } 2352 2353 /* Set Root Table Pointer */ 2354 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 2355 { 2356 vtd_root_table_setup(s); 2357 /* Ok - report back to driver */ 2358 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 2359 vtd_reset_caches(s); 2360 vtd_address_space_refresh_all(s); 2361 } 2362 2363 /* Set Interrupt Remap Table Pointer */ 2364 static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s) 2365 { 2366 vtd_interrupt_remap_table_setup(s); 2367 /* Ok - report back to driver */ 2368 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS); 2369 } 2370 2371 /* Handle Translation Enable/Disable */ 2372 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 2373 { 2374 if (s->dmar_enabled == en) { 2375 return; 2376 } 2377 2378 trace_vtd_dmar_enable(en); 2379 2380 if (en) { 2381 s->dmar_enabled = true; 2382 /* Ok - report back to driver */ 2383 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 2384 } else { 2385 s->dmar_enabled = false; 2386 2387 /* Clear the index of Fault Recording Register */ 2388 s->next_frcd_reg = 0; 2389 /* Ok - report back to driver */ 2390 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 2391 } 2392 2393 vtd_reset_caches(s); 2394 vtd_address_space_refresh_all(s); 2395 } 2396 2397 /* Handle Interrupt Remap Enable/Disable */ 2398 static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en) 2399 { 2400 trace_vtd_ir_enable(en); 2401 2402 if (en) { 2403 s->intr_enabled = true; 2404 /* Ok - report back to driver */ 2405 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES); 2406 } else { 2407 s->intr_enabled = false; 2408 /* Ok - report back to driver */ 2409 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0); 2410 } 2411 } 2412 2413 /* Handle write to Global Command Register */ 2414 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 2415 { 2416 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 2417 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 2418 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 2419 uint32_t changed = status ^ val; 2420 2421 trace_vtd_reg_write_gcmd(status, val); 2422 if ((changed & VTD_GCMD_TE) && s->dma_translation) { 2423 /* Translation enable/disable */ 2424 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 2425 } 2426 if (val & VTD_GCMD_SRTP) { 2427 /* Set/update the root-table pointer */ 2428 vtd_handle_gcmd_srtp(s); 2429 } 2430 if (changed & VTD_GCMD_QIE) { 2431 /* Queued Invalidation Enable */ 2432 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 2433 } 2434 if (val & VTD_GCMD_SIRTP) { 2435 /* Set/update the interrupt remapping root-table pointer */ 2436 vtd_handle_gcmd_sirtp(s); 2437 } 2438 if ((changed & VTD_GCMD_IRE) && 2439 x86_iommu_ir_supported(x86_iommu)) { 2440 /* Interrupt remap enable/disable */ 2441 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE); 2442 } 2443 } 2444 2445 /* Handle write to Context Command Register */ 2446 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 2447 { 2448 uint64_t ret; 2449 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 2450 2451 /* Context-cache invalidation request */ 2452 if (val & VTD_CCMD_ICC) { 2453 if (s->qi_enabled) { 2454 error_report_once("Queued Invalidation enabled, " 2455 "should not use register-based invalidation"); 2456 return; 2457 } 2458 ret = vtd_context_cache_invalidate(s, val); 2459 /* Invalidation completed. Change something to show */ 2460 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 2461 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 2462 ret); 2463 } 2464 } 2465 2466 /* Handle write to IOTLB Invalidation Register */ 2467 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 2468 { 2469 uint64_t ret; 2470 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 2471 2472 /* IOTLB invalidation request */ 2473 if (val & VTD_TLB_IVT) { 2474 if (s->qi_enabled) { 2475 error_report_once("Queued Invalidation enabled, " 2476 "should not use register-based invalidation"); 2477 return; 2478 } 2479 ret = vtd_iotlb_flush(s, val); 2480 /* Invalidation completed. Change something to show */ 2481 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 2482 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 2483 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 2484 } 2485 } 2486 2487 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 2488 static bool vtd_get_inv_desc(IntelIOMMUState *s, 2489 VTDInvDesc *inv_desc) 2490 { 2491 dma_addr_t base_addr = s->iq; 2492 uint32_t offset = s->iq_head; 2493 uint32_t dw = s->iq_dw ? 32 : 16; 2494 dma_addr_t addr = base_addr + offset * dw; 2495 2496 if (dma_memory_read(&address_space_memory, addr, 2497 inv_desc, dw, MEMTXATTRS_UNSPECIFIED)) { 2498 error_report_once("Read INV DESC failed."); 2499 return false; 2500 } 2501 inv_desc->lo = le64_to_cpu(inv_desc->lo); 2502 inv_desc->hi = le64_to_cpu(inv_desc->hi); 2503 if (dw == 32) { 2504 inv_desc->val[2] = le64_to_cpu(inv_desc->val[2]); 2505 inv_desc->val[3] = le64_to_cpu(inv_desc->val[3]); 2506 } 2507 return true; 2508 } 2509 2510 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 2511 { 2512 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 2513 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 2514 error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64 2515 " (reserved nonzero)", __func__, inv_desc->hi, 2516 inv_desc->lo); 2517 return false; 2518 } 2519 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 2520 /* Status Write */ 2521 uint32_t status_data = (uint32_t)(inv_desc->lo >> 2522 VTD_INV_DESC_WAIT_DATA_SHIFT); 2523 2524 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 2525 2526 /* FIXME: need to be masked with HAW? */ 2527 dma_addr_t status_addr = inv_desc->hi; 2528 trace_vtd_inv_desc_wait_sw(status_addr, status_data); 2529 status_data = cpu_to_le32(status_data); 2530 if (dma_memory_write(&address_space_memory, status_addr, 2531 &status_data, sizeof(status_data), 2532 MEMTXATTRS_UNSPECIFIED)) { 2533 trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo); 2534 return false; 2535 } 2536 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 2537 /* Interrupt flag */ 2538 vtd_generate_completion_event(s); 2539 } else { 2540 error_report_once("%s: invalid wait desc: hi=%"PRIx64", lo=%"PRIx64 2541 " (unknown type)", __func__, inv_desc->hi, 2542 inv_desc->lo); 2543 return false; 2544 } 2545 return true; 2546 } 2547 2548 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 2549 VTDInvDesc *inv_desc) 2550 { 2551 uint16_t sid, fmask; 2552 2553 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 2554 error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64 2555 " (reserved nonzero)", __func__, inv_desc->hi, 2556 inv_desc->lo); 2557 return false; 2558 } 2559 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 2560 case VTD_INV_DESC_CC_DOMAIN: 2561 trace_vtd_inv_desc_cc_domain( 2562 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 2563 /* Fall through */ 2564 case VTD_INV_DESC_CC_GLOBAL: 2565 vtd_context_global_invalidate(s); 2566 break; 2567 2568 case VTD_INV_DESC_CC_DEVICE: 2569 sid = VTD_INV_DESC_CC_SID(inv_desc->lo); 2570 fmask = VTD_INV_DESC_CC_FM(inv_desc->lo); 2571 vtd_context_device_invalidate(s, sid, fmask); 2572 break; 2573 2574 default: 2575 error_report_once("%s: invalid cc inv desc: hi=%"PRIx64", lo=%"PRIx64 2576 " (invalid type)", __func__, inv_desc->hi, 2577 inv_desc->lo); 2578 return false; 2579 } 2580 return true; 2581 } 2582 2583 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 2584 { 2585 uint16_t domain_id; 2586 uint8_t am; 2587 hwaddr addr; 2588 2589 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 2590 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 2591 error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2592 ", lo=0x%"PRIx64" (reserved bits unzero)", 2593 __func__, inv_desc->hi, inv_desc->lo); 2594 return false; 2595 } 2596 2597 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 2598 case VTD_INV_DESC_IOTLB_GLOBAL: 2599 vtd_iotlb_global_invalidate(s); 2600 break; 2601 2602 case VTD_INV_DESC_IOTLB_DOMAIN: 2603 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 2604 vtd_iotlb_domain_invalidate(s, domain_id); 2605 break; 2606 2607 case VTD_INV_DESC_IOTLB_PAGE: 2608 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 2609 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 2610 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 2611 if (am > VTD_MAMV) { 2612 error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2613 ", lo=0x%"PRIx64" (am=%u > VTD_MAMV=%u)", 2614 __func__, inv_desc->hi, inv_desc->lo, 2615 am, (unsigned)VTD_MAMV); 2616 return false; 2617 } 2618 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 2619 break; 2620 2621 default: 2622 error_report_once("%s: invalid iotlb inv desc: hi=0x%"PRIx64 2623 ", lo=0x%"PRIx64" (type mismatch: 0x%llx)", 2624 __func__, inv_desc->hi, inv_desc->lo, 2625 inv_desc->lo & VTD_INV_DESC_IOTLB_G); 2626 return false; 2627 } 2628 return true; 2629 } 2630 2631 static bool vtd_process_inv_iec_desc(IntelIOMMUState *s, 2632 VTDInvDesc *inv_desc) 2633 { 2634 trace_vtd_inv_desc_iec(inv_desc->iec.granularity, 2635 inv_desc->iec.index, 2636 inv_desc->iec.index_mask); 2637 2638 vtd_iec_notify_all(s, !inv_desc->iec.granularity, 2639 inv_desc->iec.index, 2640 inv_desc->iec.index_mask); 2641 return true; 2642 } 2643 2644 static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s, 2645 VTDInvDesc *inv_desc) 2646 { 2647 VTDAddressSpace *vtd_dev_as; 2648 IOMMUTLBEvent event; 2649 hwaddr addr; 2650 uint64_t sz; 2651 uint16_t sid; 2652 bool size; 2653 2654 addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi); 2655 sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo); 2656 size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi); 2657 2658 if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) || 2659 (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) { 2660 error_report_once("%s: invalid dev-iotlb inv desc: hi=%"PRIx64 2661 ", lo=%"PRIx64" (reserved nonzero)", __func__, 2662 inv_desc->hi, inv_desc->lo); 2663 return false; 2664 } 2665 2666 /* 2667 * Using sid is OK since the guest should have finished the 2668 * initialization of both the bus and device. 2669 */ 2670 vtd_dev_as = vtd_get_as_by_sid(s, sid); 2671 if (!vtd_dev_as) { 2672 goto done; 2673 } 2674 2675 /* According to ATS spec table 2.4: 2676 * S = 0, bits 15:12 = xxxx range size: 4K 2677 * S = 1, bits 15:12 = xxx0 range size: 8K 2678 * S = 1, bits 15:12 = xx01 range size: 16K 2679 * S = 1, bits 15:12 = x011 range size: 32K 2680 * S = 1, bits 15:12 = 0111 range size: 64K 2681 * ... 2682 */ 2683 if (size) { 2684 sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT); 2685 addr &= ~(sz - 1); 2686 } else { 2687 sz = VTD_PAGE_SIZE; 2688 } 2689 2690 event.type = IOMMU_NOTIFIER_DEVIOTLB_UNMAP; 2691 event.entry.target_as = &vtd_dev_as->as; 2692 event.entry.addr_mask = sz - 1; 2693 event.entry.iova = addr; 2694 event.entry.perm = IOMMU_NONE; 2695 event.entry.translated_addr = 0; 2696 memory_region_notify_iommu(&vtd_dev_as->iommu, 0, event); 2697 2698 done: 2699 return true; 2700 } 2701 2702 static bool vtd_process_inv_desc(IntelIOMMUState *s) 2703 { 2704 VTDInvDesc inv_desc; 2705 uint8_t desc_type; 2706 2707 trace_vtd_inv_qi_head(s->iq_head); 2708 if (!vtd_get_inv_desc(s, &inv_desc)) { 2709 s->iq_last_desc_type = VTD_INV_DESC_NONE; 2710 return false; 2711 } 2712 2713 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 2714 /* FIXME: should update at first or at last? */ 2715 s->iq_last_desc_type = desc_type; 2716 2717 switch (desc_type) { 2718 case VTD_INV_DESC_CC: 2719 trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo); 2720 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 2721 return false; 2722 } 2723 break; 2724 2725 case VTD_INV_DESC_IOTLB: 2726 trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo); 2727 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 2728 return false; 2729 } 2730 break; 2731 2732 /* 2733 * TODO: the entity of below two cases will be implemented in future series. 2734 * To make guest (which integrates scalable mode support patch set in 2735 * iommu driver) work, just return true is enough so far. 2736 */ 2737 case VTD_INV_DESC_PC: 2738 break; 2739 2740 case VTD_INV_DESC_PIOTLB: 2741 break; 2742 2743 case VTD_INV_DESC_WAIT: 2744 trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo); 2745 if (!vtd_process_wait_desc(s, &inv_desc)) { 2746 return false; 2747 } 2748 break; 2749 2750 case VTD_INV_DESC_IEC: 2751 trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo); 2752 if (!vtd_process_inv_iec_desc(s, &inv_desc)) { 2753 return false; 2754 } 2755 break; 2756 2757 case VTD_INV_DESC_DEVICE: 2758 trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo); 2759 if (!vtd_process_device_iotlb_desc(s, &inv_desc)) { 2760 return false; 2761 } 2762 break; 2763 2764 default: 2765 error_report_once("%s: invalid inv desc: hi=%"PRIx64", lo=%"PRIx64 2766 " (unknown type)", __func__, inv_desc.hi, 2767 inv_desc.lo); 2768 return false; 2769 } 2770 s->iq_head++; 2771 if (s->iq_head == s->iq_size) { 2772 s->iq_head = 0; 2773 } 2774 return true; 2775 } 2776 2777 /* Try to fetch and process more Invalidation Descriptors */ 2778 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 2779 { 2780 int qi_shift; 2781 2782 /* Refer to 10.4.23 of VT-d spec 3.0 */ 2783 qi_shift = s->iq_dw ? VTD_IQH_QH_SHIFT_5 : VTD_IQH_QH_SHIFT_4; 2784 2785 trace_vtd_inv_qi_fetch(); 2786 2787 if (s->iq_tail >= s->iq_size) { 2788 /* Detects an invalid Tail pointer */ 2789 error_report_once("%s: detected invalid QI tail " 2790 "(tail=0x%x, size=0x%x)", 2791 __func__, s->iq_tail, s->iq_size); 2792 vtd_handle_inv_queue_error(s); 2793 return; 2794 } 2795 while (s->iq_head != s->iq_tail) { 2796 if (!vtd_process_inv_desc(s)) { 2797 /* Invalidation Queue Errors */ 2798 vtd_handle_inv_queue_error(s); 2799 break; 2800 } 2801 /* Must update the IQH_REG in time */ 2802 vtd_set_quad_raw(s, DMAR_IQH_REG, 2803 (((uint64_t)(s->iq_head)) << qi_shift) & 2804 VTD_IQH_QH_MASK); 2805 } 2806 } 2807 2808 /* Handle write to Invalidation Queue Tail Register */ 2809 static void vtd_handle_iqt_write(IntelIOMMUState *s) 2810 { 2811 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 2812 2813 if (s->iq_dw && (val & VTD_IQT_QT_256_RSV_BIT)) { 2814 error_report_once("%s: RSV bit is set: val=0x%"PRIx64, 2815 __func__, val); 2816 return; 2817 } 2818 s->iq_tail = VTD_IQT_QT(s->iq_dw, val); 2819 trace_vtd_inv_qi_tail(s->iq_tail); 2820 2821 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 2822 /* Process Invalidation Queue here */ 2823 vtd_fetch_inv_desc(s); 2824 } 2825 } 2826 2827 static void vtd_handle_fsts_write(IntelIOMMUState *s) 2828 { 2829 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 2830 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 2831 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 2832 2833 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 2834 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 2835 trace_vtd_fsts_clear_ip(); 2836 } 2837 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 2838 * Descriptors if there are any when Queued Invalidation is enabled? 2839 */ 2840 } 2841 2842 static void vtd_handle_fectl_write(IntelIOMMUState *s) 2843 { 2844 uint32_t fectl_reg; 2845 /* FIXME: when software clears the IM field, check the IP field. But do we 2846 * need to compare the old value and the new value to conclude that 2847 * software clears the IM field? Or just check if the IM field is zero? 2848 */ 2849 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 2850 2851 trace_vtd_reg_write_fectl(fectl_reg); 2852 2853 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 2854 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 2855 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 2856 } 2857 } 2858 2859 static void vtd_handle_ics_write(IntelIOMMUState *s) 2860 { 2861 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 2862 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 2863 2864 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 2865 trace_vtd_reg_ics_clear_ip(); 2866 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 2867 } 2868 } 2869 2870 static void vtd_handle_iectl_write(IntelIOMMUState *s) 2871 { 2872 uint32_t iectl_reg; 2873 /* FIXME: when software clears the IM field, check the IP field. But do we 2874 * need to compare the old value and the new value to conclude that 2875 * software clears the IM field? Or just check if the IM field is zero? 2876 */ 2877 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 2878 2879 trace_vtd_reg_write_iectl(iectl_reg); 2880 2881 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 2882 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 2883 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 2884 } 2885 } 2886 2887 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 2888 { 2889 IntelIOMMUState *s = opaque; 2890 uint64_t val; 2891 2892 trace_vtd_reg_read(addr, size); 2893 2894 if (addr + size > DMAR_REG_SIZE) { 2895 error_report_once("%s: MMIO over range: addr=0x%" PRIx64 2896 " size=0x%x", __func__, addr, size); 2897 return (uint64_t)-1; 2898 } 2899 2900 switch (addr) { 2901 /* Root Table Address Register, 64-bit */ 2902 case DMAR_RTADDR_REG: 2903 val = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 2904 if (size == 4) { 2905 val = val & ((1ULL << 32) - 1); 2906 } 2907 break; 2908 2909 case DMAR_RTADDR_REG_HI: 2910 assert(size == 4); 2911 val = vtd_get_quad_raw(s, DMAR_RTADDR_REG) >> 32; 2912 break; 2913 2914 /* Invalidation Queue Address Register, 64-bit */ 2915 case DMAR_IQA_REG: 2916 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 2917 if (size == 4) { 2918 val = val & ((1ULL << 32) - 1); 2919 } 2920 break; 2921 2922 case DMAR_IQA_REG_HI: 2923 assert(size == 4); 2924 val = s->iq >> 32; 2925 break; 2926 2927 default: 2928 if (size == 4) { 2929 val = vtd_get_long(s, addr); 2930 } else { 2931 val = vtd_get_quad(s, addr); 2932 } 2933 } 2934 2935 return val; 2936 } 2937 2938 static void vtd_mem_write(void *opaque, hwaddr addr, 2939 uint64_t val, unsigned size) 2940 { 2941 IntelIOMMUState *s = opaque; 2942 2943 trace_vtd_reg_write(addr, size, val); 2944 2945 if (addr + size > DMAR_REG_SIZE) { 2946 error_report_once("%s: MMIO over range: addr=0x%" PRIx64 2947 " size=0x%x", __func__, addr, size); 2948 return; 2949 } 2950 2951 switch (addr) { 2952 /* Global Command Register, 32-bit */ 2953 case DMAR_GCMD_REG: 2954 vtd_set_long(s, addr, val); 2955 vtd_handle_gcmd_write(s); 2956 break; 2957 2958 /* Context Command Register, 64-bit */ 2959 case DMAR_CCMD_REG: 2960 if (size == 4) { 2961 vtd_set_long(s, addr, val); 2962 } else { 2963 vtd_set_quad(s, addr, val); 2964 vtd_handle_ccmd_write(s); 2965 } 2966 break; 2967 2968 case DMAR_CCMD_REG_HI: 2969 assert(size == 4); 2970 vtd_set_long(s, addr, val); 2971 vtd_handle_ccmd_write(s); 2972 break; 2973 2974 /* IOTLB Invalidation Register, 64-bit */ 2975 case DMAR_IOTLB_REG: 2976 if (size == 4) { 2977 vtd_set_long(s, addr, val); 2978 } else { 2979 vtd_set_quad(s, addr, val); 2980 vtd_handle_iotlb_write(s); 2981 } 2982 break; 2983 2984 case DMAR_IOTLB_REG_HI: 2985 assert(size == 4); 2986 vtd_set_long(s, addr, val); 2987 vtd_handle_iotlb_write(s); 2988 break; 2989 2990 /* Invalidate Address Register, 64-bit */ 2991 case DMAR_IVA_REG: 2992 if (size == 4) { 2993 vtd_set_long(s, addr, val); 2994 } else { 2995 vtd_set_quad(s, addr, val); 2996 } 2997 break; 2998 2999 case DMAR_IVA_REG_HI: 3000 assert(size == 4); 3001 vtd_set_long(s, addr, val); 3002 break; 3003 3004 /* Fault Status Register, 32-bit */ 3005 case DMAR_FSTS_REG: 3006 assert(size == 4); 3007 vtd_set_long(s, addr, val); 3008 vtd_handle_fsts_write(s); 3009 break; 3010 3011 /* Fault Event Control Register, 32-bit */ 3012 case DMAR_FECTL_REG: 3013 assert(size == 4); 3014 vtd_set_long(s, addr, val); 3015 vtd_handle_fectl_write(s); 3016 break; 3017 3018 /* Fault Event Data Register, 32-bit */ 3019 case DMAR_FEDATA_REG: 3020 assert(size == 4); 3021 vtd_set_long(s, addr, val); 3022 break; 3023 3024 /* Fault Event Address Register, 32-bit */ 3025 case DMAR_FEADDR_REG: 3026 if (size == 4) { 3027 vtd_set_long(s, addr, val); 3028 } else { 3029 /* 3030 * While the register is 32-bit only, some guests (Xen...) write to 3031 * it with 64-bit. 3032 */ 3033 vtd_set_quad(s, addr, val); 3034 } 3035 break; 3036 3037 /* Fault Event Upper Address Register, 32-bit */ 3038 case DMAR_FEUADDR_REG: 3039 assert(size == 4); 3040 vtd_set_long(s, addr, val); 3041 break; 3042 3043 /* Protected Memory Enable Register, 32-bit */ 3044 case DMAR_PMEN_REG: 3045 assert(size == 4); 3046 vtd_set_long(s, addr, val); 3047 break; 3048 3049 /* Root Table Address Register, 64-bit */ 3050 case DMAR_RTADDR_REG: 3051 if (size == 4) { 3052 vtd_set_long(s, addr, val); 3053 } else { 3054 vtd_set_quad(s, addr, val); 3055 } 3056 break; 3057 3058 case DMAR_RTADDR_REG_HI: 3059 assert(size == 4); 3060 vtd_set_long(s, addr, val); 3061 break; 3062 3063 /* Invalidation Queue Tail Register, 64-bit */ 3064 case DMAR_IQT_REG: 3065 if (size == 4) { 3066 vtd_set_long(s, addr, val); 3067 } else { 3068 vtd_set_quad(s, addr, val); 3069 } 3070 vtd_handle_iqt_write(s); 3071 break; 3072 3073 case DMAR_IQT_REG_HI: 3074 assert(size == 4); 3075 vtd_set_long(s, addr, val); 3076 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 3077 break; 3078 3079 /* Invalidation Queue Address Register, 64-bit */ 3080 case DMAR_IQA_REG: 3081 if (size == 4) { 3082 vtd_set_long(s, addr, val); 3083 } else { 3084 vtd_set_quad(s, addr, val); 3085 } 3086 vtd_update_iq_dw(s); 3087 break; 3088 3089 case DMAR_IQA_REG_HI: 3090 assert(size == 4); 3091 vtd_set_long(s, addr, val); 3092 break; 3093 3094 /* Invalidation Completion Status Register, 32-bit */ 3095 case DMAR_ICS_REG: 3096 assert(size == 4); 3097 vtd_set_long(s, addr, val); 3098 vtd_handle_ics_write(s); 3099 break; 3100 3101 /* Invalidation Event Control Register, 32-bit */ 3102 case DMAR_IECTL_REG: 3103 assert(size == 4); 3104 vtd_set_long(s, addr, val); 3105 vtd_handle_iectl_write(s); 3106 break; 3107 3108 /* Invalidation Event Data Register, 32-bit */ 3109 case DMAR_IEDATA_REG: 3110 assert(size == 4); 3111 vtd_set_long(s, addr, val); 3112 break; 3113 3114 /* Invalidation Event Address Register, 32-bit */ 3115 case DMAR_IEADDR_REG: 3116 assert(size == 4); 3117 vtd_set_long(s, addr, val); 3118 break; 3119 3120 /* Invalidation Event Upper Address Register, 32-bit */ 3121 case DMAR_IEUADDR_REG: 3122 assert(size == 4); 3123 vtd_set_long(s, addr, val); 3124 break; 3125 3126 /* Fault Recording Registers, 128-bit */ 3127 case DMAR_FRCD_REG_0_0: 3128 if (size == 4) { 3129 vtd_set_long(s, addr, val); 3130 } else { 3131 vtd_set_quad(s, addr, val); 3132 } 3133 break; 3134 3135 case DMAR_FRCD_REG_0_1: 3136 assert(size == 4); 3137 vtd_set_long(s, addr, val); 3138 break; 3139 3140 case DMAR_FRCD_REG_0_2: 3141 if (size == 4) { 3142 vtd_set_long(s, addr, val); 3143 } else { 3144 vtd_set_quad(s, addr, val); 3145 /* May clear bit 127 (Fault), update PPF */ 3146 vtd_update_fsts_ppf(s); 3147 } 3148 break; 3149 3150 case DMAR_FRCD_REG_0_3: 3151 assert(size == 4); 3152 vtd_set_long(s, addr, val); 3153 /* May clear bit 127 (Fault), update PPF */ 3154 vtd_update_fsts_ppf(s); 3155 break; 3156 3157 case DMAR_IRTA_REG: 3158 if (size == 4) { 3159 vtd_set_long(s, addr, val); 3160 } else { 3161 vtd_set_quad(s, addr, val); 3162 } 3163 break; 3164 3165 case DMAR_IRTA_REG_HI: 3166 assert(size == 4); 3167 vtd_set_long(s, addr, val); 3168 break; 3169 3170 default: 3171 if (size == 4) { 3172 vtd_set_long(s, addr, val); 3173 } else { 3174 vtd_set_quad(s, addr, val); 3175 } 3176 } 3177 } 3178 3179 static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, 3180 IOMMUAccessFlags flag, int iommu_idx) 3181 { 3182 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 3183 IntelIOMMUState *s = vtd_as->iommu_state; 3184 IOMMUTLBEntry iotlb = { 3185 /* We'll fill in the rest later. */ 3186 .target_as = &address_space_memory, 3187 }; 3188 bool success; 3189 3190 if (likely(s->dmar_enabled)) { 3191 success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, 3192 addr, flag & IOMMU_WO, &iotlb); 3193 } else { 3194 /* DMAR disabled, passthrough, use 4k-page*/ 3195 iotlb.iova = addr & VTD_PAGE_MASK_4K; 3196 iotlb.translated_addr = addr & VTD_PAGE_MASK_4K; 3197 iotlb.addr_mask = ~VTD_PAGE_MASK_4K; 3198 iotlb.perm = IOMMU_RW; 3199 success = true; 3200 } 3201 3202 if (likely(success)) { 3203 trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus), 3204 VTD_PCI_SLOT(vtd_as->devfn), 3205 VTD_PCI_FUNC(vtd_as->devfn), 3206 iotlb.iova, iotlb.translated_addr, 3207 iotlb.addr_mask); 3208 } else { 3209 error_report_once("%s: detected translation failure " 3210 "(dev=%02x:%02x:%02x, iova=0x%" PRIx64 ")", 3211 __func__, pci_bus_num(vtd_as->bus), 3212 VTD_PCI_SLOT(vtd_as->devfn), 3213 VTD_PCI_FUNC(vtd_as->devfn), 3214 addr); 3215 } 3216 3217 return iotlb; 3218 } 3219 3220 static int vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu, 3221 IOMMUNotifierFlag old, 3222 IOMMUNotifierFlag new, 3223 Error **errp) 3224 { 3225 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 3226 IntelIOMMUState *s = vtd_as->iommu_state; 3227 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3228 3229 /* TODO: add support for VFIO and vhost users */ 3230 if (s->snoop_control) { 3231 error_setg_errno(errp, ENOTSUP, 3232 "Snoop Control with vhost or VFIO is not supported"); 3233 return -ENOTSUP; 3234 } 3235 if (!s->caching_mode && (new & IOMMU_NOTIFIER_MAP)) { 3236 error_setg_errno(errp, ENOTSUP, 3237 "device %02x.%02x.%x requires caching mode", 3238 pci_bus_num(vtd_as->bus), PCI_SLOT(vtd_as->devfn), 3239 PCI_FUNC(vtd_as->devfn)); 3240 return -ENOTSUP; 3241 } 3242 if (!x86_iommu->dt_supported && (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP)) { 3243 error_setg_errno(errp, ENOTSUP, 3244 "device %02x.%02x.%x requires device IOTLB mode", 3245 pci_bus_num(vtd_as->bus), PCI_SLOT(vtd_as->devfn), 3246 PCI_FUNC(vtd_as->devfn)); 3247 return -ENOTSUP; 3248 } 3249 3250 /* Update per-address-space notifier flags */ 3251 vtd_as->notifier_flags = new; 3252 3253 if (old == IOMMU_NOTIFIER_NONE) { 3254 QLIST_INSERT_HEAD(&s->vtd_as_with_notifiers, vtd_as, next); 3255 } else if (new == IOMMU_NOTIFIER_NONE) { 3256 QLIST_REMOVE(vtd_as, next); 3257 } 3258 return 0; 3259 } 3260 3261 static int vtd_post_load(void *opaque, int version_id) 3262 { 3263 IntelIOMMUState *iommu = opaque; 3264 3265 /* 3266 * We don't need to migrate the root_scalable because we can 3267 * simply do the calculation after the loading is complete. We 3268 * can actually do similar things with root, dmar_enabled, etc. 3269 * however since we've had them already so we'd better keep them 3270 * for compatibility of migration. 3271 */ 3272 vtd_update_scalable_state(iommu); 3273 3274 vtd_update_iq_dw(iommu); 3275 3276 /* 3277 * Memory regions are dynamically turned on/off depending on 3278 * context entry configurations from the guest. After migration, 3279 * we need to make sure the memory regions are still correct. 3280 */ 3281 vtd_switch_address_space_all(iommu); 3282 3283 return 0; 3284 } 3285 3286 static const VMStateDescription vtd_vmstate = { 3287 .name = "iommu-intel", 3288 .version_id = 1, 3289 .minimum_version_id = 1, 3290 .priority = MIG_PRI_IOMMU, 3291 .post_load = vtd_post_load, 3292 .fields = (const VMStateField[]) { 3293 VMSTATE_UINT64(root, IntelIOMMUState), 3294 VMSTATE_UINT64(intr_root, IntelIOMMUState), 3295 VMSTATE_UINT64(iq, IntelIOMMUState), 3296 VMSTATE_UINT32(intr_size, IntelIOMMUState), 3297 VMSTATE_UINT16(iq_head, IntelIOMMUState), 3298 VMSTATE_UINT16(iq_tail, IntelIOMMUState), 3299 VMSTATE_UINT16(iq_size, IntelIOMMUState), 3300 VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState), 3301 VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE), 3302 VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState), 3303 VMSTATE_UNUSED(1), /* bool root_extended is obsolete by VT-d */ 3304 VMSTATE_BOOL(dmar_enabled, IntelIOMMUState), 3305 VMSTATE_BOOL(qi_enabled, IntelIOMMUState), 3306 VMSTATE_BOOL(intr_enabled, IntelIOMMUState), 3307 VMSTATE_BOOL(intr_eime, IntelIOMMUState), 3308 VMSTATE_END_OF_LIST() 3309 } 3310 }; 3311 3312 static const MemoryRegionOps vtd_mem_ops = { 3313 .read = vtd_mem_read, 3314 .write = vtd_mem_write, 3315 .endianness = DEVICE_LITTLE_ENDIAN, 3316 .impl = { 3317 .min_access_size = 4, 3318 .max_access_size = 8, 3319 }, 3320 .valid = { 3321 .min_access_size = 4, 3322 .max_access_size = 8, 3323 }, 3324 }; 3325 3326 static Property vtd_properties[] = { 3327 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 3328 DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim, 3329 ON_OFF_AUTO_AUTO), 3330 DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false), 3331 DEFINE_PROP_UINT8("aw-bits", IntelIOMMUState, aw_bits, 3332 VTD_HOST_ADDRESS_WIDTH), 3333 DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE), 3334 DEFINE_PROP_BOOL("x-scalable-mode", IntelIOMMUState, scalable_mode, FALSE), 3335 DEFINE_PROP_BOOL("snoop-control", IntelIOMMUState, snoop_control, false), 3336 DEFINE_PROP_BOOL("x-pasid-mode", IntelIOMMUState, pasid, false), 3337 DEFINE_PROP_BOOL("dma-drain", IntelIOMMUState, dma_drain, true), 3338 DEFINE_PROP_BOOL("dma-translation", IntelIOMMUState, dma_translation, true), 3339 DEFINE_PROP_END_OF_LIST(), 3340 }; 3341 3342 /* Read IRTE entry with specific index */ 3343 static bool vtd_irte_get(IntelIOMMUState *iommu, uint16_t index, 3344 VTD_IR_TableEntry *entry, uint16_t sid, 3345 bool do_fault) 3346 { 3347 static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \ 3348 {0xffff, 0xfffb, 0xfff9, 0xfff8}; 3349 dma_addr_t addr = 0x00; 3350 uint16_t mask, source_id; 3351 uint8_t bus, bus_max, bus_min; 3352 3353 if (index >= iommu->intr_size) { 3354 error_report_once("%s: index too large: ind=0x%x", 3355 __func__, index); 3356 if (do_fault) { 3357 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_INDEX_OVER, index); 3358 } 3359 return false; 3360 } 3361 3362 addr = iommu->intr_root + index * sizeof(*entry); 3363 if (dma_memory_read(&address_space_memory, addr, 3364 entry, sizeof(*entry), MEMTXATTRS_UNSPECIFIED)) { 3365 error_report_once("%s: read failed: ind=0x%x addr=0x%" PRIx64, 3366 __func__, index, addr); 3367 if (do_fault) { 3368 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_ROOT_INVAL, index); 3369 } 3370 return false; 3371 } 3372 3373 entry->data[0] = le64_to_cpu(entry->data[0]); 3374 entry->data[1] = le64_to_cpu(entry->data[1]); 3375 3376 trace_vtd_ir_irte_get(index, entry->data[1], entry->data[0]); 3377 3378 /* 3379 * The remaining potential fault conditions are "qualified" by the 3380 * Fault Processing Disable bit in the IRTE. Even "not present". 3381 * So just clear the do_fault flag if PFD is set, which will 3382 * prevent faults being raised. 3383 */ 3384 if (entry->irte.fault_disable) { 3385 do_fault = false; 3386 } 3387 3388 if (!entry->irte.present) { 3389 error_report_once("%s: detected non-present IRTE " 3390 "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")", 3391 __func__, index, entry->data[1], entry->data[0]); 3392 if (do_fault) { 3393 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_ENTRY_P, index); 3394 } 3395 return false; 3396 } 3397 3398 if (entry->irte.__reserved_0 || entry->irte.__reserved_1 || 3399 entry->irte.__reserved_2) { 3400 error_report_once("%s: detected non-zero reserved IRTE " 3401 "(index=%u, high=0x%" PRIx64 ", low=0x%" PRIx64 ")", 3402 __func__, index, entry->data[1], entry->data[0]); 3403 if (do_fault) { 3404 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_IRTE_RSVD, index); 3405 } 3406 return false; 3407 } 3408 3409 if (sid != X86_IOMMU_SID_INVALID) { 3410 /* Validate IRTE SID */ 3411 source_id = entry->irte.source_id; 3412 switch (entry->irte.sid_vtype) { 3413 case VTD_SVT_NONE: 3414 break; 3415 3416 case VTD_SVT_ALL: 3417 mask = vtd_svt_mask[entry->irte.sid_q]; 3418 if ((source_id & mask) != (sid & mask)) { 3419 error_report_once("%s: invalid IRTE SID " 3420 "(index=%u, sid=%u, source_id=%u)", 3421 __func__, index, sid, source_id); 3422 if (do_fault) { 3423 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3424 } 3425 return false; 3426 } 3427 break; 3428 3429 case VTD_SVT_BUS: 3430 bus_max = source_id >> 8; 3431 bus_min = source_id & 0xff; 3432 bus = sid >> 8; 3433 if (bus > bus_max || bus < bus_min) { 3434 error_report_once("%s: invalid SVT_BUS " 3435 "(index=%u, bus=%u, min=%u, max=%u)", 3436 __func__, index, bus, bus_min, bus_max); 3437 if (do_fault) { 3438 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3439 } 3440 return false; 3441 } 3442 break; 3443 3444 default: 3445 error_report_once("%s: detected invalid IRTE SVT " 3446 "(index=%u, type=%d)", __func__, 3447 index, entry->irte.sid_vtype); 3448 /* Take this as verification failure. */ 3449 if (do_fault) { 3450 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_SID_ERR, index); 3451 } 3452 return false; 3453 } 3454 } 3455 3456 return true; 3457 } 3458 3459 /* Fetch IRQ information of specific IR index */ 3460 static bool vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index, 3461 X86IOMMUIrq *irq, uint16_t sid, bool do_fault) 3462 { 3463 VTD_IR_TableEntry irte = {}; 3464 3465 if (!vtd_irte_get(iommu, index, &irte, sid, do_fault)) { 3466 return false; 3467 } 3468 3469 irq->trigger_mode = irte.irte.trigger_mode; 3470 irq->vector = irte.irte.vector; 3471 irq->delivery_mode = irte.irte.delivery_mode; 3472 irq->dest = irte.irte.dest_id; 3473 if (!iommu->intr_eime) { 3474 #define VTD_IR_APIC_DEST_MASK (0xff00ULL) 3475 #define VTD_IR_APIC_DEST_SHIFT (8) 3476 irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >> 3477 VTD_IR_APIC_DEST_SHIFT; 3478 } 3479 irq->dest_mode = irte.irte.dest_mode; 3480 irq->redir_hint = irte.irte.redir_hint; 3481 3482 trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector, 3483 irq->delivery_mode, irq->dest, irq->dest_mode); 3484 3485 return true; 3486 } 3487 3488 /* Interrupt remapping for MSI/MSI-X entry */ 3489 static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu, 3490 MSIMessage *origin, 3491 MSIMessage *translated, 3492 uint16_t sid, bool do_fault) 3493 { 3494 VTD_IR_MSIAddress addr; 3495 uint16_t index; 3496 X86IOMMUIrq irq = {}; 3497 3498 assert(origin && translated); 3499 3500 trace_vtd_ir_remap_msi_req(origin->address, origin->data); 3501 3502 if (!iommu || !iommu->intr_enabled) { 3503 memcpy(translated, origin, sizeof(*origin)); 3504 goto out; 3505 } 3506 3507 if (origin->address & VTD_MSI_ADDR_HI_MASK) { 3508 error_report_once("%s: MSI address high 32 bits non-zero detected: " 3509 "address=0x%" PRIx64, __func__, origin->address); 3510 if (do_fault) { 3511 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 3512 } 3513 return -EINVAL; 3514 } 3515 3516 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK; 3517 if (addr.addr.__head != 0xfee) { 3518 error_report_once("%s: MSI address low 32 bit invalid: 0x%" PRIx32, 3519 __func__, addr.data); 3520 if (do_fault) { 3521 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 3522 } 3523 return -EINVAL; 3524 } 3525 3526 /* This is compatible mode. */ 3527 if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) { 3528 memcpy(translated, origin, sizeof(*origin)); 3529 goto out; 3530 } 3531 3532 index = addr.addr.index_h << 15 | addr.addr.index_l; 3533 3534 #define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff) 3535 #define VTD_IR_MSI_DATA_RESERVED (0xffff0000) 3536 3537 if (addr.addr.sub_valid) { 3538 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */ 3539 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE; 3540 } 3541 3542 if (!vtd_remap_irq_get(iommu, index, &irq, sid, do_fault)) { 3543 return -EINVAL; 3544 } 3545 3546 if (addr.addr.sub_valid) { 3547 trace_vtd_ir_remap_type("MSI"); 3548 if (origin->data & VTD_IR_MSI_DATA_RESERVED) { 3549 error_report_once("%s: invalid IR MSI " 3550 "(sid=%u, address=0x%" PRIx64 3551 ", data=0x%" PRIx32 ")", 3552 __func__, sid, origin->address, origin->data); 3553 if (do_fault) { 3554 vtd_report_ir_fault(iommu, sid, VTD_FR_IR_REQ_RSVD, 0); 3555 } 3556 return -EINVAL; 3557 } 3558 } else { 3559 uint8_t vector = origin->data & 0xff; 3560 uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1; 3561 3562 trace_vtd_ir_remap_type("IOAPIC"); 3563 /* IOAPIC entry vector should be aligned with IRTE vector 3564 * (see vt-d spec 5.1.5.1). */ 3565 if (vector != irq.vector) { 3566 trace_vtd_warn_ir_vector(sid, index, vector, irq.vector); 3567 } 3568 3569 /* The Trigger Mode field must match the Trigger Mode in the IRTE. 3570 * (see vt-d spec 5.1.5.1). */ 3571 if (trigger_mode != irq.trigger_mode) { 3572 trace_vtd_warn_ir_trigger(sid, index, trigger_mode, 3573 irq.trigger_mode); 3574 } 3575 } 3576 3577 /* 3578 * We'd better keep the last two bits, assuming that guest OS 3579 * might modify it. Keep it does not hurt after all. 3580 */ 3581 irq.msi_addr_last_bits = addr.addr.__not_care; 3582 3583 /* Translate X86IOMMUIrq to MSI message */ 3584 x86_iommu_irq_to_msi_message(&irq, translated); 3585 3586 out: 3587 trace_vtd_ir_remap_msi(origin->address, origin->data, 3588 translated->address, translated->data); 3589 return 0; 3590 } 3591 3592 static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src, 3593 MSIMessage *dst, uint16_t sid) 3594 { 3595 return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu), 3596 src, dst, sid, false); 3597 } 3598 3599 static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr, 3600 uint64_t *data, unsigned size, 3601 MemTxAttrs attrs) 3602 { 3603 return MEMTX_OK; 3604 } 3605 3606 static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr, 3607 uint64_t value, unsigned size, 3608 MemTxAttrs attrs) 3609 { 3610 int ret = 0; 3611 MSIMessage from = {}, to = {}; 3612 uint16_t sid = X86_IOMMU_SID_INVALID; 3613 3614 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST; 3615 from.data = (uint32_t) value; 3616 3617 if (!attrs.unspecified) { 3618 /* We have explicit Source ID */ 3619 sid = attrs.requester_id; 3620 } 3621 3622 ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid, true); 3623 if (ret) { 3624 /* Drop this interrupt */ 3625 return MEMTX_ERROR; 3626 } 3627 3628 apic_get_class(NULL)->send_msi(&to); 3629 3630 return MEMTX_OK; 3631 } 3632 3633 static const MemoryRegionOps vtd_mem_ir_ops = { 3634 .read_with_attrs = vtd_mem_ir_read, 3635 .write_with_attrs = vtd_mem_ir_write, 3636 .endianness = DEVICE_LITTLE_ENDIAN, 3637 .impl = { 3638 .min_access_size = 4, 3639 .max_access_size = 4, 3640 }, 3641 .valid = { 3642 .min_access_size = 4, 3643 .max_access_size = 4, 3644 }, 3645 }; 3646 3647 static void vtd_report_ir_illegal_access(VTDAddressSpace *vtd_as, 3648 hwaddr addr, bool is_write) 3649 { 3650 IntelIOMMUState *s = vtd_as->iommu_state; 3651 uint8_t bus_n = pci_bus_num(vtd_as->bus); 3652 uint16_t sid = PCI_BUILD_BDF(bus_n, vtd_as->devfn); 3653 bool is_fpd_set = false; 3654 VTDContextEntry ce; 3655 3656 assert(vtd_as->pasid != PCI_NO_PASID); 3657 3658 /* Try out best to fetch FPD, we can't do anything more */ 3659 if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 3660 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 3661 if (!is_fpd_set && s->root_scalable) { 3662 vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, vtd_as->pasid); 3663 } 3664 } 3665 3666 vtd_report_fault(s, VTD_FR_SM_INTERRUPT_ADDR, 3667 is_fpd_set, sid, addr, is_write, 3668 true, vtd_as->pasid); 3669 } 3670 3671 static MemTxResult vtd_mem_ir_fault_read(void *opaque, hwaddr addr, 3672 uint64_t *data, unsigned size, 3673 MemTxAttrs attrs) 3674 { 3675 vtd_report_ir_illegal_access(opaque, addr, false); 3676 3677 return MEMTX_ERROR; 3678 } 3679 3680 static MemTxResult vtd_mem_ir_fault_write(void *opaque, hwaddr addr, 3681 uint64_t value, unsigned size, 3682 MemTxAttrs attrs) 3683 { 3684 vtd_report_ir_illegal_access(opaque, addr, true); 3685 3686 return MEMTX_ERROR; 3687 } 3688 3689 static const MemoryRegionOps vtd_mem_ir_fault_ops = { 3690 .read_with_attrs = vtd_mem_ir_fault_read, 3691 .write_with_attrs = vtd_mem_ir_fault_write, 3692 .endianness = DEVICE_LITTLE_ENDIAN, 3693 .impl = { 3694 .min_access_size = 1, 3695 .max_access_size = 8, 3696 }, 3697 .valid = { 3698 .min_access_size = 1, 3699 .max_access_size = 8, 3700 }, 3701 }; 3702 3703 VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, 3704 int devfn, unsigned int pasid) 3705 { 3706 /* 3707 * We can't simply use sid here since the bus number might not be 3708 * initialized by the guest. 3709 */ 3710 struct vtd_as_key key = { 3711 .bus = bus, 3712 .devfn = devfn, 3713 .pasid = pasid, 3714 }; 3715 VTDAddressSpace *vtd_dev_as; 3716 char name[128]; 3717 3718 vtd_dev_as = g_hash_table_lookup(s->vtd_address_spaces, &key); 3719 if (!vtd_dev_as) { 3720 struct vtd_as_key *new_key = g_malloc(sizeof(*new_key)); 3721 3722 new_key->bus = bus; 3723 new_key->devfn = devfn; 3724 new_key->pasid = pasid; 3725 3726 if (pasid == PCI_NO_PASID) { 3727 snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn), 3728 PCI_FUNC(devfn)); 3729 } else { 3730 snprintf(name, sizeof(name), "vtd-%02x.%x-pasid-%x", PCI_SLOT(devfn), 3731 PCI_FUNC(devfn), pasid); 3732 } 3733 3734 vtd_dev_as = g_new0(VTDAddressSpace, 1); 3735 3736 vtd_dev_as->bus = bus; 3737 vtd_dev_as->devfn = (uint8_t)devfn; 3738 vtd_dev_as->pasid = pasid; 3739 vtd_dev_as->iommu_state = s; 3740 vtd_dev_as->context_cache_entry.context_cache_gen = 0; 3741 vtd_dev_as->iova_tree = iova_tree_new(); 3742 3743 memory_region_init(&vtd_dev_as->root, OBJECT(s), name, UINT64_MAX); 3744 address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, "vtd-root"); 3745 3746 /* 3747 * Build the DMAR-disabled container with aliases to the 3748 * shared MRs. Note that aliasing to a shared memory region 3749 * could help the memory API to detect same FlatViews so we 3750 * can have devices to share the same FlatView when DMAR is 3751 * disabled (either by not providing "intel_iommu=on" or with 3752 * "iommu=pt"). It will greatly reduce the total number of 3753 * FlatViews of the system hence VM runs faster. 3754 */ 3755 memory_region_init_alias(&vtd_dev_as->nodmar, OBJECT(s), 3756 "vtd-nodmar", &s->mr_nodmar, 0, 3757 memory_region_size(&s->mr_nodmar)); 3758 3759 /* 3760 * Build the per-device DMAR-enabled container. 3761 * 3762 * TODO: currently we have per-device IOMMU memory region only 3763 * because we have per-device IOMMU notifiers for devices. If 3764 * one day we can abstract the IOMMU notifiers out of the 3765 * memory regions then we can also share the same memory 3766 * region here just like what we've done above with the nodmar 3767 * region. 3768 */ 3769 strcat(name, "-dmar"); 3770 memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu), 3771 TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s), 3772 name, UINT64_MAX); 3773 memory_region_init_alias(&vtd_dev_as->iommu_ir, OBJECT(s), "vtd-ir", 3774 &s->mr_ir, 0, memory_region_size(&s->mr_ir)); 3775 memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->iommu), 3776 VTD_INTERRUPT_ADDR_FIRST, 3777 &vtd_dev_as->iommu_ir, 1); 3778 3779 /* 3780 * This region is used for catching fault to access interrupt 3781 * range via passthrough + PASID. See also 3782 * vtd_switch_address_space(). We can't use alias since we 3783 * need to know the sid which is valid for MSI who uses 3784 * bus_master_as (see msi_send_message()). 3785 */ 3786 memory_region_init_io(&vtd_dev_as->iommu_ir_fault, OBJECT(s), 3787 &vtd_mem_ir_fault_ops, vtd_dev_as, "vtd-no-ir", 3788 VTD_INTERRUPT_ADDR_SIZE); 3789 /* 3790 * Hook to root since when PT is enabled vtd_dev_as->iommu 3791 * will be disabled. 3792 */ 3793 memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->root), 3794 VTD_INTERRUPT_ADDR_FIRST, 3795 &vtd_dev_as->iommu_ir_fault, 2); 3796 3797 /* 3798 * Hook both the containers under the root container, we 3799 * switch between DMAR & noDMAR by enable/disable 3800 * corresponding sub-containers 3801 */ 3802 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 3803 MEMORY_REGION(&vtd_dev_as->iommu), 3804 0); 3805 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 3806 &vtd_dev_as->nodmar, 0); 3807 3808 vtd_switch_address_space(vtd_dev_as); 3809 3810 g_hash_table_insert(s->vtd_address_spaces, new_key, vtd_dev_as); 3811 } 3812 return vtd_dev_as; 3813 } 3814 3815 /* Unmap the whole range in the notifier's scope. */ 3816 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n) 3817 { 3818 hwaddr total, remain; 3819 hwaddr start = n->start; 3820 hwaddr end = n->end; 3821 IntelIOMMUState *s = as->iommu_state; 3822 DMAMap map; 3823 3824 /* 3825 * Note: all the codes in this function has a assumption that IOVA 3826 * bits are no more than VTD_MGAW bits (which is restricted by 3827 * VT-d spec), otherwise we need to consider overflow of 64 bits. 3828 */ 3829 3830 if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) { 3831 /* 3832 * Don't need to unmap regions that is bigger than the whole 3833 * VT-d supported address space size 3834 */ 3835 end = VTD_ADDRESS_SIZE(s->aw_bits) - 1; 3836 } 3837 3838 assert(start <= end); 3839 total = remain = end - start + 1; 3840 3841 while (remain >= VTD_PAGE_SIZE) { 3842 IOMMUTLBEvent event; 3843 uint64_t mask = dma_aligned_pow2_mask(start, end, s->aw_bits); 3844 uint64_t size = mask + 1; 3845 3846 assert(size); 3847 3848 event.type = IOMMU_NOTIFIER_UNMAP; 3849 event.entry.iova = start; 3850 event.entry.addr_mask = mask; 3851 event.entry.target_as = &address_space_memory; 3852 event.entry.perm = IOMMU_NONE; 3853 /* This field is meaningless for unmap */ 3854 event.entry.translated_addr = 0; 3855 3856 memory_region_notify_iommu_one(n, &event); 3857 3858 start += size; 3859 remain -= size; 3860 } 3861 3862 assert(!remain); 3863 3864 trace_vtd_as_unmap_whole(pci_bus_num(as->bus), 3865 VTD_PCI_SLOT(as->devfn), 3866 VTD_PCI_FUNC(as->devfn), 3867 n->start, total); 3868 3869 map.iova = n->start; 3870 map.size = total - 1; /* Inclusive */ 3871 iova_tree_remove(as->iova_tree, map); 3872 } 3873 3874 static void vtd_address_space_unmap_all(IntelIOMMUState *s) 3875 { 3876 VTDAddressSpace *vtd_as; 3877 IOMMUNotifier *n; 3878 3879 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 3880 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 3881 vtd_address_space_unmap(vtd_as, n); 3882 } 3883 } 3884 } 3885 3886 static void vtd_address_space_refresh_all(IntelIOMMUState *s) 3887 { 3888 vtd_address_space_unmap_all(s); 3889 vtd_switch_address_space_all(s); 3890 } 3891 3892 static int vtd_replay_hook(IOMMUTLBEvent *event, void *private) 3893 { 3894 memory_region_notify_iommu_one(private, event); 3895 return 0; 3896 } 3897 3898 static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) 3899 { 3900 VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu); 3901 IntelIOMMUState *s = vtd_as->iommu_state; 3902 uint8_t bus_n = pci_bus_num(vtd_as->bus); 3903 VTDContextEntry ce; 3904 DMAMap map = { .iova = 0, .size = HWADDR_MAX }; 3905 3906 /* replay is protected by BQL, page walk will re-setup it safely */ 3907 iova_tree_remove(vtd_as->iova_tree, map); 3908 3909 if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 3910 trace_vtd_replay_ce_valid(s->root_scalable ? "scalable mode" : 3911 "legacy mode", 3912 bus_n, PCI_SLOT(vtd_as->devfn), 3913 PCI_FUNC(vtd_as->devfn), 3914 vtd_get_domain_id(s, &ce, vtd_as->pasid), 3915 ce.hi, ce.lo); 3916 if (n->notifier_flags & IOMMU_NOTIFIER_MAP) { 3917 /* This is required only for MAP typed notifiers */ 3918 vtd_page_walk_info info = { 3919 .hook_fn = vtd_replay_hook, 3920 .private = (void *)n, 3921 .notify_unmap = false, 3922 .aw = s->aw_bits, 3923 .as = vtd_as, 3924 .domain_id = vtd_get_domain_id(s, &ce, vtd_as->pasid), 3925 }; 3926 3927 vtd_page_walk(s, &ce, 0, ~0ULL, &info, vtd_as->pasid); 3928 } 3929 } else { 3930 trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn), 3931 PCI_FUNC(vtd_as->devfn)); 3932 } 3933 3934 return; 3935 } 3936 3937 /* Do the initialization. It will also be called when reset, so pay 3938 * attention when adding new initialization stuff. 3939 */ 3940 static void vtd_init(IntelIOMMUState *s) 3941 { 3942 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3943 3944 memset(s->csr, 0, DMAR_REG_SIZE); 3945 memset(s->wmask, 0, DMAR_REG_SIZE); 3946 memset(s->w1cmask, 0, DMAR_REG_SIZE); 3947 memset(s->womask, 0, DMAR_REG_SIZE); 3948 3949 s->root = 0; 3950 s->root_scalable = false; 3951 s->dmar_enabled = false; 3952 s->intr_enabled = false; 3953 s->iq_head = 0; 3954 s->iq_tail = 0; 3955 s->iq = 0; 3956 s->iq_size = 0; 3957 s->qi_enabled = false; 3958 s->iq_last_desc_type = VTD_INV_DESC_NONE; 3959 s->iq_dw = false; 3960 s->next_frcd_reg = 0; 3961 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | 3962 VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS | 3963 VTD_CAP_MGAW(s->aw_bits); 3964 if (s->dma_drain) { 3965 s->cap |= VTD_CAP_DRAIN; 3966 } 3967 if (s->dma_translation) { 3968 if (s->aw_bits >= VTD_HOST_AW_39BIT) { 3969 s->cap |= VTD_CAP_SAGAW_39bit; 3970 } 3971 if (s->aw_bits >= VTD_HOST_AW_48BIT) { 3972 s->cap |= VTD_CAP_SAGAW_48bit; 3973 } 3974 } 3975 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 3976 3977 /* 3978 * Rsvd field masks for spte 3979 */ 3980 vtd_spte_rsvd[0] = ~0ULL; 3981 vtd_spte_rsvd[1] = VTD_SPTE_PAGE_L1_RSVD_MASK(s->aw_bits, 3982 x86_iommu->dt_supported); 3983 vtd_spte_rsvd[2] = VTD_SPTE_PAGE_L2_RSVD_MASK(s->aw_bits); 3984 vtd_spte_rsvd[3] = VTD_SPTE_PAGE_L3_RSVD_MASK(s->aw_bits); 3985 vtd_spte_rsvd[4] = VTD_SPTE_PAGE_L4_RSVD_MASK(s->aw_bits); 3986 3987 vtd_spte_rsvd_large[2] = VTD_SPTE_LPAGE_L2_RSVD_MASK(s->aw_bits, 3988 x86_iommu->dt_supported); 3989 vtd_spte_rsvd_large[3] = VTD_SPTE_LPAGE_L3_RSVD_MASK(s->aw_bits, 3990 x86_iommu->dt_supported); 3991 3992 if (s->scalable_mode || s->snoop_control) { 3993 vtd_spte_rsvd[1] &= ~VTD_SPTE_SNP; 3994 vtd_spte_rsvd_large[2] &= ~VTD_SPTE_SNP; 3995 vtd_spte_rsvd_large[3] &= ~VTD_SPTE_SNP; 3996 } 3997 3998 if (x86_iommu_ir_supported(x86_iommu)) { 3999 s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV; 4000 if (s->intr_eim == ON_OFF_AUTO_ON) { 4001 s->ecap |= VTD_ECAP_EIM; 4002 } 4003 assert(s->intr_eim != ON_OFF_AUTO_AUTO); 4004 } 4005 4006 if (x86_iommu->dt_supported) { 4007 s->ecap |= VTD_ECAP_DT; 4008 } 4009 4010 if (x86_iommu->pt_supported) { 4011 s->ecap |= VTD_ECAP_PT; 4012 } 4013 4014 if (s->caching_mode) { 4015 s->cap |= VTD_CAP_CM; 4016 } 4017 4018 /* TODO: read cap/ecap from host to decide which cap to be exposed. */ 4019 if (s->scalable_mode) { 4020 s->ecap |= VTD_ECAP_SMTS | VTD_ECAP_SRS | VTD_ECAP_SLTS; 4021 } 4022 4023 if (s->snoop_control) { 4024 s->ecap |= VTD_ECAP_SC; 4025 } 4026 4027 if (s->pasid) { 4028 s->ecap |= VTD_ECAP_PASID; 4029 } 4030 4031 vtd_reset_caches(s); 4032 4033 /* Define registers with default values and bit semantics */ 4034 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 4035 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 4036 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 4037 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 4038 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 4039 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 4040 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffffc00ULL, 0); 4041 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 4042 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 4043 4044 /* Advanced Fault Logging not supported */ 4045 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 4046 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 4047 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 4048 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 4049 4050 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 4051 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 4052 */ 4053 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 4054 4055 /* Treated as RO for implementations that PLMR and PHMR fields reported 4056 * as Clear in the CAP_REG. 4057 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 4058 */ 4059 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 4060 4061 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 4062 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 4063 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff807ULL, 0); 4064 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 4065 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 4066 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 4067 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 4068 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 4069 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 4070 4071 /* IOTLB registers */ 4072 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 4073 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 4074 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 4075 4076 /* Fault Recording Registers, 128-bit */ 4077 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 4078 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 4079 4080 /* 4081 * Interrupt remapping registers. 4082 */ 4083 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0); 4084 } 4085 4086 /* Should not reset address_spaces when reset because devices will still use 4087 * the address space they got at first (won't ask the bus again). 4088 */ 4089 static void vtd_reset(DeviceState *dev) 4090 { 4091 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 4092 4093 vtd_init(s); 4094 vtd_address_space_refresh_all(s); 4095 } 4096 4097 static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) 4098 { 4099 IntelIOMMUState *s = opaque; 4100 VTDAddressSpace *vtd_as; 4101 4102 assert(0 <= devfn && devfn < PCI_DEVFN_MAX); 4103 4104 vtd_as = vtd_find_add_as(s, bus, devfn, PCI_NO_PASID); 4105 return &vtd_as->as; 4106 } 4107 4108 static PCIIOMMUOps vtd_iommu_ops = { 4109 .get_address_space = vtd_host_dma_iommu, 4110 }; 4111 4112 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) 4113 { 4114 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 4115 4116 if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu_ir_supported(x86_iommu)) { 4117 error_setg(errp, "eim=on cannot be selected without intremap=on"); 4118 return false; 4119 } 4120 4121 if (s->intr_eim == ON_OFF_AUTO_AUTO) { 4122 s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim) 4123 && x86_iommu_ir_supported(x86_iommu) ? 4124 ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 4125 } 4126 if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) { 4127 if (kvm_irqchip_is_split() && !kvm_enable_x2apic()) { 4128 error_setg(errp, "eim=on requires support on the KVM side" 4129 "(X2APIC_API, first shipped in v4.7)"); 4130 return false; 4131 } 4132 } 4133 4134 /* Currently only address widths supported are 39 and 48 bits */ 4135 if ((s->aw_bits != VTD_HOST_AW_39BIT) && 4136 (s->aw_bits != VTD_HOST_AW_48BIT)) { 4137 error_setg(errp, "Supported values for aw-bits are: %d, %d", 4138 VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT); 4139 return false; 4140 } 4141 4142 if (s->scalable_mode && !s->dma_drain) { 4143 error_setg(errp, "Need to set dma_drain for scalable mode"); 4144 return false; 4145 } 4146 4147 if (s->pasid && !s->scalable_mode) { 4148 error_setg(errp, "Need to set scalable mode for PASID"); 4149 return false; 4150 } 4151 4152 return true; 4153 } 4154 4155 static int vtd_machine_done_notify_one(Object *child, void *unused) 4156 { 4157 IntelIOMMUState *iommu = INTEL_IOMMU_DEVICE(x86_iommu_get_default()); 4158 4159 /* 4160 * We hard-coded here because vfio-pci is the only special case 4161 * here. Let's be more elegant in the future when we can, but so 4162 * far there seems to be no better way. 4163 */ 4164 if (object_dynamic_cast(child, "vfio-pci") && !iommu->caching_mode) { 4165 vtd_panic_require_caching_mode(); 4166 } 4167 4168 return 0; 4169 } 4170 4171 static void vtd_machine_done_hook(Notifier *notifier, void *unused) 4172 { 4173 object_child_foreach_recursive(object_get_root(), 4174 vtd_machine_done_notify_one, NULL); 4175 } 4176 4177 static Notifier vtd_machine_done_notify = { 4178 .notify = vtd_machine_done_hook, 4179 }; 4180 4181 static void vtd_realize(DeviceState *dev, Error **errp) 4182 { 4183 MachineState *ms = MACHINE(qdev_get_machine()); 4184 PCMachineState *pcms = PC_MACHINE(ms); 4185 X86MachineState *x86ms = X86_MACHINE(ms); 4186 PCIBus *bus = pcms->bus; 4187 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 4188 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 4189 4190 if (s->pasid && x86_iommu->dt_supported) { 4191 /* 4192 * PASID-based-Device-TLB Invalidate Descriptor is not 4193 * implemented and it requires support from vhost layer which 4194 * needs to be implemented in the future. 4195 */ 4196 error_setg(errp, "PASID based device IOTLB is not supported"); 4197 return; 4198 } 4199 4200 if (!vtd_decide_config(s, errp)) { 4201 return; 4202 } 4203 4204 QLIST_INIT(&s->vtd_as_with_notifiers); 4205 qemu_mutex_init(&s->iommu_lock); 4206 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 4207 "intel_iommu", DMAR_REG_SIZE); 4208 memory_region_add_subregion(get_system_memory(), 4209 Q35_HOST_BRIDGE_IOMMU_ADDR, &s->csrmem); 4210 4211 /* Create the shared memory regions by all devices */ 4212 memory_region_init(&s->mr_nodmar, OBJECT(s), "vtd-nodmar", 4213 UINT64_MAX); 4214 memory_region_init_io(&s->mr_ir, OBJECT(s), &vtd_mem_ir_ops, 4215 s, "vtd-ir", VTD_INTERRUPT_ADDR_SIZE); 4216 memory_region_init_alias(&s->mr_sys_alias, OBJECT(s), 4217 "vtd-sys-alias", get_system_memory(), 0, 4218 memory_region_size(get_system_memory())); 4219 memory_region_add_subregion_overlap(&s->mr_nodmar, 0, 4220 &s->mr_sys_alias, 0); 4221 memory_region_add_subregion_overlap(&s->mr_nodmar, 4222 VTD_INTERRUPT_ADDR_FIRST, 4223 &s->mr_ir, 1); 4224 /* No corresponding destroy */ 4225 s->iotlb = g_hash_table_new_full(vtd_iotlb_hash, vtd_iotlb_equal, 4226 g_free, g_free); 4227 s->vtd_address_spaces = g_hash_table_new_full(vtd_as_hash, vtd_as_equal, 4228 g_free, g_free); 4229 vtd_init(s); 4230 pci_setup_iommu(bus, &vtd_iommu_ops, dev); 4231 /* Pseudo address space under root PCI bus. */ 4232 x86ms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC); 4233 qemu_add_machine_init_done_notifier(&vtd_machine_done_notify); 4234 } 4235 4236 static void vtd_class_init(ObjectClass *klass, void *data) 4237 { 4238 DeviceClass *dc = DEVICE_CLASS(klass); 4239 X86IOMMUClass *x86_class = X86_IOMMU_DEVICE_CLASS(klass); 4240 4241 dc->reset = vtd_reset; 4242 dc->vmsd = &vtd_vmstate; 4243 device_class_set_props(dc, vtd_properties); 4244 dc->hotpluggable = false; 4245 x86_class->realize = vtd_realize; 4246 x86_class->int_remap = vtd_int_remap; 4247 /* Supported by the pc-q35-* machine types */ 4248 dc->user_creatable = true; 4249 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 4250 dc->desc = "Intel IOMMU (VT-d) DMA Remapping device"; 4251 } 4252 4253 static const TypeInfo vtd_info = { 4254 .name = TYPE_INTEL_IOMMU_DEVICE, 4255 .parent = TYPE_X86_IOMMU_DEVICE, 4256 .instance_size = sizeof(IntelIOMMUState), 4257 .class_init = vtd_class_init, 4258 }; 4259 4260 static void vtd_iommu_memory_region_class_init(ObjectClass *klass, 4261 void *data) 4262 { 4263 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 4264 4265 imrc->translate = vtd_iommu_translate; 4266 imrc->notify_flag_changed = vtd_iommu_notify_flag_changed; 4267 imrc->replay = vtd_iommu_replay; 4268 } 4269 4270 static const TypeInfo vtd_iommu_memory_region_info = { 4271 .parent = TYPE_IOMMU_MEMORY_REGION, 4272 .name = TYPE_INTEL_IOMMU_MEMORY_REGION, 4273 .class_init = vtd_iommu_memory_region_class_init, 4274 }; 4275 4276 static void vtd_register_types(void) 4277 { 4278 type_register_static(&vtd_info); 4279 type_register_static(&vtd_iommu_memory_region_info); 4280 } 4281 4282 type_init(vtd_register_types) 4283