1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "qemu/osdep.h" 23 #include "qemu/error-report.h" 24 #include "qapi/error.h" 25 #include "hw/sysbus.h" 26 #include "exec/address-spaces.h" 27 #include "intel_iommu_internal.h" 28 #include "hw/pci/pci.h" 29 #include "hw/pci/pci_bus.h" 30 #include "hw/i386/pc.h" 31 #include "hw/i386/apic-msidef.h" 32 #include "hw/boards.h" 33 #include "hw/i386/x86-iommu.h" 34 #include "hw/pci-host/q35.h" 35 #include "sysemu/kvm.h" 36 #include "hw/i386/apic_internal.h" 37 #include "kvm_i386.h" 38 #include "trace.h" 39 40 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 41 uint64_t wmask, uint64_t w1cmask) 42 { 43 stq_le_p(&s->csr[addr], val); 44 stq_le_p(&s->wmask[addr], wmask); 45 stq_le_p(&s->w1cmask[addr], w1cmask); 46 } 47 48 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 49 { 50 stq_le_p(&s->womask[addr], mask); 51 } 52 53 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 54 uint32_t wmask, uint32_t w1cmask) 55 { 56 stl_le_p(&s->csr[addr], val); 57 stl_le_p(&s->wmask[addr], wmask); 58 stl_le_p(&s->w1cmask[addr], w1cmask); 59 } 60 61 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 62 { 63 stl_le_p(&s->womask[addr], mask); 64 } 65 66 /* "External" get/set operations */ 67 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 68 { 69 uint64_t oldval = ldq_le_p(&s->csr[addr]); 70 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 71 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 72 stq_le_p(&s->csr[addr], 73 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 74 } 75 76 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 77 { 78 uint32_t oldval = ldl_le_p(&s->csr[addr]); 79 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 80 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 81 stl_le_p(&s->csr[addr], 82 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 83 } 84 85 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 86 { 87 uint64_t val = ldq_le_p(&s->csr[addr]); 88 uint64_t womask = ldq_le_p(&s->womask[addr]); 89 return val & ~womask; 90 } 91 92 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 93 { 94 uint32_t val = ldl_le_p(&s->csr[addr]); 95 uint32_t womask = ldl_le_p(&s->womask[addr]); 96 return val & ~womask; 97 } 98 99 /* "Internal" get/set operations */ 100 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 101 { 102 return ldq_le_p(&s->csr[addr]); 103 } 104 105 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 106 { 107 return ldl_le_p(&s->csr[addr]); 108 } 109 110 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 111 { 112 stq_le_p(&s->csr[addr], val); 113 } 114 115 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 116 uint32_t clear, uint32_t mask) 117 { 118 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 119 stl_le_p(&s->csr[addr], new_val); 120 return new_val; 121 } 122 123 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 124 uint64_t clear, uint64_t mask) 125 { 126 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 127 stq_le_p(&s->csr[addr], new_val); 128 return new_val; 129 } 130 131 static inline void vtd_iommu_lock(IntelIOMMUState *s) 132 { 133 qemu_mutex_lock(&s->iommu_lock); 134 } 135 136 static inline void vtd_iommu_unlock(IntelIOMMUState *s) 137 { 138 qemu_mutex_unlock(&s->iommu_lock); 139 } 140 141 /* Whether the address space needs to notify new mappings */ 142 static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as) 143 { 144 return as->notifier_flags & IOMMU_NOTIFIER_MAP; 145 } 146 147 /* GHashTable functions */ 148 static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2) 149 { 150 return *((const uint64_t *)v1) == *((const uint64_t *)v2); 151 } 152 153 static guint vtd_uint64_hash(gconstpointer v) 154 { 155 return (guint)*(const uint64_t *)v; 156 } 157 158 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 159 gpointer user_data) 160 { 161 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 162 uint16_t domain_id = *(uint16_t *)user_data; 163 return entry->domain_id == domain_id; 164 } 165 166 /* The shift of an addr for a certain level of paging structure */ 167 static inline uint32_t vtd_slpt_level_shift(uint32_t level) 168 { 169 assert(level != 0); 170 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 171 } 172 173 static inline uint64_t vtd_slpt_level_page_mask(uint32_t level) 174 { 175 return ~((1ULL << vtd_slpt_level_shift(level)) - 1); 176 } 177 178 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 179 gpointer user_data) 180 { 181 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 182 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 183 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask; 184 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K; 185 return (entry->domain_id == info->domain_id) && 186 (((entry->gfn & info->mask) == gfn) || 187 (entry->gfn == gfn_tlb)); 188 } 189 190 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 191 * IntelIOMMUState to 1. Must be called with IOMMU lock held. 192 */ 193 static void vtd_reset_context_cache_locked(IntelIOMMUState *s) 194 { 195 VTDAddressSpace *vtd_as; 196 VTDBus *vtd_bus; 197 GHashTableIter bus_it; 198 uint32_t devfn_it; 199 200 trace_vtd_context_cache_reset(); 201 202 g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr); 203 204 while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) { 205 for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) { 206 vtd_as = vtd_bus->dev_as[devfn_it]; 207 if (!vtd_as) { 208 continue; 209 } 210 vtd_as->context_cache_entry.context_cache_gen = 0; 211 } 212 } 213 s->context_cache_gen = 1; 214 } 215 216 /* Must be called with IOMMU lock held. */ 217 static void vtd_reset_iotlb_locked(IntelIOMMUState *s) 218 { 219 assert(s->iotlb); 220 g_hash_table_remove_all(s->iotlb); 221 } 222 223 static void vtd_reset_iotlb(IntelIOMMUState *s) 224 { 225 vtd_iommu_lock(s); 226 vtd_reset_iotlb_locked(s); 227 vtd_iommu_unlock(s); 228 } 229 230 static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint16_t source_id, 231 uint32_t level) 232 { 233 return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) | 234 ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT); 235 } 236 237 static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) 238 { 239 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K; 240 } 241 242 /* Must be called with IOMMU lock held */ 243 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 244 hwaddr addr) 245 { 246 VTDIOTLBEntry *entry; 247 uint64_t key; 248 int level; 249 250 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) { 251 key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level), 252 source_id, level); 253 entry = g_hash_table_lookup(s->iotlb, &key); 254 if (entry) { 255 goto out; 256 } 257 } 258 259 out: 260 return entry; 261 } 262 263 /* Must be with IOMMU lock held */ 264 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 265 uint16_t domain_id, hwaddr addr, uint64_t slpte, 266 uint8_t access_flags, uint32_t level) 267 { 268 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 269 uint64_t *key = g_malloc(sizeof(*key)); 270 uint64_t gfn = vtd_get_iotlb_gfn(addr, level); 271 272 trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id); 273 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 274 trace_vtd_iotlb_reset("iotlb exceeds size limit"); 275 vtd_reset_iotlb_locked(s); 276 } 277 278 entry->gfn = gfn; 279 entry->domain_id = domain_id; 280 entry->slpte = slpte; 281 entry->access_flags = access_flags; 282 entry->mask = vtd_slpt_level_page_mask(level); 283 *key = vtd_get_iotlb_key(gfn, source_id, level); 284 g_hash_table_replace(s->iotlb, key, entry); 285 } 286 287 /* Given the reg addr of both the message data and address, generate an 288 * interrupt via MSI. 289 */ 290 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 291 hwaddr mesg_data_reg) 292 { 293 MSIMessage msi; 294 295 assert(mesg_data_reg < DMAR_REG_SIZE); 296 assert(mesg_addr_reg < DMAR_REG_SIZE); 297 298 msi.address = vtd_get_long_raw(s, mesg_addr_reg); 299 msi.data = vtd_get_long_raw(s, mesg_data_reg); 300 301 trace_vtd_irq_generate(msi.address, msi.data); 302 303 apic_get_class()->send_msi(&msi); 304 } 305 306 /* Generate a fault event to software via MSI if conditions are met. 307 * Notice that the value of FSTS_REG being passed to it should be the one 308 * before any update. 309 */ 310 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 311 { 312 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 313 pre_fsts & VTD_FSTS_IQE) { 314 trace_vtd_err("There are previous interrupt conditions " 315 "to be serviced by software, fault event " 316 "is not generated."); 317 return; 318 } 319 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 320 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 321 trace_vtd_err("Interrupt Mask set, irq is not generated."); 322 } else { 323 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 324 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 325 } 326 } 327 328 /* Check if the Fault (F) field of the Fault Recording Register referenced by 329 * @index is Set. 330 */ 331 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 332 { 333 /* Each reg is 128-bit */ 334 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 335 addr += 8; /* Access the high 64-bit half */ 336 337 assert(index < DMAR_FRCD_REG_NR); 338 339 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 340 } 341 342 /* Update the PPF field of Fault Status Register. 343 * Should be called whenever change the F field of any fault recording 344 * registers. 345 */ 346 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 347 { 348 uint32_t i; 349 uint32_t ppf_mask = 0; 350 351 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 352 if (vtd_is_frcd_set(s, i)) { 353 ppf_mask = VTD_FSTS_PPF; 354 break; 355 } 356 } 357 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 358 trace_vtd_fsts_ppf(!!ppf_mask); 359 } 360 361 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 362 { 363 /* Each reg is 128-bit */ 364 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 365 addr += 8; /* Access the high 64-bit half */ 366 367 assert(index < DMAR_FRCD_REG_NR); 368 369 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 370 vtd_update_fsts_ppf(s); 371 } 372 373 /* Must not update F field now, should be done later */ 374 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 375 uint16_t source_id, hwaddr addr, 376 VTDFaultReason fault, bool is_write) 377 { 378 uint64_t hi = 0, lo; 379 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 380 381 assert(index < DMAR_FRCD_REG_NR); 382 383 lo = VTD_FRCD_FI(addr); 384 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 385 if (!is_write) { 386 hi |= VTD_FRCD_T; 387 } 388 vtd_set_quad_raw(s, frcd_reg_addr, lo); 389 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 390 391 trace_vtd_frr_new(index, hi, lo); 392 } 393 394 /* Try to collapse multiple pending faults from the same requester */ 395 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 396 { 397 uint32_t i; 398 uint64_t frcd_reg; 399 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 400 401 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 402 frcd_reg = vtd_get_quad_raw(s, addr); 403 if ((frcd_reg & VTD_FRCD_F) && 404 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 405 return true; 406 } 407 addr += 16; /* 128-bit for each */ 408 } 409 return false; 410 } 411 412 /* Log and report an DMAR (address translation) fault to software */ 413 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 414 hwaddr addr, VTDFaultReason fault, 415 bool is_write) 416 { 417 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 418 419 assert(fault < VTD_FR_MAX); 420 421 if (fault == VTD_FR_RESERVED_ERR) { 422 /* This is not a normal fault reason case. Drop it. */ 423 return; 424 } 425 426 trace_vtd_dmar_fault(source_id, fault, addr, is_write); 427 428 if (fsts_reg & VTD_FSTS_PFO) { 429 trace_vtd_err("New fault is not recorded due to " 430 "Primary Fault Overflow."); 431 return; 432 } 433 434 if (vtd_try_collapse_fault(s, source_id)) { 435 trace_vtd_err("New fault is not recorded due to " 436 "compression of faults."); 437 return; 438 } 439 440 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 441 trace_vtd_err("Next Fault Recording Reg is used, " 442 "new fault is not recorded, set PFO field."); 443 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 444 return; 445 } 446 447 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write); 448 449 if (fsts_reg & VTD_FSTS_PPF) { 450 trace_vtd_err("There are pending faults already, " 451 "fault event is not generated."); 452 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 453 s->next_frcd_reg++; 454 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 455 s->next_frcd_reg = 0; 456 } 457 } else { 458 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 459 VTD_FSTS_FRI(s->next_frcd_reg)); 460 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 461 s->next_frcd_reg++; 462 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 463 s->next_frcd_reg = 0; 464 } 465 /* This case actually cause the PPF to be Set. 466 * So generate fault event (interrupt). 467 */ 468 vtd_generate_fault_event(s, fsts_reg); 469 } 470 } 471 472 /* Handle Invalidation Queue Errors of queued invalidation interface error 473 * conditions. 474 */ 475 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 476 { 477 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 478 479 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 480 vtd_generate_fault_event(s, fsts_reg); 481 } 482 483 /* Set the IWC field and try to generate an invalidation completion interrupt */ 484 static void vtd_generate_completion_event(IntelIOMMUState *s) 485 { 486 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 487 trace_vtd_inv_desc_wait_irq("One pending, skip current"); 488 return; 489 } 490 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 491 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 492 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 493 trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, " 494 "new event not generated"); 495 return; 496 } else { 497 /* Generate the interrupt event */ 498 trace_vtd_inv_desc_wait_irq("Generating complete event"); 499 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 500 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 501 } 502 } 503 504 static inline bool vtd_root_entry_present(VTDRootEntry *root) 505 { 506 return root->val & VTD_ROOT_ENTRY_P; 507 } 508 509 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 510 VTDRootEntry *re) 511 { 512 dma_addr_t addr; 513 514 addr = s->root + index * sizeof(*re); 515 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) { 516 trace_vtd_re_invalid(re->rsvd, re->val); 517 re->val = 0; 518 return -VTD_FR_ROOT_TABLE_INV; 519 } 520 re->val = le64_to_cpu(re->val); 521 return 0; 522 } 523 524 static inline bool vtd_ce_present(VTDContextEntry *context) 525 { 526 return context->lo & VTD_CONTEXT_ENTRY_P; 527 } 528 529 static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index, 530 VTDContextEntry *ce) 531 { 532 dma_addr_t addr; 533 534 /* we have checked that root entry is present */ 535 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce); 536 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) { 537 trace_vtd_re_invalid(root->rsvd, root->val); 538 return -VTD_FR_CONTEXT_TABLE_INV; 539 } 540 ce->lo = le64_to_cpu(ce->lo); 541 ce->hi = le64_to_cpu(ce->hi); 542 return 0; 543 } 544 545 static inline dma_addr_t vtd_ce_get_slpt_base(VTDContextEntry *ce) 546 { 547 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 548 } 549 550 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte, uint8_t aw) 551 { 552 return slpte & VTD_SL_PT_BASE_ADDR_MASK(aw); 553 } 554 555 /* Whether the pte indicates the address of the page frame */ 556 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 557 { 558 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 559 } 560 561 /* Get the content of a spte located in @base_addr[@index] */ 562 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 563 { 564 uint64_t slpte; 565 566 assert(index < VTD_SL_PT_ENTRY_NR); 567 568 if (dma_memory_read(&address_space_memory, 569 base_addr + index * sizeof(slpte), &slpte, 570 sizeof(slpte))) { 571 slpte = (uint64_t)-1; 572 return slpte; 573 } 574 slpte = le64_to_cpu(slpte); 575 return slpte; 576 } 577 578 /* Given an iova and the level of paging structure, return the offset 579 * of current level. 580 */ 581 static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level) 582 { 583 return (iova >> vtd_slpt_level_shift(level)) & 584 ((1ULL << VTD_SL_LEVEL_BITS) - 1); 585 } 586 587 /* Check Capability Register to see if the @level of page-table is supported */ 588 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 589 { 590 return VTD_CAP_SAGAW_MASK & s->cap & 591 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 592 } 593 594 /* Get the page-table level that hardware should use for the second-level 595 * page-table walk from the Address Width field of context-entry. 596 */ 597 static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce) 598 { 599 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 600 } 601 602 static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce) 603 { 604 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 605 } 606 607 static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce) 608 { 609 return ce->lo & VTD_CONTEXT_ENTRY_TT; 610 } 611 612 /* Return true if check passed, otherwise false */ 613 static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu, 614 VTDContextEntry *ce) 615 { 616 switch (vtd_ce_get_type(ce)) { 617 case VTD_CONTEXT_TT_MULTI_LEVEL: 618 /* Always supported */ 619 break; 620 case VTD_CONTEXT_TT_DEV_IOTLB: 621 if (!x86_iommu->dt_supported) { 622 return false; 623 } 624 break; 625 case VTD_CONTEXT_TT_PASS_THROUGH: 626 if (!x86_iommu->pt_supported) { 627 return false; 628 } 629 break; 630 default: 631 /* Unknwon type */ 632 return false; 633 } 634 return true; 635 } 636 637 static inline uint64_t vtd_iova_limit(VTDContextEntry *ce, uint8_t aw) 638 { 639 uint32_t ce_agaw = vtd_ce_get_agaw(ce); 640 return 1ULL << MIN(ce_agaw, aw); 641 } 642 643 /* Return true if IOVA passes range check, otherwise false. */ 644 static inline bool vtd_iova_range_check(uint64_t iova, VTDContextEntry *ce, 645 uint8_t aw) 646 { 647 /* 648 * Check if @iova is above 2^X-1, where X is the minimum of MGAW 649 * in CAP_REG and AW in context-entry. 650 */ 651 return !(iova & ~(vtd_iova_limit(ce, aw) - 1)); 652 } 653 654 /* 655 * Rsvd field masks for spte: 656 * Index [1] to [4] 4k pages 657 * Index [5] to [8] large pages 658 */ 659 static uint64_t vtd_paging_entry_rsvd_field[9]; 660 661 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 662 { 663 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) { 664 /* Maybe large page */ 665 return slpte & vtd_paging_entry_rsvd_field[level + 4]; 666 } else { 667 return slpte & vtd_paging_entry_rsvd_field[level]; 668 } 669 } 670 671 /* Find the VTD address space associated with a given bus number */ 672 static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num) 673 { 674 VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num]; 675 if (!vtd_bus) { 676 /* 677 * Iterate over the registered buses to find the one which 678 * currently hold this bus number, and update the bus_num 679 * lookup table: 680 */ 681 GHashTableIter iter; 682 683 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); 684 while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) { 685 if (pci_bus_num(vtd_bus->bus) == bus_num) { 686 s->vtd_as_by_bus_num[bus_num] = vtd_bus; 687 return vtd_bus; 688 } 689 } 690 } 691 return vtd_bus; 692 } 693 694 /* Given the @iova, get relevant @slptep. @slpte_level will be the last level 695 * of the translation, can be used for deciding the size of large page. 696 */ 697 static int vtd_iova_to_slpte(VTDContextEntry *ce, uint64_t iova, bool is_write, 698 uint64_t *slptep, uint32_t *slpte_level, 699 bool *reads, bool *writes, uint8_t aw_bits) 700 { 701 dma_addr_t addr = vtd_ce_get_slpt_base(ce); 702 uint32_t level = vtd_ce_get_level(ce); 703 uint32_t offset; 704 uint64_t slpte; 705 uint64_t access_right_check; 706 707 if (!vtd_iova_range_check(iova, ce, aw_bits)) { 708 trace_vtd_err_dmar_iova_overflow(iova); 709 return -VTD_FR_ADDR_BEYOND_MGAW; 710 } 711 712 /* FIXME: what is the Atomics request here? */ 713 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 714 715 while (true) { 716 offset = vtd_iova_level_offset(iova, level); 717 slpte = vtd_get_slpte(addr, offset); 718 719 if (slpte == (uint64_t)-1) { 720 trace_vtd_err_dmar_slpte_read_error(iova, level); 721 if (level == vtd_ce_get_level(ce)) { 722 /* Invalid programming of context-entry */ 723 return -VTD_FR_CONTEXT_ENTRY_INV; 724 } else { 725 return -VTD_FR_PAGING_ENTRY_INV; 726 } 727 } 728 *reads = (*reads) && (slpte & VTD_SL_R); 729 *writes = (*writes) && (slpte & VTD_SL_W); 730 if (!(slpte & access_right_check)) { 731 trace_vtd_err_dmar_slpte_perm_error(iova, level, slpte, is_write); 732 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 733 } 734 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 735 trace_vtd_err_dmar_slpte_resv_error(iova, level, slpte); 736 return -VTD_FR_PAGING_ENTRY_RSVD; 737 } 738 739 if (vtd_is_last_slpte(slpte, level)) { 740 *slptep = slpte; 741 *slpte_level = level; 742 return 0; 743 } 744 addr = vtd_get_slpte_addr(slpte, aw_bits); 745 level--; 746 } 747 } 748 749 typedef int (*vtd_page_walk_hook)(IOMMUTLBEntry *entry, void *private); 750 751 /** 752 * Constant information used during page walking 753 * 754 * @hook_fn: hook func to be called when detected page 755 * @private: private data to be passed into hook func 756 * @notify_unmap: whether we should notify invalid entries 757 * @as: VT-d address space of the device 758 * @aw: maximum address width 759 * @domain: domain ID of the page walk 760 */ 761 typedef struct { 762 VTDAddressSpace *as; 763 vtd_page_walk_hook hook_fn; 764 void *private; 765 bool notify_unmap; 766 uint8_t aw; 767 uint16_t domain_id; 768 } vtd_page_walk_info; 769 770 static int vtd_page_walk_one(IOMMUTLBEntry *entry, vtd_page_walk_info *info) 771 { 772 VTDAddressSpace *as = info->as; 773 vtd_page_walk_hook hook_fn = info->hook_fn; 774 void *private = info->private; 775 DMAMap target = { 776 .iova = entry->iova, 777 .size = entry->addr_mask, 778 .translated_addr = entry->translated_addr, 779 .perm = entry->perm, 780 }; 781 DMAMap *mapped = iova_tree_find(as->iova_tree, &target); 782 783 if (entry->perm == IOMMU_NONE && !info->notify_unmap) { 784 trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 785 return 0; 786 } 787 788 assert(hook_fn); 789 790 /* Update local IOVA mapped ranges */ 791 if (entry->perm) { 792 if (mapped) { 793 /* If it's exactly the same translation, skip */ 794 if (!memcmp(mapped, &target, sizeof(target))) { 795 trace_vtd_page_walk_one_skip_map(entry->iova, entry->addr_mask, 796 entry->translated_addr); 797 return 0; 798 } else { 799 /* 800 * Translation changed. Normally this should not 801 * happen, but it can happen when with buggy guest 802 * OSes. Note that there will be a small window that 803 * we don't have map at all. But that's the best 804 * effort we can do. The ideal way to emulate this is 805 * atomically modify the PTE to follow what has 806 * changed, but we can't. One example is that vfio 807 * driver only has VFIO_IOMMU_[UN]MAP_DMA but no 808 * interface to modify a mapping (meanwhile it seems 809 * meaningless to even provide one). Anyway, let's 810 * mark this as a TODO in case one day we'll have 811 * a better solution. 812 */ 813 IOMMUAccessFlags cache_perm = entry->perm; 814 int ret; 815 816 /* Emulate an UNMAP */ 817 entry->perm = IOMMU_NONE; 818 trace_vtd_page_walk_one(info->domain_id, 819 entry->iova, 820 entry->translated_addr, 821 entry->addr_mask, 822 entry->perm); 823 ret = hook_fn(entry, private); 824 if (ret) { 825 return ret; 826 } 827 /* Drop any existing mapping */ 828 iova_tree_remove(as->iova_tree, &target); 829 /* Recover the correct permission */ 830 entry->perm = cache_perm; 831 } 832 } 833 iova_tree_insert(as->iova_tree, &target); 834 } else { 835 if (!mapped) { 836 /* Skip since we didn't map this range at all */ 837 trace_vtd_page_walk_one_skip_unmap(entry->iova, entry->addr_mask); 838 return 0; 839 } 840 iova_tree_remove(as->iova_tree, &target); 841 } 842 843 trace_vtd_page_walk_one(info->domain_id, entry->iova, 844 entry->translated_addr, entry->addr_mask, 845 entry->perm); 846 return hook_fn(entry, private); 847 } 848 849 /** 850 * vtd_page_walk_level - walk over specific level for IOVA range 851 * 852 * @addr: base GPA addr to start the walk 853 * @start: IOVA range start address 854 * @end: IOVA range end address (start <= addr < end) 855 * @read: whether parent level has read permission 856 * @write: whether parent level has write permission 857 * @info: constant information for the page walk 858 */ 859 static int vtd_page_walk_level(dma_addr_t addr, uint64_t start, 860 uint64_t end, uint32_t level, bool read, 861 bool write, vtd_page_walk_info *info) 862 { 863 bool read_cur, write_cur, entry_valid; 864 uint32_t offset; 865 uint64_t slpte; 866 uint64_t subpage_size, subpage_mask; 867 IOMMUTLBEntry entry; 868 uint64_t iova = start; 869 uint64_t iova_next; 870 int ret = 0; 871 872 trace_vtd_page_walk_level(addr, level, start, end); 873 874 subpage_size = 1ULL << vtd_slpt_level_shift(level); 875 subpage_mask = vtd_slpt_level_page_mask(level); 876 877 while (iova < end) { 878 iova_next = (iova & subpage_mask) + subpage_size; 879 880 offset = vtd_iova_level_offset(iova, level); 881 slpte = vtd_get_slpte(addr, offset); 882 883 if (slpte == (uint64_t)-1) { 884 trace_vtd_page_walk_skip_read(iova, iova_next); 885 goto next; 886 } 887 888 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 889 trace_vtd_page_walk_skip_reserve(iova, iova_next); 890 goto next; 891 } 892 893 /* Permissions are stacked with parents' */ 894 read_cur = read && (slpte & VTD_SL_R); 895 write_cur = write && (slpte & VTD_SL_W); 896 897 /* 898 * As long as we have either read/write permission, this is a 899 * valid entry. The rule works for both page entries and page 900 * table entries. 901 */ 902 entry_valid = read_cur | write_cur; 903 904 if (!vtd_is_last_slpte(slpte, level) && entry_valid) { 905 /* 906 * This is a valid PDE (or even bigger than PDE). We need 907 * to walk one further level. 908 */ 909 ret = vtd_page_walk_level(vtd_get_slpte_addr(slpte, info->aw), 910 iova, MIN(iova_next, end), level - 1, 911 read_cur, write_cur, info); 912 } else { 913 /* 914 * This means we are either: 915 * 916 * (1) the real page entry (either 4K page, or huge page) 917 * (2) the whole range is invalid 918 * 919 * In either case, we send an IOTLB notification down. 920 */ 921 entry.target_as = &address_space_memory; 922 entry.iova = iova & subpage_mask; 923 entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur); 924 entry.addr_mask = ~subpage_mask; 925 /* NOTE: this is only meaningful if entry_valid == true */ 926 entry.translated_addr = vtd_get_slpte_addr(slpte, info->aw); 927 ret = vtd_page_walk_one(&entry, info); 928 } 929 930 if (ret < 0) { 931 return ret; 932 } 933 934 next: 935 iova = iova_next; 936 } 937 938 return 0; 939 } 940 941 /** 942 * vtd_page_walk - walk specific IOVA range, and call the hook 943 * 944 * @ce: context entry to walk upon 945 * @start: IOVA address to start the walk 946 * @end: IOVA range end address (start <= addr < end) 947 * @info: page walking information struct 948 */ 949 static int vtd_page_walk(VTDContextEntry *ce, uint64_t start, uint64_t end, 950 vtd_page_walk_info *info) 951 { 952 dma_addr_t addr = vtd_ce_get_slpt_base(ce); 953 uint32_t level = vtd_ce_get_level(ce); 954 955 if (!vtd_iova_range_check(start, ce, info->aw)) { 956 return -VTD_FR_ADDR_BEYOND_MGAW; 957 } 958 959 if (!vtd_iova_range_check(end, ce, info->aw)) { 960 /* Fix end so that it reaches the maximum */ 961 end = vtd_iova_limit(ce, info->aw); 962 } 963 964 return vtd_page_walk_level(addr, start, end, level, true, true, info); 965 } 966 967 /* Map a device to its corresponding domain (context-entry) */ 968 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 969 uint8_t devfn, VTDContextEntry *ce) 970 { 971 VTDRootEntry re; 972 int ret_fr; 973 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 974 975 ret_fr = vtd_get_root_entry(s, bus_num, &re); 976 if (ret_fr) { 977 return ret_fr; 978 } 979 980 if (!vtd_root_entry_present(&re)) { 981 /* Not error - it's okay we don't have root entry. */ 982 trace_vtd_re_not_present(bus_num); 983 return -VTD_FR_ROOT_ENTRY_P; 984 } 985 986 if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD(s->aw_bits))) { 987 trace_vtd_re_invalid(re.rsvd, re.val); 988 return -VTD_FR_ROOT_ENTRY_RSVD; 989 } 990 991 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce); 992 if (ret_fr) { 993 return ret_fr; 994 } 995 996 if (!vtd_ce_present(ce)) { 997 /* Not error - it's okay we don't have context entry. */ 998 trace_vtd_ce_not_present(bus_num, devfn); 999 return -VTD_FR_CONTEXT_ENTRY_P; 1000 } 1001 1002 if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) || 1003 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) { 1004 trace_vtd_ce_invalid(ce->hi, ce->lo); 1005 return -VTD_FR_CONTEXT_ENTRY_RSVD; 1006 } 1007 1008 /* Check if the programming of context-entry is valid */ 1009 if (!vtd_is_level_supported(s, vtd_ce_get_level(ce))) { 1010 trace_vtd_ce_invalid(ce->hi, ce->lo); 1011 return -VTD_FR_CONTEXT_ENTRY_INV; 1012 } 1013 1014 /* Do translation type check */ 1015 if (!vtd_ce_type_check(x86_iommu, ce)) { 1016 trace_vtd_ce_invalid(ce->hi, ce->lo); 1017 return -VTD_FR_CONTEXT_ENTRY_INV; 1018 } 1019 1020 return 0; 1021 } 1022 1023 static int vtd_sync_shadow_page_hook(IOMMUTLBEntry *entry, 1024 void *private) 1025 { 1026 memory_region_notify_iommu((IOMMUMemoryRegion *)private, *entry); 1027 return 0; 1028 } 1029 1030 /* If context entry is NULL, we'll try to fetch it on our own. */ 1031 static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as, 1032 VTDContextEntry *ce, 1033 hwaddr addr, hwaddr size) 1034 { 1035 IntelIOMMUState *s = vtd_as->iommu_state; 1036 vtd_page_walk_info info = { 1037 .hook_fn = vtd_sync_shadow_page_hook, 1038 .private = (void *)&vtd_as->iommu, 1039 .notify_unmap = true, 1040 .aw = s->aw_bits, 1041 .as = vtd_as, 1042 }; 1043 VTDContextEntry ce_cache; 1044 int ret; 1045 1046 if (ce) { 1047 /* If the caller provided context entry, use it */ 1048 ce_cache = *ce; 1049 } else { 1050 /* If the caller didn't provide ce, try to fetch */ 1051 ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 1052 vtd_as->devfn, &ce_cache); 1053 if (ret) { 1054 /* 1055 * This should not really happen, but in case it happens, 1056 * we just skip the sync for this time. After all we even 1057 * don't have the root table pointer! 1058 */ 1059 trace_vtd_err("Detected invalid context entry when " 1060 "trying to sync shadow page table"); 1061 return 0; 1062 } 1063 } 1064 1065 info.domain_id = VTD_CONTEXT_ENTRY_DID(ce_cache.hi); 1066 1067 return vtd_page_walk(&ce_cache, addr, addr + size, &info); 1068 } 1069 1070 static int vtd_sync_shadow_page_table(VTDAddressSpace *vtd_as) 1071 { 1072 return vtd_sync_shadow_page_table_range(vtd_as, NULL, 0, UINT64_MAX); 1073 } 1074 1075 /* 1076 * Fetch translation type for specific device. Returns <0 if error 1077 * happens, otherwise return the shifted type to check against 1078 * VTD_CONTEXT_TT_*. 1079 */ 1080 static int vtd_dev_get_trans_type(VTDAddressSpace *as) 1081 { 1082 IntelIOMMUState *s; 1083 VTDContextEntry ce; 1084 int ret; 1085 1086 s = as->iommu_state; 1087 1088 ret = vtd_dev_to_context_entry(s, pci_bus_num(as->bus), 1089 as->devfn, &ce); 1090 if (ret) { 1091 return ret; 1092 } 1093 1094 return vtd_ce_get_type(&ce); 1095 } 1096 1097 static bool vtd_dev_pt_enabled(VTDAddressSpace *as) 1098 { 1099 int ret; 1100 1101 assert(as); 1102 1103 ret = vtd_dev_get_trans_type(as); 1104 if (ret < 0) { 1105 /* 1106 * Possibly failed to parse the context entry for some reason 1107 * (e.g., during init, or any guest configuration errors on 1108 * context entries). We should assume PT not enabled for 1109 * safety. 1110 */ 1111 return false; 1112 } 1113 1114 return ret == VTD_CONTEXT_TT_PASS_THROUGH; 1115 } 1116 1117 /* Return whether the device is using IOMMU translation. */ 1118 static bool vtd_switch_address_space(VTDAddressSpace *as) 1119 { 1120 bool use_iommu; 1121 /* Whether we need to take the BQL on our own */ 1122 bool take_bql = !qemu_mutex_iothread_locked(); 1123 1124 assert(as); 1125 1126 use_iommu = as->iommu_state->dmar_enabled & !vtd_dev_pt_enabled(as); 1127 1128 trace_vtd_switch_address_space(pci_bus_num(as->bus), 1129 VTD_PCI_SLOT(as->devfn), 1130 VTD_PCI_FUNC(as->devfn), 1131 use_iommu); 1132 1133 /* 1134 * It's possible that we reach here without BQL, e.g., when called 1135 * from vtd_pt_enable_fast_path(). However the memory APIs need 1136 * it. We'd better make sure we have had it already, or, take it. 1137 */ 1138 if (take_bql) { 1139 qemu_mutex_lock_iothread(); 1140 } 1141 1142 /* Turn off first then on the other */ 1143 if (use_iommu) { 1144 memory_region_set_enabled(&as->sys_alias, false); 1145 memory_region_set_enabled(MEMORY_REGION(&as->iommu), true); 1146 } else { 1147 memory_region_set_enabled(MEMORY_REGION(&as->iommu), false); 1148 memory_region_set_enabled(&as->sys_alias, true); 1149 } 1150 1151 if (take_bql) { 1152 qemu_mutex_unlock_iothread(); 1153 } 1154 1155 return use_iommu; 1156 } 1157 1158 static void vtd_switch_address_space_all(IntelIOMMUState *s) 1159 { 1160 GHashTableIter iter; 1161 VTDBus *vtd_bus; 1162 int i; 1163 1164 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); 1165 while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) { 1166 for (i = 0; i < PCI_DEVFN_MAX; i++) { 1167 if (!vtd_bus->dev_as[i]) { 1168 continue; 1169 } 1170 vtd_switch_address_space(vtd_bus->dev_as[i]); 1171 } 1172 } 1173 } 1174 1175 static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn) 1176 { 1177 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL); 1178 } 1179 1180 static const bool vtd_qualified_faults[] = { 1181 [VTD_FR_RESERVED] = false, 1182 [VTD_FR_ROOT_ENTRY_P] = false, 1183 [VTD_FR_CONTEXT_ENTRY_P] = true, 1184 [VTD_FR_CONTEXT_ENTRY_INV] = true, 1185 [VTD_FR_ADDR_BEYOND_MGAW] = true, 1186 [VTD_FR_WRITE] = true, 1187 [VTD_FR_READ] = true, 1188 [VTD_FR_PAGING_ENTRY_INV] = true, 1189 [VTD_FR_ROOT_TABLE_INV] = false, 1190 [VTD_FR_CONTEXT_TABLE_INV] = false, 1191 [VTD_FR_ROOT_ENTRY_RSVD] = false, 1192 [VTD_FR_PAGING_ENTRY_RSVD] = true, 1193 [VTD_FR_CONTEXT_ENTRY_TT] = true, 1194 [VTD_FR_RESERVED_ERR] = false, 1195 [VTD_FR_MAX] = false, 1196 }; 1197 1198 /* To see if a fault condition is "qualified", which is reported to software 1199 * only if the FPD field in the context-entry used to process the faulting 1200 * request is 0. 1201 */ 1202 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 1203 { 1204 return vtd_qualified_faults[fault]; 1205 } 1206 1207 static inline bool vtd_is_interrupt_addr(hwaddr addr) 1208 { 1209 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 1210 } 1211 1212 static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id) 1213 { 1214 VTDBus *vtd_bus; 1215 VTDAddressSpace *vtd_as; 1216 bool success = false; 1217 1218 vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id)); 1219 if (!vtd_bus) { 1220 goto out; 1221 } 1222 1223 vtd_as = vtd_bus->dev_as[VTD_SID_TO_DEVFN(source_id)]; 1224 if (!vtd_as) { 1225 goto out; 1226 } 1227 1228 if (vtd_switch_address_space(vtd_as) == false) { 1229 /* We switched off IOMMU region successfully. */ 1230 success = true; 1231 } 1232 1233 out: 1234 trace_vtd_pt_enable_fast_path(source_id, success); 1235 } 1236 1237 /* Map dev to context-entry then do a paging-structures walk to do a iommu 1238 * translation. 1239 * 1240 * Called from RCU critical section. 1241 * 1242 * @bus_num: The bus number 1243 * @devfn: The devfn, which is the combined of device and function number 1244 * @is_write: The access is a write operation 1245 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 1246 * 1247 * Returns true if translation is successful, otherwise false. 1248 */ 1249 static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, 1250 uint8_t devfn, hwaddr addr, bool is_write, 1251 IOMMUTLBEntry *entry) 1252 { 1253 IntelIOMMUState *s = vtd_as->iommu_state; 1254 VTDContextEntry ce; 1255 uint8_t bus_num = pci_bus_num(bus); 1256 VTDContextCacheEntry *cc_entry; 1257 uint64_t slpte, page_mask; 1258 uint32_t level; 1259 uint16_t source_id = vtd_make_source_id(bus_num, devfn); 1260 int ret_fr; 1261 bool is_fpd_set = false; 1262 bool reads = true; 1263 bool writes = true; 1264 uint8_t access_flags; 1265 VTDIOTLBEntry *iotlb_entry; 1266 1267 /* 1268 * We have standalone memory region for interrupt addresses, we 1269 * should never receive translation requests in this region. 1270 */ 1271 assert(!vtd_is_interrupt_addr(addr)); 1272 1273 vtd_iommu_lock(s); 1274 1275 cc_entry = &vtd_as->context_cache_entry; 1276 1277 /* Try to fetch slpte form IOTLB */ 1278 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr); 1279 if (iotlb_entry) { 1280 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, 1281 iotlb_entry->domain_id); 1282 slpte = iotlb_entry->slpte; 1283 access_flags = iotlb_entry->access_flags; 1284 page_mask = iotlb_entry->mask; 1285 goto out; 1286 } 1287 1288 /* Try to fetch context-entry from cache first */ 1289 if (cc_entry->context_cache_gen == s->context_cache_gen) { 1290 trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi, 1291 cc_entry->context_entry.lo, 1292 cc_entry->context_cache_gen); 1293 ce = cc_entry->context_entry; 1294 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1295 } else { 1296 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 1297 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1298 if (ret_fr) { 1299 ret_fr = -ret_fr; 1300 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 1301 trace_vtd_fault_disabled(); 1302 } else { 1303 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 1304 } 1305 goto error; 1306 } 1307 /* Update context-cache */ 1308 trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo, 1309 cc_entry->context_cache_gen, 1310 s->context_cache_gen); 1311 cc_entry->context_entry = ce; 1312 cc_entry->context_cache_gen = s->context_cache_gen; 1313 } 1314 1315 /* 1316 * We don't need to translate for pass-through context entries. 1317 * Also, let's ignore IOTLB caching as well for PT devices. 1318 */ 1319 if (vtd_ce_get_type(&ce) == VTD_CONTEXT_TT_PASS_THROUGH) { 1320 entry->iova = addr & VTD_PAGE_MASK_4K; 1321 entry->translated_addr = entry->iova; 1322 entry->addr_mask = ~VTD_PAGE_MASK_4K; 1323 entry->perm = IOMMU_RW; 1324 trace_vtd_translate_pt(source_id, entry->iova); 1325 1326 /* 1327 * When this happens, it means firstly caching-mode is not 1328 * enabled, and this is the first passthrough translation for 1329 * the device. Let's enable the fast path for passthrough. 1330 * 1331 * When passthrough is disabled again for the device, we can 1332 * capture it via the context entry invalidation, then the 1333 * IOMMU region can be swapped back. 1334 */ 1335 vtd_pt_enable_fast_path(s, source_id); 1336 vtd_iommu_unlock(s); 1337 return true; 1338 } 1339 1340 ret_fr = vtd_iova_to_slpte(&ce, addr, is_write, &slpte, &level, 1341 &reads, &writes, s->aw_bits); 1342 if (ret_fr) { 1343 ret_fr = -ret_fr; 1344 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 1345 trace_vtd_fault_disabled(); 1346 } else { 1347 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 1348 } 1349 goto error; 1350 } 1351 1352 page_mask = vtd_slpt_level_page_mask(level); 1353 access_flags = IOMMU_ACCESS_FLAG(reads, writes); 1354 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte, 1355 access_flags, level); 1356 out: 1357 vtd_iommu_unlock(s); 1358 entry->iova = addr & page_mask; 1359 entry->translated_addr = vtd_get_slpte_addr(slpte, s->aw_bits) & page_mask; 1360 entry->addr_mask = ~page_mask; 1361 entry->perm = access_flags; 1362 return true; 1363 1364 error: 1365 vtd_iommu_unlock(s); 1366 entry->iova = 0; 1367 entry->translated_addr = 0; 1368 entry->addr_mask = 0; 1369 entry->perm = IOMMU_NONE; 1370 return false; 1371 } 1372 1373 static void vtd_root_table_setup(IntelIOMMUState *s) 1374 { 1375 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 1376 s->root_extended = s->root & VTD_RTADDR_RTT; 1377 s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits); 1378 1379 trace_vtd_reg_dmar_root(s->root, s->root_extended); 1380 } 1381 1382 static void vtd_iec_notify_all(IntelIOMMUState *s, bool global, 1383 uint32_t index, uint32_t mask) 1384 { 1385 x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask); 1386 } 1387 1388 static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s) 1389 { 1390 uint64_t value = 0; 1391 value = vtd_get_quad_raw(s, DMAR_IRTA_REG); 1392 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1); 1393 s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits); 1394 s->intr_eime = value & VTD_IRTA_EIME; 1395 1396 /* Notify global invalidation */ 1397 vtd_iec_notify_all(s, true, 0, 0); 1398 1399 trace_vtd_reg_ir_root(s->intr_root, s->intr_size); 1400 } 1401 1402 static void vtd_iommu_replay_all(IntelIOMMUState *s) 1403 { 1404 VTDAddressSpace *vtd_as; 1405 1406 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 1407 vtd_sync_shadow_page_table(vtd_as); 1408 } 1409 } 1410 1411 static void vtd_context_global_invalidate(IntelIOMMUState *s) 1412 { 1413 trace_vtd_inv_desc_cc_global(); 1414 /* Protects context cache */ 1415 vtd_iommu_lock(s); 1416 s->context_cache_gen++; 1417 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 1418 vtd_reset_context_cache_locked(s); 1419 } 1420 vtd_iommu_unlock(s); 1421 vtd_switch_address_space_all(s); 1422 /* 1423 * From VT-d spec 6.5.2.1, a global context entry invalidation 1424 * should be followed by a IOTLB global invalidation, so we should 1425 * be safe even without this. Hoewever, let's replay the region as 1426 * well to be safer, and go back here when we need finer tunes for 1427 * VT-d emulation codes. 1428 */ 1429 vtd_iommu_replay_all(s); 1430 } 1431 1432 /* Do a context-cache device-selective invalidation. 1433 * @func_mask: FM field after shifting 1434 */ 1435 static void vtd_context_device_invalidate(IntelIOMMUState *s, 1436 uint16_t source_id, 1437 uint16_t func_mask) 1438 { 1439 uint16_t mask; 1440 VTDBus *vtd_bus; 1441 VTDAddressSpace *vtd_as; 1442 uint8_t bus_n, devfn; 1443 uint16_t devfn_it; 1444 1445 trace_vtd_inv_desc_cc_devices(source_id, func_mask); 1446 1447 switch (func_mask & 3) { 1448 case 0: 1449 mask = 0; /* No bits in the SID field masked */ 1450 break; 1451 case 1: 1452 mask = 4; /* Mask bit 2 in the SID field */ 1453 break; 1454 case 2: 1455 mask = 6; /* Mask bit 2:1 in the SID field */ 1456 break; 1457 case 3: 1458 mask = 7; /* Mask bit 2:0 in the SID field */ 1459 break; 1460 } 1461 mask = ~mask; 1462 1463 bus_n = VTD_SID_TO_BUS(source_id); 1464 vtd_bus = vtd_find_as_from_bus_num(s, bus_n); 1465 if (vtd_bus) { 1466 devfn = VTD_SID_TO_DEVFN(source_id); 1467 for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) { 1468 vtd_as = vtd_bus->dev_as[devfn_it]; 1469 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) { 1470 trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(devfn_it), 1471 VTD_PCI_FUNC(devfn_it)); 1472 vtd_iommu_lock(s); 1473 vtd_as->context_cache_entry.context_cache_gen = 0; 1474 vtd_iommu_unlock(s); 1475 /* 1476 * Do switch address space when needed, in case if the 1477 * device passthrough bit is switched. 1478 */ 1479 vtd_switch_address_space(vtd_as); 1480 /* 1481 * So a device is moving out of (or moving into) a 1482 * domain, resync the shadow page table. 1483 * This won't bring bad even if we have no such 1484 * notifier registered - the IOMMU notification 1485 * framework will skip MAP notifications if that 1486 * happened. 1487 */ 1488 vtd_sync_shadow_page_table(vtd_as); 1489 } 1490 } 1491 } 1492 } 1493 1494 /* Context-cache invalidation 1495 * Returns the Context Actual Invalidation Granularity. 1496 * @val: the content of the CCMD_REG 1497 */ 1498 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 1499 { 1500 uint64_t caig; 1501 uint64_t type = val & VTD_CCMD_CIRG_MASK; 1502 1503 switch (type) { 1504 case VTD_CCMD_DOMAIN_INVL: 1505 /* Fall through */ 1506 case VTD_CCMD_GLOBAL_INVL: 1507 caig = VTD_CCMD_GLOBAL_INVL_A; 1508 vtd_context_global_invalidate(s); 1509 break; 1510 1511 case VTD_CCMD_DEVICE_INVL: 1512 caig = VTD_CCMD_DEVICE_INVL_A; 1513 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 1514 break; 1515 1516 default: 1517 trace_vtd_err("Context cache invalidate type error."); 1518 caig = 0; 1519 } 1520 return caig; 1521 } 1522 1523 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 1524 { 1525 trace_vtd_inv_desc_iotlb_global(); 1526 vtd_reset_iotlb(s); 1527 vtd_iommu_replay_all(s); 1528 } 1529 1530 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 1531 { 1532 VTDContextEntry ce; 1533 VTDAddressSpace *vtd_as; 1534 1535 trace_vtd_inv_desc_iotlb_domain(domain_id); 1536 1537 vtd_iommu_lock(s); 1538 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 1539 &domain_id); 1540 vtd_iommu_unlock(s); 1541 1542 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 1543 if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 1544 vtd_as->devfn, &ce) && 1545 domain_id == VTD_CONTEXT_ENTRY_DID(ce.hi)) { 1546 vtd_sync_shadow_page_table(vtd_as); 1547 } 1548 } 1549 } 1550 1551 static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, 1552 uint16_t domain_id, hwaddr addr, 1553 uint8_t am) 1554 { 1555 VTDAddressSpace *vtd_as; 1556 VTDContextEntry ce; 1557 int ret; 1558 hwaddr size = (1 << am) * VTD_PAGE_SIZE; 1559 1560 QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) { 1561 ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 1562 vtd_as->devfn, &ce); 1563 if (!ret && domain_id == VTD_CONTEXT_ENTRY_DID(ce.hi)) { 1564 if (vtd_as_has_map_notifier(vtd_as)) { 1565 /* 1566 * As long as we have MAP notifications registered in 1567 * any of our IOMMU notifiers, we need to sync the 1568 * shadow page table. 1569 */ 1570 vtd_sync_shadow_page_table_range(vtd_as, &ce, addr, size); 1571 } else { 1572 /* 1573 * For UNMAP-only notifiers, we don't need to walk the 1574 * page tables. We just deliver the PSI down to 1575 * invalidate caches. 1576 */ 1577 IOMMUTLBEntry entry = { 1578 .target_as = &address_space_memory, 1579 .iova = addr, 1580 .translated_addr = 0, 1581 .addr_mask = size - 1, 1582 .perm = IOMMU_NONE, 1583 }; 1584 memory_region_notify_iommu(&vtd_as->iommu, entry); 1585 } 1586 } 1587 } 1588 } 1589 1590 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 1591 hwaddr addr, uint8_t am) 1592 { 1593 VTDIOTLBPageInvInfo info; 1594 1595 trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am); 1596 1597 assert(am <= VTD_MAMV); 1598 info.domain_id = domain_id; 1599 info.addr = addr; 1600 info.mask = ~((1 << am) - 1); 1601 vtd_iommu_lock(s); 1602 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 1603 vtd_iommu_unlock(s); 1604 vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am); 1605 } 1606 1607 /* Flush IOTLB 1608 * Returns the IOTLB Actual Invalidation Granularity. 1609 * @val: the content of the IOTLB_REG 1610 */ 1611 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 1612 { 1613 uint64_t iaig; 1614 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 1615 uint16_t domain_id; 1616 hwaddr addr; 1617 uint8_t am; 1618 1619 switch (type) { 1620 case VTD_TLB_GLOBAL_FLUSH: 1621 iaig = VTD_TLB_GLOBAL_FLUSH_A; 1622 vtd_iotlb_global_invalidate(s); 1623 break; 1624 1625 case VTD_TLB_DSI_FLUSH: 1626 domain_id = VTD_TLB_DID(val); 1627 iaig = VTD_TLB_DSI_FLUSH_A; 1628 vtd_iotlb_domain_invalidate(s, domain_id); 1629 break; 1630 1631 case VTD_TLB_PSI_FLUSH: 1632 domain_id = VTD_TLB_DID(val); 1633 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 1634 am = VTD_IVA_AM(addr); 1635 addr = VTD_IVA_ADDR(addr); 1636 if (am > VTD_MAMV) { 1637 trace_vtd_err("IOTLB PSI flush: address mask overflow."); 1638 iaig = 0; 1639 break; 1640 } 1641 iaig = VTD_TLB_PSI_FLUSH_A; 1642 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1643 break; 1644 1645 default: 1646 trace_vtd_err("IOTLB flush: invalid granularity."); 1647 iaig = 0; 1648 } 1649 return iaig; 1650 } 1651 1652 static void vtd_fetch_inv_desc(IntelIOMMUState *s); 1653 1654 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 1655 { 1656 return s->qi_enabled && (s->iq_tail == s->iq_head) && 1657 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 1658 } 1659 1660 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 1661 { 1662 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 1663 1664 trace_vtd_inv_qi_enable(en); 1665 1666 if (en) { 1667 s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits); 1668 /* 2^(x+8) entries */ 1669 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8); 1670 s->qi_enabled = true; 1671 trace_vtd_inv_qi_setup(s->iq, s->iq_size); 1672 /* Ok - report back to driver */ 1673 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 1674 1675 if (s->iq_tail != 0) { 1676 /* 1677 * This is a spec violation but Windows guests are known to set up 1678 * Queued Invalidation this way so we allow the write and process 1679 * Invalidation Descriptors right away. 1680 */ 1681 trace_vtd_warn_invalid_qi_tail(s->iq_tail); 1682 if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 1683 vtd_fetch_inv_desc(s); 1684 } 1685 } 1686 } else { 1687 if (vtd_queued_inv_disable_check(s)) { 1688 /* disable Queued Invalidation */ 1689 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 1690 s->iq_head = 0; 1691 s->qi_enabled = false; 1692 /* Ok - report back to driver */ 1693 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 1694 } else { 1695 trace_vtd_err_qi_disable(s->iq_head, s->iq_tail, s->iq_last_desc_type); 1696 } 1697 } 1698 } 1699 1700 /* Set Root Table Pointer */ 1701 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 1702 { 1703 vtd_root_table_setup(s); 1704 /* Ok - report back to driver */ 1705 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 1706 } 1707 1708 /* Set Interrupt Remap Table Pointer */ 1709 static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s) 1710 { 1711 vtd_interrupt_remap_table_setup(s); 1712 /* Ok - report back to driver */ 1713 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS); 1714 } 1715 1716 /* Handle Translation Enable/Disable */ 1717 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 1718 { 1719 if (s->dmar_enabled == en) { 1720 return; 1721 } 1722 1723 trace_vtd_dmar_enable(en); 1724 1725 if (en) { 1726 s->dmar_enabled = true; 1727 /* Ok - report back to driver */ 1728 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 1729 } else { 1730 s->dmar_enabled = false; 1731 1732 /* Clear the index of Fault Recording Register */ 1733 s->next_frcd_reg = 0; 1734 /* Ok - report back to driver */ 1735 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 1736 } 1737 1738 vtd_switch_address_space_all(s); 1739 } 1740 1741 /* Handle Interrupt Remap Enable/Disable */ 1742 static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en) 1743 { 1744 trace_vtd_ir_enable(en); 1745 1746 if (en) { 1747 s->intr_enabled = true; 1748 /* Ok - report back to driver */ 1749 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES); 1750 } else { 1751 s->intr_enabled = false; 1752 /* Ok - report back to driver */ 1753 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0); 1754 } 1755 } 1756 1757 /* Handle write to Global Command Register */ 1758 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 1759 { 1760 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 1761 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 1762 uint32_t changed = status ^ val; 1763 1764 trace_vtd_reg_write_gcmd(status, val); 1765 if (changed & VTD_GCMD_TE) { 1766 /* Translation enable/disable */ 1767 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 1768 } 1769 if (val & VTD_GCMD_SRTP) { 1770 /* Set/update the root-table pointer */ 1771 vtd_handle_gcmd_srtp(s); 1772 } 1773 if (changed & VTD_GCMD_QIE) { 1774 /* Queued Invalidation Enable */ 1775 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 1776 } 1777 if (val & VTD_GCMD_SIRTP) { 1778 /* Set/update the interrupt remapping root-table pointer */ 1779 vtd_handle_gcmd_sirtp(s); 1780 } 1781 if (changed & VTD_GCMD_IRE) { 1782 /* Interrupt remap enable/disable */ 1783 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE); 1784 } 1785 } 1786 1787 /* Handle write to Context Command Register */ 1788 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 1789 { 1790 uint64_t ret; 1791 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 1792 1793 /* Context-cache invalidation request */ 1794 if (val & VTD_CCMD_ICC) { 1795 if (s->qi_enabled) { 1796 trace_vtd_err("Queued Invalidation enabled, " 1797 "should not use register-based invalidation"); 1798 return; 1799 } 1800 ret = vtd_context_cache_invalidate(s, val); 1801 /* Invalidation completed. Change something to show */ 1802 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 1803 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 1804 ret); 1805 } 1806 } 1807 1808 /* Handle write to IOTLB Invalidation Register */ 1809 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 1810 { 1811 uint64_t ret; 1812 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 1813 1814 /* IOTLB invalidation request */ 1815 if (val & VTD_TLB_IVT) { 1816 if (s->qi_enabled) { 1817 trace_vtd_err("Queued Invalidation enabled, " 1818 "should not use register-based invalidation."); 1819 return; 1820 } 1821 ret = vtd_iotlb_flush(s, val); 1822 /* Invalidation completed. Change something to show */ 1823 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 1824 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 1825 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 1826 } 1827 } 1828 1829 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 1830 static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset, 1831 VTDInvDesc *inv_desc) 1832 { 1833 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc); 1834 if (dma_memory_read(&address_space_memory, addr, inv_desc, 1835 sizeof(*inv_desc))) { 1836 trace_vtd_err("Read INV DESC failed."); 1837 inv_desc->lo = 0; 1838 inv_desc->hi = 0; 1839 return false; 1840 } 1841 inv_desc->lo = le64_to_cpu(inv_desc->lo); 1842 inv_desc->hi = le64_to_cpu(inv_desc->hi); 1843 return true; 1844 } 1845 1846 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1847 { 1848 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 1849 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 1850 trace_vtd_inv_desc_wait_invalid(inv_desc->hi, inv_desc->lo); 1851 return false; 1852 } 1853 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 1854 /* Status Write */ 1855 uint32_t status_data = (uint32_t)(inv_desc->lo >> 1856 VTD_INV_DESC_WAIT_DATA_SHIFT); 1857 1858 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 1859 1860 /* FIXME: need to be masked with HAW? */ 1861 dma_addr_t status_addr = inv_desc->hi; 1862 trace_vtd_inv_desc_wait_sw(status_addr, status_data); 1863 status_data = cpu_to_le32(status_data); 1864 if (dma_memory_write(&address_space_memory, status_addr, &status_data, 1865 sizeof(status_data))) { 1866 trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo); 1867 return false; 1868 } 1869 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 1870 /* Interrupt flag */ 1871 vtd_generate_completion_event(s); 1872 } else { 1873 trace_vtd_inv_desc_wait_invalid(inv_desc->hi, inv_desc->lo); 1874 return false; 1875 } 1876 return true; 1877 } 1878 1879 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 1880 VTDInvDesc *inv_desc) 1881 { 1882 uint16_t sid, fmask; 1883 1884 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 1885 trace_vtd_inv_desc_cc_invalid(inv_desc->hi, inv_desc->lo); 1886 return false; 1887 } 1888 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 1889 case VTD_INV_DESC_CC_DOMAIN: 1890 trace_vtd_inv_desc_cc_domain( 1891 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 1892 /* Fall through */ 1893 case VTD_INV_DESC_CC_GLOBAL: 1894 vtd_context_global_invalidate(s); 1895 break; 1896 1897 case VTD_INV_DESC_CC_DEVICE: 1898 sid = VTD_INV_DESC_CC_SID(inv_desc->lo); 1899 fmask = VTD_INV_DESC_CC_FM(inv_desc->lo); 1900 vtd_context_device_invalidate(s, sid, fmask); 1901 break; 1902 1903 default: 1904 trace_vtd_inv_desc_cc_invalid(inv_desc->hi, inv_desc->lo); 1905 return false; 1906 } 1907 return true; 1908 } 1909 1910 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1911 { 1912 uint16_t domain_id; 1913 uint8_t am; 1914 hwaddr addr; 1915 1916 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 1917 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 1918 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo); 1919 return false; 1920 } 1921 1922 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 1923 case VTD_INV_DESC_IOTLB_GLOBAL: 1924 vtd_iotlb_global_invalidate(s); 1925 break; 1926 1927 case VTD_INV_DESC_IOTLB_DOMAIN: 1928 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1929 vtd_iotlb_domain_invalidate(s, domain_id); 1930 break; 1931 1932 case VTD_INV_DESC_IOTLB_PAGE: 1933 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1934 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 1935 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 1936 if (am > VTD_MAMV) { 1937 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo); 1938 return false; 1939 } 1940 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1941 break; 1942 1943 default: 1944 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo); 1945 return false; 1946 } 1947 return true; 1948 } 1949 1950 static bool vtd_process_inv_iec_desc(IntelIOMMUState *s, 1951 VTDInvDesc *inv_desc) 1952 { 1953 trace_vtd_inv_desc_iec(inv_desc->iec.granularity, 1954 inv_desc->iec.index, 1955 inv_desc->iec.index_mask); 1956 1957 vtd_iec_notify_all(s, !inv_desc->iec.granularity, 1958 inv_desc->iec.index, 1959 inv_desc->iec.index_mask); 1960 return true; 1961 } 1962 1963 static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s, 1964 VTDInvDesc *inv_desc) 1965 { 1966 VTDAddressSpace *vtd_dev_as; 1967 IOMMUTLBEntry entry; 1968 struct VTDBus *vtd_bus; 1969 hwaddr addr; 1970 uint64_t sz; 1971 uint16_t sid; 1972 uint8_t devfn; 1973 bool size; 1974 uint8_t bus_num; 1975 1976 addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi); 1977 sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo); 1978 devfn = sid & 0xff; 1979 bus_num = sid >> 8; 1980 size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi); 1981 1982 if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) || 1983 (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) { 1984 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo); 1985 return false; 1986 } 1987 1988 vtd_bus = vtd_find_as_from_bus_num(s, bus_num); 1989 if (!vtd_bus) { 1990 goto done; 1991 } 1992 1993 vtd_dev_as = vtd_bus->dev_as[devfn]; 1994 if (!vtd_dev_as) { 1995 goto done; 1996 } 1997 1998 /* According to ATS spec table 2.4: 1999 * S = 0, bits 15:12 = xxxx range size: 4K 2000 * S = 1, bits 15:12 = xxx0 range size: 8K 2001 * S = 1, bits 15:12 = xx01 range size: 16K 2002 * S = 1, bits 15:12 = x011 range size: 32K 2003 * S = 1, bits 15:12 = 0111 range size: 64K 2004 * ... 2005 */ 2006 if (size) { 2007 sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT); 2008 addr &= ~(sz - 1); 2009 } else { 2010 sz = VTD_PAGE_SIZE; 2011 } 2012 2013 entry.target_as = &vtd_dev_as->as; 2014 entry.addr_mask = sz - 1; 2015 entry.iova = addr; 2016 entry.perm = IOMMU_NONE; 2017 entry.translated_addr = 0; 2018 memory_region_notify_iommu(&vtd_dev_as->iommu, entry); 2019 2020 done: 2021 return true; 2022 } 2023 2024 static bool vtd_process_inv_desc(IntelIOMMUState *s) 2025 { 2026 VTDInvDesc inv_desc; 2027 uint8_t desc_type; 2028 2029 trace_vtd_inv_qi_head(s->iq_head); 2030 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) { 2031 s->iq_last_desc_type = VTD_INV_DESC_NONE; 2032 return false; 2033 } 2034 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 2035 /* FIXME: should update at first or at last? */ 2036 s->iq_last_desc_type = desc_type; 2037 2038 switch (desc_type) { 2039 case VTD_INV_DESC_CC: 2040 trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo); 2041 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 2042 return false; 2043 } 2044 break; 2045 2046 case VTD_INV_DESC_IOTLB: 2047 trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo); 2048 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 2049 return false; 2050 } 2051 break; 2052 2053 case VTD_INV_DESC_WAIT: 2054 trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo); 2055 if (!vtd_process_wait_desc(s, &inv_desc)) { 2056 return false; 2057 } 2058 break; 2059 2060 case VTD_INV_DESC_IEC: 2061 trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo); 2062 if (!vtd_process_inv_iec_desc(s, &inv_desc)) { 2063 return false; 2064 } 2065 break; 2066 2067 case VTD_INV_DESC_DEVICE: 2068 trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo); 2069 if (!vtd_process_device_iotlb_desc(s, &inv_desc)) { 2070 return false; 2071 } 2072 break; 2073 2074 default: 2075 trace_vtd_inv_desc_invalid(inv_desc.hi, inv_desc.lo); 2076 return false; 2077 } 2078 s->iq_head++; 2079 if (s->iq_head == s->iq_size) { 2080 s->iq_head = 0; 2081 } 2082 return true; 2083 } 2084 2085 /* Try to fetch and process more Invalidation Descriptors */ 2086 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 2087 { 2088 trace_vtd_inv_qi_fetch(); 2089 2090 if (s->iq_tail >= s->iq_size) { 2091 /* Detects an invalid Tail pointer */ 2092 trace_vtd_err_qi_tail(s->iq_tail, s->iq_size); 2093 vtd_handle_inv_queue_error(s); 2094 return; 2095 } 2096 while (s->iq_head != s->iq_tail) { 2097 if (!vtd_process_inv_desc(s)) { 2098 /* Invalidation Queue Errors */ 2099 vtd_handle_inv_queue_error(s); 2100 break; 2101 } 2102 /* Must update the IQH_REG in time */ 2103 vtd_set_quad_raw(s, DMAR_IQH_REG, 2104 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) & 2105 VTD_IQH_QH_MASK); 2106 } 2107 } 2108 2109 /* Handle write to Invalidation Queue Tail Register */ 2110 static void vtd_handle_iqt_write(IntelIOMMUState *s) 2111 { 2112 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 2113 2114 s->iq_tail = VTD_IQT_QT(val); 2115 trace_vtd_inv_qi_tail(s->iq_tail); 2116 2117 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 2118 /* Process Invalidation Queue here */ 2119 vtd_fetch_inv_desc(s); 2120 } 2121 } 2122 2123 static void vtd_handle_fsts_write(IntelIOMMUState *s) 2124 { 2125 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 2126 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 2127 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 2128 2129 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 2130 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 2131 trace_vtd_fsts_clear_ip(); 2132 } 2133 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 2134 * Descriptors if there are any when Queued Invalidation is enabled? 2135 */ 2136 } 2137 2138 static void vtd_handle_fectl_write(IntelIOMMUState *s) 2139 { 2140 uint32_t fectl_reg; 2141 /* FIXME: when software clears the IM field, check the IP field. But do we 2142 * need to compare the old value and the new value to conclude that 2143 * software clears the IM field? Or just check if the IM field is zero? 2144 */ 2145 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 2146 2147 trace_vtd_reg_write_fectl(fectl_reg); 2148 2149 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 2150 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 2151 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 2152 } 2153 } 2154 2155 static void vtd_handle_ics_write(IntelIOMMUState *s) 2156 { 2157 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 2158 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 2159 2160 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 2161 trace_vtd_reg_ics_clear_ip(); 2162 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 2163 } 2164 } 2165 2166 static void vtd_handle_iectl_write(IntelIOMMUState *s) 2167 { 2168 uint32_t iectl_reg; 2169 /* FIXME: when software clears the IM field, check the IP field. But do we 2170 * need to compare the old value and the new value to conclude that 2171 * software clears the IM field? Or just check if the IM field is zero? 2172 */ 2173 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 2174 2175 trace_vtd_reg_write_iectl(iectl_reg); 2176 2177 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 2178 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 2179 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 2180 } 2181 } 2182 2183 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 2184 { 2185 IntelIOMMUState *s = opaque; 2186 uint64_t val; 2187 2188 trace_vtd_reg_read(addr, size); 2189 2190 if (addr + size > DMAR_REG_SIZE) { 2191 trace_vtd_err("Read MMIO over range."); 2192 return (uint64_t)-1; 2193 } 2194 2195 switch (addr) { 2196 /* Root Table Address Register, 64-bit */ 2197 case DMAR_RTADDR_REG: 2198 if (size == 4) { 2199 val = s->root & ((1ULL << 32) - 1); 2200 } else { 2201 val = s->root; 2202 } 2203 break; 2204 2205 case DMAR_RTADDR_REG_HI: 2206 assert(size == 4); 2207 val = s->root >> 32; 2208 break; 2209 2210 /* Invalidation Queue Address Register, 64-bit */ 2211 case DMAR_IQA_REG: 2212 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 2213 if (size == 4) { 2214 val = val & ((1ULL << 32) - 1); 2215 } 2216 break; 2217 2218 case DMAR_IQA_REG_HI: 2219 assert(size == 4); 2220 val = s->iq >> 32; 2221 break; 2222 2223 default: 2224 if (size == 4) { 2225 val = vtd_get_long(s, addr); 2226 } else { 2227 val = vtd_get_quad(s, addr); 2228 } 2229 } 2230 2231 return val; 2232 } 2233 2234 static void vtd_mem_write(void *opaque, hwaddr addr, 2235 uint64_t val, unsigned size) 2236 { 2237 IntelIOMMUState *s = opaque; 2238 2239 trace_vtd_reg_write(addr, size, val); 2240 2241 if (addr + size > DMAR_REG_SIZE) { 2242 trace_vtd_err("Write MMIO over range."); 2243 return; 2244 } 2245 2246 switch (addr) { 2247 /* Global Command Register, 32-bit */ 2248 case DMAR_GCMD_REG: 2249 vtd_set_long(s, addr, val); 2250 vtd_handle_gcmd_write(s); 2251 break; 2252 2253 /* Context Command Register, 64-bit */ 2254 case DMAR_CCMD_REG: 2255 if (size == 4) { 2256 vtd_set_long(s, addr, val); 2257 } else { 2258 vtd_set_quad(s, addr, val); 2259 vtd_handle_ccmd_write(s); 2260 } 2261 break; 2262 2263 case DMAR_CCMD_REG_HI: 2264 assert(size == 4); 2265 vtd_set_long(s, addr, val); 2266 vtd_handle_ccmd_write(s); 2267 break; 2268 2269 /* IOTLB Invalidation Register, 64-bit */ 2270 case DMAR_IOTLB_REG: 2271 if (size == 4) { 2272 vtd_set_long(s, addr, val); 2273 } else { 2274 vtd_set_quad(s, addr, val); 2275 vtd_handle_iotlb_write(s); 2276 } 2277 break; 2278 2279 case DMAR_IOTLB_REG_HI: 2280 assert(size == 4); 2281 vtd_set_long(s, addr, val); 2282 vtd_handle_iotlb_write(s); 2283 break; 2284 2285 /* Invalidate Address Register, 64-bit */ 2286 case DMAR_IVA_REG: 2287 if (size == 4) { 2288 vtd_set_long(s, addr, val); 2289 } else { 2290 vtd_set_quad(s, addr, val); 2291 } 2292 break; 2293 2294 case DMAR_IVA_REG_HI: 2295 assert(size == 4); 2296 vtd_set_long(s, addr, val); 2297 break; 2298 2299 /* Fault Status Register, 32-bit */ 2300 case DMAR_FSTS_REG: 2301 assert(size == 4); 2302 vtd_set_long(s, addr, val); 2303 vtd_handle_fsts_write(s); 2304 break; 2305 2306 /* Fault Event Control Register, 32-bit */ 2307 case DMAR_FECTL_REG: 2308 assert(size == 4); 2309 vtd_set_long(s, addr, val); 2310 vtd_handle_fectl_write(s); 2311 break; 2312 2313 /* Fault Event Data Register, 32-bit */ 2314 case DMAR_FEDATA_REG: 2315 assert(size == 4); 2316 vtd_set_long(s, addr, val); 2317 break; 2318 2319 /* Fault Event Address Register, 32-bit */ 2320 case DMAR_FEADDR_REG: 2321 if (size == 4) { 2322 vtd_set_long(s, addr, val); 2323 } else { 2324 /* 2325 * While the register is 32-bit only, some guests (Xen...) write to 2326 * it with 64-bit. 2327 */ 2328 vtd_set_quad(s, addr, val); 2329 } 2330 break; 2331 2332 /* Fault Event Upper Address Register, 32-bit */ 2333 case DMAR_FEUADDR_REG: 2334 assert(size == 4); 2335 vtd_set_long(s, addr, val); 2336 break; 2337 2338 /* Protected Memory Enable Register, 32-bit */ 2339 case DMAR_PMEN_REG: 2340 assert(size == 4); 2341 vtd_set_long(s, addr, val); 2342 break; 2343 2344 /* Root Table Address Register, 64-bit */ 2345 case DMAR_RTADDR_REG: 2346 if (size == 4) { 2347 vtd_set_long(s, addr, val); 2348 } else { 2349 vtd_set_quad(s, addr, val); 2350 } 2351 break; 2352 2353 case DMAR_RTADDR_REG_HI: 2354 assert(size == 4); 2355 vtd_set_long(s, addr, val); 2356 break; 2357 2358 /* Invalidation Queue Tail Register, 64-bit */ 2359 case DMAR_IQT_REG: 2360 if (size == 4) { 2361 vtd_set_long(s, addr, val); 2362 } else { 2363 vtd_set_quad(s, addr, val); 2364 } 2365 vtd_handle_iqt_write(s); 2366 break; 2367 2368 case DMAR_IQT_REG_HI: 2369 assert(size == 4); 2370 vtd_set_long(s, addr, val); 2371 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 2372 break; 2373 2374 /* Invalidation Queue Address Register, 64-bit */ 2375 case DMAR_IQA_REG: 2376 if (size == 4) { 2377 vtd_set_long(s, addr, val); 2378 } else { 2379 vtd_set_quad(s, addr, val); 2380 } 2381 break; 2382 2383 case DMAR_IQA_REG_HI: 2384 assert(size == 4); 2385 vtd_set_long(s, addr, val); 2386 break; 2387 2388 /* Invalidation Completion Status Register, 32-bit */ 2389 case DMAR_ICS_REG: 2390 assert(size == 4); 2391 vtd_set_long(s, addr, val); 2392 vtd_handle_ics_write(s); 2393 break; 2394 2395 /* Invalidation Event Control Register, 32-bit */ 2396 case DMAR_IECTL_REG: 2397 assert(size == 4); 2398 vtd_set_long(s, addr, val); 2399 vtd_handle_iectl_write(s); 2400 break; 2401 2402 /* Invalidation Event Data Register, 32-bit */ 2403 case DMAR_IEDATA_REG: 2404 assert(size == 4); 2405 vtd_set_long(s, addr, val); 2406 break; 2407 2408 /* Invalidation Event Address Register, 32-bit */ 2409 case DMAR_IEADDR_REG: 2410 assert(size == 4); 2411 vtd_set_long(s, addr, val); 2412 break; 2413 2414 /* Invalidation Event Upper Address Register, 32-bit */ 2415 case DMAR_IEUADDR_REG: 2416 assert(size == 4); 2417 vtd_set_long(s, addr, val); 2418 break; 2419 2420 /* Fault Recording Registers, 128-bit */ 2421 case DMAR_FRCD_REG_0_0: 2422 if (size == 4) { 2423 vtd_set_long(s, addr, val); 2424 } else { 2425 vtd_set_quad(s, addr, val); 2426 } 2427 break; 2428 2429 case DMAR_FRCD_REG_0_1: 2430 assert(size == 4); 2431 vtd_set_long(s, addr, val); 2432 break; 2433 2434 case DMAR_FRCD_REG_0_2: 2435 if (size == 4) { 2436 vtd_set_long(s, addr, val); 2437 } else { 2438 vtd_set_quad(s, addr, val); 2439 /* May clear bit 127 (Fault), update PPF */ 2440 vtd_update_fsts_ppf(s); 2441 } 2442 break; 2443 2444 case DMAR_FRCD_REG_0_3: 2445 assert(size == 4); 2446 vtd_set_long(s, addr, val); 2447 /* May clear bit 127 (Fault), update PPF */ 2448 vtd_update_fsts_ppf(s); 2449 break; 2450 2451 case DMAR_IRTA_REG: 2452 if (size == 4) { 2453 vtd_set_long(s, addr, val); 2454 } else { 2455 vtd_set_quad(s, addr, val); 2456 } 2457 break; 2458 2459 case DMAR_IRTA_REG_HI: 2460 assert(size == 4); 2461 vtd_set_long(s, addr, val); 2462 break; 2463 2464 default: 2465 if (size == 4) { 2466 vtd_set_long(s, addr, val); 2467 } else { 2468 vtd_set_quad(s, addr, val); 2469 } 2470 } 2471 } 2472 2473 static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, 2474 IOMMUAccessFlags flag) 2475 { 2476 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 2477 IntelIOMMUState *s = vtd_as->iommu_state; 2478 IOMMUTLBEntry iotlb = { 2479 /* We'll fill in the rest later. */ 2480 .target_as = &address_space_memory, 2481 }; 2482 bool success; 2483 2484 if (likely(s->dmar_enabled)) { 2485 success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, 2486 addr, flag & IOMMU_WO, &iotlb); 2487 } else { 2488 /* DMAR disabled, passthrough, use 4k-page*/ 2489 iotlb.iova = addr & VTD_PAGE_MASK_4K; 2490 iotlb.translated_addr = addr & VTD_PAGE_MASK_4K; 2491 iotlb.addr_mask = ~VTD_PAGE_MASK_4K; 2492 iotlb.perm = IOMMU_RW; 2493 success = true; 2494 } 2495 2496 if (likely(success)) { 2497 trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus), 2498 VTD_PCI_SLOT(vtd_as->devfn), 2499 VTD_PCI_FUNC(vtd_as->devfn), 2500 iotlb.iova, iotlb.translated_addr, 2501 iotlb.addr_mask); 2502 } else { 2503 trace_vtd_err_dmar_translate(pci_bus_num(vtd_as->bus), 2504 VTD_PCI_SLOT(vtd_as->devfn), 2505 VTD_PCI_FUNC(vtd_as->devfn), 2506 iotlb.iova); 2507 } 2508 2509 return iotlb; 2510 } 2511 2512 static void vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu, 2513 IOMMUNotifierFlag old, 2514 IOMMUNotifierFlag new) 2515 { 2516 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 2517 IntelIOMMUState *s = vtd_as->iommu_state; 2518 2519 if (!s->caching_mode && new & IOMMU_NOTIFIER_MAP) { 2520 error_report("We need to set caching-mode=1 for intel-iommu to enable " 2521 "device assignment with IOMMU protection."); 2522 exit(1); 2523 } 2524 2525 /* Update per-address-space notifier flags */ 2526 vtd_as->notifier_flags = new; 2527 2528 if (old == IOMMU_NOTIFIER_NONE) { 2529 QLIST_INSERT_HEAD(&s->vtd_as_with_notifiers, vtd_as, next); 2530 } else if (new == IOMMU_NOTIFIER_NONE) { 2531 QLIST_REMOVE(vtd_as, next); 2532 } 2533 } 2534 2535 static int vtd_post_load(void *opaque, int version_id) 2536 { 2537 IntelIOMMUState *iommu = opaque; 2538 2539 /* 2540 * Memory regions are dynamically turned on/off depending on 2541 * context entry configurations from the guest. After migration, 2542 * we need to make sure the memory regions are still correct. 2543 */ 2544 vtd_switch_address_space_all(iommu); 2545 2546 return 0; 2547 } 2548 2549 static const VMStateDescription vtd_vmstate = { 2550 .name = "iommu-intel", 2551 .version_id = 1, 2552 .minimum_version_id = 1, 2553 .priority = MIG_PRI_IOMMU, 2554 .post_load = vtd_post_load, 2555 .fields = (VMStateField[]) { 2556 VMSTATE_UINT64(root, IntelIOMMUState), 2557 VMSTATE_UINT64(intr_root, IntelIOMMUState), 2558 VMSTATE_UINT64(iq, IntelIOMMUState), 2559 VMSTATE_UINT32(intr_size, IntelIOMMUState), 2560 VMSTATE_UINT16(iq_head, IntelIOMMUState), 2561 VMSTATE_UINT16(iq_tail, IntelIOMMUState), 2562 VMSTATE_UINT16(iq_size, IntelIOMMUState), 2563 VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState), 2564 VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE), 2565 VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState), 2566 VMSTATE_BOOL(root_extended, IntelIOMMUState), 2567 VMSTATE_BOOL(dmar_enabled, IntelIOMMUState), 2568 VMSTATE_BOOL(qi_enabled, IntelIOMMUState), 2569 VMSTATE_BOOL(intr_enabled, IntelIOMMUState), 2570 VMSTATE_BOOL(intr_eime, IntelIOMMUState), 2571 VMSTATE_END_OF_LIST() 2572 } 2573 }; 2574 2575 static const MemoryRegionOps vtd_mem_ops = { 2576 .read = vtd_mem_read, 2577 .write = vtd_mem_write, 2578 .endianness = DEVICE_LITTLE_ENDIAN, 2579 .impl = { 2580 .min_access_size = 4, 2581 .max_access_size = 8, 2582 }, 2583 .valid = { 2584 .min_access_size = 4, 2585 .max_access_size = 8, 2586 }, 2587 }; 2588 2589 static Property vtd_properties[] = { 2590 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 2591 DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim, 2592 ON_OFF_AUTO_AUTO), 2593 DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false), 2594 DEFINE_PROP_UINT8("x-aw-bits", IntelIOMMUState, aw_bits, 2595 VTD_HOST_ADDRESS_WIDTH), 2596 DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE), 2597 DEFINE_PROP_END_OF_LIST(), 2598 }; 2599 2600 /* Read IRTE entry with specific index */ 2601 static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index, 2602 VTD_IR_TableEntry *entry, uint16_t sid) 2603 { 2604 static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \ 2605 {0xffff, 0xfffb, 0xfff9, 0xfff8}; 2606 dma_addr_t addr = 0x00; 2607 uint16_t mask, source_id; 2608 uint8_t bus, bus_max, bus_min; 2609 2610 addr = iommu->intr_root + index * sizeof(*entry); 2611 if (dma_memory_read(&address_space_memory, addr, entry, 2612 sizeof(*entry))) { 2613 trace_vtd_err("Memory read failed for IRTE."); 2614 return -VTD_FR_IR_ROOT_INVAL; 2615 } 2616 2617 trace_vtd_ir_irte_get(index, le64_to_cpu(entry->data[1]), 2618 le64_to_cpu(entry->data[0])); 2619 2620 if (!entry->irte.present) { 2621 trace_vtd_err_irte(index, le64_to_cpu(entry->data[1]), 2622 le64_to_cpu(entry->data[0])); 2623 return -VTD_FR_IR_ENTRY_P; 2624 } 2625 2626 if (entry->irte.__reserved_0 || entry->irte.__reserved_1 || 2627 entry->irte.__reserved_2) { 2628 trace_vtd_err_irte(index, le64_to_cpu(entry->data[1]), 2629 le64_to_cpu(entry->data[0])); 2630 return -VTD_FR_IR_IRTE_RSVD; 2631 } 2632 2633 if (sid != X86_IOMMU_SID_INVALID) { 2634 /* Validate IRTE SID */ 2635 source_id = le32_to_cpu(entry->irte.source_id); 2636 switch (entry->irte.sid_vtype) { 2637 case VTD_SVT_NONE: 2638 break; 2639 2640 case VTD_SVT_ALL: 2641 mask = vtd_svt_mask[entry->irte.sid_q]; 2642 if ((source_id & mask) != (sid & mask)) { 2643 trace_vtd_err_irte_sid(index, sid, source_id); 2644 return -VTD_FR_IR_SID_ERR; 2645 } 2646 break; 2647 2648 case VTD_SVT_BUS: 2649 bus_max = source_id >> 8; 2650 bus_min = source_id & 0xff; 2651 bus = sid >> 8; 2652 if (bus > bus_max || bus < bus_min) { 2653 trace_vtd_err_irte_sid_bus(index, bus, bus_min, bus_max); 2654 return -VTD_FR_IR_SID_ERR; 2655 } 2656 break; 2657 2658 default: 2659 trace_vtd_err_irte_svt(index, entry->irte.sid_vtype); 2660 /* Take this as verification failure. */ 2661 return -VTD_FR_IR_SID_ERR; 2662 break; 2663 } 2664 } 2665 2666 return 0; 2667 } 2668 2669 /* Fetch IRQ information of specific IR index */ 2670 static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index, 2671 VTDIrq *irq, uint16_t sid) 2672 { 2673 VTD_IR_TableEntry irte = {}; 2674 int ret = 0; 2675 2676 ret = vtd_irte_get(iommu, index, &irte, sid); 2677 if (ret) { 2678 return ret; 2679 } 2680 2681 irq->trigger_mode = irte.irte.trigger_mode; 2682 irq->vector = irte.irte.vector; 2683 irq->delivery_mode = irte.irte.delivery_mode; 2684 irq->dest = le32_to_cpu(irte.irte.dest_id); 2685 if (!iommu->intr_eime) { 2686 #define VTD_IR_APIC_DEST_MASK (0xff00ULL) 2687 #define VTD_IR_APIC_DEST_SHIFT (8) 2688 irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >> 2689 VTD_IR_APIC_DEST_SHIFT; 2690 } 2691 irq->dest_mode = irte.irte.dest_mode; 2692 irq->redir_hint = irte.irte.redir_hint; 2693 2694 trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector, 2695 irq->delivery_mode, irq->dest, irq->dest_mode); 2696 2697 return 0; 2698 } 2699 2700 /* Generate one MSI message from VTDIrq info */ 2701 static void vtd_generate_msi_message(VTDIrq *irq, MSIMessage *msg_out) 2702 { 2703 VTD_MSIMessage msg = {}; 2704 2705 /* Generate address bits */ 2706 msg.dest_mode = irq->dest_mode; 2707 msg.redir_hint = irq->redir_hint; 2708 msg.dest = irq->dest; 2709 msg.__addr_hi = irq->dest & 0xffffff00; 2710 msg.__addr_head = cpu_to_le32(0xfee); 2711 /* Keep this from original MSI address bits */ 2712 msg.__not_used = irq->msi_addr_last_bits; 2713 2714 /* Generate data bits */ 2715 msg.vector = irq->vector; 2716 msg.delivery_mode = irq->delivery_mode; 2717 msg.level = 1; 2718 msg.trigger_mode = irq->trigger_mode; 2719 2720 msg_out->address = msg.msi_addr; 2721 msg_out->data = msg.msi_data; 2722 } 2723 2724 /* Interrupt remapping for MSI/MSI-X entry */ 2725 static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu, 2726 MSIMessage *origin, 2727 MSIMessage *translated, 2728 uint16_t sid) 2729 { 2730 int ret = 0; 2731 VTD_IR_MSIAddress addr; 2732 uint16_t index; 2733 VTDIrq irq = {}; 2734 2735 assert(origin && translated); 2736 2737 trace_vtd_ir_remap_msi_req(origin->address, origin->data); 2738 2739 if (!iommu || !iommu->intr_enabled) { 2740 memcpy(translated, origin, sizeof(*origin)); 2741 goto out; 2742 } 2743 2744 if (origin->address & VTD_MSI_ADDR_HI_MASK) { 2745 trace_vtd_err("MSI address high 32 bits non-zero when " 2746 "Interrupt Remapping enabled."); 2747 return -VTD_FR_IR_REQ_RSVD; 2748 } 2749 2750 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK; 2751 if (addr.addr.__head != 0xfee) { 2752 trace_vtd_err("MSI addr low 32 bit invalid."); 2753 return -VTD_FR_IR_REQ_RSVD; 2754 } 2755 2756 /* This is compatible mode. */ 2757 if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) { 2758 memcpy(translated, origin, sizeof(*origin)); 2759 goto out; 2760 } 2761 2762 index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l); 2763 2764 #define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff) 2765 #define VTD_IR_MSI_DATA_RESERVED (0xffff0000) 2766 2767 if (addr.addr.sub_valid) { 2768 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */ 2769 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE; 2770 } 2771 2772 ret = vtd_remap_irq_get(iommu, index, &irq, sid); 2773 if (ret) { 2774 return ret; 2775 } 2776 2777 if (addr.addr.sub_valid) { 2778 trace_vtd_ir_remap_type("MSI"); 2779 if (origin->data & VTD_IR_MSI_DATA_RESERVED) { 2780 trace_vtd_err_ir_msi_invalid(sid, origin->address, origin->data); 2781 return -VTD_FR_IR_REQ_RSVD; 2782 } 2783 } else { 2784 uint8_t vector = origin->data & 0xff; 2785 uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1; 2786 2787 trace_vtd_ir_remap_type("IOAPIC"); 2788 /* IOAPIC entry vector should be aligned with IRTE vector 2789 * (see vt-d spec 5.1.5.1). */ 2790 if (vector != irq.vector) { 2791 trace_vtd_warn_ir_vector(sid, index, vector, irq.vector); 2792 } 2793 2794 /* The Trigger Mode field must match the Trigger Mode in the IRTE. 2795 * (see vt-d spec 5.1.5.1). */ 2796 if (trigger_mode != irq.trigger_mode) { 2797 trace_vtd_warn_ir_trigger(sid, index, trigger_mode, 2798 irq.trigger_mode); 2799 } 2800 } 2801 2802 /* 2803 * We'd better keep the last two bits, assuming that guest OS 2804 * might modify it. Keep it does not hurt after all. 2805 */ 2806 irq.msi_addr_last_bits = addr.addr.__not_care; 2807 2808 /* Translate VTDIrq to MSI message */ 2809 vtd_generate_msi_message(&irq, translated); 2810 2811 out: 2812 trace_vtd_ir_remap_msi(origin->address, origin->data, 2813 translated->address, translated->data); 2814 return 0; 2815 } 2816 2817 static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src, 2818 MSIMessage *dst, uint16_t sid) 2819 { 2820 return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu), 2821 src, dst, sid); 2822 } 2823 2824 static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr, 2825 uint64_t *data, unsigned size, 2826 MemTxAttrs attrs) 2827 { 2828 return MEMTX_OK; 2829 } 2830 2831 static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr, 2832 uint64_t value, unsigned size, 2833 MemTxAttrs attrs) 2834 { 2835 int ret = 0; 2836 MSIMessage from = {}, to = {}; 2837 uint16_t sid = X86_IOMMU_SID_INVALID; 2838 2839 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST; 2840 from.data = (uint32_t) value; 2841 2842 if (!attrs.unspecified) { 2843 /* We have explicit Source ID */ 2844 sid = attrs.requester_id; 2845 } 2846 2847 ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid); 2848 if (ret) { 2849 /* TODO: report error */ 2850 /* Drop this interrupt */ 2851 return MEMTX_ERROR; 2852 } 2853 2854 apic_get_class()->send_msi(&to); 2855 2856 return MEMTX_OK; 2857 } 2858 2859 static const MemoryRegionOps vtd_mem_ir_ops = { 2860 .read_with_attrs = vtd_mem_ir_read, 2861 .write_with_attrs = vtd_mem_ir_write, 2862 .endianness = DEVICE_LITTLE_ENDIAN, 2863 .impl = { 2864 .min_access_size = 4, 2865 .max_access_size = 4, 2866 }, 2867 .valid = { 2868 .min_access_size = 4, 2869 .max_access_size = 4, 2870 }, 2871 }; 2872 2873 VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) 2874 { 2875 uintptr_t key = (uintptr_t)bus; 2876 VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key); 2877 VTDAddressSpace *vtd_dev_as; 2878 char name[128]; 2879 2880 if (!vtd_bus) { 2881 uintptr_t *new_key = g_malloc(sizeof(*new_key)); 2882 *new_key = (uintptr_t)bus; 2883 /* No corresponding free() */ 2884 vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \ 2885 PCI_DEVFN_MAX); 2886 vtd_bus->bus = bus; 2887 g_hash_table_insert(s->vtd_as_by_busptr, new_key, vtd_bus); 2888 } 2889 2890 vtd_dev_as = vtd_bus->dev_as[devfn]; 2891 2892 if (!vtd_dev_as) { 2893 snprintf(name, sizeof(name), "intel_iommu_devfn_%d", devfn); 2894 vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace)); 2895 2896 vtd_dev_as->bus = bus; 2897 vtd_dev_as->devfn = (uint8_t)devfn; 2898 vtd_dev_as->iommu_state = s; 2899 vtd_dev_as->context_cache_entry.context_cache_gen = 0; 2900 vtd_dev_as->iova_tree = iova_tree_new(); 2901 2902 /* 2903 * Memory region relationships looks like (Address range shows 2904 * only lower 32 bits to make it short in length...): 2905 * 2906 * |-----------------+-------------------+----------| 2907 * | Name | Address range | Priority | 2908 * |-----------------+-------------------+----------+ 2909 * | vtd_root | 00000000-ffffffff | 0 | 2910 * | intel_iommu | 00000000-ffffffff | 1 | 2911 * | vtd_sys_alias | 00000000-ffffffff | 1 | 2912 * | intel_iommu_ir | fee00000-feefffff | 64 | 2913 * |-----------------+-------------------+----------| 2914 * 2915 * We enable/disable DMAR by switching enablement for 2916 * vtd_sys_alias and intel_iommu regions. IR region is always 2917 * enabled. 2918 */ 2919 memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu), 2920 TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s), 2921 "intel_iommu_dmar", 2922 UINT64_MAX); 2923 memory_region_init_alias(&vtd_dev_as->sys_alias, OBJECT(s), 2924 "vtd_sys_alias", get_system_memory(), 2925 0, memory_region_size(get_system_memory())); 2926 memory_region_init_io(&vtd_dev_as->iommu_ir, OBJECT(s), 2927 &vtd_mem_ir_ops, s, "intel_iommu_ir", 2928 VTD_INTERRUPT_ADDR_SIZE); 2929 memory_region_init(&vtd_dev_as->root, OBJECT(s), 2930 "vtd_root", UINT64_MAX); 2931 memory_region_add_subregion_overlap(&vtd_dev_as->root, 2932 VTD_INTERRUPT_ADDR_FIRST, 2933 &vtd_dev_as->iommu_ir, 64); 2934 address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, name); 2935 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 2936 &vtd_dev_as->sys_alias, 1); 2937 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 2938 MEMORY_REGION(&vtd_dev_as->iommu), 2939 1); 2940 vtd_switch_address_space(vtd_dev_as); 2941 } 2942 return vtd_dev_as; 2943 } 2944 2945 /* Unmap the whole range in the notifier's scope. */ 2946 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n) 2947 { 2948 IOMMUTLBEntry entry; 2949 hwaddr size; 2950 hwaddr start = n->start; 2951 hwaddr end = n->end; 2952 IntelIOMMUState *s = as->iommu_state; 2953 DMAMap map; 2954 2955 /* 2956 * Note: all the codes in this function has a assumption that IOVA 2957 * bits are no more than VTD_MGAW bits (which is restricted by 2958 * VT-d spec), otherwise we need to consider overflow of 64 bits. 2959 */ 2960 2961 if (end > VTD_ADDRESS_SIZE(s->aw_bits)) { 2962 /* 2963 * Don't need to unmap regions that is bigger than the whole 2964 * VT-d supported address space size 2965 */ 2966 end = VTD_ADDRESS_SIZE(s->aw_bits); 2967 } 2968 2969 assert(start <= end); 2970 size = end - start; 2971 2972 if (ctpop64(size) != 1) { 2973 /* 2974 * This size cannot format a correct mask. Let's enlarge it to 2975 * suite the minimum available mask. 2976 */ 2977 int n = 64 - clz64(size); 2978 if (n > s->aw_bits) { 2979 /* should not happen, but in case it happens, limit it */ 2980 n = s->aw_bits; 2981 } 2982 size = 1ULL << n; 2983 } 2984 2985 entry.target_as = &address_space_memory; 2986 /* Adjust iova for the size */ 2987 entry.iova = n->start & ~(size - 1); 2988 /* This field is meaningless for unmap */ 2989 entry.translated_addr = 0; 2990 entry.perm = IOMMU_NONE; 2991 entry.addr_mask = size - 1; 2992 2993 trace_vtd_as_unmap_whole(pci_bus_num(as->bus), 2994 VTD_PCI_SLOT(as->devfn), 2995 VTD_PCI_FUNC(as->devfn), 2996 entry.iova, size); 2997 2998 map.iova = entry.iova; 2999 map.size = entry.addr_mask; 3000 iova_tree_remove(as->iova_tree, &map); 3001 3002 memory_region_notify_one(n, &entry); 3003 } 3004 3005 static void vtd_address_space_unmap_all(IntelIOMMUState *s) 3006 { 3007 VTDAddressSpace *vtd_as; 3008 IOMMUNotifier *n; 3009 3010 QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { 3011 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 3012 vtd_address_space_unmap(vtd_as, n); 3013 } 3014 } 3015 } 3016 3017 static int vtd_replay_hook(IOMMUTLBEntry *entry, void *private) 3018 { 3019 memory_region_notify_one((IOMMUNotifier *)private, entry); 3020 return 0; 3021 } 3022 3023 static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) 3024 { 3025 VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu); 3026 IntelIOMMUState *s = vtd_as->iommu_state; 3027 uint8_t bus_n = pci_bus_num(vtd_as->bus); 3028 VTDContextEntry ce; 3029 3030 /* 3031 * The replay can be triggered by either a invalidation or a newly 3032 * created entry. No matter what, we release existing mappings 3033 * (it means flushing caches for UNMAP-only registers). 3034 */ 3035 vtd_address_space_unmap(vtd_as, n); 3036 3037 if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 3038 trace_vtd_replay_ce_valid(bus_n, PCI_SLOT(vtd_as->devfn), 3039 PCI_FUNC(vtd_as->devfn), 3040 VTD_CONTEXT_ENTRY_DID(ce.hi), 3041 ce.hi, ce.lo); 3042 if (vtd_as_has_map_notifier(vtd_as)) { 3043 /* This is required only for MAP typed notifiers */ 3044 vtd_page_walk_info info = { 3045 .hook_fn = vtd_replay_hook, 3046 .private = (void *)n, 3047 .notify_unmap = false, 3048 .aw = s->aw_bits, 3049 .as = vtd_as, 3050 .domain_id = VTD_CONTEXT_ENTRY_DID(ce.hi), 3051 }; 3052 3053 vtd_page_walk(&ce, 0, ~0ULL, &info); 3054 } 3055 } else { 3056 trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn), 3057 PCI_FUNC(vtd_as->devfn)); 3058 } 3059 3060 return; 3061 } 3062 3063 /* Do the initialization. It will also be called when reset, so pay 3064 * attention when adding new initialization stuff. 3065 */ 3066 static void vtd_init(IntelIOMMUState *s) 3067 { 3068 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3069 3070 memset(s->csr, 0, DMAR_REG_SIZE); 3071 memset(s->wmask, 0, DMAR_REG_SIZE); 3072 memset(s->w1cmask, 0, DMAR_REG_SIZE); 3073 memset(s->womask, 0, DMAR_REG_SIZE); 3074 3075 s->root = 0; 3076 s->root_extended = false; 3077 s->dmar_enabled = false; 3078 s->iq_head = 0; 3079 s->iq_tail = 0; 3080 s->iq = 0; 3081 s->iq_size = 0; 3082 s->qi_enabled = false; 3083 s->iq_last_desc_type = VTD_INV_DESC_NONE; 3084 s->next_frcd_reg = 0; 3085 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | 3086 VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS | 3087 VTD_CAP_SAGAW_39bit | VTD_CAP_MGAW(s->aw_bits); 3088 if (s->aw_bits == VTD_HOST_AW_48BIT) { 3089 s->cap |= VTD_CAP_SAGAW_48bit; 3090 } 3091 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 3092 3093 /* 3094 * Rsvd field masks for spte 3095 */ 3096 vtd_paging_entry_rsvd_field[0] = ~0ULL; 3097 vtd_paging_entry_rsvd_field[1] = VTD_SPTE_PAGE_L1_RSVD_MASK(s->aw_bits); 3098 vtd_paging_entry_rsvd_field[2] = VTD_SPTE_PAGE_L2_RSVD_MASK(s->aw_bits); 3099 vtd_paging_entry_rsvd_field[3] = VTD_SPTE_PAGE_L3_RSVD_MASK(s->aw_bits); 3100 vtd_paging_entry_rsvd_field[4] = VTD_SPTE_PAGE_L4_RSVD_MASK(s->aw_bits); 3101 vtd_paging_entry_rsvd_field[5] = VTD_SPTE_LPAGE_L1_RSVD_MASK(s->aw_bits); 3102 vtd_paging_entry_rsvd_field[6] = VTD_SPTE_LPAGE_L2_RSVD_MASK(s->aw_bits); 3103 vtd_paging_entry_rsvd_field[7] = VTD_SPTE_LPAGE_L3_RSVD_MASK(s->aw_bits); 3104 vtd_paging_entry_rsvd_field[8] = VTD_SPTE_LPAGE_L4_RSVD_MASK(s->aw_bits); 3105 3106 if (x86_iommu->intr_supported) { 3107 s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV; 3108 if (s->intr_eim == ON_OFF_AUTO_ON) { 3109 s->ecap |= VTD_ECAP_EIM; 3110 } 3111 assert(s->intr_eim != ON_OFF_AUTO_AUTO); 3112 } 3113 3114 if (x86_iommu->dt_supported) { 3115 s->ecap |= VTD_ECAP_DT; 3116 } 3117 3118 if (x86_iommu->pt_supported) { 3119 s->ecap |= VTD_ECAP_PT; 3120 } 3121 3122 if (s->caching_mode) { 3123 s->cap |= VTD_CAP_CM; 3124 } 3125 3126 vtd_iommu_lock(s); 3127 vtd_reset_context_cache_locked(s); 3128 vtd_reset_iotlb_locked(s); 3129 vtd_iommu_unlock(s); 3130 3131 /* Define registers with default values and bit semantics */ 3132 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 3133 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 3134 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 3135 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 3136 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 3137 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 3138 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0); 3139 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 3140 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 3141 3142 /* Advanced Fault Logging not supported */ 3143 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 3144 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 3145 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 3146 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 3147 3148 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 3149 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 3150 */ 3151 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 3152 3153 /* Treated as RO for implementations that PLMR and PHMR fields reported 3154 * as Clear in the CAP_REG. 3155 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 3156 */ 3157 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 3158 3159 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 3160 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 3161 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0); 3162 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 3163 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 3164 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 3165 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 3166 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 3167 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 3168 3169 /* IOTLB registers */ 3170 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 3171 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 3172 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 3173 3174 /* Fault Recording Registers, 128-bit */ 3175 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 3176 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 3177 3178 /* 3179 * Interrupt remapping registers. 3180 */ 3181 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0); 3182 } 3183 3184 /* Should not reset address_spaces when reset because devices will still use 3185 * the address space they got at first (won't ask the bus again). 3186 */ 3187 static void vtd_reset(DeviceState *dev) 3188 { 3189 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 3190 3191 vtd_init(s); 3192 3193 /* 3194 * When device reset, throw away all mappings and external caches 3195 */ 3196 vtd_address_space_unmap_all(s); 3197 } 3198 3199 static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) 3200 { 3201 IntelIOMMUState *s = opaque; 3202 VTDAddressSpace *vtd_as; 3203 3204 assert(0 <= devfn && devfn < PCI_DEVFN_MAX); 3205 3206 vtd_as = vtd_find_add_as(s, bus, devfn); 3207 return &vtd_as->as; 3208 } 3209 3210 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) 3211 { 3212 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3213 3214 /* Currently Intel IOMMU IR only support "kernel-irqchip={off|split}" */ 3215 if (x86_iommu->intr_supported && kvm_irqchip_in_kernel() && 3216 !kvm_irqchip_is_split()) { 3217 error_setg(errp, "Intel Interrupt Remapping cannot work with " 3218 "kernel-irqchip=on, please use 'split|off'."); 3219 return false; 3220 } 3221 if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu->intr_supported) { 3222 error_setg(errp, "eim=on cannot be selected without intremap=on"); 3223 return false; 3224 } 3225 3226 if (s->intr_eim == ON_OFF_AUTO_AUTO) { 3227 s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim) 3228 && x86_iommu->intr_supported ? 3229 ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 3230 } 3231 if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) { 3232 if (!kvm_irqchip_in_kernel()) { 3233 error_setg(errp, "eim=on requires accel=kvm,kernel-irqchip=split"); 3234 return false; 3235 } 3236 if (!kvm_enable_x2apic()) { 3237 error_setg(errp, "eim=on requires support on the KVM side" 3238 "(X2APIC_API, first shipped in v4.7)"); 3239 return false; 3240 } 3241 } 3242 3243 /* Currently only address widths supported are 39 and 48 bits */ 3244 if ((s->aw_bits != VTD_HOST_AW_39BIT) && 3245 (s->aw_bits != VTD_HOST_AW_48BIT)) { 3246 error_setg(errp, "Supported values for x-aw-bits are: %d, %d", 3247 VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT); 3248 return false; 3249 } 3250 3251 return true; 3252 } 3253 3254 static void vtd_realize(DeviceState *dev, Error **errp) 3255 { 3256 MachineState *ms = MACHINE(qdev_get_machine()); 3257 PCMachineState *pcms = PC_MACHINE(ms); 3258 PCIBus *bus = pcms->bus; 3259 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 3260 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev); 3261 3262 x86_iommu->type = TYPE_INTEL; 3263 3264 if (!vtd_decide_config(s, errp)) { 3265 return; 3266 } 3267 3268 QLIST_INIT(&s->vtd_as_with_notifiers); 3269 qemu_mutex_init(&s->iommu_lock); 3270 memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num)); 3271 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 3272 "intel_iommu", DMAR_REG_SIZE); 3273 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem); 3274 /* No corresponding destroy */ 3275 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 3276 g_free, g_free); 3277 s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 3278 g_free, g_free); 3279 vtd_init(s); 3280 sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR); 3281 pci_setup_iommu(bus, vtd_host_dma_iommu, dev); 3282 /* Pseudo address space under root PCI bus. */ 3283 pcms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC); 3284 } 3285 3286 static void vtd_class_init(ObjectClass *klass, void *data) 3287 { 3288 DeviceClass *dc = DEVICE_CLASS(klass); 3289 X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass); 3290 3291 dc->reset = vtd_reset; 3292 dc->vmsd = &vtd_vmstate; 3293 dc->props = vtd_properties; 3294 dc->hotpluggable = false; 3295 x86_class->realize = vtd_realize; 3296 x86_class->int_remap = vtd_int_remap; 3297 /* Supported by the pc-q35-* machine types */ 3298 dc->user_creatable = true; 3299 } 3300 3301 static const TypeInfo vtd_info = { 3302 .name = TYPE_INTEL_IOMMU_DEVICE, 3303 .parent = TYPE_X86_IOMMU_DEVICE, 3304 .instance_size = sizeof(IntelIOMMUState), 3305 .class_init = vtd_class_init, 3306 }; 3307 3308 static void vtd_iommu_memory_region_class_init(ObjectClass *klass, 3309 void *data) 3310 { 3311 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 3312 3313 imrc->translate = vtd_iommu_translate; 3314 imrc->notify_flag_changed = vtd_iommu_notify_flag_changed; 3315 imrc->replay = vtd_iommu_replay; 3316 } 3317 3318 static const TypeInfo vtd_iommu_memory_region_info = { 3319 .parent = TYPE_IOMMU_MEMORY_REGION, 3320 .name = TYPE_INTEL_IOMMU_MEMORY_REGION, 3321 .class_init = vtd_iommu_memory_region_class_init, 3322 }; 3323 3324 static void vtd_register_types(void) 3325 { 3326 type_register_static(&vtd_info); 3327 type_register_static(&vtd_iommu_memory_region_info); 3328 } 3329 3330 type_init(vtd_register_types) 3331