1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "qemu/osdep.h" 23 #include "qemu/error-report.h" 24 #include "qapi/error.h" 25 #include "hw/sysbus.h" 26 #include "exec/address-spaces.h" 27 #include "intel_iommu_internal.h" 28 #include "hw/pci/pci.h" 29 #include "hw/pci/pci_bus.h" 30 #include "hw/i386/pc.h" 31 #include "hw/i386/apic-msidef.h" 32 #include "hw/boards.h" 33 #include "hw/i386/x86-iommu.h" 34 #include "hw/pci-host/q35.h" 35 #include "sysemu/kvm.h" 36 #include "hw/i386/apic_internal.h" 37 #include "kvm_i386.h" 38 #include "trace.h" 39 40 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 41 uint64_t wmask, uint64_t w1cmask) 42 { 43 stq_le_p(&s->csr[addr], val); 44 stq_le_p(&s->wmask[addr], wmask); 45 stq_le_p(&s->w1cmask[addr], w1cmask); 46 } 47 48 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 49 { 50 stq_le_p(&s->womask[addr], mask); 51 } 52 53 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 54 uint32_t wmask, uint32_t w1cmask) 55 { 56 stl_le_p(&s->csr[addr], val); 57 stl_le_p(&s->wmask[addr], wmask); 58 stl_le_p(&s->w1cmask[addr], w1cmask); 59 } 60 61 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 62 { 63 stl_le_p(&s->womask[addr], mask); 64 } 65 66 /* "External" get/set operations */ 67 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 68 { 69 uint64_t oldval = ldq_le_p(&s->csr[addr]); 70 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 71 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 72 stq_le_p(&s->csr[addr], 73 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 74 } 75 76 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 77 { 78 uint32_t oldval = ldl_le_p(&s->csr[addr]); 79 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 80 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 81 stl_le_p(&s->csr[addr], 82 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 83 } 84 85 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 86 { 87 uint64_t val = ldq_le_p(&s->csr[addr]); 88 uint64_t womask = ldq_le_p(&s->womask[addr]); 89 return val & ~womask; 90 } 91 92 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 93 { 94 uint32_t val = ldl_le_p(&s->csr[addr]); 95 uint32_t womask = ldl_le_p(&s->womask[addr]); 96 return val & ~womask; 97 } 98 99 /* "Internal" get/set operations */ 100 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 101 { 102 return ldq_le_p(&s->csr[addr]); 103 } 104 105 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 106 { 107 return ldl_le_p(&s->csr[addr]); 108 } 109 110 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 111 { 112 stq_le_p(&s->csr[addr], val); 113 } 114 115 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 116 uint32_t clear, uint32_t mask) 117 { 118 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 119 stl_le_p(&s->csr[addr], new_val); 120 return new_val; 121 } 122 123 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 124 uint64_t clear, uint64_t mask) 125 { 126 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 127 stq_le_p(&s->csr[addr], new_val); 128 return new_val; 129 } 130 131 /* GHashTable functions */ 132 static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2) 133 { 134 return *((const uint64_t *)v1) == *((const uint64_t *)v2); 135 } 136 137 static guint vtd_uint64_hash(gconstpointer v) 138 { 139 return (guint)*(const uint64_t *)v; 140 } 141 142 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 143 gpointer user_data) 144 { 145 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 146 uint16_t domain_id = *(uint16_t *)user_data; 147 return entry->domain_id == domain_id; 148 } 149 150 /* The shift of an addr for a certain level of paging structure */ 151 static inline uint32_t vtd_slpt_level_shift(uint32_t level) 152 { 153 assert(level != 0); 154 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 155 } 156 157 static inline uint64_t vtd_slpt_level_page_mask(uint32_t level) 158 { 159 return ~((1ULL << vtd_slpt_level_shift(level)) - 1); 160 } 161 162 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 163 gpointer user_data) 164 { 165 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 166 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 167 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask; 168 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K; 169 return (entry->domain_id == info->domain_id) && 170 (((entry->gfn & info->mask) == gfn) || 171 (entry->gfn == gfn_tlb)); 172 } 173 174 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 175 * IntelIOMMUState to 1. 176 */ 177 static void vtd_reset_context_cache(IntelIOMMUState *s) 178 { 179 VTDAddressSpace *vtd_as; 180 VTDBus *vtd_bus; 181 GHashTableIter bus_it; 182 uint32_t devfn_it; 183 184 trace_vtd_context_cache_reset(); 185 186 g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr); 187 188 while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) { 189 for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) { 190 vtd_as = vtd_bus->dev_as[devfn_it]; 191 if (!vtd_as) { 192 continue; 193 } 194 vtd_as->context_cache_entry.context_cache_gen = 0; 195 } 196 } 197 s->context_cache_gen = 1; 198 } 199 200 static void vtd_reset_iotlb(IntelIOMMUState *s) 201 { 202 assert(s->iotlb); 203 g_hash_table_remove_all(s->iotlb); 204 } 205 206 static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint16_t source_id, 207 uint32_t level) 208 { 209 return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) | 210 ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT); 211 } 212 213 static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) 214 { 215 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K; 216 } 217 218 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 219 hwaddr addr) 220 { 221 VTDIOTLBEntry *entry; 222 uint64_t key; 223 int level; 224 225 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) { 226 key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level), 227 source_id, level); 228 entry = g_hash_table_lookup(s->iotlb, &key); 229 if (entry) { 230 goto out; 231 } 232 } 233 234 out: 235 return entry; 236 } 237 238 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 239 uint16_t domain_id, hwaddr addr, uint64_t slpte, 240 uint8_t access_flags, uint32_t level) 241 { 242 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 243 uint64_t *key = g_malloc(sizeof(*key)); 244 uint64_t gfn = vtd_get_iotlb_gfn(addr, level); 245 246 trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id); 247 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 248 trace_vtd_iotlb_reset("iotlb exceeds size limit"); 249 vtd_reset_iotlb(s); 250 } 251 252 entry->gfn = gfn; 253 entry->domain_id = domain_id; 254 entry->slpte = slpte; 255 entry->access_flags = access_flags; 256 entry->mask = vtd_slpt_level_page_mask(level); 257 *key = vtd_get_iotlb_key(gfn, source_id, level); 258 g_hash_table_replace(s->iotlb, key, entry); 259 } 260 261 /* Given the reg addr of both the message data and address, generate an 262 * interrupt via MSI. 263 */ 264 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 265 hwaddr mesg_data_reg) 266 { 267 MSIMessage msi; 268 269 assert(mesg_data_reg < DMAR_REG_SIZE); 270 assert(mesg_addr_reg < DMAR_REG_SIZE); 271 272 msi.address = vtd_get_long_raw(s, mesg_addr_reg); 273 msi.data = vtd_get_long_raw(s, mesg_data_reg); 274 275 trace_vtd_irq_generate(msi.address, msi.data); 276 277 apic_get_class()->send_msi(&msi); 278 } 279 280 /* Generate a fault event to software via MSI if conditions are met. 281 * Notice that the value of FSTS_REG being passed to it should be the one 282 * before any update. 283 */ 284 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 285 { 286 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 287 pre_fsts & VTD_FSTS_IQE) { 288 trace_vtd_err("There are previous interrupt conditions " 289 "to be serviced by software, fault event " 290 "is not generated."); 291 return; 292 } 293 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 294 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 295 trace_vtd_err("Interrupt Mask set, irq is not generated."); 296 } else { 297 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 298 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 299 } 300 } 301 302 /* Check if the Fault (F) field of the Fault Recording Register referenced by 303 * @index is Set. 304 */ 305 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 306 { 307 /* Each reg is 128-bit */ 308 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 309 addr += 8; /* Access the high 64-bit half */ 310 311 assert(index < DMAR_FRCD_REG_NR); 312 313 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 314 } 315 316 /* Update the PPF field of Fault Status Register. 317 * Should be called whenever change the F field of any fault recording 318 * registers. 319 */ 320 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 321 { 322 uint32_t i; 323 uint32_t ppf_mask = 0; 324 325 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 326 if (vtd_is_frcd_set(s, i)) { 327 ppf_mask = VTD_FSTS_PPF; 328 break; 329 } 330 } 331 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 332 trace_vtd_fsts_ppf(!!ppf_mask); 333 } 334 335 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 336 { 337 /* Each reg is 128-bit */ 338 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 339 addr += 8; /* Access the high 64-bit half */ 340 341 assert(index < DMAR_FRCD_REG_NR); 342 343 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 344 vtd_update_fsts_ppf(s); 345 } 346 347 /* Must not update F field now, should be done later */ 348 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 349 uint16_t source_id, hwaddr addr, 350 VTDFaultReason fault, bool is_write) 351 { 352 uint64_t hi = 0, lo; 353 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 354 355 assert(index < DMAR_FRCD_REG_NR); 356 357 lo = VTD_FRCD_FI(addr); 358 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 359 if (!is_write) { 360 hi |= VTD_FRCD_T; 361 } 362 vtd_set_quad_raw(s, frcd_reg_addr, lo); 363 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 364 365 trace_vtd_frr_new(index, hi, lo); 366 } 367 368 /* Try to collapse multiple pending faults from the same requester */ 369 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 370 { 371 uint32_t i; 372 uint64_t frcd_reg; 373 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 374 375 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 376 frcd_reg = vtd_get_quad_raw(s, addr); 377 if ((frcd_reg & VTD_FRCD_F) && 378 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 379 return true; 380 } 381 addr += 16; /* 128-bit for each */ 382 } 383 return false; 384 } 385 386 /* Log and report an DMAR (address translation) fault to software */ 387 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 388 hwaddr addr, VTDFaultReason fault, 389 bool is_write) 390 { 391 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 392 393 assert(fault < VTD_FR_MAX); 394 395 if (fault == VTD_FR_RESERVED_ERR) { 396 /* This is not a normal fault reason case. Drop it. */ 397 return; 398 } 399 400 trace_vtd_dmar_fault(source_id, fault, addr, is_write); 401 402 if (fsts_reg & VTD_FSTS_PFO) { 403 trace_vtd_err("New fault is not recorded due to " 404 "Primary Fault Overflow."); 405 return; 406 } 407 408 if (vtd_try_collapse_fault(s, source_id)) { 409 trace_vtd_err("New fault is not recorded due to " 410 "compression of faults."); 411 return; 412 } 413 414 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 415 trace_vtd_err("Next Fault Recording Reg is used, " 416 "new fault is not recorded, set PFO field."); 417 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 418 return; 419 } 420 421 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write); 422 423 if (fsts_reg & VTD_FSTS_PPF) { 424 trace_vtd_err("There are pending faults already, " 425 "fault event is not generated."); 426 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 427 s->next_frcd_reg++; 428 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 429 s->next_frcd_reg = 0; 430 } 431 } else { 432 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 433 VTD_FSTS_FRI(s->next_frcd_reg)); 434 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 435 s->next_frcd_reg++; 436 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 437 s->next_frcd_reg = 0; 438 } 439 /* This case actually cause the PPF to be Set. 440 * So generate fault event (interrupt). 441 */ 442 vtd_generate_fault_event(s, fsts_reg); 443 } 444 } 445 446 /* Handle Invalidation Queue Errors of queued invalidation interface error 447 * conditions. 448 */ 449 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 450 { 451 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 452 453 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 454 vtd_generate_fault_event(s, fsts_reg); 455 } 456 457 /* Set the IWC field and try to generate an invalidation completion interrupt */ 458 static void vtd_generate_completion_event(IntelIOMMUState *s) 459 { 460 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 461 trace_vtd_inv_desc_wait_irq("One pending, skip current"); 462 return; 463 } 464 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 465 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 466 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 467 trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, " 468 "new event not generated"); 469 return; 470 } else { 471 /* Generate the interrupt event */ 472 trace_vtd_inv_desc_wait_irq("Generating complete event"); 473 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 474 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 475 } 476 } 477 478 static inline bool vtd_root_entry_present(VTDRootEntry *root) 479 { 480 return root->val & VTD_ROOT_ENTRY_P; 481 } 482 483 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 484 VTDRootEntry *re) 485 { 486 dma_addr_t addr; 487 488 addr = s->root + index * sizeof(*re); 489 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) { 490 trace_vtd_re_invalid(re->rsvd, re->val); 491 re->val = 0; 492 return -VTD_FR_ROOT_TABLE_INV; 493 } 494 re->val = le64_to_cpu(re->val); 495 return 0; 496 } 497 498 static inline bool vtd_ce_present(VTDContextEntry *context) 499 { 500 return context->lo & VTD_CONTEXT_ENTRY_P; 501 } 502 503 static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index, 504 VTDContextEntry *ce) 505 { 506 dma_addr_t addr; 507 508 /* we have checked that root entry is present */ 509 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce); 510 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) { 511 trace_vtd_re_invalid(root->rsvd, root->val); 512 return -VTD_FR_CONTEXT_TABLE_INV; 513 } 514 ce->lo = le64_to_cpu(ce->lo); 515 ce->hi = le64_to_cpu(ce->hi); 516 return 0; 517 } 518 519 static inline dma_addr_t vtd_ce_get_slpt_base(VTDContextEntry *ce) 520 { 521 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 522 } 523 524 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte, uint8_t aw) 525 { 526 return slpte & VTD_SL_PT_BASE_ADDR_MASK(aw); 527 } 528 529 /* Whether the pte indicates the address of the page frame */ 530 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 531 { 532 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 533 } 534 535 /* Get the content of a spte located in @base_addr[@index] */ 536 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 537 { 538 uint64_t slpte; 539 540 assert(index < VTD_SL_PT_ENTRY_NR); 541 542 if (dma_memory_read(&address_space_memory, 543 base_addr + index * sizeof(slpte), &slpte, 544 sizeof(slpte))) { 545 slpte = (uint64_t)-1; 546 return slpte; 547 } 548 slpte = le64_to_cpu(slpte); 549 return slpte; 550 } 551 552 /* Given an iova and the level of paging structure, return the offset 553 * of current level. 554 */ 555 static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level) 556 { 557 return (iova >> vtd_slpt_level_shift(level)) & 558 ((1ULL << VTD_SL_LEVEL_BITS) - 1); 559 } 560 561 /* Check Capability Register to see if the @level of page-table is supported */ 562 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 563 { 564 return VTD_CAP_SAGAW_MASK & s->cap & 565 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 566 } 567 568 /* Get the page-table level that hardware should use for the second-level 569 * page-table walk from the Address Width field of context-entry. 570 */ 571 static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce) 572 { 573 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 574 } 575 576 static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce) 577 { 578 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 579 } 580 581 static inline uint32_t vtd_ce_get_type(VTDContextEntry *ce) 582 { 583 return ce->lo & VTD_CONTEXT_ENTRY_TT; 584 } 585 586 /* Return true if check passed, otherwise false */ 587 static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu, 588 VTDContextEntry *ce) 589 { 590 switch (vtd_ce_get_type(ce)) { 591 case VTD_CONTEXT_TT_MULTI_LEVEL: 592 /* Always supported */ 593 break; 594 case VTD_CONTEXT_TT_DEV_IOTLB: 595 if (!x86_iommu->dt_supported) { 596 return false; 597 } 598 break; 599 case VTD_CONTEXT_TT_PASS_THROUGH: 600 if (!x86_iommu->pt_supported) { 601 return false; 602 } 603 break; 604 default: 605 /* Unknwon type */ 606 return false; 607 } 608 return true; 609 } 610 611 static inline uint64_t vtd_iova_limit(VTDContextEntry *ce, uint8_t aw) 612 { 613 uint32_t ce_agaw = vtd_ce_get_agaw(ce); 614 return 1ULL << MIN(ce_agaw, aw); 615 } 616 617 /* Return true if IOVA passes range check, otherwise false. */ 618 static inline bool vtd_iova_range_check(uint64_t iova, VTDContextEntry *ce, 619 uint8_t aw) 620 { 621 /* 622 * Check if @iova is above 2^X-1, where X is the minimum of MGAW 623 * in CAP_REG and AW in context-entry. 624 */ 625 return !(iova & ~(vtd_iova_limit(ce, aw) - 1)); 626 } 627 628 /* 629 * Rsvd field masks for spte: 630 * Index [1] to [4] 4k pages 631 * Index [5] to [8] large pages 632 */ 633 static uint64_t vtd_paging_entry_rsvd_field[9]; 634 635 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 636 { 637 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) { 638 /* Maybe large page */ 639 return slpte & vtd_paging_entry_rsvd_field[level + 4]; 640 } else { 641 return slpte & vtd_paging_entry_rsvd_field[level]; 642 } 643 } 644 645 /* Find the VTD address space associated with a given bus number */ 646 static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num) 647 { 648 VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num]; 649 if (!vtd_bus) { 650 /* 651 * Iterate over the registered buses to find the one which 652 * currently hold this bus number, and update the bus_num 653 * lookup table: 654 */ 655 GHashTableIter iter; 656 657 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); 658 while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) { 659 if (pci_bus_num(vtd_bus->bus) == bus_num) { 660 s->vtd_as_by_bus_num[bus_num] = vtd_bus; 661 return vtd_bus; 662 } 663 } 664 } 665 return vtd_bus; 666 } 667 668 /* Given the @iova, get relevant @slptep. @slpte_level will be the last level 669 * of the translation, can be used for deciding the size of large page. 670 */ 671 static int vtd_iova_to_slpte(VTDContextEntry *ce, uint64_t iova, bool is_write, 672 uint64_t *slptep, uint32_t *slpte_level, 673 bool *reads, bool *writes, uint8_t aw_bits) 674 { 675 dma_addr_t addr = vtd_ce_get_slpt_base(ce); 676 uint32_t level = vtd_ce_get_level(ce); 677 uint32_t offset; 678 uint64_t slpte; 679 uint64_t access_right_check; 680 681 if (!vtd_iova_range_check(iova, ce, aw_bits)) { 682 trace_vtd_err_dmar_iova_overflow(iova); 683 return -VTD_FR_ADDR_BEYOND_MGAW; 684 } 685 686 /* FIXME: what is the Atomics request here? */ 687 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 688 689 while (true) { 690 offset = vtd_iova_level_offset(iova, level); 691 slpte = vtd_get_slpte(addr, offset); 692 693 if (slpte == (uint64_t)-1) { 694 trace_vtd_err_dmar_slpte_read_error(iova, level); 695 if (level == vtd_ce_get_level(ce)) { 696 /* Invalid programming of context-entry */ 697 return -VTD_FR_CONTEXT_ENTRY_INV; 698 } else { 699 return -VTD_FR_PAGING_ENTRY_INV; 700 } 701 } 702 *reads = (*reads) && (slpte & VTD_SL_R); 703 *writes = (*writes) && (slpte & VTD_SL_W); 704 if (!(slpte & access_right_check)) { 705 trace_vtd_err_dmar_slpte_perm_error(iova, level, slpte, is_write); 706 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 707 } 708 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 709 trace_vtd_err_dmar_slpte_resv_error(iova, level, slpte); 710 return -VTD_FR_PAGING_ENTRY_RSVD; 711 } 712 713 if (vtd_is_last_slpte(slpte, level)) { 714 *slptep = slpte; 715 *slpte_level = level; 716 return 0; 717 } 718 addr = vtd_get_slpte_addr(slpte, aw_bits); 719 level--; 720 } 721 } 722 723 typedef int (*vtd_page_walk_hook)(IOMMUTLBEntry *entry, void *private); 724 725 /** 726 * vtd_page_walk_level - walk over specific level for IOVA range 727 * 728 * @addr: base GPA addr to start the walk 729 * @start: IOVA range start address 730 * @end: IOVA range end address (start <= addr < end) 731 * @hook_fn: hook func to be called when detected page 732 * @private: private data to be passed into hook func 733 * @read: whether parent level has read permission 734 * @write: whether parent level has write permission 735 * @notify_unmap: whether we should notify invalid entries 736 * @aw: maximum address width 737 */ 738 static int vtd_page_walk_level(dma_addr_t addr, uint64_t start, 739 uint64_t end, vtd_page_walk_hook hook_fn, 740 void *private, uint32_t level, bool read, 741 bool write, bool notify_unmap, uint8_t aw) 742 { 743 bool read_cur, write_cur, entry_valid; 744 uint32_t offset; 745 uint64_t slpte; 746 uint64_t subpage_size, subpage_mask; 747 IOMMUTLBEntry entry; 748 uint64_t iova = start; 749 uint64_t iova_next; 750 int ret = 0; 751 752 trace_vtd_page_walk_level(addr, level, start, end); 753 754 subpage_size = 1ULL << vtd_slpt_level_shift(level); 755 subpage_mask = vtd_slpt_level_page_mask(level); 756 757 while (iova < end) { 758 iova_next = (iova & subpage_mask) + subpage_size; 759 760 offset = vtd_iova_level_offset(iova, level); 761 slpte = vtd_get_slpte(addr, offset); 762 763 if (slpte == (uint64_t)-1) { 764 trace_vtd_page_walk_skip_read(iova, iova_next); 765 goto next; 766 } 767 768 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 769 trace_vtd_page_walk_skip_reserve(iova, iova_next); 770 goto next; 771 } 772 773 /* Permissions are stacked with parents' */ 774 read_cur = read && (slpte & VTD_SL_R); 775 write_cur = write && (slpte & VTD_SL_W); 776 777 /* 778 * As long as we have either read/write permission, this is a 779 * valid entry. The rule works for both page entries and page 780 * table entries. 781 */ 782 entry_valid = read_cur | write_cur; 783 784 if (vtd_is_last_slpte(slpte, level)) { 785 entry.target_as = &address_space_memory; 786 entry.iova = iova & subpage_mask; 787 /* NOTE: this is only meaningful if entry_valid == true */ 788 entry.translated_addr = vtd_get_slpte_addr(slpte, aw); 789 entry.addr_mask = ~subpage_mask; 790 entry.perm = IOMMU_ACCESS_FLAG(read_cur, write_cur); 791 if (!entry_valid && !notify_unmap) { 792 trace_vtd_page_walk_skip_perm(iova, iova_next); 793 goto next; 794 } 795 trace_vtd_page_walk_one(level, entry.iova, entry.translated_addr, 796 entry.addr_mask, entry.perm); 797 if (hook_fn) { 798 ret = hook_fn(&entry, private); 799 if (ret < 0) { 800 return ret; 801 } 802 } 803 } else { 804 if (!entry_valid) { 805 trace_vtd_page_walk_skip_perm(iova, iova_next); 806 goto next; 807 } 808 ret = vtd_page_walk_level(vtd_get_slpte_addr(slpte, aw), iova, 809 MIN(iova_next, end), hook_fn, private, 810 level - 1, read_cur, write_cur, 811 notify_unmap, aw); 812 if (ret < 0) { 813 return ret; 814 } 815 } 816 817 next: 818 iova = iova_next; 819 } 820 821 return 0; 822 } 823 824 /** 825 * vtd_page_walk - walk specific IOVA range, and call the hook 826 * 827 * @ce: context entry to walk upon 828 * @start: IOVA address to start the walk 829 * @end: IOVA range end address (start <= addr < end) 830 * @hook_fn: the hook that to be called for each detected area 831 * @private: private data for the hook function 832 * @aw: maximum address width 833 */ 834 static int vtd_page_walk(VTDContextEntry *ce, uint64_t start, uint64_t end, 835 vtd_page_walk_hook hook_fn, void *private, 836 bool notify_unmap, uint8_t aw) 837 { 838 dma_addr_t addr = vtd_ce_get_slpt_base(ce); 839 uint32_t level = vtd_ce_get_level(ce); 840 841 if (!vtd_iova_range_check(start, ce, aw)) { 842 return -VTD_FR_ADDR_BEYOND_MGAW; 843 } 844 845 if (!vtd_iova_range_check(end, ce, aw)) { 846 /* Fix end so that it reaches the maximum */ 847 end = vtd_iova_limit(ce, aw); 848 } 849 850 return vtd_page_walk_level(addr, start, end, hook_fn, private, 851 level, true, true, notify_unmap, aw); 852 } 853 854 /* Map a device to its corresponding domain (context-entry) */ 855 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 856 uint8_t devfn, VTDContextEntry *ce) 857 { 858 VTDRootEntry re; 859 int ret_fr; 860 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 861 862 ret_fr = vtd_get_root_entry(s, bus_num, &re); 863 if (ret_fr) { 864 return ret_fr; 865 } 866 867 if (!vtd_root_entry_present(&re)) { 868 /* Not error - it's okay we don't have root entry. */ 869 trace_vtd_re_not_present(bus_num); 870 return -VTD_FR_ROOT_ENTRY_P; 871 } 872 873 if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD(s->aw_bits))) { 874 trace_vtd_re_invalid(re.rsvd, re.val); 875 return -VTD_FR_ROOT_ENTRY_RSVD; 876 } 877 878 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce); 879 if (ret_fr) { 880 return ret_fr; 881 } 882 883 if (!vtd_ce_present(ce)) { 884 /* Not error - it's okay we don't have context entry. */ 885 trace_vtd_ce_not_present(bus_num, devfn); 886 return -VTD_FR_CONTEXT_ENTRY_P; 887 } 888 889 if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) || 890 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO(s->aw_bits))) { 891 trace_vtd_ce_invalid(ce->hi, ce->lo); 892 return -VTD_FR_CONTEXT_ENTRY_RSVD; 893 } 894 895 /* Check if the programming of context-entry is valid */ 896 if (!vtd_is_level_supported(s, vtd_ce_get_level(ce))) { 897 trace_vtd_ce_invalid(ce->hi, ce->lo); 898 return -VTD_FR_CONTEXT_ENTRY_INV; 899 } 900 901 /* Do translation type check */ 902 if (!vtd_ce_type_check(x86_iommu, ce)) { 903 trace_vtd_ce_invalid(ce->hi, ce->lo); 904 return -VTD_FR_CONTEXT_ENTRY_INV; 905 } 906 907 return 0; 908 } 909 910 /* 911 * Fetch translation type for specific device. Returns <0 if error 912 * happens, otherwise return the shifted type to check against 913 * VTD_CONTEXT_TT_*. 914 */ 915 static int vtd_dev_get_trans_type(VTDAddressSpace *as) 916 { 917 IntelIOMMUState *s; 918 VTDContextEntry ce; 919 int ret; 920 921 s = as->iommu_state; 922 923 ret = vtd_dev_to_context_entry(s, pci_bus_num(as->bus), 924 as->devfn, &ce); 925 if (ret) { 926 return ret; 927 } 928 929 return vtd_ce_get_type(&ce); 930 } 931 932 static bool vtd_dev_pt_enabled(VTDAddressSpace *as) 933 { 934 int ret; 935 936 assert(as); 937 938 ret = vtd_dev_get_trans_type(as); 939 if (ret < 0) { 940 /* 941 * Possibly failed to parse the context entry for some reason 942 * (e.g., during init, or any guest configuration errors on 943 * context entries). We should assume PT not enabled for 944 * safety. 945 */ 946 return false; 947 } 948 949 return ret == VTD_CONTEXT_TT_PASS_THROUGH; 950 } 951 952 /* Return whether the device is using IOMMU translation. */ 953 static bool vtd_switch_address_space(VTDAddressSpace *as) 954 { 955 bool use_iommu; 956 /* Whether we need to take the BQL on our own */ 957 bool take_bql = !qemu_mutex_iothread_locked(); 958 959 assert(as); 960 961 use_iommu = as->iommu_state->dmar_enabled & !vtd_dev_pt_enabled(as); 962 963 trace_vtd_switch_address_space(pci_bus_num(as->bus), 964 VTD_PCI_SLOT(as->devfn), 965 VTD_PCI_FUNC(as->devfn), 966 use_iommu); 967 968 /* 969 * It's possible that we reach here without BQL, e.g., when called 970 * from vtd_pt_enable_fast_path(). However the memory APIs need 971 * it. We'd better make sure we have had it already, or, take it. 972 */ 973 if (take_bql) { 974 qemu_mutex_lock_iothread(); 975 } 976 977 /* Turn off first then on the other */ 978 if (use_iommu) { 979 memory_region_set_enabled(&as->sys_alias, false); 980 memory_region_set_enabled(MEMORY_REGION(&as->iommu), true); 981 } else { 982 memory_region_set_enabled(MEMORY_REGION(&as->iommu), false); 983 memory_region_set_enabled(&as->sys_alias, true); 984 } 985 986 if (take_bql) { 987 qemu_mutex_unlock_iothread(); 988 } 989 990 return use_iommu; 991 } 992 993 static void vtd_switch_address_space_all(IntelIOMMUState *s) 994 { 995 GHashTableIter iter; 996 VTDBus *vtd_bus; 997 int i; 998 999 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); 1000 while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) { 1001 for (i = 0; i < PCI_DEVFN_MAX; i++) { 1002 if (!vtd_bus->dev_as[i]) { 1003 continue; 1004 } 1005 vtd_switch_address_space(vtd_bus->dev_as[i]); 1006 } 1007 } 1008 } 1009 1010 static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn) 1011 { 1012 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL); 1013 } 1014 1015 static const bool vtd_qualified_faults[] = { 1016 [VTD_FR_RESERVED] = false, 1017 [VTD_FR_ROOT_ENTRY_P] = false, 1018 [VTD_FR_CONTEXT_ENTRY_P] = true, 1019 [VTD_FR_CONTEXT_ENTRY_INV] = true, 1020 [VTD_FR_ADDR_BEYOND_MGAW] = true, 1021 [VTD_FR_WRITE] = true, 1022 [VTD_FR_READ] = true, 1023 [VTD_FR_PAGING_ENTRY_INV] = true, 1024 [VTD_FR_ROOT_TABLE_INV] = false, 1025 [VTD_FR_CONTEXT_TABLE_INV] = false, 1026 [VTD_FR_ROOT_ENTRY_RSVD] = false, 1027 [VTD_FR_PAGING_ENTRY_RSVD] = true, 1028 [VTD_FR_CONTEXT_ENTRY_TT] = true, 1029 [VTD_FR_RESERVED_ERR] = false, 1030 [VTD_FR_MAX] = false, 1031 }; 1032 1033 /* To see if a fault condition is "qualified", which is reported to software 1034 * only if the FPD field in the context-entry used to process the faulting 1035 * request is 0. 1036 */ 1037 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 1038 { 1039 return vtd_qualified_faults[fault]; 1040 } 1041 1042 static inline bool vtd_is_interrupt_addr(hwaddr addr) 1043 { 1044 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 1045 } 1046 1047 static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id) 1048 { 1049 VTDBus *vtd_bus; 1050 VTDAddressSpace *vtd_as; 1051 bool success = false; 1052 1053 vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id)); 1054 if (!vtd_bus) { 1055 goto out; 1056 } 1057 1058 vtd_as = vtd_bus->dev_as[VTD_SID_TO_DEVFN(source_id)]; 1059 if (!vtd_as) { 1060 goto out; 1061 } 1062 1063 if (vtd_switch_address_space(vtd_as) == false) { 1064 /* We switched off IOMMU region successfully. */ 1065 success = true; 1066 } 1067 1068 out: 1069 trace_vtd_pt_enable_fast_path(source_id, success); 1070 } 1071 1072 /* Map dev to context-entry then do a paging-structures walk to do a iommu 1073 * translation. 1074 * 1075 * Called from RCU critical section. 1076 * 1077 * @bus_num: The bus number 1078 * @devfn: The devfn, which is the combined of device and function number 1079 * @is_write: The access is a write operation 1080 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 1081 * 1082 * Returns true if translation is successful, otherwise false. 1083 */ 1084 static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, 1085 uint8_t devfn, hwaddr addr, bool is_write, 1086 IOMMUTLBEntry *entry) 1087 { 1088 IntelIOMMUState *s = vtd_as->iommu_state; 1089 VTDContextEntry ce; 1090 uint8_t bus_num = pci_bus_num(bus); 1091 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry; 1092 uint64_t slpte, page_mask; 1093 uint32_t level; 1094 uint16_t source_id = vtd_make_source_id(bus_num, devfn); 1095 int ret_fr; 1096 bool is_fpd_set = false; 1097 bool reads = true; 1098 bool writes = true; 1099 uint8_t access_flags; 1100 VTDIOTLBEntry *iotlb_entry; 1101 1102 /* 1103 * We have standalone memory region for interrupt addresses, we 1104 * should never receive translation requests in this region. 1105 */ 1106 assert(!vtd_is_interrupt_addr(addr)); 1107 1108 /* Try to fetch slpte form IOTLB */ 1109 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr); 1110 if (iotlb_entry) { 1111 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, 1112 iotlb_entry->domain_id); 1113 slpte = iotlb_entry->slpte; 1114 access_flags = iotlb_entry->access_flags; 1115 page_mask = iotlb_entry->mask; 1116 goto out; 1117 } 1118 1119 /* Try to fetch context-entry from cache first */ 1120 if (cc_entry->context_cache_gen == s->context_cache_gen) { 1121 trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi, 1122 cc_entry->context_entry.lo, 1123 cc_entry->context_cache_gen); 1124 ce = cc_entry->context_entry; 1125 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1126 } else { 1127 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 1128 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 1129 if (ret_fr) { 1130 ret_fr = -ret_fr; 1131 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 1132 trace_vtd_fault_disabled(); 1133 } else { 1134 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 1135 } 1136 goto error; 1137 } 1138 /* Update context-cache */ 1139 trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo, 1140 cc_entry->context_cache_gen, 1141 s->context_cache_gen); 1142 cc_entry->context_entry = ce; 1143 cc_entry->context_cache_gen = s->context_cache_gen; 1144 } 1145 1146 /* 1147 * We don't need to translate for pass-through context entries. 1148 * Also, let's ignore IOTLB caching as well for PT devices. 1149 */ 1150 if (vtd_ce_get_type(&ce) == VTD_CONTEXT_TT_PASS_THROUGH) { 1151 entry->iova = addr & VTD_PAGE_MASK_4K; 1152 entry->translated_addr = entry->iova; 1153 entry->addr_mask = ~VTD_PAGE_MASK_4K; 1154 entry->perm = IOMMU_RW; 1155 trace_vtd_translate_pt(source_id, entry->iova); 1156 1157 /* 1158 * When this happens, it means firstly caching-mode is not 1159 * enabled, and this is the first passthrough translation for 1160 * the device. Let's enable the fast path for passthrough. 1161 * 1162 * When passthrough is disabled again for the device, we can 1163 * capture it via the context entry invalidation, then the 1164 * IOMMU region can be swapped back. 1165 */ 1166 vtd_pt_enable_fast_path(s, source_id); 1167 1168 return true; 1169 } 1170 1171 ret_fr = vtd_iova_to_slpte(&ce, addr, is_write, &slpte, &level, 1172 &reads, &writes, s->aw_bits); 1173 if (ret_fr) { 1174 ret_fr = -ret_fr; 1175 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 1176 trace_vtd_fault_disabled(); 1177 } else { 1178 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 1179 } 1180 goto error; 1181 } 1182 1183 page_mask = vtd_slpt_level_page_mask(level); 1184 access_flags = IOMMU_ACCESS_FLAG(reads, writes); 1185 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte, 1186 access_flags, level); 1187 out: 1188 entry->iova = addr & page_mask; 1189 entry->translated_addr = vtd_get_slpte_addr(slpte, s->aw_bits) & page_mask; 1190 entry->addr_mask = ~page_mask; 1191 entry->perm = access_flags; 1192 return true; 1193 1194 error: 1195 entry->iova = 0; 1196 entry->translated_addr = 0; 1197 entry->addr_mask = 0; 1198 entry->perm = IOMMU_NONE; 1199 return false; 1200 } 1201 1202 static void vtd_root_table_setup(IntelIOMMUState *s) 1203 { 1204 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 1205 s->root_extended = s->root & VTD_RTADDR_RTT; 1206 s->root &= VTD_RTADDR_ADDR_MASK(s->aw_bits); 1207 1208 trace_vtd_reg_dmar_root(s->root, s->root_extended); 1209 } 1210 1211 static void vtd_iec_notify_all(IntelIOMMUState *s, bool global, 1212 uint32_t index, uint32_t mask) 1213 { 1214 x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask); 1215 } 1216 1217 static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s) 1218 { 1219 uint64_t value = 0; 1220 value = vtd_get_quad_raw(s, DMAR_IRTA_REG); 1221 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1); 1222 s->intr_root = value & VTD_IRTA_ADDR_MASK(s->aw_bits); 1223 s->intr_eime = value & VTD_IRTA_EIME; 1224 1225 /* Notify global invalidation */ 1226 vtd_iec_notify_all(s, true, 0, 0); 1227 1228 trace_vtd_reg_ir_root(s->intr_root, s->intr_size); 1229 } 1230 1231 static void vtd_iommu_replay_all(IntelIOMMUState *s) 1232 { 1233 IntelIOMMUNotifierNode *node; 1234 1235 QLIST_FOREACH(node, &s->notifiers_list, next) { 1236 memory_region_iommu_replay_all(&node->vtd_as->iommu); 1237 } 1238 } 1239 1240 static void vtd_context_global_invalidate(IntelIOMMUState *s) 1241 { 1242 trace_vtd_inv_desc_cc_global(); 1243 s->context_cache_gen++; 1244 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 1245 vtd_reset_context_cache(s); 1246 } 1247 vtd_switch_address_space_all(s); 1248 /* 1249 * From VT-d spec 6.5.2.1, a global context entry invalidation 1250 * should be followed by a IOTLB global invalidation, so we should 1251 * be safe even without this. Hoewever, let's replay the region as 1252 * well to be safer, and go back here when we need finer tunes for 1253 * VT-d emulation codes. 1254 */ 1255 vtd_iommu_replay_all(s); 1256 } 1257 1258 /* Do a context-cache device-selective invalidation. 1259 * @func_mask: FM field after shifting 1260 */ 1261 static void vtd_context_device_invalidate(IntelIOMMUState *s, 1262 uint16_t source_id, 1263 uint16_t func_mask) 1264 { 1265 uint16_t mask; 1266 VTDBus *vtd_bus; 1267 VTDAddressSpace *vtd_as; 1268 uint8_t bus_n, devfn; 1269 uint16_t devfn_it; 1270 1271 trace_vtd_inv_desc_cc_devices(source_id, func_mask); 1272 1273 switch (func_mask & 3) { 1274 case 0: 1275 mask = 0; /* No bits in the SID field masked */ 1276 break; 1277 case 1: 1278 mask = 4; /* Mask bit 2 in the SID field */ 1279 break; 1280 case 2: 1281 mask = 6; /* Mask bit 2:1 in the SID field */ 1282 break; 1283 case 3: 1284 mask = 7; /* Mask bit 2:0 in the SID field */ 1285 break; 1286 } 1287 mask = ~mask; 1288 1289 bus_n = VTD_SID_TO_BUS(source_id); 1290 vtd_bus = vtd_find_as_from_bus_num(s, bus_n); 1291 if (vtd_bus) { 1292 devfn = VTD_SID_TO_DEVFN(source_id); 1293 for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) { 1294 vtd_as = vtd_bus->dev_as[devfn_it]; 1295 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) { 1296 trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(devfn_it), 1297 VTD_PCI_FUNC(devfn_it)); 1298 vtd_as->context_cache_entry.context_cache_gen = 0; 1299 /* 1300 * Do switch address space when needed, in case if the 1301 * device passthrough bit is switched. 1302 */ 1303 vtd_switch_address_space(vtd_as); 1304 /* 1305 * So a device is moving out of (or moving into) a 1306 * domain, a replay() suites here to notify all the 1307 * IOMMU_NOTIFIER_MAP registers about this change. 1308 * This won't bring bad even if we have no such 1309 * notifier registered - the IOMMU notification 1310 * framework will skip MAP notifications if that 1311 * happened. 1312 */ 1313 memory_region_iommu_replay_all(&vtd_as->iommu); 1314 } 1315 } 1316 } 1317 } 1318 1319 /* Context-cache invalidation 1320 * Returns the Context Actual Invalidation Granularity. 1321 * @val: the content of the CCMD_REG 1322 */ 1323 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 1324 { 1325 uint64_t caig; 1326 uint64_t type = val & VTD_CCMD_CIRG_MASK; 1327 1328 switch (type) { 1329 case VTD_CCMD_DOMAIN_INVL: 1330 /* Fall through */ 1331 case VTD_CCMD_GLOBAL_INVL: 1332 caig = VTD_CCMD_GLOBAL_INVL_A; 1333 vtd_context_global_invalidate(s); 1334 break; 1335 1336 case VTD_CCMD_DEVICE_INVL: 1337 caig = VTD_CCMD_DEVICE_INVL_A; 1338 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 1339 break; 1340 1341 default: 1342 trace_vtd_err("Context cache invalidate type error."); 1343 caig = 0; 1344 } 1345 return caig; 1346 } 1347 1348 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 1349 { 1350 trace_vtd_inv_desc_iotlb_global(); 1351 vtd_reset_iotlb(s); 1352 vtd_iommu_replay_all(s); 1353 } 1354 1355 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 1356 { 1357 IntelIOMMUNotifierNode *node; 1358 VTDContextEntry ce; 1359 VTDAddressSpace *vtd_as; 1360 1361 trace_vtd_inv_desc_iotlb_domain(domain_id); 1362 1363 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 1364 &domain_id); 1365 1366 QLIST_FOREACH(node, &s->notifiers_list, next) { 1367 vtd_as = node->vtd_as; 1368 if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 1369 vtd_as->devfn, &ce) && 1370 domain_id == VTD_CONTEXT_ENTRY_DID(ce.hi)) { 1371 memory_region_iommu_replay_all(&vtd_as->iommu); 1372 } 1373 } 1374 } 1375 1376 static int vtd_page_invalidate_notify_hook(IOMMUTLBEntry *entry, 1377 void *private) 1378 { 1379 memory_region_notify_iommu((IOMMUMemoryRegion *)private, *entry); 1380 return 0; 1381 } 1382 1383 static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, 1384 uint16_t domain_id, hwaddr addr, 1385 uint8_t am) 1386 { 1387 IntelIOMMUNotifierNode *node; 1388 VTDContextEntry ce; 1389 int ret; 1390 1391 QLIST_FOREACH(node, &(s->notifiers_list), next) { 1392 VTDAddressSpace *vtd_as = node->vtd_as; 1393 ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), 1394 vtd_as->devfn, &ce); 1395 if (!ret && domain_id == VTD_CONTEXT_ENTRY_DID(ce.hi)) { 1396 vtd_page_walk(&ce, addr, addr + (1 << am) * VTD_PAGE_SIZE, 1397 vtd_page_invalidate_notify_hook, 1398 (void *)&vtd_as->iommu, true, s->aw_bits); 1399 } 1400 } 1401 } 1402 1403 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 1404 hwaddr addr, uint8_t am) 1405 { 1406 VTDIOTLBPageInvInfo info; 1407 1408 trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am); 1409 1410 assert(am <= VTD_MAMV); 1411 info.domain_id = domain_id; 1412 info.addr = addr; 1413 info.mask = ~((1 << am) - 1); 1414 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 1415 vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am); 1416 } 1417 1418 /* Flush IOTLB 1419 * Returns the IOTLB Actual Invalidation Granularity. 1420 * @val: the content of the IOTLB_REG 1421 */ 1422 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 1423 { 1424 uint64_t iaig; 1425 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 1426 uint16_t domain_id; 1427 hwaddr addr; 1428 uint8_t am; 1429 1430 switch (type) { 1431 case VTD_TLB_GLOBAL_FLUSH: 1432 iaig = VTD_TLB_GLOBAL_FLUSH_A; 1433 vtd_iotlb_global_invalidate(s); 1434 break; 1435 1436 case VTD_TLB_DSI_FLUSH: 1437 domain_id = VTD_TLB_DID(val); 1438 iaig = VTD_TLB_DSI_FLUSH_A; 1439 vtd_iotlb_domain_invalidate(s, domain_id); 1440 break; 1441 1442 case VTD_TLB_PSI_FLUSH: 1443 domain_id = VTD_TLB_DID(val); 1444 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 1445 am = VTD_IVA_AM(addr); 1446 addr = VTD_IVA_ADDR(addr); 1447 if (am > VTD_MAMV) { 1448 trace_vtd_err("IOTLB PSI flush: address mask overflow."); 1449 iaig = 0; 1450 break; 1451 } 1452 iaig = VTD_TLB_PSI_FLUSH_A; 1453 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1454 break; 1455 1456 default: 1457 trace_vtd_err("IOTLB flush: invalid granularity."); 1458 iaig = 0; 1459 } 1460 return iaig; 1461 } 1462 1463 static void vtd_fetch_inv_desc(IntelIOMMUState *s); 1464 1465 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 1466 { 1467 return s->qi_enabled && (s->iq_tail == s->iq_head) && 1468 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 1469 } 1470 1471 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 1472 { 1473 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 1474 1475 trace_vtd_inv_qi_enable(en); 1476 1477 if (en) { 1478 s->iq = iqa_val & VTD_IQA_IQA_MASK(s->aw_bits); 1479 /* 2^(x+8) entries */ 1480 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8); 1481 s->qi_enabled = true; 1482 trace_vtd_inv_qi_setup(s->iq, s->iq_size); 1483 /* Ok - report back to driver */ 1484 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 1485 1486 if (s->iq_tail != 0) { 1487 /* 1488 * This is a spec violation but Windows guests are known to set up 1489 * Queued Invalidation this way so we allow the write and process 1490 * Invalidation Descriptors right away. 1491 */ 1492 trace_vtd_warn_invalid_qi_tail(s->iq_tail); 1493 if (!(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 1494 vtd_fetch_inv_desc(s); 1495 } 1496 } 1497 } else { 1498 if (vtd_queued_inv_disable_check(s)) { 1499 /* disable Queued Invalidation */ 1500 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 1501 s->iq_head = 0; 1502 s->qi_enabled = false; 1503 /* Ok - report back to driver */ 1504 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 1505 } else { 1506 trace_vtd_err_qi_disable(s->iq_head, s->iq_tail, s->iq_last_desc_type); 1507 } 1508 } 1509 } 1510 1511 /* Set Root Table Pointer */ 1512 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 1513 { 1514 vtd_root_table_setup(s); 1515 /* Ok - report back to driver */ 1516 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 1517 } 1518 1519 /* Set Interrupt Remap Table Pointer */ 1520 static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s) 1521 { 1522 vtd_interrupt_remap_table_setup(s); 1523 /* Ok - report back to driver */ 1524 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS); 1525 } 1526 1527 /* Handle Translation Enable/Disable */ 1528 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 1529 { 1530 if (s->dmar_enabled == en) { 1531 return; 1532 } 1533 1534 trace_vtd_dmar_enable(en); 1535 1536 if (en) { 1537 s->dmar_enabled = true; 1538 /* Ok - report back to driver */ 1539 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 1540 } else { 1541 s->dmar_enabled = false; 1542 1543 /* Clear the index of Fault Recording Register */ 1544 s->next_frcd_reg = 0; 1545 /* Ok - report back to driver */ 1546 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 1547 } 1548 1549 vtd_switch_address_space_all(s); 1550 } 1551 1552 /* Handle Interrupt Remap Enable/Disable */ 1553 static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en) 1554 { 1555 trace_vtd_ir_enable(en); 1556 1557 if (en) { 1558 s->intr_enabled = true; 1559 /* Ok - report back to driver */ 1560 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES); 1561 } else { 1562 s->intr_enabled = false; 1563 /* Ok - report back to driver */ 1564 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0); 1565 } 1566 } 1567 1568 /* Handle write to Global Command Register */ 1569 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 1570 { 1571 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 1572 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 1573 uint32_t changed = status ^ val; 1574 1575 trace_vtd_reg_write_gcmd(status, val); 1576 if (changed & VTD_GCMD_TE) { 1577 /* Translation enable/disable */ 1578 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 1579 } 1580 if (val & VTD_GCMD_SRTP) { 1581 /* Set/update the root-table pointer */ 1582 vtd_handle_gcmd_srtp(s); 1583 } 1584 if (changed & VTD_GCMD_QIE) { 1585 /* Queued Invalidation Enable */ 1586 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 1587 } 1588 if (val & VTD_GCMD_SIRTP) { 1589 /* Set/update the interrupt remapping root-table pointer */ 1590 vtd_handle_gcmd_sirtp(s); 1591 } 1592 if (changed & VTD_GCMD_IRE) { 1593 /* Interrupt remap enable/disable */ 1594 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE); 1595 } 1596 } 1597 1598 /* Handle write to Context Command Register */ 1599 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 1600 { 1601 uint64_t ret; 1602 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 1603 1604 /* Context-cache invalidation request */ 1605 if (val & VTD_CCMD_ICC) { 1606 if (s->qi_enabled) { 1607 trace_vtd_err("Queued Invalidation enabled, " 1608 "should not use register-based invalidation"); 1609 return; 1610 } 1611 ret = vtd_context_cache_invalidate(s, val); 1612 /* Invalidation completed. Change something to show */ 1613 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 1614 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 1615 ret); 1616 } 1617 } 1618 1619 /* Handle write to IOTLB Invalidation Register */ 1620 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 1621 { 1622 uint64_t ret; 1623 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 1624 1625 /* IOTLB invalidation request */ 1626 if (val & VTD_TLB_IVT) { 1627 if (s->qi_enabled) { 1628 trace_vtd_err("Queued Invalidation enabled, " 1629 "should not use register-based invalidation."); 1630 return; 1631 } 1632 ret = vtd_iotlb_flush(s, val); 1633 /* Invalidation completed. Change something to show */ 1634 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 1635 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 1636 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 1637 } 1638 } 1639 1640 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 1641 static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset, 1642 VTDInvDesc *inv_desc) 1643 { 1644 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc); 1645 if (dma_memory_read(&address_space_memory, addr, inv_desc, 1646 sizeof(*inv_desc))) { 1647 trace_vtd_err("Read INV DESC failed."); 1648 inv_desc->lo = 0; 1649 inv_desc->hi = 0; 1650 return false; 1651 } 1652 inv_desc->lo = le64_to_cpu(inv_desc->lo); 1653 inv_desc->hi = le64_to_cpu(inv_desc->hi); 1654 return true; 1655 } 1656 1657 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1658 { 1659 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 1660 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 1661 trace_vtd_inv_desc_wait_invalid(inv_desc->hi, inv_desc->lo); 1662 return false; 1663 } 1664 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 1665 /* Status Write */ 1666 uint32_t status_data = (uint32_t)(inv_desc->lo >> 1667 VTD_INV_DESC_WAIT_DATA_SHIFT); 1668 1669 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 1670 1671 /* FIXME: need to be masked with HAW? */ 1672 dma_addr_t status_addr = inv_desc->hi; 1673 trace_vtd_inv_desc_wait_sw(status_addr, status_data); 1674 status_data = cpu_to_le32(status_data); 1675 if (dma_memory_write(&address_space_memory, status_addr, &status_data, 1676 sizeof(status_data))) { 1677 trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo); 1678 return false; 1679 } 1680 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 1681 /* Interrupt flag */ 1682 vtd_generate_completion_event(s); 1683 } else { 1684 trace_vtd_inv_desc_wait_invalid(inv_desc->hi, inv_desc->lo); 1685 return false; 1686 } 1687 return true; 1688 } 1689 1690 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 1691 VTDInvDesc *inv_desc) 1692 { 1693 uint16_t sid, fmask; 1694 1695 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 1696 trace_vtd_inv_desc_cc_invalid(inv_desc->hi, inv_desc->lo); 1697 return false; 1698 } 1699 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 1700 case VTD_INV_DESC_CC_DOMAIN: 1701 trace_vtd_inv_desc_cc_domain( 1702 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 1703 /* Fall through */ 1704 case VTD_INV_DESC_CC_GLOBAL: 1705 vtd_context_global_invalidate(s); 1706 break; 1707 1708 case VTD_INV_DESC_CC_DEVICE: 1709 sid = VTD_INV_DESC_CC_SID(inv_desc->lo); 1710 fmask = VTD_INV_DESC_CC_FM(inv_desc->lo); 1711 vtd_context_device_invalidate(s, sid, fmask); 1712 break; 1713 1714 default: 1715 trace_vtd_inv_desc_cc_invalid(inv_desc->hi, inv_desc->lo); 1716 return false; 1717 } 1718 return true; 1719 } 1720 1721 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1722 { 1723 uint16_t domain_id; 1724 uint8_t am; 1725 hwaddr addr; 1726 1727 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 1728 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 1729 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo); 1730 return false; 1731 } 1732 1733 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 1734 case VTD_INV_DESC_IOTLB_GLOBAL: 1735 vtd_iotlb_global_invalidate(s); 1736 break; 1737 1738 case VTD_INV_DESC_IOTLB_DOMAIN: 1739 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1740 vtd_iotlb_domain_invalidate(s, domain_id); 1741 break; 1742 1743 case VTD_INV_DESC_IOTLB_PAGE: 1744 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1745 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 1746 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 1747 if (am > VTD_MAMV) { 1748 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo); 1749 return false; 1750 } 1751 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1752 break; 1753 1754 default: 1755 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo); 1756 return false; 1757 } 1758 return true; 1759 } 1760 1761 static bool vtd_process_inv_iec_desc(IntelIOMMUState *s, 1762 VTDInvDesc *inv_desc) 1763 { 1764 trace_vtd_inv_desc_iec(inv_desc->iec.granularity, 1765 inv_desc->iec.index, 1766 inv_desc->iec.index_mask); 1767 1768 vtd_iec_notify_all(s, !inv_desc->iec.granularity, 1769 inv_desc->iec.index, 1770 inv_desc->iec.index_mask); 1771 return true; 1772 } 1773 1774 static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s, 1775 VTDInvDesc *inv_desc) 1776 { 1777 VTDAddressSpace *vtd_dev_as; 1778 IOMMUTLBEntry entry; 1779 struct VTDBus *vtd_bus; 1780 hwaddr addr; 1781 uint64_t sz; 1782 uint16_t sid; 1783 uint8_t devfn; 1784 bool size; 1785 uint8_t bus_num; 1786 1787 addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi); 1788 sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo); 1789 devfn = sid & 0xff; 1790 bus_num = sid >> 8; 1791 size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi); 1792 1793 if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) || 1794 (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) { 1795 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo); 1796 return false; 1797 } 1798 1799 vtd_bus = vtd_find_as_from_bus_num(s, bus_num); 1800 if (!vtd_bus) { 1801 goto done; 1802 } 1803 1804 vtd_dev_as = vtd_bus->dev_as[devfn]; 1805 if (!vtd_dev_as) { 1806 goto done; 1807 } 1808 1809 /* According to ATS spec table 2.4: 1810 * S = 0, bits 15:12 = xxxx range size: 4K 1811 * S = 1, bits 15:12 = xxx0 range size: 8K 1812 * S = 1, bits 15:12 = xx01 range size: 16K 1813 * S = 1, bits 15:12 = x011 range size: 32K 1814 * S = 1, bits 15:12 = 0111 range size: 64K 1815 * ... 1816 */ 1817 if (size) { 1818 sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT); 1819 addr &= ~(sz - 1); 1820 } else { 1821 sz = VTD_PAGE_SIZE; 1822 } 1823 1824 entry.target_as = &vtd_dev_as->as; 1825 entry.addr_mask = sz - 1; 1826 entry.iova = addr; 1827 entry.perm = IOMMU_NONE; 1828 entry.translated_addr = 0; 1829 memory_region_notify_iommu(&vtd_dev_as->iommu, entry); 1830 1831 done: 1832 return true; 1833 } 1834 1835 static bool vtd_process_inv_desc(IntelIOMMUState *s) 1836 { 1837 VTDInvDesc inv_desc; 1838 uint8_t desc_type; 1839 1840 trace_vtd_inv_qi_head(s->iq_head); 1841 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) { 1842 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1843 return false; 1844 } 1845 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 1846 /* FIXME: should update at first or at last? */ 1847 s->iq_last_desc_type = desc_type; 1848 1849 switch (desc_type) { 1850 case VTD_INV_DESC_CC: 1851 trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo); 1852 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 1853 return false; 1854 } 1855 break; 1856 1857 case VTD_INV_DESC_IOTLB: 1858 trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo); 1859 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 1860 return false; 1861 } 1862 break; 1863 1864 case VTD_INV_DESC_WAIT: 1865 trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo); 1866 if (!vtd_process_wait_desc(s, &inv_desc)) { 1867 return false; 1868 } 1869 break; 1870 1871 case VTD_INV_DESC_IEC: 1872 trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo); 1873 if (!vtd_process_inv_iec_desc(s, &inv_desc)) { 1874 return false; 1875 } 1876 break; 1877 1878 case VTD_INV_DESC_DEVICE: 1879 trace_vtd_inv_desc("device", inv_desc.hi, inv_desc.lo); 1880 if (!vtd_process_device_iotlb_desc(s, &inv_desc)) { 1881 return false; 1882 } 1883 break; 1884 1885 default: 1886 trace_vtd_inv_desc_invalid(inv_desc.hi, inv_desc.lo); 1887 return false; 1888 } 1889 s->iq_head++; 1890 if (s->iq_head == s->iq_size) { 1891 s->iq_head = 0; 1892 } 1893 return true; 1894 } 1895 1896 /* Try to fetch and process more Invalidation Descriptors */ 1897 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 1898 { 1899 trace_vtd_inv_qi_fetch(); 1900 1901 if (s->iq_tail >= s->iq_size) { 1902 /* Detects an invalid Tail pointer */ 1903 trace_vtd_err_qi_tail(s->iq_tail, s->iq_size); 1904 vtd_handle_inv_queue_error(s); 1905 return; 1906 } 1907 while (s->iq_head != s->iq_tail) { 1908 if (!vtd_process_inv_desc(s)) { 1909 /* Invalidation Queue Errors */ 1910 vtd_handle_inv_queue_error(s); 1911 break; 1912 } 1913 /* Must update the IQH_REG in time */ 1914 vtd_set_quad_raw(s, DMAR_IQH_REG, 1915 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) & 1916 VTD_IQH_QH_MASK); 1917 } 1918 } 1919 1920 /* Handle write to Invalidation Queue Tail Register */ 1921 static void vtd_handle_iqt_write(IntelIOMMUState *s) 1922 { 1923 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 1924 1925 s->iq_tail = VTD_IQT_QT(val); 1926 trace_vtd_inv_qi_tail(s->iq_tail); 1927 1928 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 1929 /* Process Invalidation Queue here */ 1930 vtd_fetch_inv_desc(s); 1931 } 1932 } 1933 1934 static void vtd_handle_fsts_write(IntelIOMMUState *s) 1935 { 1936 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 1937 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1938 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 1939 1940 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 1941 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1942 trace_vtd_fsts_clear_ip(); 1943 } 1944 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 1945 * Descriptors if there are any when Queued Invalidation is enabled? 1946 */ 1947 } 1948 1949 static void vtd_handle_fectl_write(IntelIOMMUState *s) 1950 { 1951 uint32_t fectl_reg; 1952 /* FIXME: when software clears the IM field, check the IP field. But do we 1953 * need to compare the old value and the new value to conclude that 1954 * software clears the IM field? Or just check if the IM field is zero? 1955 */ 1956 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1957 1958 trace_vtd_reg_write_fectl(fectl_reg); 1959 1960 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 1961 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 1962 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1963 } 1964 } 1965 1966 static void vtd_handle_ics_write(IntelIOMMUState *s) 1967 { 1968 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 1969 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1970 1971 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 1972 trace_vtd_reg_ics_clear_ip(); 1973 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1974 } 1975 } 1976 1977 static void vtd_handle_iectl_write(IntelIOMMUState *s) 1978 { 1979 uint32_t iectl_reg; 1980 /* FIXME: when software clears the IM field, check the IP field. But do we 1981 * need to compare the old value and the new value to conclude that 1982 * software clears the IM field? Or just check if the IM field is zero? 1983 */ 1984 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1985 1986 trace_vtd_reg_write_iectl(iectl_reg); 1987 1988 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 1989 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 1990 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1991 } 1992 } 1993 1994 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 1995 { 1996 IntelIOMMUState *s = opaque; 1997 uint64_t val; 1998 1999 trace_vtd_reg_read(addr, size); 2000 2001 if (addr + size > DMAR_REG_SIZE) { 2002 trace_vtd_err("Read MMIO over range."); 2003 return (uint64_t)-1; 2004 } 2005 2006 switch (addr) { 2007 /* Root Table Address Register, 64-bit */ 2008 case DMAR_RTADDR_REG: 2009 if (size == 4) { 2010 val = s->root & ((1ULL << 32) - 1); 2011 } else { 2012 val = s->root; 2013 } 2014 break; 2015 2016 case DMAR_RTADDR_REG_HI: 2017 assert(size == 4); 2018 val = s->root >> 32; 2019 break; 2020 2021 /* Invalidation Queue Address Register, 64-bit */ 2022 case DMAR_IQA_REG: 2023 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 2024 if (size == 4) { 2025 val = val & ((1ULL << 32) - 1); 2026 } 2027 break; 2028 2029 case DMAR_IQA_REG_HI: 2030 assert(size == 4); 2031 val = s->iq >> 32; 2032 break; 2033 2034 default: 2035 if (size == 4) { 2036 val = vtd_get_long(s, addr); 2037 } else { 2038 val = vtd_get_quad(s, addr); 2039 } 2040 } 2041 2042 return val; 2043 } 2044 2045 static void vtd_mem_write(void *opaque, hwaddr addr, 2046 uint64_t val, unsigned size) 2047 { 2048 IntelIOMMUState *s = opaque; 2049 2050 trace_vtd_reg_write(addr, size, val); 2051 2052 if (addr + size > DMAR_REG_SIZE) { 2053 trace_vtd_err("Write MMIO over range."); 2054 return; 2055 } 2056 2057 switch (addr) { 2058 /* Global Command Register, 32-bit */ 2059 case DMAR_GCMD_REG: 2060 vtd_set_long(s, addr, val); 2061 vtd_handle_gcmd_write(s); 2062 break; 2063 2064 /* Context Command Register, 64-bit */ 2065 case DMAR_CCMD_REG: 2066 if (size == 4) { 2067 vtd_set_long(s, addr, val); 2068 } else { 2069 vtd_set_quad(s, addr, val); 2070 vtd_handle_ccmd_write(s); 2071 } 2072 break; 2073 2074 case DMAR_CCMD_REG_HI: 2075 assert(size == 4); 2076 vtd_set_long(s, addr, val); 2077 vtd_handle_ccmd_write(s); 2078 break; 2079 2080 /* IOTLB Invalidation Register, 64-bit */ 2081 case DMAR_IOTLB_REG: 2082 if (size == 4) { 2083 vtd_set_long(s, addr, val); 2084 } else { 2085 vtd_set_quad(s, addr, val); 2086 vtd_handle_iotlb_write(s); 2087 } 2088 break; 2089 2090 case DMAR_IOTLB_REG_HI: 2091 assert(size == 4); 2092 vtd_set_long(s, addr, val); 2093 vtd_handle_iotlb_write(s); 2094 break; 2095 2096 /* Invalidate Address Register, 64-bit */ 2097 case DMAR_IVA_REG: 2098 if (size == 4) { 2099 vtd_set_long(s, addr, val); 2100 } else { 2101 vtd_set_quad(s, addr, val); 2102 } 2103 break; 2104 2105 case DMAR_IVA_REG_HI: 2106 assert(size == 4); 2107 vtd_set_long(s, addr, val); 2108 break; 2109 2110 /* Fault Status Register, 32-bit */ 2111 case DMAR_FSTS_REG: 2112 assert(size == 4); 2113 vtd_set_long(s, addr, val); 2114 vtd_handle_fsts_write(s); 2115 break; 2116 2117 /* Fault Event Control Register, 32-bit */ 2118 case DMAR_FECTL_REG: 2119 assert(size == 4); 2120 vtd_set_long(s, addr, val); 2121 vtd_handle_fectl_write(s); 2122 break; 2123 2124 /* Fault Event Data Register, 32-bit */ 2125 case DMAR_FEDATA_REG: 2126 assert(size == 4); 2127 vtd_set_long(s, addr, val); 2128 break; 2129 2130 /* Fault Event Address Register, 32-bit */ 2131 case DMAR_FEADDR_REG: 2132 assert(size == 4); 2133 vtd_set_long(s, addr, val); 2134 break; 2135 2136 /* Fault Event Upper Address Register, 32-bit */ 2137 case DMAR_FEUADDR_REG: 2138 assert(size == 4); 2139 vtd_set_long(s, addr, val); 2140 break; 2141 2142 /* Protected Memory Enable Register, 32-bit */ 2143 case DMAR_PMEN_REG: 2144 assert(size == 4); 2145 vtd_set_long(s, addr, val); 2146 break; 2147 2148 /* Root Table Address Register, 64-bit */ 2149 case DMAR_RTADDR_REG: 2150 if (size == 4) { 2151 vtd_set_long(s, addr, val); 2152 } else { 2153 vtd_set_quad(s, addr, val); 2154 } 2155 break; 2156 2157 case DMAR_RTADDR_REG_HI: 2158 assert(size == 4); 2159 vtd_set_long(s, addr, val); 2160 break; 2161 2162 /* Invalidation Queue Tail Register, 64-bit */ 2163 case DMAR_IQT_REG: 2164 if (size == 4) { 2165 vtd_set_long(s, addr, val); 2166 } else { 2167 vtd_set_quad(s, addr, val); 2168 } 2169 vtd_handle_iqt_write(s); 2170 break; 2171 2172 case DMAR_IQT_REG_HI: 2173 assert(size == 4); 2174 vtd_set_long(s, addr, val); 2175 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 2176 break; 2177 2178 /* Invalidation Queue Address Register, 64-bit */ 2179 case DMAR_IQA_REG: 2180 if (size == 4) { 2181 vtd_set_long(s, addr, val); 2182 } else { 2183 vtd_set_quad(s, addr, val); 2184 } 2185 break; 2186 2187 case DMAR_IQA_REG_HI: 2188 assert(size == 4); 2189 vtd_set_long(s, addr, val); 2190 break; 2191 2192 /* Invalidation Completion Status Register, 32-bit */ 2193 case DMAR_ICS_REG: 2194 assert(size == 4); 2195 vtd_set_long(s, addr, val); 2196 vtd_handle_ics_write(s); 2197 break; 2198 2199 /* Invalidation Event Control Register, 32-bit */ 2200 case DMAR_IECTL_REG: 2201 assert(size == 4); 2202 vtd_set_long(s, addr, val); 2203 vtd_handle_iectl_write(s); 2204 break; 2205 2206 /* Invalidation Event Data Register, 32-bit */ 2207 case DMAR_IEDATA_REG: 2208 assert(size == 4); 2209 vtd_set_long(s, addr, val); 2210 break; 2211 2212 /* Invalidation Event Address Register, 32-bit */ 2213 case DMAR_IEADDR_REG: 2214 assert(size == 4); 2215 vtd_set_long(s, addr, val); 2216 break; 2217 2218 /* Invalidation Event Upper Address Register, 32-bit */ 2219 case DMAR_IEUADDR_REG: 2220 assert(size == 4); 2221 vtd_set_long(s, addr, val); 2222 break; 2223 2224 /* Fault Recording Registers, 128-bit */ 2225 case DMAR_FRCD_REG_0_0: 2226 if (size == 4) { 2227 vtd_set_long(s, addr, val); 2228 } else { 2229 vtd_set_quad(s, addr, val); 2230 } 2231 break; 2232 2233 case DMAR_FRCD_REG_0_1: 2234 assert(size == 4); 2235 vtd_set_long(s, addr, val); 2236 break; 2237 2238 case DMAR_FRCD_REG_0_2: 2239 if (size == 4) { 2240 vtd_set_long(s, addr, val); 2241 } else { 2242 vtd_set_quad(s, addr, val); 2243 /* May clear bit 127 (Fault), update PPF */ 2244 vtd_update_fsts_ppf(s); 2245 } 2246 break; 2247 2248 case DMAR_FRCD_REG_0_3: 2249 assert(size == 4); 2250 vtd_set_long(s, addr, val); 2251 /* May clear bit 127 (Fault), update PPF */ 2252 vtd_update_fsts_ppf(s); 2253 break; 2254 2255 case DMAR_IRTA_REG: 2256 if (size == 4) { 2257 vtd_set_long(s, addr, val); 2258 } else { 2259 vtd_set_quad(s, addr, val); 2260 } 2261 break; 2262 2263 case DMAR_IRTA_REG_HI: 2264 assert(size == 4); 2265 vtd_set_long(s, addr, val); 2266 break; 2267 2268 default: 2269 if (size == 4) { 2270 vtd_set_long(s, addr, val); 2271 } else { 2272 vtd_set_quad(s, addr, val); 2273 } 2274 } 2275 } 2276 2277 static IOMMUTLBEntry vtd_iommu_translate(IOMMUMemoryRegion *iommu, hwaddr addr, 2278 IOMMUAccessFlags flag) 2279 { 2280 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 2281 IntelIOMMUState *s = vtd_as->iommu_state; 2282 IOMMUTLBEntry iotlb = { 2283 /* We'll fill in the rest later. */ 2284 .target_as = &address_space_memory, 2285 }; 2286 bool success; 2287 2288 if (likely(s->dmar_enabled)) { 2289 success = vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, 2290 addr, flag & IOMMU_WO, &iotlb); 2291 } else { 2292 /* DMAR disabled, passthrough, use 4k-page*/ 2293 iotlb.iova = addr & VTD_PAGE_MASK_4K; 2294 iotlb.translated_addr = addr & VTD_PAGE_MASK_4K; 2295 iotlb.addr_mask = ~VTD_PAGE_MASK_4K; 2296 iotlb.perm = IOMMU_RW; 2297 success = true; 2298 } 2299 2300 if (likely(success)) { 2301 trace_vtd_dmar_translate(pci_bus_num(vtd_as->bus), 2302 VTD_PCI_SLOT(vtd_as->devfn), 2303 VTD_PCI_FUNC(vtd_as->devfn), 2304 iotlb.iova, iotlb.translated_addr, 2305 iotlb.addr_mask); 2306 } else { 2307 trace_vtd_err_dmar_translate(pci_bus_num(vtd_as->bus), 2308 VTD_PCI_SLOT(vtd_as->devfn), 2309 VTD_PCI_FUNC(vtd_as->devfn), 2310 iotlb.iova); 2311 } 2312 2313 return iotlb; 2314 } 2315 2316 static void vtd_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu, 2317 IOMMUNotifierFlag old, 2318 IOMMUNotifierFlag new) 2319 { 2320 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 2321 IntelIOMMUState *s = vtd_as->iommu_state; 2322 IntelIOMMUNotifierNode *node = NULL; 2323 IntelIOMMUNotifierNode *next_node = NULL; 2324 2325 if (!s->caching_mode && new & IOMMU_NOTIFIER_MAP) { 2326 error_report("We need to set caching-mode=1 for intel-iommu to enable " 2327 "device assignment with IOMMU protection."); 2328 exit(1); 2329 } 2330 2331 if (old == IOMMU_NOTIFIER_NONE) { 2332 node = g_malloc0(sizeof(*node)); 2333 node->vtd_as = vtd_as; 2334 QLIST_INSERT_HEAD(&s->notifiers_list, node, next); 2335 return; 2336 } 2337 2338 /* update notifier node with new flags */ 2339 QLIST_FOREACH_SAFE(node, &s->notifiers_list, next, next_node) { 2340 if (node->vtd_as == vtd_as) { 2341 if (new == IOMMU_NOTIFIER_NONE) { 2342 QLIST_REMOVE(node, next); 2343 g_free(node); 2344 } 2345 return; 2346 } 2347 } 2348 } 2349 2350 static int vtd_post_load(void *opaque, int version_id) 2351 { 2352 IntelIOMMUState *iommu = opaque; 2353 2354 /* 2355 * Memory regions are dynamically turned on/off depending on 2356 * context entry configurations from the guest. After migration, 2357 * we need to make sure the memory regions are still correct. 2358 */ 2359 vtd_switch_address_space_all(iommu); 2360 2361 return 0; 2362 } 2363 2364 static const VMStateDescription vtd_vmstate = { 2365 .name = "iommu-intel", 2366 .version_id = 1, 2367 .minimum_version_id = 1, 2368 .priority = MIG_PRI_IOMMU, 2369 .post_load = vtd_post_load, 2370 .fields = (VMStateField[]) { 2371 VMSTATE_UINT64(root, IntelIOMMUState), 2372 VMSTATE_UINT64(intr_root, IntelIOMMUState), 2373 VMSTATE_UINT64(iq, IntelIOMMUState), 2374 VMSTATE_UINT32(intr_size, IntelIOMMUState), 2375 VMSTATE_UINT16(iq_head, IntelIOMMUState), 2376 VMSTATE_UINT16(iq_tail, IntelIOMMUState), 2377 VMSTATE_UINT16(iq_size, IntelIOMMUState), 2378 VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState), 2379 VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE), 2380 VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState), 2381 VMSTATE_BOOL(root_extended, IntelIOMMUState), 2382 VMSTATE_BOOL(dmar_enabled, IntelIOMMUState), 2383 VMSTATE_BOOL(qi_enabled, IntelIOMMUState), 2384 VMSTATE_BOOL(intr_enabled, IntelIOMMUState), 2385 VMSTATE_BOOL(intr_eime, IntelIOMMUState), 2386 VMSTATE_END_OF_LIST() 2387 } 2388 }; 2389 2390 static const MemoryRegionOps vtd_mem_ops = { 2391 .read = vtd_mem_read, 2392 .write = vtd_mem_write, 2393 .endianness = DEVICE_LITTLE_ENDIAN, 2394 .impl = { 2395 .min_access_size = 4, 2396 .max_access_size = 8, 2397 }, 2398 .valid = { 2399 .min_access_size = 4, 2400 .max_access_size = 8, 2401 }, 2402 }; 2403 2404 static Property vtd_properties[] = { 2405 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 2406 DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim, 2407 ON_OFF_AUTO_AUTO), 2408 DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false), 2409 DEFINE_PROP_UINT8("x-aw-bits", IntelIOMMUState, aw_bits, 2410 VTD_HOST_ADDRESS_WIDTH), 2411 DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE), 2412 DEFINE_PROP_END_OF_LIST(), 2413 }; 2414 2415 /* Read IRTE entry with specific index */ 2416 static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index, 2417 VTD_IR_TableEntry *entry, uint16_t sid) 2418 { 2419 static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \ 2420 {0xffff, 0xfffb, 0xfff9, 0xfff8}; 2421 dma_addr_t addr = 0x00; 2422 uint16_t mask, source_id; 2423 uint8_t bus, bus_max, bus_min; 2424 2425 addr = iommu->intr_root + index * sizeof(*entry); 2426 if (dma_memory_read(&address_space_memory, addr, entry, 2427 sizeof(*entry))) { 2428 trace_vtd_err("Memory read failed for IRTE."); 2429 return -VTD_FR_IR_ROOT_INVAL; 2430 } 2431 2432 trace_vtd_ir_irte_get(index, le64_to_cpu(entry->data[1]), 2433 le64_to_cpu(entry->data[0])); 2434 2435 if (!entry->irte.present) { 2436 trace_vtd_err_irte(index, le64_to_cpu(entry->data[1]), 2437 le64_to_cpu(entry->data[0])); 2438 return -VTD_FR_IR_ENTRY_P; 2439 } 2440 2441 if (entry->irte.__reserved_0 || entry->irte.__reserved_1 || 2442 entry->irte.__reserved_2) { 2443 trace_vtd_err_irte(index, le64_to_cpu(entry->data[1]), 2444 le64_to_cpu(entry->data[0])); 2445 return -VTD_FR_IR_IRTE_RSVD; 2446 } 2447 2448 if (sid != X86_IOMMU_SID_INVALID) { 2449 /* Validate IRTE SID */ 2450 source_id = le32_to_cpu(entry->irte.source_id); 2451 switch (entry->irte.sid_vtype) { 2452 case VTD_SVT_NONE: 2453 break; 2454 2455 case VTD_SVT_ALL: 2456 mask = vtd_svt_mask[entry->irte.sid_q]; 2457 if ((source_id & mask) != (sid & mask)) { 2458 trace_vtd_err_irte_sid(index, sid, source_id); 2459 return -VTD_FR_IR_SID_ERR; 2460 } 2461 break; 2462 2463 case VTD_SVT_BUS: 2464 bus_max = source_id >> 8; 2465 bus_min = source_id & 0xff; 2466 bus = sid >> 8; 2467 if (bus > bus_max || bus < bus_min) { 2468 trace_vtd_err_irte_sid_bus(index, bus, bus_min, bus_max); 2469 return -VTD_FR_IR_SID_ERR; 2470 } 2471 break; 2472 2473 default: 2474 trace_vtd_err_irte_svt(index, entry->irte.sid_vtype); 2475 /* Take this as verification failure. */ 2476 return -VTD_FR_IR_SID_ERR; 2477 break; 2478 } 2479 } 2480 2481 return 0; 2482 } 2483 2484 /* Fetch IRQ information of specific IR index */ 2485 static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index, 2486 VTDIrq *irq, uint16_t sid) 2487 { 2488 VTD_IR_TableEntry irte = {}; 2489 int ret = 0; 2490 2491 ret = vtd_irte_get(iommu, index, &irte, sid); 2492 if (ret) { 2493 return ret; 2494 } 2495 2496 irq->trigger_mode = irte.irte.trigger_mode; 2497 irq->vector = irte.irte.vector; 2498 irq->delivery_mode = irte.irte.delivery_mode; 2499 irq->dest = le32_to_cpu(irte.irte.dest_id); 2500 if (!iommu->intr_eime) { 2501 #define VTD_IR_APIC_DEST_MASK (0xff00ULL) 2502 #define VTD_IR_APIC_DEST_SHIFT (8) 2503 irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >> 2504 VTD_IR_APIC_DEST_SHIFT; 2505 } 2506 irq->dest_mode = irte.irte.dest_mode; 2507 irq->redir_hint = irte.irte.redir_hint; 2508 2509 trace_vtd_ir_remap(index, irq->trigger_mode, irq->vector, 2510 irq->delivery_mode, irq->dest, irq->dest_mode); 2511 2512 return 0; 2513 } 2514 2515 /* Generate one MSI message from VTDIrq info */ 2516 static void vtd_generate_msi_message(VTDIrq *irq, MSIMessage *msg_out) 2517 { 2518 VTD_MSIMessage msg = {}; 2519 2520 /* Generate address bits */ 2521 msg.dest_mode = irq->dest_mode; 2522 msg.redir_hint = irq->redir_hint; 2523 msg.dest = irq->dest; 2524 msg.__addr_hi = irq->dest & 0xffffff00; 2525 msg.__addr_head = cpu_to_le32(0xfee); 2526 /* Keep this from original MSI address bits */ 2527 msg.__not_used = irq->msi_addr_last_bits; 2528 2529 /* Generate data bits */ 2530 msg.vector = irq->vector; 2531 msg.delivery_mode = irq->delivery_mode; 2532 msg.level = 1; 2533 msg.trigger_mode = irq->trigger_mode; 2534 2535 msg_out->address = msg.msi_addr; 2536 msg_out->data = msg.msi_data; 2537 } 2538 2539 /* Interrupt remapping for MSI/MSI-X entry */ 2540 static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu, 2541 MSIMessage *origin, 2542 MSIMessage *translated, 2543 uint16_t sid) 2544 { 2545 int ret = 0; 2546 VTD_IR_MSIAddress addr; 2547 uint16_t index; 2548 VTDIrq irq = {}; 2549 2550 assert(origin && translated); 2551 2552 trace_vtd_ir_remap_msi_req(origin->address, origin->data); 2553 2554 if (!iommu || !iommu->intr_enabled) { 2555 memcpy(translated, origin, sizeof(*origin)); 2556 goto out; 2557 } 2558 2559 if (origin->address & VTD_MSI_ADDR_HI_MASK) { 2560 trace_vtd_err("MSI address high 32 bits non-zero when " 2561 "Interrupt Remapping enabled."); 2562 return -VTD_FR_IR_REQ_RSVD; 2563 } 2564 2565 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK; 2566 if (addr.addr.__head != 0xfee) { 2567 trace_vtd_err("MSI addr low 32 bit invalid."); 2568 return -VTD_FR_IR_REQ_RSVD; 2569 } 2570 2571 /* This is compatible mode. */ 2572 if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) { 2573 memcpy(translated, origin, sizeof(*origin)); 2574 goto out; 2575 } 2576 2577 index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l); 2578 2579 #define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff) 2580 #define VTD_IR_MSI_DATA_RESERVED (0xffff0000) 2581 2582 if (addr.addr.sub_valid) { 2583 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */ 2584 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE; 2585 } 2586 2587 ret = vtd_remap_irq_get(iommu, index, &irq, sid); 2588 if (ret) { 2589 return ret; 2590 } 2591 2592 if (addr.addr.sub_valid) { 2593 trace_vtd_ir_remap_type("MSI"); 2594 if (origin->data & VTD_IR_MSI_DATA_RESERVED) { 2595 trace_vtd_err_ir_msi_invalid(sid, origin->address, origin->data); 2596 return -VTD_FR_IR_REQ_RSVD; 2597 } 2598 } else { 2599 uint8_t vector = origin->data & 0xff; 2600 uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1; 2601 2602 trace_vtd_ir_remap_type("IOAPIC"); 2603 /* IOAPIC entry vector should be aligned with IRTE vector 2604 * (see vt-d spec 5.1.5.1). */ 2605 if (vector != irq.vector) { 2606 trace_vtd_warn_ir_vector(sid, index, vector, irq.vector); 2607 } 2608 2609 /* The Trigger Mode field must match the Trigger Mode in the IRTE. 2610 * (see vt-d spec 5.1.5.1). */ 2611 if (trigger_mode != irq.trigger_mode) { 2612 trace_vtd_warn_ir_trigger(sid, index, trigger_mode, 2613 irq.trigger_mode); 2614 } 2615 } 2616 2617 /* 2618 * We'd better keep the last two bits, assuming that guest OS 2619 * might modify it. Keep it does not hurt after all. 2620 */ 2621 irq.msi_addr_last_bits = addr.addr.__not_care; 2622 2623 /* Translate VTDIrq to MSI message */ 2624 vtd_generate_msi_message(&irq, translated); 2625 2626 out: 2627 trace_vtd_ir_remap_msi(origin->address, origin->data, 2628 translated->address, translated->data); 2629 return 0; 2630 } 2631 2632 static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src, 2633 MSIMessage *dst, uint16_t sid) 2634 { 2635 return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu), 2636 src, dst, sid); 2637 } 2638 2639 static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr, 2640 uint64_t *data, unsigned size, 2641 MemTxAttrs attrs) 2642 { 2643 return MEMTX_OK; 2644 } 2645 2646 static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr, 2647 uint64_t value, unsigned size, 2648 MemTxAttrs attrs) 2649 { 2650 int ret = 0; 2651 MSIMessage from = {}, to = {}; 2652 uint16_t sid = X86_IOMMU_SID_INVALID; 2653 2654 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST; 2655 from.data = (uint32_t) value; 2656 2657 if (!attrs.unspecified) { 2658 /* We have explicit Source ID */ 2659 sid = attrs.requester_id; 2660 } 2661 2662 ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid); 2663 if (ret) { 2664 /* TODO: report error */ 2665 /* Drop this interrupt */ 2666 return MEMTX_ERROR; 2667 } 2668 2669 apic_get_class()->send_msi(&to); 2670 2671 return MEMTX_OK; 2672 } 2673 2674 static const MemoryRegionOps vtd_mem_ir_ops = { 2675 .read_with_attrs = vtd_mem_ir_read, 2676 .write_with_attrs = vtd_mem_ir_write, 2677 .endianness = DEVICE_LITTLE_ENDIAN, 2678 .impl = { 2679 .min_access_size = 4, 2680 .max_access_size = 4, 2681 }, 2682 .valid = { 2683 .min_access_size = 4, 2684 .max_access_size = 4, 2685 }, 2686 }; 2687 2688 VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) 2689 { 2690 uintptr_t key = (uintptr_t)bus; 2691 VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key); 2692 VTDAddressSpace *vtd_dev_as; 2693 char name[128]; 2694 2695 if (!vtd_bus) { 2696 uintptr_t *new_key = g_malloc(sizeof(*new_key)); 2697 *new_key = (uintptr_t)bus; 2698 /* No corresponding free() */ 2699 vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \ 2700 PCI_DEVFN_MAX); 2701 vtd_bus->bus = bus; 2702 g_hash_table_insert(s->vtd_as_by_busptr, new_key, vtd_bus); 2703 } 2704 2705 vtd_dev_as = vtd_bus->dev_as[devfn]; 2706 2707 if (!vtd_dev_as) { 2708 snprintf(name, sizeof(name), "intel_iommu_devfn_%d", devfn); 2709 vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace)); 2710 2711 vtd_dev_as->bus = bus; 2712 vtd_dev_as->devfn = (uint8_t)devfn; 2713 vtd_dev_as->iommu_state = s; 2714 vtd_dev_as->context_cache_entry.context_cache_gen = 0; 2715 2716 /* 2717 * Memory region relationships looks like (Address range shows 2718 * only lower 32 bits to make it short in length...): 2719 * 2720 * |-----------------+-------------------+----------| 2721 * | Name | Address range | Priority | 2722 * |-----------------+-------------------+----------+ 2723 * | vtd_root | 00000000-ffffffff | 0 | 2724 * | intel_iommu | 00000000-ffffffff | 1 | 2725 * | vtd_sys_alias | 00000000-ffffffff | 1 | 2726 * | intel_iommu_ir | fee00000-feefffff | 64 | 2727 * |-----------------+-------------------+----------| 2728 * 2729 * We enable/disable DMAR by switching enablement for 2730 * vtd_sys_alias and intel_iommu regions. IR region is always 2731 * enabled. 2732 */ 2733 memory_region_init_iommu(&vtd_dev_as->iommu, sizeof(vtd_dev_as->iommu), 2734 TYPE_INTEL_IOMMU_MEMORY_REGION, OBJECT(s), 2735 "intel_iommu_dmar", 2736 UINT64_MAX); 2737 memory_region_init_alias(&vtd_dev_as->sys_alias, OBJECT(s), 2738 "vtd_sys_alias", get_system_memory(), 2739 0, memory_region_size(get_system_memory())); 2740 memory_region_init_io(&vtd_dev_as->iommu_ir, OBJECT(s), 2741 &vtd_mem_ir_ops, s, "intel_iommu_ir", 2742 VTD_INTERRUPT_ADDR_SIZE); 2743 memory_region_init(&vtd_dev_as->root, OBJECT(s), 2744 "vtd_root", UINT64_MAX); 2745 memory_region_add_subregion_overlap(&vtd_dev_as->root, 2746 VTD_INTERRUPT_ADDR_FIRST, 2747 &vtd_dev_as->iommu_ir, 64); 2748 address_space_init(&vtd_dev_as->as, &vtd_dev_as->root, name); 2749 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 2750 &vtd_dev_as->sys_alias, 1); 2751 memory_region_add_subregion_overlap(&vtd_dev_as->root, 0, 2752 MEMORY_REGION(&vtd_dev_as->iommu), 2753 1); 2754 vtd_switch_address_space(vtd_dev_as); 2755 } 2756 return vtd_dev_as; 2757 } 2758 2759 /* Unmap the whole range in the notifier's scope. */ 2760 static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n) 2761 { 2762 IOMMUTLBEntry entry; 2763 hwaddr size; 2764 hwaddr start = n->start; 2765 hwaddr end = n->end; 2766 IntelIOMMUState *s = as->iommu_state; 2767 2768 /* 2769 * Note: all the codes in this function has a assumption that IOVA 2770 * bits are no more than VTD_MGAW bits (which is restricted by 2771 * VT-d spec), otherwise we need to consider overflow of 64 bits. 2772 */ 2773 2774 if (end > VTD_ADDRESS_SIZE(s->aw_bits)) { 2775 /* 2776 * Don't need to unmap regions that is bigger than the whole 2777 * VT-d supported address space size 2778 */ 2779 end = VTD_ADDRESS_SIZE(s->aw_bits); 2780 } 2781 2782 assert(start <= end); 2783 size = end - start; 2784 2785 if (ctpop64(size) != 1) { 2786 /* 2787 * This size cannot format a correct mask. Let's enlarge it to 2788 * suite the minimum available mask. 2789 */ 2790 int n = 64 - clz64(size); 2791 if (n > s->aw_bits) { 2792 /* should not happen, but in case it happens, limit it */ 2793 n = s->aw_bits; 2794 } 2795 size = 1ULL << n; 2796 } 2797 2798 entry.target_as = &address_space_memory; 2799 /* Adjust iova for the size */ 2800 entry.iova = n->start & ~(size - 1); 2801 /* This field is meaningless for unmap */ 2802 entry.translated_addr = 0; 2803 entry.perm = IOMMU_NONE; 2804 entry.addr_mask = size - 1; 2805 2806 trace_vtd_as_unmap_whole(pci_bus_num(as->bus), 2807 VTD_PCI_SLOT(as->devfn), 2808 VTD_PCI_FUNC(as->devfn), 2809 entry.iova, size); 2810 2811 memory_region_notify_one(n, &entry); 2812 } 2813 2814 static void vtd_address_space_unmap_all(IntelIOMMUState *s) 2815 { 2816 IntelIOMMUNotifierNode *node; 2817 VTDAddressSpace *vtd_as; 2818 IOMMUNotifier *n; 2819 2820 QLIST_FOREACH(node, &s->notifiers_list, next) { 2821 vtd_as = node->vtd_as; 2822 IOMMU_NOTIFIER_FOREACH(n, &vtd_as->iommu) { 2823 vtd_address_space_unmap(vtd_as, n); 2824 } 2825 } 2826 } 2827 2828 static int vtd_replay_hook(IOMMUTLBEntry *entry, void *private) 2829 { 2830 memory_region_notify_one((IOMMUNotifier *)private, entry); 2831 return 0; 2832 } 2833 2834 static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) 2835 { 2836 VTDAddressSpace *vtd_as = container_of(iommu_mr, VTDAddressSpace, iommu); 2837 IntelIOMMUState *s = vtd_as->iommu_state; 2838 uint8_t bus_n = pci_bus_num(vtd_as->bus); 2839 VTDContextEntry ce; 2840 2841 /* 2842 * The replay can be triggered by either a invalidation or a newly 2843 * created entry. No matter what, we release existing mappings 2844 * (it means flushing caches for UNMAP-only registers). 2845 */ 2846 vtd_address_space_unmap(vtd_as, n); 2847 2848 if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { 2849 trace_vtd_replay_ce_valid(bus_n, PCI_SLOT(vtd_as->devfn), 2850 PCI_FUNC(vtd_as->devfn), 2851 VTD_CONTEXT_ENTRY_DID(ce.hi), 2852 ce.hi, ce.lo); 2853 vtd_page_walk(&ce, 0, ~0ULL, vtd_replay_hook, (void *)n, false, 2854 s->aw_bits); 2855 } else { 2856 trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn), 2857 PCI_FUNC(vtd_as->devfn)); 2858 } 2859 2860 return; 2861 } 2862 2863 /* Do the initialization. It will also be called when reset, so pay 2864 * attention when adding new initialization stuff. 2865 */ 2866 static void vtd_init(IntelIOMMUState *s) 2867 { 2868 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 2869 2870 memset(s->csr, 0, DMAR_REG_SIZE); 2871 memset(s->wmask, 0, DMAR_REG_SIZE); 2872 memset(s->w1cmask, 0, DMAR_REG_SIZE); 2873 memset(s->womask, 0, DMAR_REG_SIZE); 2874 2875 s->root = 0; 2876 s->root_extended = false; 2877 s->dmar_enabled = false; 2878 s->iq_head = 0; 2879 s->iq_tail = 0; 2880 s->iq = 0; 2881 s->iq_size = 0; 2882 s->qi_enabled = false; 2883 s->iq_last_desc_type = VTD_INV_DESC_NONE; 2884 s->next_frcd_reg = 0; 2885 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | 2886 VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS | 2887 VTD_CAP_SAGAW_39bit | VTD_CAP_MGAW(s->aw_bits); 2888 if (s->aw_bits == VTD_HOST_AW_48BIT) { 2889 s->cap |= VTD_CAP_SAGAW_48bit; 2890 } 2891 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 2892 2893 /* 2894 * Rsvd field masks for spte 2895 */ 2896 vtd_paging_entry_rsvd_field[0] = ~0ULL; 2897 vtd_paging_entry_rsvd_field[1] = VTD_SPTE_PAGE_L1_RSVD_MASK(s->aw_bits); 2898 vtd_paging_entry_rsvd_field[2] = VTD_SPTE_PAGE_L2_RSVD_MASK(s->aw_bits); 2899 vtd_paging_entry_rsvd_field[3] = VTD_SPTE_PAGE_L3_RSVD_MASK(s->aw_bits); 2900 vtd_paging_entry_rsvd_field[4] = VTD_SPTE_PAGE_L4_RSVD_MASK(s->aw_bits); 2901 vtd_paging_entry_rsvd_field[5] = VTD_SPTE_LPAGE_L1_RSVD_MASK(s->aw_bits); 2902 vtd_paging_entry_rsvd_field[6] = VTD_SPTE_LPAGE_L2_RSVD_MASK(s->aw_bits); 2903 vtd_paging_entry_rsvd_field[7] = VTD_SPTE_LPAGE_L3_RSVD_MASK(s->aw_bits); 2904 vtd_paging_entry_rsvd_field[8] = VTD_SPTE_LPAGE_L4_RSVD_MASK(s->aw_bits); 2905 2906 if (x86_iommu->intr_supported) { 2907 s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV; 2908 if (s->intr_eim == ON_OFF_AUTO_ON) { 2909 s->ecap |= VTD_ECAP_EIM; 2910 } 2911 assert(s->intr_eim != ON_OFF_AUTO_AUTO); 2912 } 2913 2914 if (x86_iommu->dt_supported) { 2915 s->ecap |= VTD_ECAP_DT; 2916 } 2917 2918 if (x86_iommu->pt_supported) { 2919 s->ecap |= VTD_ECAP_PT; 2920 } 2921 2922 if (s->caching_mode) { 2923 s->cap |= VTD_CAP_CM; 2924 } 2925 2926 vtd_reset_context_cache(s); 2927 vtd_reset_iotlb(s); 2928 2929 /* Define registers with default values and bit semantics */ 2930 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 2931 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 2932 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 2933 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 2934 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 2935 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 2936 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0); 2937 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 2938 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 2939 2940 /* Advanced Fault Logging not supported */ 2941 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 2942 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 2943 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 2944 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 2945 2946 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 2947 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 2948 */ 2949 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 2950 2951 /* Treated as RO for implementations that PLMR and PHMR fields reported 2952 * as Clear in the CAP_REG. 2953 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 2954 */ 2955 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 2956 2957 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 2958 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 2959 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0); 2960 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 2961 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 2962 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 2963 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 2964 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 2965 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 2966 2967 /* IOTLB registers */ 2968 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 2969 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 2970 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 2971 2972 /* Fault Recording Registers, 128-bit */ 2973 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 2974 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 2975 2976 /* 2977 * Interrupt remapping registers. 2978 */ 2979 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0); 2980 } 2981 2982 /* Should not reset address_spaces when reset because devices will still use 2983 * the address space they got at first (won't ask the bus again). 2984 */ 2985 static void vtd_reset(DeviceState *dev) 2986 { 2987 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 2988 2989 vtd_init(s); 2990 2991 /* 2992 * When device reset, throw away all mappings and external caches 2993 */ 2994 vtd_address_space_unmap_all(s); 2995 } 2996 2997 static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) 2998 { 2999 IntelIOMMUState *s = opaque; 3000 VTDAddressSpace *vtd_as; 3001 3002 assert(0 <= devfn && devfn < PCI_DEVFN_MAX); 3003 3004 vtd_as = vtd_find_add_as(s, bus, devfn); 3005 return &vtd_as->as; 3006 } 3007 3008 static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) 3009 { 3010 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); 3011 3012 /* Currently Intel IOMMU IR only support "kernel-irqchip={off|split}" */ 3013 if (x86_iommu->intr_supported && kvm_irqchip_in_kernel() && 3014 !kvm_irqchip_is_split()) { 3015 error_setg(errp, "Intel Interrupt Remapping cannot work with " 3016 "kernel-irqchip=on, please use 'split|off'."); 3017 return false; 3018 } 3019 if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu->intr_supported) { 3020 error_setg(errp, "eim=on cannot be selected without intremap=on"); 3021 return false; 3022 } 3023 3024 if (s->intr_eim == ON_OFF_AUTO_AUTO) { 3025 s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim) 3026 && x86_iommu->intr_supported ? 3027 ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; 3028 } 3029 if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) { 3030 if (!kvm_irqchip_in_kernel()) { 3031 error_setg(errp, "eim=on requires accel=kvm,kernel-irqchip=split"); 3032 return false; 3033 } 3034 if (!kvm_enable_x2apic()) { 3035 error_setg(errp, "eim=on requires support on the KVM side" 3036 "(X2APIC_API, first shipped in v4.7)"); 3037 return false; 3038 } 3039 } 3040 3041 /* Currently only address widths supported are 39 and 48 bits */ 3042 if ((s->aw_bits != VTD_HOST_AW_39BIT) && 3043 (s->aw_bits != VTD_HOST_AW_48BIT)) { 3044 error_setg(errp, "Supported values for x-aw-bits are: %d, %d", 3045 VTD_HOST_AW_39BIT, VTD_HOST_AW_48BIT); 3046 return false; 3047 } 3048 3049 return true; 3050 } 3051 3052 static void vtd_realize(DeviceState *dev, Error **errp) 3053 { 3054 MachineState *ms = MACHINE(qdev_get_machine()); 3055 PCMachineState *pcms = PC_MACHINE(ms); 3056 PCIBus *bus = pcms->bus; 3057 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 3058 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev); 3059 3060 x86_iommu->type = TYPE_INTEL; 3061 3062 if (!vtd_decide_config(s, errp)) { 3063 return; 3064 } 3065 3066 QLIST_INIT(&s->notifiers_list); 3067 memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num)); 3068 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 3069 "intel_iommu", DMAR_REG_SIZE); 3070 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem); 3071 /* No corresponding destroy */ 3072 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 3073 g_free, g_free); 3074 s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 3075 g_free, g_free); 3076 vtd_init(s); 3077 sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR); 3078 pci_setup_iommu(bus, vtd_host_dma_iommu, dev); 3079 /* Pseudo address space under root PCI bus. */ 3080 pcms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC); 3081 } 3082 3083 static void vtd_class_init(ObjectClass *klass, void *data) 3084 { 3085 DeviceClass *dc = DEVICE_CLASS(klass); 3086 X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass); 3087 3088 dc->reset = vtd_reset; 3089 dc->vmsd = &vtd_vmstate; 3090 dc->props = vtd_properties; 3091 dc->hotpluggable = false; 3092 x86_class->realize = vtd_realize; 3093 x86_class->int_remap = vtd_int_remap; 3094 /* Supported by the pc-q35-* machine types */ 3095 dc->user_creatable = true; 3096 } 3097 3098 static const TypeInfo vtd_info = { 3099 .name = TYPE_INTEL_IOMMU_DEVICE, 3100 .parent = TYPE_X86_IOMMU_DEVICE, 3101 .instance_size = sizeof(IntelIOMMUState), 3102 .class_init = vtd_class_init, 3103 }; 3104 3105 static void vtd_iommu_memory_region_class_init(ObjectClass *klass, 3106 void *data) 3107 { 3108 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 3109 3110 imrc->translate = vtd_iommu_translate; 3111 imrc->notify_flag_changed = vtd_iommu_notify_flag_changed; 3112 imrc->replay = vtd_iommu_replay; 3113 } 3114 3115 static const TypeInfo vtd_iommu_memory_region_info = { 3116 .parent = TYPE_IOMMU_MEMORY_REGION, 3117 .name = TYPE_INTEL_IOMMU_MEMORY_REGION, 3118 .class_init = vtd_iommu_memory_region_class_init, 3119 }; 3120 3121 static void vtd_register_types(void) 3122 { 3123 type_register_static(&vtd_info); 3124 type_register_static(&vtd_iommu_memory_region_info); 3125 } 3126 3127 type_init(vtd_register_types) 3128