1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "hw/sysbus.h" 23 #include "exec/address-spaces.h" 24 #include "intel_iommu_internal.h" 25 #include "hw/pci/pci.h" 26 27 /*#define DEBUG_INTEL_IOMMU*/ 28 #ifdef DEBUG_INTEL_IOMMU 29 enum { 30 DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG, 31 DEBUG_CACHE, 32 }; 33 #define VTD_DBGBIT(x) (1 << DEBUG_##x) 34 static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR); 35 36 #define VTD_DPRINTF(what, fmt, ...) do { \ 37 if (vtd_dbgflags & VTD_DBGBIT(what)) { \ 38 fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \ 39 ## __VA_ARGS__); } \ 40 } while (0) 41 #else 42 #define VTD_DPRINTF(what, fmt, ...) do {} while (0) 43 #endif 44 45 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 46 uint64_t wmask, uint64_t w1cmask) 47 { 48 stq_le_p(&s->csr[addr], val); 49 stq_le_p(&s->wmask[addr], wmask); 50 stq_le_p(&s->w1cmask[addr], w1cmask); 51 } 52 53 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 54 { 55 stq_le_p(&s->womask[addr], mask); 56 } 57 58 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 59 uint32_t wmask, uint32_t w1cmask) 60 { 61 stl_le_p(&s->csr[addr], val); 62 stl_le_p(&s->wmask[addr], wmask); 63 stl_le_p(&s->w1cmask[addr], w1cmask); 64 } 65 66 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 67 { 68 stl_le_p(&s->womask[addr], mask); 69 } 70 71 /* "External" get/set operations */ 72 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 73 { 74 uint64_t oldval = ldq_le_p(&s->csr[addr]); 75 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 76 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 77 stq_le_p(&s->csr[addr], 78 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 79 } 80 81 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 82 { 83 uint32_t oldval = ldl_le_p(&s->csr[addr]); 84 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 85 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 86 stl_le_p(&s->csr[addr], 87 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 88 } 89 90 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 91 { 92 uint64_t val = ldq_le_p(&s->csr[addr]); 93 uint64_t womask = ldq_le_p(&s->womask[addr]); 94 return val & ~womask; 95 } 96 97 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 98 { 99 uint32_t val = ldl_le_p(&s->csr[addr]); 100 uint32_t womask = ldl_le_p(&s->womask[addr]); 101 return val & ~womask; 102 } 103 104 /* "Internal" get/set operations */ 105 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 106 { 107 return ldq_le_p(&s->csr[addr]); 108 } 109 110 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 111 { 112 return ldl_le_p(&s->csr[addr]); 113 } 114 115 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 116 { 117 stq_le_p(&s->csr[addr], val); 118 } 119 120 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 121 uint32_t clear, uint32_t mask) 122 { 123 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 124 stl_le_p(&s->csr[addr], new_val); 125 return new_val; 126 } 127 128 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 129 uint64_t clear, uint64_t mask) 130 { 131 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 132 stq_le_p(&s->csr[addr], new_val); 133 return new_val; 134 } 135 136 /* GHashTable functions */ 137 static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2) 138 { 139 return *((const uint64_t *)v1) == *((const uint64_t *)v2); 140 } 141 142 static guint vtd_uint64_hash(gconstpointer v) 143 { 144 return (guint)*(const uint64_t *)v; 145 } 146 147 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 148 gpointer user_data) 149 { 150 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 151 uint16_t domain_id = *(uint16_t *)user_data; 152 return entry->domain_id == domain_id; 153 } 154 155 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 156 gpointer user_data) 157 { 158 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 159 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 160 uint64_t gfn = info->gfn & info->mask; 161 return (entry->domain_id == info->domain_id) && 162 ((entry->gfn & info->mask) == gfn); 163 } 164 165 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 166 * IntelIOMMUState to 1. 167 */ 168 static void vtd_reset_context_cache(IntelIOMMUState *s) 169 { 170 VTDAddressSpace *vtd_as; 171 VTDBus *vtd_bus; 172 GHashTableIter bus_it; 173 uint32_t devfn_it; 174 175 g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr); 176 177 VTD_DPRINTF(CACHE, "global context_cache_gen=1"); 178 while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) { 179 for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) { 180 vtd_as = vtd_bus->dev_as[devfn_it]; 181 if (!vtd_as) { 182 continue; 183 } 184 vtd_as->context_cache_entry.context_cache_gen = 0; 185 } 186 } 187 s->context_cache_gen = 1; 188 } 189 190 static void vtd_reset_iotlb(IntelIOMMUState *s) 191 { 192 assert(s->iotlb); 193 g_hash_table_remove_all(s->iotlb); 194 } 195 196 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 197 hwaddr addr) 198 { 199 uint64_t key; 200 201 key = (addr >> VTD_PAGE_SHIFT_4K) | 202 ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT); 203 return g_hash_table_lookup(s->iotlb, &key); 204 205 } 206 207 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 208 uint16_t domain_id, hwaddr addr, uint64_t slpte, 209 bool read_flags, bool write_flags) 210 { 211 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 212 uint64_t *key = g_malloc(sizeof(*key)); 213 uint64_t gfn = addr >> VTD_PAGE_SHIFT_4K; 214 215 VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64 216 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte, 217 domain_id); 218 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 219 VTD_DPRINTF(CACHE, "iotlb exceeds size limit, forced to reset"); 220 vtd_reset_iotlb(s); 221 } 222 223 entry->gfn = gfn; 224 entry->domain_id = domain_id; 225 entry->slpte = slpte; 226 entry->read_flags = read_flags; 227 entry->write_flags = write_flags; 228 *key = gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT); 229 g_hash_table_replace(s->iotlb, key, entry); 230 } 231 232 /* Given the reg addr of both the message data and address, generate an 233 * interrupt via MSI. 234 */ 235 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 236 hwaddr mesg_data_reg) 237 { 238 hwaddr addr; 239 uint32_t data; 240 241 assert(mesg_data_reg < DMAR_REG_SIZE); 242 assert(mesg_addr_reg < DMAR_REG_SIZE); 243 244 addr = vtd_get_long_raw(s, mesg_addr_reg); 245 data = vtd_get_long_raw(s, mesg_data_reg); 246 247 VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32, addr, data); 248 address_space_stl_le(&address_space_memory, addr, data, 249 MEMTXATTRS_UNSPECIFIED, NULL); 250 } 251 252 /* Generate a fault event to software via MSI if conditions are met. 253 * Notice that the value of FSTS_REG being passed to it should be the one 254 * before any update. 255 */ 256 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 257 { 258 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 259 pre_fsts & VTD_FSTS_IQE) { 260 VTD_DPRINTF(FLOG, "there are previous interrupt conditions " 261 "to be serviced by software, fault event is not generated " 262 "(FSTS_REG 0x%"PRIx32 ")", pre_fsts); 263 return; 264 } 265 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 266 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 267 VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated"); 268 } else { 269 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 270 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 271 } 272 } 273 274 /* Check if the Fault (F) field of the Fault Recording Register referenced by 275 * @index is Set. 276 */ 277 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 278 { 279 /* Each reg is 128-bit */ 280 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 281 addr += 8; /* Access the high 64-bit half */ 282 283 assert(index < DMAR_FRCD_REG_NR); 284 285 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 286 } 287 288 /* Update the PPF field of Fault Status Register. 289 * Should be called whenever change the F field of any fault recording 290 * registers. 291 */ 292 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 293 { 294 uint32_t i; 295 uint32_t ppf_mask = 0; 296 297 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 298 if (vtd_is_frcd_set(s, i)) { 299 ppf_mask = VTD_FSTS_PPF; 300 break; 301 } 302 } 303 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 304 VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0); 305 } 306 307 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 308 { 309 /* Each reg is 128-bit */ 310 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 311 addr += 8; /* Access the high 64-bit half */ 312 313 assert(index < DMAR_FRCD_REG_NR); 314 315 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 316 vtd_update_fsts_ppf(s); 317 } 318 319 /* Must not update F field now, should be done later */ 320 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 321 uint16_t source_id, hwaddr addr, 322 VTDFaultReason fault, bool is_write) 323 { 324 uint64_t hi = 0, lo; 325 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 326 327 assert(index < DMAR_FRCD_REG_NR); 328 329 lo = VTD_FRCD_FI(addr); 330 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 331 if (!is_write) { 332 hi |= VTD_FRCD_T; 333 } 334 vtd_set_quad_raw(s, frcd_reg_addr, lo); 335 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 336 VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64 337 ", lo 0x%"PRIx64, index, hi, lo); 338 } 339 340 /* Try to collapse multiple pending faults from the same requester */ 341 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 342 { 343 uint32_t i; 344 uint64_t frcd_reg; 345 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 346 347 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 348 frcd_reg = vtd_get_quad_raw(s, addr); 349 VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg); 350 if ((frcd_reg & VTD_FRCD_F) && 351 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 352 return true; 353 } 354 addr += 16; /* 128-bit for each */ 355 } 356 return false; 357 } 358 359 /* Log and report an DMAR (address translation) fault to software */ 360 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 361 hwaddr addr, VTDFaultReason fault, 362 bool is_write) 363 { 364 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 365 366 assert(fault < VTD_FR_MAX); 367 368 if (fault == VTD_FR_RESERVED_ERR) { 369 /* This is not a normal fault reason case. Drop it. */ 370 return; 371 } 372 VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64 373 ", is_write %d", source_id, fault, addr, is_write); 374 if (fsts_reg & VTD_FSTS_PFO) { 375 VTD_DPRINTF(FLOG, "new fault is not recorded due to " 376 "Primary Fault Overflow"); 377 return; 378 } 379 if (vtd_try_collapse_fault(s, source_id)) { 380 VTD_DPRINTF(FLOG, "new fault is not recorded due to " 381 "compression of faults"); 382 return; 383 } 384 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 385 VTD_DPRINTF(FLOG, "Primary Fault Overflow and " 386 "new fault is not recorded, set PFO field"); 387 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 388 return; 389 } 390 391 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write); 392 393 if (fsts_reg & VTD_FSTS_PPF) { 394 VTD_DPRINTF(FLOG, "there are pending faults already, " 395 "fault event is not generated"); 396 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 397 s->next_frcd_reg++; 398 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 399 s->next_frcd_reg = 0; 400 } 401 } else { 402 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 403 VTD_FSTS_FRI(s->next_frcd_reg)); 404 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 405 s->next_frcd_reg++; 406 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 407 s->next_frcd_reg = 0; 408 } 409 /* This case actually cause the PPF to be Set. 410 * So generate fault event (interrupt). 411 */ 412 vtd_generate_fault_event(s, fsts_reg); 413 } 414 } 415 416 /* Handle Invalidation Queue Errors of queued invalidation interface error 417 * conditions. 418 */ 419 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 420 { 421 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 422 423 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 424 vtd_generate_fault_event(s, fsts_reg); 425 } 426 427 /* Set the IWC field and try to generate an invalidation completion interrupt */ 428 static void vtd_generate_completion_event(IntelIOMMUState *s) 429 { 430 VTD_DPRINTF(INV, "completes an invalidation wait command with " 431 "Interrupt Flag"); 432 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 433 VTD_DPRINTF(INV, "there is a previous interrupt condition to be " 434 "serviced by software, " 435 "new invalidation event is not generated"); 436 return; 437 } 438 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 439 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 440 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 441 VTD_DPRINTF(INV, "IM filed in IECTL_REG is set, new invalidation " 442 "event is not generated"); 443 return; 444 } else { 445 /* Generate the interrupt event */ 446 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 447 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 448 } 449 } 450 451 static inline bool vtd_root_entry_present(VTDRootEntry *root) 452 { 453 return root->val & VTD_ROOT_ENTRY_P; 454 } 455 456 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 457 VTDRootEntry *re) 458 { 459 dma_addr_t addr; 460 461 addr = s->root + index * sizeof(*re); 462 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) { 463 VTD_DPRINTF(GENERAL, "error: fail to access root-entry at 0x%"PRIx64 464 " + %"PRIu8, s->root, index); 465 re->val = 0; 466 return -VTD_FR_ROOT_TABLE_INV; 467 } 468 re->val = le64_to_cpu(re->val); 469 return 0; 470 } 471 472 static inline bool vtd_context_entry_present(VTDContextEntry *context) 473 { 474 return context->lo & VTD_CONTEXT_ENTRY_P; 475 } 476 477 static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index, 478 VTDContextEntry *ce) 479 { 480 dma_addr_t addr; 481 482 if (!vtd_root_entry_present(root)) { 483 VTD_DPRINTF(GENERAL, "error: root-entry is not present"); 484 return -VTD_FR_ROOT_ENTRY_P; 485 } 486 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce); 487 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) { 488 VTD_DPRINTF(GENERAL, "error: fail to access context-entry at 0x%"PRIx64 489 " + %"PRIu8, 490 (uint64_t)(root->val & VTD_ROOT_ENTRY_CTP), index); 491 return -VTD_FR_CONTEXT_TABLE_INV; 492 } 493 ce->lo = le64_to_cpu(ce->lo); 494 ce->hi = le64_to_cpu(ce->hi); 495 return 0; 496 } 497 498 static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce) 499 { 500 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 501 } 502 503 /* The shift of an addr for a certain level of paging structure */ 504 static inline uint32_t vtd_slpt_level_shift(uint32_t level) 505 { 506 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 507 } 508 509 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte) 510 { 511 return slpte & VTD_SL_PT_BASE_ADDR_MASK; 512 } 513 514 /* Whether the pte indicates the address of the page frame */ 515 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 516 { 517 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 518 } 519 520 /* Get the content of a spte located in @base_addr[@index] */ 521 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 522 { 523 uint64_t slpte; 524 525 assert(index < VTD_SL_PT_ENTRY_NR); 526 527 if (dma_memory_read(&address_space_memory, 528 base_addr + index * sizeof(slpte), &slpte, 529 sizeof(slpte))) { 530 slpte = (uint64_t)-1; 531 return slpte; 532 } 533 slpte = le64_to_cpu(slpte); 534 return slpte; 535 } 536 537 /* Given a gpa and the level of paging structure, return the offset of current 538 * level. 539 */ 540 static inline uint32_t vtd_gpa_level_offset(uint64_t gpa, uint32_t level) 541 { 542 return (gpa >> vtd_slpt_level_shift(level)) & 543 ((1ULL << VTD_SL_LEVEL_BITS) - 1); 544 } 545 546 /* Check Capability Register to see if the @level of page-table is supported */ 547 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 548 { 549 return VTD_CAP_SAGAW_MASK & s->cap & 550 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 551 } 552 553 /* Get the page-table level that hardware should use for the second-level 554 * page-table walk from the Address Width field of context-entry. 555 */ 556 static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce) 557 { 558 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 559 } 560 561 static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce) 562 { 563 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 564 } 565 566 static const uint64_t vtd_paging_entry_rsvd_field[] = { 567 [0] = ~0ULL, 568 /* For not large page */ 569 [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 570 [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 571 [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 572 [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 573 /* For large page */ 574 [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 575 [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 576 [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 577 [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 578 }; 579 580 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 581 { 582 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) { 583 /* Maybe large page */ 584 return slpte & vtd_paging_entry_rsvd_field[level + 4]; 585 } else { 586 return slpte & vtd_paging_entry_rsvd_field[level]; 587 } 588 } 589 590 /* Given the @gpa, get relevant @slptep. @slpte_level will be the last level 591 * of the translation, can be used for deciding the size of large page. 592 */ 593 static int vtd_gpa_to_slpte(VTDContextEntry *ce, uint64_t gpa, bool is_write, 594 uint64_t *slptep, uint32_t *slpte_level, 595 bool *reads, bool *writes) 596 { 597 dma_addr_t addr = vtd_get_slpt_base_from_context(ce); 598 uint32_t level = vtd_get_level_from_context_entry(ce); 599 uint32_t offset; 600 uint64_t slpte; 601 uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce); 602 uint64_t access_right_check; 603 604 /* Check if @gpa is above 2^X-1, where X is the minimum of MGAW in CAP_REG 605 * and AW in context-entry. 606 */ 607 if (gpa & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) { 608 VTD_DPRINTF(GENERAL, "error: gpa 0x%"PRIx64 " exceeds limits", gpa); 609 return -VTD_FR_ADDR_BEYOND_MGAW; 610 } 611 612 /* FIXME: what is the Atomics request here? */ 613 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 614 615 while (true) { 616 offset = vtd_gpa_level_offset(gpa, level); 617 slpte = vtd_get_slpte(addr, offset); 618 619 if (slpte == (uint64_t)-1) { 620 VTD_DPRINTF(GENERAL, "error: fail to access second-level paging " 621 "entry at level %"PRIu32 " for gpa 0x%"PRIx64, 622 level, gpa); 623 if (level == vtd_get_level_from_context_entry(ce)) { 624 /* Invalid programming of context-entry */ 625 return -VTD_FR_CONTEXT_ENTRY_INV; 626 } else { 627 return -VTD_FR_PAGING_ENTRY_INV; 628 } 629 } 630 *reads = (*reads) && (slpte & VTD_SL_R); 631 *writes = (*writes) && (slpte & VTD_SL_W); 632 if (!(slpte & access_right_check)) { 633 VTD_DPRINTF(GENERAL, "error: lack of %s permission for " 634 "gpa 0x%"PRIx64 " slpte 0x%"PRIx64, 635 (is_write ? "write" : "read"), gpa, slpte); 636 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 637 } 638 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 639 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second " 640 "level paging entry level %"PRIu32 " slpte 0x%"PRIx64, 641 level, slpte); 642 return -VTD_FR_PAGING_ENTRY_RSVD; 643 } 644 645 if (vtd_is_last_slpte(slpte, level)) { 646 *slptep = slpte; 647 *slpte_level = level; 648 return 0; 649 } 650 addr = vtd_get_slpte_addr(slpte); 651 level--; 652 } 653 } 654 655 /* Map a device to its corresponding domain (context-entry) */ 656 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 657 uint8_t devfn, VTDContextEntry *ce) 658 { 659 VTDRootEntry re; 660 int ret_fr; 661 662 ret_fr = vtd_get_root_entry(s, bus_num, &re); 663 if (ret_fr) { 664 return ret_fr; 665 } 666 667 if (!vtd_root_entry_present(&re)) { 668 VTD_DPRINTF(GENERAL, "error: root-entry #%"PRIu8 " is not present", 669 bus_num); 670 return -VTD_FR_ROOT_ENTRY_P; 671 } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) { 672 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in root-entry " 673 "hi 0x%"PRIx64 " lo 0x%"PRIx64, re.rsvd, re.val); 674 return -VTD_FR_ROOT_ENTRY_RSVD; 675 } 676 677 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce); 678 if (ret_fr) { 679 return ret_fr; 680 } 681 682 if (!vtd_context_entry_present(ce)) { 683 VTD_DPRINTF(GENERAL, 684 "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") " 685 "is not present", devfn, bus_num); 686 return -VTD_FR_CONTEXT_ENTRY_P; 687 } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) || 688 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) { 689 VTD_DPRINTF(GENERAL, 690 "error: non-zero reserved field in context-entry " 691 "hi 0x%"PRIx64 " lo 0x%"PRIx64, ce->hi, ce->lo); 692 return -VTD_FR_CONTEXT_ENTRY_RSVD; 693 } 694 /* Check if the programming of context-entry is valid */ 695 if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) { 696 VTD_DPRINTF(GENERAL, "error: unsupported Address Width value in " 697 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64, 698 ce->hi, ce->lo); 699 return -VTD_FR_CONTEXT_ENTRY_INV; 700 } else if (ce->lo & VTD_CONTEXT_ENTRY_TT) { 701 VTD_DPRINTF(GENERAL, "error: unsupported Translation Type in " 702 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64, 703 ce->hi, ce->lo); 704 return -VTD_FR_CONTEXT_ENTRY_INV; 705 } 706 return 0; 707 } 708 709 static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn) 710 { 711 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL); 712 } 713 714 static const bool vtd_qualified_faults[] = { 715 [VTD_FR_RESERVED] = false, 716 [VTD_FR_ROOT_ENTRY_P] = false, 717 [VTD_FR_CONTEXT_ENTRY_P] = true, 718 [VTD_FR_CONTEXT_ENTRY_INV] = true, 719 [VTD_FR_ADDR_BEYOND_MGAW] = true, 720 [VTD_FR_WRITE] = true, 721 [VTD_FR_READ] = true, 722 [VTD_FR_PAGING_ENTRY_INV] = true, 723 [VTD_FR_ROOT_TABLE_INV] = false, 724 [VTD_FR_CONTEXT_TABLE_INV] = false, 725 [VTD_FR_ROOT_ENTRY_RSVD] = false, 726 [VTD_FR_PAGING_ENTRY_RSVD] = true, 727 [VTD_FR_CONTEXT_ENTRY_TT] = true, 728 [VTD_FR_RESERVED_ERR] = false, 729 [VTD_FR_MAX] = false, 730 }; 731 732 /* To see if a fault condition is "qualified", which is reported to software 733 * only if the FPD field in the context-entry used to process the faulting 734 * request is 0. 735 */ 736 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 737 { 738 return vtd_qualified_faults[fault]; 739 } 740 741 static inline bool vtd_is_interrupt_addr(hwaddr addr) 742 { 743 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 744 } 745 746 /* Map dev to context-entry then do a paging-structures walk to do a iommu 747 * translation. 748 * 749 * Called from RCU critical section. 750 * 751 * @bus_num: The bus number 752 * @devfn: The devfn, which is the combined of device and function number 753 * @is_write: The access is a write operation 754 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 755 */ 756 static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, 757 uint8_t devfn, hwaddr addr, bool is_write, 758 IOMMUTLBEntry *entry) 759 { 760 IntelIOMMUState *s = vtd_as->iommu_state; 761 VTDContextEntry ce; 762 uint8_t bus_num = pci_bus_num(bus); 763 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry; 764 uint64_t slpte; 765 uint32_t level; 766 uint16_t source_id = vtd_make_source_id(bus_num, devfn); 767 int ret_fr; 768 bool is_fpd_set = false; 769 bool reads = true; 770 bool writes = true; 771 VTDIOTLBEntry *iotlb_entry; 772 773 /* Check if the request is in interrupt address range */ 774 if (vtd_is_interrupt_addr(addr)) { 775 if (is_write) { 776 /* FIXME: since we don't know the length of the access here, we 777 * treat Non-DWORD length write requests without PASID as 778 * interrupt requests, too. Withoud interrupt remapping support, 779 * we just use 1:1 mapping. 780 */ 781 VTD_DPRINTF(MMU, "write request to interrupt address " 782 "gpa 0x%"PRIx64, addr); 783 entry->iova = addr & VTD_PAGE_MASK_4K; 784 entry->translated_addr = addr & VTD_PAGE_MASK_4K; 785 entry->addr_mask = ~VTD_PAGE_MASK_4K; 786 entry->perm = IOMMU_WO; 787 return; 788 } else { 789 VTD_DPRINTF(GENERAL, "error: read request from interrupt address " 790 "gpa 0x%"PRIx64, addr); 791 vtd_report_dmar_fault(s, source_id, addr, VTD_FR_READ, is_write); 792 return; 793 } 794 } 795 /* Try to fetch slpte form IOTLB */ 796 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr); 797 if (iotlb_entry) { 798 VTD_DPRINTF(CACHE, "hit iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64 799 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, 800 iotlb_entry->slpte, iotlb_entry->domain_id); 801 slpte = iotlb_entry->slpte; 802 reads = iotlb_entry->read_flags; 803 writes = iotlb_entry->write_flags; 804 goto out; 805 } 806 /* Try to fetch context-entry from cache first */ 807 if (cc_entry->context_cache_gen == s->context_cache_gen) { 808 VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d " 809 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")", 810 bus_num, devfn, cc_entry->context_entry.hi, 811 cc_entry->context_entry.lo, cc_entry->context_cache_gen); 812 ce = cc_entry->context_entry; 813 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 814 } else { 815 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 816 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 817 if (ret_fr) { 818 ret_fr = -ret_fr; 819 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 820 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA " 821 "requests through this context-entry " 822 "(with FPD Set)"); 823 } else { 824 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 825 } 826 return; 827 } 828 /* Update context-cache */ 829 VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d " 830 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")", 831 bus_num, devfn, ce.hi, ce.lo, 832 cc_entry->context_cache_gen, s->context_cache_gen); 833 cc_entry->context_entry = ce; 834 cc_entry->context_cache_gen = s->context_cache_gen; 835 } 836 837 ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level, 838 &reads, &writes); 839 if (ret_fr) { 840 ret_fr = -ret_fr; 841 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 842 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests " 843 "through this context-entry (with FPD Set)"); 844 } else { 845 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 846 } 847 return; 848 } 849 850 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte, 851 reads, writes); 852 out: 853 entry->iova = addr & VTD_PAGE_MASK_4K; 854 entry->translated_addr = vtd_get_slpte_addr(slpte) & VTD_PAGE_MASK_4K; 855 entry->addr_mask = ~VTD_PAGE_MASK_4K; 856 entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0); 857 } 858 859 static void vtd_root_table_setup(IntelIOMMUState *s) 860 { 861 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 862 s->root_extended = s->root & VTD_RTADDR_RTT; 863 s->root &= VTD_RTADDR_ADDR_MASK; 864 865 VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root, 866 (s->root_extended ? "(extended)" : "")); 867 } 868 869 static void vtd_context_global_invalidate(IntelIOMMUState *s) 870 { 871 s->context_cache_gen++; 872 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 873 vtd_reset_context_cache(s); 874 } 875 } 876 877 878 /* Find the VTD address space currently associated with a given bus number, 879 */ 880 static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num) 881 { 882 VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num]; 883 if (!vtd_bus) { 884 /* Iterate over the registered buses to find the one 885 * which currently hold this bus number, and update the bus_num lookup table: 886 */ 887 GHashTableIter iter; 888 889 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); 890 while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) { 891 if (pci_bus_num(vtd_bus->bus) == bus_num) { 892 s->vtd_as_by_bus_num[bus_num] = vtd_bus; 893 return vtd_bus; 894 } 895 } 896 } 897 return vtd_bus; 898 } 899 900 /* Do a context-cache device-selective invalidation. 901 * @func_mask: FM field after shifting 902 */ 903 static void vtd_context_device_invalidate(IntelIOMMUState *s, 904 uint16_t source_id, 905 uint16_t func_mask) 906 { 907 uint16_t mask; 908 VTDBus *vtd_bus; 909 VTDAddressSpace *vtd_as; 910 uint16_t devfn; 911 uint16_t devfn_it; 912 913 switch (func_mask & 3) { 914 case 0: 915 mask = 0; /* No bits in the SID field masked */ 916 break; 917 case 1: 918 mask = 4; /* Mask bit 2 in the SID field */ 919 break; 920 case 2: 921 mask = 6; /* Mask bit 2:1 in the SID field */ 922 break; 923 case 3: 924 mask = 7; /* Mask bit 2:0 in the SID field */ 925 break; 926 } 927 VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16 928 " mask %"PRIu16, source_id, mask); 929 vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id)); 930 if (vtd_bus) { 931 devfn = VTD_SID_TO_DEVFN(source_id); 932 for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) { 933 vtd_as = vtd_bus->dev_as[devfn_it]; 934 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) { 935 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16, 936 devfn_it); 937 vtd_as->context_cache_entry.context_cache_gen = 0; 938 } 939 } 940 } 941 } 942 943 /* Context-cache invalidation 944 * Returns the Context Actual Invalidation Granularity. 945 * @val: the content of the CCMD_REG 946 */ 947 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 948 { 949 uint64_t caig; 950 uint64_t type = val & VTD_CCMD_CIRG_MASK; 951 952 switch (type) { 953 case VTD_CCMD_DOMAIN_INVL: 954 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 955 (uint16_t)VTD_CCMD_DID(val)); 956 /* Fall through */ 957 case VTD_CCMD_GLOBAL_INVL: 958 VTD_DPRINTF(INV, "global invalidation"); 959 caig = VTD_CCMD_GLOBAL_INVL_A; 960 vtd_context_global_invalidate(s); 961 break; 962 963 case VTD_CCMD_DEVICE_INVL: 964 caig = VTD_CCMD_DEVICE_INVL_A; 965 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 966 break; 967 968 default: 969 VTD_DPRINTF(GENERAL, "error: invalid granularity"); 970 caig = 0; 971 } 972 return caig; 973 } 974 975 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 976 { 977 vtd_reset_iotlb(s); 978 } 979 980 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 981 { 982 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 983 &domain_id); 984 } 985 986 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 987 hwaddr addr, uint8_t am) 988 { 989 VTDIOTLBPageInvInfo info; 990 991 assert(am <= VTD_MAMV); 992 info.domain_id = domain_id; 993 info.gfn = addr >> VTD_PAGE_SHIFT_4K; 994 info.mask = ~((1 << am) - 1); 995 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 996 } 997 998 /* Flush IOTLB 999 * Returns the IOTLB Actual Invalidation Granularity. 1000 * @val: the content of the IOTLB_REG 1001 */ 1002 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 1003 { 1004 uint64_t iaig; 1005 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 1006 uint16_t domain_id; 1007 hwaddr addr; 1008 uint8_t am; 1009 1010 switch (type) { 1011 case VTD_TLB_GLOBAL_FLUSH: 1012 VTD_DPRINTF(INV, "global invalidation"); 1013 iaig = VTD_TLB_GLOBAL_FLUSH_A; 1014 vtd_iotlb_global_invalidate(s); 1015 break; 1016 1017 case VTD_TLB_DSI_FLUSH: 1018 domain_id = VTD_TLB_DID(val); 1019 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1020 domain_id); 1021 iaig = VTD_TLB_DSI_FLUSH_A; 1022 vtd_iotlb_domain_invalidate(s, domain_id); 1023 break; 1024 1025 case VTD_TLB_PSI_FLUSH: 1026 domain_id = VTD_TLB_DID(val); 1027 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 1028 am = VTD_IVA_AM(addr); 1029 addr = VTD_IVA_ADDR(addr); 1030 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16 1031 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am); 1032 if (am > VTD_MAMV) { 1033 VTD_DPRINTF(GENERAL, "error: supported max address mask value is " 1034 "%"PRIu8, (uint8_t)VTD_MAMV); 1035 iaig = 0; 1036 break; 1037 } 1038 iaig = VTD_TLB_PSI_FLUSH_A; 1039 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1040 break; 1041 1042 default: 1043 VTD_DPRINTF(GENERAL, "error: invalid granularity"); 1044 iaig = 0; 1045 } 1046 return iaig; 1047 } 1048 1049 static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s) 1050 { 1051 return s->iq_tail == 0; 1052 } 1053 1054 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 1055 { 1056 return s->qi_enabled && (s->iq_tail == s->iq_head) && 1057 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 1058 } 1059 1060 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 1061 { 1062 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 1063 1064 VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off")); 1065 if (en) { 1066 if (vtd_queued_inv_enable_check(s)) { 1067 s->iq = iqa_val & VTD_IQA_IQA_MASK; 1068 /* 2^(x+8) entries */ 1069 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8); 1070 s->qi_enabled = true; 1071 VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val); 1072 VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d", 1073 s->iq, s->iq_size); 1074 /* Ok - report back to driver */ 1075 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 1076 } else { 1077 VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: " 1078 "tail %"PRIu16, s->iq_tail); 1079 } 1080 } else { 1081 if (vtd_queued_inv_disable_check(s)) { 1082 /* disable Queued Invalidation */ 1083 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 1084 s->iq_head = 0; 1085 s->qi_enabled = false; 1086 /* Ok - report back to driver */ 1087 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 1088 } else { 1089 VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: " 1090 "head %"PRIu16 ", tail %"PRIu16 1091 ", last_descriptor %"PRIu8, 1092 s->iq_head, s->iq_tail, s->iq_last_desc_type); 1093 } 1094 } 1095 } 1096 1097 /* Set Root Table Pointer */ 1098 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 1099 { 1100 VTD_DPRINTF(CSR, "set Root Table Pointer"); 1101 1102 vtd_root_table_setup(s); 1103 /* Ok - report back to driver */ 1104 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 1105 } 1106 1107 /* Handle Translation Enable/Disable */ 1108 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 1109 { 1110 VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off")); 1111 1112 if (en) { 1113 s->dmar_enabled = true; 1114 /* Ok - report back to driver */ 1115 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 1116 } else { 1117 s->dmar_enabled = false; 1118 1119 /* Clear the index of Fault Recording Register */ 1120 s->next_frcd_reg = 0; 1121 /* Ok - report back to driver */ 1122 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 1123 } 1124 } 1125 1126 /* Handle write to Global Command Register */ 1127 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 1128 { 1129 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 1130 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 1131 uint32_t changed = status ^ val; 1132 1133 VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status); 1134 if (changed & VTD_GCMD_TE) { 1135 /* Translation enable/disable */ 1136 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 1137 } 1138 if (val & VTD_GCMD_SRTP) { 1139 /* Set/update the root-table pointer */ 1140 vtd_handle_gcmd_srtp(s); 1141 } 1142 if (changed & VTD_GCMD_QIE) { 1143 /* Queued Invalidation Enable */ 1144 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 1145 } 1146 } 1147 1148 /* Handle write to Context Command Register */ 1149 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 1150 { 1151 uint64_t ret; 1152 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 1153 1154 /* Context-cache invalidation request */ 1155 if (val & VTD_CCMD_ICC) { 1156 if (s->qi_enabled) { 1157 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, " 1158 "should not use register-based invalidation"); 1159 return; 1160 } 1161 ret = vtd_context_cache_invalidate(s, val); 1162 /* Invalidation completed. Change something to show */ 1163 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 1164 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 1165 ret); 1166 VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret); 1167 } 1168 } 1169 1170 /* Handle write to IOTLB Invalidation Register */ 1171 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 1172 { 1173 uint64_t ret; 1174 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 1175 1176 /* IOTLB invalidation request */ 1177 if (val & VTD_TLB_IVT) { 1178 if (s->qi_enabled) { 1179 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, " 1180 "should not use register-based invalidation"); 1181 return; 1182 } 1183 ret = vtd_iotlb_flush(s, val); 1184 /* Invalidation completed. Change something to show */ 1185 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 1186 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 1187 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 1188 VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret); 1189 } 1190 } 1191 1192 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 1193 static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset, 1194 VTDInvDesc *inv_desc) 1195 { 1196 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc); 1197 if (dma_memory_read(&address_space_memory, addr, inv_desc, 1198 sizeof(*inv_desc))) { 1199 VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor " 1200 "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset); 1201 inv_desc->lo = 0; 1202 inv_desc->hi = 0; 1203 1204 return false; 1205 } 1206 inv_desc->lo = le64_to_cpu(inv_desc->lo); 1207 inv_desc->hi = le64_to_cpu(inv_desc->hi); 1208 return true; 1209 } 1210 1211 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1212 { 1213 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 1214 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 1215 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Invalidation " 1216 "Wait Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1217 inv_desc->hi, inv_desc->lo); 1218 return false; 1219 } 1220 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 1221 /* Status Write */ 1222 uint32_t status_data = (uint32_t)(inv_desc->lo >> 1223 VTD_INV_DESC_WAIT_DATA_SHIFT); 1224 1225 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 1226 1227 /* FIXME: need to be masked with HAW? */ 1228 dma_addr_t status_addr = inv_desc->hi; 1229 VTD_DPRINTF(INV, "status data 0x%x, status addr 0x%"PRIx64, 1230 status_data, status_addr); 1231 status_data = cpu_to_le32(status_data); 1232 if (dma_memory_write(&address_space_memory, status_addr, &status_data, 1233 sizeof(status_data))) { 1234 VTD_DPRINTF(GENERAL, "error: fail to perform a coherent write"); 1235 return false; 1236 } 1237 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 1238 /* Interrupt flag */ 1239 VTD_DPRINTF(INV, "Invalidation Wait Descriptor interrupt completion"); 1240 vtd_generate_completion_event(s); 1241 } else { 1242 VTD_DPRINTF(GENERAL, "error: invalid Invalidation Wait Descriptor: " 1243 "hi 0x%"PRIx64 " lo 0x%"PRIx64, inv_desc->hi, inv_desc->lo); 1244 return false; 1245 } 1246 return true; 1247 } 1248 1249 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 1250 VTDInvDesc *inv_desc) 1251 { 1252 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 1253 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache " 1254 "Invalidate Descriptor"); 1255 return false; 1256 } 1257 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 1258 case VTD_INV_DESC_CC_DOMAIN: 1259 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1260 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 1261 /* Fall through */ 1262 case VTD_INV_DESC_CC_GLOBAL: 1263 VTD_DPRINTF(INV, "global invalidation"); 1264 vtd_context_global_invalidate(s); 1265 break; 1266 1267 case VTD_INV_DESC_CC_DEVICE: 1268 vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo), 1269 VTD_INV_DESC_CC_FM(inv_desc->lo)); 1270 break; 1271 1272 default: 1273 VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache " 1274 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1275 inv_desc->hi, inv_desc->lo); 1276 return false; 1277 } 1278 return true; 1279 } 1280 1281 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1282 { 1283 uint16_t domain_id; 1284 uint8_t am; 1285 hwaddr addr; 1286 1287 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 1288 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 1289 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in IOTLB " 1290 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1291 inv_desc->hi, inv_desc->lo); 1292 return false; 1293 } 1294 1295 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 1296 case VTD_INV_DESC_IOTLB_GLOBAL: 1297 VTD_DPRINTF(INV, "global invalidation"); 1298 vtd_iotlb_global_invalidate(s); 1299 break; 1300 1301 case VTD_INV_DESC_IOTLB_DOMAIN: 1302 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1303 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1304 domain_id); 1305 vtd_iotlb_domain_invalidate(s, domain_id); 1306 break; 1307 1308 case VTD_INV_DESC_IOTLB_PAGE: 1309 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1310 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 1311 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 1312 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16 1313 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am); 1314 if (am > VTD_MAMV) { 1315 VTD_DPRINTF(GENERAL, "error: supported max address mask value is " 1316 "%"PRIu8, (uint8_t)VTD_MAMV); 1317 return false; 1318 } 1319 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1320 break; 1321 1322 default: 1323 VTD_DPRINTF(GENERAL, "error: invalid granularity in IOTLB Invalidate " 1324 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1325 inv_desc->hi, inv_desc->lo); 1326 return false; 1327 } 1328 return true; 1329 } 1330 1331 static bool vtd_process_inv_desc(IntelIOMMUState *s) 1332 { 1333 VTDInvDesc inv_desc; 1334 uint8_t desc_type; 1335 1336 VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head); 1337 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) { 1338 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1339 return false; 1340 } 1341 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 1342 /* FIXME: should update at first or at last? */ 1343 s->iq_last_desc_type = desc_type; 1344 1345 switch (desc_type) { 1346 case VTD_INV_DESC_CC: 1347 VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64 1348 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1349 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 1350 return false; 1351 } 1352 break; 1353 1354 case VTD_INV_DESC_IOTLB: 1355 VTD_DPRINTF(INV, "IOTLB Invalidate Descriptor hi 0x%"PRIx64 1356 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1357 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 1358 return false; 1359 } 1360 break; 1361 1362 case VTD_INV_DESC_WAIT: 1363 VTD_DPRINTF(INV, "Invalidation Wait Descriptor hi 0x%"PRIx64 1364 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1365 if (!vtd_process_wait_desc(s, &inv_desc)) { 1366 return false; 1367 } 1368 break; 1369 1370 default: 1371 VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type " 1372 "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8, 1373 inv_desc.hi, inv_desc.lo, desc_type); 1374 return false; 1375 } 1376 s->iq_head++; 1377 if (s->iq_head == s->iq_size) { 1378 s->iq_head = 0; 1379 } 1380 return true; 1381 } 1382 1383 /* Try to fetch and process more Invalidation Descriptors */ 1384 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 1385 { 1386 VTD_DPRINTF(INV, "fetch Invalidation Descriptors"); 1387 if (s->iq_tail >= s->iq_size) { 1388 /* Detects an invalid Tail pointer */ 1389 VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16 1390 " while iq_size is %"PRIu16, s->iq_tail, s->iq_size); 1391 vtd_handle_inv_queue_error(s); 1392 return; 1393 } 1394 while (s->iq_head != s->iq_tail) { 1395 if (!vtd_process_inv_desc(s)) { 1396 /* Invalidation Queue Errors */ 1397 vtd_handle_inv_queue_error(s); 1398 break; 1399 } 1400 /* Must update the IQH_REG in time */ 1401 vtd_set_quad_raw(s, DMAR_IQH_REG, 1402 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) & 1403 VTD_IQH_QH_MASK); 1404 } 1405 } 1406 1407 /* Handle write to Invalidation Queue Tail Register */ 1408 static void vtd_handle_iqt_write(IntelIOMMUState *s) 1409 { 1410 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 1411 1412 s->iq_tail = VTD_IQT_QT(val); 1413 VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail); 1414 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 1415 /* Process Invalidation Queue here */ 1416 vtd_fetch_inv_desc(s); 1417 } 1418 } 1419 1420 static void vtd_handle_fsts_write(IntelIOMMUState *s) 1421 { 1422 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 1423 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1424 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 1425 1426 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 1427 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1428 VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear " 1429 "IP field of FECTL_REG"); 1430 } 1431 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 1432 * Descriptors if there are any when Queued Invalidation is enabled? 1433 */ 1434 } 1435 1436 static void vtd_handle_fectl_write(IntelIOMMUState *s) 1437 { 1438 uint32_t fectl_reg; 1439 /* FIXME: when software clears the IM field, check the IP field. But do we 1440 * need to compare the old value and the new value to conclude that 1441 * software clears the IM field? Or just check if the IM field is zero? 1442 */ 1443 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1444 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 1445 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 1446 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1447 VTD_DPRINTF(FLOG, "IM field is cleared, generate " 1448 "fault event interrupt"); 1449 } 1450 } 1451 1452 static void vtd_handle_ics_write(IntelIOMMUState *s) 1453 { 1454 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 1455 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1456 1457 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 1458 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1459 VTD_DPRINTF(INV, "pending completion interrupt condition serviced, " 1460 "clear IP field of IECTL_REG"); 1461 } 1462 } 1463 1464 static void vtd_handle_iectl_write(IntelIOMMUState *s) 1465 { 1466 uint32_t iectl_reg; 1467 /* FIXME: when software clears the IM field, check the IP field. But do we 1468 * need to compare the old value and the new value to conclude that 1469 * software clears the IM field? Or just check if the IM field is zero? 1470 */ 1471 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1472 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 1473 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 1474 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1475 VTD_DPRINTF(INV, "IM field is cleared, generate " 1476 "invalidation event interrupt"); 1477 } 1478 } 1479 1480 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 1481 { 1482 IntelIOMMUState *s = opaque; 1483 uint64_t val; 1484 1485 if (addr + size > DMAR_REG_SIZE) { 1486 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64 1487 ", got 0x%"PRIx64 " %d", 1488 (uint64_t)DMAR_REG_SIZE, addr, size); 1489 return (uint64_t)-1; 1490 } 1491 1492 switch (addr) { 1493 /* Root Table Address Register, 64-bit */ 1494 case DMAR_RTADDR_REG: 1495 if (size == 4) { 1496 val = s->root & ((1ULL << 32) - 1); 1497 } else { 1498 val = s->root; 1499 } 1500 break; 1501 1502 case DMAR_RTADDR_REG_HI: 1503 assert(size == 4); 1504 val = s->root >> 32; 1505 break; 1506 1507 /* Invalidation Queue Address Register, 64-bit */ 1508 case DMAR_IQA_REG: 1509 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 1510 if (size == 4) { 1511 val = val & ((1ULL << 32) - 1); 1512 } 1513 break; 1514 1515 case DMAR_IQA_REG_HI: 1516 assert(size == 4); 1517 val = s->iq >> 32; 1518 break; 1519 1520 default: 1521 if (size == 4) { 1522 val = vtd_get_long(s, addr); 1523 } else { 1524 val = vtd_get_quad(s, addr); 1525 } 1526 } 1527 VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64, 1528 addr, size, val); 1529 return val; 1530 } 1531 1532 static void vtd_mem_write(void *opaque, hwaddr addr, 1533 uint64_t val, unsigned size) 1534 { 1535 IntelIOMMUState *s = opaque; 1536 1537 if (addr + size > DMAR_REG_SIZE) { 1538 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64 1539 ", got 0x%"PRIx64 " %d", 1540 (uint64_t)DMAR_REG_SIZE, addr, size); 1541 return; 1542 } 1543 1544 switch (addr) { 1545 /* Global Command Register, 32-bit */ 1546 case DMAR_GCMD_REG: 1547 VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64 1548 ", size %d, val 0x%"PRIx64, addr, size, val); 1549 vtd_set_long(s, addr, val); 1550 vtd_handle_gcmd_write(s); 1551 break; 1552 1553 /* Context Command Register, 64-bit */ 1554 case DMAR_CCMD_REG: 1555 VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64 1556 ", size %d, val 0x%"PRIx64, addr, size, val); 1557 if (size == 4) { 1558 vtd_set_long(s, addr, val); 1559 } else { 1560 vtd_set_quad(s, addr, val); 1561 vtd_handle_ccmd_write(s); 1562 } 1563 break; 1564 1565 case DMAR_CCMD_REG_HI: 1566 VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64 1567 ", size %d, val 0x%"PRIx64, addr, size, val); 1568 assert(size == 4); 1569 vtd_set_long(s, addr, val); 1570 vtd_handle_ccmd_write(s); 1571 break; 1572 1573 /* IOTLB Invalidation Register, 64-bit */ 1574 case DMAR_IOTLB_REG: 1575 VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64 1576 ", size %d, val 0x%"PRIx64, addr, size, val); 1577 if (size == 4) { 1578 vtd_set_long(s, addr, val); 1579 } else { 1580 vtd_set_quad(s, addr, val); 1581 vtd_handle_iotlb_write(s); 1582 } 1583 break; 1584 1585 case DMAR_IOTLB_REG_HI: 1586 VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64 1587 ", size %d, val 0x%"PRIx64, addr, size, val); 1588 assert(size == 4); 1589 vtd_set_long(s, addr, val); 1590 vtd_handle_iotlb_write(s); 1591 break; 1592 1593 /* Invalidate Address Register, 64-bit */ 1594 case DMAR_IVA_REG: 1595 VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64 1596 ", size %d, val 0x%"PRIx64, addr, size, val); 1597 if (size == 4) { 1598 vtd_set_long(s, addr, val); 1599 } else { 1600 vtd_set_quad(s, addr, val); 1601 } 1602 break; 1603 1604 case DMAR_IVA_REG_HI: 1605 VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64 1606 ", size %d, val 0x%"PRIx64, addr, size, val); 1607 assert(size == 4); 1608 vtd_set_long(s, addr, val); 1609 break; 1610 1611 /* Fault Status Register, 32-bit */ 1612 case DMAR_FSTS_REG: 1613 VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64 1614 ", size %d, val 0x%"PRIx64, addr, size, val); 1615 assert(size == 4); 1616 vtd_set_long(s, addr, val); 1617 vtd_handle_fsts_write(s); 1618 break; 1619 1620 /* Fault Event Control Register, 32-bit */ 1621 case DMAR_FECTL_REG: 1622 VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64 1623 ", size %d, val 0x%"PRIx64, addr, size, val); 1624 assert(size == 4); 1625 vtd_set_long(s, addr, val); 1626 vtd_handle_fectl_write(s); 1627 break; 1628 1629 /* Fault Event Data Register, 32-bit */ 1630 case DMAR_FEDATA_REG: 1631 VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64 1632 ", size %d, val 0x%"PRIx64, addr, size, val); 1633 assert(size == 4); 1634 vtd_set_long(s, addr, val); 1635 break; 1636 1637 /* Fault Event Address Register, 32-bit */ 1638 case DMAR_FEADDR_REG: 1639 VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64 1640 ", size %d, val 0x%"PRIx64, addr, size, val); 1641 assert(size == 4); 1642 vtd_set_long(s, addr, val); 1643 break; 1644 1645 /* Fault Event Upper Address Register, 32-bit */ 1646 case DMAR_FEUADDR_REG: 1647 VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64 1648 ", size %d, val 0x%"PRIx64, addr, size, val); 1649 assert(size == 4); 1650 vtd_set_long(s, addr, val); 1651 break; 1652 1653 /* Protected Memory Enable Register, 32-bit */ 1654 case DMAR_PMEN_REG: 1655 VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64 1656 ", size %d, val 0x%"PRIx64, addr, size, val); 1657 assert(size == 4); 1658 vtd_set_long(s, addr, val); 1659 break; 1660 1661 /* Root Table Address Register, 64-bit */ 1662 case DMAR_RTADDR_REG: 1663 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64 1664 ", size %d, val 0x%"PRIx64, addr, size, val); 1665 if (size == 4) { 1666 vtd_set_long(s, addr, val); 1667 } else { 1668 vtd_set_quad(s, addr, val); 1669 } 1670 break; 1671 1672 case DMAR_RTADDR_REG_HI: 1673 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64 1674 ", size %d, val 0x%"PRIx64, addr, size, val); 1675 assert(size == 4); 1676 vtd_set_long(s, addr, val); 1677 break; 1678 1679 /* Invalidation Queue Tail Register, 64-bit */ 1680 case DMAR_IQT_REG: 1681 VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64 1682 ", size %d, val 0x%"PRIx64, addr, size, val); 1683 if (size == 4) { 1684 vtd_set_long(s, addr, val); 1685 } else { 1686 vtd_set_quad(s, addr, val); 1687 } 1688 vtd_handle_iqt_write(s); 1689 break; 1690 1691 case DMAR_IQT_REG_HI: 1692 VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64 1693 ", size %d, val 0x%"PRIx64, addr, size, val); 1694 assert(size == 4); 1695 vtd_set_long(s, addr, val); 1696 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 1697 break; 1698 1699 /* Invalidation Queue Address Register, 64-bit */ 1700 case DMAR_IQA_REG: 1701 VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64 1702 ", size %d, val 0x%"PRIx64, addr, size, val); 1703 if (size == 4) { 1704 vtd_set_long(s, addr, val); 1705 } else { 1706 vtd_set_quad(s, addr, val); 1707 } 1708 break; 1709 1710 case DMAR_IQA_REG_HI: 1711 VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64 1712 ", size %d, val 0x%"PRIx64, addr, size, val); 1713 assert(size == 4); 1714 vtd_set_long(s, addr, val); 1715 break; 1716 1717 /* Invalidation Completion Status Register, 32-bit */ 1718 case DMAR_ICS_REG: 1719 VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64 1720 ", size %d, val 0x%"PRIx64, addr, size, val); 1721 assert(size == 4); 1722 vtd_set_long(s, addr, val); 1723 vtd_handle_ics_write(s); 1724 break; 1725 1726 /* Invalidation Event Control Register, 32-bit */ 1727 case DMAR_IECTL_REG: 1728 VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64 1729 ", size %d, val 0x%"PRIx64, addr, size, val); 1730 assert(size == 4); 1731 vtd_set_long(s, addr, val); 1732 vtd_handle_iectl_write(s); 1733 break; 1734 1735 /* Invalidation Event Data Register, 32-bit */ 1736 case DMAR_IEDATA_REG: 1737 VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64 1738 ", size %d, val 0x%"PRIx64, addr, size, val); 1739 assert(size == 4); 1740 vtd_set_long(s, addr, val); 1741 break; 1742 1743 /* Invalidation Event Address Register, 32-bit */ 1744 case DMAR_IEADDR_REG: 1745 VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64 1746 ", size %d, val 0x%"PRIx64, addr, size, val); 1747 assert(size == 4); 1748 vtd_set_long(s, addr, val); 1749 break; 1750 1751 /* Invalidation Event Upper Address Register, 32-bit */ 1752 case DMAR_IEUADDR_REG: 1753 VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64 1754 ", size %d, val 0x%"PRIx64, addr, size, val); 1755 assert(size == 4); 1756 vtd_set_long(s, addr, val); 1757 break; 1758 1759 /* Fault Recording Registers, 128-bit */ 1760 case DMAR_FRCD_REG_0_0: 1761 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64 1762 ", size %d, val 0x%"PRIx64, addr, size, val); 1763 if (size == 4) { 1764 vtd_set_long(s, addr, val); 1765 } else { 1766 vtd_set_quad(s, addr, val); 1767 } 1768 break; 1769 1770 case DMAR_FRCD_REG_0_1: 1771 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64 1772 ", size %d, val 0x%"PRIx64, addr, size, val); 1773 assert(size == 4); 1774 vtd_set_long(s, addr, val); 1775 break; 1776 1777 case DMAR_FRCD_REG_0_2: 1778 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64 1779 ", size %d, val 0x%"PRIx64, addr, size, val); 1780 if (size == 4) { 1781 vtd_set_long(s, addr, val); 1782 } else { 1783 vtd_set_quad(s, addr, val); 1784 /* May clear bit 127 (Fault), update PPF */ 1785 vtd_update_fsts_ppf(s); 1786 } 1787 break; 1788 1789 case DMAR_FRCD_REG_0_3: 1790 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64 1791 ", size %d, val 0x%"PRIx64, addr, size, val); 1792 assert(size == 4); 1793 vtd_set_long(s, addr, val); 1794 /* May clear bit 127 (Fault), update PPF */ 1795 vtd_update_fsts_ppf(s); 1796 break; 1797 1798 default: 1799 VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64 1800 ", size %d, val 0x%"PRIx64, addr, size, val); 1801 if (size == 4) { 1802 vtd_set_long(s, addr, val); 1803 } else { 1804 vtd_set_quad(s, addr, val); 1805 } 1806 } 1807 } 1808 1809 static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr, 1810 bool is_write) 1811 { 1812 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 1813 IntelIOMMUState *s = vtd_as->iommu_state; 1814 IOMMUTLBEntry ret = { 1815 .target_as = &address_space_memory, 1816 .iova = addr, 1817 .translated_addr = 0, 1818 .addr_mask = ~(hwaddr)0, 1819 .perm = IOMMU_NONE, 1820 }; 1821 1822 if (!s->dmar_enabled) { 1823 /* DMAR disabled, passthrough, use 4k-page*/ 1824 ret.iova = addr & VTD_PAGE_MASK_4K; 1825 ret.translated_addr = addr & VTD_PAGE_MASK_4K; 1826 ret.addr_mask = ~VTD_PAGE_MASK_4K; 1827 ret.perm = IOMMU_RW; 1828 return ret; 1829 } 1830 1831 vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr, 1832 is_write, &ret); 1833 VTD_DPRINTF(MMU, 1834 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8 1835 " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus), 1836 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn), 1837 vtd_as->devfn, addr, ret.translated_addr); 1838 return ret; 1839 } 1840 1841 static const VMStateDescription vtd_vmstate = { 1842 .name = "iommu-intel", 1843 .unmigratable = 1, 1844 }; 1845 1846 static const MemoryRegionOps vtd_mem_ops = { 1847 .read = vtd_mem_read, 1848 .write = vtd_mem_write, 1849 .endianness = DEVICE_LITTLE_ENDIAN, 1850 .impl = { 1851 .min_access_size = 4, 1852 .max_access_size = 8, 1853 }, 1854 .valid = { 1855 .min_access_size = 4, 1856 .max_access_size = 8, 1857 }, 1858 }; 1859 1860 static Property vtd_properties[] = { 1861 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 1862 DEFINE_PROP_END_OF_LIST(), 1863 }; 1864 1865 1866 VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) 1867 { 1868 uintptr_t key = (uintptr_t)bus; 1869 VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key); 1870 VTDAddressSpace *vtd_dev_as; 1871 1872 if (!vtd_bus) { 1873 /* No corresponding free() */ 1874 vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * VTD_PCI_DEVFN_MAX); 1875 vtd_bus->bus = bus; 1876 key = (uintptr_t)bus; 1877 g_hash_table_insert(s->vtd_as_by_busptr, &key, vtd_bus); 1878 } 1879 1880 vtd_dev_as = vtd_bus->dev_as[devfn]; 1881 1882 if (!vtd_dev_as) { 1883 vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace)); 1884 1885 vtd_dev_as->bus = bus; 1886 vtd_dev_as->devfn = (uint8_t)devfn; 1887 vtd_dev_as->iommu_state = s; 1888 vtd_dev_as->context_cache_entry.context_cache_gen = 0; 1889 memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s), 1890 &s->iommu_ops, "intel_iommu", UINT64_MAX); 1891 address_space_init(&vtd_dev_as->as, 1892 &vtd_dev_as->iommu, "intel_iommu"); 1893 } 1894 return vtd_dev_as; 1895 } 1896 1897 /* Do the initialization. It will also be called when reset, so pay 1898 * attention when adding new initialization stuff. 1899 */ 1900 static void vtd_init(IntelIOMMUState *s) 1901 { 1902 memset(s->csr, 0, DMAR_REG_SIZE); 1903 memset(s->wmask, 0, DMAR_REG_SIZE); 1904 memset(s->w1cmask, 0, DMAR_REG_SIZE); 1905 memset(s->womask, 0, DMAR_REG_SIZE); 1906 1907 s->iommu_ops.translate = vtd_iommu_translate; 1908 s->root = 0; 1909 s->root_extended = false; 1910 s->dmar_enabled = false; 1911 s->iq_head = 0; 1912 s->iq_tail = 0; 1913 s->iq = 0; 1914 s->iq_size = 0; 1915 s->qi_enabled = false; 1916 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1917 s->next_frcd_reg = 0; 1918 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW | 1919 VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI; 1920 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 1921 1922 vtd_reset_context_cache(s); 1923 vtd_reset_iotlb(s); 1924 1925 /* Define registers with default values and bit semantics */ 1926 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 1927 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 1928 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 1929 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 1930 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 1931 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 1932 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0); 1933 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 1934 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 1935 1936 /* Advanced Fault Logging not supported */ 1937 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 1938 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 1939 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 1940 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 1941 1942 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 1943 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 1944 */ 1945 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 1946 1947 /* Treated as RO for implementations that PLMR and PHMR fields reported 1948 * as Clear in the CAP_REG. 1949 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 1950 */ 1951 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 1952 1953 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 1954 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 1955 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0); 1956 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 1957 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 1958 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 1959 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 1960 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 1961 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 1962 1963 /* IOTLB registers */ 1964 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 1965 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 1966 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 1967 1968 /* Fault Recording Registers, 128-bit */ 1969 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 1970 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 1971 } 1972 1973 /* Should not reset address_spaces when reset because devices will still use 1974 * the address space they got at first (won't ask the bus again). 1975 */ 1976 static void vtd_reset(DeviceState *dev) 1977 { 1978 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 1979 1980 VTD_DPRINTF(GENERAL, ""); 1981 vtd_init(s); 1982 } 1983 1984 static void vtd_realize(DeviceState *dev, Error **errp) 1985 { 1986 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 1987 1988 VTD_DPRINTF(GENERAL, ""); 1989 memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num)); 1990 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 1991 "intel_iommu", DMAR_REG_SIZE); 1992 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem); 1993 /* No corresponding destroy */ 1994 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 1995 g_free, g_free); 1996 s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 1997 g_free, g_free); 1998 vtd_init(s); 1999 } 2000 2001 static void vtd_class_init(ObjectClass *klass, void *data) 2002 { 2003 DeviceClass *dc = DEVICE_CLASS(klass); 2004 2005 dc->reset = vtd_reset; 2006 dc->realize = vtd_realize; 2007 dc->vmsd = &vtd_vmstate; 2008 dc->props = vtd_properties; 2009 } 2010 2011 static const TypeInfo vtd_info = { 2012 .name = TYPE_INTEL_IOMMU_DEVICE, 2013 .parent = TYPE_SYS_BUS_DEVICE, 2014 .instance_size = sizeof(IntelIOMMUState), 2015 .class_init = vtd_class_init, 2016 }; 2017 2018 static void vtd_register_types(void) 2019 { 2020 VTD_DPRINTF(GENERAL, ""); 2021 type_register_static(&vtd_info); 2022 } 2023 2024 type_init(vtd_register_types) 2025