1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "hw/sysbus.h" 23 #include "exec/address-spaces.h" 24 #include "intel_iommu_internal.h" 25 26 /*#define DEBUG_INTEL_IOMMU*/ 27 #ifdef DEBUG_INTEL_IOMMU 28 enum { 29 DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG, 30 DEBUG_CACHE, 31 }; 32 #define VTD_DBGBIT(x) (1 << DEBUG_##x) 33 static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR); 34 35 #define VTD_DPRINTF(what, fmt, ...) do { \ 36 if (vtd_dbgflags & VTD_DBGBIT(what)) { \ 37 fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \ 38 ## __VA_ARGS__); } \ 39 } while (0) 40 #else 41 #define VTD_DPRINTF(what, fmt, ...) do {} while (0) 42 #endif 43 44 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 45 uint64_t wmask, uint64_t w1cmask) 46 { 47 stq_le_p(&s->csr[addr], val); 48 stq_le_p(&s->wmask[addr], wmask); 49 stq_le_p(&s->w1cmask[addr], w1cmask); 50 } 51 52 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 53 { 54 stq_le_p(&s->womask[addr], mask); 55 } 56 57 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 58 uint32_t wmask, uint32_t w1cmask) 59 { 60 stl_le_p(&s->csr[addr], val); 61 stl_le_p(&s->wmask[addr], wmask); 62 stl_le_p(&s->w1cmask[addr], w1cmask); 63 } 64 65 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 66 { 67 stl_le_p(&s->womask[addr], mask); 68 } 69 70 /* "External" get/set operations */ 71 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 72 { 73 uint64_t oldval = ldq_le_p(&s->csr[addr]); 74 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 75 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 76 stq_le_p(&s->csr[addr], 77 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 78 } 79 80 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 81 { 82 uint32_t oldval = ldl_le_p(&s->csr[addr]); 83 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 84 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 85 stl_le_p(&s->csr[addr], 86 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 87 } 88 89 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 90 { 91 uint64_t val = ldq_le_p(&s->csr[addr]); 92 uint64_t womask = ldq_le_p(&s->womask[addr]); 93 return val & ~womask; 94 } 95 96 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 97 { 98 uint32_t val = ldl_le_p(&s->csr[addr]); 99 uint32_t womask = ldl_le_p(&s->womask[addr]); 100 return val & ~womask; 101 } 102 103 /* "Internal" get/set operations */ 104 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 105 { 106 return ldq_le_p(&s->csr[addr]); 107 } 108 109 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 110 { 111 return ldl_le_p(&s->csr[addr]); 112 } 113 114 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 115 { 116 stq_le_p(&s->csr[addr], val); 117 } 118 119 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 120 uint32_t clear, uint32_t mask) 121 { 122 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 123 stl_le_p(&s->csr[addr], new_val); 124 return new_val; 125 } 126 127 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 128 uint64_t clear, uint64_t mask) 129 { 130 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 131 stq_le_p(&s->csr[addr], new_val); 132 return new_val; 133 } 134 135 /* GHashTable functions */ 136 static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2) 137 { 138 return *((const uint64_t *)v1) == *((const uint64_t *)v2); 139 } 140 141 static guint vtd_uint64_hash(gconstpointer v) 142 { 143 return (guint)*(const uint64_t *)v; 144 } 145 146 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 147 gpointer user_data) 148 { 149 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 150 uint16_t domain_id = *(uint16_t *)user_data; 151 return entry->domain_id == domain_id; 152 } 153 154 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 155 gpointer user_data) 156 { 157 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 158 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 159 uint64_t gfn = info->gfn & info->mask; 160 return (entry->domain_id == info->domain_id) && 161 ((entry->gfn & info->mask) == gfn); 162 } 163 164 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 165 * IntelIOMMUState to 1. 166 */ 167 static void vtd_reset_context_cache(IntelIOMMUState *s) 168 { 169 VTDAddressSpace **pvtd_as; 170 VTDAddressSpace *vtd_as; 171 uint32_t bus_it; 172 uint32_t devfn_it; 173 174 VTD_DPRINTF(CACHE, "global context_cache_gen=1"); 175 for (bus_it = 0; bus_it < VTD_PCI_BUS_MAX; ++bus_it) { 176 pvtd_as = s->address_spaces[bus_it]; 177 if (!pvtd_as) { 178 continue; 179 } 180 for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) { 181 vtd_as = pvtd_as[devfn_it]; 182 if (!vtd_as) { 183 continue; 184 } 185 vtd_as->context_cache_entry.context_cache_gen = 0; 186 } 187 } 188 s->context_cache_gen = 1; 189 } 190 191 static void vtd_reset_iotlb(IntelIOMMUState *s) 192 { 193 assert(s->iotlb); 194 g_hash_table_remove_all(s->iotlb); 195 } 196 197 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 198 hwaddr addr) 199 { 200 uint64_t key; 201 202 key = (addr >> VTD_PAGE_SHIFT_4K) | 203 ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT); 204 return g_hash_table_lookup(s->iotlb, &key); 205 206 } 207 208 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 209 uint16_t domain_id, hwaddr addr, uint64_t slpte, 210 bool read_flags, bool write_flags) 211 { 212 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 213 uint64_t *key = g_malloc(sizeof(*key)); 214 uint64_t gfn = addr >> VTD_PAGE_SHIFT_4K; 215 216 VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64 217 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte, 218 domain_id); 219 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 220 VTD_DPRINTF(CACHE, "iotlb exceeds size limit, forced to reset"); 221 vtd_reset_iotlb(s); 222 } 223 224 entry->gfn = gfn; 225 entry->domain_id = domain_id; 226 entry->slpte = slpte; 227 entry->read_flags = read_flags; 228 entry->write_flags = write_flags; 229 *key = gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT); 230 g_hash_table_replace(s->iotlb, key, entry); 231 } 232 233 /* Given the reg addr of both the message data and address, generate an 234 * interrupt via MSI. 235 */ 236 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 237 hwaddr mesg_data_reg) 238 { 239 hwaddr addr; 240 uint32_t data; 241 242 assert(mesg_data_reg < DMAR_REG_SIZE); 243 assert(mesg_addr_reg < DMAR_REG_SIZE); 244 245 addr = vtd_get_long_raw(s, mesg_addr_reg); 246 data = vtd_get_long_raw(s, mesg_data_reg); 247 248 VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32, addr, data); 249 stl_le_phys(&address_space_memory, addr, data); 250 } 251 252 /* Generate a fault event to software via MSI if conditions are met. 253 * Notice that the value of FSTS_REG being passed to it should be the one 254 * before any update. 255 */ 256 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 257 { 258 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 259 pre_fsts & VTD_FSTS_IQE) { 260 VTD_DPRINTF(FLOG, "there are previous interrupt conditions " 261 "to be serviced by software, fault event is not generated " 262 "(FSTS_REG 0x%"PRIx32 ")", pre_fsts); 263 return; 264 } 265 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 266 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 267 VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated"); 268 } else { 269 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 270 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 271 } 272 } 273 274 /* Check if the Fault (F) field of the Fault Recording Register referenced by 275 * @index is Set. 276 */ 277 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 278 { 279 /* Each reg is 128-bit */ 280 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 281 addr += 8; /* Access the high 64-bit half */ 282 283 assert(index < DMAR_FRCD_REG_NR); 284 285 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 286 } 287 288 /* Update the PPF field of Fault Status Register. 289 * Should be called whenever change the F field of any fault recording 290 * registers. 291 */ 292 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 293 { 294 uint32_t i; 295 uint32_t ppf_mask = 0; 296 297 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 298 if (vtd_is_frcd_set(s, i)) { 299 ppf_mask = VTD_FSTS_PPF; 300 break; 301 } 302 } 303 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 304 VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0); 305 } 306 307 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 308 { 309 /* Each reg is 128-bit */ 310 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 311 addr += 8; /* Access the high 64-bit half */ 312 313 assert(index < DMAR_FRCD_REG_NR); 314 315 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 316 vtd_update_fsts_ppf(s); 317 } 318 319 /* Must not update F field now, should be done later */ 320 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 321 uint16_t source_id, hwaddr addr, 322 VTDFaultReason fault, bool is_write) 323 { 324 uint64_t hi = 0, lo; 325 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 326 327 assert(index < DMAR_FRCD_REG_NR); 328 329 lo = VTD_FRCD_FI(addr); 330 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 331 if (!is_write) { 332 hi |= VTD_FRCD_T; 333 } 334 vtd_set_quad_raw(s, frcd_reg_addr, lo); 335 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 336 VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64 337 ", lo 0x%"PRIx64, index, hi, lo); 338 } 339 340 /* Try to collapse multiple pending faults from the same requester */ 341 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 342 { 343 uint32_t i; 344 uint64_t frcd_reg; 345 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 346 347 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 348 frcd_reg = vtd_get_quad_raw(s, addr); 349 VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg); 350 if ((frcd_reg & VTD_FRCD_F) && 351 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 352 return true; 353 } 354 addr += 16; /* 128-bit for each */ 355 } 356 return false; 357 } 358 359 /* Log and report an DMAR (address translation) fault to software */ 360 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 361 hwaddr addr, VTDFaultReason fault, 362 bool is_write) 363 { 364 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 365 366 assert(fault < VTD_FR_MAX); 367 368 if (fault == VTD_FR_RESERVED_ERR) { 369 /* This is not a normal fault reason case. Drop it. */ 370 return; 371 } 372 VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64 373 ", is_write %d", source_id, fault, addr, is_write); 374 if (fsts_reg & VTD_FSTS_PFO) { 375 VTD_DPRINTF(FLOG, "new fault is not recorded due to " 376 "Primary Fault Overflow"); 377 return; 378 } 379 if (vtd_try_collapse_fault(s, source_id)) { 380 VTD_DPRINTF(FLOG, "new fault is not recorded due to " 381 "compression of faults"); 382 return; 383 } 384 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 385 VTD_DPRINTF(FLOG, "Primary Fault Overflow and " 386 "new fault is not recorded, set PFO field"); 387 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 388 return; 389 } 390 391 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write); 392 393 if (fsts_reg & VTD_FSTS_PPF) { 394 VTD_DPRINTF(FLOG, "there are pending faults already, " 395 "fault event is not generated"); 396 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 397 s->next_frcd_reg++; 398 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 399 s->next_frcd_reg = 0; 400 } 401 } else { 402 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 403 VTD_FSTS_FRI(s->next_frcd_reg)); 404 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 405 s->next_frcd_reg++; 406 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 407 s->next_frcd_reg = 0; 408 } 409 /* This case actually cause the PPF to be Set. 410 * So generate fault event (interrupt). 411 */ 412 vtd_generate_fault_event(s, fsts_reg); 413 } 414 } 415 416 /* Handle Invalidation Queue Errors of queued invalidation interface error 417 * conditions. 418 */ 419 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 420 { 421 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 422 423 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 424 vtd_generate_fault_event(s, fsts_reg); 425 } 426 427 /* Set the IWC field and try to generate an invalidation completion interrupt */ 428 static void vtd_generate_completion_event(IntelIOMMUState *s) 429 { 430 VTD_DPRINTF(INV, "completes an invalidation wait command with " 431 "Interrupt Flag"); 432 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 433 VTD_DPRINTF(INV, "there is a previous interrupt condition to be " 434 "serviced by software, " 435 "new invalidation event is not generated"); 436 return; 437 } 438 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 439 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 440 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 441 VTD_DPRINTF(INV, "IM filed in IECTL_REG is set, new invalidation " 442 "event is not generated"); 443 return; 444 } else { 445 /* Generate the interrupt event */ 446 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 447 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 448 } 449 } 450 451 static inline bool vtd_root_entry_present(VTDRootEntry *root) 452 { 453 return root->val & VTD_ROOT_ENTRY_P; 454 } 455 456 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 457 VTDRootEntry *re) 458 { 459 dma_addr_t addr; 460 461 addr = s->root + index * sizeof(*re); 462 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) { 463 VTD_DPRINTF(GENERAL, "error: fail to access root-entry at 0x%"PRIx64 464 " + %"PRIu8, s->root, index); 465 re->val = 0; 466 return -VTD_FR_ROOT_TABLE_INV; 467 } 468 re->val = le64_to_cpu(re->val); 469 return 0; 470 } 471 472 static inline bool vtd_context_entry_present(VTDContextEntry *context) 473 { 474 return context->lo & VTD_CONTEXT_ENTRY_P; 475 } 476 477 static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index, 478 VTDContextEntry *ce) 479 { 480 dma_addr_t addr; 481 482 if (!vtd_root_entry_present(root)) { 483 VTD_DPRINTF(GENERAL, "error: root-entry is not present"); 484 return -VTD_FR_ROOT_ENTRY_P; 485 } 486 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce); 487 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) { 488 VTD_DPRINTF(GENERAL, "error: fail to access context-entry at 0x%"PRIx64 489 " + %"PRIu8, 490 (uint64_t)(root->val & VTD_ROOT_ENTRY_CTP), index); 491 return -VTD_FR_CONTEXT_TABLE_INV; 492 } 493 ce->lo = le64_to_cpu(ce->lo); 494 ce->hi = le64_to_cpu(ce->hi); 495 return 0; 496 } 497 498 static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce) 499 { 500 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 501 } 502 503 /* The shift of an addr for a certain level of paging structure */ 504 static inline uint32_t vtd_slpt_level_shift(uint32_t level) 505 { 506 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 507 } 508 509 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte) 510 { 511 return slpte & VTD_SL_PT_BASE_ADDR_MASK; 512 } 513 514 /* Whether the pte indicates the address of the page frame */ 515 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 516 { 517 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 518 } 519 520 /* Get the content of a spte located in @base_addr[@index] */ 521 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 522 { 523 uint64_t slpte; 524 525 assert(index < VTD_SL_PT_ENTRY_NR); 526 527 if (dma_memory_read(&address_space_memory, 528 base_addr + index * sizeof(slpte), &slpte, 529 sizeof(slpte))) { 530 slpte = (uint64_t)-1; 531 return slpte; 532 } 533 slpte = le64_to_cpu(slpte); 534 return slpte; 535 } 536 537 /* Given a gpa and the level of paging structure, return the offset of current 538 * level. 539 */ 540 static inline uint32_t vtd_gpa_level_offset(uint64_t gpa, uint32_t level) 541 { 542 return (gpa >> vtd_slpt_level_shift(level)) & 543 ((1ULL << VTD_SL_LEVEL_BITS) - 1); 544 } 545 546 /* Check Capability Register to see if the @level of page-table is supported */ 547 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 548 { 549 return VTD_CAP_SAGAW_MASK & s->cap & 550 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 551 } 552 553 /* Get the page-table level that hardware should use for the second-level 554 * page-table walk from the Address Width field of context-entry. 555 */ 556 static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce) 557 { 558 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 559 } 560 561 static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce) 562 { 563 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 564 } 565 566 static const uint64_t vtd_paging_entry_rsvd_field[] = { 567 [0] = ~0ULL, 568 /* For not large page */ 569 [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 570 [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 571 [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 572 [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 573 /* For large page */ 574 [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 575 [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 576 [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 577 [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 578 }; 579 580 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 581 { 582 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) { 583 /* Maybe large page */ 584 return slpte & vtd_paging_entry_rsvd_field[level + 4]; 585 } else { 586 return slpte & vtd_paging_entry_rsvd_field[level]; 587 } 588 } 589 590 /* Given the @gpa, get relevant @slptep. @slpte_level will be the last level 591 * of the translation, can be used for deciding the size of large page. 592 */ 593 static int vtd_gpa_to_slpte(VTDContextEntry *ce, uint64_t gpa, bool is_write, 594 uint64_t *slptep, uint32_t *slpte_level, 595 bool *reads, bool *writes) 596 { 597 dma_addr_t addr = vtd_get_slpt_base_from_context(ce); 598 uint32_t level = vtd_get_level_from_context_entry(ce); 599 uint32_t offset; 600 uint64_t slpte; 601 uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce); 602 uint64_t access_right_check; 603 604 /* Check if @gpa is above 2^X-1, where X is the minimum of MGAW in CAP_REG 605 * and AW in context-entry. 606 */ 607 if (gpa & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) { 608 VTD_DPRINTF(GENERAL, "error: gpa 0x%"PRIx64 " exceeds limits", gpa); 609 return -VTD_FR_ADDR_BEYOND_MGAW; 610 } 611 612 /* FIXME: what is the Atomics request here? */ 613 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 614 615 while (true) { 616 offset = vtd_gpa_level_offset(gpa, level); 617 slpte = vtd_get_slpte(addr, offset); 618 619 if (slpte == (uint64_t)-1) { 620 VTD_DPRINTF(GENERAL, "error: fail to access second-level paging " 621 "entry at level %"PRIu32 " for gpa 0x%"PRIx64, 622 level, gpa); 623 if (level == vtd_get_level_from_context_entry(ce)) { 624 /* Invalid programming of context-entry */ 625 return -VTD_FR_CONTEXT_ENTRY_INV; 626 } else { 627 return -VTD_FR_PAGING_ENTRY_INV; 628 } 629 } 630 *reads = (*reads) && (slpte & VTD_SL_R); 631 *writes = (*writes) && (slpte & VTD_SL_W); 632 if (!(slpte & access_right_check)) { 633 VTD_DPRINTF(GENERAL, "error: lack of %s permission for " 634 "gpa 0x%"PRIx64 " slpte 0x%"PRIx64, 635 (is_write ? "write" : "read"), gpa, slpte); 636 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 637 } 638 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 639 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second " 640 "level paging entry level %"PRIu32 " slpte 0x%"PRIx64, 641 level, slpte); 642 return -VTD_FR_PAGING_ENTRY_RSVD; 643 } 644 645 if (vtd_is_last_slpte(slpte, level)) { 646 *slptep = slpte; 647 *slpte_level = level; 648 return 0; 649 } 650 addr = vtd_get_slpte_addr(slpte); 651 level--; 652 } 653 } 654 655 /* Map a device to its corresponding domain (context-entry) */ 656 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 657 uint8_t devfn, VTDContextEntry *ce) 658 { 659 VTDRootEntry re; 660 int ret_fr; 661 662 ret_fr = vtd_get_root_entry(s, bus_num, &re); 663 if (ret_fr) { 664 return ret_fr; 665 } 666 667 if (!vtd_root_entry_present(&re)) { 668 VTD_DPRINTF(GENERAL, "error: root-entry #%"PRIu8 " is not present", 669 bus_num); 670 return -VTD_FR_ROOT_ENTRY_P; 671 } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) { 672 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in root-entry " 673 "hi 0x%"PRIx64 " lo 0x%"PRIx64, re.rsvd, re.val); 674 return -VTD_FR_ROOT_ENTRY_RSVD; 675 } 676 677 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce); 678 if (ret_fr) { 679 return ret_fr; 680 } 681 682 if (!vtd_context_entry_present(ce)) { 683 VTD_DPRINTF(GENERAL, 684 "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") " 685 "is not present", devfn, bus_num); 686 return -VTD_FR_CONTEXT_ENTRY_P; 687 } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) || 688 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) { 689 VTD_DPRINTF(GENERAL, 690 "error: non-zero reserved field in context-entry " 691 "hi 0x%"PRIx64 " lo 0x%"PRIx64, ce->hi, ce->lo); 692 return -VTD_FR_CONTEXT_ENTRY_RSVD; 693 } 694 /* Check if the programming of context-entry is valid */ 695 if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) { 696 VTD_DPRINTF(GENERAL, "error: unsupported Address Width value in " 697 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64, 698 ce->hi, ce->lo); 699 return -VTD_FR_CONTEXT_ENTRY_INV; 700 } else if (ce->lo & VTD_CONTEXT_ENTRY_TT) { 701 VTD_DPRINTF(GENERAL, "error: unsupported Translation Type in " 702 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64, 703 ce->hi, ce->lo); 704 return -VTD_FR_CONTEXT_ENTRY_INV; 705 } 706 return 0; 707 } 708 709 static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn) 710 { 711 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL); 712 } 713 714 static const bool vtd_qualified_faults[] = { 715 [VTD_FR_RESERVED] = false, 716 [VTD_FR_ROOT_ENTRY_P] = false, 717 [VTD_FR_CONTEXT_ENTRY_P] = true, 718 [VTD_FR_CONTEXT_ENTRY_INV] = true, 719 [VTD_FR_ADDR_BEYOND_MGAW] = true, 720 [VTD_FR_WRITE] = true, 721 [VTD_FR_READ] = true, 722 [VTD_FR_PAGING_ENTRY_INV] = true, 723 [VTD_FR_ROOT_TABLE_INV] = false, 724 [VTD_FR_CONTEXT_TABLE_INV] = false, 725 [VTD_FR_ROOT_ENTRY_RSVD] = false, 726 [VTD_FR_PAGING_ENTRY_RSVD] = true, 727 [VTD_FR_CONTEXT_ENTRY_TT] = true, 728 [VTD_FR_RESERVED_ERR] = false, 729 [VTD_FR_MAX] = false, 730 }; 731 732 /* To see if a fault condition is "qualified", which is reported to software 733 * only if the FPD field in the context-entry used to process the faulting 734 * request is 0. 735 */ 736 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 737 { 738 return vtd_qualified_faults[fault]; 739 } 740 741 static inline bool vtd_is_interrupt_addr(hwaddr addr) 742 { 743 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 744 } 745 746 /* Map dev to context-entry then do a paging-structures walk to do a iommu 747 * translation. 748 * 749 * Called from RCU critical section. 750 * 751 * @bus_num: The bus number 752 * @devfn: The devfn, which is the combined of device and function number 753 * @is_write: The access is a write operation 754 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 755 */ 756 static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num, 757 uint8_t devfn, hwaddr addr, bool is_write, 758 IOMMUTLBEntry *entry) 759 { 760 IntelIOMMUState *s = vtd_as->iommu_state; 761 VTDContextEntry ce; 762 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry; 763 uint64_t slpte; 764 uint32_t level; 765 uint16_t source_id = vtd_make_source_id(bus_num, devfn); 766 int ret_fr; 767 bool is_fpd_set = false; 768 bool reads = true; 769 bool writes = true; 770 VTDIOTLBEntry *iotlb_entry; 771 772 /* Check if the request is in interrupt address range */ 773 if (vtd_is_interrupt_addr(addr)) { 774 if (is_write) { 775 /* FIXME: since we don't know the length of the access here, we 776 * treat Non-DWORD length write requests without PASID as 777 * interrupt requests, too. Withoud interrupt remapping support, 778 * we just use 1:1 mapping. 779 */ 780 VTD_DPRINTF(MMU, "write request to interrupt address " 781 "gpa 0x%"PRIx64, addr); 782 entry->iova = addr & VTD_PAGE_MASK_4K; 783 entry->translated_addr = addr & VTD_PAGE_MASK_4K; 784 entry->addr_mask = ~VTD_PAGE_MASK_4K; 785 entry->perm = IOMMU_WO; 786 return; 787 } else { 788 VTD_DPRINTF(GENERAL, "error: read request from interrupt address " 789 "gpa 0x%"PRIx64, addr); 790 vtd_report_dmar_fault(s, source_id, addr, VTD_FR_READ, is_write); 791 return; 792 } 793 } 794 /* Try to fetch slpte form IOTLB */ 795 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr); 796 if (iotlb_entry) { 797 VTD_DPRINTF(CACHE, "hit iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64 798 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, 799 iotlb_entry->slpte, iotlb_entry->domain_id); 800 slpte = iotlb_entry->slpte; 801 reads = iotlb_entry->read_flags; 802 writes = iotlb_entry->write_flags; 803 goto out; 804 } 805 /* Try to fetch context-entry from cache first */ 806 if (cc_entry->context_cache_gen == s->context_cache_gen) { 807 VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d " 808 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")", 809 bus_num, devfn, cc_entry->context_entry.hi, 810 cc_entry->context_entry.lo, cc_entry->context_cache_gen); 811 ce = cc_entry->context_entry; 812 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 813 } else { 814 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 815 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 816 if (ret_fr) { 817 ret_fr = -ret_fr; 818 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 819 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA " 820 "requests through this context-entry " 821 "(with FPD Set)"); 822 } else { 823 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 824 } 825 return; 826 } 827 /* Update context-cache */ 828 VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d " 829 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")", 830 bus_num, devfn, ce.hi, ce.lo, 831 cc_entry->context_cache_gen, s->context_cache_gen); 832 cc_entry->context_entry = ce; 833 cc_entry->context_cache_gen = s->context_cache_gen; 834 } 835 836 ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level, 837 &reads, &writes); 838 if (ret_fr) { 839 ret_fr = -ret_fr; 840 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 841 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests " 842 "through this context-entry (with FPD Set)"); 843 } else { 844 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 845 } 846 return; 847 } 848 849 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte, 850 reads, writes); 851 out: 852 entry->iova = addr & VTD_PAGE_MASK_4K; 853 entry->translated_addr = vtd_get_slpte_addr(slpte) & VTD_PAGE_MASK_4K; 854 entry->addr_mask = ~VTD_PAGE_MASK_4K; 855 entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0); 856 } 857 858 static void vtd_root_table_setup(IntelIOMMUState *s) 859 { 860 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 861 s->root_extended = s->root & VTD_RTADDR_RTT; 862 s->root &= VTD_RTADDR_ADDR_MASK; 863 864 VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root, 865 (s->root_extended ? "(extended)" : "")); 866 } 867 868 static void vtd_context_global_invalidate(IntelIOMMUState *s) 869 { 870 s->context_cache_gen++; 871 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 872 vtd_reset_context_cache(s); 873 } 874 } 875 876 /* Do a context-cache device-selective invalidation. 877 * @func_mask: FM field after shifting 878 */ 879 static void vtd_context_device_invalidate(IntelIOMMUState *s, 880 uint16_t source_id, 881 uint16_t func_mask) 882 { 883 uint16_t mask; 884 VTDAddressSpace **pvtd_as; 885 VTDAddressSpace *vtd_as; 886 uint16_t devfn; 887 uint16_t devfn_it; 888 889 switch (func_mask & 3) { 890 case 0: 891 mask = 0; /* No bits in the SID field masked */ 892 break; 893 case 1: 894 mask = 4; /* Mask bit 2 in the SID field */ 895 break; 896 case 2: 897 mask = 6; /* Mask bit 2:1 in the SID field */ 898 break; 899 case 3: 900 mask = 7; /* Mask bit 2:0 in the SID field */ 901 break; 902 } 903 VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16 904 " mask %"PRIu16, source_id, mask); 905 pvtd_as = s->address_spaces[VTD_SID_TO_BUS(source_id)]; 906 if (pvtd_as) { 907 devfn = VTD_SID_TO_DEVFN(source_id); 908 for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) { 909 vtd_as = pvtd_as[devfn_it]; 910 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) { 911 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16, 912 devfn_it); 913 vtd_as->context_cache_entry.context_cache_gen = 0; 914 } 915 } 916 } 917 } 918 919 /* Context-cache invalidation 920 * Returns the Context Actual Invalidation Granularity. 921 * @val: the content of the CCMD_REG 922 */ 923 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 924 { 925 uint64_t caig; 926 uint64_t type = val & VTD_CCMD_CIRG_MASK; 927 928 switch (type) { 929 case VTD_CCMD_DOMAIN_INVL: 930 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 931 (uint16_t)VTD_CCMD_DID(val)); 932 /* Fall through */ 933 case VTD_CCMD_GLOBAL_INVL: 934 VTD_DPRINTF(INV, "global invalidation"); 935 caig = VTD_CCMD_GLOBAL_INVL_A; 936 vtd_context_global_invalidate(s); 937 break; 938 939 case VTD_CCMD_DEVICE_INVL: 940 caig = VTD_CCMD_DEVICE_INVL_A; 941 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 942 break; 943 944 default: 945 VTD_DPRINTF(GENERAL, "error: invalid granularity"); 946 caig = 0; 947 } 948 return caig; 949 } 950 951 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 952 { 953 vtd_reset_iotlb(s); 954 } 955 956 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 957 { 958 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 959 &domain_id); 960 } 961 962 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 963 hwaddr addr, uint8_t am) 964 { 965 VTDIOTLBPageInvInfo info; 966 967 assert(am <= VTD_MAMV); 968 info.domain_id = domain_id; 969 info.gfn = addr >> VTD_PAGE_SHIFT_4K; 970 info.mask = ~((1 << am) - 1); 971 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 972 } 973 974 /* Flush IOTLB 975 * Returns the IOTLB Actual Invalidation Granularity. 976 * @val: the content of the IOTLB_REG 977 */ 978 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 979 { 980 uint64_t iaig; 981 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 982 uint16_t domain_id; 983 hwaddr addr; 984 uint8_t am; 985 986 switch (type) { 987 case VTD_TLB_GLOBAL_FLUSH: 988 VTD_DPRINTF(INV, "global invalidation"); 989 iaig = VTD_TLB_GLOBAL_FLUSH_A; 990 vtd_iotlb_global_invalidate(s); 991 break; 992 993 case VTD_TLB_DSI_FLUSH: 994 domain_id = VTD_TLB_DID(val); 995 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 996 domain_id); 997 iaig = VTD_TLB_DSI_FLUSH_A; 998 vtd_iotlb_domain_invalidate(s, domain_id); 999 break; 1000 1001 case VTD_TLB_PSI_FLUSH: 1002 domain_id = VTD_TLB_DID(val); 1003 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 1004 am = VTD_IVA_AM(addr); 1005 addr = VTD_IVA_ADDR(addr); 1006 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16 1007 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am); 1008 if (am > VTD_MAMV) { 1009 VTD_DPRINTF(GENERAL, "error: supported max address mask value is " 1010 "%"PRIu8, (uint8_t)VTD_MAMV); 1011 iaig = 0; 1012 break; 1013 } 1014 iaig = VTD_TLB_PSI_FLUSH_A; 1015 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1016 break; 1017 1018 default: 1019 VTD_DPRINTF(GENERAL, "error: invalid granularity"); 1020 iaig = 0; 1021 } 1022 return iaig; 1023 } 1024 1025 static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s) 1026 { 1027 return s->iq_tail == 0; 1028 } 1029 1030 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 1031 { 1032 return s->qi_enabled && (s->iq_tail == s->iq_head) && 1033 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 1034 } 1035 1036 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 1037 { 1038 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 1039 1040 VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off")); 1041 if (en) { 1042 if (vtd_queued_inv_enable_check(s)) { 1043 s->iq = iqa_val & VTD_IQA_IQA_MASK; 1044 /* 2^(x+8) entries */ 1045 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8); 1046 s->qi_enabled = true; 1047 VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val); 1048 VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d", 1049 s->iq, s->iq_size); 1050 /* Ok - report back to driver */ 1051 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 1052 } else { 1053 VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: " 1054 "tail %"PRIu16, s->iq_tail); 1055 } 1056 } else { 1057 if (vtd_queued_inv_disable_check(s)) { 1058 /* disable Queued Invalidation */ 1059 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 1060 s->iq_head = 0; 1061 s->qi_enabled = false; 1062 /* Ok - report back to driver */ 1063 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 1064 } else { 1065 VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: " 1066 "head %"PRIu16 ", tail %"PRIu16 1067 ", last_descriptor %"PRIu8, 1068 s->iq_head, s->iq_tail, s->iq_last_desc_type); 1069 } 1070 } 1071 } 1072 1073 /* Set Root Table Pointer */ 1074 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 1075 { 1076 VTD_DPRINTF(CSR, "set Root Table Pointer"); 1077 1078 vtd_root_table_setup(s); 1079 /* Ok - report back to driver */ 1080 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 1081 } 1082 1083 /* Handle Translation Enable/Disable */ 1084 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 1085 { 1086 VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off")); 1087 1088 if (en) { 1089 s->dmar_enabled = true; 1090 /* Ok - report back to driver */ 1091 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 1092 } else { 1093 s->dmar_enabled = false; 1094 1095 /* Clear the index of Fault Recording Register */ 1096 s->next_frcd_reg = 0; 1097 /* Ok - report back to driver */ 1098 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 1099 } 1100 } 1101 1102 /* Handle write to Global Command Register */ 1103 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 1104 { 1105 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 1106 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 1107 uint32_t changed = status ^ val; 1108 1109 VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status); 1110 if (changed & VTD_GCMD_TE) { 1111 /* Translation enable/disable */ 1112 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 1113 } 1114 if (val & VTD_GCMD_SRTP) { 1115 /* Set/update the root-table pointer */ 1116 vtd_handle_gcmd_srtp(s); 1117 } 1118 if (changed & VTD_GCMD_QIE) { 1119 /* Queued Invalidation Enable */ 1120 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 1121 } 1122 } 1123 1124 /* Handle write to Context Command Register */ 1125 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 1126 { 1127 uint64_t ret; 1128 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 1129 1130 /* Context-cache invalidation request */ 1131 if (val & VTD_CCMD_ICC) { 1132 if (s->qi_enabled) { 1133 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, " 1134 "should not use register-based invalidation"); 1135 return; 1136 } 1137 ret = vtd_context_cache_invalidate(s, val); 1138 /* Invalidation completed. Change something to show */ 1139 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 1140 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 1141 ret); 1142 VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret); 1143 } 1144 } 1145 1146 /* Handle write to IOTLB Invalidation Register */ 1147 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 1148 { 1149 uint64_t ret; 1150 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 1151 1152 /* IOTLB invalidation request */ 1153 if (val & VTD_TLB_IVT) { 1154 if (s->qi_enabled) { 1155 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, " 1156 "should not use register-based invalidation"); 1157 return; 1158 } 1159 ret = vtd_iotlb_flush(s, val); 1160 /* Invalidation completed. Change something to show */ 1161 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 1162 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 1163 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 1164 VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret); 1165 } 1166 } 1167 1168 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 1169 static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset, 1170 VTDInvDesc *inv_desc) 1171 { 1172 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc); 1173 if (dma_memory_read(&address_space_memory, addr, inv_desc, 1174 sizeof(*inv_desc))) { 1175 VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor " 1176 "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset); 1177 inv_desc->lo = 0; 1178 inv_desc->hi = 0; 1179 1180 return false; 1181 } 1182 inv_desc->lo = le64_to_cpu(inv_desc->lo); 1183 inv_desc->hi = le64_to_cpu(inv_desc->hi); 1184 return true; 1185 } 1186 1187 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1188 { 1189 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 1190 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 1191 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Invalidation " 1192 "Wait Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1193 inv_desc->hi, inv_desc->lo); 1194 return false; 1195 } 1196 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 1197 /* Status Write */ 1198 uint32_t status_data = (uint32_t)(inv_desc->lo >> 1199 VTD_INV_DESC_WAIT_DATA_SHIFT); 1200 1201 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 1202 1203 /* FIXME: need to be masked with HAW? */ 1204 dma_addr_t status_addr = inv_desc->hi; 1205 VTD_DPRINTF(INV, "status data 0x%x, status addr 0x%"PRIx64, 1206 status_data, status_addr); 1207 status_data = cpu_to_le32(status_data); 1208 if (dma_memory_write(&address_space_memory, status_addr, &status_data, 1209 sizeof(status_data))) { 1210 VTD_DPRINTF(GENERAL, "error: fail to perform a coherent write"); 1211 return false; 1212 } 1213 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 1214 /* Interrupt flag */ 1215 VTD_DPRINTF(INV, "Invalidation Wait Descriptor interrupt completion"); 1216 vtd_generate_completion_event(s); 1217 } else { 1218 VTD_DPRINTF(GENERAL, "error: invalid Invalidation Wait Descriptor: " 1219 "hi 0x%"PRIx64 " lo 0x%"PRIx64, inv_desc->hi, inv_desc->lo); 1220 return false; 1221 } 1222 return true; 1223 } 1224 1225 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 1226 VTDInvDesc *inv_desc) 1227 { 1228 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 1229 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache " 1230 "Invalidate Descriptor"); 1231 return false; 1232 } 1233 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 1234 case VTD_INV_DESC_CC_DOMAIN: 1235 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1236 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 1237 /* Fall through */ 1238 case VTD_INV_DESC_CC_GLOBAL: 1239 VTD_DPRINTF(INV, "global invalidation"); 1240 vtd_context_global_invalidate(s); 1241 break; 1242 1243 case VTD_INV_DESC_CC_DEVICE: 1244 vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo), 1245 VTD_INV_DESC_CC_FM(inv_desc->lo)); 1246 break; 1247 1248 default: 1249 VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache " 1250 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1251 inv_desc->hi, inv_desc->lo); 1252 return false; 1253 } 1254 return true; 1255 } 1256 1257 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1258 { 1259 uint16_t domain_id; 1260 uint8_t am; 1261 hwaddr addr; 1262 1263 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 1264 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 1265 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in IOTLB " 1266 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1267 inv_desc->hi, inv_desc->lo); 1268 return false; 1269 } 1270 1271 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 1272 case VTD_INV_DESC_IOTLB_GLOBAL: 1273 VTD_DPRINTF(INV, "global invalidation"); 1274 vtd_iotlb_global_invalidate(s); 1275 break; 1276 1277 case VTD_INV_DESC_IOTLB_DOMAIN: 1278 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1279 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1280 domain_id); 1281 vtd_iotlb_domain_invalidate(s, domain_id); 1282 break; 1283 1284 case VTD_INV_DESC_IOTLB_PAGE: 1285 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1286 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 1287 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 1288 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16 1289 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am); 1290 if (am > VTD_MAMV) { 1291 VTD_DPRINTF(GENERAL, "error: supported max address mask value is " 1292 "%"PRIu8, (uint8_t)VTD_MAMV); 1293 return false; 1294 } 1295 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1296 break; 1297 1298 default: 1299 VTD_DPRINTF(GENERAL, "error: invalid granularity in IOTLB Invalidate " 1300 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1301 inv_desc->hi, inv_desc->lo); 1302 return false; 1303 } 1304 return true; 1305 } 1306 1307 static bool vtd_process_inv_desc(IntelIOMMUState *s) 1308 { 1309 VTDInvDesc inv_desc; 1310 uint8_t desc_type; 1311 1312 VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head); 1313 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) { 1314 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1315 return false; 1316 } 1317 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 1318 /* FIXME: should update at first or at last? */ 1319 s->iq_last_desc_type = desc_type; 1320 1321 switch (desc_type) { 1322 case VTD_INV_DESC_CC: 1323 VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64 1324 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1325 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 1326 return false; 1327 } 1328 break; 1329 1330 case VTD_INV_DESC_IOTLB: 1331 VTD_DPRINTF(INV, "IOTLB Invalidate Descriptor hi 0x%"PRIx64 1332 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1333 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 1334 return false; 1335 } 1336 break; 1337 1338 case VTD_INV_DESC_WAIT: 1339 VTD_DPRINTF(INV, "Invalidation Wait Descriptor hi 0x%"PRIx64 1340 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1341 if (!vtd_process_wait_desc(s, &inv_desc)) { 1342 return false; 1343 } 1344 break; 1345 1346 default: 1347 VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type " 1348 "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8, 1349 inv_desc.hi, inv_desc.lo, desc_type); 1350 return false; 1351 } 1352 s->iq_head++; 1353 if (s->iq_head == s->iq_size) { 1354 s->iq_head = 0; 1355 } 1356 return true; 1357 } 1358 1359 /* Try to fetch and process more Invalidation Descriptors */ 1360 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 1361 { 1362 VTD_DPRINTF(INV, "fetch Invalidation Descriptors"); 1363 if (s->iq_tail >= s->iq_size) { 1364 /* Detects an invalid Tail pointer */ 1365 VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16 1366 " while iq_size is %"PRIu16, s->iq_tail, s->iq_size); 1367 vtd_handle_inv_queue_error(s); 1368 return; 1369 } 1370 while (s->iq_head != s->iq_tail) { 1371 if (!vtd_process_inv_desc(s)) { 1372 /* Invalidation Queue Errors */ 1373 vtd_handle_inv_queue_error(s); 1374 break; 1375 } 1376 /* Must update the IQH_REG in time */ 1377 vtd_set_quad_raw(s, DMAR_IQH_REG, 1378 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) & 1379 VTD_IQH_QH_MASK); 1380 } 1381 } 1382 1383 /* Handle write to Invalidation Queue Tail Register */ 1384 static void vtd_handle_iqt_write(IntelIOMMUState *s) 1385 { 1386 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 1387 1388 s->iq_tail = VTD_IQT_QT(val); 1389 VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail); 1390 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 1391 /* Process Invalidation Queue here */ 1392 vtd_fetch_inv_desc(s); 1393 } 1394 } 1395 1396 static void vtd_handle_fsts_write(IntelIOMMUState *s) 1397 { 1398 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 1399 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1400 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 1401 1402 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 1403 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1404 VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear " 1405 "IP field of FECTL_REG"); 1406 } 1407 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 1408 * Descriptors if there are any when Queued Invalidation is enabled? 1409 */ 1410 } 1411 1412 static void vtd_handle_fectl_write(IntelIOMMUState *s) 1413 { 1414 uint32_t fectl_reg; 1415 /* FIXME: when software clears the IM field, check the IP field. But do we 1416 * need to compare the old value and the new value to conclude that 1417 * software clears the IM field? Or just check if the IM field is zero? 1418 */ 1419 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1420 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 1421 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 1422 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1423 VTD_DPRINTF(FLOG, "IM field is cleared, generate " 1424 "fault event interrupt"); 1425 } 1426 } 1427 1428 static void vtd_handle_ics_write(IntelIOMMUState *s) 1429 { 1430 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 1431 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1432 1433 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 1434 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1435 VTD_DPRINTF(INV, "pending completion interrupt condition serviced, " 1436 "clear IP field of IECTL_REG"); 1437 } 1438 } 1439 1440 static void vtd_handle_iectl_write(IntelIOMMUState *s) 1441 { 1442 uint32_t iectl_reg; 1443 /* FIXME: when software clears the IM field, check the IP field. But do we 1444 * need to compare the old value and the new value to conclude that 1445 * software clears the IM field? Or just check if the IM field is zero? 1446 */ 1447 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1448 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 1449 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 1450 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1451 VTD_DPRINTF(INV, "IM field is cleared, generate " 1452 "invalidation event interrupt"); 1453 } 1454 } 1455 1456 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 1457 { 1458 IntelIOMMUState *s = opaque; 1459 uint64_t val; 1460 1461 if (addr + size > DMAR_REG_SIZE) { 1462 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64 1463 ", got 0x%"PRIx64 " %d", 1464 (uint64_t)DMAR_REG_SIZE, addr, size); 1465 return (uint64_t)-1; 1466 } 1467 1468 switch (addr) { 1469 /* Root Table Address Register, 64-bit */ 1470 case DMAR_RTADDR_REG: 1471 if (size == 4) { 1472 val = s->root & ((1ULL << 32) - 1); 1473 } else { 1474 val = s->root; 1475 } 1476 break; 1477 1478 case DMAR_RTADDR_REG_HI: 1479 assert(size == 4); 1480 val = s->root >> 32; 1481 break; 1482 1483 /* Invalidation Queue Address Register, 64-bit */ 1484 case DMAR_IQA_REG: 1485 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 1486 if (size == 4) { 1487 val = val & ((1ULL << 32) - 1); 1488 } 1489 break; 1490 1491 case DMAR_IQA_REG_HI: 1492 assert(size == 4); 1493 val = s->iq >> 32; 1494 break; 1495 1496 default: 1497 if (size == 4) { 1498 val = vtd_get_long(s, addr); 1499 } else { 1500 val = vtd_get_quad(s, addr); 1501 } 1502 } 1503 VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64, 1504 addr, size, val); 1505 return val; 1506 } 1507 1508 static void vtd_mem_write(void *opaque, hwaddr addr, 1509 uint64_t val, unsigned size) 1510 { 1511 IntelIOMMUState *s = opaque; 1512 1513 if (addr + size > DMAR_REG_SIZE) { 1514 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64 1515 ", got 0x%"PRIx64 " %d", 1516 (uint64_t)DMAR_REG_SIZE, addr, size); 1517 return; 1518 } 1519 1520 switch (addr) { 1521 /* Global Command Register, 32-bit */ 1522 case DMAR_GCMD_REG: 1523 VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64 1524 ", size %d, val 0x%"PRIx64, addr, size, val); 1525 vtd_set_long(s, addr, val); 1526 vtd_handle_gcmd_write(s); 1527 break; 1528 1529 /* Context Command Register, 64-bit */ 1530 case DMAR_CCMD_REG: 1531 VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64 1532 ", size %d, val 0x%"PRIx64, addr, size, val); 1533 if (size == 4) { 1534 vtd_set_long(s, addr, val); 1535 } else { 1536 vtd_set_quad(s, addr, val); 1537 vtd_handle_ccmd_write(s); 1538 } 1539 break; 1540 1541 case DMAR_CCMD_REG_HI: 1542 VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64 1543 ", size %d, val 0x%"PRIx64, addr, size, val); 1544 assert(size == 4); 1545 vtd_set_long(s, addr, val); 1546 vtd_handle_ccmd_write(s); 1547 break; 1548 1549 /* IOTLB Invalidation Register, 64-bit */ 1550 case DMAR_IOTLB_REG: 1551 VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64 1552 ", size %d, val 0x%"PRIx64, addr, size, val); 1553 if (size == 4) { 1554 vtd_set_long(s, addr, val); 1555 } else { 1556 vtd_set_quad(s, addr, val); 1557 vtd_handle_iotlb_write(s); 1558 } 1559 break; 1560 1561 case DMAR_IOTLB_REG_HI: 1562 VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64 1563 ", size %d, val 0x%"PRIx64, addr, size, val); 1564 assert(size == 4); 1565 vtd_set_long(s, addr, val); 1566 vtd_handle_iotlb_write(s); 1567 break; 1568 1569 /* Invalidate Address Register, 64-bit */ 1570 case DMAR_IVA_REG: 1571 VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64 1572 ", size %d, val 0x%"PRIx64, addr, size, val); 1573 if (size == 4) { 1574 vtd_set_long(s, addr, val); 1575 } else { 1576 vtd_set_quad(s, addr, val); 1577 } 1578 break; 1579 1580 case DMAR_IVA_REG_HI: 1581 VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64 1582 ", size %d, val 0x%"PRIx64, addr, size, val); 1583 assert(size == 4); 1584 vtd_set_long(s, addr, val); 1585 break; 1586 1587 /* Fault Status Register, 32-bit */ 1588 case DMAR_FSTS_REG: 1589 VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64 1590 ", size %d, val 0x%"PRIx64, addr, size, val); 1591 assert(size == 4); 1592 vtd_set_long(s, addr, val); 1593 vtd_handle_fsts_write(s); 1594 break; 1595 1596 /* Fault Event Control Register, 32-bit */ 1597 case DMAR_FECTL_REG: 1598 VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64 1599 ", size %d, val 0x%"PRIx64, addr, size, val); 1600 assert(size == 4); 1601 vtd_set_long(s, addr, val); 1602 vtd_handle_fectl_write(s); 1603 break; 1604 1605 /* Fault Event Data Register, 32-bit */ 1606 case DMAR_FEDATA_REG: 1607 VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64 1608 ", size %d, val 0x%"PRIx64, addr, size, val); 1609 assert(size == 4); 1610 vtd_set_long(s, addr, val); 1611 break; 1612 1613 /* Fault Event Address Register, 32-bit */ 1614 case DMAR_FEADDR_REG: 1615 VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64 1616 ", size %d, val 0x%"PRIx64, addr, size, val); 1617 assert(size == 4); 1618 vtd_set_long(s, addr, val); 1619 break; 1620 1621 /* Fault Event Upper Address Register, 32-bit */ 1622 case DMAR_FEUADDR_REG: 1623 VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64 1624 ", size %d, val 0x%"PRIx64, addr, size, val); 1625 assert(size == 4); 1626 vtd_set_long(s, addr, val); 1627 break; 1628 1629 /* Protected Memory Enable Register, 32-bit */ 1630 case DMAR_PMEN_REG: 1631 VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64 1632 ", size %d, val 0x%"PRIx64, addr, size, val); 1633 assert(size == 4); 1634 vtd_set_long(s, addr, val); 1635 break; 1636 1637 /* Root Table Address Register, 64-bit */ 1638 case DMAR_RTADDR_REG: 1639 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64 1640 ", size %d, val 0x%"PRIx64, addr, size, val); 1641 if (size == 4) { 1642 vtd_set_long(s, addr, val); 1643 } else { 1644 vtd_set_quad(s, addr, val); 1645 } 1646 break; 1647 1648 case DMAR_RTADDR_REG_HI: 1649 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64 1650 ", size %d, val 0x%"PRIx64, addr, size, val); 1651 assert(size == 4); 1652 vtd_set_long(s, addr, val); 1653 break; 1654 1655 /* Invalidation Queue Tail Register, 64-bit */ 1656 case DMAR_IQT_REG: 1657 VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64 1658 ", size %d, val 0x%"PRIx64, addr, size, val); 1659 if (size == 4) { 1660 vtd_set_long(s, addr, val); 1661 } else { 1662 vtd_set_quad(s, addr, val); 1663 } 1664 vtd_handle_iqt_write(s); 1665 break; 1666 1667 case DMAR_IQT_REG_HI: 1668 VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64 1669 ", size %d, val 0x%"PRIx64, addr, size, val); 1670 assert(size == 4); 1671 vtd_set_long(s, addr, val); 1672 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 1673 break; 1674 1675 /* Invalidation Queue Address Register, 64-bit */ 1676 case DMAR_IQA_REG: 1677 VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64 1678 ", size %d, val 0x%"PRIx64, addr, size, val); 1679 if (size == 4) { 1680 vtd_set_long(s, addr, val); 1681 } else { 1682 vtd_set_quad(s, addr, val); 1683 } 1684 break; 1685 1686 case DMAR_IQA_REG_HI: 1687 VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64 1688 ", size %d, val 0x%"PRIx64, addr, size, val); 1689 assert(size == 4); 1690 vtd_set_long(s, addr, val); 1691 break; 1692 1693 /* Invalidation Completion Status Register, 32-bit */ 1694 case DMAR_ICS_REG: 1695 VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64 1696 ", size %d, val 0x%"PRIx64, addr, size, val); 1697 assert(size == 4); 1698 vtd_set_long(s, addr, val); 1699 vtd_handle_ics_write(s); 1700 break; 1701 1702 /* Invalidation Event Control Register, 32-bit */ 1703 case DMAR_IECTL_REG: 1704 VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64 1705 ", size %d, val 0x%"PRIx64, addr, size, val); 1706 assert(size == 4); 1707 vtd_set_long(s, addr, val); 1708 vtd_handle_iectl_write(s); 1709 break; 1710 1711 /* Invalidation Event Data Register, 32-bit */ 1712 case DMAR_IEDATA_REG: 1713 VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64 1714 ", size %d, val 0x%"PRIx64, addr, size, val); 1715 assert(size == 4); 1716 vtd_set_long(s, addr, val); 1717 break; 1718 1719 /* Invalidation Event Address Register, 32-bit */ 1720 case DMAR_IEADDR_REG: 1721 VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64 1722 ", size %d, val 0x%"PRIx64, addr, size, val); 1723 assert(size == 4); 1724 vtd_set_long(s, addr, val); 1725 break; 1726 1727 /* Invalidation Event Upper Address Register, 32-bit */ 1728 case DMAR_IEUADDR_REG: 1729 VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64 1730 ", size %d, val 0x%"PRIx64, addr, size, val); 1731 assert(size == 4); 1732 vtd_set_long(s, addr, val); 1733 break; 1734 1735 /* Fault Recording Registers, 128-bit */ 1736 case DMAR_FRCD_REG_0_0: 1737 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64 1738 ", size %d, val 0x%"PRIx64, addr, size, val); 1739 if (size == 4) { 1740 vtd_set_long(s, addr, val); 1741 } else { 1742 vtd_set_quad(s, addr, val); 1743 } 1744 break; 1745 1746 case DMAR_FRCD_REG_0_1: 1747 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64 1748 ", size %d, val 0x%"PRIx64, addr, size, val); 1749 assert(size == 4); 1750 vtd_set_long(s, addr, val); 1751 break; 1752 1753 case DMAR_FRCD_REG_0_2: 1754 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64 1755 ", size %d, val 0x%"PRIx64, addr, size, val); 1756 if (size == 4) { 1757 vtd_set_long(s, addr, val); 1758 } else { 1759 vtd_set_quad(s, addr, val); 1760 /* May clear bit 127 (Fault), update PPF */ 1761 vtd_update_fsts_ppf(s); 1762 } 1763 break; 1764 1765 case DMAR_FRCD_REG_0_3: 1766 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64 1767 ", size %d, val 0x%"PRIx64, addr, size, val); 1768 assert(size == 4); 1769 vtd_set_long(s, addr, val); 1770 /* May clear bit 127 (Fault), update PPF */ 1771 vtd_update_fsts_ppf(s); 1772 break; 1773 1774 default: 1775 VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64 1776 ", size %d, val 0x%"PRIx64, addr, size, val); 1777 if (size == 4) { 1778 vtd_set_long(s, addr, val); 1779 } else { 1780 vtd_set_quad(s, addr, val); 1781 } 1782 } 1783 } 1784 1785 static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr, 1786 bool is_write) 1787 { 1788 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 1789 IntelIOMMUState *s = vtd_as->iommu_state; 1790 IOMMUTLBEntry ret = { 1791 .target_as = &address_space_memory, 1792 .iova = addr, 1793 .translated_addr = 0, 1794 .addr_mask = ~(hwaddr)0, 1795 .perm = IOMMU_NONE, 1796 }; 1797 1798 if (!s->dmar_enabled) { 1799 /* DMAR disabled, passthrough, use 4k-page*/ 1800 ret.iova = addr & VTD_PAGE_MASK_4K; 1801 ret.translated_addr = addr & VTD_PAGE_MASK_4K; 1802 ret.addr_mask = ~VTD_PAGE_MASK_4K; 1803 ret.perm = IOMMU_RW; 1804 return ret; 1805 } 1806 1807 vtd_do_iommu_translate(vtd_as, vtd_as->bus_num, vtd_as->devfn, addr, 1808 is_write, &ret); 1809 VTD_DPRINTF(MMU, 1810 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8 1811 " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, vtd_as->bus_num, 1812 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn), 1813 vtd_as->devfn, addr, ret.translated_addr); 1814 return ret; 1815 } 1816 1817 static const VMStateDescription vtd_vmstate = { 1818 .name = "iommu-intel", 1819 .unmigratable = 1, 1820 }; 1821 1822 static const MemoryRegionOps vtd_mem_ops = { 1823 .read = vtd_mem_read, 1824 .write = vtd_mem_write, 1825 .endianness = DEVICE_LITTLE_ENDIAN, 1826 .impl = { 1827 .min_access_size = 4, 1828 .max_access_size = 8, 1829 }, 1830 .valid = { 1831 .min_access_size = 4, 1832 .max_access_size = 8, 1833 }, 1834 }; 1835 1836 static Property vtd_properties[] = { 1837 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 1838 DEFINE_PROP_END_OF_LIST(), 1839 }; 1840 1841 /* Do the initialization. It will also be called when reset, so pay 1842 * attention when adding new initialization stuff. 1843 */ 1844 static void vtd_init(IntelIOMMUState *s) 1845 { 1846 memset(s->csr, 0, DMAR_REG_SIZE); 1847 memset(s->wmask, 0, DMAR_REG_SIZE); 1848 memset(s->w1cmask, 0, DMAR_REG_SIZE); 1849 memset(s->womask, 0, DMAR_REG_SIZE); 1850 1851 s->iommu_ops.translate = vtd_iommu_translate; 1852 s->root = 0; 1853 s->root_extended = false; 1854 s->dmar_enabled = false; 1855 s->iq_head = 0; 1856 s->iq_tail = 0; 1857 s->iq = 0; 1858 s->iq_size = 0; 1859 s->qi_enabled = false; 1860 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1861 s->next_frcd_reg = 0; 1862 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW | 1863 VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI; 1864 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 1865 1866 vtd_reset_context_cache(s); 1867 vtd_reset_iotlb(s); 1868 1869 /* Define registers with default values and bit semantics */ 1870 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 1871 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 1872 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 1873 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 1874 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 1875 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 1876 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0); 1877 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 1878 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 1879 1880 /* Advanced Fault Logging not supported */ 1881 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 1882 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 1883 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 1884 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 1885 1886 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 1887 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 1888 */ 1889 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 1890 1891 /* Treated as RO for implementations that PLMR and PHMR fields reported 1892 * as Clear in the CAP_REG. 1893 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 1894 */ 1895 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 1896 1897 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 1898 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 1899 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0); 1900 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 1901 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 1902 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 1903 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 1904 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 1905 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 1906 1907 /* IOTLB registers */ 1908 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 1909 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 1910 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 1911 1912 /* Fault Recording Registers, 128-bit */ 1913 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 1914 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 1915 } 1916 1917 /* Should not reset address_spaces when reset because devices will still use 1918 * the address space they got at first (won't ask the bus again). 1919 */ 1920 static void vtd_reset(DeviceState *dev) 1921 { 1922 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 1923 1924 VTD_DPRINTF(GENERAL, ""); 1925 vtd_init(s); 1926 } 1927 1928 static void vtd_realize(DeviceState *dev, Error **errp) 1929 { 1930 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 1931 1932 VTD_DPRINTF(GENERAL, ""); 1933 memset(s->address_spaces, 0, sizeof(s->address_spaces)); 1934 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 1935 "intel_iommu", DMAR_REG_SIZE); 1936 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem); 1937 /* No corresponding destroy */ 1938 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 1939 g_free, g_free); 1940 vtd_init(s); 1941 } 1942 1943 static void vtd_class_init(ObjectClass *klass, void *data) 1944 { 1945 DeviceClass *dc = DEVICE_CLASS(klass); 1946 1947 dc->reset = vtd_reset; 1948 dc->realize = vtd_realize; 1949 dc->vmsd = &vtd_vmstate; 1950 dc->props = vtd_properties; 1951 } 1952 1953 static const TypeInfo vtd_info = { 1954 .name = TYPE_INTEL_IOMMU_DEVICE, 1955 .parent = TYPE_SYS_BUS_DEVICE, 1956 .instance_size = sizeof(IntelIOMMUState), 1957 .class_init = vtd_class_init, 1958 }; 1959 1960 static void vtd_register_types(void) 1961 { 1962 VTD_DPRINTF(GENERAL, ""); 1963 type_register_static(&vtd_info); 1964 } 1965 1966 type_init(vtd_register_types) 1967