1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "hw/sysbus.h" 23 #include "exec/address-spaces.h" 24 #include "intel_iommu_internal.h" 25 26 /*#define DEBUG_INTEL_IOMMU*/ 27 #ifdef DEBUG_INTEL_IOMMU 28 enum { 29 DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG, 30 DEBUG_CACHE, 31 }; 32 #define VTD_DBGBIT(x) (1 << DEBUG_##x) 33 static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR); 34 35 #define VTD_DPRINTF(what, fmt, ...) do { \ 36 if (vtd_dbgflags & VTD_DBGBIT(what)) { \ 37 fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \ 38 ## __VA_ARGS__); } \ 39 } while (0) 40 #else 41 #define VTD_DPRINTF(what, fmt, ...) do {} while (0) 42 #endif 43 44 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 45 uint64_t wmask, uint64_t w1cmask) 46 { 47 stq_le_p(&s->csr[addr], val); 48 stq_le_p(&s->wmask[addr], wmask); 49 stq_le_p(&s->w1cmask[addr], w1cmask); 50 } 51 52 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 53 { 54 stq_le_p(&s->womask[addr], mask); 55 } 56 57 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 58 uint32_t wmask, uint32_t w1cmask) 59 { 60 stl_le_p(&s->csr[addr], val); 61 stl_le_p(&s->wmask[addr], wmask); 62 stl_le_p(&s->w1cmask[addr], w1cmask); 63 } 64 65 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 66 { 67 stl_le_p(&s->womask[addr], mask); 68 } 69 70 /* "External" get/set operations */ 71 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 72 { 73 uint64_t oldval = ldq_le_p(&s->csr[addr]); 74 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 75 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 76 stq_le_p(&s->csr[addr], 77 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 78 } 79 80 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 81 { 82 uint32_t oldval = ldl_le_p(&s->csr[addr]); 83 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 84 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 85 stl_le_p(&s->csr[addr], 86 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 87 } 88 89 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 90 { 91 uint64_t val = ldq_le_p(&s->csr[addr]); 92 uint64_t womask = ldq_le_p(&s->womask[addr]); 93 return val & ~womask; 94 } 95 96 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 97 { 98 uint32_t val = ldl_le_p(&s->csr[addr]); 99 uint32_t womask = ldl_le_p(&s->womask[addr]); 100 return val & ~womask; 101 } 102 103 /* "Internal" get/set operations */ 104 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 105 { 106 return ldq_le_p(&s->csr[addr]); 107 } 108 109 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 110 { 111 return ldl_le_p(&s->csr[addr]); 112 } 113 114 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 115 { 116 stq_le_p(&s->csr[addr], val); 117 } 118 119 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 120 uint32_t clear, uint32_t mask) 121 { 122 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 123 stl_le_p(&s->csr[addr], new_val); 124 return new_val; 125 } 126 127 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 128 uint64_t clear, uint64_t mask) 129 { 130 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 131 stq_le_p(&s->csr[addr], new_val); 132 return new_val; 133 } 134 135 /* GHashTable functions */ 136 static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2) 137 { 138 return *((const uint64_t *)v1) == *((const uint64_t *)v2); 139 } 140 141 static guint vtd_uint64_hash(gconstpointer v) 142 { 143 return (guint)*(const uint64_t *)v; 144 } 145 146 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 147 gpointer user_data) 148 { 149 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 150 uint16_t domain_id = *(uint16_t *)user_data; 151 return entry->domain_id == domain_id; 152 } 153 154 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 155 gpointer user_data) 156 { 157 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 158 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 159 uint64_t gfn = info->gfn & info->mask; 160 return (entry->domain_id == info->domain_id) && 161 ((entry->gfn & info->mask) == gfn); 162 } 163 164 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 165 * IntelIOMMUState to 1. 166 */ 167 static void vtd_reset_context_cache(IntelIOMMUState *s) 168 { 169 VTDAddressSpace **pvtd_as; 170 VTDAddressSpace *vtd_as; 171 uint32_t bus_it; 172 uint32_t devfn_it; 173 174 VTD_DPRINTF(CACHE, "global context_cache_gen=1"); 175 for (bus_it = 0; bus_it < VTD_PCI_BUS_MAX; ++bus_it) { 176 pvtd_as = s->address_spaces[bus_it]; 177 if (!pvtd_as) { 178 continue; 179 } 180 for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) { 181 vtd_as = pvtd_as[devfn_it]; 182 if (!vtd_as) { 183 continue; 184 } 185 vtd_as->context_cache_entry.context_cache_gen = 0; 186 } 187 } 188 s->context_cache_gen = 1; 189 } 190 191 static void vtd_reset_iotlb(IntelIOMMUState *s) 192 { 193 assert(s->iotlb); 194 g_hash_table_remove_all(s->iotlb); 195 } 196 197 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 198 hwaddr addr) 199 { 200 uint64_t key; 201 202 key = (addr >> VTD_PAGE_SHIFT_4K) | 203 ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT); 204 return g_hash_table_lookup(s->iotlb, &key); 205 206 } 207 208 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 209 uint16_t domain_id, hwaddr addr, uint64_t slpte, 210 bool read_flags, bool write_flags) 211 { 212 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 213 uint64_t *key = g_malloc(sizeof(*key)); 214 uint64_t gfn = addr >> VTD_PAGE_SHIFT_4K; 215 216 VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64 217 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte, 218 domain_id); 219 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 220 VTD_DPRINTF(CACHE, "iotlb exceeds size limit, forced to reset"); 221 vtd_reset_iotlb(s); 222 } 223 224 entry->gfn = gfn; 225 entry->domain_id = domain_id; 226 entry->slpte = slpte; 227 entry->read_flags = read_flags; 228 entry->write_flags = write_flags; 229 *key = gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT); 230 g_hash_table_replace(s->iotlb, key, entry); 231 } 232 233 /* Given the reg addr of both the message data and address, generate an 234 * interrupt via MSI. 235 */ 236 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 237 hwaddr mesg_data_reg) 238 { 239 hwaddr addr; 240 uint32_t data; 241 242 assert(mesg_data_reg < DMAR_REG_SIZE); 243 assert(mesg_addr_reg < DMAR_REG_SIZE); 244 245 addr = vtd_get_long_raw(s, mesg_addr_reg); 246 data = vtd_get_long_raw(s, mesg_data_reg); 247 248 VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32, addr, data); 249 address_space_stl_le(&address_space_memory, addr, data, 250 MEMTXATTRS_UNSPECIFIED, NULL); 251 } 252 253 /* Generate a fault event to software via MSI if conditions are met. 254 * Notice that the value of FSTS_REG being passed to it should be the one 255 * before any update. 256 */ 257 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 258 { 259 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 260 pre_fsts & VTD_FSTS_IQE) { 261 VTD_DPRINTF(FLOG, "there are previous interrupt conditions " 262 "to be serviced by software, fault event is not generated " 263 "(FSTS_REG 0x%"PRIx32 ")", pre_fsts); 264 return; 265 } 266 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 267 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 268 VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated"); 269 } else { 270 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 271 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 272 } 273 } 274 275 /* Check if the Fault (F) field of the Fault Recording Register referenced by 276 * @index is Set. 277 */ 278 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 279 { 280 /* Each reg is 128-bit */ 281 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 282 addr += 8; /* Access the high 64-bit half */ 283 284 assert(index < DMAR_FRCD_REG_NR); 285 286 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 287 } 288 289 /* Update the PPF field of Fault Status Register. 290 * Should be called whenever change the F field of any fault recording 291 * registers. 292 */ 293 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 294 { 295 uint32_t i; 296 uint32_t ppf_mask = 0; 297 298 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 299 if (vtd_is_frcd_set(s, i)) { 300 ppf_mask = VTD_FSTS_PPF; 301 break; 302 } 303 } 304 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 305 VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0); 306 } 307 308 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 309 { 310 /* Each reg is 128-bit */ 311 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 312 addr += 8; /* Access the high 64-bit half */ 313 314 assert(index < DMAR_FRCD_REG_NR); 315 316 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 317 vtd_update_fsts_ppf(s); 318 } 319 320 /* Must not update F field now, should be done later */ 321 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 322 uint16_t source_id, hwaddr addr, 323 VTDFaultReason fault, bool is_write) 324 { 325 uint64_t hi = 0, lo; 326 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 327 328 assert(index < DMAR_FRCD_REG_NR); 329 330 lo = VTD_FRCD_FI(addr); 331 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 332 if (!is_write) { 333 hi |= VTD_FRCD_T; 334 } 335 vtd_set_quad_raw(s, frcd_reg_addr, lo); 336 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 337 VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64 338 ", lo 0x%"PRIx64, index, hi, lo); 339 } 340 341 /* Try to collapse multiple pending faults from the same requester */ 342 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 343 { 344 uint32_t i; 345 uint64_t frcd_reg; 346 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 347 348 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 349 frcd_reg = vtd_get_quad_raw(s, addr); 350 VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg); 351 if ((frcd_reg & VTD_FRCD_F) && 352 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 353 return true; 354 } 355 addr += 16; /* 128-bit for each */ 356 } 357 return false; 358 } 359 360 /* Log and report an DMAR (address translation) fault to software */ 361 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 362 hwaddr addr, VTDFaultReason fault, 363 bool is_write) 364 { 365 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 366 367 assert(fault < VTD_FR_MAX); 368 369 if (fault == VTD_FR_RESERVED_ERR) { 370 /* This is not a normal fault reason case. Drop it. */ 371 return; 372 } 373 VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64 374 ", is_write %d", source_id, fault, addr, is_write); 375 if (fsts_reg & VTD_FSTS_PFO) { 376 VTD_DPRINTF(FLOG, "new fault is not recorded due to " 377 "Primary Fault Overflow"); 378 return; 379 } 380 if (vtd_try_collapse_fault(s, source_id)) { 381 VTD_DPRINTF(FLOG, "new fault is not recorded due to " 382 "compression of faults"); 383 return; 384 } 385 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 386 VTD_DPRINTF(FLOG, "Primary Fault Overflow and " 387 "new fault is not recorded, set PFO field"); 388 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 389 return; 390 } 391 392 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write); 393 394 if (fsts_reg & VTD_FSTS_PPF) { 395 VTD_DPRINTF(FLOG, "there are pending faults already, " 396 "fault event is not generated"); 397 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 398 s->next_frcd_reg++; 399 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 400 s->next_frcd_reg = 0; 401 } 402 } else { 403 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 404 VTD_FSTS_FRI(s->next_frcd_reg)); 405 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 406 s->next_frcd_reg++; 407 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 408 s->next_frcd_reg = 0; 409 } 410 /* This case actually cause the PPF to be Set. 411 * So generate fault event (interrupt). 412 */ 413 vtd_generate_fault_event(s, fsts_reg); 414 } 415 } 416 417 /* Handle Invalidation Queue Errors of queued invalidation interface error 418 * conditions. 419 */ 420 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 421 { 422 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 423 424 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 425 vtd_generate_fault_event(s, fsts_reg); 426 } 427 428 /* Set the IWC field and try to generate an invalidation completion interrupt */ 429 static void vtd_generate_completion_event(IntelIOMMUState *s) 430 { 431 VTD_DPRINTF(INV, "completes an invalidation wait command with " 432 "Interrupt Flag"); 433 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 434 VTD_DPRINTF(INV, "there is a previous interrupt condition to be " 435 "serviced by software, " 436 "new invalidation event is not generated"); 437 return; 438 } 439 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 440 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 441 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 442 VTD_DPRINTF(INV, "IM filed in IECTL_REG is set, new invalidation " 443 "event is not generated"); 444 return; 445 } else { 446 /* Generate the interrupt event */ 447 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 448 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 449 } 450 } 451 452 static inline bool vtd_root_entry_present(VTDRootEntry *root) 453 { 454 return root->val & VTD_ROOT_ENTRY_P; 455 } 456 457 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 458 VTDRootEntry *re) 459 { 460 dma_addr_t addr; 461 462 addr = s->root + index * sizeof(*re); 463 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) { 464 VTD_DPRINTF(GENERAL, "error: fail to access root-entry at 0x%"PRIx64 465 " + %"PRIu8, s->root, index); 466 re->val = 0; 467 return -VTD_FR_ROOT_TABLE_INV; 468 } 469 re->val = le64_to_cpu(re->val); 470 return 0; 471 } 472 473 static inline bool vtd_context_entry_present(VTDContextEntry *context) 474 { 475 return context->lo & VTD_CONTEXT_ENTRY_P; 476 } 477 478 static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index, 479 VTDContextEntry *ce) 480 { 481 dma_addr_t addr; 482 483 if (!vtd_root_entry_present(root)) { 484 VTD_DPRINTF(GENERAL, "error: root-entry is not present"); 485 return -VTD_FR_ROOT_ENTRY_P; 486 } 487 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce); 488 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) { 489 VTD_DPRINTF(GENERAL, "error: fail to access context-entry at 0x%"PRIx64 490 " + %"PRIu8, 491 (uint64_t)(root->val & VTD_ROOT_ENTRY_CTP), index); 492 return -VTD_FR_CONTEXT_TABLE_INV; 493 } 494 ce->lo = le64_to_cpu(ce->lo); 495 ce->hi = le64_to_cpu(ce->hi); 496 return 0; 497 } 498 499 static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce) 500 { 501 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 502 } 503 504 /* The shift of an addr for a certain level of paging structure */ 505 static inline uint32_t vtd_slpt_level_shift(uint32_t level) 506 { 507 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 508 } 509 510 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte) 511 { 512 return slpte & VTD_SL_PT_BASE_ADDR_MASK; 513 } 514 515 /* Whether the pte indicates the address of the page frame */ 516 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 517 { 518 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 519 } 520 521 /* Get the content of a spte located in @base_addr[@index] */ 522 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 523 { 524 uint64_t slpte; 525 526 assert(index < VTD_SL_PT_ENTRY_NR); 527 528 if (dma_memory_read(&address_space_memory, 529 base_addr + index * sizeof(slpte), &slpte, 530 sizeof(slpte))) { 531 slpte = (uint64_t)-1; 532 return slpte; 533 } 534 slpte = le64_to_cpu(slpte); 535 return slpte; 536 } 537 538 /* Given a gpa and the level of paging structure, return the offset of current 539 * level. 540 */ 541 static inline uint32_t vtd_gpa_level_offset(uint64_t gpa, uint32_t level) 542 { 543 return (gpa >> vtd_slpt_level_shift(level)) & 544 ((1ULL << VTD_SL_LEVEL_BITS) - 1); 545 } 546 547 /* Check Capability Register to see if the @level of page-table is supported */ 548 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 549 { 550 return VTD_CAP_SAGAW_MASK & s->cap & 551 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 552 } 553 554 /* Get the page-table level that hardware should use for the second-level 555 * page-table walk from the Address Width field of context-entry. 556 */ 557 static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce) 558 { 559 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 560 } 561 562 static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce) 563 { 564 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 565 } 566 567 static const uint64_t vtd_paging_entry_rsvd_field[] = { 568 [0] = ~0ULL, 569 /* For not large page */ 570 [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 571 [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 572 [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 573 [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 574 /* For large page */ 575 [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 576 [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 577 [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 578 [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 579 }; 580 581 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 582 { 583 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) { 584 /* Maybe large page */ 585 return slpte & vtd_paging_entry_rsvd_field[level + 4]; 586 } else { 587 return slpte & vtd_paging_entry_rsvd_field[level]; 588 } 589 } 590 591 /* Given the @gpa, get relevant @slptep. @slpte_level will be the last level 592 * of the translation, can be used for deciding the size of large page. 593 */ 594 static int vtd_gpa_to_slpte(VTDContextEntry *ce, uint64_t gpa, bool is_write, 595 uint64_t *slptep, uint32_t *slpte_level, 596 bool *reads, bool *writes) 597 { 598 dma_addr_t addr = vtd_get_slpt_base_from_context(ce); 599 uint32_t level = vtd_get_level_from_context_entry(ce); 600 uint32_t offset; 601 uint64_t slpte; 602 uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce); 603 uint64_t access_right_check; 604 605 /* Check if @gpa is above 2^X-1, where X is the minimum of MGAW in CAP_REG 606 * and AW in context-entry. 607 */ 608 if (gpa & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) { 609 VTD_DPRINTF(GENERAL, "error: gpa 0x%"PRIx64 " exceeds limits", gpa); 610 return -VTD_FR_ADDR_BEYOND_MGAW; 611 } 612 613 /* FIXME: what is the Atomics request here? */ 614 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 615 616 while (true) { 617 offset = vtd_gpa_level_offset(gpa, level); 618 slpte = vtd_get_slpte(addr, offset); 619 620 if (slpte == (uint64_t)-1) { 621 VTD_DPRINTF(GENERAL, "error: fail to access second-level paging " 622 "entry at level %"PRIu32 " for gpa 0x%"PRIx64, 623 level, gpa); 624 if (level == vtd_get_level_from_context_entry(ce)) { 625 /* Invalid programming of context-entry */ 626 return -VTD_FR_CONTEXT_ENTRY_INV; 627 } else { 628 return -VTD_FR_PAGING_ENTRY_INV; 629 } 630 } 631 *reads = (*reads) && (slpte & VTD_SL_R); 632 *writes = (*writes) && (slpte & VTD_SL_W); 633 if (!(slpte & access_right_check)) { 634 VTD_DPRINTF(GENERAL, "error: lack of %s permission for " 635 "gpa 0x%"PRIx64 " slpte 0x%"PRIx64, 636 (is_write ? "write" : "read"), gpa, slpte); 637 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 638 } 639 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 640 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second " 641 "level paging entry level %"PRIu32 " slpte 0x%"PRIx64, 642 level, slpte); 643 return -VTD_FR_PAGING_ENTRY_RSVD; 644 } 645 646 if (vtd_is_last_slpte(slpte, level)) { 647 *slptep = slpte; 648 *slpte_level = level; 649 return 0; 650 } 651 addr = vtd_get_slpte_addr(slpte); 652 level--; 653 } 654 } 655 656 /* Map a device to its corresponding domain (context-entry) */ 657 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 658 uint8_t devfn, VTDContextEntry *ce) 659 { 660 VTDRootEntry re; 661 int ret_fr; 662 663 ret_fr = vtd_get_root_entry(s, bus_num, &re); 664 if (ret_fr) { 665 return ret_fr; 666 } 667 668 if (!vtd_root_entry_present(&re)) { 669 VTD_DPRINTF(GENERAL, "error: root-entry #%"PRIu8 " is not present", 670 bus_num); 671 return -VTD_FR_ROOT_ENTRY_P; 672 } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) { 673 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in root-entry " 674 "hi 0x%"PRIx64 " lo 0x%"PRIx64, re.rsvd, re.val); 675 return -VTD_FR_ROOT_ENTRY_RSVD; 676 } 677 678 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce); 679 if (ret_fr) { 680 return ret_fr; 681 } 682 683 if (!vtd_context_entry_present(ce)) { 684 VTD_DPRINTF(GENERAL, 685 "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") " 686 "is not present", devfn, bus_num); 687 return -VTD_FR_CONTEXT_ENTRY_P; 688 } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) || 689 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) { 690 VTD_DPRINTF(GENERAL, 691 "error: non-zero reserved field in context-entry " 692 "hi 0x%"PRIx64 " lo 0x%"PRIx64, ce->hi, ce->lo); 693 return -VTD_FR_CONTEXT_ENTRY_RSVD; 694 } 695 /* Check if the programming of context-entry is valid */ 696 if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) { 697 VTD_DPRINTF(GENERAL, "error: unsupported Address Width value in " 698 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64, 699 ce->hi, ce->lo); 700 return -VTD_FR_CONTEXT_ENTRY_INV; 701 } else if (ce->lo & VTD_CONTEXT_ENTRY_TT) { 702 VTD_DPRINTF(GENERAL, "error: unsupported Translation Type in " 703 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64, 704 ce->hi, ce->lo); 705 return -VTD_FR_CONTEXT_ENTRY_INV; 706 } 707 return 0; 708 } 709 710 static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn) 711 { 712 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL); 713 } 714 715 static const bool vtd_qualified_faults[] = { 716 [VTD_FR_RESERVED] = false, 717 [VTD_FR_ROOT_ENTRY_P] = false, 718 [VTD_FR_CONTEXT_ENTRY_P] = true, 719 [VTD_FR_CONTEXT_ENTRY_INV] = true, 720 [VTD_FR_ADDR_BEYOND_MGAW] = true, 721 [VTD_FR_WRITE] = true, 722 [VTD_FR_READ] = true, 723 [VTD_FR_PAGING_ENTRY_INV] = true, 724 [VTD_FR_ROOT_TABLE_INV] = false, 725 [VTD_FR_CONTEXT_TABLE_INV] = false, 726 [VTD_FR_ROOT_ENTRY_RSVD] = false, 727 [VTD_FR_PAGING_ENTRY_RSVD] = true, 728 [VTD_FR_CONTEXT_ENTRY_TT] = true, 729 [VTD_FR_RESERVED_ERR] = false, 730 [VTD_FR_MAX] = false, 731 }; 732 733 /* To see if a fault condition is "qualified", which is reported to software 734 * only if the FPD field in the context-entry used to process the faulting 735 * request is 0. 736 */ 737 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 738 { 739 return vtd_qualified_faults[fault]; 740 } 741 742 static inline bool vtd_is_interrupt_addr(hwaddr addr) 743 { 744 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 745 } 746 747 /* Map dev to context-entry then do a paging-structures walk to do a iommu 748 * translation. 749 * 750 * Called from RCU critical section. 751 * 752 * @bus_num: The bus number 753 * @devfn: The devfn, which is the combined of device and function number 754 * @is_write: The access is a write operation 755 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 756 */ 757 static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num, 758 uint8_t devfn, hwaddr addr, bool is_write, 759 IOMMUTLBEntry *entry) 760 { 761 IntelIOMMUState *s = vtd_as->iommu_state; 762 VTDContextEntry ce; 763 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry; 764 uint64_t slpte; 765 uint32_t level; 766 uint16_t source_id = vtd_make_source_id(bus_num, devfn); 767 int ret_fr; 768 bool is_fpd_set = false; 769 bool reads = true; 770 bool writes = true; 771 VTDIOTLBEntry *iotlb_entry; 772 773 /* Check if the request is in interrupt address range */ 774 if (vtd_is_interrupt_addr(addr)) { 775 if (is_write) { 776 /* FIXME: since we don't know the length of the access here, we 777 * treat Non-DWORD length write requests without PASID as 778 * interrupt requests, too. Withoud interrupt remapping support, 779 * we just use 1:1 mapping. 780 */ 781 VTD_DPRINTF(MMU, "write request to interrupt address " 782 "gpa 0x%"PRIx64, addr); 783 entry->iova = addr & VTD_PAGE_MASK_4K; 784 entry->translated_addr = addr & VTD_PAGE_MASK_4K; 785 entry->addr_mask = ~VTD_PAGE_MASK_4K; 786 entry->perm = IOMMU_WO; 787 return; 788 } else { 789 VTD_DPRINTF(GENERAL, "error: read request from interrupt address " 790 "gpa 0x%"PRIx64, addr); 791 vtd_report_dmar_fault(s, source_id, addr, VTD_FR_READ, is_write); 792 return; 793 } 794 } 795 /* Try to fetch slpte form IOTLB */ 796 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr); 797 if (iotlb_entry) { 798 VTD_DPRINTF(CACHE, "hit iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64 799 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, 800 iotlb_entry->slpte, iotlb_entry->domain_id); 801 slpte = iotlb_entry->slpte; 802 reads = iotlb_entry->read_flags; 803 writes = iotlb_entry->write_flags; 804 goto out; 805 } 806 /* Try to fetch context-entry from cache first */ 807 if (cc_entry->context_cache_gen == s->context_cache_gen) { 808 VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d " 809 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")", 810 bus_num, devfn, cc_entry->context_entry.hi, 811 cc_entry->context_entry.lo, cc_entry->context_cache_gen); 812 ce = cc_entry->context_entry; 813 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 814 } else { 815 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 816 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 817 if (ret_fr) { 818 ret_fr = -ret_fr; 819 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 820 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA " 821 "requests through this context-entry " 822 "(with FPD Set)"); 823 } else { 824 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 825 } 826 return; 827 } 828 /* Update context-cache */ 829 VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d " 830 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")", 831 bus_num, devfn, ce.hi, ce.lo, 832 cc_entry->context_cache_gen, s->context_cache_gen); 833 cc_entry->context_entry = ce; 834 cc_entry->context_cache_gen = s->context_cache_gen; 835 } 836 837 ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level, 838 &reads, &writes); 839 if (ret_fr) { 840 ret_fr = -ret_fr; 841 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 842 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests " 843 "through this context-entry (with FPD Set)"); 844 } else { 845 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 846 } 847 return; 848 } 849 850 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte, 851 reads, writes); 852 out: 853 entry->iova = addr & VTD_PAGE_MASK_4K; 854 entry->translated_addr = vtd_get_slpte_addr(slpte) & VTD_PAGE_MASK_4K; 855 entry->addr_mask = ~VTD_PAGE_MASK_4K; 856 entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0); 857 } 858 859 static void vtd_root_table_setup(IntelIOMMUState *s) 860 { 861 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 862 s->root_extended = s->root & VTD_RTADDR_RTT; 863 s->root &= VTD_RTADDR_ADDR_MASK; 864 865 VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root, 866 (s->root_extended ? "(extended)" : "")); 867 } 868 869 static void vtd_context_global_invalidate(IntelIOMMUState *s) 870 { 871 s->context_cache_gen++; 872 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 873 vtd_reset_context_cache(s); 874 } 875 } 876 877 /* Do a context-cache device-selective invalidation. 878 * @func_mask: FM field after shifting 879 */ 880 static void vtd_context_device_invalidate(IntelIOMMUState *s, 881 uint16_t source_id, 882 uint16_t func_mask) 883 { 884 uint16_t mask; 885 VTDAddressSpace **pvtd_as; 886 VTDAddressSpace *vtd_as; 887 uint16_t devfn; 888 uint16_t devfn_it; 889 890 switch (func_mask & 3) { 891 case 0: 892 mask = 0; /* No bits in the SID field masked */ 893 break; 894 case 1: 895 mask = 4; /* Mask bit 2 in the SID field */ 896 break; 897 case 2: 898 mask = 6; /* Mask bit 2:1 in the SID field */ 899 break; 900 case 3: 901 mask = 7; /* Mask bit 2:0 in the SID field */ 902 break; 903 } 904 VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16 905 " mask %"PRIu16, source_id, mask); 906 pvtd_as = s->address_spaces[VTD_SID_TO_BUS(source_id)]; 907 if (pvtd_as) { 908 devfn = VTD_SID_TO_DEVFN(source_id); 909 for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) { 910 vtd_as = pvtd_as[devfn_it]; 911 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) { 912 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16, 913 devfn_it); 914 vtd_as->context_cache_entry.context_cache_gen = 0; 915 } 916 } 917 } 918 } 919 920 /* Context-cache invalidation 921 * Returns the Context Actual Invalidation Granularity. 922 * @val: the content of the CCMD_REG 923 */ 924 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 925 { 926 uint64_t caig; 927 uint64_t type = val & VTD_CCMD_CIRG_MASK; 928 929 switch (type) { 930 case VTD_CCMD_DOMAIN_INVL: 931 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 932 (uint16_t)VTD_CCMD_DID(val)); 933 /* Fall through */ 934 case VTD_CCMD_GLOBAL_INVL: 935 VTD_DPRINTF(INV, "global invalidation"); 936 caig = VTD_CCMD_GLOBAL_INVL_A; 937 vtd_context_global_invalidate(s); 938 break; 939 940 case VTD_CCMD_DEVICE_INVL: 941 caig = VTD_CCMD_DEVICE_INVL_A; 942 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 943 break; 944 945 default: 946 VTD_DPRINTF(GENERAL, "error: invalid granularity"); 947 caig = 0; 948 } 949 return caig; 950 } 951 952 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 953 { 954 vtd_reset_iotlb(s); 955 } 956 957 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 958 { 959 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 960 &domain_id); 961 } 962 963 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 964 hwaddr addr, uint8_t am) 965 { 966 VTDIOTLBPageInvInfo info; 967 968 assert(am <= VTD_MAMV); 969 info.domain_id = domain_id; 970 info.gfn = addr >> VTD_PAGE_SHIFT_4K; 971 info.mask = ~((1 << am) - 1); 972 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 973 } 974 975 /* Flush IOTLB 976 * Returns the IOTLB Actual Invalidation Granularity. 977 * @val: the content of the IOTLB_REG 978 */ 979 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 980 { 981 uint64_t iaig; 982 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 983 uint16_t domain_id; 984 hwaddr addr; 985 uint8_t am; 986 987 switch (type) { 988 case VTD_TLB_GLOBAL_FLUSH: 989 VTD_DPRINTF(INV, "global invalidation"); 990 iaig = VTD_TLB_GLOBAL_FLUSH_A; 991 vtd_iotlb_global_invalidate(s); 992 break; 993 994 case VTD_TLB_DSI_FLUSH: 995 domain_id = VTD_TLB_DID(val); 996 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 997 domain_id); 998 iaig = VTD_TLB_DSI_FLUSH_A; 999 vtd_iotlb_domain_invalidate(s, domain_id); 1000 break; 1001 1002 case VTD_TLB_PSI_FLUSH: 1003 domain_id = VTD_TLB_DID(val); 1004 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 1005 am = VTD_IVA_AM(addr); 1006 addr = VTD_IVA_ADDR(addr); 1007 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16 1008 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am); 1009 if (am > VTD_MAMV) { 1010 VTD_DPRINTF(GENERAL, "error: supported max address mask value is " 1011 "%"PRIu8, (uint8_t)VTD_MAMV); 1012 iaig = 0; 1013 break; 1014 } 1015 iaig = VTD_TLB_PSI_FLUSH_A; 1016 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1017 break; 1018 1019 default: 1020 VTD_DPRINTF(GENERAL, "error: invalid granularity"); 1021 iaig = 0; 1022 } 1023 return iaig; 1024 } 1025 1026 static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s) 1027 { 1028 return s->iq_tail == 0; 1029 } 1030 1031 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 1032 { 1033 return s->qi_enabled && (s->iq_tail == s->iq_head) && 1034 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 1035 } 1036 1037 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 1038 { 1039 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 1040 1041 VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off")); 1042 if (en) { 1043 if (vtd_queued_inv_enable_check(s)) { 1044 s->iq = iqa_val & VTD_IQA_IQA_MASK; 1045 /* 2^(x+8) entries */ 1046 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8); 1047 s->qi_enabled = true; 1048 VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val); 1049 VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d", 1050 s->iq, s->iq_size); 1051 /* Ok - report back to driver */ 1052 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 1053 } else { 1054 VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: " 1055 "tail %"PRIu16, s->iq_tail); 1056 } 1057 } else { 1058 if (vtd_queued_inv_disable_check(s)) { 1059 /* disable Queued Invalidation */ 1060 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 1061 s->iq_head = 0; 1062 s->qi_enabled = false; 1063 /* Ok - report back to driver */ 1064 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 1065 } else { 1066 VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: " 1067 "head %"PRIu16 ", tail %"PRIu16 1068 ", last_descriptor %"PRIu8, 1069 s->iq_head, s->iq_tail, s->iq_last_desc_type); 1070 } 1071 } 1072 } 1073 1074 /* Set Root Table Pointer */ 1075 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 1076 { 1077 VTD_DPRINTF(CSR, "set Root Table Pointer"); 1078 1079 vtd_root_table_setup(s); 1080 /* Ok - report back to driver */ 1081 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 1082 } 1083 1084 /* Handle Translation Enable/Disable */ 1085 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 1086 { 1087 VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off")); 1088 1089 if (en) { 1090 s->dmar_enabled = true; 1091 /* Ok - report back to driver */ 1092 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 1093 } else { 1094 s->dmar_enabled = false; 1095 1096 /* Clear the index of Fault Recording Register */ 1097 s->next_frcd_reg = 0; 1098 /* Ok - report back to driver */ 1099 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 1100 } 1101 } 1102 1103 /* Handle write to Global Command Register */ 1104 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 1105 { 1106 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 1107 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 1108 uint32_t changed = status ^ val; 1109 1110 VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status); 1111 if (changed & VTD_GCMD_TE) { 1112 /* Translation enable/disable */ 1113 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 1114 } 1115 if (val & VTD_GCMD_SRTP) { 1116 /* Set/update the root-table pointer */ 1117 vtd_handle_gcmd_srtp(s); 1118 } 1119 if (changed & VTD_GCMD_QIE) { 1120 /* Queued Invalidation Enable */ 1121 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 1122 } 1123 } 1124 1125 /* Handle write to Context Command Register */ 1126 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 1127 { 1128 uint64_t ret; 1129 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 1130 1131 /* Context-cache invalidation request */ 1132 if (val & VTD_CCMD_ICC) { 1133 if (s->qi_enabled) { 1134 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, " 1135 "should not use register-based invalidation"); 1136 return; 1137 } 1138 ret = vtd_context_cache_invalidate(s, val); 1139 /* Invalidation completed. Change something to show */ 1140 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 1141 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 1142 ret); 1143 VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret); 1144 } 1145 } 1146 1147 /* Handle write to IOTLB Invalidation Register */ 1148 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 1149 { 1150 uint64_t ret; 1151 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 1152 1153 /* IOTLB invalidation request */ 1154 if (val & VTD_TLB_IVT) { 1155 if (s->qi_enabled) { 1156 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, " 1157 "should not use register-based invalidation"); 1158 return; 1159 } 1160 ret = vtd_iotlb_flush(s, val); 1161 /* Invalidation completed. Change something to show */ 1162 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 1163 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 1164 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 1165 VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret); 1166 } 1167 } 1168 1169 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 1170 static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset, 1171 VTDInvDesc *inv_desc) 1172 { 1173 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc); 1174 if (dma_memory_read(&address_space_memory, addr, inv_desc, 1175 sizeof(*inv_desc))) { 1176 VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor " 1177 "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset); 1178 inv_desc->lo = 0; 1179 inv_desc->hi = 0; 1180 1181 return false; 1182 } 1183 inv_desc->lo = le64_to_cpu(inv_desc->lo); 1184 inv_desc->hi = le64_to_cpu(inv_desc->hi); 1185 return true; 1186 } 1187 1188 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1189 { 1190 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 1191 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 1192 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Invalidation " 1193 "Wait Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1194 inv_desc->hi, inv_desc->lo); 1195 return false; 1196 } 1197 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 1198 /* Status Write */ 1199 uint32_t status_data = (uint32_t)(inv_desc->lo >> 1200 VTD_INV_DESC_WAIT_DATA_SHIFT); 1201 1202 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 1203 1204 /* FIXME: need to be masked with HAW? */ 1205 dma_addr_t status_addr = inv_desc->hi; 1206 VTD_DPRINTF(INV, "status data 0x%x, status addr 0x%"PRIx64, 1207 status_data, status_addr); 1208 status_data = cpu_to_le32(status_data); 1209 if (dma_memory_write(&address_space_memory, status_addr, &status_data, 1210 sizeof(status_data))) { 1211 VTD_DPRINTF(GENERAL, "error: fail to perform a coherent write"); 1212 return false; 1213 } 1214 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 1215 /* Interrupt flag */ 1216 VTD_DPRINTF(INV, "Invalidation Wait Descriptor interrupt completion"); 1217 vtd_generate_completion_event(s); 1218 } else { 1219 VTD_DPRINTF(GENERAL, "error: invalid Invalidation Wait Descriptor: " 1220 "hi 0x%"PRIx64 " lo 0x%"PRIx64, inv_desc->hi, inv_desc->lo); 1221 return false; 1222 } 1223 return true; 1224 } 1225 1226 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 1227 VTDInvDesc *inv_desc) 1228 { 1229 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 1230 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache " 1231 "Invalidate Descriptor"); 1232 return false; 1233 } 1234 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 1235 case VTD_INV_DESC_CC_DOMAIN: 1236 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1237 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 1238 /* Fall through */ 1239 case VTD_INV_DESC_CC_GLOBAL: 1240 VTD_DPRINTF(INV, "global invalidation"); 1241 vtd_context_global_invalidate(s); 1242 break; 1243 1244 case VTD_INV_DESC_CC_DEVICE: 1245 vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo), 1246 VTD_INV_DESC_CC_FM(inv_desc->lo)); 1247 break; 1248 1249 default: 1250 VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache " 1251 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1252 inv_desc->hi, inv_desc->lo); 1253 return false; 1254 } 1255 return true; 1256 } 1257 1258 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1259 { 1260 uint16_t domain_id; 1261 uint8_t am; 1262 hwaddr addr; 1263 1264 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 1265 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 1266 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in IOTLB " 1267 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1268 inv_desc->hi, inv_desc->lo); 1269 return false; 1270 } 1271 1272 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 1273 case VTD_INV_DESC_IOTLB_GLOBAL: 1274 VTD_DPRINTF(INV, "global invalidation"); 1275 vtd_iotlb_global_invalidate(s); 1276 break; 1277 1278 case VTD_INV_DESC_IOTLB_DOMAIN: 1279 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1280 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1281 domain_id); 1282 vtd_iotlb_domain_invalidate(s, domain_id); 1283 break; 1284 1285 case VTD_INV_DESC_IOTLB_PAGE: 1286 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1287 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 1288 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 1289 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16 1290 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am); 1291 if (am > VTD_MAMV) { 1292 VTD_DPRINTF(GENERAL, "error: supported max address mask value is " 1293 "%"PRIu8, (uint8_t)VTD_MAMV); 1294 return false; 1295 } 1296 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1297 break; 1298 1299 default: 1300 VTD_DPRINTF(GENERAL, "error: invalid granularity in IOTLB Invalidate " 1301 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1302 inv_desc->hi, inv_desc->lo); 1303 return false; 1304 } 1305 return true; 1306 } 1307 1308 static bool vtd_process_inv_desc(IntelIOMMUState *s) 1309 { 1310 VTDInvDesc inv_desc; 1311 uint8_t desc_type; 1312 1313 VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head); 1314 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) { 1315 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1316 return false; 1317 } 1318 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 1319 /* FIXME: should update at first or at last? */ 1320 s->iq_last_desc_type = desc_type; 1321 1322 switch (desc_type) { 1323 case VTD_INV_DESC_CC: 1324 VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64 1325 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1326 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 1327 return false; 1328 } 1329 break; 1330 1331 case VTD_INV_DESC_IOTLB: 1332 VTD_DPRINTF(INV, "IOTLB Invalidate Descriptor hi 0x%"PRIx64 1333 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1334 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 1335 return false; 1336 } 1337 break; 1338 1339 case VTD_INV_DESC_WAIT: 1340 VTD_DPRINTF(INV, "Invalidation Wait Descriptor hi 0x%"PRIx64 1341 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1342 if (!vtd_process_wait_desc(s, &inv_desc)) { 1343 return false; 1344 } 1345 break; 1346 1347 default: 1348 VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type " 1349 "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8, 1350 inv_desc.hi, inv_desc.lo, desc_type); 1351 return false; 1352 } 1353 s->iq_head++; 1354 if (s->iq_head == s->iq_size) { 1355 s->iq_head = 0; 1356 } 1357 return true; 1358 } 1359 1360 /* Try to fetch and process more Invalidation Descriptors */ 1361 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 1362 { 1363 VTD_DPRINTF(INV, "fetch Invalidation Descriptors"); 1364 if (s->iq_tail >= s->iq_size) { 1365 /* Detects an invalid Tail pointer */ 1366 VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16 1367 " while iq_size is %"PRIu16, s->iq_tail, s->iq_size); 1368 vtd_handle_inv_queue_error(s); 1369 return; 1370 } 1371 while (s->iq_head != s->iq_tail) { 1372 if (!vtd_process_inv_desc(s)) { 1373 /* Invalidation Queue Errors */ 1374 vtd_handle_inv_queue_error(s); 1375 break; 1376 } 1377 /* Must update the IQH_REG in time */ 1378 vtd_set_quad_raw(s, DMAR_IQH_REG, 1379 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) & 1380 VTD_IQH_QH_MASK); 1381 } 1382 } 1383 1384 /* Handle write to Invalidation Queue Tail Register */ 1385 static void vtd_handle_iqt_write(IntelIOMMUState *s) 1386 { 1387 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 1388 1389 s->iq_tail = VTD_IQT_QT(val); 1390 VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail); 1391 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 1392 /* Process Invalidation Queue here */ 1393 vtd_fetch_inv_desc(s); 1394 } 1395 } 1396 1397 static void vtd_handle_fsts_write(IntelIOMMUState *s) 1398 { 1399 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 1400 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1401 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 1402 1403 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 1404 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1405 VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear " 1406 "IP field of FECTL_REG"); 1407 } 1408 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 1409 * Descriptors if there are any when Queued Invalidation is enabled? 1410 */ 1411 } 1412 1413 static void vtd_handle_fectl_write(IntelIOMMUState *s) 1414 { 1415 uint32_t fectl_reg; 1416 /* FIXME: when software clears the IM field, check the IP field. But do we 1417 * need to compare the old value and the new value to conclude that 1418 * software clears the IM field? Or just check if the IM field is zero? 1419 */ 1420 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1421 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 1422 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 1423 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1424 VTD_DPRINTF(FLOG, "IM field is cleared, generate " 1425 "fault event interrupt"); 1426 } 1427 } 1428 1429 static void vtd_handle_ics_write(IntelIOMMUState *s) 1430 { 1431 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 1432 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1433 1434 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 1435 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1436 VTD_DPRINTF(INV, "pending completion interrupt condition serviced, " 1437 "clear IP field of IECTL_REG"); 1438 } 1439 } 1440 1441 static void vtd_handle_iectl_write(IntelIOMMUState *s) 1442 { 1443 uint32_t iectl_reg; 1444 /* FIXME: when software clears the IM field, check the IP field. But do we 1445 * need to compare the old value and the new value to conclude that 1446 * software clears the IM field? Or just check if the IM field is zero? 1447 */ 1448 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1449 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 1450 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 1451 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1452 VTD_DPRINTF(INV, "IM field is cleared, generate " 1453 "invalidation event interrupt"); 1454 } 1455 } 1456 1457 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 1458 { 1459 IntelIOMMUState *s = opaque; 1460 uint64_t val; 1461 1462 if (addr + size > DMAR_REG_SIZE) { 1463 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64 1464 ", got 0x%"PRIx64 " %d", 1465 (uint64_t)DMAR_REG_SIZE, addr, size); 1466 return (uint64_t)-1; 1467 } 1468 1469 switch (addr) { 1470 /* Root Table Address Register, 64-bit */ 1471 case DMAR_RTADDR_REG: 1472 if (size == 4) { 1473 val = s->root & ((1ULL << 32) - 1); 1474 } else { 1475 val = s->root; 1476 } 1477 break; 1478 1479 case DMAR_RTADDR_REG_HI: 1480 assert(size == 4); 1481 val = s->root >> 32; 1482 break; 1483 1484 /* Invalidation Queue Address Register, 64-bit */ 1485 case DMAR_IQA_REG: 1486 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 1487 if (size == 4) { 1488 val = val & ((1ULL << 32) - 1); 1489 } 1490 break; 1491 1492 case DMAR_IQA_REG_HI: 1493 assert(size == 4); 1494 val = s->iq >> 32; 1495 break; 1496 1497 default: 1498 if (size == 4) { 1499 val = vtd_get_long(s, addr); 1500 } else { 1501 val = vtd_get_quad(s, addr); 1502 } 1503 } 1504 VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64, 1505 addr, size, val); 1506 return val; 1507 } 1508 1509 static void vtd_mem_write(void *opaque, hwaddr addr, 1510 uint64_t val, unsigned size) 1511 { 1512 IntelIOMMUState *s = opaque; 1513 1514 if (addr + size > DMAR_REG_SIZE) { 1515 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64 1516 ", got 0x%"PRIx64 " %d", 1517 (uint64_t)DMAR_REG_SIZE, addr, size); 1518 return; 1519 } 1520 1521 switch (addr) { 1522 /* Global Command Register, 32-bit */ 1523 case DMAR_GCMD_REG: 1524 VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64 1525 ", size %d, val 0x%"PRIx64, addr, size, val); 1526 vtd_set_long(s, addr, val); 1527 vtd_handle_gcmd_write(s); 1528 break; 1529 1530 /* Context Command Register, 64-bit */ 1531 case DMAR_CCMD_REG: 1532 VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64 1533 ", size %d, val 0x%"PRIx64, addr, size, val); 1534 if (size == 4) { 1535 vtd_set_long(s, addr, val); 1536 } else { 1537 vtd_set_quad(s, addr, val); 1538 vtd_handle_ccmd_write(s); 1539 } 1540 break; 1541 1542 case DMAR_CCMD_REG_HI: 1543 VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64 1544 ", size %d, val 0x%"PRIx64, addr, size, val); 1545 assert(size == 4); 1546 vtd_set_long(s, addr, val); 1547 vtd_handle_ccmd_write(s); 1548 break; 1549 1550 /* IOTLB Invalidation Register, 64-bit */ 1551 case DMAR_IOTLB_REG: 1552 VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64 1553 ", size %d, val 0x%"PRIx64, addr, size, val); 1554 if (size == 4) { 1555 vtd_set_long(s, addr, val); 1556 } else { 1557 vtd_set_quad(s, addr, val); 1558 vtd_handle_iotlb_write(s); 1559 } 1560 break; 1561 1562 case DMAR_IOTLB_REG_HI: 1563 VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64 1564 ", size %d, val 0x%"PRIx64, addr, size, val); 1565 assert(size == 4); 1566 vtd_set_long(s, addr, val); 1567 vtd_handle_iotlb_write(s); 1568 break; 1569 1570 /* Invalidate Address Register, 64-bit */ 1571 case DMAR_IVA_REG: 1572 VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64 1573 ", size %d, val 0x%"PRIx64, addr, size, val); 1574 if (size == 4) { 1575 vtd_set_long(s, addr, val); 1576 } else { 1577 vtd_set_quad(s, addr, val); 1578 } 1579 break; 1580 1581 case DMAR_IVA_REG_HI: 1582 VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64 1583 ", size %d, val 0x%"PRIx64, addr, size, val); 1584 assert(size == 4); 1585 vtd_set_long(s, addr, val); 1586 break; 1587 1588 /* Fault Status Register, 32-bit */ 1589 case DMAR_FSTS_REG: 1590 VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64 1591 ", size %d, val 0x%"PRIx64, addr, size, val); 1592 assert(size == 4); 1593 vtd_set_long(s, addr, val); 1594 vtd_handle_fsts_write(s); 1595 break; 1596 1597 /* Fault Event Control Register, 32-bit */ 1598 case DMAR_FECTL_REG: 1599 VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64 1600 ", size %d, val 0x%"PRIx64, addr, size, val); 1601 assert(size == 4); 1602 vtd_set_long(s, addr, val); 1603 vtd_handle_fectl_write(s); 1604 break; 1605 1606 /* Fault Event Data Register, 32-bit */ 1607 case DMAR_FEDATA_REG: 1608 VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64 1609 ", size %d, val 0x%"PRIx64, addr, size, val); 1610 assert(size == 4); 1611 vtd_set_long(s, addr, val); 1612 break; 1613 1614 /* Fault Event Address Register, 32-bit */ 1615 case DMAR_FEADDR_REG: 1616 VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64 1617 ", size %d, val 0x%"PRIx64, addr, size, val); 1618 assert(size == 4); 1619 vtd_set_long(s, addr, val); 1620 break; 1621 1622 /* Fault Event Upper Address Register, 32-bit */ 1623 case DMAR_FEUADDR_REG: 1624 VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64 1625 ", size %d, val 0x%"PRIx64, addr, size, val); 1626 assert(size == 4); 1627 vtd_set_long(s, addr, val); 1628 break; 1629 1630 /* Protected Memory Enable Register, 32-bit */ 1631 case DMAR_PMEN_REG: 1632 VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64 1633 ", size %d, val 0x%"PRIx64, addr, size, val); 1634 assert(size == 4); 1635 vtd_set_long(s, addr, val); 1636 break; 1637 1638 /* Root Table Address Register, 64-bit */ 1639 case DMAR_RTADDR_REG: 1640 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64 1641 ", size %d, val 0x%"PRIx64, addr, size, val); 1642 if (size == 4) { 1643 vtd_set_long(s, addr, val); 1644 } else { 1645 vtd_set_quad(s, addr, val); 1646 } 1647 break; 1648 1649 case DMAR_RTADDR_REG_HI: 1650 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64 1651 ", size %d, val 0x%"PRIx64, addr, size, val); 1652 assert(size == 4); 1653 vtd_set_long(s, addr, val); 1654 break; 1655 1656 /* Invalidation Queue Tail Register, 64-bit */ 1657 case DMAR_IQT_REG: 1658 VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64 1659 ", size %d, val 0x%"PRIx64, addr, size, val); 1660 if (size == 4) { 1661 vtd_set_long(s, addr, val); 1662 } else { 1663 vtd_set_quad(s, addr, val); 1664 } 1665 vtd_handle_iqt_write(s); 1666 break; 1667 1668 case DMAR_IQT_REG_HI: 1669 VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64 1670 ", size %d, val 0x%"PRIx64, addr, size, val); 1671 assert(size == 4); 1672 vtd_set_long(s, addr, val); 1673 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 1674 break; 1675 1676 /* Invalidation Queue Address Register, 64-bit */ 1677 case DMAR_IQA_REG: 1678 VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64 1679 ", size %d, val 0x%"PRIx64, addr, size, val); 1680 if (size == 4) { 1681 vtd_set_long(s, addr, val); 1682 } else { 1683 vtd_set_quad(s, addr, val); 1684 } 1685 break; 1686 1687 case DMAR_IQA_REG_HI: 1688 VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64 1689 ", size %d, val 0x%"PRIx64, addr, size, val); 1690 assert(size == 4); 1691 vtd_set_long(s, addr, val); 1692 break; 1693 1694 /* Invalidation Completion Status Register, 32-bit */ 1695 case DMAR_ICS_REG: 1696 VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64 1697 ", size %d, val 0x%"PRIx64, addr, size, val); 1698 assert(size == 4); 1699 vtd_set_long(s, addr, val); 1700 vtd_handle_ics_write(s); 1701 break; 1702 1703 /* Invalidation Event Control Register, 32-bit */ 1704 case DMAR_IECTL_REG: 1705 VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64 1706 ", size %d, val 0x%"PRIx64, addr, size, val); 1707 assert(size == 4); 1708 vtd_set_long(s, addr, val); 1709 vtd_handle_iectl_write(s); 1710 break; 1711 1712 /* Invalidation Event Data Register, 32-bit */ 1713 case DMAR_IEDATA_REG: 1714 VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64 1715 ", size %d, val 0x%"PRIx64, addr, size, val); 1716 assert(size == 4); 1717 vtd_set_long(s, addr, val); 1718 break; 1719 1720 /* Invalidation Event Address Register, 32-bit */ 1721 case DMAR_IEADDR_REG: 1722 VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64 1723 ", size %d, val 0x%"PRIx64, addr, size, val); 1724 assert(size == 4); 1725 vtd_set_long(s, addr, val); 1726 break; 1727 1728 /* Invalidation Event Upper Address Register, 32-bit */ 1729 case DMAR_IEUADDR_REG: 1730 VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64 1731 ", size %d, val 0x%"PRIx64, addr, size, val); 1732 assert(size == 4); 1733 vtd_set_long(s, addr, val); 1734 break; 1735 1736 /* Fault Recording Registers, 128-bit */ 1737 case DMAR_FRCD_REG_0_0: 1738 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64 1739 ", size %d, val 0x%"PRIx64, addr, size, val); 1740 if (size == 4) { 1741 vtd_set_long(s, addr, val); 1742 } else { 1743 vtd_set_quad(s, addr, val); 1744 } 1745 break; 1746 1747 case DMAR_FRCD_REG_0_1: 1748 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64 1749 ", size %d, val 0x%"PRIx64, addr, size, val); 1750 assert(size == 4); 1751 vtd_set_long(s, addr, val); 1752 break; 1753 1754 case DMAR_FRCD_REG_0_2: 1755 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64 1756 ", size %d, val 0x%"PRIx64, addr, size, val); 1757 if (size == 4) { 1758 vtd_set_long(s, addr, val); 1759 } else { 1760 vtd_set_quad(s, addr, val); 1761 /* May clear bit 127 (Fault), update PPF */ 1762 vtd_update_fsts_ppf(s); 1763 } 1764 break; 1765 1766 case DMAR_FRCD_REG_0_3: 1767 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64 1768 ", size %d, val 0x%"PRIx64, addr, size, val); 1769 assert(size == 4); 1770 vtd_set_long(s, addr, val); 1771 /* May clear bit 127 (Fault), update PPF */ 1772 vtd_update_fsts_ppf(s); 1773 break; 1774 1775 default: 1776 VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64 1777 ", size %d, val 0x%"PRIx64, addr, size, val); 1778 if (size == 4) { 1779 vtd_set_long(s, addr, val); 1780 } else { 1781 vtd_set_quad(s, addr, val); 1782 } 1783 } 1784 } 1785 1786 static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr, 1787 bool is_write) 1788 { 1789 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 1790 IntelIOMMUState *s = vtd_as->iommu_state; 1791 IOMMUTLBEntry ret = { 1792 .target_as = &address_space_memory, 1793 .iova = addr, 1794 .translated_addr = 0, 1795 .addr_mask = ~(hwaddr)0, 1796 .perm = IOMMU_NONE, 1797 }; 1798 1799 if (!s->dmar_enabled) { 1800 /* DMAR disabled, passthrough, use 4k-page*/ 1801 ret.iova = addr & VTD_PAGE_MASK_4K; 1802 ret.translated_addr = addr & VTD_PAGE_MASK_4K; 1803 ret.addr_mask = ~VTD_PAGE_MASK_4K; 1804 ret.perm = IOMMU_RW; 1805 return ret; 1806 } 1807 1808 vtd_do_iommu_translate(vtd_as, vtd_as->bus_num, vtd_as->devfn, addr, 1809 is_write, &ret); 1810 VTD_DPRINTF(MMU, 1811 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8 1812 " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, vtd_as->bus_num, 1813 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn), 1814 vtd_as->devfn, addr, ret.translated_addr); 1815 return ret; 1816 } 1817 1818 static const VMStateDescription vtd_vmstate = { 1819 .name = "iommu-intel", 1820 .unmigratable = 1, 1821 }; 1822 1823 static const MemoryRegionOps vtd_mem_ops = { 1824 .read = vtd_mem_read, 1825 .write = vtd_mem_write, 1826 .endianness = DEVICE_LITTLE_ENDIAN, 1827 .impl = { 1828 .min_access_size = 4, 1829 .max_access_size = 8, 1830 }, 1831 .valid = { 1832 .min_access_size = 4, 1833 .max_access_size = 8, 1834 }, 1835 }; 1836 1837 static Property vtd_properties[] = { 1838 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 1839 DEFINE_PROP_END_OF_LIST(), 1840 }; 1841 1842 /* Do the initialization. It will also be called when reset, so pay 1843 * attention when adding new initialization stuff. 1844 */ 1845 static void vtd_init(IntelIOMMUState *s) 1846 { 1847 memset(s->csr, 0, DMAR_REG_SIZE); 1848 memset(s->wmask, 0, DMAR_REG_SIZE); 1849 memset(s->w1cmask, 0, DMAR_REG_SIZE); 1850 memset(s->womask, 0, DMAR_REG_SIZE); 1851 1852 s->iommu_ops.translate = vtd_iommu_translate; 1853 s->root = 0; 1854 s->root_extended = false; 1855 s->dmar_enabled = false; 1856 s->iq_head = 0; 1857 s->iq_tail = 0; 1858 s->iq = 0; 1859 s->iq_size = 0; 1860 s->qi_enabled = false; 1861 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1862 s->next_frcd_reg = 0; 1863 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW | 1864 VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI; 1865 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 1866 1867 vtd_reset_context_cache(s); 1868 vtd_reset_iotlb(s); 1869 1870 /* Define registers with default values and bit semantics */ 1871 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 1872 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 1873 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 1874 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 1875 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 1876 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 1877 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0); 1878 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 1879 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 1880 1881 /* Advanced Fault Logging not supported */ 1882 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 1883 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 1884 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 1885 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 1886 1887 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 1888 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 1889 */ 1890 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 1891 1892 /* Treated as RO for implementations that PLMR and PHMR fields reported 1893 * as Clear in the CAP_REG. 1894 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 1895 */ 1896 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 1897 1898 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 1899 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 1900 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0); 1901 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 1902 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 1903 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 1904 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 1905 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 1906 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 1907 1908 /* IOTLB registers */ 1909 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 1910 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 1911 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 1912 1913 /* Fault Recording Registers, 128-bit */ 1914 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 1915 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 1916 } 1917 1918 /* Should not reset address_spaces when reset because devices will still use 1919 * the address space they got at first (won't ask the bus again). 1920 */ 1921 static void vtd_reset(DeviceState *dev) 1922 { 1923 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 1924 1925 VTD_DPRINTF(GENERAL, ""); 1926 vtd_init(s); 1927 } 1928 1929 static void vtd_realize(DeviceState *dev, Error **errp) 1930 { 1931 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 1932 1933 VTD_DPRINTF(GENERAL, ""); 1934 memset(s->address_spaces, 0, sizeof(s->address_spaces)); 1935 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 1936 "intel_iommu", DMAR_REG_SIZE); 1937 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem); 1938 /* No corresponding destroy */ 1939 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 1940 g_free, g_free); 1941 vtd_init(s); 1942 } 1943 1944 static void vtd_class_init(ObjectClass *klass, void *data) 1945 { 1946 DeviceClass *dc = DEVICE_CLASS(klass); 1947 1948 dc->reset = vtd_reset; 1949 dc->realize = vtd_realize; 1950 dc->vmsd = &vtd_vmstate; 1951 dc->props = vtd_properties; 1952 } 1953 1954 static const TypeInfo vtd_info = { 1955 .name = TYPE_INTEL_IOMMU_DEVICE, 1956 .parent = TYPE_SYS_BUS_DEVICE, 1957 .instance_size = sizeof(IntelIOMMUState), 1958 .class_init = vtd_class_init, 1959 }; 1960 1961 static void vtd_register_types(void) 1962 { 1963 VTD_DPRINTF(GENERAL, ""); 1964 type_register_static(&vtd_info); 1965 } 1966 1967 type_init(vtd_register_types) 1968