1 /* 2 * QEMU emulation of AMD IOMMU (AMD-Vi) 3 * 4 * Copyright (C) 2011 Eduard - Gabriel Munteanu 5 * Copyright (C) 2015, 2016 David Kiarie Kahurani 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 17 * You should have received a copy of the GNU General Public License along 18 * with this program; if not, see <http://www.gnu.org/licenses/>. 19 * 20 * Cache implementation inspired by hw/i386/intel_iommu.c 21 */ 22 23 #include "qemu/osdep.h" 24 #include "hw/i386/pc.h" 25 #include "hw/pci/msi.h" 26 #include "hw/pci/pci_bus.h" 27 #include "migration/vmstate.h" 28 #include "amd_iommu.h" 29 #include "qapi/error.h" 30 #include "qemu/error-report.h" 31 #include "hw/i386/apic_internal.h" 32 #include "trace.h" 33 #include "hw/i386/apic-msidef.h" 34 #include "hw/qdev-properties.h" 35 36 /* used AMD-Vi MMIO registers */ 37 const char *amdvi_mmio_low[] = { 38 "AMDVI_MMIO_DEVTAB_BASE", 39 "AMDVI_MMIO_CMDBUF_BASE", 40 "AMDVI_MMIO_EVTLOG_BASE", 41 "AMDVI_MMIO_CONTROL", 42 "AMDVI_MMIO_EXCL_BASE", 43 "AMDVI_MMIO_EXCL_LIMIT", 44 "AMDVI_MMIO_EXT_FEATURES", 45 "AMDVI_MMIO_PPR_BASE", 46 "UNHANDLED" 47 }; 48 const char *amdvi_mmio_high[] = { 49 "AMDVI_MMIO_COMMAND_HEAD", 50 "AMDVI_MMIO_COMMAND_TAIL", 51 "AMDVI_MMIO_EVTLOG_HEAD", 52 "AMDVI_MMIO_EVTLOG_TAIL", 53 "AMDVI_MMIO_STATUS", 54 "AMDVI_MMIO_PPR_HEAD", 55 "AMDVI_MMIO_PPR_TAIL", 56 "UNHANDLED" 57 }; 58 59 struct AMDVIAddressSpace { 60 uint8_t bus_num; /* bus number */ 61 uint8_t devfn; /* device function */ 62 AMDVIState *iommu_state; /* AMDVI - one per machine */ 63 MemoryRegion root; /* AMDVI Root memory map region */ 64 IOMMUMemoryRegion iommu; /* Device's address translation region */ 65 MemoryRegion iommu_ir; /* Device's interrupt remapping region */ 66 AddressSpace as; /* device's corresponding address space */ 67 }; 68 69 /* AMDVI cache entry */ 70 typedef struct AMDVIIOTLBEntry { 71 uint16_t domid; /* assigned domain id */ 72 uint16_t devid; /* device owning entry */ 73 uint64_t perms; /* access permissions */ 74 uint64_t translated_addr; /* translated address */ 75 uint64_t page_mask; /* physical page size */ 76 } AMDVIIOTLBEntry; 77 78 uint64_t amdvi_extended_feature_register(AMDVIState *s) 79 { 80 uint64_t feature = AMDVI_DEFAULT_EXT_FEATURES; 81 if (s->xtsup) { 82 feature |= AMDVI_FEATURE_XT; 83 } 84 85 return feature; 86 } 87 88 /* configure MMIO registers at startup/reset */ 89 static void amdvi_set_quad(AMDVIState *s, hwaddr addr, uint64_t val, 90 uint64_t romask, uint64_t w1cmask) 91 { 92 stq_le_p(&s->mmior[addr], val); 93 stq_le_p(&s->romask[addr], romask); 94 stq_le_p(&s->w1cmask[addr], w1cmask); 95 } 96 97 static uint16_t amdvi_readw(AMDVIState *s, hwaddr addr) 98 { 99 return lduw_le_p(&s->mmior[addr]); 100 } 101 102 static uint32_t amdvi_readl(AMDVIState *s, hwaddr addr) 103 { 104 return ldl_le_p(&s->mmior[addr]); 105 } 106 107 static uint64_t amdvi_readq(AMDVIState *s, hwaddr addr) 108 { 109 return ldq_le_p(&s->mmior[addr]); 110 } 111 112 /* internal write */ 113 static void amdvi_writeq_raw(AMDVIState *s, hwaddr addr, uint64_t val) 114 { 115 stq_le_p(&s->mmior[addr], val); 116 } 117 118 /* external write */ 119 static void amdvi_writew(AMDVIState *s, hwaddr addr, uint16_t val) 120 { 121 uint16_t romask = lduw_le_p(&s->romask[addr]); 122 uint16_t w1cmask = lduw_le_p(&s->w1cmask[addr]); 123 uint16_t oldval = lduw_le_p(&s->mmior[addr]); 124 stw_le_p(&s->mmior[addr], 125 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask)); 126 } 127 128 static void amdvi_writel(AMDVIState *s, hwaddr addr, uint32_t val) 129 { 130 uint32_t romask = ldl_le_p(&s->romask[addr]); 131 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 132 uint32_t oldval = ldl_le_p(&s->mmior[addr]); 133 stl_le_p(&s->mmior[addr], 134 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask)); 135 } 136 137 static void amdvi_writeq(AMDVIState *s, hwaddr addr, uint64_t val) 138 { 139 uint64_t romask = ldq_le_p(&s->romask[addr]); 140 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 141 uint32_t oldval = ldq_le_p(&s->mmior[addr]); 142 stq_le_p(&s->mmior[addr], 143 ((oldval & romask) | (val & ~romask)) & ~(val & w1cmask)); 144 } 145 146 /* OR a 64-bit register with a 64-bit value */ 147 static bool amdvi_test_mask(AMDVIState *s, hwaddr addr, uint64_t val) 148 { 149 return amdvi_readq(s, addr) | val; 150 } 151 152 /* OR a 64-bit register with a 64-bit value storing result in the register */ 153 static void amdvi_assign_orq(AMDVIState *s, hwaddr addr, uint64_t val) 154 { 155 amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) | val); 156 } 157 158 /* AND a 64-bit register with a 64-bit value storing result in the register */ 159 static void amdvi_assign_andq(AMDVIState *s, hwaddr addr, uint64_t val) 160 { 161 amdvi_writeq_raw(s, addr, amdvi_readq(s, addr) & val); 162 } 163 164 static void amdvi_generate_msi_interrupt(AMDVIState *s) 165 { 166 MSIMessage msg = {}; 167 MemTxAttrs attrs = { 168 .requester_id = pci_requester_id(&s->pci.dev) 169 }; 170 171 if (msi_enabled(&s->pci.dev)) { 172 msg = msi_get_message(&s->pci.dev, 0); 173 address_space_stl_le(&address_space_memory, msg.address, msg.data, 174 attrs, NULL); 175 } 176 } 177 178 static void amdvi_log_event(AMDVIState *s, uint64_t *evt) 179 { 180 /* event logging not enabled */ 181 if (!s->evtlog_enabled || amdvi_test_mask(s, AMDVI_MMIO_STATUS, 182 AMDVI_MMIO_STATUS_EVT_OVF)) { 183 return; 184 } 185 186 /* event log buffer full */ 187 if (s->evtlog_tail >= s->evtlog_len) { 188 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_OVF); 189 /* generate interrupt */ 190 amdvi_generate_msi_interrupt(s); 191 return; 192 } 193 194 if (dma_memory_write(&address_space_memory, s->evtlog + s->evtlog_tail, 195 evt, AMDVI_EVENT_LEN, MEMTXATTRS_UNSPECIFIED)) { 196 trace_amdvi_evntlog_fail(s->evtlog, s->evtlog_tail); 197 } 198 199 s->evtlog_tail += AMDVI_EVENT_LEN; 200 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT); 201 amdvi_generate_msi_interrupt(s); 202 } 203 204 static void amdvi_setevent_bits(uint64_t *buffer, uint64_t value, int start, 205 int length) 206 { 207 int index = start / 64, bitpos = start % 64; 208 uint64_t mask = MAKE_64BIT_MASK(start, length); 209 buffer[index] &= ~mask; 210 buffer[index] |= (value << bitpos) & mask; 211 } 212 /* 213 * AMDVi event structure 214 * 0:15 -> DeviceID 215 * 48:63 -> event type + miscellaneous info 216 * 64:127 -> related address 217 */ 218 static void amdvi_encode_event(uint64_t *evt, uint16_t devid, uint64_t addr, 219 uint16_t info) 220 { 221 evt[0] = 0; 222 evt[1] = 0; 223 224 amdvi_setevent_bits(evt, devid, 0, 16); 225 amdvi_setevent_bits(evt, info, 48, 16); 226 amdvi_setevent_bits(evt, addr, 64, 64); 227 } 228 /* log an error encountered during a page walk 229 * 230 * @addr: virtual address in translation request 231 */ 232 static void amdvi_page_fault(AMDVIState *s, uint16_t devid, 233 hwaddr addr, uint16_t info) 234 { 235 uint64_t evt[2]; 236 237 info |= AMDVI_EVENT_IOPF_I | AMDVI_EVENT_IOPF; 238 amdvi_encode_event(evt, devid, addr, info); 239 amdvi_log_event(s, evt); 240 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS, 241 PCI_STATUS_SIG_TARGET_ABORT); 242 } 243 /* 244 * log a master abort accessing device table 245 * @devtab : address of device table entry 246 * @info : error flags 247 */ 248 static void amdvi_log_devtab_error(AMDVIState *s, uint16_t devid, 249 hwaddr devtab, uint16_t info) 250 { 251 uint64_t evt[2]; 252 253 info |= AMDVI_EVENT_DEV_TAB_HW_ERROR; 254 255 amdvi_encode_event(evt, devid, devtab, info); 256 amdvi_log_event(s, evt); 257 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS, 258 PCI_STATUS_SIG_TARGET_ABORT); 259 } 260 /* log an event trying to access command buffer 261 * @addr : address that couldn't be accessed 262 */ 263 static void amdvi_log_command_error(AMDVIState *s, hwaddr addr) 264 { 265 uint64_t evt[2]; 266 uint16_t info = AMDVI_EVENT_COMMAND_HW_ERROR; 267 268 amdvi_encode_event(evt, 0, addr, info); 269 amdvi_log_event(s, evt); 270 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS, 271 PCI_STATUS_SIG_TARGET_ABORT); 272 } 273 /* log an illegal command event 274 * @addr : address of illegal command 275 */ 276 static void amdvi_log_illegalcom_error(AMDVIState *s, uint16_t info, 277 hwaddr addr) 278 { 279 uint64_t evt[2]; 280 281 info |= AMDVI_EVENT_ILLEGAL_COMMAND_ERROR; 282 amdvi_encode_event(evt, 0, addr, info); 283 amdvi_log_event(s, evt); 284 } 285 /* log an error accessing device table 286 * 287 * @devid : device owning the table entry 288 * @devtab : address of device table entry 289 * @info : error flags 290 */ 291 static void amdvi_log_illegaldevtab_error(AMDVIState *s, uint16_t devid, 292 hwaddr addr, uint16_t info) 293 { 294 uint64_t evt[2]; 295 296 info |= AMDVI_EVENT_ILLEGAL_DEVTAB_ENTRY; 297 amdvi_encode_event(evt, devid, addr, info); 298 amdvi_log_event(s, evt); 299 } 300 /* log an error accessing a PTE entry 301 * @addr : address that couldn't be accessed 302 */ 303 static void amdvi_log_pagetab_error(AMDVIState *s, uint16_t devid, 304 hwaddr addr, uint16_t info) 305 { 306 uint64_t evt[2]; 307 308 info |= AMDVI_EVENT_PAGE_TAB_HW_ERROR; 309 amdvi_encode_event(evt, devid, addr, info); 310 amdvi_log_event(s, evt); 311 pci_word_test_and_set_mask(s->pci.dev.config + PCI_STATUS, 312 PCI_STATUS_SIG_TARGET_ABORT); 313 } 314 315 static gboolean amdvi_uint64_equal(gconstpointer v1, gconstpointer v2) 316 { 317 return *((const uint64_t *)v1) == *((const uint64_t *)v2); 318 } 319 320 static guint amdvi_uint64_hash(gconstpointer v) 321 { 322 return (guint)*(const uint64_t *)v; 323 } 324 325 static AMDVIIOTLBEntry *amdvi_iotlb_lookup(AMDVIState *s, hwaddr addr, 326 uint64_t devid) 327 { 328 uint64_t key = (addr >> AMDVI_PAGE_SHIFT_4K) | 329 ((uint64_t)(devid) << AMDVI_DEVID_SHIFT); 330 return g_hash_table_lookup(s->iotlb, &key); 331 } 332 333 static void amdvi_iotlb_reset(AMDVIState *s) 334 { 335 assert(s->iotlb); 336 trace_amdvi_iotlb_reset(); 337 g_hash_table_remove_all(s->iotlb); 338 } 339 340 static gboolean amdvi_iotlb_remove_by_devid(gpointer key, gpointer value, 341 gpointer user_data) 342 { 343 AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value; 344 uint16_t devid = *(uint16_t *)user_data; 345 return entry->devid == devid; 346 } 347 348 static void amdvi_iotlb_remove_page(AMDVIState *s, hwaddr addr, 349 uint64_t devid) 350 { 351 uint64_t key = (addr >> AMDVI_PAGE_SHIFT_4K) | 352 ((uint64_t)(devid) << AMDVI_DEVID_SHIFT); 353 g_hash_table_remove(s->iotlb, &key); 354 } 355 356 static void amdvi_update_iotlb(AMDVIState *s, uint16_t devid, 357 uint64_t gpa, IOMMUTLBEntry to_cache, 358 uint16_t domid) 359 { 360 AMDVIIOTLBEntry *entry = g_new(AMDVIIOTLBEntry, 1); 361 uint64_t *key = g_new(uint64_t, 1); 362 uint64_t gfn = gpa >> AMDVI_PAGE_SHIFT_4K; 363 364 /* don't cache erroneous translations */ 365 if (to_cache.perm != IOMMU_NONE) { 366 trace_amdvi_cache_update(domid, PCI_BUS_NUM(devid), PCI_SLOT(devid), 367 PCI_FUNC(devid), gpa, to_cache.translated_addr); 368 369 if (g_hash_table_size(s->iotlb) >= AMDVI_IOTLB_MAX_SIZE) { 370 amdvi_iotlb_reset(s); 371 } 372 373 entry->domid = domid; 374 entry->perms = to_cache.perm; 375 entry->translated_addr = to_cache.translated_addr; 376 entry->page_mask = to_cache.addr_mask; 377 *key = gfn | ((uint64_t)(devid) << AMDVI_DEVID_SHIFT); 378 g_hash_table_replace(s->iotlb, key, entry); 379 } 380 } 381 382 static void amdvi_completion_wait(AMDVIState *s, uint64_t *cmd) 383 { 384 /* pad the last 3 bits */ 385 hwaddr addr = cpu_to_le64(extract64(cmd[0], 3, 49)) << 3; 386 uint64_t data = cpu_to_le64(cmd[1]); 387 388 if (extract64(cmd[0], 52, 8)) { 389 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4), 390 s->cmdbuf + s->cmdbuf_head); 391 } 392 if (extract64(cmd[0], 0, 1)) { 393 if (dma_memory_write(&address_space_memory, addr, &data, 394 AMDVI_COMPLETION_DATA_SIZE, 395 MEMTXATTRS_UNSPECIFIED)) { 396 trace_amdvi_completion_wait_fail(addr); 397 } 398 } 399 /* set completion interrupt */ 400 if (extract64(cmd[0], 1, 1)) { 401 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_COMP_INT); 402 /* generate interrupt */ 403 amdvi_generate_msi_interrupt(s); 404 } 405 trace_amdvi_completion_wait(addr, data); 406 } 407 408 /* log error without aborting since linux seems to be using reserved bits */ 409 static void amdvi_inval_devtab_entry(AMDVIState *s, uint64_t *cmd) 410 { 411 uint16_t devid = cpu_to_le16((uint16_t)extract64(cmd[0], 0, 16)); 412 413 /* This command should invalidate internal caches of which there isn't */ 414 if (extract64(cmd[0], 16, 44) || cmd[1]) { 415 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4), 416 s->cmdbuf + s->cmdbuf_head); 417 } 418 trace_amdvi_devtab_inval(PCI_BUS_NUM(devid), PCI_SLOT(devid), 419 PCI_FUNC(devid)); 420 } 421 422 static void amdvi_complete_ppr(AMDVIState *s, uint64_t *cmd) 423 { 424 if (extract64(cmd[0], 16, 16) || extract64(cmd[0], 52, 8) || 425 extract64(cmd[1], 0, 2) || extract64(cmd[1], 3, 29) 426 || extract64(cmd[1], 48, 16)) { 427 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4), 428 s->cmdbuf + s->cmdbuf_head); 429 } 430 trace_amdvi_ppr_exec(); 431 } 432 433 static void amdvi_inval_all(AMDVIState *s, uint64_t *cmd) 434 { 435 if (extract64(cmd[0], 0, 60) || cmd[1]) { 436 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4), 437 s->cmdbuf + s->cmdbuf_head); 438 } 439 440 amdvi_iotlb_reset(s); 441 trace_amdvi_all_inval(); 442 } 443 444 static gboolean amdvi_iotlb_remove_by_domid(gpointer key, gpointer value, 445 gpointer user_data) 446 { 447 AMDVIIOTLBEntry *entry = (AMDVIIOTLBEntry *)value; 448 uint16_t domid = *(uint16_t *)user_data; 449 return entry->domid == domid; 450 } 451 452 /* we don't have devid - we can't remove pages by address */ 453 static void amdvi_inval_pages(AMDVIState *s, uint64_t *cmd) 454 { 455 uint16_t domid = cpu_to_le16((uint16_t)extract64(cmd[0], 32, 16)); 456 457 if (extract64(cmd[0], 20, 12) || extract64(cmd[0], 48, 12) || 458 extract64(cmd[1], 3, 9)) { 459 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4), 460 s->cmdbuf + s->cmdbuf_head); 461 } 462 463 g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_domid, 464 &domid); 465 trace_amdvi_pages_inval(domid); 466 } 467 468 static void amdvi_prefetch_pages(AMDVIState *s, uint64_t *cmd) 469 { 470 if (extract64(cmd[0], 16, 8) || extract64(cmd[0], 52, 8) || 471 extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) || 472 extract64(cmd[1], 5, 7)) { 473 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4), 474 s->cmdbuf + s->cmdbuf_head); 475 } 476 477 trace_amdvi_prefetch_pages(); 478 } 479 480 static void amdvi_inval_inttable(AMDVIState *s, uint64_t *cmd) 481 { 482 if (extract64(cmd[0], 16, 44) || cmd[1]) { 483 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4), 484 s->cmdbuf + s->cmdbuf_head); 485 return; 486 } 487 488 trace_amdvi_intr_inval(); 489 } 490 491 /* FIXME: Try to work with the specified size instead of all the pages 492 * when the S bit is on 493 */ 494 static void iommu_inval_iotlb(AMDVIState *s, uint64_t *cmd) 495 { 496 497 uint16_t devid = extract64(cmd[0], 0, 16); 498 if (extract64(cmd[1], 1, 1) || extract64(cmd[1], 3, 1) || 499 extract64(cmd[1], 6, 6)) { 500 amdvi_log_illegalcom_error(s, extract64(cmd[0], 60, 4), 501 s->cmdbuf + s->cmdbuf_head); 502 return; 503 } 504 505 if (extract64(cmd[1], 0, 1)) { 506 g_hash_table_foreach_remove(s->iotlb, amdvi_iotlb_remove_by_devid, 507 &devid); 508 } else { 509 amdvi_iotlb_remove_page(s, cpu_to_le64(extract64(cmd[1], 12, 52)) << 12, 510 cpu_to_le16(extract64(cmd[1], 0, 16))); 511 } 512 trace_amdvi_iotlb_inval(); 513 } 514 515 /* not honouring reserved bits is regarded as an illegal command */ 516 static void amdvi_cmdbuf_exec(AMDVIState *s) 517 { 518 uint64_t cmd[2]; 519 520 if (dma_memory_read(&address_space_memory, s->cmdbuf + s->cmdbuf_head, 521 cmd, AMDVI_COMMAND_SIZE, MEMTXATTRS_UNSPECIFIED)) { 522 trace_amdvi_command_read_fail(s->cmdbuf, s->cmdbuf_head); 523 amdvi_log_command_error(s, s->cmdbuf + s->cmdbuf_head); 524 return; 525 } 526 527 switch (extract64(cmd[0], 60, 4)) { 528 case AMDVI_CMD_COMPLETION_WAIT: 529 amdvi_completion_wait(s, cmd); 530 break; 531 case AMDVI_CMD_INVAL_DEVTAB_ENTRY: 532 amdvi_inval_devtab_entry(s, cmd); 533 break; 534 case AMDVI_CMD_INVAL_AMDVI_PAGES: 535 amdvi_inval_pages(s, cmd); 536 break; 537 case AMDVI_CMD_INVAL_IOTLB_PAGES: 538 iommu_inval_iotlb(s, cmd); 539 break; 540 case AMDVI_CMD_INVAL_INTR_TABLE: 541 amdvi_inval_inttable(s, cmd); 542 break; 543 case AMDVI_CMD_PREFETCH_AMDVI_PAGES: 544 amdvi_prefetch_pages(s, cmd); 545 break; 546 case AMDVI_CMD_COMPLETE_PPR_REQUEST: 547 amdvi_complete_ppr(s, cmd); 548 break; 549 case AMDVI_CMD_INVAL_AMDVI_ALL: 550 amdvi_inval_all(s, cmd); 551 break; 552 default: 553 trace_amdvi_unhandled_command(extract64(cmd[1], 60, 4)); 554 /* log illegal command */ 555 amdvi_log_illegalcom_error(s, extract64(cmd[1], 60, 4), 556 s->cmdbuf + s->cmdbuf_head); 557 } 558 } 559 560 static void amdvi_cmdbuf_run(AMDVIState *s) 561 { 562 if (!s->cmdbuf_enabled) { 563 trace_amdvi_command_error(amdvi_readq(s, AMDVI_MMIO_CONTROL)); 564 return; 565 } 566 567 /* check if there is work to do. */ 568 while (s->cmdbuf_head != s->cmdbuf_tail) { 569 trace_amdvi_command_exec(s->cmdbuf_head, s->cmdbuf_tail, s->cmdbuf); 570 amdvi_cmdbuf_exec(s); 571 s->cmdbuf_head += AMDVI_COMMAND_SIZE; 572 amdvi_writeq_raw(s, AMDVI_MMIO_COMMAND_HEAD, s->cmdbuf_head); 573 574 /* wrap head pointer */ 575 if (s->cmdbuf_head >= s->cmdbuf_len * AMDVI_COMMAND_SIZE) { 576 s->cmdbuf_head = 0; 577 } 578 } 579 } 580 581 static void amdvi_mmio_trace(hwaddr addr, unsigned size) 582 { 583 uint8_t index = (addr & ~0x2000) / 8; 584 585 if ((addr & 0x2000)) { 586 /* high table */ 587 index = index >= AMDVI_MMIO_REGS_HIGH ? AMDVI_MMIO_REGS_HIGH : index; 588 trace_amdvi_mmio_read(amdvi_mmio_high[index], addr, size, addr & ~0x07); 589 } else { 590 index = index >= AMDVI_MMIO_REGS_LOW ? AMDVI_MMIO_REGS_LOW : index; 591 trace_amdvi_mmio_read(amdvi_mmio_low[index], addr, size, addr & ~0x07); 592 } 593 } 594 595 static uint64_t amdvi_mmio_read(void *opaque, hwaddr addr, unsigned size) 596 { 597 AMDVIState *s = opaque; 598 599 uint64_t val = -1; 600 if (addr + size > AMDVI_MMIO_SIZE) { 601 trace_amdvi_mmio_read_invalid(AMDVI_MMIO_SIZE, addr, size); 602 return (uint64_t)-1; 603 } 604 605 if (size == 2) { 606 val = amdvi_readw(s, addr); 607 } else if (size == 4) { 608 val = amdvi_readl(s, addr); 609 } else if (size == 8) { 610 val = amdvi_readq(s, addr); 611 } 612 amdvi_mmio_trace(addr, size); 613 614 return val; 615 } 616 617 static void amdvi_handle_control_write(AMDVIState *s) 618 { 619 unsigned long control = amdvi_readq(s, AMDVI_MMIO_CONTROL); 620 s->enabled = !!(control & AMDVI_MMIO_CONTROL_AMDVIEN); 621 622 s->ats_enabled = !!(control & AMDVI_MMIO_CONTROL_HTTUNEN); 623 s->evtlog_enabled = s->enabled && !!(control & 624 AMDVI_MMIO_CONTROL_EVENTLOGEN); 625 626 s->evtlog_intr = !!(control & AMDVI_MMIO_CONTROL_EVENTINTEN); 627 s->completion_wait_intr = !!(control & AMDVI_MMIO_CONTROL_COMWAITINTEN); 628 s->cmdbuf_enabled = s->enabled && !!(control & 629 AMDVI_MMIO_CONTROL_CMDBUFLEN); 630 s->ga_enabled = !!(control & AMDVI_MMIO_CONTROL_GAEN); 631 632 /* update the flags depending on the control register */ 633 if (s->cmdbuf_enabled) { 634 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_CMDBUF_RUN); 635 } else { 636 amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_CMDBUF_RUN); 637 } 638 if (s->evtlog_enabled) { 639 amdvi_assign_orq(s, AMDVI_MMIO_STATUS, AMDVI_MMIO_STATUS_EVT_RUN); 640 } else { 641 amdvi_assign_andq(s, AMDVI_MMIO_STATUS, ~AMDVI_MMIO_STATUS_EVT_RUN); 642 } 643 644 trace_amdvi_control_status(control); 645 amdvi_cmdbuf_run(s); 646 } 647 648 static inline void amdvi_handle_devtab_write(AMDVIState *s) 649 650 { 651 uint64_t val = amdvi_readq(s, AMDVI_MMIO_DEVICE_TABLE); 652 s->devtab = (val & AMDVI_MMIO_DEVTAB_BASE_MASK); 653 654 /* set device table length */ 655 s->devtab_len = ((val & AMDVI_MMIO_DEVTAB_SIZE_MASK) + 1 * 656 (AMDVI_MMIO_DEVTAB_SIZE_UNIT / 657 AMDVI_MMIO_DEVTAB_ENTRY_SIZE)); 658 } 659 660 static inline void amdvi_handle_cmdhead_write(AMDVIState *s) 661 { 662 s->cmdbuf_head = amdvi_readq(s, AMDVI_MMIO_COMMAND_HEAD) 663 & AMDVI_MMIO_CMDBUF_HEAD_MASK; 664 amdvi_cmdbuf_run(s); 665 } 666 667 static inline void amdvi_handle_cmdbase_write(AMDVIState *s) 668 { 669 s->cmdbuf = amdvi_readq(s, AMDVI_MMIO_COMMAND_BASE) 670 & AMDVI_MMIO_CMDBUF_BASE_MASK; 671 s->cmdbuf_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_CMDBUF_SIZE_BYTE) 672 & AMDVI_MMIO_CMDBUF_SIZE_MASK); 673 s->cmdbuf_head = s->cmdbuf_tail = 0; 674 } 675 676 static inline void amdvi_handle_cmdtail_write(AMDVIState *s) 677 { 678 s->cmdbuf_tail = amdvi_readq(s, AMDVI_MMIO_COMMAND_TAIL) 679 & AMDVI_MMIO_CMDBUF_TAIL_MASK; 680 amdvi_cmdbuf_run(s); 681 } 682 683 static inline void amdvi_handle_excllim_write(AMDVIState *s) 684 { 685 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EXCL_LIMIT); 686 s->excl_limit = (val & AMDVI_MMIO_EXCL_LIMIT_MASK) | 687 AMDVI_MMIO_EXCL_LIMIT_LOW; 688 } 689 690 static inline void amdvi_handle_evtbase_write(AMDVIState *s) 691 { 692 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_BASE); 693 s->evtlog = val & AMDVI_MMIO_EVTLOG_BASE_MASK; 694 s->evtlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_EVTLOG_SIZE_BYTE) 695 & AMDVI_MMIO_EVTLOG_SIZE_MASK); 696 } 697 698 static inline void amdvi_handle_evttail_write(AMDVIState *s) 699 { 700 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_TAIL); 701 s->evtlog_tail = val & AMDVI_MMIO_EVTLOG_TAIL_MASK; 702 } 703 704 static inline void amdvi_handle_evthead_write(AMDVIState *s) 705 { 706 uint64_t val = amdvi_readq(s, AMDVI_MMIO_EVENT_HEAD); 707 s->evtlog_head = val & AMDVI_MMIO_EVTLOG_HEAD_MASK; 708 } 709 710 static inline void amdvi_handle_pprbase_write(AMDVIState *s) 711 { 712 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_BASE); 713 s->ppr_log = val & AMDVI_MMIO_PPRLOG_BASE_MASK; 714 s->pprlog_len = 1UL << (amdvi_readq(s, AMDVI_MMIO_PPRLOG_SIZE_BYTE) 715 & AMDVI_MMIO_PPRLOG_SIZE_MASK); 716 } 717 718 static inline void amdvi_handle_pprhead_write(AMDVIState *s) 719 { 720 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_HEAD); 721 s->pprlog_head = val & AMDVI_MMIO_PPRLOG_HEAD_MASK; 722 } 723 724 static inline void amdvi_handle_pprtail_write(AMDVIState *s) 725 { 726 uint64_t val = amdvi_readq(s, AMDVI_MMIO_PPR_TAIL); 727 s->pprlog_tail = val & AMDVI_MMIO_PPRLOG_TAIL_MASK; 728 } 729 730 /* FIXME: something might go wrong if System Software writes in chunks 731 * of one byte but linux writes in chunks of 4 bytes so currently it 732 * works correctly with linux but will definitely be busted if software 733 * reads/writes 8 bytes 734 */ 735 static void amdvi_mmio_reg_write(AMDVIState *s, unsigned size, uint64_t val, 736 hwaddr addr) 737 { 738 if (size == 2) { 739 amdvi_writew(s, addr, val); 740 } else if (size == 4) { 741 amdvi_writel(s, addr, val); 742 } else if (size == 8) { 743 amdvi_writeq(s, addr, val); 744 } 745 } 746 747 static void amdvi_mmio_write(void *opaque, hwaddr addr, uint64_t val, 748 unsigned size) 749 { 750 AMDVIState *s = opaque; 751 unsigned long offset = addr & 0x07; 752 753 if (addr + size > AMDVI_MMIO_SIZE) { 754 trace_amdvi_mmio_write("error: addr outside region: max ", 755 (uint64_t)AMDVI_MMIO_SIZE, size, val, offset); 756 return; 757 } 758 759 amdvi_mmio_trace(addr, size); 760 switch (addr & ~0x07) { 761 case AMDVI_MMIO_CONTROL: 762 amdvi_mmio_reg_write(s, size, val, addr); 763 amdvi_handle_control_write(s); 764 break; 765 case AMDVI_MMIO_DEVICE_TABLE: 766 amdvi_mmio_reg_write(s, size, val, addr); 767 /* set device table address 768 * This also suffers from inability to tell whether software 769 * is done writing 770 */ 771 if (offset || (size == 8)) { 772 amdvi_handle_devtab_write(s); 773 } 774 break; 775 case AMDVI_MMIO_COMMAND_HEAD: 776 amdvi_mmio_reg_write(s, size, val, addr); 777 amdvi_handle_cmdhead_write(s); 778 break; 779 case AMDVI_MMIO_COMMAND_BASE: 780 amdvi_mmio_reg_write(s, size, val, addr); 781 /* FIXME - make sure System Software has finished writing in case 782 * it writes in chucks less than 8 bytes in a robust way.As for 783 * now, this hacks works for the linux driver 784 */ 785 if (offset || (size == 8)) { 786 amdvi_handle_cmdbase_write(s); 787 } 788 break; 789 case AMDVI_MMIO_COMMAND_TAIL: 790 amdvi_mmio_reg_write(s, size, val, addr); 791 amdvi_handle_cmdtail_write(s); 792 break; 793 case AMDVI_MMIO_EVENT_BASE: 794 amdvi_mmio_reg_write(s, size, val, addr); 795 amdvi_handle_evtbase_write(s); 796 break; 797 case AMDVI_MMIO_EVENT_HEAD: 798 amdvi_mmio_reg_write(s, size, val, addr); 799 amdvi_handle_evthead_write(s); 800 break; 801 case AMDVI_MMIO_EVENT_TAIL: 802 amdvi_mmio_reg_write(s, size, val, addr); 803 amdvi_handle_evttail_write(s); 804 break; 805 case AMDVI_MMIO_EXCL_LIMIT: 806 amdvi_mmio_reg_write(s, size, val, addr); 807 amdvi_handle_excllim_write(s); 808 break; 809 /* PPR log base - unused for now */ 810 case AMDVI_MMIO_PPR_BASE: 811 amdvi_mmio_reg_write(s, size, val, addr); 812 amdvi_handle_pprbase_write(s); 813 break; 814 /* PPR log head - also unused for now */ 815 case AMDVI_MMIO_PPR_HEAD: 816 amdvi_mmio_reg_write(s, size, val, addr); 817 amdvi_handle_pprhead_write(s); 818 break; 819 /* PPR log tail - unused for now */ 820 case AMDVI_MMIO_PPR_TAIL: 821 amdvi_mmio_reg_write(s, size, val, addr); 822 amdvi_handle_pprtail_write(s); 823 break; 824 } 825 } 826 827 static inline uint64_t amdvi_get_perms(uint64_t entry) 828 { 829 return (entry & (AMDVI_DEV_PERM_READ | AMDVI_DEV_PERM_WRITE)) >> 830 AMDVI_DEV_PERM_SHIFT; 831 } 832 833 /* validate that reserved bits are honoured */ 834 static bool amdvi_validate_dte(AMDVIState *s, uint16_t devid, 835 uint64_t *dte) 836 { 837 if ((dte[0] & AMDVI_DTE_LOWER_QUAD_RESERVED) 838 || (dte[1] & AMDVI_DTE_MIDDLE_QUAD_RESERVED) 839 || (dte[2] & AMDVI_DTE_UPPER_QUAD_RESERVED) || dte[3]) { 840 amdvi_log_illegaldevtab_error(s, devid, 841 s->devtab + 842 devid * AMDVI_DEVTAB_ENTRY_SIZE, 0); 843 return false; 844 } 845 846 return true; 847 } 848 849 /* get a device table entry given the devid */ 850 static bool amdvi_get_dte(AMDVIState *s, int devid, uint64_t *entry) 851 { 852 uint32_t offset = devid * AMDVI_DEVTAB_ENTRY_SIZE; 853 854 if (dma_memory_read(&address_space_memory, s->devtab + offset, entry, 855 AMDVI_DEVTAB_ENTRY_SIZE, MEMTXATTRS_UNSPECIFIED)) { 856 trace_amdvi_dte_get_fail(s->devtab, offset); 857 /* log error accessing dte */ 858 amdvi_log_devtab_error(s, devid, s->devtab + offset, 0); 859 return false; 860 } 861 862 *entry = le64_to_cpu(*entry); 863 if (!amdvi_validate_dte(s, devid, entry)) { 864 trace_amdvi_invalid_dte(entry[0]); 865 return false; 866 } 867 868 return true; 869 } 870 871 /* get pte translation mode */ 872 static inline uint8_t get_pte_translation_mode(uint64_t pte) 873 { 874 return (pte >> AMDVI_DEV_MODE_RSHIFT) & AMDVI_DEV_MODE_MASK; 875 } 876 877 static inline uint64_t pte_override_page_mask(uint64_t pte) 878 { 879 uint8_t page_mask = 13; 880 uint64_t addr = (pte & AMDVI_DEV_PT_ROOT_MASK) >> 12; 881 /* find the first zero bit */ 882 while (addr & 1) { 883 page_mask++; 884 addr = addr >> 1; 885 } 886 887 return ~((1ULL << page_mask) - 1); 888 } 889 890 static inline uint64_t pte_get_page_mask(uint64_t oldlevel) 891 { 892 return ~((1UL << ((oldlevel * 9) + 3)) - 1); 893 } 894 895 static inline uint64_t amdvi_get_pte_entry(AMDVIState *s, uint64_t pte_addr, 896 uint16_t devid) 897 { 898 uint64_t pte; 899 900 if (dma_memory_read(&address_space_memory, pte_addr, 901 &pte, sizeof(pte), MEMTXATTRS_UNSPECIFIED)) { 902 trace_amdvi_get_pte_hwerror(pte_addr); 903 amdvi_log_pagetab_error(s, devid, pte_addr, 0); 904 pte = 0; 905 return pte; 906 } 907 908 pte = le64_to_cpu(pte); 909 return pte; 910 } 911 912 static void amdvi_page_walk(AMDVIAddressSpace *as, uint64_t *dte, 913 IOMMUTLBEntry *ret, unsigned perms, 914 hwaddr addr) 915 { 916 unsigned level, present, pte_perms, oldlevel; 917 uint64_t pte = dte[0], pte_addr, page_mask; 918 919 /* make sure the DTE has TV = 1 */ 920 if (pte & AMDVI_DEV_TRANSLATION_VALID) { 921 level = get_pte_translation_mode(pte); 922 if (level >= 7) { 923 trace_amdvi_mode_invalid(level, addr); 924 return; 925 } 926 if (level == 0) { 927 goto no_remap; 928 } 929 930 /* we are at the leaf page table or page table encodes a huge page */ 931 do { 932 pte_perms = amdvi_get_perms(pte); 933 present = pte & 1; 934 if (!present || perms != (perms & pte_perms)) { 935 amdvi_page_fault(as->iommu_state, as->devfn, addr, perms); 936 trace_amdvi_page_fault(addr); 937 return; 938 } 939 940 /* go to the next lower level */ 941 pte_addr = pte & AMDVI_DEV_PT_ROOT_MASK; 942 /* add offset and load pte */ 943 pte_addr += ((addr >> (3 + 9 * level)) & 0x1FF) << 3; 944 pte = amdvi_get_pte_entry(as->iommu_state, pte_addr, as->devfn); 945 if (!pte) { 946 return; 947 } 948 oldlevel = level; 949 level = get_pte_translation_mode(pte); 950 } while (level > 0 && level < 7); 951 952 if (level == 0x7) { 953 page_mask = pte_override_page_mask(pte); 954 } else { 955 page_mask = pte_get_page_mask(oldlevel); 956 } 957 958 /* get access permissions from pte */ 959 ret->iova = addr & page_mask; 960 ret->translated_addr = (pte & AMDVI_DEV_PT_ROOT_MASK) & page_mask; 961 ret->addr_mask = ~page_mask; 962 ret->perm = amdvi_get_perms(pte); 963 return; 964 } 965 no_remap: 966 ret->iova = addr & AMDVI_PAGE_MASK_4K; 967 ret->translated_addr = addr & AMDVI_PAGE_MASK_4K; 968 ret->addr_mask = ~AMDVI_PAGE_MASK_4K; 969 ret->perm = amdvi_get_perms(pte); 970 } 971 972 static void amdvi_do_translate(AMDVIAddressSpace *as, hwaddr addr, 973 bool is_write, IOMMUTLBEntry *ret) 974 { 975 AMDVIState *s = as->iommu_state; 976 uint16_t devid = PCI_BUILD_BDF(as->bus_num, as->devfn); 977 AMDVIIOTLBEntry *iotlb_entry = amdvi_iotlb_lookup(s, addr, devid); 978 uint64_t entry[4]; 979 980 if (iotlb_entry) { 981 trace_amdvi_iotlb_hit(PCI_BUS_NUM(devid), PCI_SLOT(devid), 982 PCI_FUNC(devid), addr, iotlb_entry->translated_addr); 983 ret->iova = addr & ~iotlb_entry->page_mask; 984 ret->translated_addr = iotlb_entry->translated_addr; 985 ret->addr_mask = iotlb_entry->page_mask; 986 ret->perm = iotlb_entry->perms; 987 return; 988 } 989 990 if (!amdvi_get_dte(s, devid, entry)) { 991 return; 992 } 993 994 /* devices with V = 0 are not translated */ 995 if (!(entry[0] & AMDVI_DEV_VALID)) { 996 goto out; 997 } 998 999 amdvi_page_walk(as, entry, ret, 1000 is_write ? AMDVI_PERM_WRITE : AMDVI_PERM_READ, addr); 1001 1002 amdvi_update_iotlb(s, devid, addr, *ret, 1003 entry[1] & AMDVI_DEV_DOMID_ID_MASK); 1004 return; 1005 1006 out: 1007 ret->iova = addr & AMDVI_PAGE_MASK_4K; 1008 ret->translated_addr = addr & AMDVI_PAGE_MASK_4K; 1009 ret->addr_mask = ~AMDVI_PAGE_MASK_4K; 1010 ret->perm = IOMMU_RW; 1011 } 1012 1013 static inline bool amdvi_is_interrupt_addr(hwaddr addr) 1014 { 1015 return addr >= AMDVI_INT_ADDR_FIRST && addr <= AMDVI_INT_ADDR_LAST; 1016 } 1017 1018 static IOMMUTLBEntry amdvi_translate(IOMMUMemoryRegion *iommu, hwaddr addr, 1019 IOMMUAccessFlags flag, int iommu_idx) 1020 { 1021 AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu); 1022 AMDVIState *s = as->iommu_state; 1023 IOMMUTLBEntry ret = { 1024 .target_as = &address_space_memory, 1025 .iova = addr, 1026 .translated_addr = 0, 1027 .addr_mask = ~(hwaddr)0, 1028 .perm = IOMMU_NONE 1029 }; 1030 1031 if (!s->enabled) { 1032 /* AMDVI disabled - corresponds to iommu=off not 1033 * failure to provide any parameter 1034 */ 1035 ret.iova = addr & AMDVI_PAGE_MASK_4K; 1036 ret.translated_addr = addr & AMDVI_PAGE_MASK_4K; 1037 ret.addr_mask = ~AMDVI_PAGE_MASK_4K; 1038 ret.perm = IOMMU_RW; 1039 return ret; 1040 } else if (amdvi_is_interrupt_addr(addr)) { 1041 ret.iova = addr & AMDVI_PAGE_MASK_4K; 1042 ret.translated_addr = addr & AMDVI_PAGE_MASK_4K; 1043 ret.addr_mask = ~AMDVI_PAGE_MASK_4K; 1044 ret.perm = IOMMU_WO; 1045 return ret; 1046 } 1047 1048 amdvi_do_translate(as, addr, flag & IOMMU_WO, &ret); 1049 trace_amdvi_translation_result(as->bus_num, PCI_SLOT(as->devfn), 1050 PCI_FUNC(as->devfn), addr, ret.translated_addr); 1051 return ret; 1052 } 1053 1054 static int amdvi_get_irte(AMDVIState *s, MSIMessage *origin, uint64_t *dte, 1055 union irte *irte, uint16_t devid) 1056 { 1057 uint64_t irte_root, offset; 1058 1059 irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK; 1060 offset = (origin->data & AMDVI_IRTE_OFFSET) << 2; 1061 1062 trace_amdvi_ir_irte(irte_root, offset); 1063 1064 if (dma_memory_read(&address_space_memory, irte_root + offset, 1065 irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) { 1066 trace_amdvi_ir_err("failed to get irte"); 1067 return -AMDVI_IR_GET_IRTE; 1068 } 1069 1070 trace_amdvi_ir_irte_val(irte->val); 1071 1072 return 0; 1073 } 1074 1075 static int amdvi_int_remap_legacy(AMDVIState *iommu, 1076 MSIMessage *origin, 1077 MSIMessage *translated, 1078 uint64_t *dte, 1079 X86IOMMUIrq *irq, 1080 uint16_t sid) 1081 { 1082 int ret; 1083 union irte irte; 1084 1085 /* get interrupt remapping table */ 1086 ret = amdvi_get_irte(iommu, origin, dte, &irte, sid); 1087 if (ret < 0) { 1088 return ret; 1089 } 1090 1091 if (!irte.fields.valid) { 1092 trace_amdvi_ir_target_abort("RemapEn is disabled"); 1093 return -AMDVI_IR_TARGET_ABORT; 1094 } 1095 1096 if (irte.fields.guest_mode) { 1097 error_report_once("guest mode is not zero"); 1098 return -AMDVI_IR_ERR; 1099 } 1100 1101 if (irte.fields.int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) { 1102 error_report_once("reserved int_type"); 1103 return -AMDVI_IR_ERR; 1104 } 1105 1106 irq->delivery_mode = irte.fields.int_type; 1107 irq->vector = irte.fields.vector; 1108 irq->dest_mode = irte.fields.dm; 1109 irq->redir_hint = irte.fields.rq_eoi; 1110 irq->dest = irte.fields.destination; 1111 1112 return 0; 1113 } 1114 1115 static int amdvi_get_irte_ga(AMDVIState *s, MSIMessage *origin, uint64_t *dte, 1116 struct irte_ga *irte, uint16_t devid) 1117 { 1118 uint64_t irte_root, offset; 1119 1120 irte_root = dte[2] & AMDVI_IR_PHYS_ADDR_MASK; 1121 offset = (origin->data & AMDVI_IRTE_OFFSET) << 4; 1122 trace_amdvi_ir_irte(irte_root, offset); 1123 1124 if (dma_memory_read(&address_space_memory, irte_root + offset, 1125 irte, sizeof(*irte), MEMTXATTRS_UNSPECIFIED)) { 1126 trace_amdvi_ir_err("failed to get irte_ga"); 1127 return -AMDVI_IR_GET_IRTE; 1128 } 1129 1130 trace_amdvi_ir_irte_ga_val(irte->hi.val, irte->lo.val); 1131 return 0; 1132 } 1133 1134 static int amdvi_int_remap_ga(AMDVIState *iommu, 1135 MSIMessage *origin, 1136 MSIMessage *translated, 1137 uint64_t *dte, 1138 X86IOMMUIrq *irq, 1139 uint16_t sid) 1140 { 1141 int ret; 1142 struct irte_ga irte; 1143 1144 /* get interrupt remapping table */ 1145 ret = amdvi_get_irte_ga(iommu, origin, dte, &irte, sid); 1146 if (ret < 0) { 1147 return ret; 1148 } 1149 1150 if (!irte.lo.fields_remap.valid) { 1151 trace_amdvi_ir_target_abort("RemapEn is disabled"); 1152 return -AMDVI_IR_TARGET_ABORT; 1153 } 1154 1155 if (irte.lo.fields_remap.guest_mode) { 1156 error_report_once("guest mode is not zero"); 1157 return -AMDVI_IR_ERR; 1158 } 1159 1160 if (irte.lo.fields_remap.int_type > AMDVI_IOAPIC_INT_TYPE_ARBITRATED) { 1161 error_report_once("reserved int_type is set"); 1162 return -AMDVI_IR_ERR; 1163 } 1164 1165 irq->delivery_mode = irte.lo.fields_remap.int_type; 1166 irq->vector = irte.hi.fields.vector; 1167 irq->dest_mode = irte.lo.fields_remap.dm; 1168 irq->redir_hint = irte.lo.fields_remap.rq_eoi; 1169 if (iommu->xtsup) { 1170 irq->dest = irte.lo.fields_remap.destination | 1171 (irte.hi.fields.destination_hi << 24); 1172 } else { 1173 irq->dest = irte.lo.fields_remap.destination & 0xff; 1174 } 1175 1176 return 0; 1177 } 1178 1179 static int __amdvi_int_remap_msi(AMDVIState *iommu, 1180 MSIMessage *origin, 1181 MSIMessage *translated, 1182 uint64_t *dte, 1183 X86IOMMUIrq *irq, 1184 uint16_t sid) 1185 { 1186 int ret; 1187 uint8_t int_ctl; 1188 1189 int_ctl = (dte[2] >> AMDVI_IR_INTCTL_SHIFT) & 3; 1190 trace_amdvi_ir_intctl(int_ctl); 1191 1192 switch (int_ctl) { 1193 case AMDVI_IR_INTCTL_PASS: 1194 memcpy(translated, origin, sizeof(*origin)); 1195 return 0; 1196 case AMDVI_IR_INTCTL_REMAP: 1197 break; 1198 case AMDVI_IR_INTCTL_ABORT: 1199 trace_amdvi_ir_target_abort("int_ctl abort"); 1200 return -AMDVI_IR_TARGET_ABORT; 1201 default: 1202 trace_amdvi_ir_err("int_ctl reserved"); 1203 return -AMDVI_IR_ERR; 1204 } 1205 1206 if (iommu->ga_enabled) { 1207 ret = amdvi_int_remap_ga(iommu, origin, translated, dte, irq, sid); 1208 } else { 1209 ret = amdvi_int_remap_legacy(iommu, origin, translated, dte, irq, sid); 1210 } 1211 1212 return ret; 1213 } 1214 1215 /* Interrupt remapping for MSI/MSI-X entry */ 1216 static int amdvi_int_remap_msi(AMDVIState *iommu, 1217 MSIMessage *origin, 1218 MSIMessage *translated, 1219 uint16_t sid) 1220 { 1221 int ret = 0; 1222 uint64_t pass = 0; 1223 uint64_t dte[4] = { 0 }; 1224 X86IOMMUIrq irq = { 0 }; 1225 uint8_t dest_mode, delivery_mode; 1226 1227 assert(origin && translated); 1228 1229 /* 1230 * When IOMMU is enabled, interrupt remap request will come either from 1231 * IO-APIC or PCI device. If interrupt is from PCI device then it will 1232 * have a valid requester id but if the interrupt is from IO-APIC 1233 * then requester id will be invalid. 1234 */ 1235 if (sid == X86_IOMMU_SID_INVALID) { 1236 sid = AMDVI_IOAPIC_SB_DEVID; 1237 } 1238 1239 trace_amdvi_ir_remap_msi_req(origin->address, origin->data, sid); 1240 1241 /* check if device table entry is set before we go further. */ 1242 if (!iommu || !iommu->devtab_len) { 1243 memcpy(translated, origin, sizeof(*origin)); 1244 goto out; 1245 } 1246 1247 if (!amdvi_get_dte(iommu, sid, dte)) { 1248 return -AMDVI_IR_ERR; 1249 } 1250 1251 /* Check if IR is enabled in DTE */ 1252 if (!(dte[2] & AMDVI_IR_REMAP_ENABLE)) { 1253 memcpy(translated, origin, sizeof(*origin)); 1254 goto out; 1255 } 1256 1257 /* validate that we are configure with intremap=on */ 1258 if (!x86_iommu_ir_supported(X86_IOMMU_DEVICE(iommu))) { 1259 trace_amdvi_err("Interrupt remapping is enabled in the guest but " 1260 "not in the host. Use intremap=on to enable interrupt " 1261 "remapping in amd-iommu."); 1262 return -AMDVI_IR_ERR; 1263 } 1264 1265 if (origin->address < AMDVI_INT_ADDR_FIRST || 1266 origin->address + sizeof(origin->data) > AMDVI_INT_ADDR_LAST + 1) { 1267 trace_amdvi_err("MSI is not from IOAPIC."); 1268 return -AMDVI_IR_ERR; 1269 } 1270 1271 /* 1272 * The MSI data register [10:8] are used to get the upstream interrupt type. 1273 * 1274 * See MSI/MSI-X format: 1275 * https://pdfs.semanticscholar.org/presentation/9420/c279e942eca568157711ef5c92b800c40a79.pdf 1276 * (page 5) 1277 */ 1278 delivery_mode = (origin->data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 7; 1279 1280 switch (delivery_mode) { 1281 case AMDVI_IOAPIC_INT_TYPE_FIXED: 1282 case AMDVI_IOAPIC_INT_TYPE_ARBITRATED: 1283 trace_amdvi_ir_delivery_mode("fixed/arbitrated"); 1284 ret = __amdvi_int_remap_msi(iommu, origin, translated, dte, &irq, sid); 1285 if (ret < 0) { 1286 goto remap_fail; 1287 } else { 1288 /* Translate IRQ to MSI messages */ 1289 x86_iommu_irq_to_msi_message(&irq, translated); 1290 goto out; 1291 } 1292 break; 1293 case AMDVI_IOAPIC_INT_TYPE_SMI: 1294 error_report("SMI is not supported!"); 1295 ret = -AMDVI_IR_ERR; 1296 break; 1297 case AMDVI_IOAPIC_INT_TYPE_NMI: 1298 pass = dte[3] & AMDVI_DEV_NMI_PASS_MASK; 1299 trace_amdvi_ir_delivery_mode("nmi"); 1300 break; 1301 case AMDVI_IOAPIC_INT_TYPE_INIT: 1302 pass = dte[3] & AMDVI_DEV_INT_PASS_MASK; 1303 trace_amdvi_ir_delivery_mode("init"); 1304 break; 1305 case AMDVI_IOAPIC_INT_TYPE_EINT: 1306 pass = dte[3] & AMDVI_DEV_EINT_PASS_MASK; 1307 trace_amdvi_ir_delivery_mode("eint"); 1308 break; 1309 default: 1310 trace_amdvi_ir_delivery_mode("unsupported delivery_mode"); 1311 ret = -AMDVI_IR_ERR; 1312 break; 1313 } 1314 1315 if (ret < 0) { 1316 goto remap_fail; 1317 } 1318 1319 /* 1320 * The MSI address register bit[2] is used to get the destination 1321 * mode. The dest_mode 1 is valid for fixed and arbitrated interrupts 1322 * only. 1323 */ 1324 dest_mode = (origin->address >> MSI_ADDR_DEST_MODE_SHIFT) & 1; 1325 if (dest_mode) { 1326 trace_amdvi_ir_err("invalid dest_mode"); 1327 ret = -AMDVI_IR_ERR; 1328 goto remap_fail; 1329 } 1330 1331 if (pass) { 1332 memcpy(translated, origin, sizeof(*origin)); 1333 } else { 1334 trace_amdvi_ir_err("passthrough is not enabled"); 1335 ret = -AMDVI_IR_ERR; 1336 goto remap_fail; 1337 } 1338 1339 out: 1340 trace_amdvi_ir_remap_msi(origin->address, origin->data, 1341 translated->address, translated->data); 1342 return 0; 1343 1344 remap_fail: 1345 return ret; 1346 } 1347 1348 static int amdvi_int_remap(X86IOMMUState *iommu, 1349 MSIMessage *origin, 1350 MSIMessage *translated, 1351 uint16_t sid) 1352 { 1353 return amdvi_int_remap_msi(AMD_IOMMU_DEVICE(iommu), origin, 1354 translated, sid); 1355 } 1356 1357 static MemTxResult amdvi_mem_ir_write(void *opaque, hwaddr addr, 1358 uint64_t value, unsigned size, 1359 MemTxAttrs attrs) 1360 { 1361 int ret; 1362 MSIMessage from = { 0, 0 }, to = { 0, 0 }; 1363 uint16_t sid = AMDVI_IOAPIC_SB_DEVID; 1364 1365 from.address = (uint64_t) addr + AMDVI_INT_ADDR_FIRST; 1366 from.data = (uint32_t) value; 1367 1368 trace_amdvi_mem_ir_write_req(addr, value, size); 1369 1370 if (!attrs.unspecified) { 1371 /* We have explicit Source ID */ 1372 sid = attrs.requester_id; 1373 } 1374 1375 ret = amdvi_int_remap_msi(opaque, &from, &to, sid); 1376 if (ret < 0) { 1377 /* TODO: log the event using IOMMU log event interface */ 1378 error_report_once("failed to remap interrupt from devid 0x%x", sid); 1379 return MEMTX_ERROR; 1380 } 1381 1382 apic_get_class(NULL)->send_msi(&to); 1383 1384 trace_amdvi_mem_ir_write(to.address, to.data); 1385 return MEMTX_OK; 1386 } 1387 1388 static MemTxResult amdvi_mem_ir_read(void *opaque, hwaddr addr, 1389 uint64_t *data, unsigned size, 1390 MemTxAttrs attrs) 1391 { 1392 return MEMTX_OK; 1393 } 1394 1395 static const MemoryRegionOps amdvi_ir_ops = { 1396 .read_with_attrs = amdvi_mem_ir_read, 1397 .write_with_attrs = amdvi_mem_ir_write, 1398 .endianness = DEVICE_LITTLE_ENDIAN, 1399 .impl = { 1400 .min_access_size = 4, 1401 .max_access_size = 4, 1402 }, 1403 .valid = { 1404 .min_access_size = 4, 1405 .max_access_size = 4, 1406 } 1407 }; 1408 1409 static AddressSpace *amdvi_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) 1410 { 1411 char name[128]; 1412 AMDVIState *s = opaque; 1413 AMDVIAddressSpace **iommu_as, *amdvi_dev_as; 1414 int bus_num = pci_bus_num(bus); 1415 1416 iommu_as = s->address_spaces[bus_num]; 1417 1418 /* allocate memory during the first run */ 1419 if (!iommu_as) { 1420 iommu_as = g_new0(AMDVIAddressSpace *, PCI_DEVFN_MAX); 1421 s->address_spaces[bus_num] = iommu_as; 1422 } 1423 1424 /* set up AMD-Vi region */ 1425 if (!iommu_as[devfn]) { 1426 snprintf(name, sizeof(name), "amd_iommu_devfn_%d", devfn); 1427 1428 iommu_as[devfn] = g_new0(AMDVIAddressSpace, 1); 1429 iommu_as[devfn]->bus_num = (uint8_t)bus_num; 1430 iommu_as[devfn]->devfn = (uint8_t)devfn; 1431 iommu_as[devfn]->iommu_state = s; 1432 1433 amdvi_dev_as = iommu_as[devfn]; 1434 1435 /* 1436 * Memory region relationships looks like (Address range shows 1437 * only lower 32 bits to make it short in length...): 1438 * 1439 * |-----------------+-------------------+----------| 1440 * | Name | Address range | Priority | 1441 * |-----------------+-------------------+----------+ 1442 * | amdvi_root | 00000000-ffffffff | 0 | 1443 * | amdvi_iommu | 00000000-ffffffff | 1 | 1444 * | amdvi_iommu_ir | fee00000-feefffff | 64 | 1445 * |-----------------+-------------------+----------| 1446 */ 1447 memory_region_init_iommu(&amdvi_dev_as->iommu, 1448 sizeof(amdvi_dev_as->iommu), 1449 TYPE_AMD_IOMMU_MEMORY_REGION, 1450 OBJECT(s), 1451 "amd_iommu", UINT64_MAX); 1452 memory_region_init(&amdvi_dev_as->root, OBJECT(s), 1453 "amdvi_root", UINT64_MAX); 1454 address_space_init(&amdvi_dev_as->as, &amdvi_dev_as->root, name); 1455 memory_region_init_io(&amdvi_dev_as->iommu_ir, OBJECT(s), 1456 &amdvi_ir_ops, s, "amd_iommu_ir", 1457 AMDVI_INT_ADDR_SIZE); 1458 memory_region_add_subregion_overlap(&amdvi_dev_as->root, 1459 AMDVI_INT_ADDR_FIRST, 1460 &amdvi_dev_as->iommu_ir, 1461 64); 1462 memory_region_add_subregion_overlap(&amdvi_dev_as->root, 0, 1463 MEMORY_REGION(&amdvi_dev_as->iommu), 1464 1); 1465 } 1466 return &iommu_as[devfn]->as; 1467 } 1468 1469 static const PCIIOMMUOps amdvi_iommu_ops = { 1470 .get_address_space = amdvi_host_dma_iommu, 1471 }; 1472 1473 static const MemoryRegionOps mmio_mem_ops = { 1474 .read = amdvi_mmio_read, 1475 .write = amdvi_mmio_write, 1476 .endianness = DEVICE_LITTLE_ENDIAN, 1477 .impl = { 1478 .min_access_size = 1, 1479 .max_access_size = 8, 1480 .unaligned = false, 1481 }, 1482 .valid = { 1483 .min_access_size = 1, 1484 .max_access_size = 8, 1485 } 1486 }; 1487 1488 static int amdvi_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu, 1489 IOMMUNotifierFlag old, 1490 IOMMUNotifierFlag new, 1491 Error **errp) 1492 { 1493 AMDVIAddressSpace *as = container_of(iommu, AMDVIAddressSpace, iommu); 1494 1495 if (new & IOMMU_NOTIFIER_MAP) { 1496 error_setg(errp, 1497 "device %02x.%02x.%x requires iommu notifier which is not " 1498 "currently supported", as->bus_num, PCI_SLOT(as->devfn), 1499 PCI_FUNC(as->devfn)); 1500 return -EINVAL; 1501 } 1502 return 0; 1503 } 1504 1505 static void amdvi_init(AMDVIState *s) 1506 { 1507 amdvi_iotlb_reset(s); 1508 1509 s->devtab_len = 0; 1510 s->cmdbuf_len = 0; 1511 s->cmdbuf_head = 0; 1512 s->cmdbuf_tail = 0; 1513 s->evtlog_head = 0; 1514 s->evtlog_tail = 0; 1515 s->excl_enabled = false; 1516 s->excl_allow = false; 1517 s->mmio_enabled = false; 1518 s->enabled = false; 1519 s->ats_enabled = false; 1520 s->cmdbuf_enabled = false; 1521 1522 /* reset MMIO */ 1523 memset(s->mmior, 0, AMDVI_MMIO_SIZE); 1524 amdvi_set_quad(s, AMDVI_MMIO_EXT_FEATURES, 1525 amdvi_extended_feature_register(s), 1526 0xffffffffffffffef, 0); 1527 amdvi_set_quad(s, AMDVI_MMIO_STATUS, 0, 0x98, 0x67); 1528 } 1529 1530 static void amdvi_pci_realize(PCIDevice *pdev, Error **errp) 1531 { 1532 AMDVIPCIState *s = AMD_IOMMU_PCI(pdev); 1533 int ret; 1534 1535 ret = pci_add_capability(pdev, AMDVI_CAPAB_ID_SEC, 0, 1536 AMDVI_CAPAB_SIZE, errp); 1537 if (ret < 0) { 1538 return; 1539 } 1540 s->capab_offset = ret; 1541 1542 ret = pci_add_capability(pdev, PCI_CAP_ID_MSI, 0, 1543 AMDVI_CAPAB_REG_SIZE, errp); 1544 if (ret < 0) { 1545 return; 1546 } 1547 ret = pci_add_capability(pdev, PCI_CAP_ID_HT, 0, 1548 AMDVI_CAPAB_REG_SIZE, errp); 1549 if (ret < 0) { 1550 return; 1551 } 1552 1553 if (msi_init(pdev, 0, 1, true, false, errp) < 0) { 1554 return; 1555 } 1556 1557 /* reset device ident */ 1558 pci_config_set_prog_interface(pdev->config, 0); 1559 1560 /* reset AMDVI specific capabilities, all r/o */ 1561 pci_set_long(pdev->config + s->capab_offset, AMDVI_CAPAB_FEATURES); 1562 pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_BAR_LOW, 1563 AMDVI_BASE_ADDR & ~(0xffff0000)); 1564 pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_BAR_HIGH, 1565 (AMDVI_BASE_ADDR & ~(0xffff)) >> 16); 1566 pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_RANGE, 1567 0xff000000); 1568 pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_MISC, 0); 1569 pci_set_long(pdev->config + s->capab_offset + AMDVI_CAPAB_MISC, 1570 AMDVI_MAX_PH_ADDR | AMDVI_MAX_GVA_ADDR | AMDVI_MAX_VA_ADDR); 1571 } 1572 1573 static void amdvi_sysbus_reset(DeviceState *dev) 1574 { 1575 AMDVIState *s = AMD_IOMMU_DEVICE(dev); 1576 1577 msi_reset(&s->pci.dev); 1578 amdvi_init(s); 1579 } 1580 1581 static void amdvi_sysbus_realize(DeviceState *dev, Error **errp) 1582 { 1583 AMDVIState *s = AMD_IOMMU_DEVICE(dev); 1584 MachineState *ms = MACHINE(qdev_get_machine()); 1585 PCMachineState *pcms = PC_MACHINE(ms); 1586 X86MachineState *x86ms = X86_MACHINE(ms); 1587 PCIBus *bus = pcms->pcibus; 1588 1589 s->iotlb = g_hash_table_new_full(amdvi_uint64_hash, 1590 amdvi_uint64_equal, g_free, g_free); 1591 1592 /* This device should take care of IOMMU PCI properties */ 1593 if (!qdev_realize(DEVICE(&s->pci), &bus->qbus, errp)) { 1594 return; 1595 } 1596 1597 /* Pseudo address space under root PCI bus. */ 1598 x86ms->ioapic_as = amdvi_host_dma_iommu(bus, s, AMDVI_IOAPIC_SB_DEVID); 1599 1600 /* set up MMIO */ 1601 memory_region_init_io(&s->mmio, OBJECT(s), &mmio_mem_ops, s, "amdvi-mmio", 1602 AMDVI_MMIO_SIZE); 1603 memory_region_add_subregion(get_system_memory(), AMDVI_BASE_ADDR, 1604 &s->mmio); 1605 pci_setup_iommu(bus, &amdvi_iommu_ops, s); 1606 amdvi_init(s); 1607 } 1608 1609 static Property amdvi_properties[] = { 1610 DEFINE_PROP_BOOL("xtsup", AMDVIState, xtsup, false), 1611 DEFINE_PROP_END_OF_LIST(), 1612 }; 1613 1614 static const VMStateDescription vmstate_amdvi_sysbus = { 1615 .name = "amd-iommu", 1616 .unmigratable = 1 1617 }; 1618 1619 static void amdvi_sysbus_instance_init(Object *klass) 1620 { 1621 AMDVIState *s = AMD_IOMMU_DEVICE(klass); 1622 1623 object_initialize(&s->pci, sizeof(s->pci), TYPE_AMD_IOMMU_PCI); 1624 } 1625 1626 static void amdvi_sysbus_class_init(ObjectClass *klass, void *data) 1627 { 1628 DeviceClass *dc = DEVICE_CLASS(klass); 1629 X86IOMMUClass *dc_class = X86_IOMMU_DEVICE_CLASS(klass); 1630 1631 dc->reset = amdvi_sysbus_reset; 1632 dc->vmsd = &vmstate_amdvi_sysbus; 1633 dc->hotpluggable = false; 1634 dc_class->realize = amdvi_sysbus_realize; 1635 dc_class->int_remap = amdvi_int_remap; 1636 /* Supported by the pc-q35-* machine types */ 1637 dc->user_creatable = true; 1638 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1639 dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device"; 1640 device_class_set_props(dc, amdvi_properties); 1641 } 1642 1643 static const TypeInfo amdvi_sysbus = { 1644 .name = TYPE_AMD_IOMMU_DEVICE, 1645 .parent = TYPE_X86_IOMMU_DEVICE, 1646 .instance_size = sizeof(AMDVIState), 1647 .instance_init = amdvi_sysbus_instance_init, 1648 .class_init = amdvi_sysbus_class_init 1649 }; 1650 1651 static void amdvi_pci_class_init(ObjectClass *klass, void *data) 1652 { 1653 DeviceClass *dc = DEVICE_CLASS(klass); 1654 PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); 1655 1656 k->vendor_id = PCI_VENDOR_ID_AMD; 1657 k->class_id = 0x0806; 1658 k->realize = amdvi_pci_realize; 1659 1660 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 1661 dc->desc = "AMD IOMMU (AMD-Vi) DMA Remapping device"; 1662 } 1663 1664 static const TypeInfo amdvi_pci = { 1665 .name = TYPE_AMD_IOMMU_PCI, 1666 .parent = TYPE_PCI_DEVICE, 1667 .instance_size = sizeof(AMDVIPCIState), 1668 .class_init = amdvi_pci_class_init, 1669 .interfaces = (InterfaceInfo[]) { 1670 { INTERFACE_CONVENTIONAL_PCI_DEVICE }, 1671 { }, 1672 }, 1673 }; 1674 1675 static void amdvi_iommu_memory_region_class_init(ObjectClass *klass, void *data) 1676 { 1677 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); 1678 1679 imrc->translate = amdvi_translate; 1680 imrc->notify_flag_changed = amdvi_iommu_notify_flag_changed; 1681 } 1682 1683 static const TypeInfo amdvi_iommu_memory_region_info = { 1684 .parent = TYPE_IOMMU_MEMORY_REGION, 1685 .name = TYPE_AMD_IOMMU_MEMORY_REGION, 1686 .class_init = amdvi_iommu_memory_region_class_init, 1687 }; 1688 1689 static void amdvi_register_types(void) 1690 { 1691 type_register_static(&amdvi_pci); 1692 type_register_static(&amdvi_sysbus); 1693 type_register_static(&amdvi_iommu_memory_region_info); 1694 } 1695 1696 type_init(amdvi_register_types); 1697