1 /* 2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM 3 * 4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Kevin Tian <kevin.tian@intel.com> 27 * Jike Song <jike.song@intel.com> 28 * Xiaoguang Chen <xiaoguang.chen@intel.com> 29 */ 30 31 #include <linux/init.h> 32 #include <linux/device.h> 33 #include <linux/mm.h> 34 #include <linux/mmu_context.h> 35 #include <linux/types.h> 36 #include <linux/list.h> 37 #include <linux/rbtree.h> 38 #include <linux/spinlock.h> 39 #include <linux/eventfd.h> 40 #include <linux/uuid.h> 41 #include <linux/kvm_host.h> 42 #include <linux/vfio.h> 43 #include <linux/mdev.h> 44 45 #include "i915_drv.h" 46 #include "gvt.h" 47 48 static const struct intel_gvt_ops *intel_gvt_ops; 49 50 /* helper macros copied from vfio-pci */ 51 #define VFIO_PCI_OFFSET_SHIFT 40 52 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) 53 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 54 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 55 56 struct vfio_region { 57 u32 type; 58 u32 subtype; 59 size_t size; 60 u32 flags; 61 }; 62 63 struct kvmgt_pgfn { 64 gfn_t gfn; 65 struct hlist_node hnode; 66 }; 67 68 struct kvmgt_guest_info { 69 struct kvm *kvm; 70 struct intel_vgpu *vgpu; 71 struct kvm_page_track_notifier_node track_node; 72 #define NR_BKT (1 << 18) 73 struct hlist_head ptable[NR_BKT]; 74 #undef NR_BKT 75 }; 76 77 struct gvt_dma { 78 struct rb_node node; 79 gfn_t gfn; 80 unsigned long iova; 81 }; 82 83 static inline bool handle_valid(unsigned long handle) 84 { 85 return !!(handle & ~0xff); 86 } 87 88 static int kvmgt_guest_init(struct mdev_device *mdev); 89 static void intel_vgpu_release_work(struct work_struct *work); 90 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); 91 92 static int gvt_dma_map_iova(struct intel_vgpu *vgpu, kvm_pfn_t pfn, 93 unsigned long *iova) 94 { 95 struct page *page; 96 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 97 dma_addr_t daddr; 98 99 if (unlikely(!pfn_valid(pfn))) 100 return -EFAULT; 101 102 page = pfn_to_page(pfn); 103 daddr = dma_map_page(dev, page, 0, PAGE_SIZE, 104 PCI_DMA_BIDIRECTIONAL); 105 if (dma_mapping_error(dev, daddr)) 106 return -ENOMEM; 107 108 *iova = (unsigned long)(daddr >> PAGE_SHIFT); 109 return 0; 110 } 111 112 static void gvt_dma_unmap_iova(struct intel_vgpu *vgpu, unsigned long iova) 113 { 114 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 115 dma_addr_t daddr; 116 117 daddr = (dma_addr_t)(iova << PAGE_SHIFT); 118 dma_unmap_page(dev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 119 } 120 121 static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 122 { 123 struct rb_node *node = vgpu->vdev.cache.rb_node; 124 struct gvt_dma *ret = NULL; 125 126 while (node) { 127 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node); 128 129 if (gfn < itr->gfn) 130 node = node->rb_left; 131 else if (gfn > itr->gfn) 132 node = node->rb_right; 133 else { 134 ret = itr; 135 goto out; 136 } 137 } 138 139 out: 140 return ret; 141 } 142 143 static unsigned long gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 144 { 145 struct gvt_dma *entry; 146 unsigned long iova; 147 148 mutex_lock(&vgpu->vdev.cache_lock); 149 150 entry = __gvt_cache_find(vgpu, gfn); 151 iova = (entry == NULL) ? INTEL_GVT_INVALID_ADDR : entry->iova; 152 153 mutex_unlock(&vgpu->vdev.cache_lock); 154 return iova; 155 } 156 157 static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 158 unsigned long iova) 159 { 160 struct gvt_dma *new, *itr; 161 struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL; 162 163 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 164 if (!new) 165 return; 166 167 new->gfn = gfn; 168 new->iova = iova; 169 170 mutex_lock(&vgpu->vdev.cache_lock); 171 while (*link) { 172 parent = *link; 173 itr = rb_entry(parent, struct gvt_dma, node); 174 175 if (gfn == itr->gfn) 176 goto out; 177 else if (gfn < itr->gfn) 178 link = &parent->rb_left; 179 else 180 link = &parent->rb_right; 181 } 182 183 rb_link_node(&new->node, parent, link); 184 rb_insert_color(&new->node, &vgpu->vdev.cache); 185 mutex_unlock(&vgpu->vdev.cache_lock); 186 return; 187 188 out: 189 mutex_unlock(&vgpu->vdev.cache_lock); 190 kfree(new); 191 } 192 193 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 194 struct gvt_dma *entry) 195 { 196 rb_erase(&entry->node, &vgpu->vdev.cache); 197 kfree(entry); 198 } 199 200 static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn) 201 { 202 struct device *dev = mdev_dev(vgpu->vdev.mdev); 203 struct gvt_dma *this; 204 unsigned long g1; 205 int rc; 206 207 mutex_lock(&vgpu->vdev.cache_lock); 208 this = __gvt_cache_find(vgpu, gfn); 209 if (!this) { 210 mutex_unlock(&vgpu->vdev.cache_lock); 211 return; 212 } 213 214 g1 = gfn; 215 gvt_dma_unmap_iova(vgpu, this->iova); 216 rc = vfio_unpin_pages(dev, &g1, 1); 217 WARN_ON(rc != 1); 218 __gvt_cache_remove_entry(vgpu, this); 219 mutex_unlock(&vgpu->vdev.cache_lock); 220 } 221 222 static void gvt_cache_init(struct intel_vgpu *vgpu) 223 { 224 vgpu->vdev.cache = RB_ROOT; 225 mutex_init(&vgpu->vdev.cache_lock); 226 } 227 228 static void gvt_cache_destroy(struct intel_vgpu *vgpu) 229 { 230 struct gvt_dma *dma; 231 struct rb_node *node = NULL; 232 struct device *dev = mdev_dev(vgpu->vdev.mdev); 233 unsigned long gfn; 234 235 for (;;) { 236 mutex_lock(&vgpu->vdev.cache_lock); 237 node = rb_first(&vgpu->vdev.cache); 238 if (!node) { 239 mutex_unlock(&vgpu->vdev.cache_lock); 240 break; 241 } 242 dma = rb_entry(node, struct gvt_dma, node); 243 gvt_dma_unmap_iova(vgpu, dma->iova); 244 gfn = dma->gfn; 245 __gvt_cache_remove_entry(vgpu, dma); 246 mutex_unlock(&vgpu->vdev.cache_lock); 247 vfio_unpin_pages(dev, &gfn, 1); 248 } 249 } 250 251 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) 252 { 253 hash_init(info->ptable); 254 } 255 256 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info) 257 { 258 struct kvmgt_pgfn *p; 259 struct hlist_node *tmp; 260 int i; 261 262 hash_for_each_safe(info->ptable, i, tmp, p, hnode) { 263 hash_del(&p->hnode); 264 kfree(p); 265 } 266 } 267 268 static struct kvmgt_pgfn * 269 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn) 270 { 271 struct kvmgt_pgfn *p, *res = NULL; 272 273 hash_for_each_possible(info->ptable, p, hnode, gfn) { 274 if (gfn == p->gfn) { 275 res = p; 276 break; 277 } 278 } 279 280 return res; 281 } 282 283 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info, 284 gfn_t gfn) 285 { 286 struct kvmgt_pgfn *p; 287 288 p = __kvmgt_protect_table_find(info, gfn); 289 return !!p; 290 } 291 292 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn) 293 { 294 struct kvmgt_pgfn *p; 295 296 if (kvmgt_gfn_is_write_protected(info, gfn)) 297 return; 298 299 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); 300 if (WARN(!p, "gfn: 0x%llx\n", gfn)) 301 return; 302 303 p->gfn = gfn; 304 hash_add(info->ptable, &p->hnode, gfn); 305 } 306 307 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, 308 gfn_t gfn) 309 { 310 struct kvmgt_pgfn *p; 311 312 p = __kvmgt_protect_table_find(info, gfn); 313 if (p) { 314 hash_del(&p->hnode); 315 kfree(p); 316 } 317 } 318 319 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) 320 { 321 struct intel_vgpu *vgpu = NULL; 322 struct intel_vgpu_type *type; 323 struct device *pdev; 324 void *gvt; 325 int ret; 326 327 pdev = mdev_parent_dev(mdev); 328 gvt = kdev_to_i915(pdev)->gvt; 329 330 type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj)); 331 if (!type) { 332 gvt_vgpu_err("failed to find type %s to create\n", 333 kobject_name(kobj)); 334 ret = -EINVAL; 335 goto out; 336 } 337 338 vgpu = intel_gvt_ops->vgpu_create(gvt, type); 339 if (IS_ERR_OR_NULL(vgpu)) { 340 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu); 341 gvt_vgpu_err("failed to create intel vgpu: %d\n", ret); 342 goto out; 343 } 344 345 INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work); 346 347 vgpu->vdev.mdev = mdev; 348 mdev_set_drvdata(mdev, vgpu); 349 350 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", 351 dev_name(mdev_dev(mdev))); 352 ret = 0; 353 354 out: 355 return ret; 356 } 357 358 static int intel_vgpu_remove(struct mdev_device *mdev) 359 { 360 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 361 362 if (handle_valid(vgpu->handle)) 363 return -EBUSY; 364 365 intel_gvt_ops->vgpu_destroy(vgpu); 366 return 0; 367 } 368 369 static int intel_vgpu_iommu_notifier(struct notifier_block *nb, 370 unsigned long action, void *data) 371 { 372 struct intel_vgpu *vgpu = container_of(nb, 373 struct intel_vgpu, 374 vdev.iommu_notifier); 375 376 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 377 struct vfio_iommu_type1_dma_unmap *unmap = data; 378 unsigned long gfn, end_gfn; 379 380 gfn = unmap->iova >> PAGE_SHIFT; 381 end_gfn = gfn + unmap->size / PAGE_SIZE; 382 383 while (gfn < end_gfn) 384 gvt_cache_remove(vgpu, gfn++); 385 } 386 387 return NOTIFY_OK; 388 } 389 390 static int intel_vgpu_group_notifier(struct notifier_block *nb, 391 unsigned long action, void *data) 392 { 393 struct intel_vgpu *vgpu = container_of(nb, 394 struct intel_vgpu, 395 vdev.group_notifier); 396 397 /* the only action we care about */ 398 if (action == VFIO_GROUP_NOTIFY_SET_KVM) { 399 vgpu->vdev.kvm = data; 400 401 if (!data) 402 schedule_work(&vgpu->vdev.release_work); 403 } 404 405 return NOTIFY_OK; 406 } 407 408 static int intel_vgpu_open(struct mdev_device *mdev) 409 { 410 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 411 unsigned long events; 412 int ret; 413 414 vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier; 415 vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier; 416 417 events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; 418 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events, 419 &vgpu->vdev.iommu_notifier); 420 if (ret != 0) { 421 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n", 422 ret); 423 goto out; 424 } 425 426 events = VFIO_GROUP_NOTIFY_SET_KVM; 427 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events, 428 &vgpu->vdev.group_notifier); 429 if (ret != 0) { 430 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n", 431 ret); 432 goto undo_iommu; 433 } 434 435 ret = kvmgt_guest_init(mdev); 436 if (ret) 437 goto undo_group; 438 439 intel_gvt_ops->vgpu_activate(vgpu); 440 441 atomic_set(&vgpu->vdev.released, 0); 442 return ret; 443 444 undo_group: 445 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, 446 &vgpu->vdev.group_notifier); 447 448 undo_iommu: 449 vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, 450 &vgpu->vdev.iommu_notifier); 451 out: 452 return ret; 453 } 454 455 static void __intel_vgpu_release(struct intel_vgpu *vgpu) 456 { 457 struct kvmgt_guest_info *info; 458 int ret; 459 460 if (!handle_valid(vgpu->handle)) 461 return; 462 463 if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1)) 464 return; 465 466 intel_gvt_ops->vgpu_deactivate(vgpu); 467 468 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY, 469 &vgpu->vdev.iommu_notifier); 470 WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret); 471 472 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY, 473 &vgpu->vdev.group_notifier); 474 WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret); 475 476 info = (struct kvmgt_guest_info *)vgpu->handle; 477 kvmgt_guest_exit(info); 478 479 vgpu->vdev.kvm = NULL; 480 vgpu->handle = 0; 481 } 482 483 static void intel_vgpu_release(struct mdev_device *mdev) 484 { 485 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 486 487 __intel_vgpu_release(vgpu); 488 } 489 490 static void intel_vgpu_release_work(struct work_struct *work) 491 { 492 struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu, 493 vdev.release_work); 494 495 __intel_vgpu_release(vgpu); 496 } 497 498 static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) 499 { 500 u32 start_lo, start_hi; 501 u32 mem_type; 502 503 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 504 PCI_BASE_ADDRESS_MEM_MASK; 505 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 506 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 507 508 switch (mem_type) { 509 case PCI_BASE_ADDRESS_MEM_TYPE_64: 510 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space 511 + bar + 4)); 512 break; 513 case PCI_BASE_ADDRESS_MEM_TYPE_32: 514 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 515 /* 1M mem BAR treated as 32-bit BAR */ 516 default: 517 /* mem unknown type treated as 32-bit BAR */ 518 start_hi = 0; 519 break; 520 } 521 522 return ((u64)start_hi << 32) | start_lo; 523 } 524 525 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off, 526 void *buf, unsigned int count, bool is_write) 527 { 528 uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar); 529 int ret; 530 531 if (is_write) 532 ret = intel_gvt_ops->emulate_mmio_write(vgpu, 533 bar_start + off, buf, count); 534 else 535 ret = intel_gvt_ops->emulate_mmio_read(vgpu, 536 bar_start + off, buf, count); 537 return ret; 538 } 539 540 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, 541 size_t count, loff_t *ppos, bool is_write) 542 { 543 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 544 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 545 uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 546 int ret = -EINVAL; 547 548 549 if (index >= VFIO_PCI_NUM_REGIONS) { 550 gvt_vgpu_err("invalid index: %u\n", index); 551 return -EINVAL; 552 } 553 554 switch (index) { 555 case VFIO_PCI_CONFIG_REGION_INDEX: 556 if (is_write) 557 ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos, 558 buf, count); 559 else 560 ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos, 561 buf, count); 562 break; 563 case VFIO_PCI_BAR0_REGION_INDEX: 564 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos, 565 buf, count, is_write); 566 break; 567 case VFIO_PCI_BAR2_REGION_INDEX: 568 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_2, pos, 569 buf, count, is_write); 570 break; 571 case VFIO_PCI_BAR1_REGION_INDEX: 572 case VFIO_PCI_BAR3_REGION_INDEX: 573 case VFIO_PCI_BAR4_REGION_INDEX: 574 case VFIO_PCI_BAR5_REGION_INDEX: 575 case VFIO_PCI_VGA_REGION_INDEX: 576 case VFIO_PCI_ROM_REGION_INDEX: 577 default: 578 gvt_vgpu_err("unsupported region: %u\n", index); 579 } 580 581 return ret == 0 ? count : ret; 582 } 583 584 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, 585 size_t count, loff_t *ppos) 586 { 587 unsigned int done = 0; 588 int ret; 589 590 while (count) { 591 size_t filled; 592 593 if (count >= 4 && !(*ppos % 4)) { 594 u32 val; 595 596 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 597 ppos, false); 598 if (ret <= 0) 599 goto read_err; 600 601 if (copy_to_user(buf, &val, sizeof(val))) 602 goto read_err; 603 604 filled = 4; 605 } else if (count >= 2 && !(*ppos % 2)) { 606 u16 val; 607 608 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 609 ppos, false); 610 if (ret <= 0) 611 goto read_err; 612 613 if (copy_to_user(buf, &val, sizeof(val))) 614 goto read_err; 615 616 filled = 2; 617 } else { 618 u8 val; 619 620 ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos, 621 false); 622 if (ret <= 0) 623 goto read_err; 624 625 if (copy_to_user(buf, &val, sizeof(val))) 626 goto read_err; 627 628 filled = 1; 629 } 630 631 count -= filled; 632 done += filled; 633 *ppos += filled; 634 buf += filled; 635 } 636 637 return done; 638 639 read_err: 640 return -EFAULT; 641 } 642 643 static ssize_t intel_vgpu_write(struct mdev_device *mdev, 644 const char __user *buf, 645 size_t count, loff_t *ppos) 646 { 647 unsigned int done = 0; 648 int ret; 649 650 while (count) { 651 size_t filled; 652 653 if (count >= 4 && !(*ppos % 4)) { 654 u32 val; 655 656 if (copy_from_user(&val, buf, sizeof(val))) 657 goto write_err; 658 659 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 660 ppos, true); 661 if (ret <= 0) 662 goto write_err; 663 664 filled = 4; 665 } else if (count >= 2 && !(*ppos % 2)) { 666 u16 val; 667 668 if (copy_from_user(&val, buf, sizeof(val))) 669 goto write_err; 670 671 ret = intel_vgpu_rw(mdev, (char *)&val, 672 sizeof(val), ppos, true); 673 if (ret <= 0) 674 goto write_err; 675 676 filled = 2; 677 } else { 678 u8 val; 679 680 if (copy_from_user(&val, buf, sizeof(val))) 681 goto write_err; 682 683 ret = intel_vgpu_rw(mdev, &val, sizeof(val), 684 ppos, true); 685 if (ret <= 0) 686 goto write_err; 687 688 filled = 1; 689 } 690 691 count -= filled; 692 done += filled; 693 *ppos += filled; 694 buf += filled; 695 } 696 697 return done; 698 write_err: 699 return -EFAULT; 700 } 701 702 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) 703 { 704 unsigned int index; 705 u64 virtaddr; 706 unsigned long req_size, pgoff = 0; 707 pgprot_t pg_prot; 708 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 709 710 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 711 if (index >= VFIO_PCI_ROM_REGION_INDEX) 712 return -EINVAL; 713 714 if (vma->vm_end < vma->vm_start) 715 return -EINVAL; 716 if ((vma->vm_flags & VM_SHARED) == 0) 717 return -EINVAL; 718 if (index != VFIO_PCI_BAR2_REGION_INDEX) 719 return -EINVAL; 720 721 pg_prot = vma->vm_page_prot; 722 virtaddr = vma->vm_start; 723 req_size = vma->vm_end - vma->vm_start; 724 pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; 725 726 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); 727 } 728 729 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) 730 { 731 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) 732 return 1; 733 734 return 0; 735 } 736 737 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, 738 unsigned int index, unsigned int start, 739 unsigned int count, uint32_t flags, 740 void *data) 741 { 742 return 0; 743 } 744 745 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, 746 unsigned int index, unsigned int start, 747 unsigned int count, uint32_t flags, void *data) 748 { 749 return 0; 750 } 751 752 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, 753 unsigned int index, unsigned int start, unsigned int count, 754 uint32_t flags, void *data) 755 { 756 return 0; 757 } 758 759 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, 760 unsigned int index, unsigned int start, unsigned int count, 761 uint32_t flags, void *data) 762 { 763 struct eventfd_ctx *trigger; 764 765 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 766 int fd = *(int *)data; 767 768 trigger = eventfd_ctx_fdget(fd); 769 if (IS_ERR(trigger)) { 770 gvt_vgpu_err("eventfd_ctx_fdget failed\n"); 771 return PTR_ERR(trigger); 772 } 773 vgpu->vdev.msi_trigger = trigger; 774 } 775 776 return 0; 777 } 778 779 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags, 780 unsigned int index, unsigned int start, unsigned int count, 781 void *data) 782 { 783 int (*func)(struct intel_vgpu *vgpu, unsigned int index, 784 unsigned int start, unsigned int count, uint32_t flags, 785 void *data) = NULL; 786 787 switch (index) { 788 case VFIO_PCI_INTX_IRQ_INDEX: 789 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 790 case VFIO_IRQ_SET_ACTION_MASK: 791 func = intel_vgpu_set_intx_mask; 792 break; 793 case VFIO_IRQ_SET_ACTION_UNMASK: 794 func = intel_vgpu_set_intx_unmask; 795 break; 796 case VFIO_IRQ_SET_ACTION_TRIGGER: 797 func = intel_vgpu_set_intx_trigger; 798 break; 799 } 800 break; 801 case VFIO_PCI_MSI_IRQ_INDEX: 802 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 803 case VFIO_IRQ_SET_ACTION_MASK: 804 case VFIO_IRQ_SET_ACTION_UNMASK: 805 /* XXX Need masking support exported */ 806 break; 807 case VFIO_IRQ_SET_ACTION_TRIGGER: 808 func = intel_vgpu_set_msi_trigger; 809 break; 810 } 811 break; 812 } 813 814 if (!func) 815 return -ENOTTY; 816 817 return func(vgpu, index, start, count, flags, data); 818 } 819 820 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, 821 unsigned long arg) 822 { 823 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 824 unsigned long minsz; 825 826 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); 827 828 if (cmd == VFIO_DEVICE_GET_INFO) { 829 struct vfio_device_info info; 830 831 minsz = offsetofend(struct vfio_device_info, num_irqs); 832 833 if (copy_from_user(&info, (void __user *)arg, minsz)) 834 return -EFAULT; 835 836 if (info.argsz < minsz) 837 return -EINVAL; 838 839 info.flags = VFIO_DEVICE_FLAGS_PCI; 840 info.flags |= VFIO_DEVICE_FLAGS_RESET; 841 info.num_regions = VFIO_PCI_NUM_REGIONS; 842 info.num_irqs = VFIO_PCI_NUM_IRQS; 843 844 return copy_to_user((void __user *)arg, &info, minsz) ? 845 -EFAULT : 0; 846 847 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 848 struct vfio_region_info info; 849 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 850 int i, ret; 851 struct vfio_region_info_cap_sparse_mmap *sparse = NULL; 852 size_t size; 853 int nr_areas = 1; 854 int cap_type_id; 855 856 minsz = offsetofend(struct vfio_region_info, offset); 857 858 if (copy_from_user(&info, (void __user *)arg, minsz)) 859 return -EFAULT; 860 861 if (info.argsz < minsz) 862 return -EINVAL; 863 864 switch (info.index) { 865 case VFIO_PCI_CONFIG_REGION_INDEX: 866 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 867 info.size = vgpu->gvt->device_info.cfg_space_size; 868 info.flags = VFIO_REGION_INFO_FLAG_READ | 869 VFIO_REGION_INFO_FLAG_WRITE; 870 break; 871 case VFIO_PCI_BAR0_REGION_INDEX: 872 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 873 info.size = vgpu->cfg_space.bar[info.index].size; 874 if (!info.size) { 875 info.flags = 0; 876 break; 877 } 878 879 info.flags = VFIO_REGION_INFO_FLAG_READ | 880 VFIO_REGION_INFO_FLAG_WRITE; 881 break; 882 case VFIO_PCI_BAR1_REGION_INDEX: 883 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 884 info.size = 0; 885 info.flags = 0; 886 break; 887 case VFIO_PCI_BAR2_REGION_INDEX: 888 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 889 info.flags = VFIO_REGION_INFO_FLAG_CAPS | 890 VFIO_REGION_INFO_FLAG_MMAP | 891 VFIO_REGION_INFO_FLAG_READ | 892 VFIO_REGION_INFO_FLAG_WRITE; 893 info.size = gvt_aperture_sz(vgpu->gvt); 894 895 size = sizeof(*sparse) + 896 (nr_areas * sizeof(*sparse->areas)); 897 sparse = kzalloc(size, GFP_KERNEL); 898 if (!sparse) 899 return -ENOMEM; 900 901 sparse->nr_areas = nr_areas; 902 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 903 sparse->areas[0].offset = 904 PAGE_ALIGN(vgpu_aperture_offset(vgpu)); 905 sparse->areas[0].size = vgpu_aperture_sz(vgpu); 906 break; 907 908 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 909 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 910 info.size = 0; 911 912 info.flags = 0; 913 gvt_dbg_core("get region info bar:%d\n", info.index); 914 break; 915 916 case VFIO_PCI_ROM_REGION_INDEX: 917 case VFIO_PCI_VGA_REGION_INDEX: 918 gvt_dbg_core("get region info index:%d\n", info.index); 919 break; 920 default: 921 { 922 struct vfio_region_info_cap_type cap_type; 923 924 if (info.index >= VFIO_PCI_NUM_REGIONS + 925 vgpu->vdev.num_regions) 926 return -EINVAL; 927 928 i = info.index - VFIO_PCI_NUM_REGIONS; 929 930 info.offset = 931 VFIO_PCI_INDEX_TO_OFFSET(info.index); 932 info.size = vgpu->vdev.region[i].size; 933 info.flags = vgpu->vdev.region[i].flags; 934 935 cap_type.type = vgpu->vdev.region[i].type; 936 cap_type.subtype = vgpu->vdev.region[i].subtype; 937 938 ret = vfio_info_add_capability(&caps, 939 VFIO_REGION_INFO_CAP_TYPE, 940 &cap_type); 941 if (ret) 942 return ret; 943 } 944 } 945 946 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { 947 switch (cap_type_id) { 948 case VFIO_REGION_INFO_CAP_SPARSE_MMAP: 949 ret = vfio_info_add_capability(&caps, 950 VFIO_REGION_INFO_CAP_SPARSE_MMAP, 951 sparse); 952 kfree(sparse); 953 if (ret) 954 return ret; 955 break; 956 default: 957 return -EINVAL; 958 } 959 } 960 961 if (caps.size) { 962 if (info.argsz < sizeof(info) + caps.size) { 963 info.argsz = sizeof(info) + caps.size; 964 info.cap_offset = 0; 965 } else { 966 vfio_info_cap_shift(&caps, sizeof(info)); 967 if (copy_to_user((void __user *)arg + 968 sizeof(info), caps.buf, 969 caps.size)) { 970 kfree(caps.buf); 971 return -EFAULT; 972 } 973 info.cap_offset = sizeof(info); 974 } 975 976 kfree(caps.buf); 977 } 978 979 return copy_to_user((void __user *)arg, &info, minsz) ? 980 -EFAULT : 0; 981 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 982 struct vfio_irq_info info; 983 984 minsz = offsetofend(struct vfio_irq_info, count); 985 986 if (copy_from_user(&info, (void __user *)arg, minsz)) 987 return -EFAULT; 988 989 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 990 return -EINVAL; 991 992 switch (info.index) { 993 case VFIO_PCI_INTX_IRQ_INDEX: 994 case VFIO_PCI_MSI_IRQ_INDEX: 995 break; 996 default: 997 return -EINVAL; 998 } 999 1000 info.flags = VFIO_IRQ_INFO_EVENTFD; 1001 1002 info.count = intel_vgpu_get_irq_count(vgpu, info.index); 1003 1004 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1005 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1006 VFIO_IRQ_INFO_AUTOMASKED); 1007 else 1008 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1009 1010 return copy_to_user((void __user *)arg, &info, minsz) ? 1011 -EFAULT : 0; 1012 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1013 struct vfio_irq_set hdr; 1014 u8 *data = NULL; 1015 int ret = 0; 1016 size_t data_size = 0; 1017 1018 minsz = offsetofend(struct vfio_irq_set, count); 1019 1020 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1021 return -EFAULT; 1022 1023 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 1024 int max = intel_vgpu_get_irq_count(vgpu, hdr.index); 1025 1026 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1027 VFIO_PCI_NUM_IRQS, &data_size); 1028 if (ret) { 1029 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); 1030 return -EINVAL; 1031 } 1032 if (data_size) { 1033 data = memdup_user((void __user *)(arg + minsz), 1034 data_size); 1035 if (IS_ERR(data)) 1036 return PTR_ERR(data); 1037 } 1038 } 1039 1040 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, 1041 hdr.start, hdr.count, data); 1042 kfree(data); 1043 1044 return ret; 1045 } else if (cmd == VFIO_DEVICE_RESET) { 1046 intel_gvt_ops->vgpu_reset(vgpu); 1047 return 0; 1048 } 1049 1050 return 0; 1051 } 1052 1053 static ssize_t 1054 vgpu_id_show(struct device *dev, struct device_attribute *attr, 1055 char *buf) 1056 { 1057 struct mdev_device *mdev = mdev_from_dev(dev); 1058 1059 if (mdev) { 1060 struct intel_vgpu *vgpu = (struct intel_vgpu *) 1061 mdev_get_drvdata(mdev); 1062 return sprintf(buf, "%d\n", vgpu->id); 1063 } 1064 return sprintf(buf, "\n"); 1065 } 1066 1067 static ssize_t 1068 hw_id_show(struct device *dev, struct device_attribute *attr, 1069 char *buf) 1070 { 1071 struct mdev_device *mdev = mdev_from_dev(dev); 1072 1073 if (mdev) { 1074 struct intel_vgpu *vgpu = (struct intel_vgpu *) 1075 mdev_get_drvdata(mdev); 1076 return sprintf(buf, "%u\n", 1077 vgpu->submission.shadow_ctx->hw_id); 1078 } 1079 return sprintf(buf, "\n"); 1080 } 1081 1082 static DEVICE_ATTR_RO(vgpu_id); 1083 static DEVICE_ATTR_RO(hw_id); 1084 1085 static struct attribute *intel_vgpu_attrs[] = { 1086 &dev_attr_vgpu_id.attr, 1087 &dev_attr_hw_id.attr, 1088 NULL 1089 }; 1090 1091 static const struct attribute_group intel_vgpu_group = { 1092 .name = "intel_vgpu", 1093 .attrs = intel_vgpu_attrs, 1094 }; 1095 1096 static const struct attribute_group *intel_vgpu_groups[] = { 1097 &intel_vgpu_group, 1098 NULL, 1099 }; 1100 1101 static struct mdev_parent_ops intel_vgpu_ops = { 1102 .mdev_attr_groups = intel_vgpu_groups, 1103 .create = intel_vgpu_create, 1104 .remove = intel_vgpu_remove, 1105 1106 .open = intel_vgpu_open, 1107 .release = intel_vgpu_release, 1108 1109 .read = intel_vgpu_read, 1110 .write = intel_vgpu_write, 1111 .mmap = intel_vgpu_mmap, 1112 .ioctl = intel_vgpu_ioctl, 1113 }; 1114 1115 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) 1116 { 1117 struct attribute **kvm_type_attrs; 1118 struct attribute_group **kvm_vgpu_type_groups; 1119 1120 intel_gvt_ops = ops; 1121 if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs, 1122 &kvm_vgpu_type_groups)) 1123 return -EFAULT; 1124 intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups; 1125 1126 return mdev_register_device(dev, &intel_vgpu_ops); 1127 } 1128 1129 static void kvmgt_host_exit(struct device *dev, void *gvt) 1130 { 1131 mdev_unregister_device(dev); 1132 } 1133 1134 static int kvmgt_write_protect_add(unsigned long handle, u64 gfn) 1135 { 1136 struct kvmgt_guest_info *info; 1137 struct kvm *kvm; 1138 struct kvm_memory_slot *slot; 1139 int idx; 1140 1141 if (!handle_valid(handle)) 1142 return -ESRCH; 1143 1144 info = (struct kvmgt_guest_info *)handle; 1145 kvm = info->kvm; 1146 1147 idx = srcu_read_lock(&kvm->srcu); 1148 slot = gfn_to_memslot(kvm, gfn); 1149 if (!slot) { 1150 srcu_read_unlock(&kvm->srcu, idx); 1151 return -EINVAL; 1152 } 1153 1154 spin_lock(&kvm->mmu_lock); 1155 1156 if (kvmgt_gfn_is_write_protected(info, gfn)) 1157 goto out; 1158 1159 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1160 kvmgt_protect_table_add(info, gfn); 1161 1162 out: 1163 spin_unlock(&kvm->mmu_lock); 1164 srcu_read_unlock(&kvm->srcu, idx); 1165 return 0; 1166 } 1167 1168 static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn) 1169 { 1170 struct kvmgt_guest_info *info; 1171 struct kvm *kvm; 1172 struct kvm_memory_slot *slot; 1173 int idx; 1174 1175 if (!handle_valid(handle)) 1176 return 0; 1177 1178 info = (struct kvmgt_guest_info *)handle; 1179 kvm = info->kvm; 1180 1181 idx = srcu_read_lock(&kvm->srcu); 1182 slot = gfn_to_memslot(kvm, gfn); 1183 if (!slot) { 1184 srcu_read_unlock(&kvm->srcu, idx); 1185 return -EINVAL; 1186 } 1187 1188 spin_lock(&kvm->mmu_lock); 1189 1190 if (!kvmgt_gfn_is_write_protected(info, gfn)) 1191 goto out; 1192 1193 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1194 kvmgt_protect_table_del(info, gfn); 1195 1196 out: 1197 spin_unlock(&kvm->mmu_lock); 1198 srcu_read_unlock(&kvm->srcu, idx); 1199 return 0; 1200 } 1201 1202 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 1203 const u8 *val, int len, 1204 struct kvm_page_track_notifier_node *node) 1205 { 1206 struct kvmgt_guest_info *info = container_of(node, 1207 struct kvmgt_guest_info, track_node); 1208 1209 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) 1210 intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa, 1211 (void *)val, len); 1212 } 1213 1214 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 1215 struct kvm_memory_slot *slot, 1216 struct kvm_page_track_notifier_node *node) 1217 { 1218 int i; 1219 gfn_t gfn; 1220 struct kvmgt_guest_info *info = container_of(node, 1221 struct kvmgt_guest_info, track_node); 1222 1223 spin_lock(&kvm->mmu_lock); 1224 for (i = 0; i < slot->npages; i++) { 1225 gfn = slot->base_gfn + i; 1226 if (kvmgt_gfn_is_write_protected(info, gfn)) { 1227 kvm_slot_page_track_remove_page(kvm, slot, gfn, 1228 KVM_PAGE_TRACK_WRITE); 1229 kvmgt_protect_table_del(info, gfn); 1230 } 1231 } 1232 spin_unlock(&kvm->mmu_lock); 1233 } 1234 1235 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm) 1236 { 1237 struct intel_vgpu *itr; 1238 struct kvmgt_guest_info *info; 1239 int id; 1240 bool ret = false; 1241 1242 mutex_lock(&vgpu->gvt->lock); 1243 for_each_active_vgpu(vgpu->gvt, itr, id) { 1244 if (!handle_valid(itr->handle)) 1245 continue; 1246 1247 info = (struct kvmgt_guest_info *)itr->handle; 1248 if (kvm && kvm == info->kvm) { 1249 ret = true; 1250 goto out; 1251 } 1252 } 1253 out: 1254 mutex_unlock(&vgpu->gvt->lock); 1255 return ret; 1256 } 1257 1258 static int kvmgt_guest_init(struct mdev_device *mdev) 1259 { 1260 struct kvmgt_guest_info *info; 1261 struct intel_vgpu *vgpu; 1262 struct kvm *kvm; 1263 1264 vgpu = mdev_get_drvdata(mdev); 1265 if (handle_valid(vgpu->handle)) 1266 return -EEXIST; 1267 1268 kvm = vgpu->vdev.kvm; 1269 if (!kvm || kvm->mm != current->mm) { 1270 gvt_vgpu_err("KVM is required to use Intel vGPU\n"); 1271 return -ESRCH; 1272 } 1273 1274 if (__kvmgt_vgpu_exist(vgpu, kvm)) 1275 return -EEXIST; 1276 1277 info = vzalloc(sizeof(struct kvmgt_guest_info)); 1278 if (!info) 1279 return -ENOMEM; 1280 1281 vgpu->handle = (unsigned long)info; 1282 info->vgpu = vgpu; 1283 info->kvm = kvm; 1284 kvm_get_kvm(info->kvm); 1285 1286 kvmgt_protect_table_init(info); 1287 gvt_cache_init(vgpu); 1288 1289 info->track_node.track_write = kvmgt_page_track_write; 1290 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 1291 kvm_page_track_register_notifier(kvm, &info->track_node); 1292 1293 return 0; 1294 } 1295 1296 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info) 1297 { 1298 kvm_page_track_unregister_notifier(info->kvm, &info->track_node); 1299 kvm_put_kvm(info->kvm); 1300 kvmgt_protect_table_destroy(info); 1301 gvt_cache_destroy(info->vgpu); 1302 vfree(info); 1303 1304 return true; 1305 } 1306 1307 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) 1308 { 1309 /* nothing to do here */ 1310 return 0; 1311 } 1312 1313 static void kvmgt_detach_vgpu(unsigned long handle) 1314 { 1315 /* nothing to do here */ 1316 } 1317 1318 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) 1319 { 1320 struct kvmgt_guest_info *info; 1321 struct intel_vgpu *vgpu; 1322 1323 if (!handle_valid(handle)) 1324 return -ESRCH; 1325 1326 info = (struct kvmgt_guest_info *)handle; 1327 vgpu = info->vgpu; 1328 1329 if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1) 1330 return 0; 1331 1332 return -EFAULT; 1333 } 1334 1335 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) 1336 { 1337 unsigned long iova, pfn; 1338 struct kvmgt_guest_info *info; 1339 struct device *dev; 1340 struct intel_vgpu *vgpu; 1341 int rc; 1342 1343 if (!handle_valid(handle)) 1344 return INTEL_GVT_INVALID_ADDR; 1345 1346 info = (struct kvmgt_guest_info *)handle; 1347 vgpu = info->vgpu; 1348 iova = gvt_cache_find(info->vgpu, gfn); 1349 if (iova != INTEL_GVT_INVALID_ADDR) 1350 return iova; 1351 1352 pfn = INTEL_GVT_INVALID_ADDR; 1353 dev = mdev_dev(info->vgpu->vdev.mdev); 1354 rc = vfio_pin_pages(dev, &gfn, 1, IOMMU_READ | IOMMU_WRITE, &pfn); 1355 if (rc != 1) { 1356 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n", 1357 gfn, rc); 1358 return INTEL_GVT_INVALID_ADDR; 1359 } 1360 /* transfer to host iova for GFX to use DMA */ 1361 rc = gvt_dma_map_iova(info->vgpu, pfn, &iova); 1362 if (rc) { 1363 gvt_vgpu_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn); 1364 vfio_unpin_pages(dev, &gfn, 1); 1365 return INTEL_GVT_INVALID_ADDR; 1366 } 1367 1368 gvt_cache_add(info->vgpu, gfn, iova); 1369 return iova; 1370 } 1371 1372 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, 1373 void *buf, unsigned long len, bool write) 1374 { 1375 struct kvmgt_guest_info *info; 1376 struct kvm *kvm; 1377 int idx, ret; 1378 bool kthread = current->mm == NULL; 1379 1380 if (!handle_valid(handle)) 1381 return -ESRCH; 1382 1383 info = (struct kvmgt_guest_info *)handle; 1384 kvm = info->kvm; 1385 1386 if (kthread) 1387 use_mm(kvm->mm); 1388 1389 idx = srcu_read_lock(&kvm->srcu); 1390 ret = write ? kvm_write_guest(kvm, gpa, buf, len) : 1391 kvm_read_guest(kvm, gpa, buf, len); 1392 srcu_read_unlock(&kvm->srcu, idx); 1393 1394 if (kthread) 1395 unuse_mm(kvm->mm); 1396 1397 return ret; 1398 } 1399 1400 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa, 1401 void *buf, unsigned long len) 1402 { 1403 return kvmgt_rw_gpa(handle, gpa, buf, len, false); 1404 } 1405 1406 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa, 1407 void *buf, unsigned long len) 1408 { 1409 return kvmgt_rw_gpa(handle, gpa, buf, len, true); 1410 } 1411 1412 static unsigned long kvmgt_virt_to_pfn(void *addr) 1413 { 1414 return PFN_DOWN(__pa(addr)); 1415 } 1416 1417 struct intel_gvt_mpt kvmgt_mpt = { 1418 .host_init = kvmgt_host_init, 1419 .host_exit = kvmgt_host_exit, 1420 .attach_vgpu = kvmgt_attach_vgpu, 1421 .detach_vgpu = kvmgt_detach_vgpu, 1422 .inject_msi = kvmgt_inject_msi, 1423 .from_virt_to_mfn = kvmgt_virt_to_pfn, 1424 .set_wp_page = kvmgt_write_protect_add, 1425 .unset_wp_page = kvmgt_write_protect_remove, 1426 .read_gpa = kvmgt_read_gpa, 1427 .write_gpa = kvmgt_write_gpa, 1428 .gfn_to_mfn = kvmgt_gfn_to_pfn, 1429 }; 1430 EXPORT_SYMBOL_GPL(kvmgt_mpt); 1431 1432 static int __init kvmgt_init(void) 1433 { 1434 return 0; 1435 } 1436 1437 static void __exit kvmgt_exit(void) 1438 { 1439 } 1440 1441 module_init(kvmgt_init); 1442 module_exit(kvmgt_exit); 1443 1444 MODULE_LICENSE("GPL and additional rights"); 1445 MODULE_AUTHOR("Intel Corporation"); 1446