1 /* 2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM 3 * 4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Kevin Tian <kevin.tian@intel.com> 27 * Jike Song <jike.song@intel.com> 28 * Xiaoguang Chen <xiaoguang.chen@intel.com> 29 */ 30 31 #include <linux/init.h> 32 #include <linux/device.h> 33 #include <linux/mm.h> 34 #include <linux/mmu_context.h> 35 #include <linux/types.h> 36 #include <linux/list.h> 37 #include <linux/rbtree.h> 38 #include <linux/spinlock.h> 39 #include <linux/eventfd.h> 40 #include <linux/uuid.h> 41 #include <linux/kvm_host.h> 42 #include <linux/vfio.h> 43 #include <linux/mdev.h> 44 45 #include "i915_drv.h" 46 #include "gvt.h" 47 48 static const struct intel_gvt_ops *intel_gvt_ops; 49 50 /* helper macros copied from vfio-pci */ 51 #define VFIO_PCI_OFFSET_SHIFT 40 52 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) 53 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 54 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 55 56 struct vfio_region { 57 u32 type; 58 u32 subtype; 59 size_t size; 60 u32 flags; 61 }; 62 63 struct kvmgt_pgfn { 64 gfn_t gfn; 65 struct hlist_node hnode; 66 }; 67 68 struct kvmgt_guest_info { 69 struct kvm *kvm; 70 struct intel_vgpu *vgpu; 71 struct kvm_page_track_notifier_node track_node; 72 #define NR_BKT (1 << 18) 73 struct hlist_head ptable[NR_BKT]; 74 #undef NR_BKT 75 }; 76 77 struct gvt_dma { 78 struct rb_node node; 79 gfn_t gfn; 80 unsigned long iova; 81 }; 82 83 static inline bool handle_valid(unsigned long handle) 84 { 85 return !!(handle & ~0xff); 86 } 87 88 static int kvmgt_guest_init(struct mdev_device *mdev); 89 static void intel_vgpu_release_work(struct work_struct *work); 90 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); 91 92 static int gvt_dma_map_iova(struct intel_vgpu *vgpu, kvm_pfn_t pfn, 93 unsigned long *iova) 94 { 95 struct page *page; 96 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 97 dma_addr_t daddr; 98 99 if (unlikely(!pfn_valid(pfn))) 100 return -EFAULT; 101 102 page = pfn_to_page(pfn); 103 daddr = dma_map_page(dev, page, 0, PAGE_SIZE, 104 PCI_DMA_BIDIRECTIONAL); 105 if (dma_mapping_error(dev, daddr)) 106 return -ENOMEM; 107 108 *iova = (unsigned long)(daddr >> PAGE_SHIFT); 109 return 0; 110 } 111 112 static void gvt_dma_unmap_iova(struct intel_vgpu *vgpu, unsigned long iova) 113 { 114 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 115 dma_addr_t daddr; 116 117 daddr = (dma_addr_t)(iova << PAGE_SHIFT); 118 dma_unmap_page(dev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 119 } 120 121 static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 122 { 123 struct rb_node *node = vgpu->vdev.cache.rb_node; 124 struct gvt_dma *ret = NULL; 125 126 while (node) { 127 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node); 128 129 if (gfn < itr->gfn) 130 node = node->rb_left; 131 else if (gfn > itr->gfn) 132 node = node->rb_right; 133 else { 134 ret = itr; 135 goto out; 136 } 137 } 138 139 out: 140 return ret; 141 } 142 143 static unsigned long gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 144 { 145 struct gvt_dma *entry; 146 unsigned long iova; 147 148 mutex_lock(&vgpu->vdev.cache_lock); 149 150 entry = __gvt_cache_find(vgpu, gfn); 151 iova = (entry == NULL) ? INTEL_GVT_INVALID_ADDR : entry->iova; 152 153 mutex_unlock(&vgpu->vdev.cache_lock); 154 return iova; 155 } 156 157 static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 158 unsigned long iova) 159 { 160 struct gvt_dma *new, *itr; 161 struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL; 162 163 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 164 if (!new) 165 return; 166 167 new->gfn = gfn; 168 new->iova = iova; 169 170 mutex_lock(&vgpu->vdev.cache_lock); 171 while (*link) { 172 parent = *link; 173 itr = rb_entry(parent, struct gvt_dma, node); 174 175 if (gfn == itr->gfn) 176 goto out; 177 else if (gfn < itr->gfn) 178 link = &parent->rb_left; 179 else 180 link = &parent->rb_right; 181 } 182 183 rb_link_node(&new->node, parent, link); 184 rb_insert_color(&new->node, &vgpu->vdev.cache); 185 mutex_unlock(&vgpu->vdev.cache_lock); 186 return; 187 188 out: 189 mutex_unlock(&vgpu->vdev.cache_lock); 190 kfree(new); 191 } 192 193 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 194 struct gvt_dma *entry) 195 { 196 rb_erase(&entry->node, &vgpu->vdev.cache); 197 kfree(entry); 198 } 199 200 static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn) 201 { 202 struct device *dev = mdev_dev(vgpu->vdev.mdev); 203 struct gvt_dma *this; 204 unsigned long g1; 205 int rc; 206 207 mutex_lock(&vgpu->vdev.cache_lock); 208 this = __gvt_cache_find(vgpu, gfn); 209 if (!this) { 210 mutex_unlock(&vgpu->vdev.cache_lock); 211 return; 212 } 213 214 g1 = gfn; 215 gvt_dma_unmap_iova(vgpu, this->iova); 216 rc = vfio_unpin_pages(dev, &g1, 1); 217 WARN_ON(rc != 1); 218 __gvt_cache_remove_entry(vgpu, this); 219 mutex_unlock(&vgpu->vdev.cache_lock); 220 } 221 222 static void gvt_cache_init(struct intel_vgpu *vgpu) 223 { 224 vgpu->vdev.cache = RB_ROOT; 225 mutex_init(&vgpu->vdev.cache_lock); 226 } 227 228 static void gvt_cache_destroy(struct intel_vgpu *vgpu) 229 { 230 struct gvt_dma *dma; 231 struct rb_node *node = NULL; 232 struct device *dev = mdev_dev(vgpu->vdev.mdev); 233 unsigned long gfn; 234 235 mutex_lock(&vgpu->vdev.cache_lock); 236 while ((node = rb_first(&vgpu->vdev.cache))) { 237 dma = rb_entry(node, struct gvt_dma, node); 238 gvt_dma_unmap_iova(vgpu, dma->iova); 239 gfn = dma->gfn; 240 241 vfio_unpin_pages(dev, &gfn, 1); 242 __gvt_cache_remove_entry(vgpu, dma); 243 } 244 mutex_unlock(&vgpu->vdev.cache_lock); 245 } 246 247 static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt, 248 const char *name) 249 { 250 int i; 251 struct intel_vgpu_type *t; 252 const char *driver_name = dev_driver_string( 253 &gvt->dev_priv->drm.pdev->dev); 254 255 for (i = 0; i < gvt->num_types; i++) { 256 t = &gvt->types[i]; 257 if (!strncmp(t->name, name + strlen(driver_name) + 1, 258 sizeof(t->name))) 259 return t; 260 } 261 262 return NULL; 263 } 264 265 static ssize_t available_instances_show(struct kobject *kobj, 266 struct device *dev, char *buf) 267 { 268 struct intel_vgpu_type *type; 269 unsigned int num = 0; 270 void *gvt = kdev_to_i915(dev)->gvt; 271 272 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 273 if (!type) 274 num = 0; 275 else 276 num = type->avail_instance; 277 278 return sprintf(buf, "%u\n", num); 279 } 280 281 static ssize_t device_api_show(struct kobject *kobj, struct device *dev, 282 char *buf) 283 { 284 return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); 285 } 286 287 static ssize_t description_show(struct kobject *kobj, struct device *dev, 288 char *buf) 289 { 290 struct intel_vgpu_type *type; 291 void *gvt = kdev_to_i915(dev)->gvt; 292 293 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 294 if (!type) 295 return 0; 296 297 return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" 298 "fence: %d\nresolution: %s\n", 299 BYTES_TO_MB(type->low_gm_size), 300 BYTES_TO_MB(type->high_gm_size), 301 type->fence, vgpu_edid_str(type->resolution)); 302 } 303 304 static MDEV_TYPE_ATTR_RO(available_instances); 305 static MDEV_TYPE_ATTR_RO(device_api); 306 static MDEV_TYPE_ATTR_RO(description); 307 308 static struct attribute *type_attrs[] = { 309 &mdev_type_attr_available_instances.attr, 310 &mdev_type_attr_device_api.attr, 311 &mdev_type_attr_description.attr, 312 NULL, 313 }; 314 315 static struct attribute_group *intel_vgpu_type_groups[] = { 316 [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, 317 }; 318 319 static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) 320 { 321 int i, j; 322 struct intel_vgpu_type *type; 323 struct attribute_group *group; 324 325 for (i = 0; i < gvt->num_types; i++) { 326 type = &gvt->types[i]; 327 328 group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); 329 if (WARN_ON(!group)) 330 goto unwind; 331 332 group->name = type->name; 333 group->attrs = type_attrs; 334 intel_vgpu_type_groups[i] = group; 335 } 336 337 return true; 338 339 unwind: 340 for (j = 0; j < i; j++) { 341 group = intel_vgpu_type_groups[j]; 342 kfree(group); 343 } 344 345 return false; 346 } 347 348 static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) 349 { 350 int i; 351 struct attribute_group *group; 352 353 for (i = 0; i < gvt->num_types; i++) { 354 group = intel_vgpu_type_groups[i]; 355 kfree(group); 356 } 357 } 358 359 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) 360 { 361 hash_init(info->ptable); 362 } 363 364 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info) 365 { 366 struct kvmgt_pgfn *p; 367 struct hlist_node *tmp; 368 int i; 369 370 hash_for_each_safe(info->ptable, i, tmp, p, hnode) { 371 hash_del(&p->hnode); 372 kfree(p); 373 } 374 } 375 376 static struct kvmgt_pgfn * 377 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn) 378 { 379 struct kvmgt_pgfn *p, *res = NULL; 380 381 hash_for_each_possible(info->ptable, p, hnode, gfn) { 382 if (gfn == p->gfn) { 383 res = p; 384 break; 385 } 386 } 387 388 return res; 389 } 390 391 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info, 392 gfn_t gfn) 393 { 394 struct kvmgt_pgfn *p; 395 396 p = __kvmgt_protect_table_find(info, gfn); 397 return !!p; 398 } 399 400 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn) 401 { 402 struct kvmgt_pgfn *p; 403 404 if (kvmgt_gfn_is_write_protected(info, gfn)) 405 return; 406 407 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); 408 if (WARN(!p, "gfn: 0x%llx\n", gfn)) 409 return; 410 411 p->gfn = gfn; 412 hash_add(info->ptable, &p->hnode, gfn); 413 } 414 415 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, 416 gfn_t gfn) 417 { 418 struct kvmgt_pgfn *p; 419 420 p = __kvmgt_protect_table_find(info, gfn); 421 if (p) { 422 hash_del(&p->hnode); 423 kfree(p); 424 } 425 } 426 427 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) 428 { 429 struct intel_vgpu *vgpu = NULL; 430 struct intel_vgpu_type *type; 431 struct device *pdev; 432 void *gvt; 433 int ret; 434 435 pdev = mdev_parent_dev(mdev); 436 gvt = kdev_to_i915(pdev)->gvt; 437 438 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 439 if (!type) { 440 gvt_vgpu_err("failed to find type %s to create\n", 441 kobject_name(kobj)); 442 ret = -EINVAL; 443 goto out; 444 } 445 446 vgpu = intel_gvt_ops->vgpu_create(gvt, type); 447 if (IS_ERR_OR_NULL(vgpu)) { 448 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu); 449 gvt_vgpu_err("failed to create intel vgpu: %d\n", ret); 450 goto out; 451 } 452 453 INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work); 454 455 vgpu->vdev.mdev = mdev; 456 mdev_set_drvdata(mdev, vgpu); 457 458 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", 459 dev_name(mdev_dev(mdev))); 460 ret = 0; 461 462 out: 463 return ret; 464 } 465 466 static int intel_vgpu_remove(struct mdev_device *mdev) 467 { 468 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 469 470 if (handle_valid(vgpu->handle)) 471 return -EBUSY; 472 473 intel_gvt_ops->vgpu_destroy(vgpu); 474 return 0; 475 } 476 477 static int intel_vgpu_iommu_notifier(struct notifier_block *nb, 478 unsigned long action, void *data) 479 { 480 struct intel_vgpu *vgpu = container_of(nb, 481 struct intel_vgpu, 482 vdev.iommu_notifier); 483 484 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 485 struct vfio_iommu_type1_dma_unmap *unmap = data; 486 unsigned long gfn, end_gfn; 487 488 gfn = unmap->iova >> PAGE_SHIFT; 489 end_gfn = gfn + unmap->size / PAGE_SIZE; 490 491 while (gfn < end_gfn) 492 gvt_cache_remove(vgpu, gfn++); 493 } 494 495 return NOTIFY_OK; 496 } 497 498 static int intel_vgpu_group_notifier(struct notifier_block *nb, 499 unsigned long action, void *data) 500 { 501 struct intel_vgpu *vgpu = container_of(nb, 502 struct intel_vgpu, 503 vdev.group_notifier); 504 505 /* the only action we care about */ 506 if (action == VFIO_GROUP_NOTIFY_SET_KVM) { 507 vgpu->vdev.kvm = data; 508 509 if (!data) 510 schedule_work(&vgpu->vdev.release_work); 511 } 512 513 return NOTIFY_OK; 514 } 515 516 static int intel_vgpu_open(struct mdev_device *mdev) 517 { 518 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 519 unsigned long events; 520 int ret; 521 522 vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier; 523 vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier; 524 525 events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; 526 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events, 527 &vgpu->vdev.iommu_notifier); 528 if (ret != 0) { 529 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n", 530 ret); 531 goto out; 532 } 533 534 events = VFIO_GROUP_NOTIFY_SET_KVM; 535 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events, 536 &vgpu->vdev.group_notifier); 537 if (ret != 0) { 538 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n", 539 ret); 540 goto undo_iommu; 541 } 542 543 ret = kvmgt_guest_init(mdev); 544 if (ret) 545 goto undo_group; 546 547 intel_gvt_ops->vgpu_activate(vgpu); 548 549 atomic_set(&vgpu->vdev.released, 0); 550 return ret; 551 552 undo_group: 553 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, 554 &vgpu->vdev.group_notifier); 555 556 undo_iommu: 557 vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, 558 &vgpu->vdev.iommu_notifier); 559 out: 560 return ret; 561 } 562 563 static void __intel_vgpu_release(struct intel_vgpu *vgpu) 564 { 565 struct kvmgt_guest_info *info; 566 int ret; 567 568 if (!handle_valid(vgpu->handle)) 569 return; 570 571 if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1)) 572 return; 573 574 intel_gvt_ops->vgpu_deactivate(vgpu); 575 576 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY, 577 &vgpu->vdev.iommu_notifier); 578 WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret); 579 580 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY, 581 &vgpu->vdev.group_notifier); 582 WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret); 583 584 info = (struct kvmgt_guest_info *)vgpu->handle; 585 kvmgt_guest_exit(info); 586 587 vgpu->vdev.kvm = NULL; 588 vgpu->handle = 0; 589 } 590 591 static void intel_vgpu_release(struct mdev_device *mdev) 592 { 593 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 594 595 __intel_vgpu_release(vgpu); 596 } 597 598 static void intel_vgpu_release_work(struct work_struct *work) 599 { 600 struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu, 601 vdev.release_work); 602 603 __intel_vgpu_release(vgpu); 604 } 605 606 static uint64_t intel_vgpu_get_bar0_addr(struct intel_vgpu *vgpu) 607 { 608 u32 start_lo, start_hi; 609 u32 mem_type; 610 int pos = PCI_BASE_ADDRESS_0; 611 612 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) & 613 PCI_BASE_ADDRESS_MEM_MASK; 614 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) & 615 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 616 617 switch (mem_type) { 618 case PCI_BASE_ADDRESS_MEM_TYPE_64: 619 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space 620 + pos + 4)); 621 break; 622 case PCI_BASE_ADDRESS_MEM_TYPE_32: 623 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 624 /* 1M mem BAR treated as 32-bit BAR */ 625 default: 626 /* mem unknown type treated as 32-bit BAR */ 627 start_hi = 0; 628 break; 629 } 630 631 return ((u64)start_hi << 32) | start_lo; 632 } 633 634 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, 635 size_t count, loff_t *ppos, bool is_write) 636 { 637 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 638 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 639 uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 640 int ret = -EINVAL; 641 642 643 if (index >= VFIO_PCI_NUM_REGIONS) { 644 gvt_vgpu_err("invalid index: %u\n", index); 645 return -EINVAL; 646 } 647 648 switch (index) { 649 case VFIO_PCI_CONFIG_REGION_INDEX: 650 if (is_write) 651 ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos, 652 buf, count); 653 else 654 ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos, 655 buf, count); 656 break; 657 case VFIO_PCI_BAR0_REGION_INDEX: 658 case VFIO_PCI_BAR1_REGION_INDEX: 659 if (is_write) { 660 uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu); 661 662 ret = intel_gvt_ops->emulate_mmio_write(vgpu, 663 bar0_start + pos, buf, count); 664 } else { 665 uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu); 666 667 ret = intel_gvt_ops->emulate_mmio_read(vgpu, 668 bar0_start + pos, buf, count); 669 } 670 break; 671 case VFIO_PCI_BAR2_REGION_INDEX: 672 case VFIO_PCI_BAR3_REGION_INDEX: 673 case VFIO_PCI_BAR4_REGION_INDEX: 674 case VFIO_PCI_BAR5_REGION_INDEX: 675 case VFIO_PCI_VGA_REGION_INDEX: 676 case VFIO_PCI_ROM_REGION_INDEX: 677 default: 678 gvt_vgpu_err("unsupported region: %u\n", index); 679 } 680 681 return ret == 0 ? count : ret; 682 } 683 684 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, 685 size_t count, loff_t *ppos) 686 { 687 unsigned int done = 0; 688 int ret; 689 690 while (count) { 691 size_t filled; 692 693 if (count >= 4 && !(*ppos % 4)) { 694 u32 val; 695 696 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 697 ppos, false); 698 if (ret <= 0) 699 goto read_err; 700 701 if (copy_to_user(buf, &val, sizeof(val))) 702 goto read_err; 703 704 filled = 4; 705 } else if (count >= 2 && !(*ppos % 2)) { 706 u16 val; 707 708 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 709 ppos, false); 710 if (ret <= 0) 711 goto read_err; 712 713 if (copy_to_user(buf, &val, sizeof(val))) 714 goto read_err; 715 716 filled = 2; 717 } else { 718 u8 val; 719 720 ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos, 721 false); 722 if (ret <= 0) 723 goto read_err; 724 725 if (copy_to_user(buf, &val, sizeof(val))) 726 goto read_err; 727 728 filled = 1; 729 } 730 731 count -= filled; 732 done += filled; 733 *ppos += filled; 734 buf += filled; 735 } 736 737 return done; 738 739 read_err: 740 return -EFAULT; 741 } 742 743 static ssize_t intel_vgpu_write(struct mdev_device *mdev, 744 const char __user *buf, 745 size_t count, loff_t *ppos) 746 { 747 unsigned int done = 0; 748 int ret; 749 750 while (count) { 751 size_t filled; 752 753 if (count >= 4 && !(*ppos % 4)) { 754 u32 val; 755 756 if (copy_from_user(&val, buf, sizeof(val))) 757 goto write_err; 758 759 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 760 ppos, true); 761 if (ret <= 0) 762 goto write_err; 763 764 filled = 4; 765 } else if (count >= 2 && !(*ppos % 2)) { 766 u16 val; 767 768 if (copy_from_user(&val, buf, sizeof(val))) 769 goto write_err; 770 771 ret = intel_vgpu_rw(mdev, (char *)&val, 772 sizeof(val), ppos, true); 773 if (ret <= 0) 774 goto write_err; 775 776 filled = 2; 777 } else { 778 u8 val; 779 780 if (copy_from_user(&val, buf, sizeof(val))) 781 goto write_err; 782 783 ret = intel_vgpu_rw(mdev, &val, sizeof(val), 784 ppos, true); 785 if (ret <= 0) 786 goto write_err; 787 788 filled = 1; 789 } 790 791 count -= filled; 792 done += filled; 793 *ppos += filled; 794 buf += filled; 795 } 796 797 return done; 798 write_err: 799 return -EFAULT; 800 } 801 802 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) 803 { 804 unsigned int index; 805 u64 virtaddr; 806 unsigned long req_size, pgoff = 0; 807 pgprot_t pg_prot; 808 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 809 810 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 811 if (index >= VFIO_PCI_ROM_REGION_INDEX) 812 return -EINVAL; 813 814 if (vma->vm_end < vma->vm_start) 815 return -EINVAL; 816 if ((vma->vm_flags & VM_SHARED) == 0) 817 return -EINVAL; 818 if (index != VFIO_PCI_BAR2_REGION_INDEX) 819 return -EINVAL; 820 821 pg_prot = vma->vm_page_prot; 822 virtaddr = vma->vm_start; 823 req_size = vma->vm_end - vma->vm_start; 824 pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; 825 826 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); 827 } 828 829 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) 830 { 831 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) 832 return 1; 833 834 return 0; 835 } 836 837 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, 838 unsigned int index, unsigned int start, 839 unsigned int count, uint32_t flags, 840 void *data) 841 { 842 return 0; 843 } 844 845 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, 846 unsigned int index, unsigned int start, 847 unsigned int count, uint32_t flags, void *data) 848 { 849 return 0; 850 } 851 852 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, 853 unsigned int index, unsigned int start, unsigned int count, 854 uint32_t flags, void *data) 855 { 856 return 0; 857 } 858 859 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, 860 unsigned int index, unsigned int start, unsigned int count, 861 uint32_t flags, void *data) 862 { 863 struct eventfd_ctx *trigger; 864 865 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 866 int fd = *(int *)data; 867 868 trigger = eventfd_ctx_fdget(fd); 869 if (IS_ERR(trigger)) { 870 gvt_vgpu_err("eventfd_ctx_fdget failed\n"); 871 return PTR_ERR(trigger); 872 } 873 vgpu->vdev.msi_trigger = trigger; 874 } 875 876 return 0; 877 } 878 879 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags, 880 unsigned int index, unsigned int start, unsigned int count, 881 void *data) 882 { 883 int (*func)(struct intel_vgpu *vgpu, unsigned int index, 884 unsigned int start, unsigned int count, uint32_t flags, 885 void *data) = NULL; 886 887 switch (index) { 888 case VFIO_PCI_INTX_IRQ_INDEX: 889 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 890 case VFIO_IRQ_SET_ACTION_MASK: 891 func = intel_vgpu_set_intx_mask; 892 break; 893 case VFIO_IRQ_SET_ACTION_UNMASK: 894 func = intel_vgpu_set_intx_unmask; 895 break; 896 case VFIO_IRQ_SET_ACTION_TRIGGER: 897 func = intel_vgpu_set_intx_trigger; 898 break; 899 } 900 break; 901 case VFIO_PCI_MSI_IRQ_INDEX: 902 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 903 case VFIO_IRQ_SET_ACTION_MASK: 904 case VFIO_IRQ_SET_ACTION_UNMASK: 905 /* XXX Need masking support exported */ 906 break; 907 case VFIO_IRQ_SET_ACTION_TRIGGER: 908 func = intel_vgpu_set_msi_trigger; 909 break; 910 } 911 break; 912 } 913 914 if (!func) 915 return -ENOTTY; 916 917 return func(vgpu, index, start, count, flags, data); 918 } 919 920 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, 921 unsigned long arg) 922 { 923 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 924 unsigned long minsz; 925 926 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); 927 928 if (cmd == VFIO_DEVICE_GET_INFO) { 929 struct vfio_device_info info; 930 931 minsz = offsetofend(struct vfio_device_info, num_irqs); 932 933 if (copy_from_user(&info, (void __user *)arg, minsz)) 934 return -EFAULT; 935 936 if (info.argsz < minsz) 937 return -EINVAL; 938 939 info.flags = VFIO_DEVICE_FLAGS_PCI; 940 info.flags |= VFIO_DEVICE_FLAGS_RESET; 941 info.num_regions = VFIO_PCI_NUM_REGIONS; 942 info.num_irqs = VFIO_PCI_NUM_IRQS; 943 944 return copy_to_user((void __user *)arg, &info, minsz) ? 945 -EFAULT : 0; 946 947 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 948 struct vfio_region_info info; 949 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 950 int i, ret; 951 struct vfio_region_info_cap_sparse_mmap *sparse = NULL; 952 size_t size; 953 int nr_areas = 1; 954 int cap_type_id; 955 956 minsz = offsetofend(struct vfio_region_info, offset); 957 958 if (copy_from_user(&info, (void __user *)arg, minsz)) 959 return -EFAULT; 960 961 if (info.argsz < minsz) 962 return -EINVAL; 963 964 switch (info.index) { 965 case VFIO_PCI_CONFIG_REGION_INDEX: 966 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 967 info.size = INTEL_GVT_MAX_CFG_SPACE_SZ; 968 info.flags = VFIO_REGION_INFO_FLAG_READ | 969 VFIO_REGION_INFO_FLAG_WRITE; 970 break; 971 case VFIO_PCI_BAR0_REGION_INDEX: 972 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 973 info.size = vgpu->cfg_space.bar[info.index].size; 974 if (!info.size) { 975 info.flags = 0; 976 break; 977 } 978 979 info.flags = VFIO_REGION_INFO_FLAG_READ | 980 VFIO_REGION_INFO_FLAG_WRITE; 981 break; 982 case VFIO_PCI_BAR1_REGION_INDEX: 983 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 984 info.size = 0; 985 info.flags = 0; 986 break; 987 case VFIO_PCI_BAR2_REGION_INDEX: 988 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 989 info.flags = VFIO_REGION_INFO_FLAG_CAPS | 990 VFIO_REGION_INFO_FLAG_MMAP | 991 VFIO_REGION_INFO_FLAG_READ | 992 VFIO_REGION_INFO_FLAG_WRITE; 993 info.size = gvt_aperture_sz(vgpu->gvt); 994 995 size = sizeof(*sparse) + 996 (nr_areas * sizeof(*sparse->areas)); 997 sparse = kzalloc(size, GFP_KERNEL); 998 if (!sparse) 999 return -ENOMEM; 1000 1001 sparse->nr_areas = nr_areas; 1002 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1003 sparse->areas[0].offset = 1004 PAGE_ALIGN(vgpu_aperture_offset(vgpu)); 1005 sparse->areas[0].size = vgpu_aperture_sz(vgpu); 1006 break; 1007 1008 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1009 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1010 info.size = 0; 1011 1012 info.flags = 0; 1013 gvt_dbg_core("get region info bar:%d\n", info.index); 1014 break; 1015 1016 case VFIO_PCI_ROM_REGION_INDEX: 1017 case VFIO_PCI_VGA_REGION_INDEX: 1018 gvt_dbg_core("get region info index:%d\n", info.index); 1019 break; 1020 default: 1021 { 1022 struct vfio_region_info_cap_type cap_type; 1023 1024 if (info.index >= VFIO_PCI_NUM_REGIONS + 1025 vgpu->vdev.num_regions) 1026 return -EINVAL; 1027 1028 i = info.index - VFIO_PCI_NUM_REGIONS; 1029 1030 info.offset = 1031 VFIO_PCI_INDEX_TO_OFFSET(info.index); 1032 info.size = vgpu->vdev.region[i].size; 1033 info.flags = vgpu->vdev.region[i].flags; 1034 1035 cap_type.type = vgpu->vdev.region[i].type; 1036 cap_type.subtype = vgpu->vdev.region[i].subtype; 1037 1038 ret = vfio_info_add_capability(&caps, 1039 VFIO_REGION_INFO_CAP_TYPE, 1040 &cap_type); 1041 if (ret) 1042 return ret; 1043 } 1044 } 1045 1046 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { 1047 switch (cap_type_id) { 1048 case VFIO_REGION_INFO_CAP_SPARSE_MMAP: 1049 ret = vfio_info_add_capability(&caps, 1050 VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1051 sparse); 1052 kfree(sparse); 1053 if (ret) 1054 return ret; 1055 break; 1056 default: 1057 return -EINVAL; 1058 } 1059 } 1060 1061 if (caps.size) { 1062 if (info.argsz < sizeof(info) + caps.size) { 1063 info.argsz = sizeof(info) + caps.size; 1064 info.cap_offset = 0; 1065 } else { 1066 vfio_info_cap_shift(&caps, sizeof(info)); 1067 if (copy_to_user((void __user *)arg + 1068 sizeof(info), caps.buf, 1069 caps.size)) { 1070 kfree(caps.buf); 1071 return -EFAULT; 1072 } 1073 info.cap_offset = sizeof(info); 1074 } 1075 1076 kfree(caps.buf); 1077 } 1078 1079 return copy_to_user((void __user *)arg, &info, minsz) ? 1080 -EFAULT : 0; 1081 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 1082 struct vfio_irq_info info; 1083 1084 minsz = offsetofend(struct vfio_irq_info, count); 1085 1086 if (copy_from_user(&info, (void __user *)arg, minsz)) 1087 return -EFAULT; 1088 1089 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1090 return -EINVAL; 1091 1092 switch (info.index) { 1093 case VFIO_PCI_INTX_IRQ_INDEX: 1094 case VFIO_PCI_MSI_IRQ_INDEX: 1095 break; 1096 default: 1097 return -EINVAL; 1098 } 1099 1100 info.flags = VFIO_IRQ_INFO_EVENTFD; 1101 1102 info.count = intel_vgpu_get_irq_count(vgpu, info.index); 1103 1104 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1105 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1106 VFIO_IRQ_INFO_AUTOMASKED); 1107 else 1108 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1109 1110 return copy_to_user((void __user *)arg, &info, minsz) ? 1111 -EFAULT : 0; 1112 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1113 struct vfio_irq_set hdr; 1114 u8 *data = NULL; 1115 int ret = 0; 1116 size_t data_size = 0; 1117 1118 minsz = offsetofend(struct vfio_irq_set, count); 1119 1120 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1121 return -EFAULT; 1122 1123 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 1124 int max = intel_vgpu_get_irq_count(vgpu, hdr.index); 1125 1126 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1127 VFIO_PCI_NUM_IRQS, &data_size); 1128 if (ret) { 1129 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); 1130 return -EINVAL; 1131 } 1132 if (data_size) { 1133 data = memdup_user((void __user *)(arg + minsz), 1134 data_size); 1135 if (IS_ERR(data)) 1136 return PTR_ERR(data); 1137 } 1138 } 1139 1140 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, 1141 hdr.start, hdr.count, data); 1142 kfree(data); 1143 1144 return ret; 1145 } else if (cmd == VFIO_DEVICE_RESET) { 1146 intel_gvt_ops->vgpu_reset(vgpu); 1147 return 0; 1148 } 1149 1150 return 0; 1151 } 1152 1153 static const struct mdev_parent_ops intel_vgpu_ops = { 1154 .supported_type_groups = intel_vgpu_type_groups, 1155 .create = intel_vgpu_create, 1156 .remove = intel_vgpu_remove, 1157 1158 .open = intel_vgpu_open, 1159 .release = intel_vgpu_release, 1160 1161 .read = intel_vgpu_read, 1162 .write = intel_vgpu_write, 1163 .mmap = intel_vgpu_mmap, 1164 .ioctl = intel_vgpu_ioctl, 1165 }; 1166 1167 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) 1168 { 1169 if (!intel_gvt_init_vgpu_type_groups(gvt)) 1170 return -EFAULT; 1171 1172 intel_gvt_ops = ops; 1173 1174 return mdev_register_device(dev, &intel_vgpu_ops); 1175 } 1176 1177 static void kvmgt_host_exit(struct device *dev, void *gvt) 1178 { 1179 intel_gvt_cleanup_vgpu_type_groups(gvt); 1180 mdev_unregister_device(dev); 1181 } 1182 1183 static int kvmgt_write_protect_add(unsigned long handle, u64 gfn) 1184 { 1185 struct kvmgt_guest_info *info; 1186 struct kvm *kvm; 1187 struct kvm_memory_slot *slot; 1188 int idx; 1189 1190 if (!handle_valid(handle)) 1191 return -ESRCH; 1192 1193 info = (struct kvmgt_guest_info *)handle; 1194 kvm = info->kvm; 1195 1196 idx = srcu_read_lock(&kvm->srcu); 1197 slot = gfn_to_memslot(kvm, gfn); 1198 if (!slot) { 1199 srcu_read_unlock(&kvm->srcu, idx); 1200 return -EINVAL; 1201 } 1202 1203 spin_lock(&kvm->mmu_lock); 1204 1205 if (kvmgt_gfn_is_write_protected(info, gfn)) 1206 goto out; 1207 1208 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1209 kvmgt_protect_table_add(info, gfn); 1210 1211 out: 1212 spin_unlock(&kvm->mmu_lock); 1213 srcu_read_unlock(&kvm->srcu, idx); 1214 return 0; 1215 } 1216 1217 static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn) 1218 { 1219 struct kvmgt_guest_info *info; 1220 struct kvm *kvm; 1221 struct kvm_memory_slot *slot; 1222 int idx; 1223 1224 if (!handle_valid(handle)) 1225 return 0; 1226 1227 info = (struct kvmgt_guest_info *)handle; 1228 kvm = info->kvm; 1229 1230 idx = srcu_read_lock(&kvm->srcu); 1231 slot = gfn_to_memslot(kvm, gfn); 1232 if (!slot) { 1233 srcu_read_unlock(&kvm->srcu, idx); 1234 return -EINVAL; 1235 } 1236 1237 spin_lock(&kvm->mmu_lock); 1238 1239 if (!kvmgt_gfn_is_write_protected(info, gfn)) 1240 goto out; 1241 1242 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1243 kvmgt_protect_table_del(info, gfn); 1244 1245 out: 1246 spin_unlock(&kvm->mmu_lock); 1247 srcu_read_unlock(&kvm->srcu, idx); 1248 return 0; 1249 } 1250 1251 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 1252 const u8 *val, int len, 1253 struct kvm_page_track_notifier_node *node) 1254 { 1255 struct kvmgt_guest_info *info = container_of(node, 1256 struct kvmgt_guest_info, track_node); 1257 1258 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) 1259 intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa, 1260 (void *)val, len); 1261 } 1262 1263 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 1264 struct kvm_memory_slot *slot, 1265 struct kvm_page_track_notifier_node *node) 1266 { 1267 int i; 1268 gfn_t gfn; 1269 struct kvmgt_guest_info *info = container_of(node, 1270 struct kvmgt_guest_info, track_node); 1271 1272 spin_lock(&kvm->mmu_lock); 1273 for (i = 0; i < slot->npages; i++) { 1274 gfn = slot->base_gfn + i; 1275 if (kvmgt_gfn_is_write_protected(info, gfn)) { 1276 kvm_slot_page_track_remove_page(kvm, slot, gfn, 1277 KVM_PAGE_TRACK_WRITE); 1278 kvmgt_protect_table_del(info, gfn); 1279 } 1280 } 1281 spin_unlock(&kvm->mmu_lock); 1282 } 1283 1284 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm) 1285 { 1286 struct intel_vgpu *itr; 1287 struct kvmgt_guest_info *info; 1288 int id; 1289 bool ret = false; 1290 1291 mutex_lock(&vgpu->gvt->lock); 1292 for_each_active_vgpu(vgpu->gvt, itr, id) { 1293 if (!handle_valid(itr->handle)) 1294 continue; 1295 1296 info = (struct kvmgt_guest_info *)itr->handle; 1297 if (kvm && kvm == info->kvm) { 1298 ret = true; 1299 goto out; 1300 } 1301 } 1302 out: 1303 mutex_unlock(&vgpu->gvt->lock); 1304 return ret; 1305 } 1306 1307 static int kvmgt_guest_init(struct mdev_device *mdev) 1308 { 1309 struct kvmgt_guest_info *info; 1310 struct intel_vgpu *vgpu; 1311 struct kvm *kvm; 1312 1313 vgpu = mdev_get_drvdata(mdev); 1314 if (handle_valid(vgpu->handle)) 1315 return -EEXIST; 1316 1317 kvm = vgpu->vdev.kvm; 1318 if (!kvm || kvm->mm != current->mm) { 1319 gvt_vgpu_err("KVM is required to use Intel vGPU\n"); 1320 return -ESRCH; 1321 } 1322 1323 if (__kvmgt_vgpu_exist(vgpu, kvm)) 1324 return -EEXIST; 1325 1326 info = vzalloc(sizeof(struct kvmgt_guest_info)); 1327 if (!info) 1328 return -ENOMEM; 1329 1330 vgpu->handle = (unsigned long)info; 1331 info->vgpu = vgpu; 1332 info->kvm = kvm; 1333 kvm_get_kvm(info->kvm); 1334 1335 kvmgt_protect_table_init(info); 1336 gvt_cache_init(vgpu); 1337 1338 info->track_node.track_write = kvmgt_page_track_write; 1339 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 1340 kvm_page_track_register_notifier(kvm, &info->track_node); 1341 1342 return 0; 1343 } 1344 1345 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info) 1346 { 1347 kvm_page_track_unregister_notifier(info->kvm, &info->track_node); 1348 kvm_put_kvm(info->kvm); 1349 kvmgt_protect_table_destroy(info); 1350 gvt_cache_destroy(info->vgpu); 1351 vfree(info); 1352 1353 return true; 1354 } 1355 1356 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) 1357 { 1358 /* nothing to do here */ 1359 return 0; 1360 } 1361 1362 static void kvmgt_detach_vgpu(unsigned long handle) 1363 { 1364 /* nothing to do here */ 1365 } 1366 1367 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) 1368 { 1369 struct kvmgt_guest_info *info; 1370 struct intel_vgpu *vgpu; 1371 1372 if (!handle_valid(handle)) 1373 return -ESRCH; 1374 1375 info = (struct kvmgt_guest_info *)handle; 1376 vgpu = info->vgpu; 1377 1378 if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1) 1379 return 0; 1380 1381 return -EFAULT; 1382 } 1383 1384 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) 1385 { 1386 unsigned long iova, pfn; 1387 struct kvmgt_guest_info *info; 1388 struct device *dev; 1389 struct intel_vgpu *vgpu; 1390 int rc; 1391 1392 if (!handle_valid(handle)) 1393 return INTEL_GVT_INVALID_ADDR; 1394 1395 info = (struct kvmgt_guest_info *)handle; 1396 vgpu = info->vgpu; 1397 iova = gvt_cache_find(info->vgpu, gfn); 1398 if (iova != INTEL_GVT_INVALID_ADDR) 1399 return iova; 1400 1401 pfn = INTEL_GVT_INVALID_ADDR; 1402 dev = mdev_dev(info->vgpu->vdev.mdev); 1403 rc = vfio_pin_pages(dev, &gfn, 1, IOMMU_READ | IOMMU_WRITE, &pfn); 1404 if (rc != 1) { 1405 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n", 1406 gfn, rc); 1407 return INTEL_GVT_INVALID_ADDR; 1408 } 1409 /* transfer to host iova for GFX to use DMA */ 1410 rc = gvt_dma_map_iova(info->vgpu, pfn, &iova); 1411 if (rc) { 1412 gvt_vgpu_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn); 1413 vfio_unpin_pages(dev, &gfn, 1); 1414 return INTEL_GVT_INVALID_ADDR; 1415 } 1416 1417 gvt_cache_add(info->vgpu, gfn, iova); 1418 return iova; 1419 } 1420 1421 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, 1422 void *buf, unsigned long len, bool write) 1423 { 1424 struct kvmgt_guest_info *info; 1425 struct kvm *kvm; 1426 int idx, ret; 1427 bool kthread = current->mm == NULL; 1428 1429 if (!handle_valid(handle)) 1430 return -ESRCH; 1431 1432 info = (struct kvmgt_guest_info *)handle; 1433 kvm = info->kvm; 1434 1435 if (kthread) 1436 use_mm(kvm->mm); 1437 1438 idx = srcu_read_lock(&kvm->srcu); 1439 ret = write ? kvm_write_guest(kvm, gpa, buf, len) : 1440 kvm_read_guest(kvm, gpa, buf, len); 1441 srcu_read_unlock(&kvm->srcu, idx); 1442 1443 if (kthread) 1444 unuse_mm(kvm->mm); 1445 1446 return ret; 1447 } 1448 1449 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa, 1450 void *buf, unsigned long len) 1451 { 1452 return kvmgt_rw_gpa(handle, gpa, buf, len, false); 1453 } 1454 1455 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa, 1456 void *buf, unsigned long len) 1457 { 1458 return kvmgt_rw_gpa(handle, gpa, buf, len, true); 1459 } 1460 1461 static unsigned long kvmgt_virt_to_pfn(void *addr) 1462 { 1463 return PFN_DOWN(__pa(addr)); 1464 } 1465 1466 struct intel_gvt_mpt kvmgt_mpt = { 1467 .host_init = kvmgt_host_init, 1468 .host_exit = kvmgt_host_exit, 1469 .attach_vgpu = kvmgt_attach_vgpu, 1470 .detach_vgpu = kvmgt_detach_vgpu, 1471 .inject_msi = kvmgt_inject_msi, 1472 .from_virt_to_mfn = kvmgt_virt_to_pfn, 1473 .set_wp_page = kvmgt_write_protect_add, 1474 .unset_wp_page = kvmgt_write_protect_remove, 1475 .read_gpa = kvmgt_read_gpa, 1476 .write_gpa = kvmgt_write_gpa, 1477 .gfn_to_mfn = kvmgt_gfn_to_pfn, 1478 }; 1479 EXPORT_SYMBOL_GPL(kvmgt_mpt); 1480 1481 static int __init kvmgt_init(void) 1482 { 1483 return 0; 1484 } 1485 1486 static void __exit kvmgt_exit(void) 1487 { 1488 } 1489 1490 module_init(kvmgt_init); 1491 module_exit(kvmgt_exit); 1492 1493 MODULE_LICENSE("GPL and additional rights"); 1494 MODULE_AUTHOR("Intel Corporation"); 1495