1 /* 2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM 3 * 4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Kevin Tian <kevin.tian@intel.com> 27 * Jike Song <jike.song@intel.com> 28 * Xiaoguang Chen <xiaoguang.chen@intel.com> 29 */ 30 31 #include <linux/init.h> 32 #include <linux/device.h> 33 #include <linux/mm.h> 34 #include <linux/mmu_context.h> 35 #include <linux/types.h> 36 #include <linux/list.h> 37 #include <linux/rbtree.h> 38 #include <linux/spinlock.h> 39 #include <linux/eventfd.h> 40 #include <linux/uuid.h> 41 #include <linux/kvm_host.h> 42 #include <linux/vfio.h> 43 #include <linux/mdev.h> 44 45 #include "i915_drv.h" 46 #include "gvt.h" 47 48 static const struct intel_gvt_ops *intel_gvt_ops; 49 50 /* helper macros copied from vfio-pci */ 51 #define VFIO_PCI_OFFSET_SHIFT 40 52 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) 53 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 54 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 55 56 struct vfio_region { 57 u32 type; 58 u32 subtype; 59 size_t size; 60 u32 flags; 61 }; 62 63 struct kvmgt_pgfn { 64 gfn_t gfn; 65 struct hlist_node hnode; 66 }; 67 68 struct kvmgt_guest_info { 69 struct kvm *kvm; 70 struct intel_vgpu *vgpu; 71 struct kvm_page_track_notifier_node track_node; 72 #define NR_BKT (1 << 18) 73 struct hlist_head ptable[NR_BKT]; 74 #undef NR_BKT 75 }; 76 77 struct gvt_dma { 78 struct rb_node node; 79 gfn_t gfn; 80 unsigned long iova; 81 }; 82 83 static inline bool handle_valid(unsigned long handle) 84 { 85 return !!(handle & ~0xff); 86 } 87 88 static int kvmgt_guest_init(struct mdev_device *mdev); 89 static void intel_vgpu_release_work(struct work_struct *work); 90 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); 91 92 static int gvt_dma_map_iova(struct intel_vgpu *vgpu, kvm_pfn_t pfn, 93 unsigned long *iova) 94 { 95 struct page *page; 96 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 97 dma_addr_t daddr; 98 99 page = pfn_to_page(pfn); 100 if (is_error_page(page)) 101 return -EFAULT; 102 103 daddr = dma_map_page(dev, page, 0, PAGE_SIZE, 104 PCI_DMA_BIDIRECTIONAL); 105 if (dma_mapping_error(dev, daddr)) 106 return -ENOMEM; 107 108 *iova = (unsigned long)(daddr >> PAGE_SHIFT); 109 return 0; 110 } 111 112 static void gvt_dma_unmap_iova(struct intel_vgpu *vgpu, unsigned long iova) 113 { 114 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 115 dma_addr_t daddr; 116 117 daddr = (dma_addr_t)(iova << PAGE_SHIFT); 118 dma_unmap_page(dev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 119 } 120 121 static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 122 { 123 struct rb_node *node = vgpu->vdev.cache.rb_node; 124 struct gvt_dma *ret = NULL; 125 126 while (node) { 127 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node); 128 129 if (gfn < itr->gfn) 130 node = node->rb_left; 131 else if (gfn > itr->gfn) 132 node = node->rb_right; 133 else { 134 ret = itr; 135 goto out; 136 } 137 } 138 139 out: 140 return ret; 141 } 142 143 static unsigned long gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 144 { 145 struct gvt_dma *entry; 146 unsigned long iova; 147 148 mutex_lock(&vgpu->vdev.cache_lock); 149 150 entry = __gvt_cache_find(vgpu, gfn); 151 iova = (entry == NULL) ? INTEL_GVT_INVALID_ADDR : entry->iova; 152 153 mutex_unlock(&vgpu->vdev.cache_lock); 154 return iova; 155 } 156 157 static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 158 unsigned long iova) 159 { 160 struct gvt_dma *new, *itr; 161 struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL; 162 163 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 164 if (!new) 165 return; 166 167 new->gfn = gfn; 168 new->iova = iova; 169 170 mutex_lock(&vgpu->vdev.cache_lock); 171 while (*link) { 172 parent = *link; 173 itr = rb_entry(parent, struct gvt_dma, node); 174 175 if (gfn == itr->gfn) 176 goto out; 177 else if (gfn < itr->gfn) 178 link = &parent->rb_left; 179 else 180 link = &parent->rb_right; 181 } 182 183 rb_link_node(&new->node, parent, link); 184 rb_insert_color(&new->node, &vgpu->vdev.cache); 185 mutex_unlock(&vgpu->vdev.cache_lock); 186 return; 187 188 out: 189 mutex_unlock(&vgpu->vdev.cache_lock); 190 kfree(new); 191 } 192 193 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 194 struct gvt_dma *entry) 195 { 196 rb_erase(&entry->node, &vgpu->vdev.cache); 197 kfree(entry); 198 } 199 200 static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn) 201 { 202 struct device *dev = mdev_dev(vgpu->vdev.mdev); 203 struct gvt_dma *this; 204 unsigned long g1; 205 int rc; 206 207 mutex_lock(&vgpu->vdev.cache_lock); 208 this = __gvt_cache_find(vgpu, gfn); 209 if (!this) { 210 mutex_unlock(&vgpu->vdev.cache_lock); 211 return; 212 } 213 214 g1 = gfn; 215 gvt_dma_unmap_iova(vgpu, this->iova); 216 rc = vfio_unpin_pages(dev, &g1, 1); 217 WARN_ON(rc != 1); 218 __gvt_cache_remove_entry(vgpu, this); 219 mutex_unlock(&vgpu->vdev.cache_lock); 220 } 221 222 static void gvt_cache_init(struct intel_vgpu *vgpu) 223 { 224 vgpu->vdev.cache = RB_ROOT; 225 mutex_init(&vgpu->vdev.cache_lock); 226 } 227 228 static void gvt_cache_destroy(struct intel_vgpu *vgpu) 229 { 230 struct gvt_dma *dma; 231 struct rb_node *node = NULL; 232 struct device *dev = mdev_dev(vgpu->vdev.mdev); 233 unsigned long gfn; 234 235 mutex_lock(&vgpu->vdev.cache_lock); 236 while ((node = rb_first(&vgpu->vdev.cache))) { 237 dma = rb_entry(node, struct gvt_dma, node); 238 gvt_dma_unmap_iova(vgpu, dma->iova); 239 gfn = dma->gfn; 240 241 vfio_unpin_pages(dev, &gfn, 1); 242 __gvt_cache_remove_entry(vgpu, dma); 243 } 244 mutex_unlock(&vgpu->vdev.cache_lock); 245 } 246 247 static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt, 248 const char *name) 249 { 250 int i; 251 struct intel_vgpu_type *t; 252 const char *driver_name = dev_driver_string( 253 &gvt->dev_priv->drm.pdev->dev); 254 255 for (i = 0; i < gvt->num_types; i++) { 256 t = &gvt->types[i]; 257 if (!strncmp(t->name, name + strlen(driver_name) + 1, 258 sizeof(t->name))) 259 return t; 260 } 261 262 return NULL; 263 } 264 265 static ssize_t available_instances_show(struct kobject *kobj, 266 struct device *dev, char *buf) 267 { 268 struct intel_vgpu_type *type; 269 unsigned int num = 0; 270 void *gvt = kdev_to_i915(dev)->gvt; 271 272 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 273 if (!type) 274 num = 0; 275 else 276 num = type->avail_instance; 277 278 return sprintf(buf, "%u\n", num); 279 } 280 281 static ssize_t device_api_show(struct kobject *kobj, struct device *dev, 282 char *buf) 283 { 284 return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); 285 } 286 287 static ssize_t description_show(struct kobject *kobj, struct device *dev, 288 char *buf) 289 { 290 struct intel_vgpu_type *type; 291 void *gvt = kdev_to_i915(dev)->gvt; 292 293 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 294 if (!type) 295 return 0; 296 297 return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" 298 "fence: %d\n", 299 BYTES_TO_MB(type->low_gm_size), 300 BYTES_TO_MB(type->high_gm_size), 301 type->fence); 302 } 303 304 static MDEV_TYPE_ATTR_RO(available_instances); 305 static MDEV_TYPE_ATTR_RO(device_api); 306 static MDEV_TYPE_ATTR_RO(description); 307 308 static struct attribute *type_attrs[] = { 309 &mdev_type_attr_available_instances.attr, 310 &mdev_type_attr_device_api.attr, 311 &mdev_type_attr_description.attr, 312 NULL, 313 }; 314 315 static struct attribute_group *intel_vgpu_type_groups[] = { 316 [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, 317 }; 318 319 static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) 320 { 321 int i, j; 322 struct intel_vgpu_type *type; 323 struct attribute_group *group; 324 325 for (i = 0; i < gvt->num_types; i++) { 326 type = &gvt->types[i]; 327 328 group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); 329 if (WARN_ON(!group)) 330 goto unwind; 331 332 group->name = type->name; 333 group->attrs = type_attrs; 334 intel_vgpu_type_groups[i] = group; 335 } 336 337 return true; 338 339 unwind: 340 for (j = 0; j < i; j++) { 341 group = intel_vgpu_type_groups[j]; 342 kfree(group); 343 } 344 345 return false; 346 } 347 348 static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) 349 { 350 int i; 351 struct attribute_group *group; 352 353 for (i = 0; i < gvt->num_types; i++) { 354 group = intel_vgpu_type_groups[i]; 355 kfree(group); 356 } 357 } 358 359 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) 360 { 361 hash_init(info->ptable); 362 } 363 364 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info) 365 { 366 struct kvmgt_pgfn *p; 367 struct hlist_node *tmp; 368 int i; 369 370 hash_for_each_safe(info->ptable, i, tmp, p, hnode) { 371 hash_del(&p->hnode); 372 kfree(p); 373 } 374 } 375 376 static struct kvmgt_pgfn * 377 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn) 378 { 379 struct kvmgt_pgfn *p, *res = NULL; 380 381 hash_for_each_possible(info->ptable, p, hnode, gfn) { 382 if (gfn == p->gfn) { 383 res = p; 384 break; 385 } 386 } 387 388 return res; 389 } 390 391 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info, 392 gfn_t gfn) 393 { 394 struct kvmgt_pgfn *p; 395 396 p = __kvmgt_protect_table_find(info, gfn); 397 return !!p; 398 } 399 400 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn) 401 { 402 struct kvmgt_pgfn *p; 403 404 if (kvmgt_gfn_is_write_protected(info, gfn)) 405 return; 406 407 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); 408 if (WARN(!p, "gfn: 0x%llx\n", gfn)) 409 return; 410 411 p->gfn = gfn; 412 hash_add(info->ptable, &p->hnode, gfn); 413 } 414 415 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, 416 gfn_t gfn) 417 { 418 struct kvmgt_pgfn *p; 419 420 p = __kvmgt_protect_table_find(info, gfn); 421 if (p) { 422 hash_del(&p->hnode); 423 kfree(p); 424 } 425 } 426 427 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) 428 { 429 struct intel_vgpu *vgpu; 430 struct intel_vgpu_type *type; 431 struct device *pdev; 432 void *gvt; 433 int ret; 434 435 pdev = mdev_parent_dev(mdev); 436 gvt = kdev_to_i915(pdev)->gvt; 437 438 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 439 if (!type) { 440 gvt_err("failed to find type %s to create\n", 441 kobject_name(kobj)); 442 ret = -EINVAL; 443 goto out; 444 } 445 446 vgpu = intel_gvt_ops->vgpu_create(gvt, type); 447 if (IS_ERR_OR_NULL(vgpu)) { 448 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu); 449 gvt_err("failed to create intel vgpu: %d\n", ret); 450 goto out; 451 } 452 453 INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work); 454 455 vgpu->vdev.mdev = mdev; 456 mdev_set_drvdata(mdev, vgpu); 457 458 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", 459 dev_name(mdev_dev(mdev))); 460 ret = 0; 461 462 out: 463 return ret; 464 } 465 466 static int intel_vgpu_remove(struct mdev_device *mdev) 467 { 468 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 469 470 if (handle_valid(vgpu->handle)) 471 return -EBUSY; 472 473 intel_gvt_ops->vgpu_destroy(vgpu); 474 return 0; 475 } 476 477 static int intel_vgpu_iommu_notifier(struct notifier_block *nb, 478 unsigned long action, void *data) 479 { 480 struct intel_vgpu *vgpu = container_of(nb, 481 struct intel_vgpu, 482 vdev.iommu_notifier); 483 484 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 485 struct vfio_iommu_type1_dma_unmap *unmap = data; 486 unsigned long gfn, end_gfn; 487 488 gfn = unmap->iova >> PAGE_SHIFT; 489 end_gfn = gfn + unmap->size / PAGE_SIZE; 490 491 while (gfn < end_gfn) 492 gvt_cache_remove(vgpu, gfn++); 493 } 494 495 return NOTIFY_OK; 496 } 497 498 static int intel_vgpu_group_notifier(struct notifier_block *nb, 499 unsigned long action, void *data) 500 { 501 struct intel_vgpu *vgpu = container_of(nb, 502 struct intel_vgpu, 503 vdev.group_notifier); 504 505 /* the only action we care about */ 506 if (action == VFIO_GROUP_NOTIFY_SET_KVM) { 507 vgpu->vdev.kvm = data; 508 509 if (!data) 510 schedule_work(&vgpu->vdev.release_work); 511 } 512 513 return NOTIFY_OK; 514 } 515 516 static int intel_vgpu_open(struct mdev_device *mdev) 517 { 518 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 519 unsigned long events; 520 int ret; 521 522 vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier; 523 vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier; 524 525 events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; 526 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events, 527 &vgpu->vdev.iommu_notifier); 528 if (ret != 0) { 529 gvt_err("vfio_register_notifier for iommu failed: %d\n", ret); 530 goto out; 531 } 532 533 events = VFIO_GROUP_NOTIFY_SET_KVM; 534 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events, 535 &vgpu->vdev.group_notifier); 536 if (ret != 0) { 537 gvt_err("vfio_register_notifier for group failed: %d\n", ret); 538 goto undo_iommu; 539 } 540 541 ret = kvmgt_guest_init(mdev); 542 if (ret) 543 goto undo_group; 544 545 atomic_set(&vgpu->vdev.released, 0); 546 return ret; 547 548 undo_group: 549 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, 550 &vgpu->vdev.group_notifier); 551 552 undo_iommu: 553 vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, 554 &vgpu->vdev.iommu_notifier); 555 out: 556 return ret; 557 } 558 559 static void __intel_vgpu_release(struct intel_vgpu *vgpu) 560 { 561 struct kvmgt_guest_info *info; 562 int ret; 563 564 if (!handle_valid(vgpu->handle)) 565 return; 566 567 if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1)) 568 return; 569 570 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY, 571 &vgpu->vdev.iommu_notifier); 572 WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret); 573 574 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY, 575 &vgpu->vdev.group_notifier); 576 WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret); 577 578 info = (struct kvmgt_guest_info *)vgpu->handle; 579 kvmgt_guest_exit(info); 580 581 vgpu->vdev.kvm = NULL; 582 vgpu->handle = 0; 583 } 584 585 static void intel_vgpu_release(struct mdev_device *mdev) 586 { 587 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 588 589 __intel_vgpu_release(vgpu); 590 } 591 592 static void intel_vgpu_release_work(struct work_struct *work) 593 { 594 struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu, 595 vdev.release_work); 596 597 __intel_vgpu_release(vgpu); 598 } 599 600 static uint64_t intel_vgpu_get_bar0_addr(struct intel_vgpu *vgpu) 601 { 602 u32 start_lo, start_hi; 603 u32 mem_type; 604 int pos = PCI_BASE_ADDRESS_0; 605 606 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) & 607 PCI_BASE_ADDRESS_MEM_MASK; 608 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) & 609 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 610 611 switch (mem_type) { 612 case PCI_BASE_ADDRESS_MEM_TYPE_64: 613 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space 614 + pos + 4)); 615 break; 616 case PCI_BASE_ADDRESS_MEM_TYPE_32: 617 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 618 /* 1M mem BAR treated as 32-bit BAR */ 619 default: 620 /* mem unknown type treated as 32-bit BAR */ 621 start_hi = 0; 622 break; 623 } 624 625 return ((u64)start_hi << 32) | start_lo; 626 } 627 628 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, 629 size_t count, loff_t *ppos, bool is_write) 630 { 631 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 632 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 633 uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 634 int ret = -EINVAL; 635 636 637 if (index >= VFIO_PCI_NUM_REGIONS) { 638 gvt_err("invalid index: %u\n", index); 639 return -EINVAL; 640 } 641 642 switch (index) { 643 case VFIO_PCI_CONFIG_REGION_INDEX: 644 if (is_write) 645 ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos, 646 buf, count); 647 else 648 ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos, 649 buf, count); 650 break; 651 case VFIO_PCI_BAR0_REGION_INDEX: 652 case VFIO_PCI_BAR1_REGION_INDEX: 653 if (is_write) { 654 uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu); 655 656 ret = intel_gvt_ops->emulate_mmio_write(vgpu, 657 bar0_start + pos, buf, count); 658 } else { 659 uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu); 660 661 ret = intel_gvt_ops->emulate_mmio_read(vgpu, 662 bar0_start + pos, buf, count); 663 } 664 break; 665 case VFIO_PCI_BAR2_REGION_INDEX: 666 case VFIO_PCI_BAR3_REGION_INDEX: 667 case VFIO_PCI_BAR4_REGION_INDEX: 668 case VFIO_PCI_BAR5_REGION_INDEX: 669 case VFIO_PCI_VGA_REGION_INDEX: 670 case VFIO_PCI_ROM_REGION_INDEX: 671 default: 672 gvt_err("unsupported region: %u\n", index); 673 } 674 675 return ret == 0 ? count : ret; 676 } 677 678 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, 679 size_t count, loff_t *ppos) 680 { 681 unsigned int done = 0; 682 int ret; 683 684 while (count) { 685 size_t filled; 686 687 if (count >= 4 && !(*ppos % 4)) { 688 u32 val; 689 690 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 691 ppos, false); 692 if (ret <= 0) 693 goto read_err; 694 695 if (copy_to_user(buf, &val, sizeof(val))) 696 goto read_err; 697 698 filled = 4; 699 } else if (count >= 2 && !(*ppos % 2)) { 700 u16 val; 701 702 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 703 ppos, false); 704 if (ret <= 0) 705 goto read_err; 706 707 if (copy_to_user(buf, &val, sizeof(val))) 708 goto read_err; 709 710 filled = 2; 711 } else { 712 u8 val; 713 714 ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos, 715 false); 716 if (ret <= 0) 717 goto read_err; 718 719 if (copy_to_user(buf, &val, sizeof(val))) 720 goto read_err; 721 722 filled = 1; 723 } 724 725 count -= filled; 726 done += filled; 727 *ppos += filled; 728 buf += filled; 729 } 730 731 return done; 732 733 read_err: 734 return -EFAULT; 735 } 736 737 static ssize_t intel_vgpu_write(struct mdev_device *mdev, 738 const char __user *buf, 739 size_t count, loff_t *ppos) 740 { 741 unsigned int done = 0; 742 int ret; 743 744 while (count) { 745 size_t filled; 746 747 if (count >= 4 && !(*ppos % 4)) { 748 u32 val; 749 750 if (copy_from_user(&val, buf, sizeof(val))) 751 goto write_err; 752 753 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 754 ppos, true); 755 if (ret <= 0) 756 goto write_err; 757 758 filled = 4; 759 } else if (count >= 2 && !(*ppos % 2)) { 760 u16 val; 761 762 if (copy_from_user(&val, buf, sizeof(val))) 763 goto write_err; 764 765 ret = intel_vgpu_rw(mdev, (char *)&val, 766 sizeof(val), ppos, true); 767 if (ret <= 0) 768 goto write_err; 769 770 filled = 2; 771 } else { 772 u8 val; 773 774 if (copy_from_user(&val, buf, sizeof(val))) 775 goto write_err; 776 777 ret = intel_vgpu_rw(mdev, &val, sizeof(val), 778 ppos, true); 779 if (ret <= 0) 780 goto write_err; 781 782 filled = 1; 783 } 784 785 count -= filled; 786 done += filled; 787 *ppos += filled; 788 buf += filled; 789 } 790 791 return done; 792 write_err: 793 return -EFAULT; 794 } 795 796 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) 797 { 798 unsigned int index; 799 u64 virtaddr; 800 unsigned long req_size, pgoff = 0; 801 pgprot_t pg_prot; 802 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 803 804 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 805 if (index >= VFIO_PCI_ROM_REGION_INDEX) 806 return -EINVAL; 807 808 if (vma->vm_end < vma->vm_start) 809 return -EINVAL; 810 if ((vma->vm_flags & VM_SHARED) == 0) 811 return -EINVAL; 812 if (index != VFIO_PCI_BAR2_REGION_INDEX) 813 return -EINVAL; 814 815 pg_prot = vma->vm_page_prot; 816 virtaddr = vma->vm_start; 817 req_size = vma->vm_end - vma->vm_start; 818 pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; 819 820 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); 821 } 822 823 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) 824 { 825 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) 826 return 1; 827 828 return 0; 829 } 830 831 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, 832 unsigned int index, unsigned int start, 833 unsigned int count, uint32_t flags, 834 void *data) 835 { 836 return 0; 837 } 838 839 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, 840 unsigned int index, unsigned int start, 841 unsigned int count, uint32_t flags, void *data) 842 { 843 return 0; 844 } 845 846 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, 847 unsigned int index, unsigned int start, unsigned int count, 848 uint32_t flags, void *data) 849 { 850 return 0; 851 } 852 853 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, 854 unsigned int index, unsigned int start, unsigned int count, 855 uint32_t flags, void *data) 856 { 857 struct eventfd_ctx *trigger; 858 859 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 860 int fd = *(int *)data; 861 862 trigger = eventfd_ctx_fdget(fd); 863 if (IS_ERR(trigger)) { 864 gvt_err("eventfd_ctx_fdget failed\n"); 865 return PTR_ERR(trigger); 866 } 867 vgpu->vdev.msi_trigger = trigger; 868 } 869 870 return 0; 871 } 872 873 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags, 874 unsigned int index, unsigned int start, unsigned int count, 875 void *data) 876 { 877 int (*func)(struct intel_vgpu *vgpu, unsigned int index, 878 unsigned int start, unsigned int count, uint32_t flags, 879 void *data) = NULL; 880 881 switch (index) { 882 case VFIO_PCI_INTX_IRQ_INDEX: 883 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 884 case VFIO_IRQ_SET_ACTION_MASK: 885 func = intel_vgpu_set_intx_mask; 886 break; 887 case VFIO_IRQ_SET_ACTION_UNMASK: 888 func = intel_vgpu_set_intx_unmask; 889 break; 890 case VFIO_IRQ_SET_ACTION_TRIGGER: 891 func = intel_vgpu_set_intx_trigger; 892 break; 893 } 894 break; 895 case VFIO_PCI_MSI_IRQ_INDEX: 896 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 897 case VFIO_IRQ_SET_ACTION_MASK: 898 case VFIO_IRQ_SET_ACTION_UNMASK: 899 /* XXX Need masking support exported */ 900 break; 901 case VFIO_IRQ_SET_ACTION_TRIGGER: 902 func = intel_vgpu_set_msi_trigger; 903 break; 904 } 905 break; 906 } 907 908 if (!func) 909 return -ENOTTY; 910 911 return func(vgpu, index, start, count, flags, data); 912 } 913 914 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, 915 unsigned long arg) 916 { 917 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 918 unsigned long minsz; 919 920 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); 921 922 if (cmd == VFIO_DEVICE_GET_INFO) { 923 struct vfio_device_info info; 924 925 minsz = offsetofend(struct vfio_device_info, num_irqs); 926 927 if (copy_from_user(&info, (void __user *)arg, minsz)) 928 return -EFAULT; 929 930 if (info.argsz < minsz) 931 return -EINVAL; 932 933 info.flags = VFIO_DEVICE_FLAGS_PCI; 934 info.flags |= VFIO_DEVICE_FLAGS_RESET; 935 info.num_regions = VFIO_PCI_NUM_REGIONS; 936 info.num_irqs = VFIO_PCI_NUM_IRQS; 937 938 return copy_to_user((void __user *)arg, &info, minsz) ? 939 -EFAULT : 0; 940 941 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 942 struct vfio_region_info info; 943 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 944 int i, ret; 945 struct vfio_region_info_cap_sparse_mmap *sparse = NULL; 946 size_t size; 947 int nr_areas = 1; 948 int cap_type_id; 949 950 minsz = offsetofend(struct vfio_region_info, offset); 951 952 if (copy_from_user(&info, (void __user *)arg, minsz)) 953 return -EFAULT; 954 955 if (info.argsz < minsz) 956 return -EINVAL; 957 958 switch (info.index) { 959 case VFIO_PCI_CONFIG_REGION_INDEX: 960 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 961 info.size = INTEL_GVT_MAX_CFG_SPACE_SZ; 962 info.flags = VFIO_REGION_INFO_FLAG_READ | 963 VFIO_REGION_INFO_FLAG_WRITE; 964 break; 965 case VFIO_PCI_BAR0_REGION_INDEX: 966 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 967 info.size = vgpu->cfg_space.bar[info.index].size; 968 if (!info.size) { 969 info.flags = 0; 970 break; 971 } 972 973 info.flags = VFIO_REGION_INFO_FLAG_READ | 974 VFIO_REGION_INFO_FLAG_WRITE; 975 break; 976 case VFIO_PCI_BAR1_REGION_INDEX: 977 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 978 info.size = 0; 979 info.flags = 0; 980 break; 981 case VFIO_PCI_BAR2_REGION_INDEX: 982 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 983 info.flags = VFIO_REGION_INFO_FLAG_CAPS | 984 VFIO_REGION_INFO_FLAG_MMAP | 985 VFIO_REGION_INFO_FLAG_READ | 986 VFIO_REGION_INFO_FLAG_WRITE; 987 info.size = gvt_aperture_sz(vgpu->gvt); 988 989 size = sizeof(*sparse) + 990 (nr_areas * sizeof(*sparse->areas)); 991 sparse = kzalloc(size, GFP_KERNEL); 992 if (!sparse) 993 return -ENOMEM; 994 995 sparse->nr_areas = nr_areas; 996 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 997 sparse->areas[0].offset = 998 PAGE_ALIGN(vgpu_aperture_offset(vgpu)); 999 sparse->areas[0].size = vgpu_aperture_sz(vgpu); 1000 break; 1001 1002 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1003 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1004 info.size = 0; 1005 1006 info.flags = 0; 1007 gvt_dbg_core("get region info bar:%d\n", info.index); 1008 break; 1009 1010 case VFIO_PCI_ROM_REGION_INDEX: 1011 case VFIO_PCI_VGA_REGION_INDEX: 1012 gvt_dbg_core("get region info index:%d\n", info.index); 1013 break; 1014 default: 1015 { 1016 struct vfio_region_info_cap_type cap_type; 1017 1018 if (info.index >= VFIO_PCI_NUM_REGIONS + 1019 vgpu->vdev.num_regions) 1020 return -EINVAL; 1021 1022 i = info.index - VFIO_PCI_NUM_REGIONS; 1023 1024 info.offset = 1025 VFIO_PCI_INDEX_TO_OFFSET(info.index); 1026 info.size = vgpu->vdev.region[i].size; 1027 info.flags = vgpu->vdev.region[i].flags; 1028 1029 cap_type.type = vgpu->vdev.region[i].type; 1030 cap_type.subtype = vgpu->vdev.region[i].subtype; 1031 1032 ret = vfio_info_add_capability(&caps, 1033 VFIO_REGION_INFO_CAP_TYPE, 1034 &cap_type); 1035 if (ret) 1036 return ret; 1037 } 1038 } 1039 1040 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { 1041 switch (cap_type_id) { 1042 case VFIO_REGION_INFO_CAP_SPARSE_MMAP: 1043 ret = vfio_info_add_capability(&caps, 1044 VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1045 sparse); 1046 kfree(sparse); 1047 if (ret) 1048 return ret; 1049 break; 1050 default: 1051 return -EINVAL; 1052 } 1053 } 1054 1055 if (caps.size) { 1056 if (info.argsz < sizeof(info) + caps.size) { 1057 info.argsz = sizeof(info) + caps.size; 1058 info.cap_offset = 0; 1059 } else { 1060 vfio_info_cap_shift(&caps, sizeof(info)); 1061 if (copy_to_user((void __user *)arg + 1062 sizeof(info), caps.buf, 1063 caps.size)) { 1064 kfree(caps.buf); 1065 return -EFAULT; 1066 } 1067 info.cap_offset = sizeof(info); 1068 } 1069 1070 kfree(caps.buf); 1071 } 1072 1073 return copy_to_user((void __user *)arg, &info, minsz) ? 1074 -EFAULT : 0; 1075 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 1076 struct vfio_irq_info info; 1077 1078 minsz = offsetofend(struct vfio_irq_info, count); 1079 1080 if (copy_from_user(&info, (void __user *)arg, minsz)) 1081 return -EFAULT; 1082 1083 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1084 return -EINVAL; 1085 1086 switch (info.index) { 1087 case VFIO_PCI_INTX_IRQ_INDEX: 1088 case VFIO_PCI_MSI_IRQ_INDEX: 1089 break; 1090 default: 1091 return -EINVAL; 1092 } 1093 1094 info.flags = VFIO_IRQ_INFO_EVENTFD; 1095 1096 info.count = intel_vgpu_get_irq_count(vgpu, info.index); 1097 1098 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1099 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1100 VFIO_IRQ_INFO_AUTOMASKED); 1101 else 1102 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1103 1104 return copy_to_user((void __user *)arg, &info, minsz) ? 1105 -EFAULT : 0; 1106 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1107 struct vfio_irq_set hdr; 1108 u8 *data = NULL; 1109 int ret = 0; 1110 size_t data_size = 0; 1111 1112 minsz = offsetofend(struct vfio_irq_set, count); 1113 1114 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1115 return -EFAULT; 1116 1117 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 1118 int max = intel_vgpu_get_irq_count(vgpu, hdr.index); 1119 1120 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1121 VFIO_PCI_NUM_IRQS, &data_size); 1122 if (ret) { 1123 gvt_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); 1124 return -EINVAL; 1125 } 1126 if (data_size) { 1127 data = memdup_user((void __user *)(arg + minsz), 1128 data_size); 1129 if (IS_ERR(data)) 1130 return PTR_ERR(data); 1131 } 1132 } 1133 1134 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, 1135 hdr.start, hdr.count, data); 1136 kfree(data); 1137 1138 return ret; 1139 } else if (cmd == VFIO_DEVICE_RESET) { 1140 intel_gvt_ops->vgpu_reset(vgpu); 1141 return 0; 1142 } 1143 1144 return 0; 1145 } 1146 1147 static const struct mdev_parent_ops intel_vgpu_ops = { 1148 .supported_type_groups = intel_vgpu_type_groups, 1149 .create = intel_vgpu_create, 1150 .remove = intel_vgpu_remove, 1151 1152 .open = intel_vgpu_open, 1153 .release = intel_vgpu_release, 1154 1155 .read = intel_vgpu_read, 1156 .write = intel_vgpu_write, 1157 .mmap = intel_vgpu_mmap, 1158 .ioctl = intel_vgpu_ioctl, 1159 }; 1160 1161 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) 1162 { 1163 if (!intel_gvt_init_vgpu_type_groups(gvt)) 1164 return -EFAULT; 1165 1166 intel_gvt_ops = ops; 1167 1168 return mdev_register_device(dev, &intel_vgpu_ops); 1169 } 1170 1171 static void kvmgt_host_exit(struct device *dev, void *gvt) 1172 { 1173 intel_gvt_cleanup_vgpu_type_groups(gvt); 1174 mdev_unregister_device(dev); 1175 } 1176 1177 static int kvmgt_write_protect_add(unsigned long handle, u64 gfn) 1178 { 1179 struct kvmgt_guest_info *info; 1180 struct kvm *kvm; 1181 struct kvm_memory_slot *slot; 1182 int idx; 1183 1184 if (!handle_valid(handle)) 1185 return -ESRCH; 1186 1187 info = (struct kvmgt_guest_info *)handle; 1188 kvm = info->kvm; 1189 1190 idx = srcu_read_lock(&kvm->srcu); 1191 slot = gfn_to_memslot(kvm, gfn); 1192 if (!slot) { 1193 srcu_read_unlock(&kvm->srcu, idx); 1194 return -EINVAL; 1195 } 1196 1197 spin_lock(&kvm->mmu_lock); 1198 1199 if (kvmgt_gfn_is_write_protected(info, gfn)) 1200 goto out; 1201 1202 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1203 kvmgt_protect_table_add(info, gfn); 1204 1205 out: 1206 spin_unlock(&kvm->mmu_lock); 1207 srcu_read_unlock(&kvm->srcu, idx); 1208 return 0; 1209 } 1210 1211 static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn) 1212 { 1213 struct kvmgt_guest_info *info; 1214 struct kvm *kvm; 1215 struct kvm_memory_slot *slot; 1216 int idx; 1217 1218 if (!handle_valid(handle)) 1219 return 0; 1220 1221 info = (struct kvmgt_guest_info *)handle; 1222 kvm = info->kvm; 1223 1224 idx = srcu_read_lock(&kvm->srcu); 1225 slot = gfn_to_memslot(kvm, gfn); 1226 if (!slot) { 1227 srcu_read_unlock(&kvm->srcu, idx); 1228 return -EINVAL; 1229 } 1230 1231 spin_lock(&kvm->mmu_lock); 1232 1233 if (!kvmgt_gfn_is_write_protected(info, gfn)) 1234 goto out; 1235 1236 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1237 kvmgt_protect_table_del(info, gfn); 1238 1239 out: 1240 spin_unlock(&kvm->mmu_lock); 1241 srcu_read_unlock(&kvm->srcu, idx); 1242 return 0; 1243 } 1244 1245 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 1246 const u8 *val, int len, 1247 struct kvm_page_track_notifier_node *node) 1248 { 1249 struct kvmgt_guest_info *info = container_of(node, 1250 struct kvmgt_guest_info, track_node); 1251 1252 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) 1253 intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa, 1254 (void *)val, len); 1255 } 1256 1257 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 1258 struct kvm_memory_slot *slot, 1259 struct kvm_page_track_notifier_node *node) 1260 { 1261 int i; 1262 gfn_t gfn; 1263 struct kvmgt_guest_info *info = container_of(node, 1264 struct kvmgt_guest_info, track_node); 1265 1266 spin_lock(&kvm->mmu_lock); 1267 for (i = 0; i < slot->npages; i++) { 1268 gfn = slot->base_gfn + i; 1269 if (kvmgt_gfn_is_write_protected(info, gfn)) { 1270 kvm_slot_page_track_remove_page(kvm, slot, gfn, 1271 KVM_PAGE_TRACK_WRITE); 1272 kvmgt_protect_table_del(info, gfn); 1273 } 1274 } 1275 spin_unlock(&kvm->mmu_lock); 1276 } 1277 1278 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm) 1279 { 1280 struct intel_vgpu *itr; 1281 struct kvmgt_guest_info *info; 1282 int id; 1283 bool ret = false; 1284 1285 mutex_lock(&vgpu->gvt->lock); 1286 for_each_active_vgpu(vgpu->gvt, itr, id) { 1287 if (!handle_valid(itr->handle)) 1288 continue; 1289 1290 info = (struct kvmgt_guest_info *)itr->handle; 1291 if (kvm && kvm == info->kvm) { 1292 ret = true; 1293 goto out; 1294 } 1295 } 1296 out: 1297 mutex_unlock(&vgpu->gvt->lock); 1298 return ret; 1299 } 1300 1301 static int kvmgt_guest_init(struct mdev_device *mdev) 1302 { 1303 struct kvmgt_guest_info *info; 1304 struct intel_vgpu *vgpu; 1305 struct kvm *kvm; 1306 1307 vgpu = mdev_get_drvdata(mdev); 1308 if (handle_valid(vgpu->handle)) 1309 return -EEXIST; 1310 1311 kvm = vgpu->vdev.kvm; 1312 if (!kvm || kvm->mm != current->mm) { 1313 gvt_err("KVM is required to use Intel vGPU\n"); 1314 return -ESRCH; 1315 } 1316 1317 if (__kvmgt_vgpu_exist(vgpu, kvm)) 1318 return -EEXIST; 1319 1320 info = vzalloc(sizeof(struct kvmgt_guest_info)); 1321 if (!info) 1322 return -ENOMEM; 1323 1324 vgpu->handle = (unsigned long)info; 1325 info->vgpu = vgpu; 1326 info->kvm = kvm; 1327 1328 kvmgt_protect_table_init(info); 1329 gvt_cache_init(vgpu); 1330 1331 info->track_node.track_write = kvmgt_page_track_write; 1332 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 1333 kvm_page_track_register_notifier(kvm, &info->track_node); 1334 1335 return 0; 1336 } 1337 1338 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info) 1339 { 1340 if (!info) { 1341 gvt_err("kvmgt_guest_info invalid\n"); 1342 return false; 1343 } 1344 1345 kvm_page_track_unregister_notifier(info->kvm, &info->track_node); 1346 kvmgt_protect_table_destroy(info); 1347 gvt_cache_destroy(info->vgpu); 1348 vfree(info); 1349 1350 return true; 1351 } 1352 1353 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) 1354 { 1355 /* nothing to do here */ 1356 return 0; 1357 } 1358 1359 static void kvmgt_detach_vgpu(unsigned long handle) 1360 { 1361 /* nothing to do here */ 1362 } 1363 1364 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) 1365 { 1366 struct kvmgt_guest_info *info; 1367 struct intel_vgpu *vgpu; 1368 1369 if (!handle_valid(handle)) 1370 return -ESRCH; 1371 1372 info = (struct kvmgt_guest_info *)handle; 1373 vgpu = info->vgpu; 1374 1375 if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1) 1376 return 0; 1377 1378 return -EFAULT; 1379 } 1380 1381 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) 1382 { 1383 unsigned long iova, pfn; 1384 struct kvmgt_guest_info *info; 1385 struct device *dev; 1386 int rc; 1387 1388 if (!handle_valid(handle)) 1389 return INTEL_GVT_INVALID_ADDR; 1390 1391 info = (struct kvmgt_guest_info *)handle; 1392 iova = gvt_cache_find(info->vgpu, gfn); 1393 if (iova != INTEL_GVT_INVALID_ADDR) 1394 return iova; 1395 1396 pfn = INTEL_GVT_INVALID_ADDR; 1397 dev = mdev_dev(info->vgpu->vdev.mdev); 1398 rc = vfio_pin_pages(dev, &gfn, 1, IOMMU_READ | IOMMU_WRITE, &pfn); 1399 if (rc != 1) { 1400 gvt_err("vfio_pin_pages failed for gfn 0x%lx: %d\n", gfn, rc); 1401 return INTEL_GVT_INVALID_ADDR; 1402 } 1403 /* transfer to host iova for GFX to use DMA */ 1404 rc = gvt_dma_map_iova(info->vgpu, pfn, &iova); 1405 if (rc) { 1406 gvt_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn); 1407 vfio_unpin_pages(dev, &gfn, 1); 1408 return INTEL_GVT_INVALID_ADDR; 1409 } 1410 1411 gvt_cache_add(info->vgpu, gfn, iova); 1412 return iova; 1413 } 1414 1415 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, 1416 void *buf, unsigned long len, bool write) 1417 { 1418 struct kvmgt_guest_info *info; 1419 struct kvm *kvm; 1420 int ret; 1421 bool kthread = current->mm == NULL; 1422 1423 if (!handle_valid(handle)) 1424 return -ESRCH; 1425 1426 info = (struct kvmgt_guest_info *)handle; 1427 kvm = info->kvm; 1428 1429 if (kthread) 1430 use_mm(kvm->mm); 1431 1432 ret = write ? kvm_write_guest(kvm, gpa, buf, len) : 1433 kvm_read_guest(kvm, gpa, buf, len); 1434 1435 if (kthread) 1436 unuse_mm(kvm->mm); 1437 1438 return ret; 1439 } 1440 1441 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa, 1442 void *buf, unsigned long len) 1443 { 1444 return kvmgt_rw_gpa(handle, gpa, buf, len, false); 1445 } 1446 1447 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa, 1448 void *buf, unsigned long len) 1449 { 1450 return kvmgt_rw_gpa(handle, gpa, buf, len, true); 1451 } 1452 1453 static unsigned long kvmgt_virt_to_pfn(void *addr) 1454 { 1455 return PFN_DOWN(__pa(addr)); 1456 } 1457 1458 struct intel_gvt_mpt kvmgt_mpt = { 1459 .host_init = kvmgt_host_init, 1460 .host_exit = kvmgt_host_exit, 1461 .attach_vgpu = kvmgt_attach_vgpu, 1462 .detach_vgpu = kvmgt_detach_vgpu, 1463 .inject_msi = kvmgt_inject_msi, 1464 .from_virt_to_mfn = kvmgt_virt_to_pfn, 1465 .set_wp_page = kvmgt_write_protect_add, 1466 .unset_wp_page = kvmgt_write_protect_remove, 1467 .read_gpa = kvmgt_read_gpa, 1468 .write_gpa = kvmgt_write_gpa, 1469 .gfn_to_mfn = kvmgt_gfn_to_pfn, 1470 }; 1471 EXPORT_SYMBOL_GPL(kvmgt_mpt); 1472 1473 static int __init kvmgt_init(void) 1474 { 1475 return 0; 1476 } 1477 1478 static void __exit kvmgt_exit(void) 1479 { 1480 } 1481 1482 module_init(kvmgt_init); 1483 module_exit(kvmgt_exit); 1484 1485 MODULE_LICENSE("GPL and additional rights"); 1486 MODULE_AUTHOR("Intel Corporation"); 1487