1 /* 2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM 3 * 4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Kevin Tian <kevin.tian@intel.com> 27 * Jike Song <jike.song@intel.com> 28 * Xiaoguang Chen <xiaoguang.chen@intel.com> 29 */ 30 31 #include <linux/init.h> 32 #include <linux/device.h> 33 #include <linux/mm.h> 34 #include <linux/mmu_context.h> 35 #include <linux/types.h> 36 #include <linux/list.h> 37 #include <linux/rbtree.h> 38 #include <linux/spinlock.h> 39 #include <linux/eventfd.h> 40 #include <linux/uuid.h> 41 #include <linux/kvm_host.h> 42 #include <linux/vfio.h> 43 #include <linux/mdev.h> 44 45 #include "i915_drv.h" 46 #include "gvt.h" 47 48 static const struct intel_gvt_ops *intel_gvt_ops; 49 50 /* helper macros copied from vfio-pci */ 51 #define VFIO_PCI_OFFSET_SHIFT 40 52 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) 53 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 54 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 55 56 struct vfio_region { 57 u32 type; 58 u32 subtype; 59 size_t size; 60 u32 flags; 61 }; 62 63 struct kvmgt_pgfn { 64 gfn_t gfn; 65 struct hlist_node hnode; 66 }; 67 68 struct kvmgt_guest_info { 69 struct kvm *kvm; 70 struct intel_vgpu *vgpu; 71 struct kvm_page_track_notifier_node track_node; 72 #define NR_BKT (1 << 18) 73 struct hlist_head ptable[NR_BKT]; 74 #undef NR_BKT 75 }; 76 77 struct gvt_dma { 78 struct rb_node node; 79 gfn_t gfn; 80 unsigned long iova; 81 }; 82 83 static inline bool handle_valid(unsigned long handle) 84 { 85 return !!(handle & ~0xff); 86 } 87 88 static int kvmgt_guest_init(struct mdev_device *mdev); 89 static void intel_vgpu_release_work(struct work_struct *work); 90 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); 91 92 static int gvt_dma_map_iova(struct intel_vgpu *vgpu, kvm_pfn_t pfn, 93 unsigned long *iova) 94 { 95 struct page *page; 96 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 97 dma_addr_t daddr; 98 99 if (unlikely(!pfn_valid(pfn))) 100 return -EFAULT; 101 102 page = pfn_to_page(pfn); 103 daddr = dma_map_page(dev, page, 0, PAGE_SIZE, 104 PCI_DMA_BIDIRECTIONAL); 105 if (dma_mapping_error(dev, daddr)) 106 return -ENOMEM; 107 108 *iova = (unsigned long)(daddr >> PAGE_SHIFT); 109 return 0; 110 } 111 112 static void gvt_dma_unmap_iova(struct intel_vgpu *vgpu, unsigned long iova) 113 { 114 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 115 dma_addr_t daddr; 116 117 daddr = (dma_addr_t)(iova << PAGE_SHIFT); 118 dma_unmap_page(dev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 119 } 120 121 static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 122 { 123 struct rb_node *node = vgpu->vdev.cache.rb_node; 124 struct gvt_dma *ret = NULL; 125 126 while (node) { 127 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node); 128 129 if (gfn < itr->gfn) 130 node = node->rb_left; 131 else if (gfn > itr->gfn) 132 node = node->rb_right; 133 else { 134 ret = itr; 135 goto out; 136 } 137 } 138 139 out: 140 return ret; 141 } 142 143 static unsigned long gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 144 { 145 struct gvt_dma *entry; 146 unsigned long iova; 147 148 mutex_lock(&vgpu->vdev.cache_lock); 149 150 entry = __gvt_cache_find(vgpu, gfn); 151 iova = (entry == NULL) ? INTEL_GVT_INVALID_ADDR : entry->iova; 152 153 mutex_unlock(&vgpu->vdev.cache_lock); 154 return iova; 155 } 156 157 static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 158 unsigned long iova) 159 { 160 struct gvt_dma *new, *itr; 161 struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL; 162 163 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 164 if (!new) 165 return; 166 167 new->gfn = gfn; 168 new->iova = iova; 169 170 mutex_lock(&vgpu->vdev.cache_lock); 171 while (*link) { 172 parent = *link; 173 itr = rb_entry(parent, struct gvt_dma, node); 174 175 if (gfn == itr->gfn) 176 goto out; 177 else if (gfn < itr->gfn) 178 link = &parent->rb_left; 179 else 180 link = &parent->rb_right; 181 } 182 183 rb_link_node(&new->node, parent, link); 184 rb_insert_color(&new->node, &vgpu->vdev.cache); 185 mutex_unlock(&vgpu->vdev.cache_lock); 186 return; 187 188 out: 189 mutex_unlock(&vgpu->vdev.cache_lock); 190 kfree(new); 191 } 192 193 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 194 struct gvt_dma *entry) 195 { 196 rb_erase(&entry->node, &vgpu->vdev.cache); 197 kfree(entry); 198 } 199 200 static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn) 201 { 202 struct device *dev = mdev_dev(vgpu->vdev.mdev); 203 struct gvt_dma *this; 204 unsigned long g1; 205 int rc; 206 207 mutex_lock(&vgpu->vdev.cache_lock); 208 this = __gvt_cache_find(vgpu, gfn); 209 if (!this) { 210 mutex_unlock(&vgpu->vdev.cache_lock); 211 return; 212 } 213 214 g1 = gfn; 215 gvt_dma_unmap_iova(vgpu, this->iova); 216 rc = vfio_unpin_pages(dev, &g1, 1); 217 WARN_ON(rc != 1); 218 __gvt_cache_remove_entry(vgpu, this); 219 mutex_unlock(&vgpu->vdev.cache_lock); 220 } 221 222 static void gvt_cache_init(struct intel_vgpu *vgpu) 223 { 224 vgpu->vdev.cache = RB_ROOT; 225 mutex_init(&vgpu->vdev.cache_lock); 226 } 227 228 static void gvt_cache_destroy(struct intel_vgpu *vgpu) 229 { 230 struct gvt_dma *dma; 231 struct rb_node *node = NULL; 232 struct device *dev = mdev_dev(vgpu->vdev.mdev); 233 unsigned long gfn; 234 235 mutex_lock(&vgpu->vdev.cache_lock); 236 while ((node = rb_first(&vgpu->vdev.cache))) { 237 dma = rb_entry(node, struct gvt_dma, node); 238 gvt_dma_unmap_iova(vgpu, dma->iova); 239 gfn = dma->gfn; 240 241 vfio_unpin_pages(dev, &gfn, 1); 242 __gvt_cache_remove_entry(vgpu, dma); 243 } 244 mutex_unlock(&vgpu->vdev.cache_lock); 245 } 246 247 static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt, 248 const char *name) 249 { 250 int i; 251 struct intel_vgpu_type *t; 252 const char *driver_name = dev_driver_string( 253 &gvt->dev_priv->drm.pdev->dev); 254 255 for (i = 0; i < gvt->num_types; i++) { 256 t = &gvt->types[i]; 257 if (!strncmp(t->name, name + strlen(driver_name) + 1, 258 sizeof(t->name))) 259 return t; 260 } 261 262 return NULL; 263 } 264 265 static ssize_t available_instances_show(struct kobject *kobj, 266 struct device *dev, char *buf) 267 { 268 struct intel_vgpu_type *type; 269 unsigned int num = 0; 270 void *gvt = kdev_to_i915(dev)->gvt; 271 272 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 273 if (!type) 274 num = 0; 275 else 276 num = type->avail_instance; 277 278 return sprintf(buf, "%u\n", num); 279 } 280 281 static ssize_t device_api_show(struct kobject *kobj, struct device *dev, 282 char *buf) 283 { 284 return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); 285 } 286 287 static ssize_t description_show(struct kobject *kobj, struct device *dev, 288 char *buf) 289 { 290 struct intel_vgpu_type *type; 291 void *gvt = kdev_to_i915(dev)->gvt; 292 293 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 294 if (!type) 295 return 0; 296 297 return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" 298 "fence: %d\nresolution: %s\n" 299 "weight: %d\n", 300 BYTES_TO_MB(type->low_gm_size), 301 BYTES_TO_MB(type->high_gm_size), 302 type->fence, vgpu_edid_str(type->resolution), 303 type->weight); 304 } 305 306 static MDEV_TYPE_ATTR_RO(available_instances); 307 static MDEV_TYPE_ATTR_RO(device_api); 308 static MDEV_TYPE_ATTR_RO(description); 309 310 static struct attribute *type_attrs[] = { 311 &mdev_type_attr_available_instances.attr, 312 &mdev_type_attr_device_api.attr, 313 &mdev_type_attr_description.attr, 314 NULL, 315 }; 316 317 static struct attribute_group *intel_vgpu_type_groups[] = { 318 [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, 319 }; 320 321 static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) 322 { 323 int i, j; 324 struct intel_vgpu_type *type; 325 struct attribute_group *group; 326 327 for (i = 0; i < gvt->num_types; i++) { 328 type = &gvt->types[i]; 329 330 group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); 331 if (WARN_ON(!group)) 332 goto unwind; 333 334 group->name = type->name; 335 group->attrs = type_attrs; 336 intel_vgpu_type_groups[i] = group; 337 } 338 339 return true; 340 341 unwind: 342 for (j = 0; j < i; j++) { 343 group = intel_vgpu_type_groups[j]; 344 kfree(group); 345 } 346 347 return false; 348 } 349 350 static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) 351 { 352 int i; 353 struct attribute_group *group; 354 355 for (i = 0; i < gvt->num_types; i++) { 356 group = intel_vgpu_type_groups[i]; 357 kfree(group); 358 } 359 } 360 361 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) 362 { 363 hash_init(info->ptable); 364 } 365 366 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info) 367 { 368 struct kvmgt_pgfn *p; 369 struct hlist_node *tmp; 370 int i; 371 372 hash_for_each_safe(info->ptable, i, tmp, p, hnode) { 373 hash_del(&p->hnode); 374 kfree(p); 375 } 376 } 377 378 static struct kvmgt_pgfn * 379 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn) 380 { 381 struct kvmgt_pgfn *p, *res = NULL; 382 383 hash_for_each_possible(info->ptable, p, hnode, gfn) { 384 if (gfn == p->gfn) { 385 res = p; 386 break; 387 } 388 } 389 390 return res; 391 } 392 393 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info, 394 gfn_t gfn) 395 { 396 struct kvmgt_pgfn *p; 397 398 p = __kvmgt_protect_table_find(info, gfn); 399 return !!p; 400 } 401 402 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn) 403 { 404 struct kvmgt_pgfn *p; 405 406 if (kvmgt_gfn_is_write_protected(info, gfn)) 407 return; 408 409 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); 410 if (WARN(!p, "gfn: 0x%llx\n", gfn)) 411 return; 412 413 p->gfn = gfn; 414 hash_add(info->ptable, &p->hnode, gfn); 415 } 416 417 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, 418 gfn_t gfn) 419 { 420 struct kvmgt_pgfn *p; 421 422 p = __kvmgt_protect_table_find(info, gfn); 423 if (p) { 424 hash_del(&p->hnode); 425 kfree(p); 426 } 427 } 428 429 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) 430 { 431 struct intel_vgpu *vgpu = NULL; 432 struct intel_vgpu_type *type; 433 struct device *pdev; 434 void *gvt; 435 int ret; 436 437 pdev = mdev_parent_dev(mdev); 438 gvt = kdev_to_i915(pdev)->gvt; 439 440 type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj)); 441 if (!type) { 442 gvt_vgpu_err("failed to find type %s to create\n", 443 kobject_name(kobj)); 444 ret = -EINVAL; 445 goto out; 446 } 447 448 vgpu = intel_gvt_ops->vgpu_create(gvt, type); 449 if (IS_ERR_OR_NULL(vgpu)) { 450 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu); 451 gvt_vgpu_err("failed to create intel vgpu: %d\n", ret); 452 goto out; 453 } 454 455 INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work); 456 457 vgpu->vdev.mdev = mdev; 458 mdev_set_drvdata(mdev, vgpu); 459 460 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", 461 dev_name(mdev_dev(mdev))); 462 ret = 0; 463 464 out: 465 return ret; 466 } 467 468 static int intel_vgpu_remove(struct mdev_device *mdev) 469 { 470 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 471 472 if (handle_valid(vgpu->handle)) 473 return -EBUSY; 474 475 intel_gvt_ops->vgpu_destroy(vgpu); 476 return 0; 477 } 478 479 static int intel_vgpu_iommu_notifier(struct notifier_block *nb, 480 unsigned long action, void *data) 481 { 482 struct intel_vgpu *vgpu = container_of(nb, 483 struct intel_vgpu, 484 vdev.iommu_notifier); 485 486 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 487 struct vfio_iommu_type1_dma_unmap *unmap = data; 488 unsigned long gfn, end_gfn; 489 490 gfn = unmap->iova >> PAGE_SHIFT; 491 end_gfn = gfn + unmap->size / PAGE_SIZE; 492 493 while (gfn < end_gfn) 494 gvt_cache_remove(vgpu, gfn++); 495 } 496 497 return NOTIFY_OK; 498 } 499 500 static int intel_vgpu_group_notifier(struct notifier_block *nb, 501 unsigned long action, void *data) 502 { 503 struct intel_vgpu *vgpu = container_of(nb, 504 struct intel_vgpu, 505 vdev.group_notifier); 506 507 /* the only action we care about */ 508 if (action == VFIO_GROUP_NOTIFY_SET_KVM) { 509 vgpu->vdev.kvm = data; 510 511 if (!data) 512 schedule_work(&vgpu->vdev.release_work); 513 } 514 515 return NOTIFY_OK; 516 } 517 518 static int intel_vgpu_open(struct mdev_device *mdev) 519 { 520 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 521 unsigned long events; 522 int ret; 523 524 vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier; 525 vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier; 526 527 events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; 528 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events, 529 &vgpu->vdev.iommu_notifier); 530 if (ret != 0) { 531 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n", 532 ret); 533 goto out; 534 } 535 536 events = VFIO_GROUP_NOTIFY_SET_KVM; 537 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events, 538 &vgpu->vdev.group_notifier); 539 if (ret != 0) { 540 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n", 541 ret); 542 goto undo_iommu; 543 } 544 545 ret = kvmgt_guest_init(mdev); 546 if (ret) 547 goto undo_group; 548 549 intel_gvt_ops->vgpu_activate(vgpu); 550 551 atomic_set(&vgpu->vdev.released, 0); 552 return ret; 553 554 undo_group: 555 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, 556 &vgpu->vdev.group_notifier); 557 558 undo_iommu: 559 vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, 560 &vgpu->vdev.iommu_notifier); 561 out: 562 return ret; 563 } 564 565 static void __intel_vgpu_release(struct intel_vgpu *vgpu) 566 { 567 struct kvmgt_guest_info *info; 568 int ret; 569 570 if (!handle_valid(vgpu->handle)) 571 return; 572 573 if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1)) 574 return; 575 576 intel_gvt_ops->vgpu_deactivate(vgpu); 577 578 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY, 579 &vgpu->vdev.iommu_notifier); 580 WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret); 581 582 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY, 583 &vgpu->vdev.group_notifier); 584 WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret); 585 586 info = (struct kvmgt_guest_info *)vgpu->handle; 587 kvmgt_guest_exit(info); 588 589 vgpu->vdev.kvm = NULL; 590 vgpu->handle = 0; 591 } 592 593 static void intel_vgpu_release(struct mdev_device *mdev) 594 { 595 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 596 597 __intel_vgpu_release(vgpu); 598 } 599 600 static void intel_vgpu_release_work(struct work_struct *work) 601 { 602 struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu, 603 vdev.release_work); 604 605 __intel_vgpu_release(vgpu); 606 } 607 608 static uint64_t intel_vgpu_get_bar0_addr(struct intel_vgpu *vgpu) 609 { 610 u32 start_lo, start_hi; 611 u32 mem_type; 612 int pos = PCI_BASE_ADDRESS_0; 613 614 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) & 615 PCI_BASE_ADDRESS_MEM_MASK; 616 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + pos)) & 617 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 618 619 switch (mem_type) { 620 case PCI_BASE_ADDRESS_MEM_TYPE_64: 621 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space 622 + pos + 4)); 623 break; 624 case PCI_BASE_ADDRESS_MEM_TYPE_32: 625 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 626 /* 1M mem BAR treated as 32-bit BAR */ 627 default: 628 /* mem unknown type treated as 32-bit BAR */ 629 start_hi = 0; 630 break; 631 } 632 633 return ((u64)start_hi << 32) | start_lo; 634 } 635 636 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, 637 size_t count, loff_t *ppos, bool is_write) 638 { 639 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 640 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 641 uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 642 int ret = -EINVAL; 643 644 645 if (index >= VFIO_PCI_NUM_REGIONS) { 646 gvt_vgpu_err("invalid index: %u\n", index); 647 return -EINVAL; 648 } 649 650 switch (index) { 651 case VFIO_PCI_CONFIG_REGION_INDEX: 652 if (is_write) 653 ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos, 654 buf, count); 655 else 656 ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos, 657 buf, count); 658 break; 659 case VFIO_PCI_BAR0_REGION_INDEX: 660 case VFIO_PCI_BAR1_REGION_INDEX: 661 if (is_write) { 662 uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu); 663 664 ret = intel_gvt_ops->emulate_mmio_write(vgpu, 665 bar0_start + pos, buf, count); 666 } else { 667 uint64_t bar0_start = intel_vgpu_get_bar0_addr(vgpu); 668 669 ret = intel_gvt_ops->emulate_mmio_read(vgpu, 670 bar0_start + pos, buf, count); 671 } 672 break; 673 case VFIO_PCI_BAR2_REGION_INDEX: 674 case VFIO_PCI_BAR3_REGION_INDEX: 675 case VFIO_PCI_BAR4_REGION_INDEX: 676 case VFIO_PCI_BAR5_REGION_INDEX: 677 case VFIO_PCI_VGA_REGION_INDEX: 678 case VFIO_PCI_ROM_REGION_INDEX: 679 default: 680 gvt_vgpu_err("unsupported region: %u\n", index); 681 } 682 683 return ret == 0 ? count : ret; 684 } 685 686 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, 687 size_t count, loff_t *ppos) 688 { 689 unsigned int done = 0; 690 int ret; 691 692 while (count) { 693 size_t filled; 694 695 if (count >= 4 && !(*ppos % 4)) { 696 u32 val; 697 698 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 699 ppos, false); 700 if (ret <= 0) 701 goto read_err; 702 703 if (copy_to_user(buf, &val, sizeof(val))) 704 goto read_err; 705 706 filled = 4; 707 } else if (count >= 2 && !(*ppos % 2)) { 708 u16 val; 709 710 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 711 ppos, false); 712 if (ret <= 0) 713 goto read_err; 714 715 if (copy_to_user(buf, &val, sizeof(val))) 716 goto read_err; 717 718 filled = 2; 719 } else { 720 u8 val; 721 722 ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos, 723 false); 724 if (ret <= 0) 725 goto read_err; 726 727 if (copy_to_user(buf, &val, sizeof(val))) 728 goto read_err; 729 730 filled = 1; 731 } 732 733 count -= filled; 734 done += filled; 735 *ppos += filled; 736 buf += filled; 737 } 738 739 return done; 740 741 read_err: 742 return -EFAULT; 743 } 744 745 static ssize_t intel_vgpu_write(struct mdev_device *mdev, 746 const char __user *buf, 747 size_t count, loff_t *ppos) 748 { 749 unsigned int done = 0; 750 int ret; 751 752 while (count) { 753 size_t filled; 754 755 if (count >= 4 && !(*ppos % 4)) { 756 u32 val; 757 758 if (copy_from_user(&val, buf, sizeof(val))) 759 goto write_err; 760 761 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 762 ppos, true); 763 if (ret <= 0) 764 goto write_err; 765 766 filled = 4; 767 } else if (count >= 2 && !(*ppos % 2)) { 768 u16 val; 769 770 if (copy_from_user(&val, buf, sizeof(val))) 771 goto write_err; 772 773 ret = intel_vgpu_rw(mdev, (char *)&val, 774 sizeof(val), ppos, true); 775 if (ret <= 0) 776 goto write_err; 777 778 filled = 2; 779 } else { 780 u8 val; 781 782 if (copy_from_user(&val, buf, sizeof(val))) 783 goto write_err; 784 785 ret = intel_vgpu_rw(mdev, &val, sizeof(val), 786 ppos, true); 787 if (ret <= 0) 788 goto write_err; 789 790 filled = 1; 791 } 792 793 count -= filled; 794 done += filled; 795 *ppos += filled; 796 buf += filled; 797 } 798 799 return done; 800 write_err: 801 return -EFAULT; 802 } 803 804 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) 805 { 806 unsigned int index; 807 u64 virtaddr; 808 unsigned long req_size, pgoff = 0; 809 pgprot_t pg_prot; 810 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 811 812 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 813 if (index >= VFIO_PCI_ROM_REGION_INDEX) 814 return -EINVAL; 815 816 if (vma->vm_end < vma->vm_start) 817 return -EINVAL; 818 if ((vma->vm_flags & VM_SHARED) == 0) 819 return -EINVAL; 820 if (index != VFIO_PCI_BAR2_REGION_INDEX) 821 return -EINVAL; 822 823 pg_prot = vma->vm_page_prot; 824 virtaddr = vma->vm_start; 825 req_size = vma->vm_end - vma->vm_start; 826 pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; 827 828 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); 829 } 830 831 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) 832 { 833 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) 834 return 1; 835 836 return 0; 837 } 838 839 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, 840 unsigned int index, unsigned int start, 841 unsigned int count, uint32_t flags, 842 void *data) 843 { 844 return 0; 845 } 846 847 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, 848 unsigned int index, unsigned int start, 849 unsigned int count, uint32_t flags, void *data) 850 { 851 return 0; 852 } 853 854 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, 855 unsigned int index, unsigned int start, unsigned int count, 856 uint32_t flags, void *data) 857 { 858 return 0; 859 } 860 861 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, 862 unsigned int index, unsigned int start, unsigned int count, 863 uint32_t flags, void *data) 864 { 865 struct eventfd_ctx *trigger; 866 867 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 868 int fd = *(int *)data; 869 870 trigger = eventfd_ctx_fdget(fd); 871 if (IS_ERR(trigger)) { 872 gvt_vgpu_err("eventfd_ctx_fdget failed\n"); 873 return PTR_ERR(trigger); 874 } 875 vgpu->vdev.msi_trigger = trigger; 876 } 877 878 return 0; 879 } 880 881 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags, 882 unsigned int index, unsigned int start, unsigned int count, 883 void *data) 884 { 885 int (*func)(struct intel_vgpu *vgpu, unsigned int index, 886 unsigned int start, unsigned int count, uint32_t flags, 887 void *data) = NULL; 888 889 switch (index) { 890 case VFIO_PCI_INTX_IRQ_INDEX: 891 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 892 case VFIO_IRQ_SET_ACTION_MASK: 893 func = intel_vgpu_set_intx_mask; 894 break; 895 case VFIO_IRQ_SET_ACTION_UNMASK: 896 func = intel_vgpu_set_intx_unmask; 897 break; 898 case VFIO_IRQ_SET_ACTION_TRIGGER: 899 func = intel_vgpu_set_intx_trigger; 900 break; 901 } 902 break; 903 case VFIO_PCI_MSI_IRQ_INDEX: 904 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 905 case VFIO_IRQ_SET_ACTION_MASK: 906 case VFIO_IRQ_SET_ACTION_UNMASK: 907 /* XXX Need masking support exported */ 908 break; 909 case VFIO_IRQ_SET_ACTION_TRIGGER: 910 func = intel_vgpu_set_msi_trigger; 911 break; 912 } 913 break; 914 } 915 916 if (!func) 917 return -ENOTTY; 918 919 return func(vgpu, index, start, count, flags, data); 920 } 921 922 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, 923 unsigned long arg) 924 { 925 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 926 unsigned long minsz; 927 928 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); 929 930 if (cmd == VFIO_DEVICE_GET_INFO) { 931 struct vfio_device_info info; 932 933 minsz = offsetofend(struct vfio_device_info, num_irqs); 934 935 if (copy_from_user(&info, (void __user *)arg, minsz)) 936 return -EFAULT; 937 938 if (info.argsz < minsz) 939 return -EINVAL; 940 941 info.flags = VFIO_DEVICE_FLAGS_PCI; 942 info.flags |= VFIO_DEVICE_FLAGS_RESET; 943 info.num_regions = VFIO_PCI_NUM_REGIONS; 944 info.num_irqs = VFIO_PCI_NUM_IRQS; 945 946 return copy_to_user((void __user *)arg, &info, minsz) ? 947 -EFAULT : 0; 948 949 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 950 struct vfio_region_info info; 951 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 952 int i, ret; 953 struct vfio_region_info_cap_sparse_mmap *sparse = NULL; 954 size_t size; 955 int nr_areas = 1; 956 int cap_type_id; 957 958 minsz = offsetofend(struct vfio_region_info, offset); 959 960 if (copy_from_user(&info, (void __user *)arg, minsz)) 961 return -EFAULT; 962 963 if (info.argsz < minsz) 964 return -EINVAL; 965 966 switch (info.index) { 967 case VFIO_PCI_CONFIG_REGION_INDEX: 968 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 969 info.size = INTEL_GVT_MAX_CFG_SPACE_SZ; 970 info.flags = VFIO_REGION_INFO_FLAG_READ | 971 VFIO_REGION_INFO_FLAG_WRITE; 972 break; 973 case VFIO_PCI_BAR0_REGION_INDEX: 974 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 975 info.size = vgpu->cfg_space.bar[info.index].size; 976 if (!info.size) { 977 info.flags = 0; 978 break; 979 } 980 981 info.flags = VFIO_REGION_INFO_FLAG_READ | 982 VFIO_REGION_INFO_FLAG_WRITE; 983 break; 984 case VFIO_PCI_BAR1_REGION_INDEX: 985 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 986 info.size = 0; 987 info.flags = 0; 988 break; 989 case VFIO_PCI_BAR2_REGION_INDEX: 990 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 991 info.flags = VFIO_REGION_INFO_FLAG_CAPS | 992 VFIO_REGION_INFO_FLAG_MMAP | 993 VFIO_REGION_INFO_FLAG_READ | 994 VFIO_REGION_INFO_FLAG_WRITE; 995 info.size = gvt_aperture_sz(vgpu->gvt); 996 997 size = sizeof(*sparse) + 998 (nr_areas * sizeof(*sparse->areas)); 999 sparse = kzalloc(size, GFP_KERNEL); 1000 if (!sparse) 1001 return -ENOMEM; 1002 1003 sparse->nr_areas = nr_areas; 1004 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1005 sparse->areas[0].offset = 1006 PAGE_ALIGN(vgpu_aperture_offset(vgpu)); 1007 sparse->areas[0].size = vgpu_aperture_sz(vgpu); 1008 break; 1009 1010 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1011 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1012 info.size = 0; 1013 1014 info.flags = 0; 1015 gvt_dbg_core("get region info bar:%d\n", info.index); 1016 break; 1017 1018 case VFIO_PCI_ROM_REGION_INDEX: 1019 case VFIO_PCI_VGA_REGION_INDEX: 1020 gvt_dbg_core("get region info index:%d\n", info.index); 1021 break; 1022 default: 1023 { 1024 struct vfio_region_info_cap_type cap_type; 1025 1026 if (info.index >= VFIO_PCI_NUM_REGIONS + 1027 vgpu->vdev.num_regions) 1028 return -EINVAL; 1029 1030 i = info.index - VFIO_PCI_NUM_REGIONS; 1031 1032 info.offset = 1033 VFIO_PCI_INDEX_TO_OFFSET(info.index); 1034 info.size = vgpu->vdev.region[i].size; 1035 info.flags = vgpu->vdev.region[i].flags; 1036 1037 cap_type.type = vgpu->vdev.region[i].type; 1038 cap_type.subtype = vgpu->vdev.region[i].subtype; 1039 1040 ret = vfio_info_add_capability(&caps, 1041 VFIO_REGION_INFO_CAP_TYPE, 1042 &cap_type); 1043 if (ret) 1044 return ret; 1045 } 1046 } 1047 1048 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { 1049 switch (cap_type_id) { 1050 case VFIO_REGION_INFO_CAP_SPARSE_MMAP: 1051 ret = vfio_info_add_capability(&caps, 1052 VFIO_REGION_INFO_CAP_SPARSE_MMAP, 1053 sparse); 1054 kfree(sparse); 1055 if (ret) 1056 return ret; 1057 break; 1058 default: 1059 return -EINVAL; 1060 } 1061 } 1062 1063 if (caps.size) { 1064 if (info.argsz < sizeof(info) + caps.size) { 1065 info.argsz = sizeof(info) + caps.size; 1066 info.cap_offset = 0; 1067 } else { 1068 vfio_info_cap_shift(&caps, sizeof(info)); 1069 if (copy_to_user((void __user *)arg + 1070 sizeof(info), caps.buf, 1071 caps.size)) { 1072 kfree(caps.buf); 1073 return -EFAULT; 1074 } 1075 info.cap_offset = sizeof(info); 1076 } 1077 1078 kfree(caps.buf); 1079 } 1080 1081 return copy_to_user((void __user *)arg, &info, minsz) ? 1082 -EFAULT : 0; 1083 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 1084 struct vfio_irq_info info; 1085 1086 minsz = offsetofend(struct vfio_irq_info, count); 1087 1088 if (copy_from_user(&info, (void __user *)arg, minsz)) 1089 return -EFAULT; 1090 1091 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1092 return -EINVAL; 1093 1094 switch (info.index) { 1095 case VFIO_PCI_INTX_IRQ_INDEX: 1096 case VFIO_PCI_MSI_IRQ_INDEX: 1097 break; 1098 default: 1099 return -EINVAL; 1100 } 1101 1102 info.flags = VFIO_IRQ_INFO_EVENTFD; 1103 1104 info.count = intel_vgpu_get_irq_count(vgpu, info.index); 1105 1106 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1107 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1108 VFIO_IRQ_INFO_AUTOMASKED); 1109 else 1110 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1111 1112 return copy_to_user((void __user *)arg, &info, minsz) ? 1113 -EFAULT : 0; 1114 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1115 struct vfio_irq_set hdr; 1116 u8 *data = NULL; 1117 int ret = 0; 1118 size_t data_size = 0; 1119 1120 minsz = offsetofend(struct vfio_irq_set, count); 1121 1122 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1123 return -EFAULT; 1124 1125 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 1126 int max = intel_vgpu_get_irq_count(vgpu, hdr.index); 1127 1128 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1129 VFIO_PCI_NUM_IRQS, &data_size); 1130 if (ret) { 1131 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); 1132 return -EINVAL; 1133 } 1134 if (data_size) { 1135 data = memdup_user((void __user *)(arg + minsz), 1136 data_size); 1137 if (IS_ERR(data)) 1138 return PTR_ERR(data); 1139 } 1140 } 1141 1142 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, 1143 hdr.start, hdr.count, data); 1144 kfree(data); 1145 1146 return ret; 1147 } else if (cmd == VFIO_DEVICE_RESET) { 1148 intel_gvt_ops->vgpu_reset(vgpu); 1149 return 0; 1150 } 1151 1152 return 0; 1153 } 1154 1155 static ssize_t 1156 vgpu_id_show(struct device *dev, struct device_attribute *attr, 1157 char *buf) 1158 { 1159 struct mdev_device *mdev = mdev_from_dev(dev); 1160 1161 if (mdev) { 1162 struct intel_vgpu *vgpu = (struct intel_vgpu *) 1163 mdev_get_drvdata(mdev); 1164 return sprintf(buf, "%d\n", vgpu->id); 1165 } 1166 return sprintf(buf, "\n"); 1167 } 1168 1169 static DEVICE_ATTR_RO(vgpu_id); 1170 1171 static struct attribute *intel_vgpu_attrs[] = { 1172 &dev_attr_vgpu_id.attr, 1173 NULL 1174 }; 1175 1176 static const struct attribute_group intel_vgpu_group = { 1177 .name = "intel_vgpu", 1178 .attrs = intel_vgpu_attrs, 1179 }; 1180 1181 static const struct attribute_group *intel_vgpu_groups[] = { 1182 &intel_vgpu_group, 1183 NULL, 1184 }; 1185 1186 static const struct mdev_parent_ops intel_vgpu_ops = { 1187 .supported_type_groups = intel_vgpu_type_groups, 1188 .mdev_attr_groups = intel_vgpu_groups, 1189 .create = intel_vgpu_create, 1190 .remove = intel_vgpu_remove, 1191 1192 .open = intel_vgpu_open, 1193 .release = intel_vgpu_release, 1194 1195 .read = intel_vgpu_read, 1196 .write = intel_vgpu_write, 1197 .mmap = intel_vgpu_mmap, 1198 .ioctl = intel_vgpu_ioctl, 1199 }; 1200 1201 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) 1202 { 1203 if (!intel_gvt_init_vgpu_type_groups(gvt)) 1204 return -EFAULT; 1205 1206 intel_gvt_ops = ops; 1207 1208 return mdev_register_device(dev, &intel_vgpu_ops); 1209 } 1210 1211 static void kvmgt_host_exit(struct device *dev, void *gvt) 1212 { 1213 intel_gvt_cleanup_vgpu_type_groups(gvt); 1214 mdev_unregister_device(dev); 1215 } 1216 1217 static int kvmgt_write_protect_add(unsigned long handle, u64 gfn) 1218 { 1219 struct kvmgt_guest_info *info; 1220 struct kvm *kvm; 1221 struct kvm_memory_slot *slot; 1222 int idx; 1223 1224 if (!handle_valid(handle)) 1225 return -ESRCH; 1226 1227 info = (struct kvmgt_guest_info *)handle; 1228 kvm = info->kvm; 1229 1230 idx = srcu_read_lock(&kvm->srcu); 1231 slot = gfn_to_memslot(kvm, gfn); 1232 if (!slot) { 1233 srcu_read_unlock(&kvm->srcu, idx); 1234 return -EINVAL; 1235 } 1236 1237 spin_lock(&kvm->mmu_lock); 1238 1239 if (kvmgt_gfn_is_write_protected(info, gfn)) 1240 goto out; 1241 1242 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1243 kvmgt_protect_table_add(info, gfn); 1244 1245 out: 1246 spin_unlock(&kvm->mmu_lock); 1247 srcu_read_unlock(&kvm->srcu, idx); 1248 return 0; 1249 } 1250 1251 static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn) 1252 { 1253 struct kvmgt_guest_info *info; 1254 struct kvm *kvm; 1255 struct kvm_memory_slot *slot; 1256 int idx; 1257 1258 if (!handle_valid(handle)) 1259 return 0; 1260 1261 info = (struct kvmgt_guest_info *)handle; 1262 kvm = info->kvm; 1263 1264 idx = srcu_read_lock(&kvm->srcu); 1265 slot = gfn_to_memslot(kvm, gfn); 1266 if (!slot) { 1267 srcu_read_unlock(&kvm->srcu, idx); 1268 return -EINVAL; 1269 } 1270 1271 spin_lock(&kvm->mmu_lock); 1272 1273 if (!kvmgt_gfn_is_write_protected(info, gfn)) 1274 goto out; 1275 1276 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1277 kvmgt_protect_table_del(info, gfn); 1278 1279 out: 1280 spin_unlock(&kvm->mmu_lock); 1281 srcu_read_unlock(&kvm->srcu, idx); 1282 return 0; 1283 } 1284 1285 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 1286 const u8 *val, int len, 1287 struct kvm_page_track_notifier_node *node) 1288 { 1289 struct kvmgt_guest_info *info = container_of(node, 1290 struct kvmgt_guest_info, track_node); 1291 1292 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) 1293 intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa, 1294 (void *)val, len); 1295 } 1296 1297 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 1298 struct kvm_memory_slot *slot, 1299 struct kvm_page_track_notifier_node *node) 1300 { 1301 int i; 1302 gfn_t gfn; 1303 struct kvmgt_guest_info *info = container_of(node, 1304 struct kvmgt_guest_info, track_node); 1305 1306 spin_lock(&kvm->mmu_lock); 1307 for (i = 0; i < slot->npages; i++) { 1308 gfn = slot->base_gfn + i; 1309 if (kvmgt_gfn_is_write_protected(info, gfn)) { 1310 kvm_slot_page_track_remove_page(kvm, slot, gfn, 1311 KVM_PAGE_TRACK_WRITE); 1312 kvmgt_protect_table_del(info, gfn); 1313 } 1314 } 1315 spin_unlock(&kvm->mmu_lock); 1316 } 1317 1318 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm) 1319 { 1320 struct intel_vgpu *itr; 1321 struct kvmgt_guest_info *info; 1322 int id; 1323 bool ret = false; 1324 1325 mutex_lock(&vgpu->gvt->lock); 1326 for_each_active_vgpu(vgpu->gvt, itr, id) { 1327 if (!handle_valid(itr->handle)) 1328 continue; 1329 1330 info = (struct kvmgt_guest_info *)itr->handle; 1331 if (kvm && kvm == info->kvm) { 1332 ret = true; 1333 goto out; 1334 } 1335 } 1336 out: 1337 mutex_unlock(&vgpu->gvt->lock); 1338 return ret; 1339 } 1340 1341 static int kvmgt_guest_init(struct mdev_device *mdev) 1342 { 1343 struct kvmgt_guest_info *info; 1344 struct intel_vgpu *vgpu; 1345 struct kvm *kvm; 1346 1347 vgpu = mdev_get_drvdata(mdev); 1348 if (handle_valid(vgpu->handle)) 1349 return -EEXIST; 1350 1351 kvm = vgpu->vdev.kvm; 1352 if (!kvm || kvm->mm != current->mm) { 1353 gvt_vgpu_err("KVM is required to use Intel vGPU\n"); 1354 return -ESRCH; 1355 } 1356 1357 if (__kvmgt_vgpu_exist(vgpu, kvm)) 1358 return -EEXIST; 1359 1360 info = vzalloc(sizeof(struct kvmgt_guest_info)); 1361 if (!info) 1362 return -ENOMEM; 1363 1364 vgpu->handle = (unsigned long)info; 1365 info->vgpu = vgpu; 1366 info->kvm = kvm; 1367 kvm_get_kvm(info->kvm); 1368 1369 kvmgt_protect_table_init(info); 1370 gvt_cache_init(vgpu); 1371 1372 info->track_node.track_write = kvmgt_page_track_write; 1373 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 1374 kvm_page_track_register_notifier(kvm, &info->track_node); 1375 1376 return 0; 1377 } 1378 1379 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info) 1380 { 1381 kvm_page_track_unregister_notifier(info->kvm, &info->track_node); 1382 kvm_put_kvm(info->kvm); 1383 kvmgt_protect_table_destroy(info); 1384 gvt_cache_destroy(info->vgpu); 1385 vfree(info); 1386 1387 return true; 1388 } 1389 1390 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) 1391 { 1392 /* nothing to do here */ 1393 return 0; 1394 } 1395 1396 static void kvmgt_detach_vgpu(unsigned long handle) 1397 { 1398 /* nothing to do here */ 1399 } 1400 1401 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) 1402 { 1403 struct kvmgt_guest_info *info; 1404 struct intel_vgpu *vgpu; 1405 1406 if (!handle_valid(handle)) 1407 return -ESRCH; 1408 1409 info = (struct kvmgt_guest_info *)handle; 1410 vgpu = info->vgpu; 1411 1412 if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1) 1413 return 0; 1414 1415 return -EFAULT; 1416 } 1417 1418 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) 1419 { 1420 unsigned long iova, pfn; 1421 struct kvmgt_guest_info *info; 1422 struct device *dev; 1423 struct intel_vgpu *vgpu; 1424 int rc; 1425 1426 if (!handle_valid(handle)) 1427 return INTEL_GVT_INVALID_ADDR; 1428 1429 info = (struct kvmgt_guest_info *)handle; 1430 vgpu = info->vgpu; 1431 iova = gvt_cache_find(info->vgpu, gfn); 1432 if (iova != INTEL_GVT_INVALID_ADDR) 1433 return iova; 1434 1435 pfn = INTEL_GVT_INVALID_ADDR; 1436 dev = mdev_dev(info->vgpu->vdev.mdev); 1437 rc = vfio_pin_pages(dev, &gfn, 1, IOMMU_READ | IOMMU_WRITE, &pfn); 1438 if (rc != 1) { 1439 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n", 1440 gfn, rc); 1441 return INTEL_GVT_INVALID_ADDR; 1442 } 1443 /* transfer to host iova for GFX to use DMA */ 1444 rc = gvt_dma_map_iova(info->vgpu, pfn, &iova); 1445 if (rc) { 1446 gvt_vgpu_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn); 1447 vfio_unpin_pages(dev, &gfn, 1); 1448 return INTEL_GVT_INVALID_ADDR; 1449 } 1450 1451 gvt_cache_add(info->vgpu, gfn, iova); 1452 return iova; 1453 } 1454 1455 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, 1456 void *buf, unsigned long len, bool write) 1457 { 1458 struct kvmgt_guest_info *info; 1459 struct kvm *kvm; 1460 int idx, ret; 1461 bool kthread = current->mm == NULL; 1462 1463 if (!handle_valid(handle)) 1464 return -ESRCH; 1465 1466 info = (struct kvmgt_guest_info *)handle; 1467 kvm = info->kvm; 1468 1469 if (kthread) 1470 use_mm(kvm->mm); 1471 1472 idx = srcu_read_lock(&kvm->srcu); 1473 ret = write ? kvm_write_guest(kvm, gpa, buf, len) : 1474 kvm_read_guest(kvm, gpa, buf, len); 1475 srcu_read_unlock(&kvm->srcu, idx); 1476 1477 if (kthread) 1478 unuse_mm(kvm->mm); 1479 1480 return ret; 1481 } 1482 1483 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa, 1484 void *buf, unsigned long len) 1485 { 1486 return kvmgt_rw_gpa(handle, gpa, buf, len, false); 1487 } 1488 1489 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa, 1490 void *buf, unsigned long len) 1491 { 1492 return kvmgt_rw_gpa(handle, gpa, buf, len, true); 1493 } 1494 1495 static unsigned long kvmgt_virt_to_pfn(void *addr) 1496 { 1497 return PFN_DOWN(__pa(addr)); 1498 } 1499 1500 struct intel_gvt_mpt kvmgt_mpt = { 1501 .host_init = kvmgt_host_init, 1502 .host_exit = kvmgt_host_exit, 1503 .attach_vgpu = kvmgt_attach_vgpu, 1504 .detach_vgpu = kvmgt_detach_vgpu, 1505 .inject_msi = kvmgt_inject_msi, 1506 .from_virt_to_mfn = kvmgt_virt_to_pfn, 1507 .set_wp_page = kvmgt_write_protect_add, 1508 .unset_wp_page = kvmgt_write_protect_remove, 1509 .read_gpa = kvmgt_read_gpa, 1510 .write_gpa = kvmgt_write_gpa, 1511 .gfn_to_mfn = kvmgt_gfn_to_pfn, 1512 }; 1513 EXPORT_SYMBOL_GPL(kvmgt_mpt); 1514 1515 static int __init kvmgt_init(void) 1516 { 1517 return 0; 1518 } 1519 1520 static void __exit kvmgt_exit(void) 1521 { 1522 } 1523 1524 module_init(kvmgt_init); 1525 module_exit(kvmgt_exit); 1526 1527 MODULE_LICENSE("GPL and additional rights"); 1528 MODULE_AUTHOR("Intel Corporation"); 1529