1 /* 2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM 3 * 4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Kevin Tian <kevin.tian@intel.com> 27 * Jike Song <jike.song@intel.com> 28 * Xiaoguang Chen <xiaoguang.chen@intel.com> 29 */ 30 31 #include <linux/init.h> 32 #include <linux/device.h> 33 #include <linux/mm.h> 34 #include <linux/mmu_context.h> 35 #include <linux/types.h> 36 #include <linux/list.h> 37 #include <linux/rbtree.h> 38 #include <linux/spinlock.h> 39 #include <linux/eventfd.h> 40 #include <linux/uuid.h> 41 #include <linux/kvm_host.h> 42 #include <linux/vfio.h> 43 #include <linux/mdev.h> 44 45 #include "i915_drv.h" 46 #include "gvt.h" 47 48 static const struct intel_gvt_ops *intel_gvt_ops; 49 50 /* helper macros copied from vfio-pci */ 51 #define VFIO_PCI_OFFSET_SHIFT 40 52 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) 53 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 54 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 55 56 #define OPREGION_SIGNATURE "IntelGraphicsMem" 57 58 struct vfio_region; 59 struct intel_vgpu_regops { 60 size_t (*rw)(struct intel_vgpu *vgpu, char *buf, 61 size_t count, loff_t *ppos, bool iswrite); 62 void (*release)(struct intel_vgpu *vgpu, 63 struct vfio_region *region); 64 }; 65 66 struct vfio_region { 67 u32 type; 68 u32 subtype; 69 size_t size; 70 u32 flags; 71 const struct intel_vgpu_regops *ops; 72 void *data; 73 }; 74 75 struct kvmgt_pgfn { 76 gfn_t gfn; 77 struct hlist_node hnode; 78 }; 79 80 struct kvmgt_guest_info { 81 struct kvm *kvm; 82 struct intel_vgpu *vgpu; 83 struct kvm_page_track_notifier_node track_node; 84 #define NR_BKT (1 << 18) 85 struct hlist_head ptable[NR_BKT]; 86 #undef NR_BKT 87 }; 88 89 struct gvt_dma { 90 struct rb_node node; 91 gfn_t gfn; 92 unsigned long iova; 93 }; 94 95 static inline bool handle_valid(unsigned long handle) 96 { 97 return !!(handle & ~0xff); 98 } 99 100 static int kvmgt_guest_init(struct mdev_device *mdev); 101 static void intel_vgpu_release_work(struct work_struct *work); 102 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); 103 104 static int gvt_dma_map_iova(struct intel_vgpu *vgpu, kvm_pfn_t pfn, 105 unsigned long *iova) 106 { 107 struct page *page; 108 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 109 dma_addr_t daddr; 110 111 if (unlikely(!pfn_valid(pfn))) 112 return -EFAULT; 113 114 page = pfn_to_page(pfn); 115 daddr = dma_map_page(dev, page, 0, PAGE_SIZE, 116 PCI_DMA_BIDIRECTIONAL); 117 if (dma_mapping_error(dev, daddr)) 118 return -ENOMEM; 119 120 *iova = (unsigned long)(daddr >> PAGE_SHIFT); 121 return 0; 122 } 123 124 static void gvt_dma_unmap_iova(struct intel_vgpu *vgpu, unsigned long iova) 125 { 126 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 127 dma_addr_t daddr; 128 129 daddr = (dma_addr_t)(iova << PAGE_SHIFT); 130 dma_unmap_page(dev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 131 } 132 133 static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 134 { 135 struct rb_node *node = vgpu->vdev.cache.rb_node; 136 struct gvt_dma *ret = NULL; 137 138 while (node) { 139 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node); 140 141 if (gfn < itr->gfn) 142 node = node->rb_left; 143 else if (gfn > itr->gfn) 144 node = node->rb_right; 145 else { 146 ret = itr; 147 goto out; 148 } 149 } 150 151 out: 152 return ret; 153 } 154 155 static unsigned long gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 156 { 157 struct gvt_dma *entry; 158 unsigned long iova; 159 160 mutex_lock(&vgpu->vdev.cache_lock); 161 162 entry = __gvt_cache_find(vgpu, gfn); 163 iova = (entry == NULL) ? INTEL_GVT_INVALID_ADDR : entry->iova; 164 165 mutex_unlock(&vgpu->vdev.cache_lock); 166 return iova; 167 } 168 169 static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 170 unsigned long iova) 171 { 172 struct gvt_dma *new, *itr; 173 struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL; 174 175 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 176 if (!new) 177 return; 178 179 new->gfn = gfn; 180 new->iova = iova; 181 182 mutex_lock(&vgpu->vdev.cache_lock); 183 while (*link) { 184 parent = *link; 185 itr = rb_entry(parent, struct gvt_dma, node); 186 187 if (gfn == itr->gfn) 188 goto out; 189 else if (gfn < itr->gfn) 190 link = &parent->rb_left; 191 else 192 link = &parent->rb_right; 193 } 194 195 rb_link_node(&new->node, parent, link); 196 rb_insert_color(&new->node, &vgpu->vdev.cache); 197 mutex_unlock(&vgpu->vdev.cache_lock); 198 return; 199 200 out: 201 mutex_unlock(&vgpu->vdev.cache_lock); 202 kfree(new); 203 } 204 205 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 206 struct gvt_dma *entry) 207 { 208 rb_erase(&entry->node, &vgpu->vdev.cache); 209 kfree(entry); 210 } 211 212 static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn) 213 { 214 struct device *dev = mdev_dev(vgpu->vdev.mdev); 215 struct gvt_dma *this; 216 unsigned long g1; 217 int rc; 218 219 mutex_lock(&vgpu->vdev.cache_lock); 220 this = __gvt_cache_find(vgpu, gfn); 221 if (!this) { 222 mutex_unlock(&vgpu->vdev.cache_lock); 223 return; 224 } 225 226 g1 = gfn; 227 gvt_dma_unmap_iova(vgpu, this->iova); 228 rc = vfio_unpin_pages(dev, &g1, 1); 229 WARN_ON(rc != 1); 230 __gvt_cache_remove_entry(vgpu, this); 231 mutex_unlock(&vgpu->vdev.cache_lock); 232 } 233 234 static void gvt_cache_init(struct intel_vgpu *vgpu) 235 { 236 vgpu->vdev.cache = RB_ROOT; 237 mutex_init(&vgpu->vdev.cache_lock); 238 } 239 240 static void gvt_cache_destroy(struct intel_vgpu *vgpu) 241 { 242 struct gvt_dma *dma; 243 struct rb_node *node = NULL; 244 struct device *dev = mdev_dev(vgpu->vdev.mdev); 245 unsigned long gfn; 246 247 for (;;) { 248 mutex_lock(&vgpu->vdev.cache_lock); 249 node = rb_first(&vgpu->vdev.cache); 250 if (!node) { 251 mutex_unlock(&vgpu->vdev.cache_lock); 252 break; 253 } 254 dma = rb_entry(node, struct gvt_dma, node); 255 gvt_dma_unmap_iova(vgpu, dma->iova); 256 gfn = dma->gfn; 257 __gvt_cache_remove_entry(vgpu, dma); 258 mutex_unlock(&vgpu->vdev.cache_lock); 259 vfio_unpin_pages(dev, &gfn, 1); 260 } 261 } 262 263 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) 264 { 265 hash_init(info->ptable); 266 } 267 268 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info) 269 { 270 struct kvmgt_pgfn *p; 271 struct hlist_node *tmp; 272 int i; 273 274 hash_for_each_safe(info->ptable, i, tmp, p, hnode) { 275 hash_del(&p->hnode); 276 kfree(p); 277 } 278 } 279 280 static struct kvmgt_pgfn * 281 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn) 282 { 283 struct kvmgt_pgfn *p, *res = NULL; 284 285 hash_for_each_possible(info->ptable, p, hnode, gfn) { 286 if (gfn == p->gfn) { 287 res = p; 288 break; 289 } 290 } 291 292 return res; 293 } 294 295 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info, 296 gfn_t gfn) 297 { 298 struct kvmgt_pgfn *p; 299 300 p = __kvmgt_protect_table_find(info, gfn); 301 return !!p; 302 } 303 304 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn) 305 { 306 struct kvmgt_pgfn *p; 307 308 if (kvmgt_gfn_is_write_protected(info, gfn)) 309 return; 310 311 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); 312 if (WARN(!p, "gfn: 0x%llx\n", gfn)) 313 return; 314 315 p->gfn = gfn; 316 hash_add(info->ptable, &p->hnode, gfn); 317 } 318 319 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, 320 gfn_t gfn) 321 { 322 struct kvmgt_pgfn *p; 323 324 p = __kvmgt_protect_table_find(info, gfn); 325 if (p) { 326 hash_del(&p->hnode); 327 kfree(p); 328 } 329 } 330 331 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf, 332 size_t count, loff_t *ppos, bool iswrite) 333 { 334 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - 335 VFIO_PCI_NUM_REGIONS; 336 void *base = vgpu->vdev.region[i].data; 337 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 338 339 if (pos >= vgpu->vdev.region[i].size || iswrite) { 340 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n"); 341 return -EINVAL; 342 } 343 count = min(count, (size_t)(vgpu->vdev.region[i].size - pos)); 344 memcpy(buf, base + pos, count); 345 346 return count; 347 } 348 349 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu, 350 struct vfio_region *region) 351 { 352 } 353 354 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = { 355 .rw = intel_vgpu_reg_rw_opregion, 356 .release = intel_vgpu_reg_release_opregion, 357 }; 358 359 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu, 360 unsigned int type, unsigned int subtype, 361 const struct intel_vgpu_regops *ops, 362 size_t size, u32 flags, void *data) 363 { 364 struct vfio_region *region; 365 366 region = krealloc(vgpu->vdev.region, 367 (vgpu->vdev.num_regions + 1) * sizeof(*region), 368 GFP_KERNEL); 369 if (!region) 370 return -ENOMEM; 371 372 vgpu->vdev.region = region; 373 vgpu->vdev.region[vgpu->vdev.num_regions].type = type; 374 vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype; 375 vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops; 376 vgpu->vdev.region[vgpu->vdev.num_regions].size = size; 377 vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags; 378 vgpu->vdev.region[vgpu->vdev.num_regions].data = data; 379 vgpu->vdev.num_regions++; 380 return 0; 381 } 382 383 static int kvmgt_get_vfio_device(void *p_vgpu) 384 { 385 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; 386 387 vgpu->vdev.vfio_device = vfio_device_get_from_dev( 388 mdev_dev(vgpu->vdev.mdev)); 389 if (!vgpu->vdev.vfio_device) { 390 gvt_vgpu_err("failed to get vfio device\n"); 391 return -ENODEV; 392 } 393 return 0; 394 } 395 396 397 static int kvmgt_set_opregion(void *p_vgpu) 398 { 399 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; 400 void *base; 401 int ret; 402 403 /* Each vgpu has its own opregion, although VFIO would create another 404 * one later. This one is used to expose opregion to VFIO. And the 405 * other one created by VFIO later, is used by guest actually. 406 */ 407 base = vgpu_opregion(vgpu)->va; 408 if (!base) 409 return -ENOMEM; 410 411 if (memcmp(base, OPREGION_SIGNATURE, 16)) { 412 memunmap(base); 413 return -EINVAL; 414 } 415 416 ret = intel_vgpu_register_reg(vgpu, 417 PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, 418 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, 419 &intel_vgpu_regops_opregion, OPREGION_SIZE, 420 VFIO_REGION_INFO_FLAG_READ, base); 421 422 return ret; 423 } 424 425 static void kvmgt_put_vfio_device(void *vgpu) 426 { 427 if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device)) 428 return; 429 430 vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device); 431 } 432 433 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) 434 { 435 struct intel_vgpu *vgpu = NULL; 436 struct intel_vgpu_type *type; 437 struct device *pdev; 438 void *gvt; 439 int ret; 440 441 pdev = mdev_parent_dev(mdev); 442 gvt = kdev_to_i915(pdev)->gvt; 443 444 type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj)); 445 if (!type) { 446 gvt_vgpu_err("failed to find type %s to create\n", 447 kobject_name(kobj)); 448 ret = -EINVAL; 449 goto out; 450 } 451 452 vgpu = intel_gvt_ops->vgpu_create(gvt, type); 453 if (IS_ERR_OR_NULL(vgpu)) { 454 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu); 455 gvt_vgpu_err("failed to create intel vgpu: %d\n", ret); 456 goto out; 457 } 458 459 INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work); 460 461 vgpu->vdev.mdev = mdev; 462 mdev_set_drvdata(mdev, vgpu); 463 464 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", 465 dev_name(mdev_dev(mdev))); 466 ret = 0; 467 468 out: 469 return ret; 470 } 471 472 static int intel_vgpu_remove(struct mdev_device *mdev) 473 { 474 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 475 476 if (handle_valid(vgpu->handle)) 477 return -EBUSY; 478 479 intel_gvt_ops->vgpu_destroy(vgpu); 480 return 0; 481 } 482 483 static int intel_vgpu_iommu_notifier(struct notifier_block *nb, 484 unsigned long action, void *data) 485 { 486 struct intel_vgpu *vgpu = container_of(nb, 487 struct intel_vgpu, 488 vdev.iommu_notifier); 489 490 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 491 struct vfio_iommu_type1_dma_unmap *unmap = data; 492 unsigned long gfn, end_gfn; 493 494 gfn = unmap->iova >> PAGE_SHIFT; 495 end_gfn = gfn + unmap->size / PAGE_SIZE; 496 497 while (gfn < end_gfn) 498 gvt_cache_remove(vgpu, gfn++); 499 } 500 501 return NOTIFY_OK; 502 } 503 504 static int intel_vgpu_group_notifier(struct notifier_block *nb, 505 unsigned long action, void *data) 506 { 507 struct intel_vgpu *vgpu = container_of(nb, 508 struct intel_vgpu, 509 vdev.group_notifier); 510 511 /* the only action we care about */ 512 if (action == VFIO_GROUP_NOTIFY_SET_KVM) { 513 vgpu->vdev.kvm = data; 514 515 if (!data) 516 schedule_work(&vgpu->vdev.release_work); 517 } 518 519 return NOTIFY_OK; 520 } 521 522 static int intel_vgpu_open(struct mdev_device *mdev) 523 { 524 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 525 unsigned long events; 526 int ret; 527 528 vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier; 529 vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier; 530 531 events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; 532 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events, 533 &vgpu->vdev.iommu_notifier); 534 if (ret != 0) { 535 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n", 536 ret); 537 goto out; 538 } 539 540 events = VFIO_GROUP_NOTIFY_SET_KVM; 541 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events, 542 &vgpu->vdev.group_notifier); 543 if (ret != 0) { 544 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n", 545 ret); 546 goto undo_iommu; 547 } 548 549 ret = kvmgt_guest_init(mdev); 550 if (ret) 551 goto undo_group; 552 553 intel_gvt_ops->vgpu_activate(vgpu); 554 555 atomic_set(&vgpu->vdev.released, 0); 556 return ret; 557 558 undo_group: 559 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, 560 &vgpu->vdev.group_notifier); 561 562 undo_iommu: 563 vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, 564 &vgpu->vdev.iommu_notifier); 565 out: 566 return ret; 567 } 568 569 static void __intel_vgpu_release(struct intel_vgpu *vgpu) 570 { 571 struct kvmgt_guest_info *info; 572 int ret; 573 574 if (!handle_valid(vgpu->handle)) 575 return; 576 577 if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1)) 578 return; 579 580 intel_gvt_ops->vgpu_deactivate(vgpu); 581 582 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY, 583 &vgpu->vdev.iommu_notifier); 584 WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret); 585 586 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY, 587 &vgpu->vdev.group_notifier); 588 WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret); 589 590 info = (struct kvmgt_guest_info *)vgpu->handle; 591 kvmgt_guest_exit(info); 592 593 vgpu->vdev.kvm = NULL; 594 vgpu->handle = 0; 595 } 596 597 static void intel_vgpu_release(struct mdev_device *mdev) 598 { 599 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 600 601 __intel_vgpu_release(vgpu); 602 } 603 604 static void intel_vgpu_release_work(struct work_struct *work) 605 { 606 struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu, 607 vdev.release_work); 608 609 __intel_vgpu_release(vgpu); 610 } 611 612 static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) 613 { 614 u32 start_lo, start_hi; 615 u32 mem_type; 616 617 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 618 PCI_BASE_ADDRESS_MEM_MASK; 619 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 620 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 621 622 switch (mem_type) { 623 case PCI_BASE_ADDRESS_MEM_TYPE_64: 624 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space 625 + bar + 4)); 626 break; 627 case PCI_BASE_ADDRESS_MEM_TYPE_32: 628 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 629 /* 1M mem BAR treated as 32-bit BAR */ 630 default: 631 /* mem unknown type treated as 32-bit BAR */ 632 start_hi = 0; 633 break; 634 } 635 636 return ((u64)start_hi << 32) | start_lo; 637 } 638 639 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off, 640 void *buf, unsigned int count, bool is_write) 641 { 642 uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar); 643 int ret; 644 645 if (is_write) 646 ret = intel_gvt_ops->emulate_mmio_write(vgpu, 647 bar_start + off, buf, count); 648 else 649 ret = intel_gvt_ops->emulate_mmio_read(vgpu, 650 bar_start + off, buf, count); 651 return ret; 652 } 653 654 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off) 655 { 656 return off >= vgpu_aperture_offset(vgpu) && 657 off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu); 658 } 659 660 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off, 661 void *buf, unsigned long count, bool is_write) 662 { 663 void *aperture_va; 664 665 if (!intel_vgpu_in_aperture(vgpu, off) || 666 !intel_vgpu_in_aperture(vgpu, off + count)) { 667 gvt_vgpu_err("Invalid aperture offset %llu\n", off); 668 return -EINVAL; 669 } 670 671 aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap, 672 ALIGN_DOWN(off, PAGE_SIZE), 673 count + offset_in_page(off)); 674 if (!aperture_va) 675 return -EIO; 676 677 if (is_write) 678 memcpy(aperture_va + offset_in_page(off), buf, count); 679 else 680 memcpy(buf, aperture_va + offset_in_page(off), count); 681 682 io_mapping_unmap(aperture_va); 683 684 return 0; 685 } 686 687 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, 688 size_t count, loff_t *ppos, bool is_write) 689 { 690 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 691 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 692 uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 693 int ret = -EINVAL; 694 695 696 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) { 697 gvt_vgpu_err("invalid index: %u\n", index); 698 return -EINVAL; 699 } 700 701 switch (index) { 702 case VFIO_PCI_CONFIG_REGION_INDEX: 703 if (is_write) 704 ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos, 705 buf, count); 706 else 707 ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos, 708 buf, count); 709 break; 710 case VFIO_PCI_BAR0_REGION_INDEX: 711 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos, 712 buf, count, is_write); 713 break; 714 case VFIO_PCI_BAR2_REGION_INDEX: 715 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write); 716 break; 717 case VFIO_PCI_BAR1_REGION_INDEX: 718 case VFIO_PCI_BAR3_REGION_INDEX: 719 case VFIO_PCI_BAR4_REGION_INDEX: 720 case VFIO_PCI_BAR5_REGION_INDEX: 721 case VFIO_PCI_VGA_REGION_INDEX: 722 case VFIO_PCI_ROM_REGION_INDEX: 723 break; 724 default: 725 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) 726 return -EINVAL; 727 728 index -= VFIO_PCI_NUM_REGIONS; 729 return vgpu->vdev.region[index].ops->rw(vgpu, buf, count, 730 ppos, is_write); 731 } 732 733 return ret == 0 ? count : ret; 734 } 735 736 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, 737 size_t count, loff_t *ppos) 738 { 739 unsigned int done = 0; 740 int ret; 741 742 while (count) { 743 size_t filled; 744 745 if (count >= 4 && !(*ppos % 4)) { 746 u32 val; 747 748 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 749 ppos, false); 750 if (ret <= 0) 751 goto read_err; 752 753 if (copy_to_user(buf, &val, sizeof(val))) 754 goto read_err; 755 756 filled = 4; 757 } else if (count >= 2 && !(*ppos % 2)) { 758 u16 val; 759 760 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 761 ppos, false); 762 if (ret <= 0) 763 goto read_err; 764 765 if (copy_to_user(buf, &val, sizeof(val))) 766 goto read_err; 767 768 filled = 2; 769 } else { 770 u8 val; 771 772 ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos, 773 false); 774 if (ret <= 0) 775 goto read_err; 776 777 if (copy_to_user(buf, &val, sizeof(val))) 778 goto read_err; 779 780 filled = 1; 781 } 782 783 count -= filled; 784 done += filled; 785 *ppos += filled; 786 buf += filled; 787 } 788 789 return done; 790 791 read_err: 792 return -EFAULT; 793 } 794 795 static ssize_t intel_vgpu_write(struct mdev_device *mdev, 796 const char __user *buf, 797 size_t count, loff_t *ppos) 798 { 799 unsigned int done = 0; 800 int ret; 801 802 while (count) { 803 size_t filled; 804 805 if (count >= 4 && !(*ppos % 4)) { 806 u32 val; 807 808 if (copy_from_user(&val, buf, sizeof(val))) 809 goto write_err; 810 811 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 812 ppos, true); 813 if (ret <= 0) 814 goto write_err; 815 816 filled = 4; 817 } else if (count >= 2 && !(*ppos % 2)) { 818 u16 val; 819 820 if (copy_from_user(&val, buf, sizeof(val))) 821 goto write_err; 822 823 ret = intel_vgpu_rw(mdev, (char *)&val, 824 sizeof(val), ppos, true); 825 if (ret <= 0) 826 goto write_err; 827 828 filled = 2; 829 } else { 830 u8 val; 831 832 if (copy_from_user(&val, buf, sizeof(val))) 833 goto write_err; 834 835 ret = intel_vgpu_rw(mdev, &val, sizeof(val), 836 ppos, true); 837 if (ret <= 0) 838 goto write_err; 839 840 filled = 1; 841 } 842 843 count -= filled; 844 done += filled; 845 *ppos += filled; 846 buf += filled; 847 } 848 849 return done; 850 write_err: 851 return -EFAULT; 852 } 853 854 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) 855 { 856 unsigned int index; 857 u64 virtaddr; 858 unsigned long req_size, pgoff = 0; 859 pgprot_t pg_prot; 860 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 861 862 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 863 if (index >= VFIO_PCI_ROM_REGION_INDEX) 864 return -EINVAL; 865 866 if (vma->vm_end < vma->vm_start) 867 return -EINVAL; 868 if ((vma->vm_flags & VM_SHARED) == 0) 869 return -EINVAL; 870 if (index != VFIO_PCI_BAR2_REGION_INDEX) 871 return -EINVAL; 872 873 pg_prot = vma->vm_page_prot; 874 virtaddr = vma->vm_start; 875 req_size = vma->vm_end - vma->vm_start; 876 pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; 877 878 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); 879 } 880 881 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) 882 { 883 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) 884 return 1; 885 886 return 0; 887 } 888 889 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, 890 unsigned int index, unsigned int start, 891 unsigned int count, uint32_t flags, 892 void *data) 893 { 894 return 0; 895 } 896 897 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, 898 unsigned int index, unsigned int start, 899 unsigned int count, uint32_t flags, void *data) 900 { 901 return 0; 902 } 903 904 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, 905 unsigned int index, unsigned int start, unsigned int count, 906 uint32_t flags, void *data) 907 { 908 return 0; 909 } 910 911 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, 912 unsigned int index, unsigned int start, unsigned int count, 913 uint32_t flags, void *data) 914 { 915 struct eventfd_ctx *trigger; 916 917 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 918 int fd = *(int *)data; 919 920 trigger = eventfd_ctx_fdget(fd); 921 if (IS_ERR(trigger)) { 922 gvt_vgpu_err("eventfd_ctx_fdget failed\n"); 923 return PTR_ERR(trigger); 924 } 925 vgpu->vdev.msi_trigger = trigger; 926 } 927 928 return 0; 929 } 930 931 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags, 932 unsigned int index, unsigned int start, unsigned int count, 933 void *data) 934 { 935 int (*func)(struct intel_vgpu *vgpu, unsigned int index, 936 unsigned int start, unsigned int count, uint32_t flags, 937 void *data) = NULL; 938 939 switch (index) { 940 case VFIO_PCI_INTX_IRQ_INDEX: 941 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 942 case VFIO_IRQ_SET_ACTION_MASK: 943 func = intel_vgpu_set_intx_mask; 944 break; 945 case VFIO_IRQ_SET_ACTION_UNMASK: 946 func = intel_vgpu_set_intx_unmask; 947 break; 948 case VFIO_IRQ_SET_ACTION_TRIGGER: 949 func = intel_vgpu_set_intx_trigger; 950 break; 951 } 952 break; 953 case VFIO_PCI_MSI_IRQ_INDEX: 954 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 955 case VFIO_IRQ_SET_ACTION_MASK: 956 case VFIO_IRQ_SET_ACTION_UNMASK: 957 /* XXX Need masking support exported */ 958 break; 959 case VFIO_IRQ_SET_ACTION_TRIGGER: 960 func = intel_vgpu_set_msi_trigger; 961 break; 962 } 963 break; 964 } 965 966 if (!func) 967 return -ENOTTY; 968 969 return func(vgpu, index, start, count, flags, data); 970 } 971 972 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, 973 unsigned long arg) 974 { 975 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 976 unsigned long minsz; 977 978 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); 979 980 if (cmd == VFIO_DEVICE_GET_INFO) { 981 struct vfio_device_info info; 982 983 minsz = offsetofend(struct vfio_device_info, num_irqs); 984 985 if (copy_from_user(&info, (void __user *)arg, minsz)) 986 return -EFAULT; 987 988 if (info.argsz < minsz) 989 return -EINVAL; 990 991 info.flags = VFIO_DEVICE_FLAGS_PCI; 992 info.flags |= VFIO_DEVICE_FLAGS_RESET; 993 info.num_regions = VFIO_PCI_NUM_REGIONS + 994 vgpu->vdev.num_regions; 995 info.num_irqs = VFIO_PCI_NUM_IRQS; 996 997 return copy_to_user((void __user *)arg, &info, minsz) ? 998 -EFAULT : 0; 999 1000 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 1001 struct vfio_region_info info; 1002 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 1003 int i, ret; 1004 struct vfio_region_info_cap_sparse_mmap *sparse = NULL; 1005 size_t size; 1006 int nr_areas = 1; 1007 int cap_type_id; 1008 1009 minsz = offsetofend(struct vfio_region_info, offset); 1010 1011 if (copy_from_user(&info, (void __user *)arg, minsz)) 1012 return -EFAULT; 1013 1014 if (info.argsz < minsz) 1015 return -EINVAL; 1016 1017 switch (info.index) { 1018 case VFIO_PCI_CONFIG_REGION_INDEX: 1019 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1020 info.size = vgpu->gvt->device_info.cfg_space_size; 1021 info.flags = VFIO_REGION_INFO_FLAG_READ | 1022 VFIO_REGION_INFO_FLAG_WRITE; 1023 break; 1024 case VFIO_PCI_BAR0_REGION_INDEX: 1025 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1026 info.size = vgpu->cfg_space.bar[info.index].size; 1027 if (!info.size) { 1028 info.flags = 0; 1029 break; 1030 } 1031 1032 info.flags = VFIO_REGION_INFO_FLAG_READ | 1033 VFIO_REGION_INFO_FLAG_WRITE; 1034 break; 1035 case VFIO_PCI_BAR1_REGION_INDEX: 1036 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1037 info.size = 0; 1038 info.flags = 0; 1039 break; 1040 case VFIO_PCI_BAR2_REGION_INDEX: 1041 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1042 info.flags = VFIO_REGION_INFO_FLAG_CAPS | 1043 VFIO_REGION_INFO_FLAG_MMAP | 1044 VFIO_REGION_INFO_FLAG_READ | 1045 VFIO_REGION_INFO_FLAG_WRITE; 1046 info.size = gvt_aperture_sz(vgpu->gvt); 1047 1048 size = sizeof(*sparse) + 1049 (nr_areas * sizeof(*sparse->areas)); 1050 sparse = kzalloc(size, GFP_KERNEL); 1051 if (!sparse) 1052 return -ENOMEM; 1053 1054 sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1055 sparse->header.version = 1; 1056 sparse->nr_areas = nr_areas; 1057 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1058 sparse->areas[0].offset = 1059 PAGE_ALIGN(vgpu_aperture_offset(vgpu)); 1060 sparse->areas[0].size = vgpu_aperture_sz(vgpu); 1061 break; 1062 1063 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1064 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1065 info.size = 0; 1066 info.flags = 0; 1067 1068 gvt_dbg_core("get region info bar:%d\n", info.index); 1069 break; 1070 1071 case VFIO_PCI_ROM_REGION_INDEX: 1072 case VFIO_PCI_VGA_REGION_INDEX: 1073 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1074 info.size = 0; 1075 info.flags = 0; 1076 1077 gvt_dbg_core("get region info index:%d\n", info.index); 1078 break; 1079 default: 1080 { 1081 struct vfio_region_info_cap_type cap_type = { 1082 .header.id = VFIO_REGION_INFO_CAP_TYPE, 1083 .header.version = 1 }; 1084 1085 if (info.index >= VFIO_PCI_NUM_REGIONS + 1086 vgpu->vdev.num_regions) 1087 return -EINVAL; 1088 1089 i = info.index - VFIO_PCI_NUM_REGIONS; 1090 1091 info.offset = 1092 VFIO_PCI_INDEX_TO_OFFSET(info.index); 1093 info.size = vgpu->vdev.region[i].size; 1094 info.flags = vgpu->vdev.region[i].flags; 1095 1096 cap_type.type = vgpu->vdev.region[i].type; 1097 cap_type.subtype = vgpu->vdev.region[i].subtype; 1098 1099 ret = vfio_info_add_capability(&caps, 1100 &cap_type.header, 1101 sizeof(cap_type)); 1102 if (ret) 1103 return ret; 1104 } 1105 } 1106 1107 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { 1108 switch (cap_type_id) { 1109 case VFIO_REGION_INFO_CAP_SPARSE_MMAP: 1110 ret = vfio_info_add_capability(&caps, 1111 &sparse->header, sizeof(*sparse) + 1112 (sparse->nr_areas * 1113 sizeof(*sparse->areas))); 1114 kfree(sparse); 1115 if (ret) 1116 return ret; 1117 break; 1118 default: 1119 return -EINVAL; 1120 } 1121 } 1122 1123 if (caps.size) { 1124 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 1125 if (info.argsz < sizeof(info) + caps.size) { 1126 info.argsz = sizeof(info) + caps.size; 1127 info.cap_offset = 0; 1128 } else { 1129 vfio_info_cap_shift(&caps, sizeof(info)); 1130 if (copy_to_user((void __user *)arg + 1131 sizeof(info), caps.buf, 1132 caps.size)) { 1133 kfree(caps.buf); 1134 return -EFAULT; 1135 } 1136 info.cap_offset = sizeof(info); 1137 } 1138 1139 kfree(caps.buf); 1140 } 1141 1142 return copy_to_user((void __user *)arg, &info, minsz) ? 1143 -EFAULT : 0; 1144 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 1145 struct vfio_irq_info info; 1146 1147 minsz = offsetofend(struct vfio_irq_info, count); 1148 1149 if (copy_from_user(&info, (void __user *)arg, minsz)) 1150 return -EFAULT; 1151 1152 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1153 return -EINVAL; 1154 1155 switch (info.index) { 1156 case VFIO_PCI_INTX_IRQ_INDEX: 1157 case VFIO_PCI_MSI_IRQ_INDEX: 1158 break; 1159 default: 1160 return -EINVAL; 1161 } 1162 1163 info.flags = VFIO_IRQ_INFO_EVENTFD; 1164 1165 info.count = intel_vgpu_get_irq_count(vgpu, info.index); 1166 1167 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1168 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1169 VFIO_IRQ_INFO_AUTOMASKED); 1170 else 1171 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1172 1173 return copy_to_user((void __user *)arg, &info, minsz) ? 1174 -EFAULT : 0; 1175 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1176 struct vfio_irq_set hdr; 1177 u8 *data = NULL; 1178 int ret = 0; 1179 size_t data_size = 0; 1180 1181 minsz = offsetofend(struct vfio_irq_set, count); 1182 1183 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1184 return -EFAULT; 1185 1186 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 1187 int max = intel_vgpu_get_irq_count(vgpu, hdr.index); 1188 1189 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1190 VFIO_PCI_NUM_IRQS, &data_size); 1191 if (ret) { 1192 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); 1193 return -EINVAL; 1194 } 1195 if (data_size) { 1196 data = memdup_user((void __user *)(arg + minsz), 1197 data_size); 1198 if (IS_ERR(data)) 1199 return PTR_ERR(data); 1200 } 1201 } 1202 1203 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, 1204 hdr.start, hdr.count, data); 1205 kfree(data); 1206 1207 return ret; 1208 } else if (cmd == VFIO_DEVICE_RESET) { 1209 intel_gvt_ops->vgpu_reset(vgpu); 1210 return 0; 1211 } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) { 1212 struct vfio_device_gfx_plane_info dmabuf; 1213 int ret = 0; 1214 1215 minsz = offsetofend(struct vfio_device_gfx_plane_info, 1216 dmabuf_id); 1217 if (copy_from_user(&dmabuf, (void __user *)arg, minsz)) 1218 return -EFAULT; 1219 if (dmabuf.argsz < minsz) 1220 return -EINVAL; 1221 1222 ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf); 1223 if (ret != 0) 1224 return ret; 1225 1226 return copy_to_user((void __user *)arg, &dmabuf, minsz) ? 1227 -EFAULT : 0; 1228 } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) { 1229 __u32 dmabuf_id; 1230 __s32 dmabuf_fd; 1231 1232 if (get_user(dmabuf_id, (__u32 __user *)arg)) 1233 return -EFAULT; 1234 1235 dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id); 1236 return dmabuf_fd; 1237 1238 } 1239 1240 return 0; 1241 } 1242 1243 static ssize_t 1244 vgpu_id_show(struct device *dev, struct device_attribute *attr, 1245 char *buf) 1246 { 1247 struct mdev_device *mdev = mdev_from_dev(dev); 1248 1249 if (mdev) { 1250 struct intel_vgpu *vgpu = (struct intel_vgpu *) 1251 mdev_get_drvdata(mdev); 1252 return sprintf(buf, "%d\n", vgpu->id); 1253 } 1254 return sprintf(buf, "\n"); 1255 } 1256 1257 static ssize_t 1258 hw_id_show(struct device *dev, struct device_attribute *attr, 1259 char *buf) 1260 { 1261 struct mdev_device *mdev = mdev_from_dev(dev); 1262 1263 if (mdev) { 1264 struct intel_vgpu *vgpu = (struct intel_vgpu *) 1265 mdev_get_drvdata(mdev); 1266 return sprintf(buf, "%u\n", 1267 vgpu->submission.shadow_ctx->hw_id); 1268 } 1269 return sprintf(buf, "\n"); 1270 } 1271 1272 static DEVICE_ATTR_RO(vgpu_id); 1273 static DEVICE_ATTR_RO(hw_id); 1274 1275 static struct attribute *intel_vgpu_attrs[] = { 1276 &dev_attr_vgpu_id.attr, 1277 &dev_attr_hw_id.attr, 1278 NULL 1279 }; 1280 1281 static const struct attribute_group intel_vgpu_group = { 1282 .name = "intel_vgpu", 1283 .attrs = intel_vgpu_attrs, 1284 }; 1285 1286 static const struct attribute_group *intel_vgpu_groups[] = { 1287 &intel_vgpu_group, 1288 NULL, 1289 }; 1290 1291 static struct mdev_parent_ops intel_vgpu_ops = { 1292 .mdev_attr_groups = intel_vgpu_groups, 1293 .create = intel_vgpu_create, 1294 .remove = intel_vgpu_remove, 1295 1296 .open = intel_vgpu_open, 1297 .release = intel_vgpu_release, 1298 1299 .read = intel_vgpu_read, 1300 .write = intel_vgpu_write, 1301 .mmap = intel_vgpu_mmap, 1302 .ioctl = intel_vgpu_ioctl, 1303 }; 1304 1305 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) 1306 { 1307 struct attribute **kvm_type_attrs; 1308 struct attribute_group **kvm_vgpu_type_groups; 1309 1310 intel_gvt_ops = ops; 1311 if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs, 1312 &kvm_vgpu_type_groups)) 1313 return -EFAULT; 1314 intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups; 1315 1316 return mdev_register_device(dev, &intel_vgpu_ops); 1317 } 1318 1319 static void kvmgt_host_exit(struct device *dev, void *gvt) 1320 { 1321 mdev_unregister_device(dev); 1322 } 1323 1324 static int kvmgt_write_protect_add(unsigned long handle, u64 gfn) 1325 { 1326 struct kvmgt_guest_info *info; 1327 struct kvm *kvm; 1328 struct kvm_memory_slot *slot; 1329 int idx; 1330 1331 if (!handle_valid(handle)) 1332 return -ESRCH; 1333 1334 info = (struct kvmgt_guest_info *)handle; 1335 kvm = info->kvm; 1336 1337 idx = srcu_read_lock(&kvm->srcu); 1338 slot = gfn_to_memslot(kvm, gfn); 1339 if (!slot) { 1340 srcu_read_unlock(&kvm->srcu, idx); 1341 return -EINVAL; 1342 } 1343 1344 spin_lock(&kvm->mmu_lock); 1345 1346 if (kvmgt_gfn_is_write_protected(info, gfn)) 1347 goto out; 1348 1349 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1350 kvmgt_protect_table_add(info, gfn); 1351 1352 out: 1353 spin_unlock(&kvm->mmu_lock); 1354 srcu_read_unlock(&kvm->srcu, idx); 1355 return 0; 1356 } 1357 1358 static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn) 1359 { 1360 struct kvmgt_guest_info *info; 1361 struct kvm *kvm; 1362 struct kvm_memory_slot *slot; 1363 int idx; 1364 1365 if (!handle_valid(handle)) 1366 return 0; 1367 1368 info = (struct kvmgt_guest_info *)handle; 1369 kvm = info->kvm; 1370 1371 idx = srcu_read_lock(&kvm->srcu); 1372 slot = gfn_to_memslot(kvm, gfn); 1373 if (!slot) { 1374 srcu_read_unlock(&kvm->srcu, idx); 1375 return -EINVAL; 1376 } 1377 1378 spin_lock(&kvm->mmu_lock); 1379 1380 if (!kvmgt_gfn_is_write_protected(info, gfn)) 1381 goto out; 1382 1383 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1384 kvmgt_protect_table_del(info, gfn); 1385 1386 out: 1387 spin_unlock(&kvm->mmu_lock); 1388 srcu_read_unlock(&kvm->srcu, idx); 1389 return 0; 1390 } 1391 1392 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 1393 const u8 *val, int len, 1394 struct kvm_page_track_notifier_node *node) 1395 { 1396 struct kvmgt_guest_info *info = container_of(node, 1397 struct kvmgt_guest_info, track_node); 1398 1399 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) 1400 intel_gvt_ops->write_protect_handler(info->vgpu, gpa, 1401 (void *)val, len); 1402 } 1403 1404 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 1405 struct kvm_memory_slot *slot, 1406 struct kvm_page_track_notifier_node *node) 1407 { 1408 int i; 1409 gfn_t gfn; 1410 struct kvmgt_guest_info *info = container_of(node, 1411 struct kvmgt_guest_info, track_node); 1412 1413 spin_lock(&kvm->mmu_lock); 1414 for (i = 0; i < slot->npages; i++) { 1415 gfn = slot->base_gfn + i; 1416 if (kvmgt_gfn_is_write_protected(info, gfn)) { 1417 kvm_slot_page_track_remove_page(kvm, slot, gfn, 1418 KVM_PAGE_TRACK_WRITE); 1419 kvmgt_protect_table_del(info, gfn); 1420 } 1421 } 1422 spin_unlock(&kvm->mmu_lock); 1423 } 1424 1425 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm) 1426 { 1427 struct intel_vgpu *itr; 1428 struct kvmgt_guest_info *info; 1429 int id; 1430 bool ret = false; 1431 1432 mutex_lock(&vgpu->gvt->lock); 1433 for_each_active_vgpu(vgpu->gvt, itr, id) { 1434 if (!handle_valid(itr->handle)) 1435 continue; 1436 1437 info = (struct kvmgt_guest_info *)itr->handle; 1438 if (kvm && kvm == info->kvm) { 1439 ret = true; 1440 goto out; 1441 } 1442 } 1443 out: 1444 mutex_unlock(&vgpu->gvt->lock); 1445 return ret; 1446 } 1447 1448 static int kvmgt_guest_init(struct mdev_device *mdev) 1449 { 1450 struct kvmgt_guest_info *info; 1451 struct intel_vgpu *vgpu; 1452 struct kvm *kvm; 1453 1454 vgpu = mdev_get_drvdata(mdev); 1455 if (handle_valid(vgpu->handle)) 1456 return -EEXIST; 1457 1458 kvm = vgpu->vdev.kvm; 1459 if (!kvm || kvm->mm != current->mm) { 1460 gvt_vgpu_err("KVM is required to use Intel vGPU\n"); 1461 return -ESRCH; 1462 } 1463 1464 if (__kvmgt_vgpu_exist(vgpu, kvm)) 1465 return -EEXIST; 1466 1467 info = vzalloc(sizeof(struct kvmgt_guest_info)); 1468 if (!info) 1469 return -ENOMEM; 1470 1471 vgpu->handle = (unsigned long)info; 1472 info->vgpu = vgpu; 1473 info->kvm = kvm; 1474 kvm_get_kvm(info->kvm); 1475 1476 kvmgt_protect_table_init(info); 1477 gvt_cache_init(vgpu); 1478 1479 mutex_init(&vgpu->dmabuf_lock); 1480 init_completion(&vgpu->vblank_done); 1481 1482 info->track_node.track_write = kvmgt_page_track_write; 1483 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 1484 kvm_page_track_register_notifier(kvm, &info->track_node); 1485 1486 return 0; 1487 } 1488 1489 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info) 1490 { 1491 kvm_page_track_unregister_notifier(info->kvm, &info->track_node); 1492 kvm_put_kvm(info->kvm); 1493 kvmgt_protect_table_destroy(info); 1494 gvt_cache_destroy(info->vgpu); 1495 vfree(info); 1496 1497 return true; 1498 } 1499 1500 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) 1501 { 1502 /* nothing to do here */ 1503 return 0; 1504 } 1505 1506 static void kvmgt_detach_vgpu(unsigned long handle) 1507 { 1508 /* nothing to do here */ 1509 } 1510 1511 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) 1512 { 1513 struct kvmgt_guest_info *info; 1514 struct intel_vgpu *vgpu; 1515 1516 if (!handle_valid(handle)) 1517 return -ESRCH; 1518 1519 info = (struct kvmgt_guest_info *)handle; 1520 vgpu = info->vgpu; 1521 1522 if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1) 1523 return 0; 1524 1525 return -EFAULT; 1526 } 1527 1528 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) 1529 { 1530 unsigned long iova, pfn; 1531 struct kvmgt_guest_info *info; 1532 struct device *dev; 1533 struct intel_vgpu *vgpu; 1534 int rc; 1535 1536 if (!handle_valid(handle)) 1537 return INTEL_GVT_INVALID_ADDR; 1538 1539 info = (struct kvmgt_guest_info *)handle; 1540 vgpu = info->vgpu; 1541 iova = gvt_cache_find(info->vgpu, gfn); 1542 if (iova != INTEL_GVT_INVALID_ADDR) 1543 return iova; 1544 1545 pfn = INTEL_GVT_INVALID_ADDR; 1546 dev = mdev_dev(info->vgpu->vdev.mdev); 1547 rc = vfio_pin_pages(dev, &gfn, 1, IOMMU_READ | IOMMU_WRITE, &pfn); 1548 if (rc != 1) { 1549 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n", 1550 gfn, rc); 1551 return INTEL_GVT_INVALID_ADDR; 1552 } 1553 /* transfer to host iova for GFX to use DMA */ 1554 rc = gvt_dma_map_iova(info->vgpu, pfn, &iova); 1555 if (rc) { 1556 gvt_vgpu_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn); 1557 vfio_unpin_pages(dev, &gfn, 1); 1558 return INTEL_GVT_INVALID_ADDR; 1559 } 1560 1561 gvt_cache_add(info->vgpu, gfn, iova); 1562 return iova; 1563 } 1564 1565 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, 1566 void *buf, unsigned long len, bool write) 1567 { 1568 struct kvmgt_guest_info *info; 1569 struct kvm *kvm; 1570 int idx, ret; 1571 bool kthread = current->mm == NULL; 1572 1573 if (!handle_valid(handle)) 1574 return -ESRCH; 1575 1576 info = (struct kvmgt_guest_info *)handle; 1577 kvm = info->kvm; 1578 1579 if (kthread) 1580 use_mm(kvm->mm); 1581 1582 idx = srcu_read_lock(&kvm->srcu); 1583 ret = write ? kvm_write_guest(kvm, gpa, buf, len) : 1584 kvm_read_guest(kvm, gpa, buf, len); 1585 srcu_read_unlock(&kvm->srcu, idx); 1586 1587 if (kthread) 1588 unuse_mm(kvm->mm); 1589 1590 return ret; 1591 } 1592 1593 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa, 1594 void *buf, unsigned long len) 1595 { 1596 return kvmgt_rw_gpa(handle, gpa, buf, len, false); 1597 } 1598 1599 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa, 1600 void *buf, unsigned long len) 1601 { 1602 return kvmgt_rw_gpa(handle, gpa, buf, len, true); 1603 } 1604 1605 static unsigned long kvmgt_virt_to_pfn(void *addr) 1606 { 1607 return PFN_DOWN(__pa(addr)); 1608 } 1609 1610 static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn) 1611 { 1612 struct kvmgt_guest_info *info; 1613 struct kvm *kvm; 1614 1615 if (!handle_valid(handle)) 1616 return false; 1617 1618 info = (struct kvmgt_guest_info *)handle; 1619 kvm = info->kvm; 1620 1621 return kvm_is_visible_gfn(kvm, gfn); 1622 1623 } 1624 1625 struct intel_gvt_mpt kvmgt_mpt = { 1626 .host_init = kvmgt_host_init, 1627 .host_exit = kvmgt_host_exit, 1628 .attach_vgpu = kvmgt_attach_vgpu, 1629 .detach_vgpu = kvmgt_detach_vgpu, 1630 .inject_msi = kvmgt_inject_msi, 1631 .from_virt_to_mfn = kvmgt_virt_to_pfn, 1632 .set_wp_page = kvmgt_write_protect_add, 1633 .unset_wp_page = kvmgt_write_protect_remove, 1634 .read_gpa = kvmgt_read_gpa, 1635 .write_gpa = kvmgt_write_gpa, 1636 .gfn_to_mfn = kvmgt_gfn_to_pfn, 1637 .set_opregion = kvmgt_set_opregion, 1638 .get_vfio_device = kvmgt_get_vfio_device, 1639 .put_vfio_device = kvmgt_put_vfio_device, 1640 .is_valid_gfn = kvmgt_is_valid_gfn, 1641 }; 1642 EXPORT_SYMBOL_GPL(kvmgt_mpt); 1643 1644 static int __init kvmgt_init(void) 1645 { 1646 return 0; 1647 } 1648 1649 static void __exit kvmgt_exit(void) 1650 { 1651 } 1652 1653 module_init(kvmgt_init); 1654 module_exit(kvmgt_exit); 1655 1656 MODULE_LICENSE("GPL and additional rights"); 1657 MODULE_AUTHOR("Intel Corporation"); 1658