1 /* 2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM 3 * 4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Kevin Tian <kevin.tian@intel.com> 27 * Jike Song <jike.song@intel.com> 28 * Xiaoguang Chen <xiaoguang.chen@intel.com> 29 */ 30 31 #include <linux/init.h> 32 #include <linux/device.h> 33 #include <linux/mm.h> 34 #include <linux/mmu_context.h> 35 #include <linux/types.h> 36 #include <linux/list.h> 37 #include <linux/rbtree.h> 38 #include <linux/spinlock.h> 39 #include <linux/eventfd.h> 40 #include <linux/uuid.h> 41 #include <linux/kvm_host.h> 42 #include <linux/vfio.h> 43 #include <linux/mdev.h> 44 45 #include "i915_drv.h" 46 #include "gvt.h" 47 48 static const struct intel_gvt_ops *intel_gvt_ops; 49 50 /* helper macros copied from vfio-pci */ 51 #define VFIO_PCI_OFFSET_SHIFT 40 52 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) 53 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) 54 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) 55 56 #define OPREGION_SIGNATURE "IntelGraphicsMem" 57 58 struct vfio_region; 59 struct intel_vgpu_regops { 60 size_t (*rw)(struct intel_vgpu *vgpu, char *buf, 61 size_t count, loff_t *ppos, bool iswrite); 62 void (*release)(struct intel_vgpu *vgpu, 63 struct vfio_region *region); 64 }; 65 66 struct vfio_region { 67 u32 type; 68 u32 subtype; 69 size_t size; 70 u32 flags; 71 const struct intel_vgpu_regops *ops; 72 void *data; 73 }; 74 75 struct kvmgt_pgfn { 76 gfn_t gfn; 77 struct hlist_node hnode; 78 }; 79 80 struct kvmgt_guest_info { 81 struct kvm *kvm; 82 struct intel_vgpu *vgpu; 83 struct kvm_page_track_notifier_node track_node; 84 #define NR_BKT (1 << 18) 85 struct hlist_head ptable[NR_BKT]; 86 #undef NR_BKT 87 }; 88 89 struct gvt_dma { 90 struct rb_node node; 91 gfn_t gfn; 92 unsigned long iova; 93 }; 94 95 static inline bool handle_valid(unsigned long handle) 96 { 97 return !!(handle & ~0xff); 98 } 99 100 static int kvmgt_guest_init(struct mdev_device *mdev); 101 static void intel_vgpu_release_work(struct work_struct *work); 102 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); 103 104 static int gvt_dma_map_iova(struct intel_vgpu *vgpu, kvm_pfn_t pfn, 105 unsigned long *iova) 106 { 107 struct page *page; 108 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 109 dma_addr_t daddr; 110 111 if (unlikely(!pfn_valid(pfn))) 112 return -EFAULT; 113 114 page = pfn_to_page(pfn); 115 daddr = dma_map_page(dev, page, 0, PAGE_SIZE, 116 PCI_DMA_BIDIRECTIONAL); 117 if (dma_mapping_error(dev, daddr)) 118 return -ENOMEM; 119 120 *iova = (unsigned long)(daddr >> PAGE_SHIFT); 121 return 0; 122 } 123 124 static void gvt_dma_unmap_iova(struct intel_vgpu *vgpu, unsigned long iova) 125 { 126 struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev; 127 dma_addr_t daddr; 128 129 daddr = (dma_addr_t)(iova << PAGE_SHIFT); 130 dma_unmap_page(dev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 131 } 132 133 static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 134 { 135 struct rb_node *node = vgpu->vdev.cache.rb_node; 136 struct gvt_dma *ret = NULL; 137 138 while (node) { 139 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node); 140 141 if (gfn < itr->gfn) 142 node = node->rb_left; 143 else if (gfn > itr->gfn) 144 node = node->rb_right; 145 else { 146 ret = itr; 147 goto out; 148 } 149 } 150 151 out: 152 return ret; 153 } 154 155 static unsigned long gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn) 156 { 157 struct gvt_dma *entry; 158 unsigned long iova; 159 160 mutex_lock(&vgpu->vdev.cache_lock); 161 162 entry = __gvt_cache_find(vgpu, gfn); 163 iova = (entry == NULL) ? INTEL_GVT_INVALID_ADDR : entry->iova; 164 165 mutex_unlock(&vgpu->vdev.cache_lock); 166 return iova; 167 } 168 169 static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn, 170 unsigned long iova) 171 { 172 struct gvt_dma *new, *itr; 173 struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL; 174 175 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL); 176 if (!new) 177 return; 178 179 new->gfn = gfn; 180 new->iova = iova; 181 182 mutex_lock(&vgpu->vdev.cache_lock); 183 while (*link) { 184 parent = *link; 185 itr = rb_entry(parent, struct gvt_dma, node); 186 187 if (gfn == itr->gfn) 188 goto out; 189 else if (gfn < itr->gfn) 190 link = &parent->rb_left; 191 else 192 link = &parent->rb_right; 193 } 194 195 rb_link_node(&new->node, parent, link); 196 rb_insert_color(&new->node, &vgpu->vdev.cache); 197 mutex_unlock(&vgpu->vdev.cache_lock); 198 return; 199 200 out: 201 mutex_unlock(&vgpu->vdev.cache_lock); 202 kfree(new); 203 } 204 205 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu, 206 struct gvt_dma *entry) 207 { 208 rb_erase(&entry->node, &vgpu->vdev.cache); 209 kfree(entry); 210 } 211 212 static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn) 213 { 214 struct device *dev = mdev_dev(vgpu->vdev.mdev); 215 struct gvt_dma *this; 216 unsigned long g1; 217 int rc; 218 219 mutex_lock(&vgpu->vdev.cache_lock); 220 this = __gvt_cache_find(vgpu, gfn); 221 if (!this) { 222 mutex_unlock(&vgpu->vdev.cache_lock); 223 return; 224 } 225 226 g1 = gfn; 227 gvt_dma_unmap_iova(vgpu, this->iova); 228 rc = vfio_unpin_pages(dev, &g1, 1); 229 WARN_ON(rc != 1); 230 __gvt_cache_remove_entry(vgpu, this); 231 mutex_unlock(&vgpu->vdev.cache_lock); 232 } 233 234 static void gvt_cache_init(struct intel_vgpu *vgpu) 235 { 236 vgpu->vdev.cache = RB_ROOT; 237 mutex_init(&vgpu->vdev.cache_lock); 238 } 239 240 static void gvt_cache_destroy(struct intel_vgpu *vgpu) 241 { 242 struct gvt_dma *dma; 243 struct rb_node *node = NULL; 244 struct device *dev = mdev_dev(vgpu->vdev.mdev); 245 unsigned long gfn; 246 247 for (;;) { 248 mutex_lock(&vgpu->vdev.cache_lock); 249 node = rb_first(&vgpu->vdev.cache); 250 if (!node) { 251 mutex_unlock(&vgpu->vdev.cache_lock); 252 break; 253 } 254 dma = rb_entry(node, struct gvt_dma, node); 255 gvt_dma_unmap_iova(vgpu, dma->iova); 256 gfn = dma->gfn; 257 __gvt_cache_remove_entry(vgpu, dma); 258 mutex_unlock(&vgpu->vdev.cache_lock); 259 vfio_unpin_pages(dev, &gfn, 1); 260 } 261 } 262 263 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info) 264 { 265 hash_init(info->ptable); 266 } 267 268 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info) 269 { 270 struct kvmgt_pgfn *p; 271 struct hlist_node *tmp; 272 int i; 273 274 hash_for_each_safe(info->ptable, i, tmp, p, hnode) { 275 hash_del(&p->hnode); 276 kfree(p); 277 } 278 } 279 280 static struct kvmgt_pgfn * 281 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn) 282 { 283 struct kvmgt_pgfn *p, *res = NULL; 284 285 hash_for_each_possible(info->ptable, p, hnode, gfn) { 286 if (gfn == p->gfn) { 287 res = p; 288 break; 289 } 290 } 291 292 return res; 293 } 294 295 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info, 296 gfn_t gfn) 297 { 298 struct kvmgt_pgfn *p; 299 300 p = __kvmgt_protect_table_find(info, gfn); 301 return !!p; 302 } 303 304 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn) 305 { 306 struct kvmgt_pgfn *p; 307 308 if (kvmgt_gfn_is_write_protected(info, gfn)) 309 return; 310 311 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC); 312 if (WARN(!p, "gfn: 0x%llx\n", gfn)) 313 return; 314 315 p->gfn = gfn; 316 hash_add(info->ptable, &p->hnode, gfn); 317 } 318 319 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info, 320 gfn_t gfn) 321 { 322 struct kvmgt_pgfn *p; 323 324 p = __kvmgt_protect_table_find(info, gfn); 325 if (p) { 326 hash_del(&p->hnode); 327 kfree(p); 328 } 329 } 330 331 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf, 332 size_t count, loff_t *ppos, bool iswrite) 333 { 334 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - 335 VFIO_PCI_NUM_REGIONS; 336 void *base = vgpu->vdev.region[i].data; 337 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 338 339 if (pos >= vgpu->vdev.region[i].size || iswrite) { 340 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n"); 341 return -EINVAL; 342 } 343 count = min(count, (size_t)(vgpu->vdev.region[i].size - pos)); 344 memcpy(buf, base + pos, count); 345 346 return count; 347 } 348 349 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu, 350 struct vfio_region *region) 351 { 352 } 353 354 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = { 355 .rw = intel_vgpu_reg_rw_opregion, 356 .release = intel_vgpu_reg_release_opregion, 357 }; 358 359 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu, 360 unsigned int type, unsigned int subtype, 361 const struct intel_vgpu_regops *ops, 362 size_t size, u32 flags, void *data) 363 { 364 struct vfio_region *region; 365 366 region = krealloc(vgpu->vdev.region, 367 (vgpu->vdev.num_regions + 1) * sizeof(*region), 368 GFP_KERNEL); 369 if (!region) 370 return -ENOMEM; 371 372 vgpu->vdev.region = region; 373 vgpu->vdev.region[vgpu->vdev.num_regions].type = type; 374 vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype; 375 vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops; 376 vgpu->vdev.region[vgpu->vdev.num_regions].size = size; 377 vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags; 378 vgpu->vdev.region[vgpu->vdev.num_regions].data = data; 379 vgpu->vdev.num_regions++; 380 return 0; 381 } 382 383 static int kvmgt_get_vfio_device(void *p_vgpu) 384 { 385 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; 386 387 vgpu->vdev.vfio_device = vfio_device_get_from_dev( 388 mdev_dev(vgpu->vdev.mdev)); 389 if (!vgpu->vdev.vfio_device) { 390 gvt_vgpu_err("failed to get vfio device\n"); 391 return -ENODEV; 392 } 393 return 0; 394 } 395 396 397 static int kvmgt_set_opregion(void *p_vgpu) 398 { 399 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu; 400 void *base; 401 int ret; 402 403 /* Each vgpu has its own opregion, although VFIO would create another 404 * one later. This one is used to expose opregion to VFIO. And the 405 * other one created by VFIO later, is used by guest actually. 406 */ 407 base = vgpu_opregion(vgpu)->va; 408 if (!base) 409 return -ENOMEM; 410 411 if (memcmp(base, OPREGION_SIGNATURE, 16)) { 412 memunmap(base); 413 return -EINVAL; 414 } 415 416 ret = intel_vgpu_register_reg(vgpu, 417 PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, 418 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, 419 &intel_vgpu_regops_opregion, OPREGION_SIZE, 420 VFIO_REGION_INFO_FLAG_READ, base); 421 422 return ret; 423 } 424 425 static void kvmgt_put_vfio_device(void *vgpu) 426 { 427 if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device)) 428 return; 429 430 vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device); 431 } 432 433 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev) 434 { 435 struct intel_vgpu *vgpu = NULL; 436 struct intel_vgpu_type *type; 437 struct device *pdev; 438 void *gvt; 439 int ret; 440 441 pdev = mdev_parent_dev(mdev); 442 gvt = kdev_to_i915(pdev)->gvt; 443 444 type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj)); 445 if (!type) { 446 gvt_vgpu_err("failed to find type %s to create\n", 447 kobject_name(kobj)); 448 ret = -EINVAL; 449 goto out; 450 } 451 452 vgpu = intel_gvt_ops->vgpu_create(gvt, type); 453 if (IS_ERR_OR_NULL(vgpu)) { 454 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu); 455 gvt_vgpu_err("failed to create intel vgpu: %d\n", ret); 456 goto out; 457 } 458 459 INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work); 460 461 vgpu->vdev.mdev = mdev; 462 mdev_set_drvdata(mdev, vgpu); 463 464 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", 465 dev_name(mdev_dev(mdev))); 466 ret = 0; 467 468 out: 469 return ret; 470 } 471 472 static int intel_vgpu_remove(struct mdev_device *mdev) 473 { 474 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 475 476 if (handle_valid(vgpu->handle)) 477 return -EBUSY; 478 479 intel_gvt_ops->vgpu_destroy(vgpu); 480 return 0; 481 } 482 483 static int intel_vgpu_iommu_notifier(struct notifier_block *nb, 484 unsigned long action, void *data) 485 { 486 struct intel_vgpu *vgpu = container_of(nb, 487 struct intel_vgpu, 488 vdev.iommu_notifier); 489 490 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) { 491 struct vfio_iommu_type1_dma_unmap *unmap = data; 492 unsigned long gfn, end_gfn; 493 494 gfn = unmap->iova >> PAGE_SHIFT; 495 end_gfn = gfn + unmap->size / PAGE_SIZE; 496 497 while (gfn < end_gfn) 498 gvt_cache_remove(vgpu, gfn++); 499 } 500 501 return NOTIFY_OK; 502 } 503 504 static int intel_vgpu_group_notifier(struct notifier_block *nb, 505 unsigned long action, void *data) 506 { 507 struct intel_vgpu *vgpu = container_of(nb, 508 struct intel_vgpu, 509 vdev.group_notifier); 510 511 /* the only action we care about */ 512 if (action == VFIO_GROUP_NOTIFY_SET_KVM) { 513 vgpu->vdev.kvm = data; 514 515 if (!data) 516 schedule_work(&vgpu->vdev.release_work); 517 } 518 519 return NOTIFY_OK; 520 } 521 522 static int intel_vgpu_open(struct mdev_device *mdev) 523 { 524 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 525 unsigned long events; 526 int ret; 527 528 vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier; 529 vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier; 530 531 events = VFIO_IOMMU_NOTIFY_DMA_UNMAP; 532 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events, 533 &vgpu->vdev.iommu_notifier); 534 if (ret != 0) { 535 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n", 536 ret); 537 goto out; 538 } 539 540 events = VFIO_GROUP_NOTIFY_SET_KVM; 541 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events, 542 &vgpu->vdev.group_notifier); 543 if (ret != 0) { 544 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n", 545 ret); 546 goto undo_iommu; 547 } 548 549 ret = kvmgt_guest_init(mdev); 550 if (ret) 551 goto undo_group; 552 553 intel_gvt_ops->vgpu_activate(vgpu); 554 555 atomic_set(&vgpu->vdev.released, 0); 556 return ret; 557 558 undo_group: 559 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, 560 &vgpu->vdev.group_notifier); 561 562 undo_iommu: 563 vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, 564 &vgpu->vdev.iommu_notifier); 565 out: 566 return ret; 567 } 568 569 static void __intel_vgpu_release(struct intel_vgpu *vgpu) 570 { 571 struct kvmgt_guest_info *info; 572 int ret; 573 574 if (!handle_valid(vgpu->handle)) 575 return; 576 577 if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1)) 578 return; 579 580 intel_gvt_ops->vgpu_deactivate(vgpu); 581 582 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY, 583 &vgpu->vdev.iommu_notifier); 584 WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret); 585 586 ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY, 587 &vgpu->vdev.group_notifier); 588 WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret); 589 590 info = (struct kvmgt_guest_info *)vgpu->handle; 591 kvmgt_guest_exit(info); 592 593 vgpu->vdev.kvm = NULL; 594 vgpu->handle = 0; 595 } 596 597 static void intel_vgpu_release(struct mdev_device *mdev) 598 { 599 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 600 601 __intel_vgpu_release(vgpu); 602 } 603 604 static void intel_vgpu_release_work(struct work_struct *work) 605 { 606 struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu, 607 vdev.release_work); 608 609 __intel_vgpu_release(vgpu); 610 } 611 612 static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar) 613 { 614 u32 start_lo, start_hi; 615 u32 mem_type; 616 617 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 618 PCI_BASE_ADDRESS_MEM_MASK; 619 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) & 620 PCI_BASE_ADDRESS_MEM_TYPE_MASK; 621 622 switch (mem_type) { 623 case PCI_BASE_ADDRESS_MEM_TYPE_64: 624 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space 625 + bar + 4)); 626 break; 627 case PCI_BASE_ADDRESS_MEM_TYPE_32: 628 case PCI_BASE_ADDRESS_MEM_TYPE_1M: 629 /* 1M mem BAR treated as 32-bit BAR */ 630 default: 631 /* mem unknown type treated as 32-bit BAR */ 632 start_hi = 0; 633 break; 634 } 635 636 return ((u64)start_hi << 32) | start_lo; 637 } 638 639 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off, 640 void *buf, unsigned int count, bool is_write) 641 { 642 uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar); 643 int ret; 644 645 if (is_write) 646 ret = intel_gvt_ops->emulate_mmio_write(vgpu, 647 bar_start + off, buf, count); 648 else 649 ret = intel_gvt_ops->emulate_mmio_read(vgpu, 650 bar_start + off, buf, count); 651 return ret; 652 } 653 654 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off) 655 { 656 return off >= vgpu_aperture_offset(vgpu) && 657 off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu); 658 } 659 660 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off, 661 void *buf, unsigned long count, bool is_write) 662 { 663 void *aperture_va; 664 665 if (!intel_vgpu_in_aperture(vgpu, off) || 666 !intel_vgpu_in_aperture(vgpu, off + count)) { 667 gvt_vgpu_err("Invalid aperture offset %llu\n", off); 668 return -EINVAL; 669 } 670 671 aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap, 672 ALIGN_DOWN(off, PAGE_SIZE), 673 count + offset_in_page(off)); 674 if (!aperture_va) 675 return -EIO; 676 677 if (is_write) 678 memcpy(aperture_va + offset_in_page(off), buf, count); 679 else 680 memcpy(buf, aperture_va + offset_in_page(off), count); 681 682 io_mapping_unmap(aperture_va); 683 684 return 0; 685 } 686 687 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf, 688 size_t count, loff_t *ppos, bool is_write) 689 { 690 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 691 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 692 uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK; 693 int ret = -EINVAL; 694 695 696 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) { 697 gvt_vgpu_err("invalid index: %u\n", index); 698 return -EINVAL; 699 } 700 701 switch (index) { 702 case VFIO_PCI_CONFIG_REGION_INDEX: 703 if (is_write) 704 ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos, 705 buf, count); 706 else 707 ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos, 708 buf, count); 709 break; 710 case VFIO_PCI_BAR0_REGION_INDEX: 711 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos, 712 buf, count, is_write); 713 break; 714 case VFIO_PCI_BAR2_REGION_INDEX: 715 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write); 716 break; 717 case VFIO_PCI_BAR1_REGION_INDEX: 718 case VFIO_PCI_BAR3_REGION_INDEX: 719 case VFIO_PCI_BAR4_REGION_INDEX: 720 case VFIO_PCI_BAR5_REGION_INDEX: 721 case VFIO_PCI_VGA_REGION_INDEX: 722 case VFIO_PCI_ROM_REGION_INDEX: 723 break; 724 default: 725 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) 726 return -EINVAL; 727 728 index -= VFIO_PCI_NUM_REGIONS; 729 return vgpu->vdev.region[index].ops->rw(vgpu, buf, count, 730 ppos, is_write); 731 } 732 733 return ret == 0 ? count : ret; 734 } 735 736 static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos) 737 { 738 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 739 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 740 struct intel_gvt *gvt = vgpu->gvt; 741 int offset; 742 743 /* Only allow MMIO GGTT entry access */ 744 if (index != PCI_BASE_ADDRESS_0) 745 return false; 746 747 offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) - 748 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0); 749 750 return (offset >= gvt->device_info.gtt_start_offset && 751 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ? 752 true : false; 753 } 754 755 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf, 756 size_t count, loff_t *ppos) 757 { 758 unsigned int done = 0; 759 int ret; 760 761 while (count) { 762 size_t filled; 763 764 /* Only support GGTT entry 8 bytes read */ 765 if (count >= 8 && !(*ppos % 8) && 766 gtt_entry(mdev, ppos)) { 767 u64 val; 768 769 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 770 ppos, false); 771 if (ret <= 0) 772 goto read_err; 773 774 if (copy_to_user(buf, &val, sizeof(val))) 775 goto read_err; 776 777 filled = 8; 778 } else if (count >= 4 && !(*ppos % 4)) { 779 u32 val; 780 781 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 782 ppos, false); 783 if (ret <= 0) 784 goto read_err; 785 786 if (copy_to_user(buf, &val, sizeof(val))) 787 goto read_err; 788 789 filled = 4; 790 } else if (count >= 2 && !(*ppos % 2)) { 791 u16 val; 792 793 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 794 ppos, false); 795 if (ret <= 0) 796 goto read_err; 797 798 if (copy_to_user(buf, &val, sizeof(val))) 799 goto read_err; 800 801 filled = 2; 802 } else { 803 u8 val; 804 805 ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos, 806 false); 807 if (ret <= 0) 808 goto read_err; 809 810 if (copy_to_user(buf, &val, sizeof(val))) 811 goto read_err; 812 813 filled = 1; 814 } 815 816 count -= filled; 817 done += filled; 818 *ppos += filled; 819 buf += filled; 820 } 821 822 return done; 823 824 read_err: 825 return -EFAULT; 826 } 827 828 static ssize_t intel_vgpu_write(struct mdev_device *mdev, 829 const char __user *buf, 830 size_t count, loff_t *ppos) 831 { 832 unsigned int done = 0; 833 int ret; 834 835 while (count) { 836 size_t filled; 837 838 /* Only support GGTT entry 8 bytes write */ 839 if (count >= 8 && !(*ppos % 8) && 840 gtt_entry(mdev, ppos)) { 841 u64 val; 842 843 if (copy_from_user(&val, buf, sizeof(val))) 844 goto write_err; 845 846 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 847 ppos, true); 848 if (ret <= 0) 849 goto write_err; 850 851 filled = 8; 852 } else if (count >= 4 && !(*ppos % 4)) { 853 u32 val; 854 855 if (copy_from_user(&val, buf, sizeof(val))) 856 goto write_err; 857 858 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val), 859 ppos, true); 860 if (ret <= 0) 861 goto write_err; 862 863 filled = 4; 864 } else if (count >= 2 && !(*ppos % 2)) { 865 u16 val; 866 867 if (copy_from_user(&val, buf, sizeof(val))) 868 goto write_err; 869 870 ret = intel_vgpu_rw(mdev, (char *)&val, 871 sizeof(val), ppos, true); 872 if (ret <= 0) 873 goto write_err; 874 875 filled = 2; 876 } else { 877 u8 val; 878 879 if (copy_from_user(&val, buf, sizeof(val))) 880 goto write_err; 881 882 ret = intel_vgpu_rw(mdev, &val, sizeof(val), 883 ppos, true); 884 if (ret <= 0) 885 goto write_err; 886 887 filled = 1; 888 } 889 890 count -= filled; 891 done += filled; 892 *ppos += filled; 893 buf += filled; 894 } 895 896 return done; 897 write_err: 898 return -EFAULT; 899 } 900 901 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma) 902 { 903 unsigned int index; 904 u64 virtaddr; 905 unsigned long req_size, pgoff = 0; 906 pgprot_t pg_prot; 907 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 908 909 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 910 if (index >= VFIO_PCI_ROM_REGION_INDEX) 911 return -EINVAL; 912 913 if (vma->vm_end < vma->vm_start) 914 return -EINVAL; 915 if ((vma->vm_flags & VM_SHARED) == 0) 916 return -EINVAL; 917 if (index != VFIO_PCI_BAR2_REGION_INDEX) 918 return -EINVAL; 919 920 pg_prot = vma->vm_page_prot; 921 virtaddr = vma->vm_start; 922 req_size = vma->vm_end - vma->vm_start; 923 pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; 924 925 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot); 926 } 927 928 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type) 929 { 930 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX) 931 return 1; 932 933 return 0; 934 } 935 936 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu, 937 unsigned int index, unsigned int start, 938 unsigned int count, uint32_t flags, 939 void *data) 940 { 941 return 0; 942 } 943 944 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu, 945 unsigned int index, unsigned int start, 946 unsigned int count, uint32_t flags, void *data) 947 { 948 return 0; 949 } 950 951 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu, 952 unsigned int index, unsigned int start, unsigned int count, 953 uint32_t flags, void *data) 954 { 955 return 0; 956 } 957 958 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu, 959 unsigned int index, unsigned int start, unsigned int count, 960 uint32_t flags, void *data) 961 { 962 struct eventfd_ctx *trigger; 963 964 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 965 int fd = *(int *)data; 966 967 trigger = eventfd_ctx_fdget(fd); 968 if (IS_ERR(trigger)) { 969 gvt_vgpu_err("eventfd_ctx_fdget failed\n"); 970 return PTR_ERR(trigger); 971 } 972 vgpu->vdev.msi_trigger = trigger; 973 } 974 975 return 0; 976 } 977 978 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags, 979 unsigned int index, unsigned int start, unsigned int count, 980 void *data) 981 { 982 int (*func)(struct intel_vgpu *vgpu, unsigned int index, 983 unsigned int start, unsigned int count, uint32_t flags, 984 void *data) = NULL; 985 986 switch (index) { 987 case VFIO_PCI_INTX_IRQ_INDEX: 988 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 989 case VFIO_IRQ_SET_ACTION_MASK: 990 func = intel_vgpu_set_intx_mask; 991 break; 992 case VFIO_IRQ_SET_ACTION_UNMASK: 993 func = intel_vgpu_set_intx_unmask; 994 break; 995 case VFIO_IRQ_SET_ACTION_TRIGGER: 996 func = intel_vgpu_set_intx_trigger; 997 break; 998 } 999 break; 1000 case VFIO_PCI_MSI_IRQ_INDEX: 1001 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 1002 case VFIO_IRQ_SET_ACTION_MASK: 1003 case VFIO_IRQ_SET_ACTION_UNMASK: 1004 /* XXX Need masking support exported */ 1005 break; 1006 case VFIO_IRQ_SET_ACTION_TRIGGER: 1007 func = intel_vgpu_set_msi_trigger; 1008 break; 1009 } 1010 break; 1011 } 1012 1013 if (!func) 1014 return -ENOTTY; 1015 1016 return func(vgpu, index, start, count, flags, data); 1017 } 1018 1019 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, 1020 unsigned long arg) 1021 { 1022 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev); 1023 unsigned long minsz; 1024 1025 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd); 1026 1027 if (cmd == VFIO_DEVICE_GET_INFO) { 1028 struct vfio_device_info info; 1029 1030 minsz = offsetofend(struct vfio_device_info, num_irqs); 1031 1032 if (copy_from_user(&info, (void __user *)arg, minsz)) 1033 return -EFAULT; 1034 1035 if (info.argsz < minsz) 1036 return -EINVAL; 1037 1038 info.flags = VFIO_DEVICE_FLAGS_PCI; 1039 info.flags |= VFIO_DEVICE_FLAGS_RESET; 1040 info.num_regions = VFIO_PCI_NUM_REGIONS + 1041 vgpu->vdev.num_regions; 1042 info.num_irqs = VFIO_PCI_NUM_IRQS; 1043 1044 return copy_to_user((void __user *)arg, &info, minsz) ? 1045 -EFAULT : 0; 1046 1047 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 1048 struct vfio_region_info info; 1049 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 1050 int i, ret; 1051 struct vfio_region_info_cap_sparse_mmap *sparse = NULL; 1052 size_t size; 1053 int nr_areas = 1; 1054 int cap_type_id; 1055 1056 minsz = offsetofend(struct vfio_region_info, offset); 1057 1058 if (copy_from_user(&info, (void __user *)arg, minsz)) 1059 return -EFAULT; 1060 1061 if (info.argsz < minsz) 1062 return -EINVAL; 1063 1064 switch (info.index) { 1065 case VFIO_PCI_CONFIG_REGION_INDEX: 1066 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1067 info.size = vgpu->gvt->device_info.cfg_space_size; 1068 info.flags = VFIO_REGION_INFO_FLAG_READ | 1069 VFIO_REGION_INFO_FLAG_WRITE; 1070 break; 1071 case VFIO_PCI_BAR0_REGION_INDEX: 1072 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1073 info.size = vgpu->cfg_space.bar[info.index].size; 1074 if (!info.size) { 1075 info.flags = 0; 1076 break; 1077 } 1078 1079 info.flags = VFIO_REGION_INFO_FLAG_READ | 1080 VFIO_REGION_INFO_FLAG_WRITE; 1081 break; 1082 case VFIO_PCI_BAR1_REGION_INDEX: 1083 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1084 info.size = 0; 1085 info.flags = 0; 1086 break; 1087 case VFIO_PCI_BAR2_REGION_INDEX: 1088 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1089 info.flags = VFIO_REGION_INFO_FLAG_CAPS | 1090 VFIO_REGION_INFO_FLAG_MMAP | 1091 VFIO_REGION_INFO_FLAG_READ | 1092 VFIO_REGION_INFO_FLAG_WRITE; 1093 info.size = gvt_aperture_sz(vgpu->gvt); 1094 1095 size = sizeof(*sparse) + 1096 (nr_areas * sizeof(*sparse->areas)); 1097 sparse = kzalloc(size, GFP_KERNEL); 1098 if (!sparse) 1099 return -ENOMEM; 1100 1101 sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1102 sparse->header.version = 1; 1103 sparse->nr_areas = nr_areas; 1104 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP; 1105 sparse->areas[0].offset = 1106 PAGE_ALIGN(vgpu_aperture_offset(vgpu)); 1107 sparse->areas[0].size = vgpu_aperture_sz(vgpu); 1108 break; 1109 1110 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1111 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1112 info.size = 0; 1113 info.flags = 0; 1114 1115 gvt_dbg_core("get region info bar:%d\n", info.index); 1116 break; 1117 1118 case VFIO_PCI_ROM_REGION_INDEX: 1119 case VFIO_PCI_VGA_REGION_INDEX: 1120 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 1121 info.size = 0; 1122 info.flags = 0; 1123 1124 gvt_dbg_core("get region info index:%d\n", info.index); 1125 break; 1126 default: 1127 { 1128 struct vfio_region_info_cap_type cap_type = { 1129 .header.id = VFIO_REGION_INFO_CAP_TYPE, 1130 .header.version = 1 }; 1131 1132 if (info.index >= VFIO_PCI_NUM_REGIONS + 1133 vgpu->vdev.num_regions) 1134 return -EINVAL; 1135 1136 i = info.index - VFIO_PCI_NUM_REGIONS; 1137 1138 info.offset = 1139 VFIO_PCI_INDEX_TO_OFFSET(info.index); 1140 info.size = vgpu->vdev.region[i].size; 1141 info.flags = vgpu->vdev.region[i].flags; 1142 1143 cap_type.type = vgpu->vdev.region[i].type; 1144 cap_type.subtype = vgpu->vdev.region[i].subtype; 1145 1146 ret = vfio_info_add_capability(&caps, 1147 &cap_type.header, 1148 sizeof(cap_type)); 1149 if (ret) 1150 return ret; 1151 } 1152 } 1153 1154 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) { 1155 switch (cap_type_id) { 1156 case VFIO_REGION_INFO_CAP_SPARSE_MMAP: 1157 ret = vfio_info_add_capability(&caps, 1158 &sparse->header, sizeof(*sparse) + 1159 (sparse->nr_areas * 1160 sizeof(*sparse->areas))); 1161 kfree(sparse); 1162 if (ret) 1163 return ret; 1164 break; 1165 default: 1166 return -EINVAL; 1167 } 1168 } 1169 1170 if (caps.size) { 1171 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 1172 if (info.argsz < sizeof(info) + caps.size) { 1173 info.argsz = sizeof(info) + caps.size; 1174 info.cap_offset = 0; 1175 } else { 1176 vfio_info_cap_shift(&caps, sizeof(info)); 1177 if (copy_to_user((void __user *)arg + 1178 sizeof(info), caps.buf, 1179 caps.size)) { 1180 kfree(caps.buf); 1181 return -EFAULT; 1182 } 1183 info.cap_offset = sizeof(info); 1184 } 1185 1186 kfree(caps.buf); 1187 } 1188 1189 return copy_to_user((void __user *)arg, &info, minsz) ? 1190 -EFAULT : 0; 1191 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 1192 struct vfio_irq_info info; 1193 1194 minsz = offsetofend(struct vfio_irq_info, count); 1195 1196 if (copy_from_user(&info, (void __user *)arg, minsz)) 1197 return -EFAULT; 1198 1199 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1200 return -EINVAL; 1201 1202 switch (info.index) { 1203 case VFIO_PCI_INTX_IRQ_INDEX: 1204 case VFIO_PCI_MSI_IRQ_INDEX: 1205 break; 1206 default: 1207 return -EINVAL; 1208 } 1209 1210 info.flags = VFIO_IRQ_INFO_EVENTFD; 1211 1212 info.count = intel_vgpu_get_irq_count(vgpu, info.index); 1213 1214 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1215 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1216 VFIO_IRQ_INFO_AUTOMASKED); 1217 else 1218 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1219 1220 return copy_to_user((void __user *)arg, &info, minsz) ? 1221 -EFAULT : 0; 1222 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1223 struct vfio_irq_set hdr; 1224 u8 *data = NULL; 1225 int ret = 0; 1226 size_t data_size = 0; 1227 1228 minsz = offsetofend(struct vfio_irq_set, count); 1229 1230 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1231 return -EFAULT; 1232 1233 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { 1234 int max = intel_vgpu_get_irq_count(vgpu, hdr.index); 1235 1236 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1237 VFIO_PCI_NUM_IRQS, &data_size); 1238 if (ret) { 1239 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n"); 1240 return -EINVAL; 1241 } 1242 if (data_size) { 1243 data = memdup_user((void __user *)(arg + minsz), 1244 data_size); 1245 if (IS_ERR(data)) 1246 return PTR_ERR(data); 1247 } 1248 } 1249 1250 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index, 1251 hdr.start, hdr.count, data); 1252 kfree(data); 1253 1254 return ret; 1255 } else if (cmd == VFIO_DEVICE_RESET) { 1256 intel_gvt_ops->vgpu_reset(vgpu); 1257 return 0; 1258 } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) { 1259 struct vfio_device_gfx_plane_info dmabuf; 1260 int ret = 0; 1261 1262 minsz = offsetofend(struct vfio_device_gfx_plane_info, 1263 dmabuf_id); 1264 if (copy_from_user(&dmabuf, (void __user *)arg, minsz)) 1265 return -EFAULT; 1266 if (dmabuf.argsz < minsz) 1267 return -EINVAL; 1268 1269 ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf); 1270 if (ret != 0) 1271 return ret; 1272 1273 return copy_to_user((void __user *)arg, &dmabuf, minsz) ? 1274 -EFAULT : 0; 1275 } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) { 1276 __u32 dmabuf_id; 1277 __s32 dmabuf_fd; 1278 1279 if (get_user(dmabuf_id, (__u32 __user *)arg)) 1280 return -EFAULT; 1281 1282 dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id); 1283 return dmabuf_fd; 1284 1285 } 1286 1287 return 0; 1288 } 1289 1290 static ssize_t 1291 vgpu_id_show(struct device *dev, struct device_attribute *attr, 1292 char *buf) 1293 { 1294 struct mdev_device *mdev = mdev_from_dev(dev); 1295 1296 if (mdev) { 1297 struct intel_vgpu *vgpu = (struct intel_vgpu *) 1298 mdev_get_drvdata(mdev); 1299 return sprintf(buf, "%d\n", vgpu->id); 1300 } 1301 return sprintf(buf, "\n"); 1302 } 1303 1304 static ssize_t 1305 hw_id_show(struct device *dev, struct device_attribute *attr, 1306 char *buf) 1307 { 1308 struct mdev_device *mdev = mdev_from_dev(dev); 1309 1310 if (mdev) { 1311 struct intel_vgpu *vgpu = (struct intel_vgpu *) 1312 mdev_get_drvdata(mdev); 1313 return sprintf(buf, "%u\n", 1314 vgpu->submission.shadow_ctx->hw_id); 1315 } 1316 return sprintf(buf, "\n"); 1317 } 1318 1319 static DEVICE_ATTR_RO(vgpu_id); 1320 static DEVICE_ATTR_RO(hw_id); 1321 1322 static struct attribute *intel_vgpu_attrs[] = { 1323 &dev_attr_vgpu_id.attr, 1324 &dev_attr_hw_id.attr, 1325 NULL 1326 }; 1327 1328 static const struct attribute_group intel_vgpu_group = { 1329 .name = "intel_vgpu", 1330 .attrs = intel_vgpu_attrs, 1331 }; 1332 1333 static const struct attribute_group *intel_vgpu_groups[] = { 1334 &intel_vgpu_group, 1335 NULL, 1336 }; 1337 1338 static struct mdev_parent_ops intel_vgpu_ops = { 1339 .mdev_attr_groups = intel_vgpu_groups, 1340 .create = intel_vgpu_create, 1341 .remove = intel_vgpu_remove, 1342 1343 .open = intel_vgpu_open, 1344 .release = intel_vgpu_release, 1345 1346 .read = intel_vgpu_read, 1347 .write = intel_vgpu_write, 1348 .mmap = intel_vgpu_mmap, 1349 .ioctl = intel_vgpu_ioctl, 1350 }; 1351 1352 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) 1353 { 1354 struct attribute **kvm_type_attrs; 1355 struct attribute_group **kvm_vgpu_type_groups; 1356 1357 intel_gvt_ops = ops; 1358 if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs, 1359 &kvm_vgpu_type_groups)) 1360 return -EFAULT; 1361 intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups; 1362 1363 return mdev_register_device(dev, &intel_vgpu_ops); 1364 } 1365 1366 static void kvmgt_host_exit(struct device *dev, void *gvt) 1367 { 1368 mdev_unregister_device(dev); 1369 } 1370 1371 static int kvmgt_write_protect_add(unsigned long handle, u64 gfn) 1372 { 1373 struct kvmgt_guest_info *info; 1374 struct kvm *kvm; 1375 struct kvm_memory_slot *slot; 1376 int idx; 1377 1378 if (!handle_valid(handle)) 1379 return -ESRCH; 1380 1381 info = (struct kvmgt_guest_info *)handle; 1382 kvm = info->kvm; 1383 1384 idx = srcu_read_lock(&kvm->srcu); 1385 slot = gfn_to_memslot(kvm, gfn); 1386 if (!slot) { 1387 srcu_read_unlock(&kvm->srcu, idx); 1388 return -EINVAL; 1389 } 1390 1391 spin_lock(&kvm->mmu_lock); 1392 1393 if (kvmgt_gfn_is_write_protected(info, gfn)) 1394 goto out; 1395 1396 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1397 kvmgt_protect_table_add(info, gfn); 1398 1399 out: 1400 spin_unlock(&kvm->mmu_lock); 1401 srcu_read_unlock(&kvm->srcu, idx); 1402 return 0; 1403 } 1404 1405 static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn) 1406 { 1407 struct kvmgt_guest_info *info; 1408 struct kvm *kvm; 1409 struct kvm_memory_slot *slot; 1410 int idx; 1411 1412 if (!handle_valid(handle)) 1413 return 0; 1414 1415 info = (struct kvmgt_guest_info *)handle; 1416 kvm = info->kvm; 1417 1418 idx = srcu_read_lock(&kvm->srcu); 1419 slot = gfn_to_memslot(kvm, gfn); 1420 if (!slot) { 1421 srcu_read_unlock(&kvm->srcu, idx); 1422 return -EINVAL; 1423 } 1424 1425 spin_lock(&kvm->mmu_lock); 1426 1427 if (!kvmgt_gfn_is_write_protected(info, gfn)) 1428 goto out; 1429 1430 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); 1431 kvmgt_protect_table_del(info, gfn); 1432 1433 out: 1434 spin_unlock(&kvm->mmu_lock); 1435 srcu_read_unlock(&kvm->srcu, idx); 1436 return 0; 1437 } 1438 1439 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, 1440 const u8 *val, int len, 1441 struct kvm_page_track_notifier_node *node) 1442 { 1443 struct kvmgt_guest_info *info = container_of(node, 1444 struct kvmgt_guest_info, track_node); 1445 1446 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa))) 1447 intel_gvt_ops->write_protect_handler(info->vgpu, gpa, 1448 (void *)val, len); 1449 } 1450 1451 static void kvmgt_page_track_flush_slot(struct kvm *kvm, 1452 struct kvm_memory_slot *slot, 1453 struct kvm_page_track_notifier_node *node) 1454 { 1455 int i; 1456 gfn_t gfn; 1457 struct kvmgt_guest_info *info = container_of(node, 1458 struct kvmgt_guest_info, track_node); 1459 1460 spin_lock(&kvm->mmu_lock); 1461 for (i = 0; i < slot->npages; i++) { 1462 gfn = slot->base_gfn + i; 1463 if (kvmgt_gfn_is_write_protected(info, gfn)) { 1464 kvm_slot_page_track_remove_page(kvm, slot, gfn, 1465 KVM_PAGE_TRACK_WRITE); 1466 kvmgt_protect_table_del(info, gfn); 1467 } 1468 } 1469 spin_unlock(&kvm->mmu_lock); 1470 } 1471 1472 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm) 1473 { 1474 struct intel_vgpu *itr; 1475 struct kvmgt_guest_info *info; 1476 int id; 1477 bool ret = false; 1478 1479 mutex_lock(&vgpu->gvt->lock); 1480 for_each_active_vgpu(vgpu->gvt, itr, id) { 1481 if (!handle_valid(itr->handle)) 1482 continue; 1483 1484 info = (struct kvmgt_guest_info *)itr->handle; 1485 if (kvm && kvm == info->kvm) { 1486 ret = true; 1487 goto out; 1488 } 1489 } 1490 out: 1491 mutex_unlock(&vgpu->gvt->lock); 1492 return ret; 1493 } 1494 1495 static int kvmgt_guest_init(struct mdev_device *mdev) 1496 { 1497 struct kvmgt_guest_info *info; 1498 struct intel_vgpu *vgpu; 1499 struct kvm *kvm; 1500 1501 vgpu = mdev_get_drvdata(mdev); 1502 if (handle_valid(vgpu->handle)) 1503 return -EEXIST; 1504 1505 kvm = vgpu->vdev.kvm; 1506 if (!kvm || kvm->mm != current->mm) { 1507 gvt_vgpu_err("KVM is required to use Intel vGPU\n"); 1508 return -ESRCH; 1509 } 1510 1511 if (__kvmgt_vgpu_exist(vgpu, kvm)) 1512 return -EEXIST; 1513 1514 info = vzalloc(sizeof(struct kvmgt_guest_info)); 1515 if (!info) 1516 return -ENOMEM; 1517 1518 vgpu->handle = (unsigned long)info; 1519 info->vgpu = vgpu; 1520 info->kvm = kvm; 1521 kvm_get_kvm(info->kvm); 1522 1523 kvmgt_protect_table_init(info); 1524 gvt_cache_init(vgpu); 1525 1526 mutex_init(&vgpu->dmabuf_lock); 1527 init_completion(&vgpu->vblank_done); 1528 1529 info->track_node.track_write = kvmgt_page_track_write; 1530 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot; 1531 kvm_page_track_register_notifier(kvm, &info->track_node); 1532 1533 return 0; 1534 } 1535 1536 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info) 1537 { 1538 kvm_page_track_unregister_notifier(info->kvm, &info->track_node); 1539 kvm_put_kvm(info->kvm); 1540 kvmgt_protect_table_destroy(info); 1541 gvt_cache_destroy(info->vgpu); 1542 vfree(info); 1543 1544 return true; 1545 } 1546 1547 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle) 1548 { 1549 /* nothing to do here */ 1550 return 0; 1551 } 1552 1553 static void kvmgt_detach_vgpu(unsigned long handle) 1554 { 1555 /* nothing to do here */ 1556 } 1557 1558 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data) 1559 { 1560 struct kvmgt_guest_info *info; 1561 struct intel_vgpu *vgpu; 1562 1563 if (!handle_valid(handle)) 1564 return -ESRCH; 1565 1566 info = (struct kvmgt_guest_info *)handle; 1567 vgpu = info->vgpu; 1568 1569 if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1) 1570 return 0; 1571 1572 return -EFAULT; 1573 } 1574 1575 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn) 1576 { 1577 unsigned long iova, pfn; 1578 struct kvmgt_guest_info *info; 1579 struct device *dev; 1580 struct intel_vgpu *vgpu; 1581 int rc; 1582 1583 if (!handle_valid(handle)) 1584 return INTEL_GVT_INVALID_ADDR; 1585 1586 info = (struct kvmgt_guest_info *)handle; 1587 vgpu = info->vgpu; 1588 iova = gvt_cache_find(info->vgpu, gfn); 1589 if (iova != INTEL_GVT_INVALID_ADDR) 1590 return iova; 1591 1592 pfn = INTEL_GVT_INVALID_ADDR; 1593 dev = mdev_dev(info->vgpu->vdev.mdev); 1594 rc = vfio_pin_pages(dev, &gfn, 1, IOMMU_READ | IOMMU_WRITE, &pfn); 1595 if (rc != 1) { 1596 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n", 1597 gfn, rc); 1598 return INTEL_GVT_INVALID_ADDR; 1599 } 1600 /* transfer to host iova for GFX to use DMA */ 1601 rc = gvt_dma_map_iova(info->vgpu, pfn, &iova); 1602 if (rc) { 1603 gvt_vgpu_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn); 1604 vfio_unpin_pages(dev, &gfn, 1); 1605 return INTEL_GVT_INVALID_ADDR; 1606 } 1607 1608 gvt_cache_add(info->vgpu, gfn, iova); 1609 return iova; 1610 } 1611 1612 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa, 1613 void *buf, unsigned long len, bool write) 1614 { 1615 struct kvmgt_guest_info *info; 1616 struct kvm *kvm; 1617 int idx, ret; 1618 bool kthread = current->mm == NULL; 1619 1620 if (!handle_valid(handle)) 1621 return -ESRCH; 1622 1623 info = (struct kvmgt_guest_info *)handle; 1624 kvm = info->kvm; 1625 1626 if (kthread) 1627 use_mm(kvm->mm); 1628 1629 idx = srcu_read_lock(&kvm->srcu); 1630 ret = write ? kvm_write_guest(kvm, gpa, buf, len) : 1631 kvm_read_guest(kvm, gpa, buf, len); 1632 srcu_read_unlock(&kvm->srcu, idx); 1633 1634 if (kthread) 1635 unuse_mm(kvm->mm); 1636 1637 return ret; 1638 } 1639 1640 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa, 1641 void *buf, unsigned long len) 1642 { 1643 return kvmgt_rw_gpa(handle, gpa, buf, len, false); 1644 } 1645 1646 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa, 1647 void *buf, unsigned long len) 1648 { 1649 return kvmgt_rw_gpa(handle, gpa, buf, len, true); 1650 } 1651 1652 static unsigned long kvmgt_virt_to_pfn(void *addr) 1653 { 1654 return PFN_DOWN(__pa(addr)); 1655 } 1656 1657 static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn) 1658 { 1659 struct kvmgt_guest_info *info; 1660 struct kvm *kvm; 1661 1662 if (!handle_valid(handle)) 1663 return false; 1664 1665 info = (struct kvmgt_guest_info *)handle; 1666 kvm = info->kvm; 1667 1668 return kvm_is_visible_gfn(kvm, gfn); 1669 1670 } 1671 1672 struct intel_gvt_mpt kvmgt_mpt = { 1673 .host_init = kvmgt_host_init, 1674 .host_exit = kvmgt_host_exit, 1675 .attach_vgpu = kvmgt_attach_vgpu, 1676 .detach_vgpu = kvmgt_detach_vgpu, 1677 .inject_msi = kvmgt_inject_msi, 1678 .from_virt_to_mfn = kvmgt_virt_to_pfn, 1679 .set_wp_page = kvmgt_write_protect_add, 1680 .unset_wp_page = kvmgt_write_protect_remove, 1681 .read_gpa = kvmgt_read_gpa, 1682 .write_gpa = kvmgt_write_gpa, 1683 .gfn_to_mfn = kvmgt_gfn_to_pfn, 1684 .set_opregion = kvmgt_set_opregion, 1685 .get_vfio_device = kvmgt_get_vfio_device, 1686 .put_vfio_device = kvmgt_put_vfio_device, 1687 .is_valid_gfn = kvmgt_is_valid_gfn, 1688 }; 1689 EXPORT_SYMBOL_GPL(kvmgt_mpt); 1690 1691 static int __init kvmgt_init(void) 1692 { 1693 return 0; 1694 } 1695 1696 static void __exit kvmgt_exit(void) 1697 { 1698 } 1699 1700 module_init(kvmgt_init); 1701 module_exit(kvmgt_exit); 1702 1703 MODULE_LICENSE("GPL and additional rights"); 1704 MODULE_AUTHOR("Intel Corporation"); 1705